Index: unk/src/VBox/VMM/CFGM.cpp
===================================================================
--- /trunk/src/VBox/VMM/CFGM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2812 +1,0 @@
-/* $Id$ */
-/** @file
- * CFGM - Configuration Manager.
- */
-
-/*
- * Copyright (C) 2006-2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_cfgm       CFGM - The Configuration Manager
- *
- * The configuration manager is a directory containing the VM configuration at
- * run time. It works in a manner similar to the windows registry - it's like a
- * file system hierarchy, but the files (values) live in a separate name space
- * and can include the path separators.
- *
- * The configuration is normally created via a callback passed to VMR3Create()
- * via the pfnCFGMConstructor parameter. To make testcase writing a bit simpler,
- * we allow the callback to be NULL, in which case a simple default
- * configuration will be created by CFGMR3ConstructDefaultTree(). The
- * Console::configConstructor() method in Main/ConsoleImpl2.cpp creates the
- * configuration from the XML.
- *
- * Devices, drivers, services and other PDM stuff are given their own subtree
- * where they are protected from accessing information of any parents. This is
- * is implemented via the CFGMR3SetRestrictedRoot() API.
- *
- * Data validation out over the basic primitives is left to the caller. The
- * caller is in a better position to know the proper validation rules of the
- * individual properties.
- *
- * @see grp_cfgm
- *
- *
- * @section sec_cfgm_primitives     Data Primitives
- *
- * CFGM supports the following data primitives:
- *      - Integers. Representation is unsigned 64-bit. Boolean, unsigned and
- *        small integers, and pointers are all represented using this primitive.
- *      - Zero terminated character strings. These are of course UTF-8.
- *      - Variable length byte strings. This can be used to get/put binary
- *        objects like for instance RTMAC.
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_CFGM
-#include <VBox/cfgm.h>
-#include <VBox/dbgf.h>
-#include <VBox/mm.h>
-#include "CFGMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-#include <iprt/uuid.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static void cfgmR3DumpPath(PCFGMNODE pNode, PCDBGFINFOHLP pHlp);
-static void cfgmR3Dump(PCFGMNODE pRoot, unsigned iLevel, PCDBGFINFOHLP pHlp);
-static DECLCALLBACK(void) cfgmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static int  cfgmR3ResolveNode(PCFGMNODE pNode, const char *pszPath, PCFGMNODE *ppChild);
-static int  cfgmR3ResolveLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf);
-static int  cfgmR3InsertLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf);
-static void cfgmR3RemoveLeaf(PCFGMNODE pNode, PCFGMLEAF pLeaf);
-static void cfgmR3FreeValue(PCFGMLEAF pLeaf);
-
-
-
-/**
- * Constructs the configuration for the VM.
- *
- * @returns VBox status code.
- * @param   pVM                 Pointer to VM which configuration has not yet been loaded.
- * @param   pfnCFGMConstructor  Pointer to callback function for constructing the VM configuration tree.
- *                              This is called in the EM.
- * @param   pvUser              The user argument passed to pfnCFGMConstructor.
- * @thread  EMT.
- */
-VMMR3DECL(int) CFGMR3Init(PVM pVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUser)
-{
-    LogFlow(("CFGMR3Init: pfnCFGMConstructor=%p pvUser=%p\n", pfnCFGMConstructor, pvUser));
-
-    /*
-     * Init data members.
-     */
-    pVM->cfgm.s.pRoot = NULL;
-
-    /*
-     * Register DBGF into item.
-     */
-    int rc = DBGFR3InfoRegisterInternal(pVM, "cfgm", "Dumps a part of the CFGM tree. The argument indicates where to start.", cfgmR3Info);
-    AssertRCReturn(rc,rc);
-
-    /*
-     * Root Node.
-     */
-    PCFGMNODE pRoot = (PCFGMNODE)MMR3HeapAllocZ(pVM, MM_TAG_CFGM, sizeof(*pRoot));
-    if (!pRoot)
-        return VERR_NO_MEMORY;
-    pRoot->pVM        = pVM;
-    pRoot->cchName    = 0;
-    pVM->cfgm.s.pRoot = pRoot;
-
-        /*
-         * Call the constructor if specified, if not use the default one.
-         */
-    if (pfnCFGMConstructor)
-        rc = pfnCFGMConstructor(pVM, pvUser);
-    else
-        rc = CFGMR3ConstructDefaultTree(pVM);
-    if (RT_SUCCESS(rc))
-    {
-        Log(("CFGMR3Init: Successfully constructed the configuration\n"));
-        CFGMR3Dump(CFGMR3GetRoot(pVM));
-    }
-    else
-        AssertMsgFailed(("Constructor failed with rc=%Rrc pfnCFGMConstructor=%p\n", rc, pfnCFGMConstructor));
-
-    return rc;
-}
-
-
-/**
- * Terminates the configuration manager.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- */
-VMMR3DECL(int) CFGMR3Term(PVM pVM)
-{
-    CFGMR3RemoveNode(pVM->cfgm.s.pRoot);
-    pVM->cfgm.s.pRoot = NULL;
-    return 0;
-}
-
-
-/**
- * Gets the root node for the VM.
- *
- * @returns Pointer to root node.
- * @param   pVM             VM handle.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetRoot(PVM pVM)
-{
-    return pVM->cfgm.s.pRoot;
-}
-
-
-/**
- * Gets the parent of a CFGM node.
- *
- * @returns Pointer to the parent node.
- * @returns NULL if pNode is Root or pNode is the start of a
- *          restricted subtree (use CFGMr3GetParentEx() for that).
- *
- * @param   pNode           The node which parent we query.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetParent(PCFGMNODE pNode)
-{
-    if (pNode && !pNode->fRestrictedRoot)
-        return pNode->pParent;
-    return NULL;
-}
-
-
-/**
- * Gets the parent of a CFGM node.
- *
- * @returns Pointer to the parent node.
- * @returns NULL if pNode is Root or pVM is not correct.
- *
- * @param   pVM             The VM handle, used as token that the caller is trusted.
- * @param   pNode           The node which parent we query.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetParentEx(PVM pVM, PCFGMNODE pNode)
-{
-    if (pNode && pNode->pVM == pVM)
-        return pNode->pParent;
-    return NULL;
-}
-
-
-/**
- * Query a child node.
- *
- * @returns Pointer to the specified node.
- * @returns NULL if node was not found or pNode is NULL.
- * @param   pNode           Node pszPath is relative to.
- * @param   pszPath         Path to the child node or pNode.
- *                          It's good style to end this with '/'.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetChild(PCFGMNODE pNode, const char *pszPath)
-{
-    PCFGMNODE pChild;
-    int rc = cfgmR3ResolveNode(pNode, pszPath, &pChild);
-    if (RT_SUCCESS(rc))
-        return pChild;
-    return NULL;
-}
-
-
-/**
- * Query a child node by a format string.
- *
- * @returns Pointer to the specified node.
- * @returns NULL if node was not found or pNode is NULL.
- * @param   pNode           Node pszPath is relative to.
- * @param   pszPathFormat   Path to the child node or pNode.
- *                          It's good style to end this with '/'.
- * @param   ...             Arguments to pszPathFormat.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetChildF(PCFGMNODE pNode, const char *pszPathFormat, ...)
-{
-    va_list Args;
-    va_start(Args,  pszPathFormat);
-    PCFGMNODE pRet = CFGMR3GetChildFV(pNode, pszPathFormat, Args);
-    va_end(Args);
-    return pRet;
-}
-
-
-/**
- * Query a child node by a format string.
- *
- * @returns Pointer to the specified node.
- * @returns NULL if node was not found or pNode is NULL.
- * @param   pNode           Node pszPath is relative to.
- * @param   pszPathFormat   Path to the child node or pNode.
- *                          It's good style to end this with '/'.
- * @param   Args            Arguments to pszPathFormat.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetChildFV(PCFGMNODE pNode, const char *pszPathFormat, va_list Args)
-{
-    char *pszPath;
-    RTStrAPrintfV(&pszPath, pszPathFormat, Args);
-    if (pszPath)
-    {
-        PCFGMNODE pChild;
-        int rc = cfgmR3ResolveNode(pNode, pszPath, &pChild);
-        RTStrFree(pszPath);
-        if (RT_SUCCESS(rc))
-            return pChild;
-    }
-    return NULL;
-}
-
-
-/**
- * Gets the first child node.
- * Use this to start an enumeration of child nodes.
- *
- * @returns Pointer to the first child.
- * @returns NULL if no children.
- * @param   pNode           Node to enumerate children for.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetFirstChild(PCFGMNODE pNode)
-{
-    return pNode ? pNode->pFirstChild : NULL;
-}
-
-
-/**
- * Gets the next sibling node.
- * Use this to continue an enumeration.
- *
- * @returns Pointer to the first child.
- * @returns NULL if no children.
- * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
- *                          or successive calls to this function.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3GetNextChild(PCFGMNODE pCur)
-{
-    return pCur ? pCur->pNext : NULL;
-}
-
-
-/**
- * Gets the name of the current node.
- * (Needed for enumeration.)
- *
- * @returns VBox status code.
- * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
- *                          or successive calls to CFGMR3GetNextChild().
- * @param   pszName         Where to store the node name.
- * @param   cchName         Size of the buffer pointed to by pszName (with terminator).
- */
-VMMR3DECL(int) CFGMR3GetName(PCFGMNODE pCur, char *pszName, size_t cchName)
-{
-    int rc;
-    if (pCur)
-    {
-        if (cchName > pCur->cchName)
-        {
-            rc = VINF_SUCCESS;
-            memcpy(pszName, pCur->szName, pCur->cchName + 1);
-        }
-        else
-            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
-    }
-    else
-        rc = VERR_CFGM_NO_NODE;
-    return rc;
-}
-
-
-/**
- * Gets the length of the current node's name.
- * (Needed for enumeration.)
- *
- * @returns Node name length in bytes including the terminating null char.
- * @returns 0 if pCur is NULL.
- * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
- *                          or successive calls to CFGMR3GetNextChild().
- */
-VMMR3DECL(size_t) CFGMR3GetNameLen(PCFGMNODE pCur)
-{
-    return pCur ? pCur->cchName + 1 : 0;
-}
-
-
-/**
- * Validates that the child nodes are within a set of valid names.
- *
- * @returns true if all names are found in pszzAllowed.
- * @returns false if not.
- * @param   pNode           The node which children should be examined.
- * @param   pszzValid       List of valid names separated by '\\0' and ending with
- *                          a double '\\0'.
- *
- * @deprecated Use CFGMR3ValidateConfig.
- */
-VMMR3DECL(bool) CFGMR3AreChildrenValid(PCFGMNODE pNode, const char *pszzValid)
-{
-    if (pNode)
-    {
-        for (PCFGMNODE pChild = pNode->pFirstChild; pChild; pChild = pChild->pNext)
-        {
-            /* search pszzValid for the name */
-            const char *psz = pszzValid;
-            while (*psz)
-            {
-                size_t cch = strlen(psz);
-                if (    cch == pChild->cchName
-                    &&  !memcmp(psz, pChild->szName, cch))
-                    break;
-
-                /* next */
-                psz += cch + 1;
-            }
-
-            /* if at end of pszzValid we didn't find it => failure */
-            if (!*psz)
-            {
-                AssertMsgFailed(("Couldn't find '%s' in the valid values\n", pChild->szName));
-                return false;
-            }
-        }
-    }
-
-    /* all ok. */
-    return true;
-}
-
-
-/**
- * Gets the first value of a node.
- * Use this to start an enumeration of values.
- *
- * @returns Pointer to the first value.
- * @param   pCur            The node (Key) which values to enumerate.
- */
-VMMR3DECL(PCFGMLEAF) CFGMR3GetFirstValue(PCFGMNODE pCur)
-{
-    return pCur ? pCur->pFirstLeaf : NULL;
-}
-
-/**
- * Gets the next value in enumeration.
- *
- * @returns Pointer to the next value.
- * @param   pCur            The current value as returned by this function or CFGMR3GetFirstValue().
- */
-VMMR3DECL(PCFGMLEAF) CFGMR3GetNextValue(PCFGMLEAF pCur)
-{
-    return pCur ? pCur->pNext : NULL;
-}
-
-/**
- * Get the value name.
- * (Needed for enumeration.)
- *
- * @returns VBox status code.
- * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
- *                          or successive calls to CFGMR3GetNextValue().
- * @param   pszName         Where to store the value name.
- * @param   cchName         Size of the buffer pointed to by pszName (with terminator).
- */
-VMMR3DECL(int) CFGMR3GetValueName(PCFGMLEAF pCur, char *pszName, size_t cchName)
-{
-    int rc;
-    if (pCur)
-    {
-        if (cchName > pCur->cchName)
-        {
-            rc = VINF_SUCCESS;
-            memcpy(pszName, pCur->szName, pCur->cchName + 1);
-        }
-        else
-            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
-    }
-    else
-        rc = VERR_CFGM_NO_NODE;
-    return rc;
-}
-
-
-/**
- * Gets the length of the current node's name.
- * (Needed for enumeration.)
- *
- * @returns Value name length in bytes including the terminating null char.
- * @returns 0 if pCur is NULL.
- * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
- *                          or successive calls to CFGMR3GetNextValue().
- */
-VMMR3DECL(size_t) CFGMR3GetValueNameLen(PCFGMLEAF pCur)
-{
-    return pCur ? pCur->cchName + 1 : 0;
-}
-
-
-/**
- * Gets the value type.
- * (For enumeration.)
- *
- * @returns VBox status code.
- * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
- *                          or successive calls to CFGMR3GetNextValue().
- */
-VMMR3DECL(CFGMVALUETYPE) CFGMR3GetValueType(PCFGMLEAF pCur)
-{
-    Assert(pCur);
-    return pCur->enmType;
-}
-
-
-/**
- * Validates that the values are within a set of valid names.
- *
- * @returns true if all names are found in pszzAllowed.
- * @returns false if not.
- * @param   pNode           The node which values should be examined.
- * @param   pszzValid       List of valid names separated by '\\0' and ending with
- *                          a double '\\0'.
- * @deprecated Use CFGMR3ValidateConfig.
- */
-VMMR3DECL(bool) CFGMR3AreValuesValid(PCFGMNODE pNode, const char *pszzValid)
-{
-    if (pNode)
-    {
-        for (PCFGMLEAF pLeaf = pNode->pFirstLeaf; pLeaf; pLeaf = pLeaf->pNext)
-        {
-            /* search pszzValid for the name */
-            const char *psz = pszzValid;
-            while (*psz)
-            {
-                size_t cch = strlen(psz);
-                if (    cch == pLeaf->cchName
-                    &&  !memcmp(psz, pLeaf->szName, cch))
-                    break;
-
-                /* next */
-                psz += cch + 1;
-            }
-
-            /* if at end of pszzValid we didn't find it => failure */
-            if (!*psz)
-            {
-                AssertMsgFailed(("Couldn't find '%s' in the valid values\n", pLeaf->szName));
-                return false;
-            }
-        }
-    }
-
-    /* all ok. */
-    return true;
-}
-
-
-
-/**
- * Query value type.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   penmType        Where to store the type.
- */
-VMMR3DECL(int) CFGMR3QueryType(PCFGMNODE pNode, const char *pszName, PCFGMVALUETYPE penmType)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        if (penmType)
-            *penmType = pLeaf->enmType;
-    }
-    return rc;
-}
-
-
-/**
- * Query value size.
- * This works on all types of values.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pcb             Where to store the value size.
- */
-VMMR3DECL(int) CFGMR3QuerySize(PCFGMNODE pNode, const char *pszName, size_t *pcb)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        switch (pLeaf->enmType)
-        {
-            case CFGMVALUETYPE_INTEGER:
-                *pcb = sizeof(pLeaf->Value.Integer.u64);
-                break;
-
-            case CFGMVALUETYPE_STRING:
-                *pcb = pLeaf->Value.String.cb;
-                break;
-
-            case CFGMVALUETYPE_BYTES:
-                *pcb = pLeaf->Value.Bytes.cb;
-                break;
-
-            default:
-                rc = VERR_INTERNAL_ERROR;
-                AssertMsgFailed(("Invalid value type %d\n", pLeaf->enmType));
-                break;
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Query integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu64            Where to store the integer value.
- */
-VMMR3DECL(int) CFGMR3QueryInteger(PCFGMNODE pNode, const char *pszName, uint64_t *pu64)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLeaf->enmType == CFGMVALUETYPE_INTEGER)
-            *pu64 = pLeaf->Value.Integer.u64;
-        else
-            rc = VERR_CFGM_NOT_INTEGER;
-    }
-    return rc;
-}
-
-
-/**
- * Query integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu64            Where to store the integer value. This is set to the default on failure.
- * @param   u64Def          The default value. This is always set.
- */
-VMMR3DECL(int) CFGMR3QueryIntegerDef(PCFGMNODE pNode, const char *pszName, uint64_t *pu64, uint64_t u64Def)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLeaf->enmType == CFGMVALUETYPE_INTEGER)
-            *pu64 = pLeaf->Value.Integer.u64;
-        else
-            rc = VERR_CFGM_NOT_INTEGER;
-    }
-
-    if (RT_FAILURE(rc))
-    {
-        *pu64 = u64Def;
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-            rc = VINF_SUCCESS;
-    }
-
-    return rc;
-}
-
-
-/**
- * Query zero terminated character value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of a zero terminate character value.
- * @param   pszString       Where to store the string.
- * @param   cchString       Size of the string buffer. (Includes terminator.)
- */
-VMMR3DECL(int) CFGMR3QueryString(PCFGMNODE pNode, const char *pszName, char *pszString, size_t cchString)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
-        {
-            size_t cbSrc = pLeaf->Value.String.cb;
-            if (cchString >= cbSrc)
-            {
-                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
-                memset(pszString + cbSrc, 0, cchString - cbSrc);
-            }
-            else
-                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
-        }
-        else
-            rc = VERR_CFGM_NOT_STRING;
-    }
-    return rc;
-}
-
-
-/**
- * Query zero terminated character value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of a zero terminate character value.
- * @param   pszString       Where to store the string. This will not be set on overflow error.
- * @param   cchString       Size of the string buffer. (Includes terminator.)
- * @param   pszDef          The default value.
- */
-VMMR3DECL(int) CFGMR3QueryStringDef(PCFGMNODE pNode, const char *pszName, char *pszString, size_t cchString, const char *pszDef)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
-        {
-            size_t cbSrc = pLeaf->Value.String.cb;
-            if (cchString >= cbSrc)
-            {
-                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
-                memset(pszString + cbSrc, 0, cchString - cbSrc);
-            }
-            else
-                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
-        }
-        else
-            rc = VERR_CFGM_NOT_STRING;
-    }
-
-    if (RT_FAILURE(rc) && rc != VERR_CFGM_NOT_ENOUGH_SPACE)
-    {
-        size_t cchDef = strlen(pszDef);
-        if (cchString > cchDef)
-        {
-            memcpy(pszString, pszDef, cchDef);
-            memset(pszString + cchDef, 0, cchString - cchDef);
-            if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-                rc = VINF_SUCCESS;
-        }
-        else if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
-    }
-
-    return rc;
-}
-
-
-/**
- * Query byte string value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of a byte string value.
- * @param   pvData          Where to store the binary data.
- * @param   cbData          Size of buffer pvData points too.
- */
-VMMR3DECL(int) CFGMR3QueryBytes(PCFGMNODE pNode, const char *pszName, void *pvData, size_t cbData)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLeaf->enmType == CFGMVALUETYPE_BYTES)
-        {
-            if (cbData >= pLeaf->Value.Bytes.cb)
-            {
-                memcpy(pvData, pLeaf->Value.Bytes.pau8, pLeaf->Value.Bytes.cb);
-                memset((char *)pvData + pLeaf->Value.Bytes.cb, 0, cbData - pLeaf->Value.Bytes.cb);
-            }
-            else
-                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
-        }
-        else
-            rc = VERR_CFGM_NOT_BYTES;
-    }
-    return rc;
-}
-
-
-/**
- * Validate one level of a configuration node.
- *
- * This replaces the CFGMR3AreChildrenValid and CFGMR3AreValuesValid APIs.
- *
- * @returns VBox status code.
- *
- *          When an error is returned, both VMSetError and AssertLogRelMsgFailed
- *          have been called.  So, all the caller needs to do is to propagate
- *          the error status up to PDM.
- *
- * @param   pNode               The node to validate.
- * @param   pszNode             The node path, always ends with a slash.  Use
- *                              "/" for the root config node.
- * @param   pszValidValues      Patterns describing the valid value names.  See
- *                              RTStrSimplePatternMultiMatch for details on the
- *                              pattern syntax.
- * @param   pszValidNodes       Patterns describing the valid node (key) names.
- *                              See RTStrSimplePatternMultiMatch for details on
- *                              the pattern syntax.
- * @param   pszWho              Who is calling.
- * @param   uInstance           The instance number of the caller.
- */
-VMMR3DECL(int) CFGMR3ValidateConfig(PCFGMNODE pNode, const char *pszNode,
-                                    const char *pszValidValues, const char *pszValidNodes,
-                                    const char *pszWho, uint32_t uInstance)
-{
-    /* Input validation. */
-    AssertPtrNullReturn(pNode, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszNode, VERR_INVALID_POINTER);
-    Assert(*pszNode && pszNode[strlen(pszNode) - 1] == '/');
-    AssertPtrReturn(pszValidValues, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszValidNodes, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszWho, VERR_INVALID_POINTER);
-
-    if (pNode)
-    {
-        /*
-         * Enumerate the leafs and check them against pszValidValues.
-         */
-        for (PCFGMLEAF pLeaf = pNode->pFirstLeaf; pLeaf; pLeaf = pLeaf->pNext)
-        {
-            if (!RTStrSimplePatternMultiMatch(pszValidValues, RTSTR_MAX,
-                                              pLeaf->szName, pLeaf->cchName,
-                                              NULL))
-            {
-                AssertLogRelMsgFailed(("%s/%u: Value '%s/%s' didn't match '%s'\n",
-                                       pszWho, uInstance, pszNode, pLeaf->szName, pszValidValues));
-                return VMSetError(pNode->pVM, VERR_CFGM_CONFIG_UNKNOWN_VALUE, RT_SRC_POS,
-                                  N_("Unknown configuration value '%s/%s' found in the configuration of %s instance #%u"),
-                                  pszNode, pLeaf->szName, pszWho, uInstance);
-            }
-
-        }
-
-        /*
-         * Enumerate the child nodes and check them against pszValidNodes.
-         */
-        for (PCFGMNODE pChild = pNode->pFirstChild; pChild; pChild = pChild->pNext)
-        {
-            if (!RTStrSimplePatternMultiMatch(pszValidNodes, RTSTR_MAX,
-                                              pChild->szName, pChild->cchName,
-                                              NULL))
-            {
-                AssertLogRelMsgFailed(("%s/%u: Node '%s/%s' didn't match '%s'\n",
-                                       pszWho, uInstance, pszNode, pChild->szName, pszValidNodes));
-                return VMSetError(pNode->pVM, VERR_CFGM_CONFIG_UNKNOWN_NODE, RT_SRC_POS,
-                                  N_("Unknown configuration node '%s/%s' found in the configuration of %s instance #%u"),
-                                  pszNode, pChild->szName, pszWho, uInstance);
-            }
-        }
-    }
-
-    /* All is well. */
-    return VINF_SUCCESS;
-}
-
-
-
-/**
- * Populates the CFGM tree with the default configuration.
- *
- * This assumes an empty tree and is intended for testcases and such that only
- * need to do very small adjustments to the config.
- *
- * @returns VBox status code.
- * @param   pVM     VM handle.
- */
-VMMR3DECL(int) CFGMR3ConstructDefaultTree(PVM pVM)
-{
-    int rc;
-    int rcAll = VINF_SUCCESS;
-#define UPDATERC() do { if (RT_FAILURE(rc) && RT_SUCCESS(rcAll)) rcAll = rc; } while (0)
-
-    PCFGMNODE pRoot = CFGMR3GetRoot(pVM);
-    AssertReturn(pRoot, VERR_WRONG_ORDER);
-
-    /*
-     * Create VM default values.
-     */
-    rc = CFGMR3InsertString(pRoot,  "Name",                 "Default VM");
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pRoot, "RamSize",              128U * _1M);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pRoot, "RamHoleSize",          512U * _1M);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pRoot, "TimerMillies",         10);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pRoot, "RawR3Enabled",         1);
-    UPDATERC();
-    /** @todo CFGM Defaults: RawR0, PATMEnabled and CASMEnabled needs attention later. */
-    rc = CFGMR3InsertInteger(pRoot, "RawR0Enabled",         1);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pRoot, "PATMEnabled",          1);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pRoot, "CSAMEnabled",          1);
-    UPDATERC();
-
-    /*
-     * PDM.
-     */
-    PCFGMNODE pPdm;
-    rc = CFGMR3InsertNode(pRoot, "PDM", &pPdm);
-    UPDATERC();
-    PCFGMNODE pDevices = NULL;
-    rc = CFGMR3InsertNode(pPdm, "Devices", &pDevices);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pDevices, "LoadBuiltin",       1); /* boolean */
-    UPDATERC();
-    PCFGMNODE pDrivers = NULL;
-    rc = CFGMR3InsertNode(pPdm, "Drivers", &pDrivers);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pDrivers, "LoadBuiltin",       1); /* boolean */
-    UPDATERC();
-
-
-    /*
-     * Devices
-     */
-    pDevices = NULL;
-    rc = CFGMR3InsertNode(pRoot, "Devices", &pDevices);
-    UPDATERC();
-    /* device */
-    PCFGMNODE pDev = NULL;
-    PCFGMNODE pInst = NULL;
-    PCFGMNODE pCfg = NULL;
-#if 0
-    PCFGMNODE pLunL0 = NULL;
-    PCFGMNODE pLunL1 = NULL;
-#endif
-
-    /*
-     * PC Arch.
-     */
-    rc = CFGMR3InsertNode(pDevices, "pcarch", &pDev);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-
-    /*
-     * PC Bios.
-     */
-    rc = CFGMR3InsertNode(pDevices, "pcbios", &pDev);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pCfg,  "RamSize",              128U * _1M);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pCfg,  "RamHoleSize",          512U * _1M);
-    UPDATERC();
-    rc = CFGMR3InsertString(pCfg,   "BootDevice0",          "IDE");
-    UPDATERC();
-    rc = CFGMR3InsertString(pCfg,   "BootDevice1",          "NONE");
-    UPDATERC();
-    rc = CFGMR3InsertString(pCfg,   "BootDevice2",          "NONE");
-    UPDATERC();
-    rc = CFGMR3InsertString(pCfg,   "BootDevice3",          "NONE");
-    UPDATERC();
-    rc = CFGMR3InsertString(pCfg,   "HardDiskDevice",       "piix3ide");
-    UPDATERC();
-    rc = CFGMR3InsertString(pCfg,   "FloppyDevice",         "");
-    UPDATERC();
-    RTUUID Uuid;
-    RTUuidClear(&Uuid);
-    rc = CFGMR3InsertBytes(pCfg,    "UUID", &Uuid, sizeof(Uuid));
-    UPDATERC();
-
-    /*
-     * PCI bus.
-     */
-    rc = CFGMR3InsertNode(pDevices, "pci", &pDev); /* piix3 */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-
-    /*
-     * PS/2 keyboard & mouse
-     */
-    rc = CFGMR3InsertNode(pDevices, "pckbd", &pDev);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-#if 0
-    rc = CFGMR3InsertNode(pInst,    "LUN#0", &pLunL0);
-    UPDATERC();
-    rc = CFGMR3InsertString(pLunL0, "Driver",               "KeyboardQueue");
-    UPDATERC();
-    rc = CFGMR3InsertNode(pLunL0,   "Config", &pCfg);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pCfg,  "QueueSize",            64);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pLunL0,   "AttachedDriver", &pLunL1);
-    UPDATERC();
-    rc = CFGMR3InsertString(pLunL1, "Driver",               "MainKeyboard");
-    UPDATERC();
-    rc = CFGMR3InsertNode(pLunL1,   "Config", &pCfg);
-    UPDATERC();
-#endif
-#if 0
-    rc = CFGMR3InsertNode(pInst,    "LUN#1", &pLunL0);
-    UPDATERC();
-    rc = CFGMR3InsertString(pLunL0, "Driver",               "MouseQueue");
-    UPDATERC();
-    rc = CFGMR3InsertNode(pLunL0,   "Config", &pCfg);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pCfg,  "QueueSize",            128);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pLunL0,   "AttachedDriver", &pLunL1);
-    UPDATERC();
-    rc = CFGMR3InsertString(pLunL1, "Driver",               "MainMouse");
-    UPDATERC();
-    rc = CFGMR3InsertNode(pLunL1,   "Config", &pCfg);
-    UPDATERC();
-#endif
-
-    /*
-     * i8254 Programmable Interval Timer And Dummy Speaker
-     */
-    rc = CFGMR3InsertNode(pDevices, "i8254", &pDev);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-#ifdef DEBUG
-    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
-    UPDATERC();
-#endif
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-
-    /*
-     * i8259 Programmable Interrupt Controller.
-     */
-    rc = CFGMR3InsertNode(pDevices, "i8259", &pDev);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-
-    /*
-     * RTC MC146818.
-     */
-    rc = CFGMR3InsertNode(pDevices, "mc146818", &pDev);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-
-    /*
-     * VGA.
-     */
-    rc = CFGMR3InsertNode(pDevices, "vga", &pDev);
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pCfg,  "VRamSize",             4 * _1M);
-    UPDATERC();
-
-    /* Bios logo. */
-    rc = CFGMR3InsertInteger(pCfg,  "FadeIn",               1);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pCfg,  "FadeOut",              1);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pCfg,  "LogoTime",             0);
-    UPDATERC();
-    rc = CFGMR3InsertString(pCfg,   "LogoFile",             "");
-    UPDATERC();
-
-#if 0
-    rc = CFGMR3InsertNode(pInst,    "LUN#0", &pLunL0);
-    UPDATERC();
-    rc = CFGMR3InsertString(pLunL0, "Driver",               "MainDisplay");
-    UPDATERC();
-#endif
-
-    /*
-     * IDE controller.
-     */
-    rc = CFGMR3InsertNode(pDevices, "piix3ide", &pDev); /* piix3 */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
-    UPDATERC();
-    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
-    UPDATERC();
-    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
-    UPDATERC();
-
-
-
-    /*
-     * ...
-     */
-
-#undef UPDATERC
-    return rcAll;
-}
-
-
-
-
-/**
- * Resolves a path reference to a child node.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszPath         Path to the child node.
- * @param   ppChild         Where to store the pointer to the child node.
- */
-static int cfgmR3ResolveNode(PCFGMNODE pNode, const char *pszPath, PCFGMNODE *ppChild)
-{
-    if (!pNode)
-        return VERR_CFGM_NO_PARENT;
-    PCFGMNODE pChild = NULL;
-    for (;;)
-    {
-        /* skip leading slashes. */
-        while (*pszPath == '/')
-            pszPath++;
-
-        /* End of path? */
-        if (!*pszPath)
-        {
-            if (!pChild)
-                return VERR_CFGM_INVALID_CHILD_PATH;
-            *ppChild = pChild;
-            return VINF_SUCCESS;
-        }
-
-        /* find end of component. */
-        const char *pszNext = strchr(pszPath, '/');
-        if (!pszNext)
-            pszNext = strchr(pszPath,  '\0');
-        RTUINT cchName = pszNext - pszPath;
-
-        /* search child list. */
-        pChild = pNode->pFirstChild;
-        for ( ; pChild; pChild = pChild->pNext)
-            if (pChild->cchName == cchName)
-            {
-                int iDiff = memcmp(pszPath, pChild->szName, cchName);
-                if (iDiff <= 0)
-                {
-                    if (iDiff != 0)
-                        pChild = NULL;
-                    break;
-                }
-            }
-        if (!pChild)
-            return VERR_CFGM_CHILD_NOT_FOUND;
-
-        /* next iteration */
-        pNode = pChild;
-        pszPath = pszNext;
-    }
-
-    /* won't get here */
-}
-
-
-/**
- * Resolves a path reference to a child node.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of a byte string value.
- * @param   ppLeaf          Where to store the pointer to the leaf node.
- */
-static int cfgmR3ResolveLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf)
-{
-    if (!pNode)
-        return VERR_CFGM_NO_PARENT;
-
-    size_t      cchName = strlen(pszName);
-    PCFGMLEAF   pLeaf   = pNode->pFirstLeaf;
-    while (pLeaf)
-    {
-        if (cchName == pLeaf->cchName)
-        {
-            int iDiff = memcmp(pszName, pLeaf->szName, cchName);
-            if (iDiff <= 0)
-            {
-                if (iDiff != 0)
-                    break;
-                *ppLeaf = pLeaf;
-                return VINF_SUCCESS;
-            }
-        }
-
-        /* next */
-        pLeaf = pLeaf->pNext;
-    }
-    return VERR_CFGM_VALUE_NOT_FOUND;
-}
-
-
-
-/**
- * Creates a CFGM tree.
- *
- * This is intended for creating device/driver configs can be
- * passed around and later attached to the main tree in the
- * correct location.
- *
- * @returns Pointer to the root node.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(PCFGMNODE) CFGMR3CreateTree(PVM pVM)
-{
-    PCFGMNODE pNew = (PCFGMNODE)MMR3HeapAlloc(pVM, MM_TAG_CFGM, sizeof(*pNew));
-    if (pNew)
-    {
-        pNew->pPrev         = NULL;
-        pNew->pNext         = NULL;
-        pNew->pParent       = NULL;
-        pNew->pFirstChild   = NULL;
-        pNew->pFirstLeaf    = NULL;
-        pNew->pVM           = pVM;
-        pNew->fRestrictedRoot = false;
-        pNew->cchName       = 0;
-        pNew->szName[0]     = 0;
-    }
-    return pNew;
-}
-
-
-/**
- * Insert subtree.
- *
- * This function inserts (no duplication) a tree created by CFGMR3CreateTree()
- * into the main tree.
- *
- * The root node of the inserted subtree will need to be reallocated, which
- * effectually means that the passed in pSubTree handle becomes invalid
- * upon successful return. Use the value returned in ppChild instead
- * of pSubTree.
- *
- * @returns VBox status code.
- * @returns VERR_CFGM_NODE_EXISTS if the final child node name component exists.
- * @param   pNode       Parent node.
- * @param   pszName     Name or path of the new child node.
- * @param   pSubTree    The subtree to insert. Must be returned by CFGMR3CreateTree().
- * @param   ppChild     Where to store the address of the new child node. (optional)
- */
-VMMR3DECL(int) CFGMR3InsertSubTree(PCFGMNODE pNode, const char *pszName, PCFGMNODE pSubTree, PCFGMNODE *ppChild)
-{
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pSubTree, VERR_INVALID_POINTER);
-    AssertReturn(!pSubTree->pParent, VERR_INVALID_PARAMETER);
-    AssertReturn(pSubTree->pVM, VERR_INVALID_PARAMETER);
-    AssertReturn(pSubTree->pParent != pSubTree->pVM->cfgm.s.pRoot, VERR_INVALID_PARAMETER);
-    Assert(!pSubTree->pNext);
-    Assert(!pSubTree->pPrev);
-
-    /*
-     * Use CFGMR3InsertNode to create a new node and then
-     * re-attach the children and leafs of the subtree to it.
-     */
-    PCFGMNODE pNewChild;
-    int rc = CFGMR3InsertNode(pNode, pszName, &pNewChild);
-    if (RT_SUCCESS(rc))
-    {
-        Assert(!pNewChild->pFirstChild);
-        pNewChild->pFirstChild = pSubTree->pFirstChild;
-        Assert(!pNewChild->pFirstLeaf);
-        pNewChild->pFirstLeaf = pSubTree->pFirstLeaf;
-        if (ppChild)
-            *ppChild = pNewChild;
-
-        /* free the old subtree root */
-        pSubTree->pVM = NULL;
-        pSubTree->pFirstLeaf = NULL;
-        pSubTree->pFirstChild = NULL;
-        MMR3HeapFree(pSubTree);
-    }
-    return rc;
-}
-
-
-/**
- * Compares two names.
- *
- * @returns Similar to memcpy.
- * @param   pszName1            The first name.
- * @param   cchName1            The length of the first name.
- * @param   pszName2            The second name.
- * @param   cchName2            The length of the second name.
- */
-DECLINLINE(int) cfgmR3CompareNames(const char *pszName1, size_t cchName1, const char *pszName2, size_t cchName2)
-{
-    int iDiff;
-    if (cchName1 <= cchName2)
-    {
-        iDiff = memcmp(pszName1, pszName2, cchName1);
-        if (!iDiff && cchName1 < cchName2)
-            iDiff = -1;
-    }
-    else
-    {
-        iDiff = memcmp(pszName1, pszName2, cchName2);
-        if (!iDiff)
-            iDiff = 1;
-    }
-    return iDiff;
-}
-
-
-/**
- * Insert a node.
- *
- * @returns VBox status code.
- * @returns VERR_CFGM_NODE_EXISTS if the final child node name component exists.
- * @param   pNode       Parent node.
- * @param   pszName     Name or path of the new child node.
- * @param   ppChild     Where to store the address of the new child node. (optional)
- */
-VMMR3DECL(int) CFGMR3InsertNode(PCFGMNODE pNode, const char *pszName, PCFGMNODE *ppChild)
-{
-    int rc;
-    if (pNode)
-    {
-        /*
-         * If given a path we have to deal with it component by component.
-         */
-        while (*pszName == '/')
-            pszName++;
-        if (strchr(pszName, '/'))
-        {
-            char *pszDup = RTStrDup(pszName);
-            if (pszDup)
-            {
-                char *psz = pszDup;
-                for (;;)
-                {
-                    /* Terminate at '/' and find the next component. */
-                    char *pszNext = strchr(psz, '/');
-                    if (pszNext)
-                    {
-                        *pszNext++ = '\0';
-                        while (*pszNext == '/')
-                            pszNext++;
-                        if (*pszNext == '\0')
-                            pszNext = NULL;
-                    }
-
-                    /* does it exist? */
-                    PCFGMNODE pChild = CFGMR3GetChild(pNode, psz);
-                    if (!pChild)
-                    {
-                        /* no, insert it */
-                        rc = CFGMR3InsertNode(pNode, psz, &pChild);
-                        if (RT_FAILURE(rc))
-                            break;
-                        if (!pszNext)
-                        {
-                            if (ppChild)
-                                *ppChild = pChild;
-                            break;
-                        }
-
-                    }
-                    /* if last component fail */
-                    else if (!pszNext)
-                    {
-                        rc = VERR_CFGM_NODE_EXISTS;
-                        break;
-                    }
-
-                    /* next */
-                    pNode = pChild;
-                    psz = pszNext;
-                }
-                RTStrFree(pszDup);
-            }
-            else
-                rc = VERR_NO_TMP_MEMORY;
-        }
-        /*
-         * Not multicomponent, just make sure it's a non-zero name.
-         */
-        else if (*pszName)
-        {
-            /*
-             * Check if already exists and find last node in chain.
-             */
-            size_t    cchName = strlen(pszName);
-            PCFGMNODE pPrev   = NULL;
-            PCFGMNODE pNext   = pNode->pFirstChild;
-            if (pNext)
-            {
-                for ( ; pNext; pPrev = pNext, pNext = pNext->pNext)
-                {
-                    int iDiff = cfgmR3CompareNames(pszName, cchName, pNext->szName, pNext->cchName);
-                    if (iDiff <= 0)
-                    {
-                        if (!iDiff)
-                            return VERR_CFGM_NODE_EXISTS;
-                        break;
-                    }
-                }
-            }
-
-            /*
-             * Allocate and init node.
-             */
-            PCFGMNODE pNew = (PCFGMNODE)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM, sizeof(*pNew) + cchName);
-            if (pNew)
-            {
-                pNew->pParent       = pNode;
-                pNew->pFirstChild   = NULL;
-                pNew->pFirstLeaf    = NULL;
-                pNew->pVM           = pNode->pVM;
-                pNew->fRestrictedRoot = false;
-                pNew->cchName       = cchName;
-                memcpy(pNew->szName, pszName, cchName + 1);
-
-                /*
-                 * Insert into child list.
-                 */
-                pNew->pPrev         = pPrev;
-                if (pPrev)
-                    pPrev->pNext    = pNew;
-                else
-                    pNode->pFirstChild = pNew;
-                pNew->pNext         = pNext;
-                if (pNext)
-                    pNext->pPrev    = pNew;
-
-                if (ppChild)
-                    *ppChild = pNew;
-                rc = VINF_SUCCESS;
-            }
-            else
-                rc = VERR_NO_MEMORY;
-        }
-        else
-        {
-            rc = VERR_CFGM_INVALID_NODE_PATH;
-            AssertMsgFailed(("Invalid path %s\n", pszName));
-        }
-    }
-    else
-    {
-        rc = VERR_CFGM_NO_PARENT;
-        AssertMsgFailed(("No parent! path %s\n", pszName));
-    }
-
-    return rc;
-}
-
-
-/**
- * Insert a node, format string name.
- *
- * @returns VBox status     code.
- * @param   pNode           Parent node.
- * @param   ppChild         Where to store the address of the new child node. (optional)
- * @param   pszNameFormat   Name of or path the new child node.
- * @param   ...             Name format arguments.
- */
-VMMR3DECL(int) CFGMR3InsertNodeF(PCFGMNODE pNode, PCFGMNODE *ppChild, const char *pszNameFormat, ...)
-{
-    va_list Args;
-    va_start(Args,  pszNameFormat);
-    int rc = CFGMR3InsertNodeFV(pNode, ppChild, pszNameFormat, Args);
-    va_end(Args);
-    return rc;
-}
-
-
-/**
- * Insert a node, format string name.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   ppChild         Where to store the address of the new child node. (optional)
- * @param   pszNameFormat   Name or path of the new child node.
- * @param   Args            Name format arguments.
- */
-VMMR3DECL(int) CFGMR3InsertNodeFV(PCFGMNODE pNode, PCFGMNODE *ppChild, const char *pszNameFormat, va_list Args)
-{
-    int     rc;
-    char   *pszName;
-    RTStrAPrintfV(&pszName, pszNameFormat, Args);
-    if (pszName)
-    {
-        rc = CFGMR3InsertNode(pNode, pszName, ppChild);
-        RTStrFree(pszName);
-    }
-    else
-        rc = VERR_NO_MEMORY;
-    return rc;
-}
-
-
-/**
- * Marks the node as the root of a restricted subtree, i.e. the end of
- * a CFGMR3GetParent() journey.
- *
- * @param   pNode       The node to mark.
- */
-VMMR3DECL(void) CFGMR3SetRestrictedRoot(PCFGMNODE pNode)
-{
-    if (pNode)
-        pNode->fRestrictedRoot = true;
-}
-
-
-/**
- * Insert a node.
- *
- * @returns VBox status code.
- * @param   pNode       Parent node.
- * @param   pszName     Name of the new child node.
- * @param   ppLeaf      Where to store the new leaf.
- *                      The caller must fill in the enmType and Value fields!
- */
-static int cfgmR3InsertLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf)
-{
-    int rc;
-    if (*pszName)
-    {
-        if (pNode)
-        {
-            /*
-             * Check if already exists and find last node in chain.
-             */
-            size_t    cchName  = strlen(pszName);
-            PCFGMLEAF pPrev    = NULL;
-            PCFGMLEAF pNext    = pNode->pFirstLeaf;
-            if (pNext)
-            {
-                for ( ; pNext; pPrev = pNext, pNext = pNext->pNext)
-                {
-                    int iDiff = cfgmR3CompareNames(pszName, cchName, pNext->szName, pNext->cchName);
-                    if (iDiff <= 0)
-                    {
-                        if (!iDiff)
-                            return VERR_CFGM_LEAF_EXISTS;
-                        break;
-                    }
-                }
-            }
-
-            /*
-             * Allocate and init node.
-             */
-            PCFGMLEAF pNew = (PCFGMLEAF)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM, sizeof(*pNew) + cchName);
-            if (pNew)
-            {
-                pNew->cchName       = cchName;
-                memcpy(pNew->szName, pszName, cchName + 1);
-
-                /*
-                 * Insert into child list.
-                 */
-                pNew->pPrev         = pPrev;
-                if (pPrev)
-                    pPrev->pNext    = pNew;
-                else
-                    pNode->pFirstLeaf = pNew;
-                pNew->pNext         = pNext;
-                if (pNext)
-                    pNext->pPrev    = pNew;
-
-                *ppLeaf = pNew;
-                rc = VINF_SUCCESS;
-            }
-            else
-                rc = VERR_NO_MEMORY;
-        }
-        else
-            rc = VERR_CFGM_NO_PARENT;
-    }
-    else
-        rc = VERR_CFGM_INVALID_CHILD_PATH;
-    return rc;
-}
-
-
-/**
- * Remove a node.
- *
- * @param   pNode       Parent node.
- */
-VMMR3DECL(void) CFGMR3RemoveNode(PCFGMNODE pNode)
-{
-    if (pNode)
-    {
-        /*
-         * Free children.
-         */
-        while (pNode->pFirstChild)
-            CFGMR3RemoveNode(pNode->pFirstChild);
-
-        /*
-         * Free leafs.
-         */
-        while (pNode->pFirstLeaf)
-            cfgmR3RemoveLeaf(pNode, pNode->pFirstLeaf);
-
-        /*
-         * Unlink ourselves.
-         */
-        if (pNode->pPrev)
-            pNode->pPrev->pNext = pNode->pNext;
-        else
-        {
-            if (pNode->pParent)
-                pNode->pParent->pFirstChild = pNode->pNext;
-            else if (pNode == pNode->pVM->cfgm.s.pRoot) /* might be a different tree */
-                pNode->pVM->cfgm.s.pRoot = NULL;
-        }
-        if (pNode->pNext)
-            pNode->pNext->pPrev = pNode->pPrev;
-
-        /*
-         * Free ourselves. (bit of paranoia first)
-         */
-        pNode->pVM = NULL;
-        pNode->pNext = NULL;
-        pNode->pPrev = NULL;
-        pNode->pParent = NULL;
-        MMR3HeapFree(pNode);
-    }
-}
-
-
-/**
- * Removes a leaf.
- *
- * @param   pNode   Parent node.
- * @param   pLeaf   Leaf to remove.
- */
-static void cfgmR3RemoveLeaf(PCFGMNODE pNode, PCFGMLEAF pLeaf)
-{
-    if (pNode && pLeaf)
-    {
-        /*
-         * Unlink.
-         */
-        if (pLeaf->pPrev)
-            pLeaf->pPrev->pNext = pLeaf->pNext;
-        else
-            pNode->pFirstLeaf = pLeaf->pNext;
-        if (pLeaf->pNext)
-            pLeaf->pNext->pPrev = pLeaf->pPrev;
-
-        /*
-         * Free value and node.
-         */
-        cfgmR3FreeValue(pLeaf);
-        pLeaf->pNext = NULL;
-        pLeaf->pPrev = NULL;
-        MMR3HeapFree(pLeaf);
-    }
-}
-
-
-/**
- * Frees whatever resources the leaf value is owning.
- *
- * Use this before assigning a new value to a leaf.
- * The caller must either free the leaf or assign a new value to it.
- *
- * @param   pLeaf       Pointer to the leaf which value should be free.
- */
-static void cfgmR3FreeValue(PCFGMLEAF pLeaf)
-{
-    if (pLeaf)
-    {
-        switch (pLeaf->enmType)
-        {
-            case CFGMVALUETYPE_BYTES:
-                MMR3HeapFree(pLeaf->Value.Bytes.pau8);
-                pLeaf->Value.Bytes.pau8 = NULL;
-                pLeaf->Value.Bytes.cb = 0;
-                break;
-
-            case CFGMVALUETYPE_STRING:
-                MMR3HeapFree(pLeaf->Value.String.psz);
-                pLeaf->Value.String.psz = NULL;
-                pLeaf->Value.String.cb = 0;
-                break;
-
-            case CFGMVALUETYPE_INTEGER:
-                break;
-        }
-        pLeaf->enmType = (CFGMVALUETYPE)0;
-    }
-}
-
-
-/**
- * Inserts a new integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   pszName         Value name.
- * @param   u64Integer      The value.
- */
-VMMR3DECL(int) CFGMR3InsertInteger(PCFGMNODE pNode, const char *pszName, uint64_t u64Integer)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        pLeaf->enmType = CFGMVALUETYPE_INTEGER;
-        pLeaf->Value.Integer.u64 = u64Integer;
-    }
-    return rc;
-}
-
-
-/**
- * Inserts a new string value. This variant expects that the caller know the length
- * of the string already so we can avoid calling strlen() here.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   pszName         Value name.
- * @param   pszString       The value. Must not be NULL.
- * @param   cchString       The length of the string excluding the
- *                          terminator.
- */
-VMMR3DECL(int) CFGMR3InsertStringN(PCFGMNODE pNode, const char *pszName, const char *pszString, size_t cchString)
-{
-    Assert(RTStrNLen(pszString, cchString) == cchString);
-
-    int rc;
-    if (pNode)
-    {
-        /*
-         * Allocate string object first.
-         */
-        char *pszStringCopy = (char *)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_STRING, cchString + 1);
-        if (pszStringCopy)
-        {
-            memcpy(pszStringCopy, pszString, cchString);
-            pszStringCopy[cchString] = '\0';
-
-            /*
-             * Create value leaf and set it to string type.
-             */
-            PCFGMLEAF pLeaf;
-            rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
-            if (RT_SUCCESS(rc))
-            {
-                pLeaf->enmType = CFGMVALUETYPE_STRING;
-                pLeaf->Value.String.psz = pszStringCopy;
-                pLeaf->Value.String.cb  = cchString + 1;
-            }
-            else
-                MMR3HeapFree(pszStringCopy);
-        }
-        else
-            rc = VERR_NO_MEMORY;
-    }
-    else
-        rc = VERR_CFGM_NO_PARENT;
-
-    return rc;
-}
-
-
-/**
- * Inserts a new string value. Calls strlen(pszString) internally; if you know the
- * length of the string, CFGMR3InsertStringLengthKnown() is faster.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   pszName         Value name.
- * @param   pszString       The value.
- */
-VMMR3DECL(int) CFGMR3InsertString(PCFGMNODE pNode, const char *pszName, const char *pszString)
-{
-    return CFGMR3InsertStringN(pNode, pszName, pszString, strlen(pszString));
-}
-
-
-/**
- * Same as CFGMR3InsertString except the string value given in RTStrPrintfV
- * fashion.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   pszName         Value name.
- * @param   pszFormat       The value given as a format string.
- * @param   va              Argument to pszFormat.
- */
-VMMR3DECL(int) CFGMR3InsertStringFV(PCFGMNODE pNode, const char *pszName, const char *pszFormat, va_list va)
-{
-    int rc;
-    if (pNode)
-    {
-        /*
-         * Allocate string object first.
-         */
-        char *pszString = MMR3HeapAPrintfVU(pNode->pVM->pUVM, MM_TAG_CFGM_STRING, pszFormat, va);
-        if (pszString)
-        {
-            /*
-             * Create value leaf and set it to string type.
-             */
-            PCFGMLEAF pLeaf;
-            rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
-            if (RT_SUCCESS(rc))
-            {
-                pLeaf->enmType = CFGMVALUETYPE_STRING;
-                pLeaf->Value.String.psz = pszString;
-                pLeaf->Value.String.cb  = strlen(pszString) + 1;
-            }
-            else
-                MMR3HeapFree(pszString);
-        }
-        else
-            rc = VERR_NO_MEMORY;
-    }
-    else
-        rc = VERR_CFGM_NO_PARENT;
-
-    return rc;
-}
-
-
-/**
- * Same as CFGMR3InsertString except the string value given in RTStrPrintf
- * fashion.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   pszName         Value name.
- * @param   pszFormat       The value given as a format string.
- * @param   ...             Argument to pszFormat.
- */
-VMMR3DECL(int) CFGMR3InsertStringF(PCFGMNODE pNode, const char *pszName, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    int rc = CFGMR3InsertStringFV(pNode, pszName, pszFormat, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Same as CFGMR3InsertString except the string value given as a UTF-16 string.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   pszName         Value name.
- * @param   pwszValue       The string value (UTF-16).
- */
-VMMR3DECL(int) CFGMR3InsertStringW(PCFGMNODE pNode, const char *pszName, PCRTUTF16 pwszValue)
-{
-    char *pszValue;
-    int rc = RTUtf16ToUtf8(pwszValue, &pszValue);
-    if (RT_SUCCESS(rc))
-    {
-        rc = CFGMR3InsertString(pNode, pszName, pszValue);
-        RTStrFree(pszValue);
-    }
-    return rc;
-}
-
-
-/**
- * Inserts a new integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Parent node.
- * @param   pszName         Value name.
- * @param   pvBytes         The value.
- * @param   cbBytes         The value size.
- */
-VMMR3DECL(int) CFGMR3InsertBytes(PCFGMNODE pNode, const char *pszName, const void *pvBytes, size_t cbBytes)
-{
-    int rc;
-    if (pNode)
-    {
-        if (cbBytes == (RTUINT)cbBytes)
-        {
-            /*
-             * Allocate string object first.
-             */
-            void *pvCopy = MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_STRING, cbBytes);
-            if (pvCopy || !cbBytes)
-            {
-                memcpy(pvCopy, pvBytes, cbBytes);
-
-                /*
-                 * Create value leaf and set it to string type.
-                 */
-                PCFGMLEAF pLeaf;
-                rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
-                if (RT_SUCCESS(rc))
-                {
-                    pLeaf->enmType = CFGMVALUETYPE_BYTES;
-                    pLeaf->Value.Bytes.cb   = cbBytes;
-                    pLeaf->Value.Bytes.pau8 = (uint8_t *)pvCopy;
-                }
-            }
-            else
-                rc = VERR_NO_MEMORY;
-        }
-        else
-            rc = VERR_OUT_OF_RANGE;
-    }
-    else
-        rc = VERR_CFGM_NO_PARENT;
-
-    return rc;
-}
-
-
-/**
- * Remove a value.
- *
- * @returns VBox status code.
- * @param   pNode       Parent node.
- * @param   pszName     Name of the new child node.
- */
-VMMR3DECL(int) CFGMR3RemoveValue(PCFGMNODE pNode, const char *pszName)
-{
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-        cfgmR3RemoveLeaf(pNode, pLeaf);
-    return rc;
-}
-
-
-
-/*
- *  -+- helper apis -+-
- */
-
-
-/**
- * Query unsigned 64-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu64            Where to store the integer value.
- */
-VMMR3DECL(int) CFGMR3QueryU64(PCFGMNODE pNode, const char *pszName, uint64_t *pu64)
-{
-    return CFGMR3QueryInteger(pNode, pszName, pu64);
-}
-
-
-/**
- * Query unsigned 64-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu64            Where to store the integer value. Set to default on failure.
- * @param   u64Def          The default value.
- */
-VMMR3DECL(int) CFGMR3QueryU64Def(PCFGMNODE pNode, const char *pszName, uint64_t *pu64, uint64_t u64Def)
-{
-    return CFGMR3QueryIntegerDef(pNode, pszName, pu64, u64Def);
-}
-
-
-/**
- * Query signed 64-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi64            Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryS64(PCFGMNODE pNode, const char *pszName, int64_t *pi64)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-        *pi64 = (int64_t)u64;
-    return rc;
-}
-
-
-/**
- * Query signed 64-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi64            Where to store the value. Set to default on failure.
- * @param   i64Def          The default value.
- */
-VMMR3DECL(int) CFGMR3QueryS64Def(PCFGMNODE pNode, const char *pszName, int64_t *pi64, int64_t i64Def)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i64Def);
-    *pi64 = (int64_t)u64;
-    return rc;
-}
-
-
-/**
- * Query unsigned 32-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu32            Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryU32(PCFGMNODE pNode, const char *pszName, uint32_t *pu32)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        if (!(u64 & UINT64_C(0xffffffff00000000)))
-            *pu32 = (uint32_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query unsigned 32-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu32            Where to store the value. Set to default on failure.
- * @param   u32Def          The default value.
- */
-VMMR3DECL(int) CFGMR3QueryU32Def(PCFGMNODE pNode, const char *pszName, uint32_t *pu32, uint32_t u32Def)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u32Def);
-    if (RT_SUCCESS(rc))
-    {
-        if (!(u64 & UINT64_C(0xffffffff00000000)))
-            *pu32 = (uint32_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pu32 = u32Def;
-    return rc;
-}
-
-
-/**
- * Query signed 32-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi32            Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryS32(PCFGMNODE pNode, const char *pszName, int32_t *pi32)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        if (   !(u64 & UINT64_C(0xffffffff80000000))
-            ||  (u64 & UINT64_C(0xffffffff80000000)) == UINT64_C(0xffffffff80000000))
-            *pi32 = (int32_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query signed 32-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi32            Where to store the value. Set to default on failure.
- * @param   i32Def          The default value.
- */
-VMMR3DECL(int) CFGMR3QueryS32Def(PCFGMNODE pNode, const char *pszName, int32_t *pi32, int32_t i32Def)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i32Def);
-    if (RT_SUCCESS(rc))
-    {
-        if (   !(u64 & UINT64_C(0xffffffff80000000))
-            ||  (u64 & UINT64_C(0xffffffff80000000)) == UINT64_C(0xffffffff80000000))
-            *pi32 = (int32_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pi32 = i32Def;
-    return rc;
-}
-
-
-/**
- * Query unsigned 16-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu16            Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryU16(PCFGMNODE pNode, const char *pszName, uint16_t *pu16)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        if (!(u64 & UINT64_C(0xffffffffffff0000)))
-            *pu16 = (int16_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query unsigned 16-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu16            Where to store the value. Set to default on failure.
- * @param   i16Def          The default value.
- */
-VMMR3DECL(int) CFGMR3QueryU16Def(PCFGMNODE pNode, const char *pszName, uint16_t *pu16, uint16_t u16Def)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u16Def);
-    if (RT_SUCCESS(rc))
-    {
-        if (!(u64 & UINT64_C(0xffffffffffff0000)))
-            *pu16 = (int16_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pu16 = u16Def;
-    return rc;
-}
-
-
-/**
- * Query signed 16-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi16            Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryS16(PCFGMNODE pNode, const char *pszName, int16_t *pi16)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        if (   !(u64 & UINT64_C(0xffffffffffff8000))
-            ||  (u64 & UINT64_C(0xffffffffffff8000)) == UINT64_C(0xffffffffffff8000))
-            *pi16 = (int16_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query signed 16-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi16            Where to store the value. Set to default on failure.
- * @param   i16Def          The default value.
- */
-VMMR3DECL(int) CFGMR3QueryS16Def(PCFGMNODE pNode, const char *pszName, int16_t *pi16, int16_t i16Def)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i16Def);
-    if (RT_SUCCESS(rc))
-    {
-        if (   !(u64 & UINT64_C(0xffffffffffff8000))
-            ||  (u64 & UINT64_C(0xffffffffffff8000)) == UINT64_C(0xffffffffffff8000))
-            *pi16 = (int16_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pi16 = i16Def;
-    return rc;
-}
-
-
-/**
- * Query unsigned 8-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu8             Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryU8(PCFGMNODE pNode, const char *pszName, uint8_t *pu8)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        if (!(u64 & UINT64_C(0xffffffffffffff00)))
-            *pu8 = (uint8_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query unsigned 8-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu8             Where to store the value. Set to default on failure.
- * @param   u8Def           The default value.
- */
-VMMR3DECL(int) CFGMR3QueryU8Def(PCFGMNODE pNode, const char *pszName, uint8_t *pu8, uint8_t u8Def)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u8Def);
-    if (RT_SUCCESS(rc))
-    {
-        if (!(u64 & UINT64_C(0xffffffffffffff00)))
-            *pu8 = (uint8_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pu8 = u8Def;
-    return rc;
-}
-
-
-/**
- * Query signed 8-bit integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi8             Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryS8(PCFGMNODE pNode, const char *pszName, int8_t *pi8)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        if (   !(u64 & UINT64_C(0xffffffffffffff80))
-            ||  (u64 & UINT64_C(0xffffffffffffff80)) == UINT64_C(0xffffffffffffff80))
-            *pi8 = (int8_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query signed 8-bit integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi8             Where to store the value. Set to default on failure.
- * @param   i8Def           The default value.
- */
-VMMR3DECL(int) CFGMR3QueryS8Def(PCFGMNODE pNode, const char *pszName, int8_t *pi8, int8_t i8Def)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i8Def);
-    if (RT_SUCCESS(rc))
-    {
-        if (   !(u64 & UINT64_C(0xffffffffffffff80))
-            ||  (u64 & UINT64_C(0xffffffffffffff80)) == UINT64_C(0xffffffffffffff80))
-            *pi8 = (int8_t)u64;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pi8 = i8Def;
-    return rc;
-}
-
-
-/**
- * Query boolean integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pf              Where to store the value.
- * @remark  This function will interpret any non-zero value as true.
- */
-VMMR3DECL(int) CFGMR3QueryBool(PCFGMNODE pNode, const char *pszName, bool *pf)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-        *pf = u64 ? true : false;
-    return rc;
-}
-
-
-/**
- * Query boolean integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pf              Where to store the value. Set to default on failure.
- * @param   fDef            The default value.
- * @remark  This function will interpret any non-zero value as true.
- */
-VMMR3DECL(int) CFGMR3QueryBoolDef(PCFGMNODE pNode, const char *pszName, bool *pf, bool fDef)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, fDef);
-    *pf = u64 ? true : false;
-    return rc;
-}
-
-
-/**
- * Query I/O port address value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pPort           Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryPort(PCFGMNODE pNode, const char *pszName, PRTIOPORT pPort)
-{
-    AssertCompileSize(RTIOPORT, 2);
-    return CFGMR3QueryU16(pNode, pszName, pPort);
-}
-
-
-/**
- * Query I/O port address value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pPort           Where to store the value. Set to default on failure.
- * @param   PortDef         The default value.
- */
-VMMR3DECL(int) CFGMR3QueryPortDef(PCFGMNODE pNode, const char *pszName, PRTIOPORT pPort, RTIOPORT PortDef)
-{
-    AssertCompileSize(RTIOPORT, 2);
-    return CFGMR3QueryU16Def(pNode, pszName, pPort, PortDef);
-}
-
-
-/**
- * Query unsigned int address value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu              Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryUInt(PCFGMNODE pNode, const char *pszName, unsigned int *pu)
-{
-    AssertCompileSize(unsigned int, 4);
-    return CFGMR3QueryU32(pNode, pszName, (uint32_t *)pu);
-}
-
-
-/**
- * Query unsigned int address value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pu              Where to store the value. Set to default on failure.
- * @param   uDef            The default value.
- */
-VMMR3DECL(int) CFGMR3QueryUIntDef(PCFGMNODE pNode, const char *pszName, unsigned int *pu, unsigned int uDef)
-{
-    AssertCompileSize(unsigned int, 4);
-    return CFGMR3QueryU32Def(pNode, pszName, (uint32_t *)pu, uDef);
-}
-
-
-/**
- * Query signed int address value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi              Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QuerySInt(PCFGMNODE pNode, const char *pszName, signed int *pi)
-{
-    AssertCompileSize(signed int, 4);
-    return CFGMR3QueryS32(pNode, pszName, (int32_t *)pi);
-}
-
-
-/**
- * Query unsigned int address value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pi              Where to store the value. Set to default on failure.
- * @param   iDef            The default value.
- */
-VMMR3DECL(int) CFGMR3QuerySIntDef(PCFGMNODE pNode, const char *pszName, signed int *pi, signed int iDef)
-{
-    AssertCompileSize(signed int, 4);
-    return CFGMR3QueryS32Def(pNode, pszName, (int32_t *)pi, iDef);
-}
-
-
-/**
- * Query pointer integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   ppv             Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryPtr(PCFGMNODE pNode, const char *pszName, void **ppv)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        uintptr_t u = (uintptr_t)u64;
-        if (u64 == u)
-            *ppv = (void *)u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query pointer integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   ppv             Where to store the value. Set to default on failure.
- * @param   pvDef           The default value.
- */
-VMMR3DECL(int) CFGMR3QueryPtrDef(PCFGMNODE pNode, const char *pszName, void **ppv, void *pvDef)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, (uintptr_t)pvDef);
-    if (RT_SUCCESS(rc))
-    {
-        uintptr_t u = (uintptr_t)u64;
-        if (u64 == u)
-            *ppv = (void *)u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *ppv = pvDef;
-    return rc;
-}
-
-
-/**
- * Query Guest Context pointer integer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pGCPtr          Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryGCPtr(PCFGMNODE pNode, const char *pszName, PRTGCPTR pGCPtr)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        RTGCPTR u = (RTGCPTR)u64;
-        if (u64 == u)
-            *pGCPtr = u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query Guest Context pointer integer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pGCPtr          Where to store the value. Set to default on failure.
- * @param   GCPtrDef        The default value.
- */
-VMMR3DECL(int) CFGMR3QueryGCPtrDef(PCFGMNODE pNode, const char *pszName, PRTGCPTR pGCPtr, RTGCPTR GCPtrDef)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
-    if (RT_SUCCESS(rc))
-    {
-        RTGCPTR u = (RTGCPTR)u64;
-        if (u64 == u)
-            *pGCPtr = u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pGCPtr = GCPtrDef;
-    return rc;
-}
-
-
-/**
- * Query Guest Context unsigned pointer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pGCPtr          Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryGCPtrU(PCFGMNODE pNode, const char *pszName, PRTGCUINTPTR pGCPtr)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        RTGCUINTPTR u = (RTGCUINTPTR)u64;
-        if (u64 == u)
-            *pGCPtr = u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query Guest Context unsigned pointer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pGCPtr          Where to store the value. Set to default on failure.
- * @param   GCPtrDef        The default value.
- */
-VMMR3DECL(int) CFGMR3QueryGCPtrUDef(PCFGMNODE pNode, const char *pszName, PRTGCUINTPTR pGCPtr, RTGCUINTPTR GCPtrDef)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
-    if (RT_SUCCESS(rc))
-    {
-        RTGCUINTPTR u = (RTGCUINTPTR)u64;
-        if (u64 == u)
-            *pGCPtr = u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pGCPtr = GCPtrDef;
-    return rc;
-}
-
-
-/**
- * Query Guest Context signed pointer value.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pGCPtr          Where to store the value.
- */
-VMMR3DECL(int) CFGMR3QueryGCPtrS(PCFGMNODE pNode, const char *pszName, PRTGCINTPTR pGCPtr)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
-    if (RT_SUCCESS(rc))
-    {
-        RTGCINTPTR u = (RTGCINTPTR)u64;
-        if (u64 == (uint64_t)u)
-            *pGCPtr = u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    return rc;
-}
-
-
-/**
- * Query Guest Context signed pointer value with default.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Name of an integer value.
- * @param   pGCPtr          Where to store the value. Set to default on failure.
- * @param   GCPtrDef        The default value.
- */
-VMMR3DECL(int) CFGMR3QueryGCPtrSDef(PCFGMNODE pNode, const char *pszName, PRTGCINTPTR pGCPtr, RTGCINTPTR GCPtrDef)
-{
-    uint64_t u64;
-    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
-    if (RT_SUCCESS(rc))
-    {
-        RTGCINTPTR u = (RTGCINTPTR)u64;
-        if (u64 == (uint64_t)u)
-            *pGCPtr = u;
-        else
-            rc = VERR_CFGM_INTEGER_TOO_BIG;
-    }
-    if (RT_FAILURE(rc))
-        *pGCPtr = GCPtrDef;
-    return rc;
-}
-
-
-/**
- * Query zero terminated character value storing it in a
- * buffer allocated from the MM heap.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Value name. This value must be of zero terminated character string type.
- * @param   ppszString      Where to store the string pointer.
- *                          Free this using MMR3HeapFree().
- */
-VMMR3DECL(int) CFGMR3QueryStringAlloc(PCFGMNODE pNode, const char *pszName, char **ppszString)
-{
-    size_t cbString;
-    int rc = CFGMR3QuerySize(pNode, pszName, &cbString);
-    if (RT_SUCCESS(rc))
-    {
-        char *pszString = (char *)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_USER, cbString);
-        if (pszString)
-        {
-            rc = CFGMR3QueryString(pNode, pszName, pszString, cbString);
-            if (RT_SUCCESS(rc))
-                *ppszString = pszString;
-            else
-                MMR3HeapFree(pszString);
-        }
-        else
-            rc = VERR_NO_MEMORY;
-    }
-    return rc;
-}
-
-
-/**
- * Query zero terminated character value storing it in a
- * buffer allocated from the MM heap.
- *
- * @returns VBox status code.
- * @param   pNode           Which node to search for pszName in.
- * @param   pszName         Value name. This value must be of zero terminated character string type.
- * @param   ppszString      Where to store the string pointer. Not set on failure.
- *                          Free this using MMR3HeapFree().
- * @param   pszDef          The default return value.  This can be NULL.
- */
-VMMR3DECL(int) CFGMR3QueryStringAllocDef(PCFGMNODE pNode, const char *pszName, char **ppszString, const char *pszDef)
-{
-    /*
-     * (Don't call CFGMR3QuerySize and CFGMR3QueryStringDef here as the latter
-     * cannot handle pszDef being NULL.)
-     */
-    PCFGMLEAF pLeaf;
-    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
-        {
-            size_t const cbSrc = pLeaf->Value.String.cb;
-            char *pszString = (char *)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_USER, cbSrc);
-            if (pszString)
-            {
-                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
-                *ppszString = pszString;
-            }
-            else
-                rc = VERR_NO_MEMORY;
-        }
-        else
-            rc = VERR_CFGM_NOT_STRING;
-    }
-    if (RT_FAILURE(rc))
-    {
-        if (!pszDef)
-            *ppszString = NULL;
-        else
-            *ppszString = MMR3HeapStrDup(pNode->pVM, MM_TAG_CFGM_USER, pszDef);
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-            rc = VINF_SUCCESS;
-    }
-
-    return rc;
-}
-
-
-/**
- * Dumps the configuration (sub)tree to the release log.
- *
- * @param   pRoot   The root node of the dump.
- */
-VMMR3DECL(void) CFGMR3Dump(PCFGMNODE pRoot)
-{
-    LogRel(("************************* CFGM dump *************************\n"));
-    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
-    cfgmR3Dump(pRoot, 0, DBGFR3InfoLogRelHlp());
-    RTLogRelSetBuffering(fOldBuffered);
-    LogRel(("********************* End of CFGM dump **********************\n"));
-}
-
-
-/**
- * Info handler, internal version.
- *
- * @param   pVM         The VM handle.
- * @param   pHlp        Callback functions for doing output.
- * @param   pszArgs     Argument string. Optional and specific to the handler.
- */
-static DECLCALLBACK(void) cfgmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    /*
-     * Figure where to start.
-     */
-    PCFGMNODE pRoot = pVM->cfgm.s.pRoot;
-    if (pszArgs && *pszArgs)
-    {
-        int rc = cfgmR3ResolveNode(pRoot, pszArgs, &pRoot);
-        if (RT_FAILURE(rc))
-        {
-            pHlp->pfnPrintf(pHlp, "Failed to resolve CFGM path '%s', %Rrc", pszArgs, rc);
-            return;
-        }
-    }
-
-    /*
-     * Dump the specified tree.
-     */
-    pHlp->pfnPrintf(pHlp, "pRoot=%p:{", pRoot);
-    cfgmR3DumpPath(pRoot, pHlp);
-    pHlp->pfnPrintf(pHlp, "}\n");
-    cfgmR3Dump(pRoot, 0, pHlp);
-}
-
-
-/**
- * Recursively prints a path name.
- */
-static void cfgmR3DumpPath(PCFGMNODE pNode, PCDBGFINFOHLP pHlp)
-{
-    if (pNode->pParent)
-        cfgmR3DumpPath(pNode->pParent, pHlp);
-    pHlp->pfnPrintf(pHlp, "%s/", pNode->szName);
-}
-
-
-/**
- * Dumps a branch of a tree.
- */
-static void cfgmR3Dump(PCFGMNODE pRoot, unsigned iLevel, PCDBGFINFOHLP pHlp)
-{
-    /*
-     * Path.
-     */
-    pHlp->pfnPrintf(pHlp, "[");
-    cfgmR3DumpPath(pRoot, pHlp);
-    pHlp->pfnPrintf(pHlp, "] (level %d)%s\n", iLevel, pRoot->fRestrictedRoot ? " (restricted root)" : "");
-
-    /*
-     * Values.
-     */
-    PCFGMLEAF pLeaf;
-    size_t cchMax = 0;
-    for (pLeaf = CFGMR3GetFirstValue(pRoot); pLeaf; pLeaf = CFGMR3GetNextValue(pLeaf))
-        cchMax = RT_MAX(cchMax, pLeaf->cchName);
-    for (pLeaf = CFGMR3GetFirstValue(pRoot); pLeaf; pLeaf = CFGMR3GetNextValue(pLeaf))
-    {
-        switch (CFGMR3GetValueType(pLeaf))
-        {
-            case CFGMVALUETYPE_INTEGER:
-                pHlp->pfnPrintf(pHlp, "  %-*s <integer> = %#018llx (%lld)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.Integer.u64, pLeaf->Value.Integer.u64);
-                break;
-
-            case CFGMVALUETYPE_STRING:
-                pHlp->pfnPrintf(pHlp, "  %-*s <string>  = \"%s\" (cb=%zu)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.String.psz, pLeaf->Value.String.cb);
-                break;
-
-            case CFGMVALUETYPE_BYTES:
-                pHlp->pfnPrintf(pHlp, "  %-*s <bytes>   = \"%.*Rhxs\" (cb=%zu)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.Bytes.cb, pLeaf->Value.Bytes.pau8, pLeaf->Value.Bytes.cb);
-                break;
-
-            default:
-                AssertMsgFailed(("bad leaf!\n"));
-                break;
-        }
-    }
-    pHlp->pfnPrintf(pHlp, "\n");
-
-    /*
-     * Children.
-     */
-    for (PCFGMNODE pChild = CFGMR3GetFirstChild(pRoot); pChild; pChild = CFGMR3GetNextChild(pChild))
-    {
-        Assert(pChild->pNext != pChild);
-        Assert(pChild->pPrev != pChild);
-        Assert(pChild->pPrev != pChild->pNext || !pChild->pPrev);
-        Assert(pChild->pFirstChild != pChild);
-        Assert(pChild->pParent != pChild);
-        cfgmR3Dump(pChild, iLevel + 1, pHlp);
-    }
-}
-
Index: unk/src/VBox/VMM/CFGMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/CFGMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,131 +1,0 @@
-/* $Id$ */
-/** @file
- * CFGM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___CFGMInternal_h
-#define ___CFGMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-
-
-/** @defgroup grp_cfgm_int Internals.
- * @ingroup grp_cfgm
- * @{
- */
-
-
-/**
- * Configuration manager propertype value.
- */
-typedef union CFGMVALUE
-{
-    /** Integer value. */
-    struct CFGMVALUE_INTEGER
-    {
-        /** The integer represented as 64-bit unsigned. */
-        uint64_t        u64;
-    } Integer;
-
-    /** String value. (UTF-8 of course) */
-    struct CFGMVALUE_STRING
-    {
-        /** Length of string. (In bytes, including the terminator.) */
-        size_t          cb;
-        /** Pointer to the string. */
-        char           *psz;
-    } String;
-
-    /** Byte string value. */
-    struct CFGMVALUE_BYTES
-    {
-        /** Length of byte string. (in bytes) */
-        size_t          cb;
-        /** Pointer to the byte string. */
-        uint8_t        *pau8;
-    } Bytes;
-} CFGMVALUE;
-/** Pointer to configuration manager property value. */
-typedef CFGMVALUE *PCFGMVALUE;
-
-
-/**
- * Configuration manager tree node.
- */
-typedef struct CFGMLEAF
-{
-    /** Pointer to the next leaf. */
-    PCFGMLEAF       pNext;
-    /** Pointer to the previous leaf. */
-    PCFGMLEAF       pPrev;
-
-    /** Property type. */
-    CFGMVALUETYPE   enmType;
-    /** Property value. */
-    CFGMVALUE       Value;
-
-    /** Name length. (exclusive) */
-    size_t          cchName;
-    /** Name. */
-    char            szName[1];
-} CFGMLEAF;
-
-
-/**
- * Configuration manager tree node.
- */
-typedef struct CFGMNODE
-{
-    /** Pointer to the next node (on this level). */
-    PCFGMNODE       pNext;
-    /** Pointer to the previous node (on this level). */
-    PCFGMNODE       pPrev;
-    /** Pointer Parent node. */
-    PCFGMNODE       pParent;
-    /** Pointer to first child node. */
-    PCFGMNODE       pFirstChild;
-    /** Pointer to first property leaf. */
-    PCFGMLEAF       pFirstLeaf;
-
-    /** Pointer to the VM owning this node. */
-    PVM             pVM;
-
-    /** The root of a 'restricted' subtree, i.e. the parent is
-     * invisible to non-trusted users.
-     */
-    bool            fRestrictedRoot;
-
-    /** Name length. (exclusive) */
-    size_t          cchName;
-    /** Name. */
-    char            szName[1];
-} CFGMNODE;
-
-
-
-/**
- * CFGM VM Instance data.
- * Changes to this must checked against the padding of the cfgm union in VM!
- */
-typedef struct CFGM
-{
-    /** Pointer to root node. */
-    R3PTRTYPE(PCFGMNODE)    pRoot;
-} CFGM;
-
-/** @} */
-
-#endif
Index: unk/src/VBox/VMM/CPUM.cpp
===================================================================
--- /trunk/src/VBox/VMM/CPUM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,3980 +1,0 @@
-/* $Id$ */
-/** @file
- * CPUM - CPU Monitor / Manager.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_cpum CPUM - CPU Monitor / Manager
- *
- * The CPU Monitor / Manager keeps track of all the CPU registers. It is
- * also responsible for lazy FPU handling and some of the context loading
- * in raw mode.
- *
- * There are three CPU contexts, the most important one is the guest one (GC).
- * When running in raw-mode (RC) there is a special hyper context for the VMM
- * part that floats around inside the guest address space. When running in
- * raw-mode, CPUM also maintains a host context for saving and restoring
- * registers across world switches. This latter is done in cooperation with the
- * world switcher (@see pg_vmm).
- *
- * @see grp_cpum
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_CPUM
-#include <VBox/cpum.h>
-#include <VBox/cpumdis.h>
-#include <VBox/pgm.h>
-#include <VBox/mm.h>
-#include <VBox/selm.h>
-#include <VBox/dbgf.h>
-#include <VBox/patm.h>
-#include <VBox/hwaccm.h>
-#include <VBox/ssm.h>
-#include "CPUMInternal.h"
-#include <VBox/vm.h>
-
-#include <VBox/param.h>
-#include <VBox/dis.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/asm-amd64-x86.h>
-#include <iprt/string.h>
-#include <iprt/mp.h>
-#include <iprt/cpuset.h>
-#include <include/internal/pgm.h>
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The current saved state version. */
-#define CPUM_SAVED_STATE_VERSION                12
-/** The saved state version of 3.2, 3.1 and 3.3 trunk before the hidden
- * selector register change (CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID). */
-#define CPUM_SAVED_STATE_VERSION_VER3_2         11
-/** The saved state version of 3.0 and 3.1 trunk before the teleportation
- * changes. */
-#define CPUM_SAVED_STATE_VERSION_VER3_0         10
-/** The saved state version for the 2.1 trunk before the MSR changes. */
-#define CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR   9
-/** The saved state version of 2.0, used for backwards compatibility. */
-#define CPUM_SAVED_STATE_VERSION_VER2_0         8
-/** The saved state version of 1.6, used for backwards compatibility. */
-#define CPUM_SAVED_STATE_VERSION_VER1_6         6
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-
-/**
- * What kind of cpu info dump to perform.
- */
-typedef enum CPUMDUMPTYPE
-{
-    CPUMDUMPTYPE_TERSE,
-    CPUMDUMPTYPE_DEFAULT,
-    CPUMDUMPTYPE_VERBOSE
-} CPUMDUMPTYPE;
-/** Pointer to a cpu info dump type. */
-typedef CPUMDUMPTYPE *PCPUMDUMPTYPE;
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static CPUMCPUVENDOR cpumR3DetectVendor(uint32_t uEAX, uint32_t uEBX, uint32_t uECX, uint32_t uEDX);
-static int cpumR3CpuIdInit(PVM pVM);
-static DECLCALLBACK(int)  cpumR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
-static DECLCALLBACK(int)  cpumR3SaveExec(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int)  cpumR3LoadPrep(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int)  cpumR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static DECLCALLBACK(int)  cpumR3LoadDone(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(void) cpumR3InfoAll(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) cpumR3InfoGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) cpumR3InfoGuestInstr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) cpumR3InfoHyper(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) cpumR3InfoHost(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) cpumR3CpuIdInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-
-
-/**
- * Initializes the CPUM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) CPUMR3Init(PVM pVM)
-{
-    LogFlow(("CPUMR3Init\n"));
-
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompileMemberAlignment(VM, cpum.s, 32);
-    AssertCompile(sizeof(pVM->cpum.s) <= sizeof(pVM->cpum.padding));
-    AssertCompileSizeAlignment(CPUMCTX, 64);
-    AssertCompileSizeAlignment(CPUMCTXMSR, 64);
-    AssertCompileSizeAlignment(CPUMHOSTCTX, 64);
-    AssertCompileMemberAlignment(VM, cpum, 64);
-    AssertCompileMemberAlignment(VM, aCpus, 64);
-    AssertCompileMemberAlignment(VMCPU, cpum.s, 64);
-    AssertCompileMemberSizeAlignment(VM, aCpus[0].cpum.s, 64);
-
-    /* Calculate the offset from CPUM to CPUMCPU for the first CPU. */
-    pVM->cpum.s.offCPUMCPU0 = RT_OFFSETOF(VM, aCpus[0].cpum) - RT_OFFSETOF(VM, cpum);
-    Assert((uintptr_t)&pVM->cpum + pVM->cpum.s.offCPUMCPU0 == (uintptr_t)&pVM->aCpus[0].cpum);
-
-    /* Calculate the offset from CPUMCPU to CPUM. */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        /*
-         * Setup any fixed pointers and offsets.
-         */
-        pVCpu->cpum.s.pHyperCoreR3 = CPUMCTX2CORE(&pVCpu->cpum.s.Hyper);
-        pVCpu->cpum.s.pHyperCoreR0 = VM_R0_ADDR(pVM, CPUMCTX2CORE(&pVCpu->cpum.s.Hyper));
-
-        pVCpu->cpum.s.offCPUM      = RT_OFFSETOF(VM, aCpus[i].cpum) - RT_OFFSETOF(VM, cpum);
-        Assert((uintptr_t)&pVCpu->cpum - pVCpu->cpum.s.offCPUM == (uintptr_t)&pVM->cpum);
-    }
-
-    /*
-     * Check that the CPU supports the minimum features we require.
-     */
-    if (!ASMHasCpuId())
-    {
-        Log(("The CPU doesn't support CPUID!\n"));
-        return VERR_UNSUPPORTED_CPU;
-    }
-    ASMCpuId_ECX_EDX(1, &pVM->cpum.s.CPUFeatures.ecx, &pVM->cpum.s.CPUFeatures.edx);
-    ASMCpuId_ECX_EDX(0x80000001, &pVM->cpum.s.CPUFeaturesExt.ecx, &pVM->cpum.s.CPUFeaturesExt.edx);
-
-    /* Setup the CR4 AND and OR masks used in the switcher */
-    /* Depends on the presence of FXSAVE(SSE) support on the host CPU */
-    if (!pVM->cpum.s.CPUFeatures.edx.u1FXSR)
-    {
-        Log(("The CPU doesn't support FXSAVE/FXRSTOR!\n"));
-        /* No FXSAVE implies no SSE */
-        pVM->cpum.s.CR4.AndMask = X86_CR4_PVI | X86_CR4_VME;
-        pVM->cpum.s.CR4.OrMask  = 0;
-    }
-    else
-    {
-        pVM->cpum.s.CR4.AndMask = X86_CR4_OSXMMEEXCPT | X86_CR4_PVI | X86_CR4_VME;
-        pVM->cpum.s.CR4.OrMask  = X86_CR4_OSFSXR;
-    }
-
-    if (!pVM->cpum.s.CPUFeatures.edx.u1MMX)
-    {
-        Log(("The CPU doesn't support MMX!\n"));
-        return VERR_UNSUPPORTED_CPU;
-    }
-    if (!pVM->cpum.s.CPUFeatures.edx.u1TSC)
-    {
-        Log(("The CPU doesn't support TSC!\n"));
-        return VERR_UNSUPPORTED_CPU;
-    }
-    /* Bogus on AMD? */
-    if (!pVM->cpum.s.CPUFeatures.edx.u1SEP)
-        Log(("The CPU doesn't support SYSENTER/SYSEXIT!\n"));
-
-    /*
-     * Detect the host CPU vendor.
-     * (The guest CPU vendor is re-detected later on.)
-     */
-    uint32_t uEAX, uEBX, uECX, uEDX;
-    ASMCpuId(0, &uEAX, &uEBX, &uECX, &uEDX);
-    pVM->cpum.s.enmHostCpuVendor = cpumR3DetectVendor(uEAX, uEBX, uECX, uEDX);
-    pVM->cpum.s.enmGuestCpuVendor = pVM->cpum.s.enmHostCpuVendor;
-
-    /*
-     * Setup hypervisor startup values.
-     */
-
-    /*
-     * Register saved state data item.
-     */
-    int rc = SSMR3RegisterInternal(pVM, "cpum", 1, CPUM_SAVED_STATE_VERSION, sizeof(CPUM),
-                                   NULL, cpumR3LiveExec, NULL,
-                                   NULL, cpumR3SaveExec, NULL,
-                                   cpumR3LoadPrep, cpumR3LoadExec, cpumR3LoadDone);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Register info handlers.
-     */
-    DBGFR3InfoRegisterInternal(pVM, "cpum",             "Displays the all the cpu states.",         &cpumR3InfoAll);
-    DBGFR3InfoRegisterInternal(pVM, "cpumguest",        "Displays the guest cpu state.",            &cpumR3InfoGuest);
-    DBGFR3InfoRegisterInternal(pVM, "cpumhyper",        "Displays the hypervisor cpu state.",       &cpumR3InfoHyper);
-    DBGFR3InfoRegisterInternal(pVM, "cpumhost",         "Displays the host cpu state.",             &cpumR3InfoHost);
-    DBGFR3InfoRegisterInternal(pVM, "cpuid",            "Displays the guest cpuid leaves.",         &cpumR3CpuIdInfo);
-    DBGFR3InfoRegisterInternal(pVM, "cpumguestinstr",   "Displays the current guest instruction.",  &cpumR3InfoGuestInstr);
-
-    /*
-     * Initialize the Guest CPUID state.
-     */
-    rc = cpumR3CpuIdInit(pVM);
-    if (RT_FAILURE(rc))
-        return rc;
-    CPUMR3Reset(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Detect the CPU vendor give n the
- *
- * @returns The vendor.
- * @param   uEAX                EAX from CPUID(0).
- * @param   uEBX                EBX from CPUID(0).
- * @param   uECX                ECX from CPUID(0).
- * @param   uEDX                EDX from CPUID(0).
- */
-static CPUMCPUVENDOR cpumR3DetectVendor(uint32_t uEAX, uint32_t uEBX, uint32_t uECX, uint32_t uEDX)
-{
-    if (    uEAX >= 1
-        &&  uEBX == X86_CPUID_VENDOR_AMD_EBX
-        &&  uECX == X86_CPUID_VENDOR_AMD_ECX
-        &&  uEDX == X86_CPUID_VENDOR_AMD_EDX)
-        return CPUMCPUVENDOR_AMD;
-
-    if (    uEAX >= 1
-        &&  uEBX == X86_CPUID_VENDOR_INTEL_EBX
-        &&  uECX == X86_CPUID_VENDOR_INTEL_ECX
-        &&  uEDX == X86_CPUID_VENDOR_INTEL_EDX)
-        return CPUMCPUVENDOR_INTEL;
-
-    /** @todo detect the other buggers... */
-    return CPUMCPUVENDOR_UNKNOWN;
-}
-
-
-/**
- * Fetches overrides for a CPUID leaf.
- *
- * @returns VBox status code.
- * @param   pLeaf               The leaf to load the overrides into.
- * @param   pCfgNode            The CFGM node containing the overrides
- *                              (/CPUM/HostCPUID/ or /CPUM/CPUID/).
- * @param   iLeaf               The CPUID leaf number.
- */
-static int cpumR3CpuIdFetchLeafOverride(PCPUMCPUID pLeaf, PCFGMNODE pCfgNode, uint32_t iLeaf)
-{
-    PCFGMNODE pLeafNode = CFGMR3GetChildF(pCfgNode, "%RX32", iLeaf);
-    if (pLeafNode)
-    {
-        uint32_t u32;
-        int rc = CFGMR3QueryU32(pLeafNode, "eax", &u32);
-        if (RT_SUCCESS(rc))
-            pLeaf->eax = u32;
-        else
-            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
-
-        rc = CFGMR3QueryU32(pLeafNode, "ebx", &u32);
-        if (RT_SUCCESS(rc))
-            pLeaf->ebx = u32;
-        else
-            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
-
-        rc = CFGMR3QueryU32(pLeafNode, "ecx", &u32);
-        if (RT_SUCCESS(rc))
-            pLeaf->ecx = u32;
-        else
-            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
-
-        rc = CFGMR3QueryU32(pLeafNode, "edx", &u32);
-        if (RT_SUCCESS(rc))
-            pLeaf->edx = u32;
-        else
-            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
-
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Load the overrides for a set of CPUID leaves.
- *
- * @returns VBox status code.
- * @param   paLeaves            The leaf array.
- * @param   cLeaves             The number of leaves.
- * @param   uStart              The start leaf number.
- * @param   pCfgNode            The CFGM node containing the overrides
- *                              (/CPUM/HostCPUID/ or /CPUM/CPUID/).
- */
-static int cpumR3CpuIdInitLoadOverrideSet(uint32_t uStart, PCPUMCPUID paLeaves, uint32_t cLeaves, PCFGMNODE pCfgNode)
-{
-    for (uint32_t i = 0; i < cLeaves; i++)
-    {
-        int rc = cpumR3CpuIdFetchLeafOverride(&paLeaves[i], pCfgNode, uStart + i);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    return VINF_SUCCESS;
-}
-
-/**
- * Init a set of host CPUID leaves.
- *
- * @returns VBox status code.
- * @param   paLeaves            The leaf array.
- * @param   cLeaves             The number of leaves.
- * @param   uStart              The start leaf number.
- * @param   pCfgNode            The /CPUM/HostCPUID/ node.
- */
-static int cpumR3CpuIdInitHostSet(uint32_t uStart, PCPUMCPUID paLeaves, uint32_t cLeaves, PCFGMNODE pCfgNode)
-{
-    /* Using the ECX variant for all of them can't hurt... */
-    for (uint32_t i = 0; i < cLeaves; i++)
-        ASMCpuId_Idx_ECX(uStart + i, 0, &paLeaves[i].eax, &paLeaves[i].ebx, &paLeaves[i].ecx, &paLeaves[i].edx);
-
-    /* Load CPUID leaf override; we currently don't care if the user
-       specifies features the host CPU doesn't support. */
-    return cpumR3CpuIdInitLoadOverrideSet(uStart, paLeaves, cLeaves, pCfgNode);
-}
-
-
-/**
- * Initializes the emulated CPU's cpuid information.
- *
- * @returns VBox status code.
- * @param   pVM          The VM to operate on.
- */
-static int cpumR3CpuIdInit(PVM pVM)
-{
-    PCPUM       pCPUM    = &pVM->cpum.s;
-    PCFGMNODE   pCpumCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM");
-    uint32_t    i;
-    int         rc;
-
-#define PORTABLE_CLEAR_BITS_WHEN(Lvl, LeafSuffReg, FeatNm, fMask, uValue) \
-    if (pCPUM->u8PortableCpuIdLevel >= (Lvl) && (pCPUM->aGuestCpuId##LeafSuffReg & (fMask)) == (uValue) ) \
-    { \
-        LogRel(("PortableCpuId: " #LeafSuffReg "[" #FeatNm "]: %#x -> 0\n", pCPUM->aGuestCpuId##LeafSuffReg & (fMask))); \
-        pCPUM->aGuestCpuId##LeafSuffReg &= ~(uint32_t)(fMask); \
-    }
-#define PORTABLE_DISABLE_FEATURE_BIT(Lvl, LeafSuffReg, FeatNm, fBitMask) \
-    if (pCPUM->u8PortableCpuIdLevel >= (Lvl) && (pCPUM->aGuestCpuId##LeafSuffReg & (fBitMask)) ) \
-    { \
-        LogRel(("PortableCpuId: " #LeafSuffReg "[" #FeatNm "]: 1 -> 0\n")); \
-        pCPUM->aGuestCpuId##LeafSuffReg &= ~(uint32_t)(fBitMask); \
-    }
-
-    /*
-     * Read the configuration.
-     */
-    /** @cfgm{CPUM/SyntheticCpu, boolean, false}
-     * Enables the Synthetic CPU.  The Vendor ID and Processor Name are
-     * completely overridden by VirtualBox custom strings.  Some
-     * CPUID information is withheld, like the cache info. */
-    rc = CFGMR3QueryBoolDef(pCpumCfg, "SyntheticCpu",  &pCPUM->fSyntheticCpu,  false);
-    AssertRCReturn(rc, rc);
-
-    /** @cfgm{CPUM/PortableCpuIdLevel, 8-bit, 0, 3, 0}
-     * When non-zero CPUID features that could cause portability issues will be
-     * stripped.  The higher the value the more features gets stripped.  Higher
-     * values should only be used when older CPUs are involved since it may
-     * harm performance and maybe also cause problems with specific guests. */
-    rc = CFGMR3QueryU8Def(pCpumCfg, "PortableCpuIdLevel", &pCPUM->u8PortableCpuIdLevel, 0);
-    AssertRCReturn(rc, rc);
-
-    AssertLogRelReturn(!pCPUM->fSyntheticCpu || !pCPUM->u8PortableCpuIdLevel, VERR_INTERNAL_ERROR_2);
-
-    /*
-     * Get the host CPUID leaves and redetect the guest CPU vendor (could've
-     * been overridden).
-     */
-    /** @cfgm{CPUM/HostCPUID/[000000xx|800000xx|c000000x]/[eax|ebx|ecx|edx],32-bit}
-     * Overrides the host CPUID leaf values used for calculating the guest CPUID
-     * leaves.  This can be used to preserve the CPUID values when moving a VM
-     * to a different machine.  Another use is restricting (or extending) the
-     * feature set exposed to the guest. */
-    PCFGMNODE pHostOverrideCfg = CFGMR3GetChild(pCpumCfg, "HostCPUID");
-    rc = cpumR3CpuIdInitHostSet(UINT32_C(0x00000000), &pCPUM->aGuestCpuIdStd[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdStd),     pHostOverrideCfg);
-    AssertRCReturn(rc, rc);
-    rc = cpumR3CpuIdInitHostSet(UINT32_C(0x80000000), &pCPUM->aGuestCpuIdExt[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdExt),     pHostOverrideCfg);
-    AssertRCReturn(rc, rc);
-    rc = cpumR3CpuIdInitHostSet(UINT32_C(0xc0000000), &pCPUM->aGuestCpuIdCentaur[0], RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur), pHostOverrideCfg);
-    AssertRCReturn(rc, rc);
-
-    pCPUM->enmGuestCpuVendor = cpumR3DetectVendor(pCPUM->aGuestCpuIdStd[0].eax, pCPUM->aGuestCpuIdStd[0].ebx,
-                                                  pCPUM->aGuestCpuIdStd[0].ecx, pCPUM->aGuestCpuIdStd[0].edx);
-
-    /*
-     * Determine the default leaf.
-     *
-     * Intel returns values of the highest standard function, while AMD
-     * returns zeros. VIA on the other hand seems to returning nothing or
-     * perhaps some random garbage, we don't try to duplicate this behavior.
-     */
-    ASMCpuId(pCPUM->aGuestCpuIdStd[0].eax + 10, /** @todo r=bird: Use the host value here in case of overrides and more than 10 leaves being stripped already. */
-             &pCPUM->GuestCpuIdDef.eax, &pCPUM->GuestCpuIdDef.ebx,
-             &pCPUM->GuestCpuIdDef.ecx, &pCPUM->GuestCpuIdDef.edx);
-
-
-    /* Cpuid 1 & 0x80000001:
-     * Only report features we can support.
-     *
-     * Note! When enabling new features the Synthetic CPU and Portable CPUID
-     *       options may require adjusting (i.e. stripping what was enabled).
-     */
-    pCPUM->aGuestCpuIdStd[1].edx &= X86_CPUID_FEATURE_EDX_FPU
-                                  | X86_CPUID_FEATURE_EDX_VME
-                                  | X86_CPUID_FEATURE_EDX_DE
-                                  | X86_CPUID_FEATURE_EDX_PSE
-                                  | X86_CPUID_FEATURE_EDX_TSC
-                                  | X86_CPUID_FEATURE_EDX_MSR
-                                  //| X86_CPUID_FEATURE_EDX_PAE   - set later if configured.
-                                  | X86_CPUID_FEATURE_EDX_MCE
-                                  | X86_CPUID_FEATURE_EDX_CX8
-                                  //| X86_CPUID_FEATURE_EDX_APIC  - set by the APIC device if present.
-                                  /* Note! we don't report sysenter/sysexit support due to our inability to keep the IOPL part of eflags in sync while in ring 1 (see #1757) */
-                                  //| X86_CPUID_FEATURE_EDX_SEP
-                                  | X86_CPUID_FEATURE_EDX_MTRR
-                                  | X86_CPUID_FEATURE_EDX_PGE
-                                  | X86_CPUID_FEATURE_EDX_MCA
-                                  | X86_CPUID_FEATURE_EDX_CMOV
-                                  | X86_CPUID_FEATURE_EDX_PAT
-                                  | X86_CPUID_FEATURE_EDX_PSE36
-                                  //| X86_CPUID_FEATURE_EDX_PSN   - no serial number.
-                                  | X86_CPUID_FEATURE_EDX_CLFSH
-                                  //| X86_CPUID_FEATURE_EDX_DS    - no debug store.
-                                  //| X86_CPUID_FEATURE_EDX_ACPI  - not virtualized yet.
-                                  | X86_CPUID_FEATURE_EDX_MMX
-                                  | X86_CPUID_FEATURE_EDX_FXSR
-                                  | X86_CPUID_FEATURE_EDX_SSE
-                                  | X86_CPUID_FEATURE_EDX_SSE2
-                                  //| X86_CPUID_FEATURE_EDX_SS    - no self snoop.
-                                  //| X86_CPUID_FEATURE_EDX_HTT   - no hyperthreading.
-                                  //| X86_CPUID_FEATURE_EDX_TM    - no thermal monitor.
-                                  //| X86_CPUID_FEATURE_EDX_PBE   - no pending break enabled.
-                                  | 0;
-    pCPUM->aGuestCpuIdStd[1].ecx &= 0
-                                  | X86_CPUID_FEATURE_ECX_SSE3
-                                  /* Can't properly emulate monitor & mwait with guest SMP; force the guest to use hlt for idling VCPUs. */
-                                  | ((pVM->cCpus == 1) ? X86_CPUID_FEATURE_ECX_MONITOR : 0)
-                                  //| X86_CPUID_FEATURE_ECX_CPLDS - no CPL qualified debug store.
-                                  //| X86_CPUID_FEATURE_ECX_VMX   - not virtualized.
-                                  //| X86_CPUID_FEATURE_ECX_EST   - no extended speed step.
-                                  //| X86_CPUID_FEATURE_ECX_TM2   - no thermal monitor 2.
-                                    | X86_CPUID_FEATURE_ECX_SSSE3
-                                  //| X86_CPUID_FEATURE_ECX_CNTXID - no L1 context id (MSR++).
-                                  //| X86_CPUID_FEATURE_ECX_CX16  - no cmpxchg16b
-                                  /* ECX Bit 14 - xTPR Update Control. Processor supports changing IA32_MISC_ENABLES[bit 23]. */
-                                  //| X86_CPUID_FEATURE_ECX_TPRUPDATE
-                                  /* ECX Bit 21 - x2APIC support - not yet. */
-                                  // | X86_CPUID_FEATURE_ECX_X2APIC
-                                  /* ECX Bit 23 - POPCNT instruction. */
-                                  //| X86_CPUID_FEATURE_ECX_POPCNT
-                                  | 0;
-    if (pCPUM->u8PortableCpuIdLevel > 0)
-    {
-        PORTABLE_CLEAR_BITS_WHEN(1, Std[1].eax, ProcessorType, (UINT32_C(3) << 12), (UINT32_C(2) << 12));
-        PORTABLE_DISABLE_FEATURE_BIT(1, Std[1].ecx, SSSE3, X86_CPUID_FEATURE_ECX_SSSE3);
-        PORTABLE_DISABLE_FEATURE_BIT(1, Std[1].ecx, SSE3,  X86_CPUID_FEATURE_ECX_SSE3);
-        PORTABLE_DISABLE_FEATURE_BIT(2, Std[1].edx, SSE2,  X86_CPUID_FEATURE_EDX_SSE2);
-        PORTABLE_DISABLE_FEATURE_BIT(3, Std[1].edx, SSE,   X86_CPUID_FEATURE_EDX_SSE);
-        PORTABLE_DISABLE_FEATURE_BIT(3, Std[1].edx, CLFSH, X86_CPUID_FEATURE_EDX_CLFSH);
-        PORTABLE_DISABLE_FEATURE_BIT(3, Std[1].edx, CMOV,  X86_CPUID_FEATURE_EDX_CMOV);
-
-        Assert(!(pCPUM->aGuestCpuIdStd[1].edx & (  X86_CPUID_FEATURE_EDX_SEP
-                                                 | X86_CPUID_FEATURE_EDX_PSN
-                                                 | X86_CPUID_FEATURE_EDX_DS
-                                                 | X86_CPUID_FEATURE_EDX_ACPI
-                                                 | X86_CPUID_FEATURE_EDX_SS
-                                                 | X86_CPUID_FEATURE_EDX_TM
-                                                 | X86_CPUID_FEATURE_EDX_PBE
-                                                 )));
-        Assert(!(pCPUM->aGuestCpuIdStd[1].ecx & (  X86_CPUID_FEATURE_ECX_PCLMUL
-                                                 | X86_CPUID_FEATURE_ECX_DTES64
-                                                 | X86_CPUID_FEATURE_ECX_CPLDS
-                                                 | X86_CPUID_FEATURE_ECX_VMX
-                                                 | X86_CPUID_FEATURE_ECX_SMX
-                                                 | X86_CPUID_FEATURE_ECX_EST
-                                                 | X86_CPUID_FEATURE_ECX_TM2
-                                                 | X86_CPUID_FEATURE_ECX_CNTXID
-                                                 | X86_CPUID_FEATURE_ECX_FMA
-                                                 | X86_CPUID_FEATURE_ECX_CX16
-                                                 | X86_CPUID_FEATURE_ECX_TPRUPDATE
-                                                 | X86_CPUID_FEATURE_ECX_PDCM
-                                                 | X86_CPUID_FEATURE_ECX_DCA
-                                                 | X86_CPUID_FEATURE_ECX_MOVBE
-                                                 | X86_CPUID_FEATURE_ECX_AES
-                                                 | X86_CPUID_FEATURE_ECX_POPCNT
-                                                 | X86_CPUID_FEATURE_ECX_XSAVE
-                                                 | X86_CPUID_FEATURE_ECX_OSXSAVE
-                                                 | X86_CPUID_FEATURE_ECX_AVX
-                                                 )));
-    }
-
-    /* Cpuid 0x80000001:
-     * Only report features we can support.
-     *
-     * Note! When enabling new features the Synthetic CPU and Portable CPUID
-     *       options may require adjusting (i.e. stripping what was enabled).
-     *
-     * ASSUMES that this is ALWAYS the AMD defined feature set if present.
-     */
-    pCPUM->aGuestCpuIdExt[1].edx &= X86_CPUID_AMD_FEATURE_EDX_FPU
-                                  | X86_CPUID_AMD_FEATURE_EDX_VME
-                                  | X86_CPUID_AMD_FEATURE_EDX_DE
-                                  | X86_CPUID_AMD_FEATURE_EDX_PSE
-                                  | X86_CPUID_AMD_FEATURE_EDX_TSC
-                                  | X86_CPUID_AMD_FEATURE_EDX_MSR //?? this means AMD MSRs..
-                                  //| X86_CPUID_AMD_FEATURE_EDX_PAE    - not implemented yet.
-                                  //| X86_CPUID_AMD_FEATURE_EDX_MCE    - not virtualized yet.
-                                  | X86_CPUID_AMD_FEATURE_EDX_CX8
-                                  //| X86_CPUID_AMD_FEATURE_EDX_APIC   - set by the APIC device if present.
-                                  /* Note! we don't report sysenter/sysexit support due to our inability to keep the IOPL part of eflags in sync while in ring 1 (see #1757) */
-                                  //| X86_CPUID_AMD_FEATURE_EDX_SEP
-                                  | X86_CPUID_AMD_FEATURE_EDX_MTRR
-                                  | X86_CPUID_AMD_FEATURE_EDX_PGE
-                                  | X86_CPUID_AMD_FEATURE_EDX_MCA
-                                  | X86_CPUID_AMD_FEATURE_EDX_CMOV
-                                  | X86_CPUID_AMD_FEATURE_EDX_PAT
-                                  | X86_CPUID_AMD_FEATURE_EDX_PSE36
-                                  //| X86_CPUID_AMD_FEATURE_EDX_NX     - not virtualized, requires PAE.
-                                  //| X86_CPUID_AMD_FEATURE_EDX_AXMMX
-                                  | X86_CPUID_AMD_FEATURE_EDX_MMX
-                                  | X86_CPUID_AMD_FEATURE_EDX_FXSR
-                                  | X86_CPUID_AMD_FEATURE_EDX_FFXSR
-                                  //| X86_CPUID_AMD_FEATURE_EDX_PAGE1GB
-                                  //| X86_CPUID_AMD_FEATURE_EDX_RDTSCP - AMD only; turned on when necessary
-                                  //| X86_CPUID_AMD_FEATURE_EDX_LONG_MODE - turned on when necessary
-                                  | X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX
-                                  | X86_CPUID_AMD_FEATURE_EDX_3DNOW
-                                  | 0;
-    pCPUM->aGuestCpuIdExt[1].ecx &= 0
-                                  //| X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF
-                                  //| X86_CPUID_AMD_FEATURE_ECX_CMPL
-                                  //| X86_CPUID_AMD_FEATURE_ECX_SVM    - not virtualized.
-                                  //| X86_CPUID_AMD_FEATURE_ECX_EXT_APIC
-                                  /* Note: This could prevent teleporting from AMD to Intel CPUs! */
-                                  | X86_CPUID_AMD_FEATURE_ECX_CR8L         /* expose lock mov cr0 = mov cr8 hack for guests that can use this feature to access the TPR. */
-                                  //| X86_CPUID_AMD_FEATURE_ECX_ABM
-                                  //| X86_CPUID_AMD_FEATURE_ECX_SSE4A
-                                  //| X86_CPUID_AMD_FEATURE_ECX_MISALNSSE
-                                  //| X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF
-                                  //| X86_CPUID_AMD_FEATURE_ECX_OSVW
-                                  //| X86_CPUID_AMD_FEATURE_ECX_IBS
-                                  //| X86_CPUID_AMD_FEATURE_ECX_SSE5
-                                  //| X86_CPUID_AMD_FEATURE_ECX_SKINIT
-                                  //| X86_CPUID_AMD_FEATURE_ECX_WDT
-                                  | 0;
-    if (pCPUM->u8PortableCpuIdLevel > 0)
-    {
-        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].ecx, CR8L,       X86_CPUID_AMD_FEATURE_ECX_CR8L);
-        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, 3DNOW,      X86_CPUID_AMD_FEATURE_EDX_3DNOW);
-        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, 3DNOW_EX,   X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
-        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, FFXSR,      X86_CPUID_AMD_FEATURE_EDX_FFXSR);
-        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, RDTSCP,     X86_CPUID_AMD_FEATURE_EDX_RDTSCP);
-        PORTABLE_DISABLE_FEATURE_BIT(2, Ext[1].ecx, LAHF_SAHF,  X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF);
-        PORTABLE_DISABLE_FEATURE_BIT(3, Ext[1].ecx, CMOV,       X86_CPUID_AMD_FEATURE_EDX_CMOV);
-
-        Assert(!(pCPUM->aGuestCpuIdExt[1].ecx & (  X86_CPUID_AMD_FEATURE_ECX_CMPL
-                                                 | X86_CPUID_AMD_FEATURE_ECX_SVM
-                                                 | X86_CPUID_AMD_FEATURE_ECX_EXT_APIC
-                                                 | X86_CPUID_AMD_FEATURE_ECX_CR8L
-                                                 | X86_CPUID_AMD_FEATURE_ECX_ABM
-                                                 | X86_CPUID_AMD_FEATURE_ECX_SSE4A
-                                                 | X86_CPUID_AMD_FEATURE_ECX_MISALNSSE
-                                                 | X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF
-                                                 | X86_CPUID_AMD_FEATURE_ECX_OSVW
-                                                 | X86_CPUID_AMD_FEATURE_ECX_IBS
-                                                 | X86_CPUID_AMD_FEATURE_ECX_SSE5
-                                                 | X86_CPUID_AMD_FEATURE_ECX_SKINIT
-                                                 | X86_CPUID_AMD_FEATURE_ECX_WDT
-                                                 | UINT32_C(0xffffc000)
-                                                 )));
-        Assert(!(pCPUM->aGuestCpuIdExt[1].edx & (  RT_BIT(10)
-                                                 | X86_CPUID_AMD_FEATURE_EDX_SEP
-                                                 | RT_BIT(18)
-                                                 | RT_BIT(19)
-                                                 | RT_BIT(21)
-                                                 | X86_CPUID_AMD_FEATURE_EDX_AXMMX
-                                                 | X86_CPUID_AMD_FEATURE_EDX_PAGE1GB
-                                                 | RT_BIT(28)
-                                                 )));
-    }
-
-    /*
-     * Apply the Synthetic CPU modifications. (TODO: move this up)
-     */
-    if (pCPUM->fSyntheticCpu)
-    {
-        static const char s_szVendor[13]    = "VirtualBox  ";
-        static const char s_szProcessor[48] = "VirtualBox SPARCx86 Processor v1000            "; /* includes null terminator */
-
-        pCPUM->enmGuestCpuVendor = CPUMCPUVENDOR_SYNTHETIC;
-
-        /* Limit the nr of standard leaves; 5 for monitor/mwait */
-        pCPUM->aGuestCpuIdStd[0].eax = RT_MIN(pCPUM->aGuestCpuIdStd[0].eax, 5);
-
-        /* 0: Vendor */
-        pCPUM->aGuestCpuIdStd[0].ebx = pCPUM->aGuestCpuIdExt[0].ebx = ((uint32_t *)s_szVendor)[0];
-        pCPUM->aGuestCpuIdStd[0].ecx = pCPUM->aGuestCpuIdExt[0].ecx = ((uint32_t *)s_szVendor)[2];
-        pCPUM->aGuestCpuIdStd[0].edx = pCPUM->aGuestCpuIdExt[0].edx = ((uint32_t *)s_szVendor)[1];
-
-        /* 1.eax: Version information.  family : model : stepping */
-        pCPUM->aGuestCpuIdStd[1].eax = (0xf << 8) + (0x1 << 4) + 1;
-
-        /* Leaves 2 - 4 are Intel only - zero them out */
-        memset(&pCPUM->aGuestCpuIdStd[2], 0, sizeof(pCPUM->aGuestCpuIdStd[2]));
-        memset(&pCPUM->aGuestCpuIdStd[3], 0, sizeof(pCPUM->aGuestCpuIdStd[3]));
-        memset(&pCPUM->aGuestCpuIdStd[4], 0, sizeof(pCPUM->aGuestCpuIdStd[4]));
-
-        /* Leaf 5 = monitor/mwait */
-
-        /* Limit the nr of extended leaves: 0x80000008 to include the max virtual and physical address size (64 bits guests). */
-        pCPUM->aGuestCpuIdExt[0].eax = RT_MIN(pCPUM->aGuestCpuIdExt[0].eax, 0x80000008);
-        /* AMD only - set to zero. */
-        pCPUM->aGuestCpuIdExt[0].ebx = pCPUM->aGuestCpuIdExt[0].ecx = pCPUM->aGuestCpuIdExt[0].edx = 0;
-
-        /* 0x800000001: AMD only; shared feature bits are set dynamically. */
-        memset(&pCPUM->aGuestCpuIdExt[1], 0, sizeof(pCPUM->aGuestCpuIdExt[1]));
-
-        /* 0x800000002-4: Processor Name String Identifier. */
-        pCPUM->aGuestCpuIdExt[2].eax = ((uint32_t *)s_szProcessor)[0];
-        pCPUM->aGuestCpuIdExt[2].ebx = ((uint32_t *)s_szProcessor)[1];
-        pCPUM->aGuestCpuIdExt[2].ecx = ((uint32_t *)s_szProcessor)[2];
-        pCPUM->aGuestCpuIdExt[2].edx = ((uint32_t *)s_szProcessor)[3];
-        pCPUM->aGuestCpuIdExt[3].eax = ((uint32_t *)s_szProcessor)[4];
-        pCPUM->aGuestCpuIdExt[3].ebx = ((uint32_t *)s_szProcessor)[5];
-        pCPUM->aGuestCpuIdExt[3].ecx = ((uint32_t *)s_szProcessor)[6];
-        pCPUM->aGuestCpuIdExt[3].edx = ((uint32_t *)s_szProcessor)[7];
-        pCPUM->aGuestCpuIdExt[4].eax = ((uint32_t *)s_szProcessor)[8];
-        pCPUM->aGuestCpuIdExt[4].ebx = ((uint32_t *)s_szProcessor)[9];
-        pCPUM->aGuestCpuIdExt[4].ecx = ((uint32_t *)s_szProcessor)[10];
-        pCPUM->aGuestCpuIdExt[4].edx = ((uint32_t *)s_szProcessor)[11];
-
-        /* 0x800000005-7 - reserved -> zero */
-        memset(&pCPUM->aGuestCpuIdExt[5], 0, sizeof(pCPUM->aGuestCpuIdExt[5]));
-        memset(&pCPUM->aGuestCpuIdExt[6], 0, sizeof(pCPUM->aGuestCpuIdExt[6]));
-        memset(&pCPUM->aGuestCpuIdExt[7], 0, sizeof(pCPUM->aGuestCpuIdExt[7]));
-
-        /* 0x800000008: only the max virtual and physical address size. */
-        pCPUM->aGuestCpuIdExt[8].ecx = pCPUM->aGuestCpuIdExt[8].ebx = pCPUM->aGuestCpuIdExt[8].edx = 0;  /* reserved */
-    }
-
-    /*
-     * Hide HTT, multicode, SMP, whatever.
-     * (APIC-ID := 0 and #LogCpus := 0)
-     */
-    pCPUM->aGuestCpuIdStd[1].ebx &= 0x0000ffff;
-#ifdef VBOX_WITH_MULTI_CORE
-    if (    pCPUM->enmGuestCpuVendor != CPUMCPUVENDOR_SYNTHETIC
-        &&  pVM->cCpus > 1)
-    {
-        /* If CPUID Fn0000_0001_EDX[HTT] = 1 then LogicalProcessorCount is the number of threads per CPU core times the number of CPU cores per processor */
-        pCPUM->aGuestCpuIdStd[1].ebx |= (pVM->cCpus << 16);
-        pCPUM->aGuestCpuIdStd[1].edx |= X86_CPUID_FEATURE_EDX_HTT;  /* necessary for hyper-threading *or* multi-core CPUs */
-    }
-#endif
-
-    /* Cpuid 2:
-     * Intel: Cache and TLB information
-     * AMD:   Reserved
-     * Safe to expose; restrict the number of calls to 1 for the portable case.
-     */
-    if (    pCPUM->u8PortableCpuIdLevel > 0
-        &&  pCPUM->aGuestCpuIdStd[0].eax >= 2
-        && (pCPUM->aGuestCpuIdStd[2].eax & 0xff) > 1)
-    {
-        LogRel(("PortableCpuId: Std[2].al: %d -> 1\n", pCPUM->aGuestCpuIdStd[2].eax & 0xff));
-        pCPUM->aGuestCpuIdStd[2].eax &= UINT32_C(0xfffffffe);
-    }
-
-    /* Cpuid 3:
-     * Intel: EAX, EBX - reserved (transmeta uses these)
-     *        ECX, EDX - Processor Serial Number if available, otherwise reserved
-     * AMD:   Reserved
-     * Safe to expose
-     */
-    if (!(pCPUM->aGuestCpuIdStd[1].edx & X86_CPUID_FEATURE_EDX_PSN))
-    {
-        pCPUM->aGuestCpuIdStd[3].ecx = pCPUM->aGuestCpuIdStd[3].edx = 0;
-        if (pCPUM->u8PortableCpuIdLevel > 0)
-            pCPUM->aGuestCpuIdStd[3].eax = pCPUM->aGuestCpuIdStd[3].ebx = 0;
-    }
-
-    /* Cpuid 4:
-     * Intel: Deterministic Cache Parameters Leaf
-     *        Note: Depends on the ECX input! -> Feeling rather lazy now, so we just return 0
-     * AMD:   Reserved
-     * Safe to expose, except for EAX:
-     *      Bits 25-14: Maximum number of addressable IDs for logical processors sharing this cache (see note)**
-     *      Bits 31-26: Maximum number of processor cores in this physical package**
-     * Note: These SMP values are constant regardless of ECX
-     */
-    pCPUM->aGuestCpuIdStd[4].ecx = pCPUM->aGuestCpuIdStd[4].edx = 0;
-    pCPUM->aGuestCpuIdStd[4].eax = pCPUM->aGuestCpuIdStd[4].ebx = 0;
-#ifdef VBOX_WITH_MULTI_CORE
-    if (    pVM->cCpus > 1
-        &&  pVM->cpum.s.enmGuestCpuVendor == CPUMCPUVENDOR_INTEL)
-    {
-        AssertReturn(pVM->cCpus <= 64, VERR_TOO_MANY_CPUS);
-        /* One logical processor with possibly multiple cores. */
-        /* See  http://www.intel.com/Assets/PDF/appnote/241618.pdf p. 29 */
-        pCPUM->aGuestCpuIdStd[4].eax |= ((pVM->cCpus - 1) << 26);   /* 6 bits only -> 64 cores! */
-    }
-#endif
-
-    /* Cpuid 5:     Monitor/mwait Leaf
-     * Intel: ECX, EDX - reserved
-     *        EAX, EBX - Smallest and largest monitor line size
-     * AMD:   EDX - reserved
-     *        EAX, EBX - Smallest and largest monitor line size
-     *        ECX - extensions (ignored for now)
-     * Safe to expose
-     */
-    if (!(pCPUM->aGuestCpuIdStd[1].ecx & X86_CPUID_FEATURE_ECX_MONITOR))
-        pCPUM->aGuestCpuIdStd[5].eax = pCPUM->aGuestCpuIdStd[5].ebx = 0;
-
-    pCPUM->aGuestCpuIdStd[5].ecx = pCPUM->aGuestCpuIdStd[5].edx = 0;
-    /** @cfgm{/CPUM/MWaitExtensions, boolean, false}
-     * Expose MWAIT extended features to the guest.  For now we expose
-     * just MWAIT break on interrupt feature (bit 1).
-     */
-    bool fMWaitExtensions;
-    rc = CFGMR3QueryBoolDef(pCpumCfg, "MWaitExtensions", &fMWaitExtensions, false); AssertRCReturn(rc, rc);
-    if (fMWaitExtensions)
-    {
-        pCPUM->aGuestCpuIdStd[5].ecx = X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0;
-        /* @todo: for now we just expose host's MWAIT C-states, although conceptually
-           it shall be part of our power management virtualization model */
-#if 0
-        /* MWAIT sub C-states */
-        pCPUM->aGuestCpuIdStd[5].edx =
-                (0 << 0)  /* 0 in C0 */ |
-                (2 << 4)  /* 2 in C1 */ |
-                (2 << 8)  /* 2 in C2 */ |
-                (2 << 12) /* 2 in C3 */ |
-                (0 << 16) /* 0 in C4 */
-                ;
-#endif
-    }
-    else
-        pCPUM->aGuestCpuIdStd[5].ecx = pCPUM->aGuestCpuIdStd[5].edx = 0;
-
-    /* Cpuid 0x800000005 & 0x800000006 contain information about L1, L2 & L3 cache and TLB identifiers.
-     * Safe to pass on to the guest.
-     *
-     * Intel: 0x800000005 reserved
-     *        0x800000006 L2 cache information
-     * AMD:   0x800000005 L1 cache information
-     *        0x800000006 L2/L3 cache information
-     */
-
-    /* Cpuid 0x800000007:
-     * AMD:               EAX, EBX, ECX - reserved
-     *                    EDX: Advanced Power Management Information
-     * Intel:             Reserved
-     */
-    if (pCPUM->aGuestCpuIdExt[0].eax >= UINT32_C(0x80000007))
-    {
-        Assert(pVM->cpum.s.enmGuestCpuVendor != CPUMCPUVENDOR_INVALID);
-
-        pCPUM->aGuestCpuIdExt[7].eax = pCPUM->aGuestCpuIdExt[7].ebx = pCPUM->aGuestCpuIdExt[7].ecx = 0;
-
-        if (pVM->cpum.s.enmGuestCpuVendor == CPUMCPUVENDOR_AMD)
-        {
-            /* Only expose the TSC invariant capability bit to the guest. */
-            pCPUM->aGuestCpuIdExt[7].edx    &= 0
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_TS
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_FID
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_VID
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_TTP
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_TM
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_STC
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_MC
-                                            //| X86_CPUID_AMD_ADVPOWER_EDX_HWPSTATE
-#if 0 /* We don't expose X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR, because newer
-       * Linux kernels blindly assume that the AMD performance counters work
-       * if this is set for 64 bits guests. (Can't really find a CPUID feature
-       * bit for them though.) */
-                                            | X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR
-#endif
-                                            | 0;
-        }
-        else
-            pCPUM->aGuestCpuIdExt[7].edx    = 0;
-    }
-
-    /* Cpuid 0x800000008:
-     * AMD:               EBX, EDX - reserved
-     *                    EAX: Virtual/Physical/Guest address Size
-     *                    ECX: Number of cores + APICIdCoreIdSize
-     * Intel:             EAX: Virtual/Physical address Size
-     *                    EBX, ECX, EDX - reserved
-     */
-    if (pCPUM->aGuestCpuIdExt[0].eax >= UINT32_C(0x80000008))
-    {
-        /* Only expose the virtual and physical address sizes to the guest. */
-        pCPUM->aGuestCpuIdExt[8].eax &= UINT32_C(0x0000ffff);
-        pCPUM->aGuestCpuIdExt[8].ebx = pCPUM->aGuestCpuIdExt[8].edx = 0;  /* reserved */
-        /* Set APICIdCoreIdSize to zero (use legacy method to determine the number of cores per cpu)
-         * NC (0-7) Number of cores; 0 equals 1 core */
-        pCPUM->aGuestCpuIdExt[8].ecx = 0;
-#ifdef VBOX_WITH_MULTI_CORE
-        if (    pVM->cCpus > 1
-            &&  pVM->cpum.s.enmGuestCpuVendor == CPUMCPUVENDOR_AMD)
-        {
-            /* Legacy method to determine the number of cores. */
-            pCPUM->aGuestCpuIdExt[1].ecx |= X86_CPUID_AMD_FEATURE_ECX_CMPL;
-            pCPUM->aGuestCpuIdExt[8].ecx |= (pVM->cCpus - 1); /* NC: Number of CPU cores - 1; 8 bits */
-        }
-#endif
-    }
-
-    /** @cfgm{/CPUM/NT4LeafLimit, boolean, false}
-     * Limit the number of standard CPUID leaves to 0..3 to prevent NT4 from
-     * bugchecking with MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED (0x3e).
-     * This option corresponds somewhat to IA32_MISC_ENABLES.BOOT_NT4[bit 22].
-     */
-    bool fNt4LeafLimit;
-    rc = CFGMR3QueryBoolDef(pCpumCfg, "NT4LeafLimit", &fNt4LeafLimit, false); AssertRCReturn(rc, rc);
-    if (fNt4LeafLimit)
-        pCPUM->aGuestCpuIdStd[0].eax = 3; /** @todo r=bird: shouldn't we check if pCPUM->aGuestCpuIdStd[0].eax > 3 before setting it 3 here? */
-
-    /*
-     * Limit it the number of entries and fill the remaining with the defaults.
-     *
-     * The limits are masking off stuff about power saving and similar, this
-     * is perhaps a bit crudely done as there is probably some relatively harmless
-     * info too in these leaves (like words about having a constant TSC).
-     */
-    if (pCPUM->aGuestCpuIdStd[0].eax > 5)
-        pCPUM->aGuestCpuIdStd[0].eax = 5;
-    for (i = pCPUM->aGuestCpuIdStd[0].eax + 1; i < RT_ELEMENTS(pCPUM->aGuestCpuIdStd); i++)
-        pCPUM->aGuestCpuIdStd[i] = pCPUM->GuestCpuIdDef;
-
-    if (pCPUM->aGuestCpuIdExt[0].eax > UINT32_C(0x80000008))
-        pCPUM->aGuestCpuIdExt[0].eax = UINT32_C(0x80000008);
-    for (i = pCPUM->aGuestCpuIdExt[0].eax >= UINT32_C(0x80000000)
-           ? pCPUM->aGuestCpuIdExt[0].eax - UINT32_C(0x80000000) + 1
-           : 0;
-         i < RT_ELEMENTS(pCPUM->aGuestCpuIdExt);
-         i++)
-        pCPUM->aGuestCpuIdExt[i] = pCPUM->GuestCpuIdDef;
-
-    /*
-     * Centaur stuff (VIA).
-     *
-     * The important part here (we think) is to make sure the 0xc0000000
-     * function returns 0xc0000001. As for the features, we don't currently
-     * let on about any of those... 0xc0000002 seems to be some
-     * temperature/hz/++ stuff, include it as well (static).
-     */
-    if (    pCPUM->aGuestCpuIdCentaur[0].eax >= UINT32_C(0xc0000000)
-        &&  pCPUM->aGuestCpuIdCentaur[0].eax <= UINT32_C(0xc0000004))
-    {
-        pCPUM->aGuestCpuIdCentaur[0].eax = RT_MIN(pCPUM->aGuestCpuIdCentaur[0].eax, UINT32_C(0xc0000002));
-        pCPUM->aGuestCpuIdCentaur[1].edx = 0; /* all features hidden */
-        for (i = pCPUM->aGuestCpuIdCentaur[0].eax - UINT32_C(0xc0000000);
-             i < RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur);
-             i++)
-            pCPUM->aGuestCpuIdCentaur[i] = pCPUM->GuestCpuIdDef;
-    }
-    else
-        for (i = 0; i < RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur); i++)
-            pCPUM->aGuestCpuIdCentaur[i] = pCPUM->GuestCpuIdDef;
-
-
-    /*
-     * Load CPUID overrides from configuration.
-     * Note: Kind of redundant now, but allows unchanged overrides
-     */
-    /** @cfgm{CPUM/CPUID/[000000xx|800000xx|c000000x]/[eax|ebx|ecx|edx],32-bit}
-     * Overrides the CPUID leaf values. */
-    PCFGMNODE pOverrideCfg = CFGMR3GetChild(pCpumCfg, "CPUID");
-    rc = cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x00000000), &pCPUM->aGuestCpuIdStd[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdStd),     pOverrideCfg);
-    AssertRCReturn(rc, rc);
-    rc = cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x80000000), &pCPUM->aGuestCpuIdExt[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdExt),     pOverrideCfg);
-    AssertRCReturn(rc, rc);
-    rc = cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0xc0000000), &pCPUM->aGuestCpuIdCentaur[0], RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur), pOverrideCfg);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Check if PAE was explicitely enabled by the user.
-     */
-    bool fEnable;
-    rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "EnablePAE", &fEnable, false);      AssertRCReturn(rc, rc);
-    if (fEnable)
-        CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
-
-    /*
-     * We don't normally enable NX for raw-mode, so give the user a chance to
-     * force it on.
-     */
-    rc = CFGMR3QueryBoolDef(pCpumCfg, "EnableNX", &fEnable, false);                 AssertRCReturn(rc, rc);
-    if (fEnable)
-        CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
-
-    /*
-     * Log the cpuid and we're good.
-     */
-    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
-    RTCPUSET OnlineSet;
-    LogRel(("Logical host processors: %u present, %u max, %u online, online mask: %016RX64\n",
-            (unsigned)RTMpGetPresentCount(), (unsigned)RTMpGetCount(), (unsigned)RTMpGetOnlineCount(),
-            RTCpuSetToU64(RTMpGetOnlineSet(&OnlineSet)) ));
-    LogRel(("************************* CPUID dump ************************\n"));
-    DBGFR3Info(pVM, "cpuid", "verbose", DBGFR3InfoLogRelHlp());
-    LogRel(("\n"));
-    DBGFR3InfoLog(pVM, "cpuid", "verbose"); /* macro */
-    RTLogRelSetBuffering(fOldBuffered);
-    LogRel(("******************** End of CPUID dump **********************\n"));
-
-#undef PORTABLE_DISABLE_FEATURE_BIT
-#undef PORTABLE_CLEAR_BITS_WHEN
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * The CPUM will update the addresses used by the switcher.
- *
- * @param   pVM     The VM.
- */
-VMMR3DECL(void) CPUMR3Relocate(PVM pVM)
-{
-    LogFlow(("CPUMR3Relocate\n"));
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        /*
-         * Switcher pointers.
-         */
-        PVMCPU pVCpu = &pVM->aCpus[i];
-        pVCpu->cpum.s.pHyperCoreRC = MMHyperCCToRC(pVM, pVCpu->cpum.s.pHyperCoreR3);
-        Assert(pVCpu->cpum.s.pHyperCoreRC != NIL_RTRCPTR);
-
-    }
-}
-
-
-/**
- * Apply late CPUM property changes based on the fHWVirtEx setting
- *
- * @param   pVM                 The VM to operate on.
- * @param   fHWVirtExEnabled    HWVirtEx enabled/disabled
- */
-VMMR3DECL(void) CPUMR3SetHWVirtEx(PVM pVM, bool fHWVirtExEnabled)
-{
-    /*
-     * Workaround for missing cpuid(0) patches when leaf 4 returns GuestCpuIdDef:
-     * If we miss to patch a cpuid(0).eax then Linux tries to determine the number
-     * of processors from (cpuid(4).eax >> 26) + 1.
-     *
-     * Note: this code is obsolete, but let's keep it here for reference.
-     *       Purpose is valid when we artificially cap the max std id to less than 4.
-     */
-    if (!fHWVirtExEnabled)
-    {
-        Assert(pVM->cpum.s.aGuestCpuIdStd[4].eax == 0);
-        pVM->cpum.s.aGuestCpuIdStd[4].eax = 0;
-    }
-}
-
-/**
- * Terminates the CPUM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) CPUMR3Term(PVM pVM)
-{
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU   pVCpu = &pVM->aCpus[i];
-        PCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
-
-        memset(pVCpu->cpum.s.aMagic, 0, sizeof(pVCpu->cpum.s.aMagic));
-        pVCpu->cpum.s.uMagic     = 0;
-        pCtx->dr[5]              = 0;
-    }
-#endif
-    return 0;
-}
-
-
-/**
- * Resets a virtual CPU.
- *
- * Used by CPUMR3Reset and CPU hot plugging.
- *
- * @param   pVCpu               The virtual CPU handle.
- */
-VMMR3DECL(void) CPUMR3ResetCpu(PVMCPU pVCpu)
-{
-    /** @todo anything different for VCPU > 0? */
-    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
-
-    /*
-     * Initialize everything to ZERO first.
-     */
-    uint32_t fUseFlags =  pVCpu->cpum.s.fUseFlags & ~CPUM_USED_FPU_SINCE_REM;
-    memset(pCtx, 0, sizeof(*pCtx));
-    pVCpu->cpum.s.fUseFlags  = fUseFlags;
-
-    pCtx->cr0                       = X86_CR0_CD | X86_CR0_NW | X86_CR0_ET;  //0x60000010
-    pCtx->eip                       = 0x0000fff0;
-    pCtx->edx                       = 0x00000600;   /* P6 processor */
-    pCtx->eflags.Bits.u1Reserved0   = 1;
-
-    pCtx->cs                        = 0xf000;
-    pCtx->csHid.u64Base             = UINT64_C(0xffff0000);
-    pCtx->csHid.u32Limit            = 0x0000ffff;
-    pCtx->csHid.Attr.n.u1DescType   = 1; /* code/data segment */
-    pCtx->csHid.Attr.n.u1Present    = 1;
-    pCtx->csHid.Attr.n.u4Type       = X86_SEL_TYPE_READ | X86_SEL_TYPE_CODE;
-
-    pCtx->dsHid.u32Limit            = 0x0000ffff;
-    pCtx->dsHid.Attr.n.u1DescType   = 1; /* code/data segment */
-    pCtx->dsHid.Attr.n.u1Present    = 1;
-    pCtx->dsHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
-
-    pCtx->esHid.u32Limit            = 0x0000ffff;
-    pCtx->esHid.Attr.n.u1DescType   = 1; /* code/data segment */
-    pCtx->esHid.Attr.n.u1Present    = 1;
-    pCtx->esHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
-
-    pCtx->fsHid.u32Limit            = 0x0000ffff;
-    pCtx->fsHid.Attr.n.u1DescType   = 1; /* code/data segment */
-    pCtx->fsHid.Attr.n.u1Present    = 1;
-    pCtx->fsHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
-
-    pCtx->gsHid.u32Limit            = 0x0000ffff;
-    pCtx->gsHid.Attr.n.u1DescType   = 1; /* code/data segment */
-    pCtx->gsHid.Attr.n.u1Present    = 1;
-    pCtx->gsHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
-
-    pCtx->ssHid.u32Limit            = 0x0000ffff;
-    pCtx->ssHid.Attr.n.u1Present    = 1;
-    pCtx->ssHid.Attr.n.u1DescType   = 1; /* code/data segment */
-    pCtx->ssHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
-
-    pCtx->idtr.cbIdt                = 0xffff;
-    pCtx->gdtr.cbGdt                = 0xffff;
-
-    pCtx->ldtrHid.u32Limit          = 0xffff;
-    pCtx->ldtrHid.Attr.n.u1Present  = 1;
-    pCtx->ldtrHid.Attr.n.u4Type     = X86_SEL_TYPE_SYS_LDT;
-
-    pCtx->trHid.u32Limit            = 0xffff;
-    pCtx->trHid.Attr.n.u1Present    = 1;
-    pCtx->trHid.Attr.n.u4Type       = X86_SEL_TYPE_SYS_286_TSS_BUSY;
-
-    pCtx->dr[6]                     = X86_DR6_INIT_VAL;
-    pCtx->dr[7]                     = X86_DR7_INIT_VAL;
-
-    pCtx->fpu.FTW                   = 0xff;         /* All tags are set, i.e. the regs are empty. */
-    pCtx->fpu.FCW                   = 0x37f;
-
-    /* Intel 64 and IA-32 Architectures Software Developer's Manual Volume 3A, Table 8-1. IA-32 Processor States Following Power-up, Reset, or INIT */
-    pCtx->fpu.MXCSR                 = 0x1F80;
-
-    /* Init PAT MSR */
-    pCtx->msrPAT                    = UINT64_C(0x0007040600070406); /** @todo correct? */
-
-    /* Reset EFER; see AMD64 Architecture Programmer's Manual Volume 2: Table 14-1. Initial Processor State
-    * The Intel docs don't mention it.
-    */
-    pCtx->msrEFER                   = 0;
-}
-
-
-/**
- * Resets the CPU.
- *
- * @returns VINF_SUCCESS.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(void) CPUMR3Reset(PVM pVM)
-{
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        CPUMR3ResetCpu(&pVM->aCpus[i]);
-
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-        PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(&pVM->aCpus[i]);
-
-        /* Magic marker for searching in crash dumps. */
-        strcpy((char *)pVM->aCpus[i].cpum.s.aMagic, "CPUMCPU Magic");
-        pVM->aCpus[i].cpum.s.uMagic     = UINT64_C(0xDEADBEEFDEADBEEF);
-        pCtx->dr[5]                     = UINT64_C(0xDEADBEEFDEADBEEF);
-#endif
-    }
-}
-
-
-/**
- * Called both in pass 0 and the final pass.
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static void cpumR3SaveCpuId(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Save all the CPU ID leaves here so we can check them for compatibility
-     * upon loading.
-     */
-    SSMR3PutU32(pSSM, RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd));
-    SSMR3PutMem(pSSM, &pVM->cpum.s.aGuestCpuIdStd[0], sizeof(pVM->cpum.s.aGuestCpuIdStd));
-
-    SSMR3PutU32(pSSM, RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt));
-    SSMR3PutMem(pSSM, &pVM->cpum.s.aGuestCpuIdExt[0], sizeof(pVM->cpum.s.aGuestCpuIdExt));
-
-    SSMR3PutU32(pSSM, RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur));
-    SSMR3PutMem(pSSM, &pVM->cpum.s.aGuestCpuIdCentaur[0], sizeof(pVM->cpum.s.aGuestCpuIdCentaur));
-
-    SSMR3PutMem(pSSM, &pVM->cpum.s.GuestCpuIdDef, sizeof(pVM->cpum.s.GuestCpuIdDef));
-
-    /*
-     * Save a good portion of the raw CPU IDs as well as they may come in
-     * handy when validating features for raw mode.
-     */
-    CPUMCPUID   aRawStd[16];
-    for (unsigned i = 0; i < RT_ELEMENTS(aRawStd); i++)
-        ASMCpuId(i, &aRawStd[i].eax, &aRawStd[i].ebx, &aRawStd[i].ecx, &aRawStd[i].edx);
-    SSMR3PutU32(pSSM, RT_ELEMENTS(aRawStd));
-    SSMR3PutMem(pSSM, &aRawStd[0], sizeof(aRawStd));
-
-    CPUMCPUID   aRawExt[32];
-    for (unsigned i = 0; i < RT_ELEMENTS(aRawExt); i++)
-        ASMCpuId(i | UINT32_C(0x80000000), &aRawExt[i].eax, &aRawExt[i].ebx, &aRawExt[i].ecx, &aRawExt[i].edx);
-    SSMR3PutU32(pSSM, RT_ELEMENTS(aRawExt));
-    SSMR3PutMem(pSSM, &aRawExt[0], sizeof(aRawExt));
-}
-
-
-/**
- * Loads the CPU ID leaves saved by pass 0.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- * @param   uVersion            The format version.
- */
-static int cpumR3LoadCpuId(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
-{
-    AssertMsgReturn(uVersion >= CPUM_SAVED_STATE_VERSION_VER3_2, ("%u\n", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
-
-    /*
-     * Define a bunch of macros for simplifying the code.
-     */
-    /* Generic expression + failure message. */
-#define CPUID_CHECK_RET(expr, fmt) \
-    do { \
-        if (!(expr)) \
-        { \
-            char *pszMsg = RTStrAPrintf2 fmt; /* lack of variadic macros sucks */ \
-            if (fStrictCpuIdChecks) \
-            { \
-                int rcCpuid = SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, "%s", pszMsg); \
-                RTStrFree(pszMsg); \
-                return rcCpuid; \
-            } \
-            LogRel(("CPUM: %s\n", pszMsg)); \
-            RTStrFree(pszMsg); \
-        } \
-    } while (0)
-#define CPUID_CHECK_WRN(expr, fmt) \
-    do { \
-        if (!(expr)) \
-            LogRel(fmt); \
-    } while (0)
-
-    /* For comparing two values and bitch if they differs. */
-#define CPUID_CHECK2_RET(what, host, saved) \
-    do { \
-        if ((host) != (saved)) \
-        { \
-            if (fStrictCpuIdChecks) \
-                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
-                                         N_(#what " mismatch: host=%#x saved=%#x"), (host), (saved)); \
-            LogRel(("CPUM: " #what " differs: host=%#x saved=%#x\n", (host), (saved))); \
-        } \
-    } while (0)
-#define CPUID_CHECK2_WRN(what, host, saved) \
-    do { \
-        if ((host) != (saved)) \
-            LogRel(("CPUM: " #what " differs: host=%#x saved=%#x\n", (host), (saved))); \
-    } while (0)
-
-    /* For checking raw cpu features (raw mode). */
-#define CPUID_RAW_FEATURE_RET(set, reg, bit) \
-    do { \
-        if ((aHostRaw##set [1].reg & bit) != (aRaw##set [1].reg & bit)) \
-        { \
-            if (fStrictCpuIdChecks) \
-                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
-                                         N_(#bit " mismatch: host=%d saved=%d"), \
-                                         !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) ); \
-            LogRel(("CPUM: " #bit" differs: host=%d saved=%d\n", \
-                    !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) )); \
-        } \
-    } while (0)
-#define CPUID_RAW_FEATURE_WRN(set, reg, bit) \
-    do { \
-        if ((aHostRaw##set [1].reg & bit) != (aRaw##set [1].reg & bit)) \
-            LogRel(("CPUM: " #bit" differs: host=%d saved=%d\n", \
-                    !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) )); \
-    } while (0)
-#define CPUID_RAW_FEATURE_IGN(set, reg, bit) do { } while (0)
-
-    /* For checking guest features. */
-#define CPUID_GST_FEATURE_RET(set, reg, bit) \
-    do { \
-        if (    (aGuestCpuId##set [1].reg & bit) \
-            && !(aHostRaw##set [1].reg & bit) \
-            && !(aHostOverride##set [1].reg & bit) \
-            && !(aGuestOverride##set [1].reg & bit) \
-           ) \
-        { \
-            if (fStrictCpuIdChecks) \
-                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
-                                         N_(#bit " is not supported by the host but has already exposed to the guest")); \
-            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
-        } \
-    } while (0)
-#define CPUID_GST_FEATURE_WRN(set, reg, bit) \
-    do { \
-        if (    (aGuestCpuId##set [1].reg & bit) \
-            && !(aHostRaw##set [1].reg & bit) \
-            && !(aHostOverride##set [1].reg & bit) \
-            && !(aGuestOverride##set [1].reg & bit) \
-           ) \
-            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
-    } while (0)
-#define CPUID_GST_FEATURE_EMU(set, reg, bit) \
-    do { \
-        if (    (aGuestCpuId##set [1].reg & bit) \
-            && !(aHostRaw##set [1].reg & bit) \
-            && !(aHostOverride##set [1].reg & bit) \
-            && !(aGuestOverride##set [1].reg & bit) \
-           ) \
-            LogRel(("CPUM: Warning - " #bit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
-    } while (0)
-#define CPUID_GST_FEATURE_IGN(set, reg, bit) do { } while (0)
-
-    /* For checking guest features if AMD guest CPU. */
-#define CPUID_GST_AMD_FEATURE_RET(set, reg, bit) \
-    do { \
-        if (    (aGuestCpuId##set [1].reg & bit) \
-            &&  fGuestAmd \
-            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
-            && !(aHostOverride##set [1].reg & bit) \
-            && !(aGuestOverride##set [1].reg & bit) \
-           ) \
-        { \
-            if (fStrictCpuIdChecks) \
-                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
-                                         N_(#bit " is not supported by the host but has already exposed to the guest")); \
-            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
-        } \
-    } while (0)
-#define CPUID_GST_AMD_FEATURE_WRN(set, reg, bit) \
-    do { \
-        if (    (aGuestCpuId##set [1].reg & bit) \
-            &&  fGuestAmd \
-            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
-            && !(aHostOverride##set [1].reg & bit) \
-            && !(aGuestOverride##set [1].reg & bit) \
-           ) \
-            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
-    } while (0)
-#define CPUID_GST_AMD_FEATURE_EMU(set, reg, bit) \
-    do { \
-        if (    (aGuestCpuId##set [1].reg & bit) \
-            &&  fGuestAmd \
-            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
-            && !(aHostOverride##set [1].reg & bit) \
-            && !(aGuestOverride##set [1].reg & bit) \
-           ) \
-            LogRel(("CPUM: Warning - " #bit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
-    } while (0)
-#define CPUID_GST_AMD_FEATURE_IGN(set, reg, bit) do { } while (0)
-
-    /* For checking AMD features which have a corresponding bit in the standard
-       range.  (Intel defines very few bits in the extended feature sets.) */
-#define CPUID_GST_FEATURE2_RET(reg, ExtBit, StdBit) \
-    do { \
-        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
-            && !(fHostAmd  \
-                 ? aHostRawExt[1].reg      & (ExtBit) \
-                 : aHostRawStd[1].reg      & (StdBit)) \
-            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
-            && !(aGuestOverrideExt[1].reg  & (ExtBit)) \
-           ) \
-        { \
-            if (fStrictCpuIdChecks) \
-                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
-                                         N_(#ExtBit " is not supported by the host but has already exposed to the guest")); \
-            LogRel(("CPUM: " #ExtBit " is not supported by the host but has already exposed to the guest\n")); \
-        } \
-    } while (0)
-#define CPUID_GST_FEATURE2_WRN(reg, ExtBit, StdBit) \
-    do { \
-        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
-            && !(fHostAmd  \
-                 ? aHostRawExt[1].reg      & (ExtBit) \
-                 : aHostRawStd[1].reg      & (StdBit)) \
-            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
-            && !(aGuestOverrideExt[1].reg  & (ExtBit)) \
-           ) \
-            LogRel(("CPUM: " #ExtBit " is not supported by the host but has already exposed to the guest\n")); \
-    } while (0)
-#define CPUID_GST_FEATURE2_EMU(reg, ExtBit, StdBit) \
-    do { \
-        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
-            && !(fHostAmd  \
-                 ? aHostRawExt[1].reg      & (ExtBit) \
-                 : aHostRawStd[1].reg      & (StdBit)) \
-            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
-            && !(aGuestOverrideExt[1].reg  & (ExtBit)) \
-           ) \
-            LogRel(("CPUM: Warning - " #ExtBit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
-    } while (0)
-#define CPUID_GST_FEATURE2_IGN(reg, ExtBit, StdBit) do { } while (0)
-
-    /*
-     * Load them into stack buffers first.
-     */
-    CPUMCPUID   aGuestCpuIdStd[RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd)];
-    uint32_t    cGuestCpuIdStd;
-    int rc = SSMR3GetU32(pSSM, &cGuestCpuIdStd); AssertRCReturn(rc, rc);
-    if (cGuestCpuIdStd > RT_ELEMENTS(aGuestCpuIdStd))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    SSMR3GetMem(pSSM, &aGuestCpuIdStd[0], cGuestCpuIdStd * sizeof(aGuestCpuIdStd[0]));
-
-    CPUMCPUID   aGuestCpuIdExt[RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt)];
-    uint32_t    cGuestCpuIdExt;
-    rc = SSMR3GetU32(pSSM, &cGuestCpuIdExt); AssertRCReturn(rc, rc);
-    if (cGuestCpuIdExt > RT_ELEMENTS(aGuestCpuIdExt))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    SSMR3GetMem(pSSM, &aGuestCpuIdExt[0], cGuestCpuIdExt * sizeof(aGuestCpuIdExt[0]));
-
-    CPUMCPUID   aGuestCpuIdCentaur[RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur)];
-    uint32_t    cGuestCpuIdCentaur;
-    rc = SSMR3GetU32(pSSM, &cGuestCpuIdCentaur); AssertRCReturn(rc, rc);
-    if (cGuestCpuIdCentaur > RT_ELEMENTS(aGuestCpuIdCentaur))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    SSMR3GetMem(pSSM, &aGuestCpuIdCentaur[0], cGuestCpuIdCentaur * sizeof(aGuestCpuIdCentaur[0]));
-
-    CPUMCPUID   GuestCpuIdDef;
-    rc = SSMR3GetMem(pSSM, &GuestCpuIdDef, sizeof(GuestCpuIdDef));
-    AssertRCReturn(rc, rc);
-
-    CPUMCPUID   aRawStd[16];
-    uint32_t    cRawStd;
-    rc = SSMR3GetU32(pSSM, &cRawStd); AssertRCReturn(rc, rc);
-    if (cRawStd > RT_ELEMENTS(aRawStd))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    SSMR3GetMem(pSSM, &aRawStd[0], cRawStd * sizeof(aRawStd[0]));
-
-    CPUMCPUID   aRawExt[32];
-    uint32_t    cRawExt;
-    rc = SSMR3GetU32(pSSM, &cRawExt); AssertRCReturn(rc, rc);
-    if (cRawExt > RT_ELEMENTS(aRawExt))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    rc = SSMR3GetMem(pSSM, &aRawExt[0], cRawExt * sizeof(aRawExt[0]));
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Note that we support restoring less than the current amount of standard
-     * leaves because we've been allowed more is newer version of VBox.
-     *
-     * So, pad new entries with the default.
-     */
-    for (uint32_t i = cGuestCpuIdStd; i < RT_ELEMENTS(aGuestCpuIdStd); i++)
-        aGuestCpuIdStd[i] = GuestCpuIdDef;
-
-    for (uint32_t i = cGuestCpuIdExt; i < RT_ELEMENTS(aGuestCpuIdExt); i++)
-        aGuestCpuIdExt[i] = GuestCpuIdDef;
-
-    for (uint32_t i = cGuestCpuIdCentaur; i < RT_ELEMENTS(aGuestCpuIdCentaur); i++)
-        aGuestCpuIdCentaur[i] = GuestCpuIdDef;
-
-    for (uint32_t i = cRawStd; i < RT_ELEMENTS(aRawStd); i++)
-        ASMCpuId(i, &aRawStd[i].eax, &aRawStd[i].ebx, &aRawStd[i].ecx, &aRawStd[i].edx);
-
-    for (uint32_t i = cRawExt; i < RT_ELEMENTS(aRawExt); i++)
-        ASMCpuId(i | UINT32_C(0x80000000), &aRawExt[i].eax, &aRawExt[i].ebx, &aRawExt[i].ecx, &aRawExt[i].edx);
-
-    /*
-     * Get the raw CPU IDs for the current host.
-     */
-    CPUMCPUID   aHostRawStd[16];
-    for (unsigned i = 0; i < RT_ELEMENTS(aHostRawStd); i++)
-        ASMCpuId(i, &aHostRawStd[i].eax, &aHostRawStd[i].ebx, &aHostRawStd[i].ecx, &aHostRawStd[i].edx);
-
-    CPUMCPUID   aHostRawExt[32];
-    for (unsigned i = 0; i < RT_ELEMENTS(aHostRawExt); i++)
-        ASMCpuId(i | UINT32_C(0x80000000), &aHostRawExt[i].eax, &aHostRawExt[i].ebx, &aHostRawExt[i].ecx, &aHostRawExt[i].edx);
-
-    /*
-     * Get the host and guest overrides so we don't reject the state because
-     * some feature was enabled thru these interfaces.
-     * Note! We currently only need the feature leaves, so skip rest.
-     */
-    PCFGMNODE   pOverrideCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM/CPUID");
-    CPUMCPUID   aGuestOverrideStd[2];
-    memcpy(&aGuestOverrideStd[0], &aHostRawStd[0], sizeof(aGuestOverrideStd));
-    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x00000000), &aGuestOverrideStd[0], RT_ELEMENTS(aGuestOverrideStd), pOverrideCfg);
-
-    CPUMCPUID   aGuestOverrideExt[2];
-    memcpy(&aGuestOverrideExt[0], &aHostRawExt[0], sizeof(aGuestOverrideExt));
-    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x80000000), &aGuestOverrideExt[0], RT_ELEMENTS(aGuestOverrideExt), pOverrideCfg);
-
-    pOverrideCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM/HostCPUID");
-    CPUMCPUID   aHostOverrideStd[2];
-    memcpy(&aHostOverrideStd[0], &aHostRawStd[0], sizeof(aHostOverrideStd));
-    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x00000000), &aHostOverrideStd[0], RT_ELEMENTS(aHostOverrideStd), pOverrideCfg);
-
-    CPUMCPUID   aHostOverrideExt[2];
-    memcpy(&aHostOverrideExt[0], &aHostRawExt[0], sizeof(aHostOverrideExt));
-    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x80000000), &aHostOverrideExt[0], RT_ELEMENTS(aHostOverrideExt), pOverrideCfg);
-
-    /*
-     * This can be skipped.
-     */
-    bool fStrictCpuIdChecks;
-    CFGMR3QueryBoolDef(CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM"), "StrictCpuIdChecks", &fStrictCpuIdChecks, true);
-
-
-
-    /*
-     * For raw-mode we'll require that the CPUs are very similar since we don't
-     * intercept CPUID instructions for user mode applications.
-     */
-    if (!HWACCMIsEnabled(pVM))
-    {
-        /* CPUID(0) */
-        CPUID_CHECK_RET(   aHostRawStd[0].ebx == aRawStd[0].ebx
-                        && aHostRawStd[0].ecx == aRawStd[0].ecx
-                        && aHostRawStd[0].edx == aRawStd[0].edx,
-                        (N_("CPU vendor mismatch: host='%.4s%.4s%.4s' saved='%.4s%.4s%.4s'"),
-                         &aHostRawStd[0].ebx, &aHostRawStd[0].edx, &aHostRawStd[0].ecx,
-                         &aRawStd[0].ebx, &aRawStd[0].edx, &aRawStd[0].ecx));
-        CPUID_CHECK2_WRN("Std CPUID max leaf",   aHostRawStd[0].eax, aRawStd[0].eax);
-        CPUID_CHECK2_WRN("Reserved bits 15:14", (aHostRawExt[1].eax >> 14) & 3, (aRawExt[1].eax >> 14) & 3);
-        CPUID_CHECK2_WRN("Reserved bits 31:28",  aHostRawExt[1].eax >> 28,       aRawExt[1].eax >> 28);
-
-        bool const fIntel = ASMIsIntelCpuEx(aRawStd[0].ebx, aRawStd[0].ecx, aRawStd[0].edx);
-
-        /* CPUID(1).eax */
-        CPUID_CHECK2_RET("CPU family",          ASMGetCpuFamily(aHostRawStd[1].eax),        ASMGetCpuFamily(aRawStd[1].eax));
-        CPUID_CHECK2_RET("CPU model",           ASMGetCpuModel(aHostRawStd[1].eax, fIntel), ASMGetCpuModel(aRawStd[1].eax, fIntel));
-        CPUID_CHECK2_WRN("CPU type",            (aHostRawStd[1].eax >> 12) & 3,             (aRawStd[1].eax >> 12) & 3 );
-
-        /* CPUID(1).ebx - completely ignore CPU count and APIC ID. */
-        CPUID_CHECK2_RET("CPU brand ID",         aHostRawStd[1].ebx & 0xff,                 aRawStd[1].ebx & 0xff);
-        CPUID_CHECK2_WRN("CLFLUSH chunk count", (aHostRawStd[1].ebx >> 8) & 0xff,           (aRawStd[1].ebx >> 8) & 0xff);
-
-        /* CPUID(1).ecx */
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE3);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_PCLMUL);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_DTES64);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_MONITOR);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CPLDS);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_VMX);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_SMX);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_EST);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_TM2);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSSE3);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_CNTXID);
-        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(11) /*reserved*/ );
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_FMA);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CX16);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_TPRUPDATE);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_PDCM);
-        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(16) /*reserved*/);
-        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(17) /*reserved*/);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_DCA);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_1);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_2);
-        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_X2APIC);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_MOVBE);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_POPCNT);
-        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(24) /*reserved*/);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AES);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_XSAVE);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_OSXSAVE);
-        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AVX);
-        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(29) /*reserved*/);
-        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(30) /*reserved*/);
-        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(31) /*reserved*/);
-
-        /* CPUID(1).edx */
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FPU);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_VME);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_DE);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_TSC);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MSR);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PAE);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCE);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CX8);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_APIC);
-        CPUID_RAW_FEATURE_RET(Std, edx, RT_BIT_32(10) /*reserved*/);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_SEP);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MTRR);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PGE);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCA);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CMOV);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PAT);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE36);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSN);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CLFSH);
-        CPUID_RAW_FEATURE_RET(Std, edx, RT_BIT_32(20) /*reserved*/);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_DS);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_ACPI);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MMX);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FXSR);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE);
-        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE2);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_SS);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_HTT);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_TM);
-        CPUID_RAW_FEATURE_RET(Std, edx, RT_BIT_32(30) /*JMPE/IA64*/);
-        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PBE);
-
-        /* CPUID(2) - config, mostly about caches. ignore. */
-        /* CPUID(3) - processor serial number. ignore. */
-        /* CPUID(4) - config, cache and topology - takes ECX as input. ignore. */
-        /* CPUID(5) - mwait/monitor config. ignore. */
-        /* CPUID(6) - power management. ignore. */
-        /* CPUID(7) - ???. ignore. */
-        /* CPUID(8) - ???. ignore. */
-        /* CPUID(9) - DCA. ignore for now. */
-        /* CPUID(a) - PeMo info. ignore for now. */
-        /* CPUID(b) - topology info - takes ECX as input. ignore. */
-
-        /* CPUID(d) - XCR0 stuff - takes ECX as input. We only warn about the main level (ECX=0) for now. */
-        CPUID_CHECK_WRN(   aRawStd[0].eax     <  UINT32_C(0x0000000d)
-                        || aHostRawStd[0].eax >= UINT32_C(0x0000000d),
-                        ("CPUM: Standard leaf D was present on saved state host, not present on current.\n"));
-        if (   aRawStd[0].eax     >= UINT32_C(0x0000000d)
-            && aHostRawStd[0].eax >= UINT32_C(0x0000000d))
-        {
-            CPUID_CHECK2_WRN("Valid low XCR0 bits",             aHostRawStd[0xd].eax, aRawStd[0xd].eax);
-            CPUID_CHECK2_WRN("Valid high XCR0 bits",            aHostRawStd[0xd].edx, aRawStd[0xd].edx);
-            CPUID_CHECK2_WRN("Current XSAVE/XRSTOR area size",  aHostRawStd[0xd].ebx, aRawStd[0xd].ebx);
-            CPUID_CHECK2_WRN("Max XSAVE/XRSTOR area size",      aHostRawStd[0xd].ecx, aRawStd[0xd].ecx);
-        }
-
-        /* CPUID(0x80000000) - same as CPUID(0) except for eax.
-           Note! Intel have/is marking many of the fields here as reserved. We
-                 will verify them as if it's an AMD CPU. */
-        CPUID_CHECK_RET(   (aHostRawExt[0].eax >= UINT32_C(0x80000001) && aHostRawExt[0].eax <= UINT32_C(0x8000007f))
-                        || !(aRawExt[0].eax    >= UINT32_C(0x80000001) && aRawExt[0].eax     <= UINT32_C(0x8000007f)),
-                        (N_("Extended leaves was present on saved state host, but is missing on the current\n")));
-        if (aRawExt[0].eax >= UINT32_C(0x80000001) && aRawExt[0].eax     <= UINT32_C(0x8000007f))
-        {
-            CPUID_CHECK_RET(   aHostRawExt[0].ebx == aRawExt[0].ebx
-                            && aHostRawExt[0].ecx == aRawExt[0].ecx
-                            && aHostRawExt[0].edx == aRawExt[0].edx,
-                            (N_("CPU vendor mismatch: host='%.4s%.4s%.4s' saved='%.4s%.4s%.4s'"),
-                             &aHostRawExt[0].ebx, &aHostRawExt[0].edx, &aHostRawExt[0].ecx,
-                             &aRawExt[0].ebx,     &aRawExt[0].edx,     &aRawExt[0].ecx));
-            CPUID_CHECK2_WRN("Ext CPUID max leaf",   aHostRawExt[0].eax, aRawExt[0].eax);
-
-            /* CPUID(0x80000001).eax - same as CPUID(0).eax. */
-            CPUID_CHECK2_RET("CPU family",          ASMGetCpuFamily(aHostRawExt[1].eax),        ASMGetCpuFamily(aRawExt[1].eax));
-            CPUID_CHECK2_RET("CPU model",           ASMGetCpuModel(aHostRawExt[1].eax, fIntel), ASMGetCpuModel(aRawExt[1].eax, fIntel));
-            CPUID_CHECK2_WRN("CPU type",            (aHostRawExt[1].eax >> 12) & 3, (aRawExt[1].eax >> 12) & 3 );
-            CPUID_CHECK2_WRN("Reserved bits 15:14", (aHostRawExt[1].eax >> 14) & 3, (aRawExt[1].eax >> 14) & 3 );
-            CPUID_CHECK2_WRN("Reserved bits 31:28",  aHostRawExt[1].eax >> 28, aRawExt[1].eax >> 28);
-
-            /* CPUID(0x80000001).ebx - Brand ID (maybe), just warn if things differs. */
-            CPUID_CHECK2_WRN("CPU BrandID",          aHostRawExt[1].ebx & 0xffff, aRawExt[1].ebx & 0xffff);
-            CPUID_CHECK2_WRN("Reserved bits 16:27", (aHostRawExt[1].ebx >> 16) & 0xfff, (aRawExt[1].ebx >> 16) & 0xfff);
-            CPUID_CHECK2_WRN("PkgType",             (aHostRawExt[1].ebx >> 28) &   0xf, (aRawExt[1].ebx >> 28) &   0xf);
-
-            /* CPUID(0x80000001).ecx */
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF);
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CMPL);
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SVM);
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_EXT_APIC);
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CR8L);
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_ABM);
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE4A);
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_MISALNSSE);
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF);
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_OSVW);
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_IBS);
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE5);
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SKINIT);
-            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_WDT);
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(14));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(15));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(16));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(17));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(18));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(19));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(20));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(21));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(22));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(23));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(24));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(25));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(26));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(27));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(28));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(29));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(30));
-            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(31));
-
-            /* CPUID(0x80000001).edx */
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FPU);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_VME);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_DE);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PSE);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_TSC);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MSR);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAE);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MCE);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_CX8);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_APIC);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(10) /*reserved*/);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_SEP);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MTRR);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PGE);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MCA);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_CMOV);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAT);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PSE36);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(18) /*reserved*/);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(19) /*reserved*/);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_NX);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(21) /*reserved*/);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_AXMMX);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MMX);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FXSR);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FFXSR);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAGE1GB);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_RDTSCP);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(28) /*reserved*/);
-            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_LONG_MODE);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
-            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW);
-
-            /** @todo verify the rest as well. */
-        }
-    }
-
-
-
-    /*
-     * Verify that we can support the features already exposed to the guest on
-     * this host.
-     *
-     * Most of the features we're emulating requires intercepting instruction
-     * and doing it the slow way, so there is no need to warn when they aren't
-     * present in the host CPU.  Thus we use IGN instead of EMU on these.
-     *
-     * Trailing comments:
-     *      "EMU"  - Possible to emulate, could be lots of work and very slow.
-     *      "EMU?" - Can this be emulated?
-     */
-    /* CPUID(1).ecx */
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE3);    // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_PCLMUL);  // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_DTES64);  // -> EMU?
-    CPUID_GST_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_MONITOR);
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CPLDS);   // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_VMX);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SMX);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_EST);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_TM2);     // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSSE3);   // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CNTXID);  // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(11) /*reserved*/ );
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_FMA);     // -> EMU? what's this?
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CX16);    // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_TPRUPDATE);//-> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_PDCM);    // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(16) /*reserved*/);
-    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(17) /*reserved*/);
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_DCA);     // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_1);  // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_2);  // -> EMU
-    CPUID_GST_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_X2APIC);
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_MOVBE);   // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_POPCNT);  // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(24) /*reserved*/);
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AES);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_XSAVE);   // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_OSXSAVE); // -> EMU
-    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AVX);     // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(29) /*reserved*/);
-    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(30) /*reserved*/);
-    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(31) /*reserved*/);
-
-    /* CPUID(1).edx */
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FPU);
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_VME);
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_DE);      // -> EMU?
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE);
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_TSC);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MSR);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_PAE);
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCE);
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CX8);     // -> EMU?
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_APIC);
-    CPUID_GST_FEATURE_RET(Std, edx, RT_BIT_32(10) /*reserved*/);
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_SEP);
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MTRR);
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PGE);
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCA);
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CMOV);    // -> EMU
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PAT);
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE36);
-    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSN);
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CLFSH);   // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, RT_BIT_32(20) /*reserved*/);
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_DS);      // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_ACPI);    // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MMX);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FXSR);    // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE);     // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE2);    // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SS);      // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_HTT);     // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_TM);      // -> EMU?
-    CPUID_GST_FEATURE_RET(Std, edx, RT_BIT_32(30) /*JMPE/IA64*/);   // -> EMU
-    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_PBE);     // -> EMU?
-
-    /* CPUID(0x80000000). */
-    if (    aGuestCpuIdExt[0].eax >= UINT32_C(0x80000001)
-        &&  aGuestCpuIdExt[0].eax <  UINT32_C(0x8000007f))
-    {
-        /** @todo deal with no 0x80000001 on the host. */
-        bool const fHostAmd  = ASMIsAmdCpuEx(aHostRawStd[0].ebx, aHostRawStd[0].ecx, aHostRawStd[0].edx);
-        bool const fGuestAmd = ASMIsAmdCpuEx(aGuestCpuIdExt[0].ebx, aGuestCpuIdExt[0].ecx, aGuestCpuIdExt[0].edx);
-
-        /* CPUID(0x80000001).ecx */
-        CPUID_GST_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF);   // -> EMU
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CMPL);    // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SVM);     // -> EMU
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_EXT_APIC);// ???
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CR8L);    // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_ABM);     // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE4A);   // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_MISALNSSE);//-> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF);// -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_OSVW);    // -> EMU?
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_IBS);     // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE5);    // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SKINIT);  // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_WDT);     // -> EMU
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(14));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(15));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(16));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(17));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(18));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(19));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(20));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(21));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(22));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(23));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(24));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(25));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(26));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(27));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(28));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(29));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(30));
-        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(31));
-
-        /* CPUID(0x80000001).edx */
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_FPU,   X86_CPUID_FEATURE_EDX_FPU);     // -> EMU
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_VME,   X86_CPUID_FEATURE_EDX_VME);     // -> EMU
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_DE,    X86_CPUID_FEATURE_EDX_DE);      // -> EMU
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PSE,   X86_CPUID_FEATURE_EDX_PSE);
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_TSC,   X86_CPUID_FEATURE_EDX_TSC);     // -> EMU
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_MSR,   X86_CPUID_FEATURE_EDX_MSR);     // -> EMU
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_PAE,   X86_CPUID_FEATURE_EDX_PAE);
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_MCE,   X86_CPUID_FEATURE_EDX_MCE);
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_CX8,   X86_CPUID_FEATURE_EDX_CX8);     // -> EMU?
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_APIC,  X86_CPUID_FEATURE_EDX_APIC);
-        CPUID_GST_AMD_FEATURE_WRN(Ext, edx, RT_BIT_32(10) /*reserved*/);
-        CPUID_GST_FEATURE_IGN(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_SEP);                                  // Intel: long mode only.
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_MTRR,  X86_CPUID_FEATURE_EDX_MTRR);
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PGE,   X86_CPUID_FEATURE_EDX_PGE);
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_MCA,   X86_CPUID_FEATURE_EDX_MCA);
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_CMOV,  X86_CPUID_FEATURE_EDX_CMOV);    // -> EMU
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PAT,   X86_CPUID_FEATURE_EDX_PAT);
-        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PSE36, X86_CPUID_FEATURE_EDX_PSE36);
-        CPUID_GST_AMD_FEATURE_WRN(Ext, edx, RT_BIT_32(18) /*reserved*/);
-        CPUID_GST_AMD_FEATURE_WRN(Ext, edx, RT_BIT_32(19) /*reserved*/);
-        CPUID_GST_FEATURE_RET(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_NX);
-        CPUID_GST_FEATURE_WRN(    Ext, edx, RT_BIT_32(21) /*reserved*/);
-        CPUID_GST_FEATURE_RET(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_AXMMX);
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_MMX,   X86_CPUID_FEATURE_EDX_MMX);     // -> EMU
-        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_FXSR,  X86_CPUID_FEATURE_EDX_FXSR);    // -> EMU
-        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FFXSR);
-        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAGE1GB);
-        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_RDTSCP);
-        CPUID_GST_FEATURE_IGN(    Ext, edx, RT_BIT_32(28) /*reserved*/);
-        CPUID_GST_FEATURE_RET(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_LONG_MODE);
-        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
-        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW);
-    }
-
-    /*
-     * We're good, commit the CPU ID leaves.
-     */
-    memcpy(&pVM->cpum.s.aGuestCpuIdStd[0],     &aGuestCpuIdStd[0],     sizeof(aGuestCpuIdStd));
-    memcpy(&pVM->cpum.s.aGuestCpuIdExt[0],     &aGuestCpuIdExt[0],     sizeof(aGuestCpuIdExt));
-    memcpy(&pVM->cpum.s.aGuestCpuIdCentaur[0], &aGuestCpuIdCentaur[0], sizeof(aGuestCpuIdCentaur));
-    pVM->cpum.s.GuestCpuIdDef = GuestCpuIdDef;
-
-#undef CPUID_CHECK_RET
-#undef CPUID_CHECK_WRN
-#undef CPUID_CHECK2_RET
-#undef CPUID_CHECK2_WRN
-#undef CPUID_RAW_FEATURE_RET
-#undef CPUID_RAW_FEATURE_WRN
-#undef CPUID_RAW_FEATURE_IGN
-#undef CPUID_GST_FEATURE_RET
-#undef CPUID_GST_FEATURE_WRN
-#undef CPUID_GST_FEATURE_EMU
-#undef CPUID_GST_FEATURE_IGN
-#undef CPUID_GST_FEATURE2_RET
-#undef CPUID_GST_FEATURE2_WRN
-#undef CPUID_GST_FEATURE2_EMU
-#undef CPUID_GST_FEATURE2_IGN
-#undef CPUID_GST_AMD_FEATURE_RET
-#undef CPUID_GST_AMD_FEATURE_WRN
-#undef CPUID_GST_AMD_FEATURE_EMU
-#undef CPUID_GST_AMD_FEATURE_IGN
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Pass 0 live exec callback.
- *
- * @returns VINF_SSM_DONT_CALL_AGAIN.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- * @param   uPass               The pass (0).
- */
-static DECLCALLBACK(int) cpumR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    AssertReturn(uPass == 0, VERR_INTERNAL_ERROR_4);
-    cpumR3SaveCpuId(pVM, pSSM);
-    return VINF_SSM_DONT_CALL_AGAIN;
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) cpumR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Save.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        SSMR3PutMem(pSSM, &pVCpu->cpum.s.Hyper, sizeof(pVCpu->cpum.s.Hyper));
-    }
-
-    SSMR3PutU32(pSSM, pVM->cCpus);
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        SSMR3PutMem(pSSM, &pVCpu->cpum.s.Guest, sizeof(pVCpu->cpum.s.Guest));
-        SSMR3PutU32(pSSM, pVCpu->cpum.s.fUseFlags);
-        SSMR3PutU32(pSSM, pVCpu->cpum.s.fChanged);
-        SSMR3PutMem(pSSM, &pVCpu->cpum.s.GuestMsr, sizeof(pVCpu->cpum.s.GuestMsr));
-    }
-
-    cpumR3SaveCpuId(pVM, pSSM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Load a version 1.6 CPUMCTX structure.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pCpumctx16      Version 1.6 CPUMCTX
- */
-static void cpumR3LoadCPUM1_6(PVM pVM, CPUMCTX_VER1_6 *pCpumctx16)
-{
-#define CPUMCTX16_LOADREG(RegName) \
-        pVM->aCpus[0].cpum.s.Guest.RegName = pCpumctx16->RegName;
-
-#define CPUMCTX16_LOADDRXREG(RegName) \
-        pVM->aCpus[0].cpum.s.Guest.dr[RegName] = pCpumctx16->dr##RegName;
-
-#define CPUMCTX16_LOADHIDREG(RegName) \
-        pVM->aCpus[0].cpum.s.Guest.RegName##Hid.u64Base  = pCpumctx16->RegName##Hid.u32Base; \
-        pVM->aCpus[0].cpum.s.Guest.RegName##Hid.u32Limit = pCpumctx16->RegName##Hid.u32Limit; \
-        pVM->aCpus[0].cpum.s.Guest.RegName##Hid.Attr     = pCpumctx16->RegName##Hid.Attr;
-
-#define CPUMCTX16_LOADSEGREG(RegName) \
-        pVM->aCpus[0].cpum.s.Guest.RegName = pCpumctx16->RegName; \
-        CPUMCTX16_LOADHIDREG(RegName);
-
-    pVM->aCpus[0].cpum.s.Guest.fpu = pCpumctx16->fpu;
-
-    CPUMCTX16_LOADREG(rax);
-    CPUMCTX16_LOADREG(rbx);
-    CPUMCTX16_LOADREG(rcx);
-    CPUMCTX16_LOADREG(rdx);
-    CPUMCTX16_LOADREG(rdi);
-    CPUMCTX16_LOADREG(rsi);
-    CPUMCTX16_LOADREG(rbp);
-    CPUMCTX16_LOADREG(esp);
-    CPUMCTX16_LOADREG(rip);
-    CPUMCTX16_LOADREG(rflags);
-
-    CPUMCTX16_LOADSEGREG(cs);
-    CPUMCTX16_LOADSEGREG(ds);
-    CPUMCTX16_LOADSEGREG(es);
-    CPUMCTX16_LOADSEGREG(fs);
-    CPUMCTX16_LOADSEGREG(gs);
-    CPUMCTX16_LOADSEGREG(ss);
-
-    CPUMCTX16_LOADREG(r8);
-    CPUMCTX16_LOADREG(r9);
-    CPUMCTX16_LOADREG(r10);
-    CPUMCTX16_LOADREG(r11);
-    CPUMCTX16_LOADREG(r12);
-    CPUMCTX16_LOADREG(r13);
-    CPUMCTX16_LOADREG(r14);
-    CPUMCTX16_LOADREG(r15);
-
-    CPUMCTX16_LOADREG(cr0);
-    CPUMCTX16_LOADREG(cr2);
-    CPUMCTX16_LOADREG(cr3);
-    CPUMCTX16_LOADREG(cr4);
-
-    CPUMCTX16_LOADDRXREG(0);
-    CPUMCTX16_LOADDRXREG(1);
-    CPUMCTX16_LOADDRXREG(2);
-    CPUMCTX16_LOADDRXREG(3);
-    CPUMCTX16_LOADDRXREG(4);
-    CPUMCTX16_LOADDRXREG(5);
-    CPUMCTX16_LOADDRXREG(6);
-    CPUMCTX16_LOADDRXREG(7);
-
-    pVM->aCpus[0].cpum.s.Guest.gdtr.cbGdt   = pCpumctx16->gdtr.cbGdt;
-    pVM->aCpus[0].cpum.s.Guest.gdtr.pGdt    = pCpumctx16->gdtr.pGdt;
-    pVM->aCpus[0].cpum.s.Guest.idtr.cbIdt   = pCpumctx16->idtr.cbIdt;
-    pVM->aCpus[0].cpum.s.Guest.idtr.pIdt    = pCpumctx16->idtr.pIdt;
-
-    CPUMCTX16_LOADREG(ldtr);
-    CPUMCTX16_LOADREG(tr);
-
-    pVM->aCpus[0].cpum.s.Guest.SysEnter     = pCpumctx16->SysEnter;
-
-    CPUMCTX16_LOADREG(msrEFER);
-    CPUMCTX16_LOADREG(msrSTAR);
-    CPUMCTX16_LOADREG(msrPAT);
-    CPUMCTX16_LOADREG(msrLSTAR);
-    CPUMCTX16_LOADREG(msrCSTAR);
-    CPUMCTX16_LOADREG(msrSFMASK);
-    CPUMCTX16_LOADREG(msrKERNELGSBASE);
-
-    CPUMCTX16_LOADHIDREG(ldtr);
-    CPUMCTX16_LOADHIDREG(tr);
-
-#undef CPUMCTX16_LOADSEGREG
-#undef CPUMCTX16_LOADHIDREG
-#undef CPUMCTX16_LOADDRXREG
-#undef CPUMCTX16_LOADREG
-}
-
-
-/**
- * @copydoc FNSSMINTLOADPREP
- */
-static DECLCALLBACK(int) cpumR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
-{
-    pVM->cpum.s.fPendingRestore = true;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * @copydoc FNSSMINTLOADEXEC
- */
-static DECLCALLBACK(int) cpumR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    /*
-     * Validate version.
-     */
-    if (    uVersion != CPUM_SAVED_STATE_VERSION
-        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER3_2
-        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER3_0
-        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR
-        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER2_0
-        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER1_6)
-    {
-        AssertMsgFailed(("cpumR3LoadExec: Invalid version uVersion=%d!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    if (uPass == SSM_PASS_FINAL)
-    {
-        /*
-         * Set the size of RTGCPTR for SSMR3GetGCPtr. (Only necessary for
-         * really old SSM file versions.)
-         */
-        if (uVersion == CPUM_SAVED_STATE_VERSION_VER1_6)
-            SSMR3HandleSetGCPtrSize(pSSM, sizeof(RTGCPTR32));
-        else if (uVersion <= CPUM_SAVED_STATE_VERSION_VER3_0)
-            SSMR3HandleSetGCPtrSize(pSSM, HC_ARCH_BITS == 32 ? sizeof(RTGCPTR32) : sizeof(RTGCPTR));
-
-        /*
-         * Restore.
-         */
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            PVMCPU   pVCpu = &pVM->aCpus[i];
-            uint32_t uCR3  = pVCpu->cpum.s.Hyper.cr3;
-            uint32_t uESP  = pVCpu->cpum.s.Hyper.esp; /* see VMMR3Relocate(). */
-
-            SSMR3GetMem(pSSM, &pVCpu->cpum.s.Hyper, sizeof(pVCpu->cpum.s.Hyper));
-            pVCpu->cpum.s.Hyper.cr3 = uCR3;
-            pVCpu->cpum.s.Hyper.esp = uESP;
-        }
-
-        if (uVersion == CPUM_SAVED_STATE_VERSION_VER1_6)
-        {
-            CPUMCTX_VER1_6 cpumctx16;
-            memset(&pVM->aCpus[0].cpum.s.Guest, 0, sizeof(pVM->aCpus[0].cpum.s.Guest));
-            SSMR3GetMem(pSSM, &cpumctx16, sizeof(cpumctx16));
-
-            /* Save the old cpumctx state into the new one. */
-            cpumR3LoadCPUM1_6(pVM, &cpumctx16);
-
-            SSMR3GetU32(pSSM, &pVM->aCpus[0].cpum.s.fUseFlags);
-            SSMR3GetU32(pSSM, &pVM->aCpus[0].cpum.s.fChanged);
-        }
-        else
-        {
-            if (uVersion >= CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR)
-            {
-                uint32_t cCpus;
-                int rc = SSMR3GetU32(pSSM, &cCpus); AssertRCReturn(rc, rc);
-                AssertLogRelMsgReturn(cCpus == pVM->cCpus, ("Mismatching CPU counts: saved: %u; configured: %u \n", cCpus, pVM->cCpus),
-                                      VERR_SSM_UNEXPECTED_DATA);
-            }
-            AssertLogRelMsgReturn(   uVersion != CPUM_SAVED_STATE_VERSION_VER2_0
-                                  || pVM->cCpus == 1,
-                                  ("cCpus=%u\n", pVM->cCpus),
-                                  VERR_SSM_UNEXPECTED_DATA);
-
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-            {
-                SSMR3GetMem(pSSM, &pVM->aCpus[i].cpum.s.Guest, sizeof(pVM->aCpus[i].cpum.s.Guest));
-                SSMR3GetU32(pSSM, &pVM->aCpus[i].cpum.s.fUseFlags);
-                SSMR3GetU32(pSSM, &pVM->aCpus[i].cpum.s.fChanged);
-                if (uVersion >= CPUM_SAVED_STATE_VERSION_VER3_0)
-                    SSMR3GetMem(pSSM, &pVM->aCpus[i].cpum.s.GuestMsr, sizeof(pVM->aCpus[i].cpum.s.GuestMsr));
-            }
-        }
-
-        /* Older states does not set CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID for
-           raw-mode guest, so we have to do it ourselves. */
-        if (   uVersion <= CPUM_SAVED_STATE_VERSION_VER3_2
-            && !HWACCMIsEnabled(pVM))
-            for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
-                pVM->aCpus[iCpu].cpum.s.fChanged |= CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
-    }
-
-    pVM->cpum.s.fPendingRestore = false;
-
-    /*
-     * Guest CPUIDs.
-     */
-    if (uVersion > CPUM_SAVED_STATE_VERSION_VER3_0)
-        return cpumR3LoadCpuId(pVM, pSSM, uVersion);
-
-    /** @todo Merge the code below into cpumR3LoadCpuId when we've found out what is
-     *        actually required. */
-
-    /*
-     * Restore the CPUID leaves.
-     *
-     * Note that we support restoring less than the current amount of standard
-     * leaves because we've been allowed more is newer version of VBox.
-     */
-    uint32_t cElements;
-    int rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
-    if (cElements > RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdStd[0], cElements*sizeof(pVM->cpum.s.aGuestCpuIdStd[0]));
-
-    rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
-    if (cElements != RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdExt[0], sizeof(pVM->cpum.s.aGuestCpuIdExt));
-
-    rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
-    if (cElements != RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur))
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdCentaur[0], sizeof(pVM->cpum.s.aGuestCpuIdCentaur));
-
-    SSMR3GetMem(pSSM, &pVM->cpum.s.GuestCpuIdDef, sizeof(pVM->cpum.s.GuestCpuIdDef));
-
-    /*
-     * Check that the basic cpuid id information is unchanged.
-     */
-    /** @todo we should check the 64 bits capabilities too! */
-    uint32_t au32CpuId[8] = {0,0,0,0, 0,0,0,0};
-    ASMCpuId(0, &au32CpuId[0], &au32CpuId[1], &au32CpuId[2], &au32CpuId[3]);
-    ASMCpuId(1, &au32CpuId[4], &au32CpuId[5], &au32CpuId[6], &au32CpuId[7]);
-    uint32_t au32CpuIdSaved[8];
-    rc = SSMR3GetMem(pSSM, &au32CpuIdSaved[0], sizeof(au32CpuIdSaved));
-    if (RT_SUCCESS(rc))
-    {
-        /* Ignore CPU stepping. */
-        au32CpuId[4]      &=  0xfffffff0;
-        au32CpuIdSaved[4] &=  0xfffffff0;
-
-        /* Ignore APIC ID (AMD specs). */
-        au32CpuId[5]      &= ~0xff000000;
-        au32CpuIdSaved[5] &= ~0xff000000;
-
-        /* Ignore the number of Logical CPUs (AMD specs). */
-        au32CpuId[5]      &= ~0x00ff0000;
-        au32CpuIdSaved[5] &= ~0x00ff0000;
-
-        /* Ignore some advanced capability bits, that we don't expose to the guest. */
-        au32CpuId[6]      &= ~(   X86_CPUID_FEATURE_ECX_DTES64
-                               |  X86_CPUID_FEATURE_ECX_VMX
-                               |  X86_CPUID_FEATURE_ECX_SMX
-                               |  X86_CPUID_FEATURE_ECX_EST
-                               |  X86_CPUID_FEATURE_ECX_TM2
-                               |  X86_CPUID_FEATURE_ECX_CNTXID
-                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
-                               |  X86_CPUID_FEATURE_ECX_PDCM
-                               |  X86_CPUID_FEATURE_ECX_DCA
-                               |  X86_CPUID_FEATURE_ECX_X2APIC
-                              );
-        au32CpuIdSaved[6] &= ~(   X86_CPUID_FEATURE_ECX_DTES64
-                               |  X86_CPUID_FEATURE_ECX_VMX
-                               |  X86_CPUID_FEATURE_ECX_SMX
-                               |  X86_CPUID_FEATURE_ECX_EST
-                               |  X86_CPUID_FEATURE_ECX_TM2
-                               |  X86_CPUID_FEATURE_ECX_CNTXID
-                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
-                               |  X86_CPUID_FEATURE_ECX_PDCM
-                               |  X86_CPUID_FEATURE_ECX_DCA
-                               |  X86_CPUID_FEATURE_ECX_X2APIC
-                              );
-
-        /* Make sure we don't forget to update the masks when enabling
-         * features in the future.
-         */
-        AssertRelease(!(pVM->cpum.s.aGuestCpuIdStd[1].ecx &
-                              (   X86_CPUID_FEATURE_ECX_DTES64
-                               |  X86_CPUID_FEATURE_ECX_VMX
-                               |  X86_CPUID_FEATURE_ECX_SMX
-                               |  X86_CPUID_FEATURE_ECX_EST
-                               |  X86_CPUID_FEATURE_ECX_TM2
-                               |  X86_CPUID_FEATURE_ECX_CNTXID
-                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
-                               |  X86_CPUID_FEATURE_ECX_PDCM
-                               |  X86_CPUID_FEATURE_ECX_DCA
-                               |  X86_CPUID_FEATURE_ECX_X2APIC
-                              )));
-        /* do the compare */
-        if (memcmp(au32CpuIdSaved, au32CpuId, sizeof(au32CpuIdSaved)))
-        {
-            if (SSMR3HandleGetAfter(pSSM) == SSMAFTER_DEBUG_IT)
-                LogRel(("cpumR3LoadExec: CpuId mismatch! (ignored due to SSMAFTER_DEBUG_IT)\n"
-                        "Saved=%.*Rhxs\n"
-                        "Real =%.*Rhxs\n",
-                        sizeof(au32CpuIdSaved), au32CpuIdSaved,
-                        sizeof(au32CpuId), au32CpuId));
-            else
-            {
-                LogRel(("cpumR3LoadExec: CpuId mismatch!\n"
-                        "Saved=%.*Rhxs\n"
-                        "Real =%.*Rhxs\n",
-                        sizeof(au32CpuIdSaved), au32CpuIdSaved,
-                        sizeof(au32CpuId), au32CpuId));
-                rc = VERR_SSM_LOAD_CPUID_MISMATCH;
-            }
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * @copydoc FNSSMINTLOADPREP
- */
-static DECLCALLBACK(int) cpumR3LoadDone(PVM pVM, PSSMHANDLE pSSM)
-{
-    if (RT_FAILURE(SSMR3HandleGetStatus(pSSM)))
-        return VINF_SUCCESS;
-
-    /* just check this since we can. */ /** @todo Add a SSM unit flag for indicating that it's mandatory during a restore.  */
-    if (pVM->cpum.s.fPendingRestore)
-    {
-        LogRel(("CPUM: Missing state!\n"));
-        return VERR_INTERNAL_ERROR_2;
-    }
-
-    /* Notify PGM of the NXE states in case they've changed. */
-    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
-        PGMNotifyNxeChanged(&pVM->aCpus[iCpu], !!(pVM->aCpus[iCpu].cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Checks if the CPUM state restore is still pending.
- *
- * @returns true / false.
- * @param   pVM                 The VM handle.
- */
-VMMDECL(bool) CPUMR3IsStateRestorePending(PVM pVM)
-{
-    return pVM->cpum.s.fPendingRestore;
-}
-
-
-/**
- * Formats the EFLAGS value into mnemonics.
- *
- * @param   pszEFlags   Where to write the mnemonics. (Assumes sufficient buffer space.)
- * @param   efl         The EFLAGS value.
- */
-static void cpumR3InfoFormatFlags(char *pszEFlags, uint32_t efl)
-{
-    /*
-     * Format the flags.
-     */
-    static const struct
-    {
-        const char *pszSet; const char *pszClear; uint32_t fFlag;
-    }   s_aFlags[] =
-    {
-        { "vip",NULL, X86_EFL_VIP },
-        { "vif",NULL, X86_EFL_VIF },
-        { "ac", NULL, X86_EFL_AC },
-        { "vm", NULL, X86_EFL_VM },
-        { "rf", NULL, X86_EFL_RF },
-        { "nt", NULL, X86_EFL_NT },
-        { "ov", "nv", X86_EFL_OF },
-        { "dn", "up", X86_EFL_DF },
-        { "ei", "di", X86_EFL_IF },
-        { "tf", NULL, X86_EFL_TF },
-        { "nt", "pl", X86_EFL_SF },
-        { "nz", "zr", X86_EFL_ZF },
-        { "ac", "na", X86_EFL_AF },
-        { "po", "pe", X86_EFL_PF },
-        { "cy", "nc", X86_EFL_CF },
-    };
-    char *psz = pszEFlags;
-    for (unsigned i = 0; i < RT_ELEMENTS(s_aFlags); i++)
-    {
-        const char *pszAdd = s_aFlags[i].fFlag & efl ? s_aFlags[i].pszSet : s_aFlags[i].pszClear;
-        if (pszAdd)
-        {
-            strcpy(psz, pszAdd);
-            psz += strlen(pszAdd);
-            *psz++ = ' ';
-        }
-    }
-    psz[-1] = '\0';
-}
-
-
-/**
- * Formats a full register dump.
- *
- * @param   pVM         VM Handle.
- * @param   pCtx        The context to format.
- * @param   pCtxCore    The context core to format.
- * @param   pHlp        Output functions.
- * @param   enmType     The dump type.
- * @param   pszPrefix   Register name prefix.
- */
-static void cpumR3InfoOne(PVM pVM, PCPUMCTX pCtx, PCCPUMCTXCORE pCtxCore, PCDBGFINFOHLP pHlp, CPUMDUMPTYPE enmType, const char *pszPrefix)
-{
-    /*
-     * Format the EFLAGS.
-     */
-    uint32_t efl = pCtxCore->eflags.u32;
-    char szEFlags[80];
-    cpumR3InfoFormatFlags(&szEFlags[0], efl);
-
-    /*
-     * Format the registers.
-     */
-    switch (enmType)
-    {
-        case CPUMDUMPTYPE_TERSE:
-            if (CPUMIsGuestIn64BitCodeEx(pCtx))
-                pHlp->pfnPrintf(pHlp,
-                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
-                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
-                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
-                    "%sr14=%016RX64 %sr15=%016RX64\n"
-                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
-                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x                %seflags=%08x\n",
-                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
-                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
-                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
-                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
-                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
-                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, efl);
-            else
-                pHlp->pfnPrintf(pHlp,
-                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
-                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
-                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x                %seflags=%08x\n",
-                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
-                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
-                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
-                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, efl);
-            break;
-
-        case CPUMDUMPTYPE_DEFAULT:
-            if (CPUMIsGuestIn64BitCodeEx(pCtx))
-                pHlp->pfnPrintf(pHlp,
-                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
-                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
-                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
-                    "%sr14=%016RX64 %sr15=%016RX64\n"
-                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
-                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x %str=%04x      %seflags=%08x\n"
-                    "%scr0=%08RX64 %scr2=%08RX64 %scr3=%08RX64 %scr4=%08RX64 %sgdtr=%016RX64:%04x %sldtr=%04x\n"
-                    ,
-                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
-                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
-                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
-                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
-                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
-                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, (RTSEL)pCtx->tr, pszPrefix, efl,
-                    pszPrefix, pCtx->cr0, pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3, pszPrefix, pCtx->cr4,
-                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, (RTSEL)pCtx->ldtr);
-            else
-                pHlp->pfnPrintf(pHlp,
-                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
-                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
-                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x %str=%04x      %seflags=%08x\n"
-                    "%scr0=%08RX64 %scr2=%08RX64 %scr3=%08RX64 %scr4=%08RX64 %sgdtr=%08RX64:%04x %sldtr=%04x\n"
-                    ,
-                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
-                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
-                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
-                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, (RTSEL)pCtx->tr, pszPrefix, efl,
-                    pszPrefix, pCtx->cr0, pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3, pszPrefix, pCtx->cr4,
-                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, (RTSEL)pCtx->ldtr);
-            break;
-
-        case CPUMDUMPTYPE_VERBOSE:
-            if (CPUMIsGuestIn64BitCodeEx(pCtx))
-                pHlp->pfnPrintf(pHlp,
-                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
-                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
-                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
-                    "%sr14=%016RX64 %sr15=%016RX64\n"
-                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
-                    "%scs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
-                    "%sds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
-                    "%ses={%04x base=%016RX64 limit=%08x flags=%08x}\n"
-                    "%sfs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
-                    "%sgs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
-                    "%sss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
-                    "%scr0=%016RX64 %scr2=%016RX64 %scr3=%016RX64 %scr4=%016RX64\n"
-                    "%sdr0=%016RX64 %sdr1=%016RX64 %sdr2=%016RX64 %sdr3=%016RX64\n"
-                    "%sdr4=%016RX64 %sdr5=%016RX64 %sdr6=%016RX64 %sdr7=%016RX64\n"
-                    "%sgdtr=%016RX64:%04x  %sidtr=%016RX64:%04x  %seflags=%08x\n"
-                    "%sldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
-                    "%str  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
-                    "%sSysEnter={cs=%04llx eip=%016RX64 esp=%016RX64}\n"
-                    ,
-                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
-                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
-                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
-                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
-                    pszPrefix, (RTSEL)pCtxCore->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u,
-                    pszPrefix, (RTSEL)pCtxCore->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u,
-                    pszPrefix, (RTSEL)pCtxCore->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u,
-                    pszPrefix, (RTSEL)pCtxCore->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u,
-                    pszPrefix, (RTSEL)pCtxCore->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u,
-                    pszPrefix, (RTSEL)pCtxCore->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u,
-                    pszPrefix, pCtx->cr0,  pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3,  pszPrefix, pCtx->cr4,
-                    pszPrefix, pCtx->dr[0],  pszPrefix, pCtx->dr[1], pszPrefix, pCtx->dr[2],  pszPrefix, pCtx->dr[3],
-                    pszPrefix, pCtx->dr[4],  pszPrefix, pCtx->dr[5], pszPrefix, pCtx->dr[6],  pszPrefix, pCtx->dr[7],
-                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, pszPrefix, efl,
-                    pszPrefix, (RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
-                    pszPrefix, (RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
-                    pszPrefix, pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
-            else
-                pHlp->pfnPrintf(pHlp,
-                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
-                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
-                    "%scs={%04x base=%016RX64 limit=%08x flags=%08x} %sdr0=%08RX64 %sdr1=%08RX64\n"
-                    "%sds={%04x base=%016RX64 limit=%08x flags=%08x} %sdr2=%08RX64 %sdr3=%08RX64\n"
-                    "%ses={%04x base=%016RX64 limit=%08x flags=%08x} %sdr4=%08RX64 %sdr5=%08RX64\n"
-                    "%sfs={%04x base=%016RX64 limit=%08x flags=%08x} %sdr6=%08RX64 %sdr7=%08RX64\n"
-                    "%sgs={%04x base=%016RX64 limit=%08x flags=%08x} %scr0=%08RX64 %scr2=%08RX64\n"
-                    "%sss={%04x base=%016RX64 limit=%08x flags=%08x} %scr3=%08RX64 %scr4=%08RX64\n"
-                    "%sgdtr=%016RX64:%04x  %sidtr=%016RX64:%04x  %seflags=%08x\n"
-                    "%sldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
-                    "%str  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
-                    "%sSysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
-                    ,
-                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
-                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
-                    pszPrefix, (RTSEL)pCtxCore->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u, pszPrefix, pCtx->dr[0],  pszPrefix, pCtx->dr[1],
-                    pszPrefix, (RTSEL)pCtxCore->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u, pszPrefix, pCtx->dr[2],  pszPrefix, pCtx->dr[3],
-                    pszPrefix, (RTSEL)pCtxCore->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u, pszPrefix, pCtx->dr[4],  pszPrefix, pCtx->dr[5],
-                    pszPrefix, (RTSEL)pCtxCore->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u, pszPrefix, pCtx->dr[6],  pszPrefix, pCtx->dr[7],
-                    pszPrefix, (RTSEL)pCtxCore->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u, pszPrefix, pCtx->cr0,  pszPrefix, pCtx->cr2,
-                    pszPrefix, (RTSEL)pCtxCore->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u, pszPrefix, pCtx->cr3,  pszPrefix, pCtx->cr4,
-                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, pszPrefix, efl,
-                    pszPrefix, (RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
-                    pszPrefix, (RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
-                    pszPrefix, pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
-
-            pHlp->pfnPrintf(pHlp,
-                "%sFCW=%04x %sFSW=%04x %sFTW=%04x %sFOP=%04x %sMXCSR=%08x %sMXCSR_MASK=%08x\n"
-                "%sFPUIP=%08x %sCS=%04x %sRsvrd1=%04x  %sFPUDP=%08x %sDS=%04x %sRsvrd2=%04x\n"
-                ,
-                pszPrefix, pCtx->fpu.FCW,   pszPrefix, pCtx->fpu.FSW, pszPrefix, pCtx->fpu.FTW, pszPrefix, pCtx->fpu.FOP,
-                pszPrefix, pCtx->fpu.MXCSR, pszPrefix, pCtx->fpu.MXCSR_MASK,
-                pszPrefix, pCtx->fpu.FPUIP, pszPrefix, pCtx->fpu.CS,  pszPrefix, pCtx->fpu.Rsvrd1,
-                pszPrefix, pCtx->fpu.FPUDP, pszPrefix, pCtx->fpu.DS,  pszPrefix, pCtx->fpu.Rsrvd2
-                );
-            unsigned iShift = (pCtx->fpu.FSW >> 11) & 7;
-            for (unsigned iST = 0; iST < RT_ELEMENTS(pCtx->fpu.aRegs); iST++)
-            {
-                unsigned iFPR        = (iST + iShift) % RT_ELEMENTS(pCtx->fpu.aRegs);
-                unsigned uTag        = pCtx->fpu.FTW & (1 << iFPR) ? 1 : 0;
-                char     chSign      = pCtx->fpu.aRegs[0].au16[4] & 0x8000 ? '-' : '+';
-                unsigned iInteger    = (unsigned)(pCtx->fpu.aRegs[0].au64[0] >> 63);
-                uint64_t u64Fraction = pCtx->fpu.aRegs[0].au64[0] & UINT64_C(0x7fffffffffffffff);
-                unsigned uExponent   = pCtx->fpu.aRegs[0].au16[4] & 0x7fff;
-                /** @todo This isn't entirenly correct and needs more work! */
-                pHlp->pfnPrintf(pHlp,
-                                "%sST(%u)=%sFPR%u={%04RX16'%08RX32'%08RX32} t%d %c%u.%022llu ^ %u",
-                                pszPrefix, iST, pszPrefix, iFPR,
-                                pCtx->fpu.aRegs[0].au16[4], pCtx->fpu.aRegs[0].au32[1], pCtx->fpu.aRegs[0].au32[0],
-                                uTag, chSign, iInteger, u64Fraction, uExponent);
-                if (pCtx->fpu.aRegs[0].au16[5] || pCtx->fpu.aRegs[0].au16[6] || pCtx->fpu.aRegs[0].au16[7])
-                    pHlp->pfnPrintf(pHlp, " res={%04RX16,%04RX16,%04RX16}\n",
-                                    pCtx->fpu.aRegs[0].au16[5], pCtx->fpu.aRegs[0].au16[6], pCtx->fpu.aRegs[0].au16[7]);
-                else
-                    pHlp->pfnPrintf(pHlp, "\n");
-            }
-            for (unsigned iXMM = 0; iXMM < RT_ELEMENTS(pCtx->fpu.aXMM); iXMM++)
-                pHlp->pfnPrintf(pHlp,
-                                iXMM & 1
-                                ? "%sXMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32\n"
-                                : "%sXMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32  ",
-                                pszPrefix, iXMM, iXMM < 10 ? " " : "",
-                                pCtx->fpu.aXMM[iXMM].au32[3],
-                                pCtx->fpu.aXMM[iXMM].au32[2],
-                                pCtx->fpu.aXMM[iXMM].au32[1],
-                                pCtx->fpu.aXMM[iXMM].au32[0]);
-            for (unsigned i = 0; i < RT_ELEMENTS(pCtx->fpu.au32RsrvdRest); i++)
-                if (pCtx->fpu.au32RsrvdRest[i])
-                    pHlp->pfnPrintf(pHlp, "%sRsrvdRest[i]=%RX32 (offset=%#x)\n",
-                                    pszPrefix, i, pCtx->fpu.au32RsrvdRest[i], RT_OFFSETOF(X86FXSTATE, au32RsrvdRest[i]) );
-
-            pHlp->pfnPrintf(pHlp,
-                "%sEFER         =%016RX64\n"
-                "%sPAT          =%016RX64\n"
-                "%sSTAR         =%016RX64\n"
-                "%sCSTAR        =%016RX64\n"
-                "%sLSTAR        =%016RX64\n"
-                "%sSFMASK       =%016RX64\n"
-                "%sKERNELGSBASE =%016RX64\n",
-                pszPrefix, pCtx->msrEFER,
-                pszPrefix, pCtx->msrPAT,
-                pszPrefix, pCtx->msrSTAR,
-                pszPrefix, pCtx->msrCSTAR,
-                pszPrefix, pCtx->msrLSTAR,
-                pszPrefix, pCtx->msrSFMASK,
-                pszPrefix, pCtx->msrKERNELGSBASE);
-            break;
-    }
-}
-
-
-/**
- * Display all cpu states and any other cpum info.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helper functions.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) cpumR3InfoAll(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    cpumR3InfoGuest(pVM, pHlp, pszArgs);
-    cpumR3InfoGuestInstr(pVM, pHlp, pszArgs);
-    cpumR3InfoHyper(pVM, pHlp, pszArgs);
-    cpumR3InfoHost(pVM, pHlp, pszArgs);
-}
-
-
-/**
- * Parses the info argument.
- *
- * The argument starts with 'verbose', 'terse' or 'default' and then
- * continues with the comment string.
- *
- * @param   pszArgs         The pointer to the argument string.
- * @param   penmType        Where to store the dump type request.
- * @param   ppszComment     Where to store the pointer to the comment string.
- */
-static void cpumR3InfoParseArg(const char *pszArgs, CPUMDUMPTYPE *penmType, const char **ppszComment)
-{
-    if (!pszArgs)
-    {
-        *penmType = CPUMDUMPTYPE_DEFAULT;
-        *ppszComment = "";
-    }
-    else
-    {
-        if (!strncmp(pszArgs, "verbose", sizeof("verbose") - 1))
-        {
-            pszArgs += 5;
-            *penmType = CPUMDUMPTYPE_VERBOSE;
-        }
-        else if (!strncmp(pszArgs, "terse", sizeof("terse") - 1))
-        {
-            pszArgs += 5;
-            *penmType = CPUMDUMPTYPE_TERSE;
-        }
-        else if (!strncmp(pszArgs, "default", sizeof("default") - 1))
-        {
-            pszArgs += 7;
-            *penmType = CPUMDUMPTYPE_DEFAULT;
-        }
-        else
-            *penmType = CPUMDUMPTYPE_DEFAULT;
-        *ppszComment = RTStrStripL(pszArgs);
-    }
-}
-
-
-/**
- * Display the guest cpu state.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helper functions.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) cpumR3InfoGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    CPUMDUMPTYPE enmType;
-    const char *pszComment;
-    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
-
-    /* @todo SMP support! */
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    if (!pVCpu)
-        pVCpu = &pVM->aCpus[0];
-
-    pHlp->pfnPrintf(pHlp, "Guest CPUM (VCPU %d) state: %s\n", pVCpu->idCpu, pszComment);
-
-    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
-    cpumR3InfoOne(pVM, pCtx, CPUMCTX2CORE(pCtx), pHlp, enmType, "");
-}
-
-
-/**
- * Display the current guest instruction
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helper functions.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) cpumR3InfoGuestInstr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    char szInstruction[256];
-    /* @todo SMP support! */
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    if (!pVCpu)
-        pVCpu = &pVM->aCpus[0];
-
-    int rc = DBGFR3DisasInstrCurrent(pVCpu, szInstruction, sizeof(szInstruction));
-    if (RT_SUCCESS(rc))
-        pHlp->pfnPrintf(pHlp, "\nCPUM: %s\n\n", szInstruction);
-}
-
-
-/**
- * Display the hypervisor cpu state.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helper functions.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) cpumR3InfoHyper(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    CPUMDUMPTYPE enmType;
-    const char *pszComment;
-    /* @todo SMP */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
-    pHlp->pfnPrintf(pHlp, "Hypervisor CPUM state: %s\n", pszComment);
-    cpumR3InfoOne(pVM, &pVCpu->cpum.s.Hyper, pVCpu->cpum.s.pHyperCoreR3, pHlp, enmType, ".");
-    pHlp->pfnPrintf(pHlp, "CR4OrMask=%#x CR4AndMask=%#x\n", pVM->cpum.s.CR4.OrMask, pVM->cpum.s.CR4.AndMask);
-}
-
-
-/**
- * Display the host cpu state.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helper functions.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) cpumR3InfoHost(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    CPUMDUMPTYPE enmType;
-    const char *pszComment;
-    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
-    pHlp->pfnPrintf(pHlp, "Host CPUM state: %s\n", pszComment);
-
-    /*
-     * Format the EFLAGS.
-     */
-    /* @todo SMP */
-    PCPUMHOSTCTX pCtx = &pVM->aCpus[0].cpum.s.Host;
-#if HC_ARCH_BITS == 32
-    uint32_t efl = pCtx->eflags.u32;
-#else
-    uint64_t efl = pCtx->rflags;
-#endif
-    char szEFlags[80];
-    cpumR3InfoFormatFlags(&szEFlags[0], efl);
-
-    /*
-     * Format the registers.
-     */
-#if HC_ARCH_BITS == 32
-# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
-    if (!(pCtx->efer & MSR_K6_EFER_LMA))
-# endif
-    {
-        pHlp->pfnPrintf(pHlp,
-            "eax=xxxxxxxx ebx=%08x ecx=xxxxxxxx edx=xxxxxxxx esi=%08x edi=%08x\n"
-            "eip=xxxxxxxx esp=%08x ebp=%08x iopl=%d %31s\n"
-            "cs=%04x ds=%04x es=%04x fs=%04x gs=%04x                       eflags=%08x\n"
-            "cr0=%08RX64 cr2=xxxxxxxx cr3=%08RX64 cr4=%08RX64 gdtr=%08x:%04x ldtr=%04x\n"
-            "dr[0]=%08RX64 dr[1]=%08RX64x dr[2]=%08RX64 dr[3]=%08RX64x dr[6]=%08RX64 dr[7]=%08RX64\n"
-            "SysEnter={cs=%04x eip=%08x esp=%08x}\n"
-            ,
-            /*pCtx->eax,*/ pCtx->ebx, /*pCtx->ecx, pCtx->edx,*/ pCtx->esi, pCtx->edi,
-            /*pCtx->eip,*/ pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(efl), szEFlags,
-            (RTSEL)pCtx->cs, (RTSEL)pCtx->ds, (RTSEL)pCtx->es, (RTSEL)pCtx->fs, (RTSEL)pCtx->gs, efl,
-            pCtx->cr0, /*pCtx->cr2,*/ pCtx->cr3, pCtx->cr4,
-            pCtx->dr0, pCtx->dr1, pCtx->dr2, pCtx->dr3, pCtx->dr6, pCtx->dr7,
-            (uint32_t)pCtx->gdtr.uAddr, pCtx->gdtr.cb, (RTSEL)pCtx->ldtr,
-            pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
-    }
-# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
-    else
-# endif
-#endif
-#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    {
-        pHlp->pfnPrintf(pHlp,
-            "rax=xxxxxxxxxxxxxxxx rbx=%016RX64 rcx=xxxxxxxxxxxxxxxx\n"
-            "rdx=xxxxxxxxxxxxxxxx rsi=%016RX64 rdi=%016RX64\n"
-            "rip=xxxxxxxxxxxxxxxx rsp=%016RX64 rbp=%016RX64\n"
-            " r8=xxxxxxxxxxxxxxxx  r9=xxxxxxxxxxxxxxxx r10=%016RX64\n"
-            "r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
-            "r14=%016RX64 r15=%016RX64\n"
-            "iopl=%d  %31s\n"
-            "cs=%04x  ds=%04x  es=%04x  fs=%04x  gs=%04x                   eflags=%08RX64\n"
-            "cr0=%016RX64 cr2=xxxxxxxxxxxxxxxx cr3=%016RX64\n"
-            "cr4=%016RX64 ldtr=%04x tr=%04x\n"
-            "dr[0]=%016RX64 dr[1]=%016RX64 dr[2]=%016RX64\n"
-            "dr[3]=%016RX64 dr[6]=%016RX64 dr[7]=%016RX64\n"
-            "gdtr=%016RX64:%04x  idtr=%016RX64:%04x\n"
-            "SysEnter={cs=%04x eip=%08x esp=%08x}\n"
-            "FSbase=%016RX64 GSbase=%016RX64 efer=%08RX64\n"
-            ,
-            /*pCtx->rax,*/ pCtx->rbx, /*pCtx->rcx,
-            pCtx->rdx,*/ pCtx->rsi, pCtx->rdi,
-            /*pCtx->rip,*/ pCtx->rsp, pCtx->rbp,
-            /*pCtx->r8,  pCtx->r9,*/  pCtx->r10,
-            pCtx->r11, pCtx->r12, pCtx->r13,
-            pCtx->r14, pCtx->r15,
-            X86_EFL_GET_IOPL(efl), szEFlags,
-            (RTSEL)pCtx->cs, (RTSEL)pCtx->ds, (RTSEL)pCtx->es, (RTSEL)pCtx->fs, (RTSEL)pCtx->gs, efl,
-            pCtx->cr0, /*pCtx->cr2,*/ pCtx->cr3,
-            pCtx->cr4, pCtx->ldtr, pCtx->tr,
-            pCtx->dr0, pCtx->dr1, pCtx->dr2,
-            pCtx->dr3, pCtx->dr6, pCtx->dr7,
-            pCtx->gdtr.uAddr, pCtx->gdtr.cb, pCtx->idtr.uAddr, pCtx->idtr.cb,
-            pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp,
-            pCtx->FSbase, pCtx->GSbase, pCtx->efer);
-    }
-#endif
-}
-
-
-/**
- * Get L1 cache / TLS associativity.
- */
-static const char *getCacheAss(unsigned u, char *pszBuf)
-{
-    if (u == 0)
-        return "res0  ";
-    if (u == 1)
-        return "direct";
-    if (u == 255)
-        return "fully";
-    if (u >= 256)
-        return "???";
-
-    RTStrPrintf(pszBuf, 16, "%d way", u);
-    return pszBuf;
-}
-
-
-/**
- * Get L2 cache associativity.
- */
-const char *getL2CacheAss(unsigned u)
-{
-    switch (u)
-    {
-        case 0:  return "off   ";
-        case 1:  return "direct";
-        case 2:  return "2 way ";
-        case 3:  return "res3  ";
-        case 4:  return "4 way ";
-        case 5:  return "res5  ";
-        case 6:  return "8 way ";
-        case 7:  return "res7  ";
-        case 8:  return "16 way";
-        case 9:  return "res9  ";
-        case 10: return "res10 ";
-        case 11: return "res11 ";
-        case 12: return "res12 ";
-        case 13: return "res13 ";
-        case 14: return "res14 ";
-        case 15: return "fully ";
-        default: return "????";
-    }
-}
-
-
-/**
- * Display the guest CpuId leaves.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helper functions.
- * @param   pszArgs     "terse", "default" or "verbose".
- */
-static DECLCALLBACK(void) cpumR3CpuIdInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    /*
-     * Parse the argument.
-     */
-    unsigned iVerbosity = 1;
-    if (pszArgs)
-    {
-        pszArgs = RTStrStripL(pszArgs);
-        if (!strcmp(pszArgs, "terse"))
-            iVerbosity--;
-        else if (!strcmp(pszArgs, "verbose"))
-            iVerbosity++;
-    }
-
-    /*
-     * Start cracking.
-     */
-    CPUMCPUID   Host;
-    CPUMCPUID   Guest;
-    unsigned    cStdMax = pVM->cpum.s.aGuestCpuIdStd[0].eax;
-
-    pHlp->pfnPrintf(pHlp,
-                    "         RAW Standard CPUIDs\n"
-                    "     Function  eax      ebx      ecx      edx\n");
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd); i++)
-    {
-        Guest = pVM->cpum.s.aGuestCpuIdStd[i];
-        ASMCpuId_Idx_ECX(i, 0, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
-
-        pHlp->pfnPrintf(pHlp,
-                        "Gst: %08x  %08x %08x %08x %08x%s\n"
-                        "Hst:           %08x %08x %08x %08x\n",
-                        i, Guest.eax, Guest.ebx, Guest.ecx, Guest.edx,
-                        i <= cStdMax ? "" : "*",
-                        Host.eax, Host.ebx, Host.ecx, Host.edx);
-    }
-
-    /*
-     * If verbose, decode it.
-     */
-    if (iVerbosity)
-    {
-        Guest = pVM->cpum.s.aGuestCpuIdStd[0];
-        pHlp->pfnPrintf(pHlp,
-                        "Name:                            %.04s%.04s%.04s\n"
-                        "Supports:                        0-%x\n",
-                        &Guest.ebx, &Guest.edx, &Guest.ecx, Guest.eax);
-    }
-
-    /*
-     * Get Features.
-     */
-    bool const fIntel = ASMIsIntelCpuEx(pVM->cpum.s.aGuestCpuIdStd[0].ebx,
-                                        pVM->cpum.s.aGuestCpuIdStd[0].ecx,
-                                        pVM->cpum.s.aGuestCpuIdStd[0].edx);
-    if (cStdMax >= 1 && iVerbosity)
-    {
-        static const char * const s_apszTypes[4] = { "primary", "overdrive", "MP", "reserved" };
-
-        Guest = pVM->cpum.s.aGuestCpuIdStd[1];
-        uint32_t uEAX = Guest.eax;
-
-        pHlp->pfnPrintf(pHlp,
-                        "Family:                          %d  \tExtended: %d \tEffective: %d\n"
-                        "Model:                           %d  \tExtended: %d \tEffective: %d\n"
-                        "Stepping:                        %d\n"
-                        "Type:                            %d (%s)\n"
-                        "APIC ID:                         %#04x\n"
-                        "Logical CPUs:                    %d\n"
-                        "CLFLUSH Size:                    %d\n"
-                        "Brand ID:                        %#04x\n",
-                        (uEAX >> 8) & 0xf, (uEAX >> 20) & 0x7f, ASMGetCpuFamily(uEAX),
-                        (uEAX >> 4) & 0xf, (uEAX >> 16) & 0x0f, ASMGetCpuModel(uEAX, fIntel),
-                        ASMGetCpuStepping(uEAX),
-                        (uEAX >> 12) & 3, s_apszTypes[(uEAX >> 12) & 3],
-                        (Guest.ebx >> 24) & 0xff,
-                        (Guest.ebx >> 16) & 0xff,
-                        (Guest.ebx >>  8) & 0xff,
-                        (Guest.ebx >>  0) & 0xff);
-        if (iVerbosity == 1)
-        {
-            uint32_t uEDX = Guest.edx;
-            pHlp->pfnPrintf(pHlp, "Features EDX:                   ");
-            if (uEDX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " FPU");
-            if (uEDX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " VME");
-            if (uEDX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " DE");
-            if (uEDX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " PSE");
-            if (uEDX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " TSC");
-            if (uEDX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " MSR");
-            if (uEDX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " PAE");
-            if (uEDX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " MCE");
-            if (uEDX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " CX8");
-            if (uEDX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " APIC");
-            if (uEDX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " 10");
-            if (uEDX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " SEP");
-            if (uEDX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " MTRR");
-            if (uEDX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " PGE");
-            if (uEDX & RT_BIT(14))  pHlp->pfnPrintf(pHlp, " MCA");
-            if (uEDX & RT_BIT(15))  pHlp->pfnPrintf(pHlp, " CMOV");
-            if (uEDX & RT_BIT(16))  pHlp->pfnPrintf(pHlp, " PAT");
-            if (uEDX & RT_BIT(17))  pHlp->pfnPrintf(pHlp, " PSE36");
-            if (uEDX & RT_BIT(18))  pHlp->pfnPrintf(pHlp, " PSN");
-            if (uEDX & RT_BIT(19))  pHlp->pfnPrintf(pHlp, " CLFSH");
-            if (uEDX & RT_BIT(20))  pHlp->pfnPrintf(pHlp, " 20");
-            if (uEDX & RT_BIT(21))  pHlp->pfnPrintf(pHlp, " DS");
-            if (uEDX & RT_BIT(22))  pHlp->pfnPrintf(pHlp, " ACPI");
-            if (uEDX & RT_BIT(23))  pHlp->pfnPrintf(pHlp, " MMX");
-            if (uEDX & RT_BIT(24))  pHlp->pfnPrintf(pHlp, " FXSR");
-            if (uEDX & RT_BIT(25))  pHlp->pfnPrintf(pHlp, " SSE");
-            if (uEDX & RT_BIT(26))  pHlp->pfnPrintf(pHlp, " SSE2");
-            if (uEDX & RT_BIT(27))  pHlp->pfnPrintf(pHlp, " SS");
-            if (uEDX & RT_BIT(28))  pHlp->pfnPrintf(pHlp, " HTT");
-            if (uEDX & RT_BIT(29))  pHlp->pfnPrintf(pHlp, " TM");
-            if (uEDX & RT_BIT(30))  pHlp->pfnPrintf(pHlp, " 30");
-            if (uEDX & RT_BIT(31))  pHlp->pfnPrintf(pHlp, " PBE");
-            pHlp->pfnPrintf(pHlp, "\n");
-
-            uint32_t uECX = Guest.ecx;
-            pHlp->pfnPrintf(pHlp, "Features ECX:                   ");
-            if (uECX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " SSE3");
-            if (uECX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " PCLMUL");
-            if (uECX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " DTES64");
-            if (uECX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " MONITOR");
-            if (uECX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " DS-CPL");
-            if (uECX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " VMX");
-            if (uECX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " SMX");
-            if (uECX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " EST");
-            if (uECX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " TM2");
-            if (uECX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " SSSE3");
-            if (uECX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " CNXT-ID");
-            if (uECX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " 11");
-            if (uECX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " FMA");
-            if (uECX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " CX16");
-            if (uECX & RT_BIT(14))  pHlp->pfnPrintf(pHlp, " TPRUPDATE");
-            if (uECX & RT_BIT(15))  pHlp->pfnPrintf(pHlp, " PDCM");
-            if (uECX & RT_BIT(16))  pHlp->pfnPrintf(pHlp, " 16");
-            if (uECX & RT_BIT(17))  pHlp->pfnPrintf(pHlp, " PCID");
-            if (uECX & RT_BIT(18))  pHlp->pfnPrintf(pHlp, " DCA");
-            if (uECX & RT_BIT(19))  pHlp->pfnPrintf(pHlp, " SSE4.1");
-            if (uECX & RT_BIT(20))  pHlp->pfnPrintf(pHlp, " SSE4.2");
-            if (uECX & RT_BIT(21))  pHlp->pfnPrintf(pHlp, " X2APIC");
-            if (uECX & RT_BIT(22))  pHlp->pfnPrintf(pHlp, " MOVBE");
-            if (uECX & RT_BIT(23))  pHlp->pfnPrintf(pHlp, " POPCNT");
-            if (uECX & RT_BIT(24))  pHlp->pfnPrintf(pHlp, " TSCDEADL");
-            if (uECX & RT_BIT(25))  pHlp->pfnPrintf(pHlp, " AES");
-            if (uECX & RT_BIT(26))  pHlp->pfnPrintf(pHlp, " XSAVE");
-            if (uECX & RT_BIT(27))  pHlp->pfnPrintf(pHlp, " OSXSAVE");
-            if (uECX & RT_BIT(28))  pHlp->pfnPrintf(pHlp, " AVX");
-            if (uECX & RT_BIT(29))  pHlp->pfnPrintf(pHlp, " 29");
-            if (uECX & RT_BIT(30))  pHlp->pfnPrintf(pHlp, " 30");
-            if (uECX & RT_BIT(31))  pHlp->pfnPrintf(pHlp, " 31");
-            pHlp->pfnPrintf(pHlp, "\n");
-        }
-        else
-        {
-            ASMCpuId(1, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
-
-            X86CPUIDFEATEDX EdxHost  = *(PX86CPUIDFEATEDX)&Host.edx;
-            X86CPUIDFEATECX EcxHost  = *(PX86CPUIDFEATECX)&Host.ecx;
-            X86CPUIDFEATEDX EdxGuest = *(PX86CPUIDFEATEDX)&Guest.edx;
-            X86CPUIDFEATECX EcxGuest = *(PX86CPUIDFEATECX)&Guest.ecx;
-
-            pHlp->pfnPrintf(pHlp, "Mnemonic - Description                 = guest (host)\n");
-            pHlp->pfnPrintf(pHlp, "FPU - x87 FPU on Chip                  = %d (%d)\n",  EdxGuest.u1FPU,        EdxHost.u1FPU);
-            pHlp->pfnPrintf(pHlp, "VME - Virtual 8086 Mode Enhancements   = %d (%d)\n",  EdxGuest.u1VME,        EdxHost.u1VME);
-            pHlp->pfnPrintf(pHlp, "DE - Debugging extensions              = %d (%d)\n",  EdxGuest.u1DE,         EdxHost.u1DE);
-            pHlp->pfnPrintf(pHlp, "PSE - Page Size Extension              = %d (%d)\n",  EdxGuest.u1PSE,        EdxHost.u1PSE);
-            pHlp->pfnPrintf(pHlp, "TSC - Time Stamp Counter               = %d (%d)\n",  EdxGuest.u1TSC,        EdxHost.u1TSC);
-            pHlp->pfnPrintf(pHlp, "MSR - Model Specific Registers         = %d (%d)\n",  EdxGuest.u1MSR,        EdxHost.u1MSR);
-            pHlp->pfnPrintf(pHlp, "PAE - Physical Address Extension       = %d (%d)\n",  EdxGuest.u1PAE,        EdxHost.u1PAE);
-            pHlp->pfnPrintf(pHlp, "MCE - Machine Check Exception          = %d (%d)\n",  EdxGuest.u1MCE,        EdxHost.u1MCE);
-            pHlp->pfnPrintf(pHlp, "CX8 - CMPXCHG8B instruction            = %d (%d)\n",  EdxGuest.u1CX8,        EdxHost.u1CX8);
-            pHlp->pfnPrintf(pHlp, "APIC - APIC On-Chip                    = %d (%d)\n",  EdxGuest.u1APIC,       EdxHost.u1APIC);
-            pHlp->pfnPrintf(pHlp, "10 - Reserved                          = %d (%d)\n",  EdxGuest.u1Reserved1,  EdxHost.u1Reserved1);
-            pHlp->pfnPrintf(pHlp, "SEP - SYSENTER and SYSEXIT             = %d (%d)\n",  EdxGuest.u1SEP,        EdxHost.u1SEP);
-            pHlp->pfnPrintf(pHlp, "MTRR - Memory Type Range Registers     = %d (%d)\n",  EdxGuest.u1MTRR,       EdxHost.u1MTRR);
-            pHlp->pfnPrintf(pHlp, "PGE - PTE Global Bit                   = %d (%d)\n",  EdxGuest.u1PGE,        EdxHost.u1PGE);
-            pHlp->pfnPrintf(pHlp, "MCA - Machine Check Architecture       = %d (%d)\n",  EdxGuest.u1MCA,        EdxHost.u1MCA);
-            pHlp->pfnPrintf(pHlp, "CMOV - Conditional Move Instructions   = %d (%d)\n",  EdxGuest.u1CMOV,       EdxHost.u1CMOV);
-            pHlp->pfnPrintf(pHlp, "PAT - Page Attribute Table             = %d (%d)\n",  EdxGuest.u1PAT,        EdxHost.u1PAT);
-            pHlp->pfnPrintf(pHlp, "PSE-36 - 36-bit Page Size Extention    = %d (%d)\n",  EdxGuest.u1PSE36,      EdxHost.u1PSE36);
-            pHlp->pfnPrintf(pHlp, "PSN - Processor Serial Number          = %d (%d)\n",  EdxGuest.u1PSN,        EdxHost.u1PSN);
-            pHlp->pfnPrintf(pHlp, "CLFSH - CLFLUSH Instruction.           = %d (%d)\n",  EdxGuest.u1CLFSH,      EdxHost.u1CLFSH);
-            pHlp->pfnPrintf(pHlp, "20 - Reserved                          = %d (%d)\n",  EdxGuest.u1Reserved2,  EdxHost.u1Reserved2);
-            pHlp->pfnPrintf(pHlp, "DS - Debug Store                       = %d (%d)\n",  EdxGuest.u1DS,         EdxHost.u1DS);
-            pHlp->pfnPrintf(pHlp, "ACPI - Thermal Mon. & Soft. Clock Ctrl.= %d (%d)\n",  EdxGuest.u1ACPI,       EdxHost.u1ACPI);
-            pHlp->pfnPrintf(pHlp, "MMX - Intel MMX Technology             = %d (%d)\n",  EdxGuest.u1MMX,        EdxHost.u1MMX);
-            pHlp->pfnPrintf(pHlp, "FXSR - FXSAVE and FXRSTOR Instructions = %d (%d)\n",  EdxGuest.u1FXSR,       EdxHost.u1FXSR);
-            pHlp->pfnPrintf(pHlp, "SSE - SSE Support                      = %d (%d)\n",  EdxGuest.u1SSE,        EdxHost.u1SSE);
-            pHlp->pfnPrintf(pHlp, "SSE2 - SSE2 Support                    = %d (%d)\n",  EdxGuest.u1SSE2,       EdxHost.u1SSE2);
-            pHlp->pfnPrintf(pHlp, "SS - Self Snoop                        = %d (%d)\n",  EdxGuest.u1SS,         EdxHost.u1SS);
-            pHlp->pfnPrintf(pHlp, "HTT - Hyper-Threading Technology       = %d (%d)\n",  EdxGuest.u1HTT,        EdxHost.u1HTT);
-            pHlp->pfnPrintf(pHlp, "TM - Thermal Monitor                   = %d (%d)\n",  EdxGuest.u1TM,         EdxHost.u1TM);
-            pHlp->pfnPrintf(pHlp, "30 - Reserved                          = %d (%d)\n",  EdxGuest.u1Reserved3,  EdxHost.u1Reserved3);
-            pHlp->pfnPrintf(pHlp, "PBE - Pending Break Enable             = %d (%d)\n",  EdxGuest.u1PBE,        EdxHost.u1PBE);
-
-            pHlp->pfnPrintf(pHlp, "Supports SSE3                          = %d (%d)\n",  EcxGuest.u1SSE3,       EcxHost.u1SSE3);
-            pHlp->pfnPrintf(pHlp, "PCLMULQDQ                              = %d (%d)\n",  EcxGuest.u1PCLMULQDQ,  EcxHost.u1PCLMULQDQ);
-            pHlp->pfnPrintf(pHlp, "DS Area 64-bit layout                  = %d (%d)\n",  EcxGuest.u1DTE64,      EcxHost.u1DTE64);
-            pHlp->pfnPrintf(pHlp, "Supports MONITOR/MWAIT                 = %d (%d)\n",  EcxGuest.u1Monitor,    EcxHost.u1Monitor);
-            pHlp->pfnPrintf(pHlp, "CPL-DS - CPL Qualified Debug Store     = %d (%d)\n",  EcxGuest.u1CPLDS,      EcxHost.u1CPLDS);
-            pHlp->pfnPrintf(pHlp, "VMX - Virtual Machine Technology       = %d (%d)\n",  EcxGuest.u1VMX,        EcxHost.u1VMX);
-            pHlp->pfnPrintf(pHlp, "SMX - Safer Mode Extensions            = %d (%d)\n",  EcxGuest.u1SMX,        EcxHost.u1SMX);
-            pHlp->pfnPrintf(pHlp, "Enhanced SpeedStep Technology          = %d (%d)\n",  EcxGuest.u1EST,        EcxHost.u1EST);
-            pHlp->pfnPrintf(pHlp, "Terminal Monitor 2                     = %d (%d)\n",  EcxGuest.u1TM2,        EcxHost.u1TM2);
-            pHlp->pfnPrintf(pHlp, "Supplemental SSE3 instructions         = %d (%d)\n",  EcxGuest.u1SSSE3,      EcxHost.u1SSSE3);
-            pHlp->pfnPrintf(pHlp, "L1 Context ID                          = %d (%d)\n",  EcxGuest.u1CNTXID,     EcxHost.u1CNTXID);
-            pHlp->pfnPrintf(pHlp, "11 - Reserved                          = %d (%d)\n",  EcxGuest.u1Reserved1,  EcxHost.u1Reserved1);
-            pHlp->pfnPrintf(pHlp, "FMA extensions using YMM state         = %d (%d)\n",  EcxGuest.u1FMA,        EcxHost.u1FMA);
-            pHlp->pfnPrintf(pHlp, "CMPXCHG16B instruction                 = %d (%d)\n",  EcxGuest.u1CX16,       EcxHost.u1CX16);
-            pHlp->pfnPrintf(pHlp, "xTPR Update Control                    = %d (%d)\n",  EcxGuest.u1TPRUpdate,  EcxHost.u1TPRUpdate);
-            pHlp->pfnPrintf(pHlp, "Perf/Debug Capability MSR              = %d (%d)\n",  EcxGuest.u1PDCM,       EcxHost.u1PDCM);
-            pHlp->pfnPrintf(pHlp, "16 - Reserved                          = %d (%d)\n",  EcxGuest.u1Reserved2,  EcxHost.u1Reserved2);
-            pHlp->pfnPrintf(pHlp, "PCID - Process-context identifiers     = %d (%d)\n",  EcxGuest.u1PCID,       EcxHost.u1PCID);
-            pHlp->pfnPrintf(pHlp, "DCA - Direct Cache Access              = %d (%d)\n",  EcxGuest.u1DCA,        EcxHost.u1DCA);
-            pHlp->pfnPrintf(pHlp, "SSE4.1 instruction extensions          = %d (%d)\n",  EcxGuest.u1SSE4_1,     EcxHost.u1SSE4_1);
-            pHlp->pfnPrintf(pHlp, "SSE4.2 instruction extensions          = %d (%d)\n",  EcxGuest.u1SSE4_2,     EcxHost.u1SSE4_2);
-            pHlp->pfnPrintf(pHlp, "Supports the x2APIC extensions         = %d (%d)\n",  EcxGuest.u1x2APIC,     EcxHost.u1x2APIC);
-            pHlp->pfnPrintf(pHlp, "MOVBE instruction                      = %d (%d)\n",  EcxGuest.u1MOVBE,      EcxHost.u1MOVBE);
-            pHlp->pfnPrintf(pHlp, "POPCNT instruction                     = %d (%d)\n",  EcxGuest.u1POPCNT,     EcxHost.u1POPCNT);
-            pHlp->pfnPrintf(pHlp, "TSC-Deadline LAPIC timer mode          = %d (%d)\n",  EcxGuest.u1TSCDEADLINE,EcxHost.u1TSCDEADLINE);
-            pHlp->pfnPrintf(pHlp, "AESNI instruction extensions           = %d (%d)\n",  EcxGuest.u1AES,        EcxHost.u1AES);
-            pHlp->pfnPrintf(pHlp, "XSAVE/XRSTOR extended state feature    = %d (%d)\n",  EcxGuest.u1XSAVE,      EcxHost.u1XSAVE);
-            pHlp->pfnPrintf(pHlp, "Supports OSXSAVE                       = %d (%d)\n",  EcxGuest.u1OSXSAVE,    EcxHost.u1OSXSAVE);
-            pHlp->pfnPrintf(pHlp, "AVX instruction extensions             = %d (%d)\n",  EcxGuest.u1AVX,        EcxHost.u1AVX);
-            pHlp->pfnPrintf(pHlp, "29/30 - Reserved                       = %#x (%#x)\n",EcxGuest.u2Reserved3,  EcxHost.u2Reserved3);
-            pHlp->pfnPrintf(pHlp, "31 - Reserved (always 0)               = %d (%d)\n",  EcxGuest.u1Reserved4,  EcxHost.u1Reserved4);
-        }
-    }
-    if (cStdMax >= 2 && iVerbosity)
-    {
-        /** @todo */
-    }
-
-    /*
-     * Extended.
-     * Implemented after AMD specs.
-     */
-    unsigned    cExtMax = pVM->cpum.s.aGuestCpuIdExt[0].eax & 0xffff;
-
-    pHlp->pfnPrintf(pHlp,
-                    "\n"
-                    "         RAW Extended CPUIDs\n"
-                    "     Function  eax      ebx      ecx      edx\n");
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt); i++)
-    {
-        Guest = pVM->cpum.s.aGuestCpuIdExt[i];
-        ASMCpuId(0x80000000 | i, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
-
-        pHlp->pfnPrintf(pHlp,
-                        "Gst: %08x  %08x %08x %08x %08x%s\n"
-                        "Hst:           %08x %08x %08x %08x\n",
-                        0x80000000 | i, Guest.eax, Guest.ebx, Guest.ecx, Guest.edx,
-                        i <= cExtMax ? "" : "*",
-                        Host.eax, Host.ebx, Host.ecx, Host.edx);
-    }
-
-    /*
-     * Understandable output
-     */
-    if (iVerbosity)
-    {
-        Guest = pVM->cpum.s.aGuestCpuIdExt[0];
-        pHlp->pfnPrintf(pHlp,
-                        "Ext Name:                        %.4s%.4s%.4s\n"
-                        "Ext Supports:                    0x80000000-%#010x\n",
-                        &Guest.ebx, &Guest.edx, &Guest.ecx, Guest.eax);
-    }
-
-    if (iVerbosity && cExtMax >= 1)
-    {
-        Guest = pVM->cpum.s.aGuestCpuIdExt[1];
-        uint32_t uEAX = Guest.eax;
-        pHlp->pfnPrintf(pHlp,
-                        "Family:                          %d  \tExtended: %d \tEffective: %d\n"
-                        "Model:                           %d  \tExtended: %d \tEffective: %d\n"
-                        "Stepping:                        %d\n"
-                        "Brand ID:                        %#05x\n",
-                        (uEAX >> 8) & 0xf, (uEAX >> 20) & 0x7f, ASMGetCpuFamily(uEAX),
-                        (uEAX >> 4) & 0xf, (uEAX >> 16) & 0x0f, ASMGetCpuModel(uEAX, fIntel),
-                        ASMGetCpuStepping(uEAX),
-                        Guest.ebx & 0xfff);
-
-        if (iVerbosity == 1)
-        {
-            uint32_t uEDX = Guest.edx;
-            pHlp->pfnPrintf(pHlp, "Features EDX:                   ");
-            if (uEDX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " FPU");
-            if (uEDX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " VME");
-            if (uEDX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " DE");
-            if (uEDX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " PSE");
-            if (uEDX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " TSC");
-            if (uEDX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " MSR");
-            if (uEDX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " PAE");
-            if (uEDX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " MCE");
-            if (uEDX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " CX8");
-            if (uEDX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " APIC");
-            if (uEDX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " 10");
-            if (uEDX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " SCR");
-            if (uEDX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " MTRR");
-            if (uEDX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " PGE");
-            if (uEDX & RT_BIT(14))  pHlp->pfnPrintf(pHlp, " MCA");
-            if (uEDX & RT_BIT(15))  pHlp->pfnPrintf(pHlp, " CMOV");
-            if (uEDX & RT_BIT(16))  pHlp->pfnPrintf(pHlp, " PAT");
-            if (uEDX & RT_BIT(17))  pHlp->pfnPrintf(pHlp, " PSE36");
-            if (uEDX & RT_BIT(18))  pHlp->pfnPrintf(pHlp, " 18");
-            if (uEDX & RT_BIT(19))  pHlp->pfnPrintf(pHlp, " 19");
-            if (uEDX & RT_BIT(20))  pHlp->pfnPrintf(pHlp, " NX");
-            if (uEDX & RT_BIT(21))  pHlp->pfnPrintf(pHlp, " 21");
-            if (uEDX & RT_BIT(22))  pHlp->pfnPrintf(pHlp, " ExtMMX");
-            if (uEDX & RT_BIT(23))  pHlp->pfnPrintf(pHlp, " MMX");
-            if (uEDX & RT_BIT(24))  pHlp->pfnPrintf(pHlp, " FXSR");
-            if (uEDX & RT_BIT(25))  pHlp->pfnPrintf(pHlp, " FastFXSR");
-            if (uEDX & RT_BIT(26))  pHlp->pfnPrintf(pHlp, " Page1GB");
-            if (uEDX & RT_BIT(27))  pHlp->pfnPrintf(pHlp, " RDTSCP");
-            if (uEDX & RT_BIT(28))  pHlp->pfnPrintf(pHlp, " 28");
-            if (uEDX & RT_BIT(29))  pHlp->pfnPrintf(pHlp, " LongMode");
-            if (uEDX & RT_BIT(30))  pHlp->pfnPrintf(pHlp, " Ext3DNow");
-            if (uEDX & RT_BIT(31))  pHlp->pfnPrintf(pHlp, " 3DNow");
-            pHlp->pfnPrintf(pHlp, "\n");
-
-            uint32_t uECX = Guest.ecx;
-            pHlp->pfnPrintf(pHlp, "Features ECX:                   ");
-            if (uECX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " LAHF/SAHF");
-            if (uECX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " CMPL");
-            if (uECX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " SVM");
-            if (uECX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " ExtAPIC");
-            if (uECX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " CR8L");
-            if (uECX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " ABM");
-            if (uECX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " SSE4A");
-            if (uECX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " MISALNSSE");
-            if (uECX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " 3DNOWPRF");
-            if (uECX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " OSVW");
-            if (uECX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " IBS");
-            if (uECX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " SSE5");
-            if (uECX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " SKINIT");
-            if (uECX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " WDT");
-            for (unsigned iBit = 5; iBit < 32; iBit++)
-                if (uECX & RT_BIT(iBit))
-                    pHlp->pfnPrintf(pHlp, " %d", iBit);
-            pHlp->pfnPrintf(pHlp, "\n");
-        }
-        else
-        {
-            ASMCpuId(0x80000001, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
-
-            uint32_t uEdxGst = Guest.edx;
-            uint32_t uEdxHst = Host.edx;
-            pHlp->pfnPrintf(pHlp, "Mnemonic - Description                 = guest (host)\n");
-            pHlp->pfnPrintf(pHlp, "FPU - x87 FPU on Chip                  = %d (%d)\n",  !!(uEdxGst & RT_BIT( 0)),  !!(uEdxHst & RT_BIT( 0)));
-            pHlp->pfnPrintf(pHlp, "VME - Virtual 8086 Mode Enhancements   = %d (%d)\n",  !!(uEdxGst & RT_BIT( 1)),  !!(uEdxHst & RT_BIT( 1)));
-            pHlp->pfnPrintf(pHlp, "DE - Debugging extensions              = %d (%d)\n",  !!(uEdxGst & RT_BIT( 2)),  !!(uEdxHst & RT_BIT( 2)));
-            pHlp->pfnPrintf(pHlp, "PSE - Page Size Extension              = %d (%d)\n",  !!(uEdxGst & RT_BIT( 3)),  !!(uEdxHst & RT_BIT( 3)));
-            pHlp->pfnPrintf(pHlp, "TSC - Time Stamp Counter               = %d (%d)\n",  !!(uEdxGst & RT_BIT( 4)),  !!(uEdxHst & RT_BIT( 4)));
-            pHlp->pfnPrintf(pHlp, "MSR - K86 Model Specific Registers     = %d (%d)\n",  !!(uEdxGst & RT_BIT( 5)),  !!(uEdxHst & RT_BIT( 5)));
-            pHlp->pfnPrintf(pHlp, "PAE - Physical Address Extension       = %d (%d)\n",  !!(uEdxGst & RT_BIT( 6)),  !!(uEdxHst & RT_BIT( 6)));
-            pHlp->pfnPrintf(pHlp, "MCE - Machine Check Exception          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 7)),  !!(uEdxHst & RT_BIT( 7)));
-            pHlp->pfnPrintf(pHlp, "CX8 - CMPXCHG8B instruction            = %d (%d)\n",  !!(uEdxGst & RT_BIT( 8)),  !!(uEdxHst & RT_BIT( 8)));
-            pHlp->pfnPrintf(pHlp, "APIC - APIC On-Chip                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 9)),  !!(uEdxHst & RT_BIT( 9)));
-            pHlp->pfnPrintf(pHlp, "10 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(10)),  !!(uEdxHst & RT_BIT(10)));
-            pHlp->pfnPrintf(pHlp, "SEP - SYSCALL and SYSRET               = %d (%d)\n",  !!(uEdxGst & RT_BIT(11)),  !!(uEdxHst & RT_BIT(11)));
-            pHlp->pfnPrintf(pHlp, "MTRR - Memory Type Range Registers     = %d (%d)\n",  !!(uEdxGst & RT_BIT(12)),  !!(uEdxHst & RT_BIT(12)));
-            pHlp->pfnPrintf(pHlp, "PGE - PTE Global Bit                   = %d (%d)\n",  !!(uEdxGst & RT_BIT(13)),  !!(uEdxHst & RT_BIT(13)));
-            pHlp->pfnPrintf(pHlp, "MCA - Machine Check Architecture       = %d (%d)\n",  !!(uEdxGst & RT_BIT(14)),  !!(uEdxHst & RT_BIT(14)));
-            pHlp->pfnPrintf(pHlp, "CMOV - Conditional Move Instructions   = %d (%d)\n",  !!(uEdxGst & RT_BIT(15)),  !!(uEdxHst & RT_BIT(15)));
-            pHlp->pfnPrintf(pHlp, "PAT - Page Attribute Table             = %d (%d)\n",  !!(uEdxGst & RT_BIT(16)),  !!(uEdxHst & RT_BIT(16)));
-            pHlp->pfnPrintf(pHlp, "PSE-36 - 36-bit Page Size Extention    = %d (%d)\n",  !!(uEdxGst & RT_BIT(17)),  !!(uEdxHst & RT_BIT(17)));
-            pHlp->pfnPrintf(pHlp, "18 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(18)),  !!(uEdxHst & RT_BIT(18)));
-            pHlp->pfnPrintf(pHlp, "19 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(19)),  !!(uEdxHst & RT_BIT(19)));
-            pHlp->pfnPrintf(pHlp, "NX - No-Execute Page Protection        = %d (%d)\n",  !!(uEdxGst & RT_BIT(20)),  !!(uEdxHst & RT_BIT(20)));
-            pHlp->pfnPrintf(pHlp, "DS - Debug Store                       = %d (%d)\n",  !!(uEdxGst & RT_BIT(21)),  !!(uEdxHst & RT_BIT(21)));
-            pHlp->pfnPrintf(pHlp, "AXMMX - AMD Extensions to MMX Instr.   = %d (%d)\n",  !!(uEdxGst & RT_BIT(22)),  !!(uEdxHst & RT_BIT(22)));
-            pHlp->pfnPrintf(pHlp, "MMX - Intel MMX Technology             = %d (%d)\n",  !!(uEdxGst & RT_BIT(23)),  !!(uEdxHst & RT_BIT(23)));
-            pHlp->pfnPrintf(pHlp, "FXSR - FXSAVE and FXRSTOR Instructions = %d (%d)\n",  !!(uEdxGst & RT_BIT(24)),  !!(uEdxHst & RT_BIT(24)));
-            pHlp->pfnPrintf(pHlp, "25 - AMD fast FXSAVE and FXRSTOR Instr.= %d (%d)\n",  !!(uEdxGst & RT_BIT(25)),  !!(uEdxHst & RT_BIT(25)));
-            pHlp->pfnPrintf(pHlp, "26 - 1 GB large page support           = %d (%d)\n",  !!(uEdxGst & RT_BIT(26)),  !!(uEdxHst & RT_BIT(26)));
-            pHlp->pfnPrintf(pHlp, "27 - RDTSCP instruction                = %d (%d)\n",  !!(uEdxGst & RT_BIT(27)),  !!(uEdxHst & RT_BIT(27)));
-            pHlp->pfnPrintf(pHlp, "28 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(28)),  !!(uEdxHst & RT_BIT(28)));
-            pHlp->pfnPrintf(pHlp, "29 - AMD Long Mode                     = %d (%d)\n",  !!(uEdxGst & RT_BIT(29)),  !!(uEdxHst & RT_BIT(29)));
-            pHlp->pfnPrintf(pHlp, "30 - AMD Extensions to 3DNow           = %d (%d)\n",  !!(uEdxGst & RT_BIT(30)),  !!(uEdxHst & RT_BIT(30)));
-            pHlp->pfnPrintf(pHlp, "31 - AMD 3DNow                         = %d (%d)\n",  !!(uEdxGst & RT_BIT(31)),  !!(uEdxHst & RT_BIT(31)));
-
-            uint32_t uEcxGst = Guest.ecx;
-            uint32_t uEcxHst = Host.ecx;
-            pHlp->pfnPrintf(pHlp, "LahfSahf - LAHF/SAHF in 64-bit mode    = %d (%d)\n",  !!(uEcxGst & RT_BIT( 0)),  !!(uEcxHst & RT_BIT( 0)));
-            pHlp->pfnPrintf(pHlp, "CmpLegacy - Core MP legacy mode (depr) = %d (%d)\n",  !!(uEcxGst & RT_BIT( 1)),  !!(uEcxHst & RT_BIT( 1)));
-            pHlp->pfnPrintf(pHlp, "SVM - AMD VM Extensions                = %d (%d)\n",  !!(uEcxGst & RT_BIT( 2)),  !!(uEcxHst & RT_BIT( 2)));
-            pHlp->pfnPrintf(pHlp, "APIC registers starting at 0x400       = %d (%d)\n",  !!(uEcxGst & RT_BIT( 3)),  !!(uEcxHst & RT_BIT( 3)));
-            pHlp->pfnPrintf(pHlp, "AltMovCR8 - LOCK MOV CR0 means MOV CR8 = %d (%d)\n",  !!(uEcxGst & RT_BIT( 4)),  !!(uEcxHst & RT_BIT( 4)));
-            pHlp->pfnPrintf(pHlp, "Advanced bit manipulation              = %d (%d)\n",  !!(uEcxGst & RT_BIT( 5)),  !!(uEcxHst & RT_BIT( 5)));
-            pHlp->pfnPrintf(pHlp, "SSE4A instruction support              = %d (%d)\n",  !!(uEcxGst & RT_BIT( 6)),  !!(uEcxHst & RT_BIT( 6)));
-            pHlp->pfnPrintf(pHlp, "Misaligned SSE mode                    = %d (%d)\n",  !!(uEcxGst & RT_BIT( 7)),  !!(uEcxHst & RT_BIT( 7)));
-            pHlp->pfnPrintf(pHlp, "PREFETCH and PREFETCHW instruction     = %d (%d)\n",  !!(uEcxGst & RT_BIT( 8)),  !!(uEcxHst & RT_BIT( 8)));
-            pHlp->pfnPrintf(pHlp, "OS visible workaround                  = %d (%d)\n",  !!(uEcxGst & RT_BIT( 9)),  !!(uEcxHst & RT_BIT( 9)));
-            pHlp->pfnPrintf(pHlp, "Instruction based sampling             = %d (%d)\n",  !!(uEcxGst & RT_BIT(10)),  !!(uEcxHst & RT_BIT(10)));
-            pHlp->pfnPrintf(pHlp, "SSE5 support                           = %d (%d)\n",  !!(uEcxGst & RT_BIT(11)),  !!(uEcxHst & RT_BIT(11)));
-            pHlp->pfnPrintf(pHlp, "SKINIT, STGI, and DEV support          = %d (%d)\n",  !!(uEcxGst & RT_BIT(12)),  !!(uEcxHst & RT_BIT(12)));
-            pHlp->pfnPrintf(pHlp, "Watchdog timer support.                = %d (%d)\n",  !!(uEcxGst & RT_BIT(13)),  !!(uEcxHst & RT_BIT(13)));
-            pHlp->pfnPrintf(pHlp, "31:14 - Reserved                       = %#x (%#x)\n",   uEcxGst >> 14,          uEcxHst >> 14);
-        }
-    }
-
-    if (iVerbosity && cExtMax >= 2)
-    {
-        char szString[4*4*3+1] = {0};
-        uint32_t *pu32 = (uint32_t *)szString;
-        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].eax;
-        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].ebx;
-        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].ecx;
-        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].edx;
-        if (cExtMax >= 3)
-        {
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].eax;
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].ebx;
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].ecx;
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].edx;
-        }
-        if (cExtMax >= 4)
-        {
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].eax;
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].ebx;
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].ecx;
-            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].edx;
-        }
-        pHlp->pfnPrintf(pHlp, "Full Name:                       %s\n", szString);
-    }
-
-    if (iVerbosity && cExtMax >= 5)
-    {
-        uint32_t uEAX = pVM->cpum.s.aGuestCpuIdExt[5].eax;
-        uint32_t uEBX = pVM->cpum.s.aGuestCpuIdExt[5].ebx;
-        uint32_t uECX = pVM->cpum.s.aGuestCpuIdExt[5].ecx;
-        uint32_t uEDX = pVM->cpum.s.aGuestCpuIdExt[5].edx;
-        char sz1[32];
-        char sz2[32];
-
-        pHlp->pfnPrintf(pHlp,
-                        "TLB 2/4M Instr/Uni:              %s %3d entries\n"
-                        "TLB 2/4M Data:                   %s %3d entries\n",
-                        getCacheAss((uEAX >>  8) & 0xff, sz1), (uEAX >>  0) & 0xff,
-                        getCacheAss((uEAX >> 24) & 0xff, sz2), (uEAX >> 16) & 0xff);
-        pHlp->pfnPrintf(pHlp,
-                        "TLB 4K Instr/Uni:                %s %3d entries\n"
-                        "TLB 4K Data:                     %s %3d entries\n",
-                        getCacheAss((uEBX >>  8) & 0xff, sz1), (uEBX >>  0) & 0xff,
-                        getCacheAss((uEBX >> 24) & 0xff, sz2), (uEBX >> 16) & 0xff);
-        pHlp->pfnPrintf(pHlp, "L1 Instr Cache Line Size:        %d bytes\n"
-                        "L1 Instr Cache Lines Per Tag:    %d\n"
-                        "L1 Instr Cache Associativity:    %s\n"
-                        "L1 Instr Cache Size:             %d KB\n",
-                        (uEDX >> 0) & 0xff,
-                        (uEDX >> 8) & 0xff,
-                        getCacheAss((uEDX >> 16) & 0xff, sz1),
-                        (uEDX >> 24) & 0xff);
-        pHlp->pfnPrintf(pHlp,
-                        "L1 Data Cache Line Size:         %d bytes\n"
-                        "L1 Data Cache Lines Per Tag:     %d\n"
-                        "L1 Data Cache Associativity:     %s\n"
-                        "L1 Data Cache Size:              %d KB\n",
-                        (uECX >> 0) & 0xff,
-                        (uECX >> 8) & 0xff,
-                        getCacheAss((uECX >> 16) & 0xff, sz1),
-                        (uECX >> 24) & 0xff);
-    }
-
-    if (iVerbosity && cExtMax >= 6)
-    {
-        uint32_t uEAX = pVM->cpum.s.aGuestCpuIdExt[6].eax;
-        uint32_t uEBX = pVM->cpum.s.aGuestCpuIdExt[6].ebx;
-        uint32_t uEDX = pVM->cpum.s.aGuestCpuIdExt[6].edx;
-
-        pHlp->pfnPrintf(pHlp,
-                        "L2 TLB 2/4M Instr/Uni:           %s %4d entries\n"
-                        "L2 TLB 2/4M Data:                %s %4d entries\n",
-                        getL2CacheAss((uEAX >> 12) & 0xf),  (uEAX >>  0) & 0xfff,
-                        getL2CacheAss((uEAX >> 28) & 0xf),  (uEAX >> 16) & 0xfff);
-        pHlp->pfnPrintf(pHlp,
-                        "L2 TLB 4K Instr/Uni:             %s %4d entries\n"
-                        "L2 TLB 4K Data:                  %s %4d entries\n",
-                        getL2CacheAss((uEBX >> 12) & 0xf),  (uEBX >>  0) & 0xfff,
-                        getL2CacheAss((uEBX >> 28) & 0xf),  (uEBX >> 16) & 0xfff);
-        pHlp->pfnPrintf(pHlp,
-                        "L2 Cache Line Size:              %d bytes\n"
-                        "L2 Cache Lines Per Tag:          %d\n"
-                        "L2 Cache Associativity:          %s\n"
-                        "L2 Cache Size:                   %d KB\n",
-                        (uEDX >> 0) & 0xff,
-                        (uEDX >> 8) & 0xf,
-                        getL2CacheAss((uEDX >> 12) & 0xf),
-                        (uEDX >> 16) & 0xffff);
-    }
-
-    if (iVerbosity && cExtMax >= 7)
-    {
-        uint32_t uEDX = pVM->cpum.s.aGuestCpuIdExt[7].edx;
-
-        pHlp->pfnPrintf(pHlp, "APM Features:                   ");
-        if (uEDX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " TS");
-        if (uEDX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " FID");
-        if (uEDX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " VID");
-        if (uEDX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " TTP");
-        if (uEDX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " TM");
-        if (uEDX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " STC");
-        for (unsigned iBit = 6; iBit < 32; iBit++)
-            if (uEDX & RT_BIT(iBit))
-                pHlp->pfnPrintf(pHlp, " %d", iBit);
-        pHlp->pfnPrintf(pHlp, "\n");
-    }
-
-    if (iVerbosity && cExtMax >= 8)
-    {
-        uint32_t uEAX = pVM->cpum.s.aGuestCpuIdExt[8].eax;
-        uint32_t uECX = pVM->cpum.s.aGuestCpuIdExt[8].ecx;
-
-        pHlp->pfnPrintf(pHlp,
-                        "Physical Address Width:          %d bits\n"
-                        "Virtual Address Width:           %d bits\n"
-                        "Guest Physical Address Width:    %d bits\n",
-                        (uEAX >> 0) & 0xff,
-                        (uEAX >> 8) & 0xff,
-                        (uEAX >> 16) & 0xff);
-        pHlp->pfnPrintf(pHlp,
-                        "Physical Core Count:             %d\n",
-                        (uECX >> 0) & 0xff);
-    }
-
-
-    /*
-     * Centaur.
-     */
-    unsigned cCentaurMax = pVM->cpum.s.aGuestCpuIdCentaur[0].eax & 0xffff;
-
-    pHlp->pfnPrintf(pHlp,
-                    "\n"
-                    "         RAW Centaur CPUIDs\n"
-                    "     Function  eax      ebx      ecx      edx\n");
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur); i++)
-    {
-        Guest = pVM->cpum.s.aGuestCpuIdCentaur[i];
-        ASMCpuId(0xc0000000 | i, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
-
-        pHlp->pfnPrintf(pHlp,
-                        "Gst: %08x  %08x %08x %08x %08x%s\n"
-                        "Hst:           %08x %08x %08x %08x\n",
-                        0xc0000000 | i, Guest.eax, Guest.ebx, Guest.ecx, Guest.edx,
-                        i <= cCentaurMax ? "" : "*",
-                        Host.eax, Host.ebx, Host.ecx, Host.edx);
-    }
-
-    /*
-     * Understandable output
-     */
-    if (iVerbosity)
-    {
-        Guest = pVM->cpum.s.aGuestCpuIdCentaur[0];
-        pHlp->pfnPrintf(pHlp,
-                        "Centaur Supports:                0xc0000000-%#010x\n",
-                        Guest.eax);
-    }
-
-    if (iVerbosity && cCentaurMax >= 1)
-    {
-        ASMCpuId(0xc0000001, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
-        uint32_t uEdxGst = pVM->cpum.s.aGuestCpuIdExt[1].edx;
-        uint32_t uEdxHst = Host.edx;
-
-        if (iVerbosity == 1)
-        {
-            pHlp->pfnPrintf(pHlp, "Centaur Features EDX:           ");
-            if (uEdxGst & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " AIS");
-            if (uEdxGst & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " AIS-E");
-            if (uEdxGst & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " RNG");
-            if (uEdxGst & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " RNG-E");
-            if (uEdxGst & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " LH");
-            if (uEdxGst & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " FEMMS");
-            if (uEdxGst & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " ACE");
-            if (uEdxGst & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " ACE-E");
-            /* possibly indicating MM/HE and MM/HE-E on older chips... */
-            if (uEdxGst & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " ACE2");
-            if (uEdxGst & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " ACE2-E");
-            if (uEdxGst & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " PHE");
-            if (uEdxGst & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " PHE-E");
-            if (uEdxGst & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " PMM");
-            if (uEdxGst & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " PMM-E");
-            for (unsigned iBit = 14; iBit < 32; iBit++)
-                if (uEdxGst & RT_BIT(iBit))
-                    pHlp->pfnPrintf(pHlp, " %d", iBit);
-            pHlp->pfnPrintf(pHlp, "\n");
-        }
-        else
-        {
-            pHlp->pfnPrintf(pHlp, "Mnemonic - Description                 = guest (host)\n");
-            pHlp->pfnPrintf(pHlp, "AIS - Alternate Instruction Set        = %d (%d)\n",  !!(uEdxGst & RT_BIT( 0)),  !!(uEdxHst & RT_BIT( 0)));
-            pHlp->pfnPrintf(pHlp, "AIS-E - AIS enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 1)),  !!(uEdxHst & RT_BIT( 1)));
-            pHlp->pfnPrintf(pHlp, "RNG - Random Number Generator          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 2)),  !!(uEdxHst & RT_BIT( 2)));
-            pHlp->pfnPrintf(pHlp, "RNG-E - RNG enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 3)),  !!(uEdxHst & RT_BIT( 3)));
-            pHlp->pfnPrintf(pHlp, "LH - LongHaul MSR 0000_110Ah           = %d (%d)\n",  !!(uEdxGst & RT_BIT( 4)),  !!(uEdxHst & RT_BIT( 4)));
-            pHlp->pfnPrintf(pHlp, "FEMMS - FEMMS                          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 5)),  !!(uEdxHst & RT_BIT( 5)));
-            pHlp->pfnPrintf(pHlp, "ACE - Advanced Cryptography Engine     = %d (%d)\n",  !!(uEdxGst & RT_BIT( 6)),  !!(uEdxHst & RT_BIT( 6)));
-            pHlp->pfnPrintf(pHlp, "ACE-E - ACE enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 7)),  !!(uEdxHst & RT_BIT( 7)));
-            /* possibly indicating MM/HE and MM/HE-E on older chips... */
-            pHlp->pfnPrintf(pHlp, "ACE2 - Advanced Cryptography Engine 2  = %d (%d)\n",  !!(uEdxGst & RT_BIT( 8)),  !!(uEdxHst & RT_BIT( 8)));
-            pHlp->pfnPrintf(pHlp, "ACE2-E - ACE enabled                   = %d (%d)\n",  !!(uEdxGst & RT_BIT( 9)),  !!(uEdxHst & RT_BIT( 9)));
-            pHlp->pfnPrintf(pHlp, "PHE - Hash Engine                      = %d (%d)\n",  !!(uEdxGst & RT_BIT(10)),  !!(uEdxHst & RT_BIT(10)));
-            pHlp->pfnPrintf(pHlp, "PHE-E - PHE enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT(11)),  !!(uEdxHst & RT_BIT(11)));
-            pHlp->pfnPrintf(pHlp, "PMM - Montgomery Multiplier            = %d (%d)\n",  !!(uEdxGst & RT_BIT(12)),  !!(uEdxHst & RT_BIT(12)));
-            pHlp->pfnPrintf(pHlp, "PMM-E - PMM enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT(13)),  !!(uEdxHst & RT_BIT(13)));
-            for (unsigned iBit = 14; iBit < 32; iBit++)
-                if ((uEdxGst | uEdxHst) & RT_BIT(iBit))
-                    pHlp->pfnPrintf(pHlp, "Bit %d                                 = %d (%d)\n",  !!(uEdxGst & RT_BIT(iBit)),  !!(uEdxHst & RT_BIT(iBit)));
-            pHlp->pfnPrintf(pHlp, "\n");
-        }
-    }
-}
-
-
-/**
- * Structure used when disassembling and instructions in DBGF.
- * This is used so the reader function can get the stuff it needs.
- */
-typedef struct CPUMDISASSTATE
-{
-    /** Pointer to the CPU structure. */
-    PDISCPUSTATE    pCpu;
-    /** The VM handle. */
-    PVM             pVM;
-    /** The VMCPU handle. */
-    PVMCPU          pVCpu;
-    /** Pointer to the first byte in the segment. */
-    RTGCUINTPTR     GCPtrSegBase;
-    /** Pointer to the byte after the end of the segment. (might have wrapped!) */
-    RTGCUINTPTR     GCPtrSegEnd;
-    /** The size of the segment minus 1. */
-    RTGCUINTPTR     cbSegLimit;
-    /** Pointer to the current page - R3 Ptr. */
-    void const     *pvPageR3;
-    /** Pointer to the current page - GC Ptr. */
-    RTGCPTR         pvPageGC;
-    /** The lock information that PGMPhysReleasePageMappingLock needs. */
-    PGMPAGEMAPLOCK  PageMapLock;
-    /** Whether the PageMapLock is valid or not. */
-    bool            fLocked;
-    /** 64 bits mode or not. */
-    bool            f64Bits;
-} CPUMDISASSTATE, *PCPUMDISASSTATE;
-
-
-/**
- * Instruction reader.
- *
- * @returns VBox status code.
- * @param   PtrSrc      Address to read from.
- *                      In our case this is relative to the selector pointed to by the 2nd user argument of uDisCpu.
- * @param   pu8Dst      Where to store the bytes.
- * @param   cbRead      Number of bytes to read.
- * @param   uDisCpu     Pointer to the disassembler cpu state.
- *                      In this context it's always pointer to the Core of a DBGFDISASSTATE.
- */
-static DECLCALLBACK(int) cpumR3DisasInstrRead(RTUINTPTR PtrSrc, uint8_t *pu8Dst, unsigned cbRead, void *uDisCpu)
-{
-    PDISCPUSTATE pCpu = (PDISCPUSTATE)uDisCpu;
-    PCPUMDISASSTATE pState = (PCPUMDISASSTATE)pCpu->apvUserData[0];
-    Assert(cbRead > 0);
-    for (;;)
-    {
-        RTGCUINTPTR GCPtr = PtrSrc + pState->GCPtrSegBase;
-
-        /* Need to update the page translation? */
-        if (    !pState->pvPageR3
-            ||  (GCPtr >> PAGE_SHIFT) != (pState->pvPageGC >> PAGE_SHIFT))
-        {
-            int rc = VINF_SUCCESS;
-
-            /* translate the address */
-            pState->pvPageGC = GCPtr & PAGE_BASE_GC_MASK;
-            if (    MMHyperIsInsideArea(pState->pVM, pState->pvPageGC)
-                &&  !HWACCMIsEnabled(pState->pVM))
-            {
-                pState->pvPageR3 = MMHyperRCToR3(pState->pVM, (RTRCPTR)pState->pvPageGC);
-                if (!pState->pvPageR3)
-                    rc = VERR_INVALID_POINTER;
-            }
-            else
-            {
-                /* Release mapping lock previously acquired. */
-                if (pState->fLocked)
-                    PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
-                rc = PGMPhysGCPtr2CCPtrReadOnly(pState->pVCpu, pState->pvPageGC, &pState->pvPageR3, &pState->PageMapLock);
-                pState->fLocked = RT_SUCCESS_NP(rc);
-            }
-            if (RT_FAILURE(rc))
-            {
-                pState->pvPageR3 = NULL;
-                return rc;
-            }
-        }
-
-        /* check the segment limit */
-        if (!pState->f64Bits && PtrSrc > pState->cbSegLimit)
-            return VERR_OUT_OF_SELECTOR_BOUNDS;
-
-        /* calc how much we can read */
-        uint32_t cb = PAGE_SIZE - (GCPtr & PAGE_OFFSET_MASK);
-        if (!pState->f64Bits)
-        {
-            RTGCUINTPTR cbSeg = pState->GCPtrSegEnd - GCPtr;
-            if (cb > cbSeg && cbSeg)
-                cb = cbSeg;
-        }
-        if (cb > cbRead)
-            cb = cbRead;
-
-        /* read and advance */
-        memcpy(pu8Dst, (char *)pState->pvPageR3 + (GCPtr & PAGE_OFFSET_MASK), cb);
-        cbRead -= cb;
-        if (!cbRead)
-            return VINF_SUCCESS;
-        pu8Dst += cb;
-        PtrSrc += cb;
-    }
-}
-
-
-/**
- * Disassemble an instruction and return the information in the provided structure.
- *
- * @returns VBox status code.
- * @param   pVM         VM Handle
- * @param   pVCpu       VMCPU Handle
- * @param   pCtx        CPU context
- * @param   GCPtrPC     Program counter (relative to CS) to disassemble from.
- * @param   pCpu        Disassembly state
- * @param   pszPrefix   String prefix for logging (debug only)
- *
- */
-VMMR3DECL(int) CPUMR3DisasmInstrCPU(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTGCPTR GCPtrPC, PDISCPUSTATE pCpu, const char *pszPrefix)
-{
-    CPUMDISASSTATE  State;
-    int             rc;
-
-    const PGMMODE enmMode = PGMGetGuestMode(pVCpu);
-    State.pCpu            = pCpu;
-    State.pvPageGC        = 0;
-    State.pvPageR3        = NULL;
-    State.pVM             = pVM;
-    State.pVCpu           = pVCpu;
-    State.fLocked         = false;
-    State.f64Bits         = false;
-
-    /*
-     * Get selector information.
-     */
-    if (    (pCtx->cr0 & X86_CR0_PE)
-        &&   pCtx->eflags.Bits.u1VM == 0)
-    {
-        if (CPUMAreHiddenSelRegsValid(pVCpu))
-        {
-            State.f64Bits         = enmMode >= PGMMODE_AMD64 && pCtx->csHid.Attr.n.u1Long;
-            State.GCPtrSegBase    = pCtx->csHid.u64Base;
-            State.GCPtrSegEnd     = pCtx->csHid.u32Limit + 1 + (RTGCUINTPTR)pCtx->csHid.u64Base;
-            State.cbSegLimit      = pCtx->csHid.u32Limit;
-            pCpu->mode            = (State.f64Bits)
-                                    ? CPUMODE_64BIT
-                                    : pCtx->csHid.Attr.n.u1DefBig
-                                    ? CPUMODE_32BIT
-                                    : CPUMODE_16BIT;
-        }
-        else
-        {
-            DBGFSELINFO SelInfo;
-
-            rc = SELMR3GetShadowSelectorInfo(pVM, pCtx->cs, &SelInfo);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("SELMR3GetShadowSelectorInfo failed for %04X:%RGv rc=%d\n", pCtx->cs, GCPtrPC, rc));
-                return rc;
-            }
-
-            /*
-             * Validate the selector.
-             */
-            rc = DBGFR3SelInfoValidateCS(&SelInfo, pCtx->ss);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("SELMSelInfoValidateCS failed for %04X:%RGv rc=%d\n", pCtx->cs, GCPtrPC, rc));
-                return rc;
-            }
-            State.GCPtrSegBase    = SelInfo.GCPtrBase;
-            State.GCPtrSegEnd     = SelInfo.cbLimit + 1 + (RTGCUINTPTR)SelInfo.GCPtrBase;
-            State.cbSegLimit      = SelInfo.cbLimit;
-            pCpu->mode            = SelInfo.u.Raw.Gen.u1DefBig ? CPUMODE_32BIT : CPUMODE_16BIT;
-        }
-    }
-    else
-    {
-        /* real or V86 mode */
-        pCpu->mode            = CPUMODE_16BIT;
-        State.GCPtrSegBase    = pCtx->cs * 16;
-        State.GCPtrSegEnd     = 0xFFFFFFFF;
-        State.cbSegLimit      = 0xFFFFFFFF;
-    }
-
-    /*
-     * Disassemble the instruction.
-     */
-    pCpu->pfnReadBytes    = cpumR3DisasInstrRead;
-    pCpu->apvUserData[0]  = &State;
-
-    uint32_t cbInstr;
-#ifndef LOG_ENABLED
-    rc = DISInstr(pCpu, GCPtrPC, 0, &cbInstr, NULL);
-    if (RT_SUCCESS(rc))
-    {
-#else
-    char szOutput[160];
-    rc = DISInstr(pCpu, GCPtrPC, 0, &cbInstr, &szOutput[0]);
-    if (RT_SUCCESS(rc))
-    {
-        /* log it */
-        if (pszPrefix)
-            Log(("%s-CPU%d: %s", pszPrefix, pVCpu->idCpu, szOutput));
-        else
-            Log(("%s", szOutput));
-#endif
-        rc = VINF_SUCCESS;
-    }
-    else
-        Log(("CPUMR3DisasmInstrCPU: DISInstr failed for %04X:%RGv rc=%Rrc\n", pCtx->cs, GCPtrPC, rc));
-
-    /* Release mapping lock acquired in cpumR3DisasInstrRead. */
-    if (State.fLocked)
-        PGMPhysReleasePageMappingLock(pVM, &State.PageMapLock);
-
-    return rc;
-}
-
-#ifdef DEBUG
-
-/**
- * Disassemble an instruction and dump it to the log
- *
- * @returns VBox status code.
- * @param   pVM         VM Handle
- * @param   pVCpu       VMCPU Handle
- * @param   pCtx        CPU context
- * @param   pc          GC instruction pointer
- * @param   pszPrefix   String prefix for logging
- *
- * @deprecated  Use DBGFR3DisasInstrCurrentLog().
- */
-VMMR3DECL(void) CPUMR3DisasmInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTGCPTR pc, const char *pszPrefix)
-{
-    DISCPUSTATE Cpu;
-    CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pc, &Cpu, pszPrefix);
-}
-
-
-/**
- * Debug helper - Saves guest context on raw mode entry (for fatal dump)
- *
- * @internal
- */
-VMMR3DECL(void) CPUMR3SaveEntryCtx(PVM pVM)
-{
-    /** @todo SMP support!! */
-    pVM->cpum.s.GuestEntry = *CPUMQueryGuestCtxPtr(VMMGetCpu(pVM));
-}
-
-#endif /* DEBUG */
-
-/**
- * API for controlling a few of the CPU features found in CR4.
- *
- * Currently only X86_CR4_TSD is accepted as input.
- *
- * @returns VBox status code.
- *
- * @param   pVM     The VM handle.
- * @param   fOr     The CR4 OR mask.
- * @param   fAnd    The CR4 AND mask.
- */
-VMMR3DECL(int) CPUMR3SetCR4Feature(PVM pVM, RTHCUINTREG fOr, RTHCUINTREG fAnd)
-{
-    AssertMsgReturn(!(fOr & ~(X86_CR4_TSD)), ("%#x\n", fOr), VERR_INVALID_PARAMETER);
-    AssertMsgReturn((fAnd & ~(X86_CR4_TSD)) == ~(X86_CR4_TSD), ("%#x\n", fAnd), VERR_INVALID_PARAMETER);
-
-    pVM->cpum.s.CR4.OrMask &= fAnd;
-    pVM->cpum.s.CR4.OrMask |= fOr;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets a pointer to the array of standard CPUID leaves.
- *
- * CPUMR3GetGuestCpuIdStdMax() give the size of the array.
- *
- * @returns Pointer to the standard CPUID leaves (read-only).
- * @param   pVM         The VM handle.
- * @remark  Intended for PATM.
- */
-VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdStdRCPtr(PVM pVM)
-{
-    return RCPTRTYPE(PCCPUMCPUID)VM_RC_ADDR(pVM, &pVM->cpum.s.aGuestCpuIdStd[0]);
-}
-
-
-/**
- * Gets a pointer to the array of extended CPUID leaves.
- *
- * CPUMGetGuestCpuIdExtMax() give the size of the array.
- *
- * @returns Pointer to the extended CPUID leaves (read-only).
- * @param   pVM         The VM handle.
- * @remark  Intended for PATM.
- */
-VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdExtRCPtr(PVM pVM)
-{
-    return (RCPTRTYPE(PCCPUMCPUID))VM_RC_ADDR(pVM, &pVM->cpum.s.aGuestCpuIdExt[0]);
-}
-
-
-/**
- * Gets a pointer to the array of centaur CPUID leaves.
- *
- * CPUMGetGuestCpuIdCentaurMax() give the size of the array.
- *
- * @returns Pointer to the centaur CPUID leaves (read-only).
- * @param   pVM         The VM handle.
- * @remark  Intended for PATM.
- */
-VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdCentaurRCPtr(PVM pVM)
-{
-    return (RCPTRTYPE(PCCPUMCPUID))VM_RC_ADDR(pVM, &pVM->cpum.s.aGuestCpuIdCentaur[0]);
-}
-
-
-/**
- * Gets a pointer to the default CPUID leaf.
- *
- * @returns Pointer to the default CPUID leaf (read-only).
- * @param   pVM         The VM handle.
- * @remark  Intended for PATM.
- */
-VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdDefRCPtr(PVM pVM)
-{
-    return (RCPTRTYPE(PCCPUMCPUID))VM_RC_ADDR(pVM, &pVM->cpum.s.GuestCpuIdDef);
-}
-
-
-/**
- * Transforms the guest CPU state to raw-ring mode.
- *
- * This function will change the any of the cs and ss register with DPL=0 to DPL=1.
- *
- * @returns VBox status. (recompiler failure)
- * @param   pVCpu       The VMCPU handle.
- * @param   pCtxCore    The context core (for trap usage).
- * @see     @ref pg_raw
- */
-VMMR3DECL(int) CPUMR3RawEnter(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore)
-{
-    PVM pVM = pVCpu->CTX_SUFF(pVM);
-
-    Assert(!pVCpu->cpum.s.fRawEntered);
-    Assert(!pVCpu->cpum.s.fRemEntered);
-    if (!pCtxCore)
-        pCtxCore = CPUMCTX2CORE(&pVCpu->cpum.s.Guest);
-
-    /*
-     * Are we in Ring-0?
-     */
-    if (    pCtxCore->ss && (pCtxCore->ss & X86_SEL_RPL) == 0
-        &&  !pCtxCore->eflags.Bits.u1VM)
-    {
-        /*
-         * Enter execution mode.
-         */
-        PATMRawEnter(pVM, pCtxCore);
-
-        /*
-         * Set CPL to Ring-1.
-         */
-        pCtxCore->ss |= 1;
-        if (pCtxCore->cs && (pCtxCore->cs & X86_SEL_RPL) == 0)
-            pCtxCore->cs |= 1;
-    }
-    else
-    {
-        AssertMsg((pCtxCore->ss & X86_SEL_RPL) >= 2 || pCtxCore->eflags.Bits.u1VM,
-                  ("ring-1 code not supported\n"));
-        /*
-         * PATM takes care of IOPL and IF flags for Ring-3 and Ring-2 code as well.
-         */
-        PATMRawEnter(pVM, pCtxCore);
-    }
-
-    /*
-     * Invalidate the hidden registers.
-     */
-    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
-
-    /*
-     * Assert sanity.
-     */
-    AssertMsg((pCtxCore->eflags.u32 & X86_EFL_IF), ("X86_EFL_IF is clear\n"));
-    AssertReleaseMsg(   pCtxCore->eflags.Bits.u2IOPL < (unsigned)(pCtxCore->ss & X86_SEL_RPL)
-                     || pCtxCore->eflags.Bits.u1VM,
-                     ("X86_EFL_IOPL=%d CPL=%d\n", pCtxCore->eflags.Bits.u2IOPL, pCtxCore->ss & X86_SEL_RPL));
-    Assert((pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE)) == (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP));
-
-    pCtxCore->eflags.u32        |= X86_EFL_IF; /* paranoia */
-
-    pVCpu->cpum.s.fRawEntered = true;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Transforms the guest CPU state from raw-ring mode to correct values.
- *
- * This function will change any selector registers with DPL=1 to DPL=0.
- *
- * @returns Adjusted rc.
- * @param   pVCpu       The VMCPU handle.
- * @param   rc          Raw mode return code
- * @param   pCtxCore    The context core (for trap usage).
- * @see     @ref pg_raw
- */
-VMMR3DECL(int) CPUMR3RawLeave(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, int rc)
-{
-    PVM pVM = pVCpu->CTX_SUFF(pVM);
-
-    /*
-     * Don't leave if we've already left (in GC).
-     */
-    Assert(pVCpu->cpum.s.fRawEntered);
-    Assert(!pVCpu->cpum.s.fRemEntered);
-    if (!pVCpu->cpum.s.fRawEntered)
-        return rc;
-    pVCpu->cpum.s.fRawEntered = false;
-
-    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
-    if (!pCtxCore)
-        pCtxCore = CPUMCTX2CORE(pCtx);
-    Assert(pCtxCore->eflags.Bits.u1VM || (pCtxCore->ss & X86_SEL_RPL));
-    AssertMsg(pCtxCore->eflags.Bits.u1VM || pCtxCore->eflags.Bits.u2IOPL < (unsigned)(pCtxCore->ss & X86_SEL_RPL),
-              ("X86_EFL_IOPL=%d CPL=%d\n", pCtxCore->eflags.Bits.u2IOPL, pCtxCore->ss & X86_SEL_RPL));
-
-    /*
-     * Are we executing in raw ring-1?
-     */
-    if (    (pCtxCore->ss & X86_SEL_RPL) == 1
-        &&  !pCtxCore->eflags.Bits.u1VM)
-    {
-        /*
-         * Leave execution mode.
-         */
-        PATMRawLeave(pVM, pCtxCore, rc);
-        /* Not quite sure if this is really required, but shouldn't harm (too much anyways). */
-        /** @todo See what happens if we remove this. */
-        if ((pCtxCore->ds & X86_SEL_RPL) == 1)
-            pCtxCore->ds &= ~X86_SEL_RPL;
-        if ((pCtxCore->es & X86_SEL_RPL) == 1)
-            pCtxCore->es &= ~X86_SEL_RPL;
-        if ((pCtxCore->fs & X86_SEL_RPL) == 1)
-            pCtxCore->fs &= ~X86_SEL_RPL;
-        if ((pCtxCore->gs & X86_SEL_RPL) == 1)
-            pCtxCore->gs &= ~X86_SEL_RPL;
-
-        /*
-         * Ring-1 selector => Ring-0.
-         */
-        pCtxCore->ss &= ~X86_SEL_RPL;
-        if ((pCtxCore->cs & X86_SEL_RPL) == 1)
-            pCtxCore->cs &= ~X86_SEL_RPL;
-    }
-    else
-    {
-        /*
-         * PATM is taking care of the IOPL and IF flags for us.
-         */
-        PATMRawLeave(pVM, pCtxCore, rc);
-        if (!pCtxCore->eflags.Bits.u1VM)
-        {
-            /** @todo See what happens if we remove this. */
-            if ((pCtxCore->ds & X86_SEL_RPL) == 1)
-                pCtxCore->ds &= ~X86_SEL_RPL;
-            if ((pCtxCore->es & X86_SEL_RPL) == 1)
-                pCtxCore->es &= ~X86_SEL_RPL;
-            if ((pCtxCore->fs & X86_SEL_RPL) == 1)
-                pCtxCore->fs &= ~X86_SEL_RPL;
-            if ((pCtxCore->gs & X86_SEL_RPL) == 1)
-                pCtxCore->gs &= ~X86_SEL_RPL;
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * Enters REM, gets and resets the changed flags (CPUM_CHANGED_*).
- *
- * Only REM should ever call this function!
- *
- * @returns The changed flags.
- * @param   pVCpu       The VMCPU handle.
- * @param   puCpl       Where to return the current privilege level (CPL).
- */
-VMMR3DECL(uint32_t) CPUMR3RemEnter(PVMCPU pVCpu, uint32_t *puCpl)
-{
-    Assert(!pVCpu->cpum.s.fRawEntered);
-    Assert(!pVCpu->cpum.s.fRemEntered);
-
-    /*
-     * Get the CPL first.
-     */
-    *puCpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.s.Guest));
-
-    /*
-     * Get and reset the flags, leaving CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID set.
-     */
-    uint32_t fFlags = pVCpu->cpum.s.fChanged;
-    pVCpu->cpum.s.fChanged &= CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID; /* leave it set */
-
-    /** @todo change the switcher to use the fChanged flags. */
-    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_SINCE_REM)
-    {
-        fFlags |= CPUM_CHANGED_FPU_REM;
-        pVCpu->cpum.s.fUseFlags &= ~CPUM_USED_FPU_SINCE_REM;
-    }
-
-    pVCpu->cpum.s.fRemEntered = true;
-    return fFlags;
-}
-
-
-/**
- * Leaves REM and works the CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID flag.
- *
- * @param   pVCpu               The virtual CPU handle.
- * @param   fNoOutOfSyncSels    This is @c false if there are out of sync
- *                              registers.
- */
-VMMR3DECL(void) CPUMR3RemLeave(PVMCPU pVCpu, bool fNoOutOfSyncSels)
-{
-    Assert(!pVCpu->cpum.s.fRawEntered);
-    Assert(pVCpu->cpum.s.fRemEntered);
-
-    if (fNoOutOfSyncSels)
-        pVCpu->cpum.s.fChanged &= ~CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
-    else
-        pVCpu->cpum.s.fChanged |= ~CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
-
-    pVCpu->cpum.s.fRemEntered = false;
-}
-
Index: unk/src/VBox/VMM/CPUMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/CPUMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,430 +1,0 @@
-/* $Id$ */
-/** @file
- * CPUM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___CPUMInternal_h
-#define ___CPUMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/x86.h>
-
-
-
-/** @defgroup grp_cpum_int   Internals
- * @ingroup grp_cpum
- * @internal
- * @{
- */
-
-/** Flags and types for CPUM fault handlers
- * @{ */
-/** Type: Load DS */
-#define CPUM_HANDLER_DS                 1
-/** Type: Load ES */
-#define CPUM_HANDLER_ES                 2
-/** Type: Load FS */
-#define CPUM_HANDLER_FS                 3
-/** Type: Load GS */
-#define CPUM_HANDLER_GS                 4
-/** Type: IRET */
-#define CPUM_HANDLER_IRET               5
-/** Type mask. */
-#define CPUM_HANDLER_TYPEMASK           0xff
-/** If set EBP points to the CPUMCTXCORE that's being used. */
-#define CPUM_HANDLER_CTXCORE_IN_EBP     RT_BIT(31)
-/** @} */
-
-
-/** Use flags (CPUM::fUseFlags).
- * (Don't forget to sync this with CPUMInternal.mac!)
- * @{ */
-/** Used the FPU, SSE or such stuff. */
-#define CPUM_USED_FPU                   RT_BIT(0)
-/** Used the FPU, SSE or such stuff since last we were in REM.
- * REM syncing is clearing this, lazy FPU is setting it. */
-#define CPUM_USED_FPU_SINCE_REM         RT_BIT(1)
-/** Host OS is using SYSENTER and we must NULL the CS. */
-#define CPUM_USE_SYSENTER               RT_BIT(2)
-/** Host OS is using SYSENTER and we must NULL the CS. */
-#define CPUM_USE_SYSCALL                RT_BIT(3)
-/** Debug registers are used by host and must be disabled. */
-#define CPUM_USE_DEBUG_REGS_HOST        RT_BIT(4)
-/** Enabled use of debug registers in guest context. */
-#define CPUM_USE_DEBUG_REGS             RT_BIT(5)
-/** The XMM state was manually restored. (AMD only) */
-#define CPUM_MANUAL_XMM_RESTORE         RT_BIT(6)
-/** Sync the FPU state on entry (32->64 switcher only). */
-#define CPUM_SYNC_FPU_STATE             RT_BIT(7)
-/** Sync the debug state on entry (32->64 switcher only). */
-#define CPUM_SYNC_DEBUG_STATE           RT_BIT(8)
-/** Enabled use of hypervisor debug registers in guest context. */
-#define CPUM_USE_DEBUG_REGS_HYPER       RT_BIT(9)
-/** @} */
-
-/* Sanity check. */
-#if defined(VBOX_WITH_HYBRID_32BIT_KERNEL) && (HC_ARCH_BITS != 32 || R0_ARCH_BITS != 32)
-# error "VBOX_WITH_HYBRID_32BIT_KERNEL is only for 32 bit builds."
-#endif
-
-
-/**
- * The saved host CPU state.
- *
- * @remark  The special VBOX_WITH_HYBRID_32BIT_KERNEL checks here are for the 10.4.x series
- *          of Mac OS X where the OS is essentially 32-bit but the cpu mode can be 64-bit.
- */
-typedef struct CPUMHOSTCTX
-{
-    /** FPU state. (16-byte alignment)
-     * @remark On x86, the format isn't necessarily X86FXSTATE (not important). */
-    X86FXSTATE      fpu;
-
-    /** General purpose register, selectors, flags and more
-     * @{ */
-#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    /** General purpose register ++
-     * { */
-    //uint64_t        rax; - scratch
-    uint64_t        rbx;
-    //uint64_t        rcx; - scratch
-    //uint64_t        rdx; - scratch
-    uint64_t        rdi;
-    uint64_t        rsi;
-    uint64_t        rbp;
-    uint64_t        rsp;
-    //uint64_t        r8; - scratch
-    //uint64_t        r9; - scratch
-    uint64_t        r10;
-    uint64_t        r11;
-    uint64_t        r12;
-    uint64_t        r13;
-    uint64_t        r14;
-    uint64_t        r15;
-    //uint64_t        rip; - scratch
-    uint64_t        rflags;
-#endif
-
-#if HC_ARCH_BITS == 32
-    //uint32_t        eax; - scratch
-    uint32_t        ebx;
-    //uint32_t        ecx; - scratch
-    //uint32_t        edx; - scratch
-    uint32_t        edi;
-    uint32_t        esi;
-    uint32_t        ebp;
-    X86EFLAGS       eflags;
-    //uint32_t        eip; - scratch
-    /* lss pair! */
-    uint32_t        esp;
-#endif
-    /** @} */
-
-    /** Selector registers
-     * @{ */
-    RTSEL           ss;
-    RTSEL           ssPadding;
-    RTSEL           gs;
-    RTSEL           gsPadding;
-    RTSEL           fs;
-    RTSEL           fsPadding;
-    RTSEL           es;
-    RTSEL           esPadding;
-    RTSEL           ds;
-    RTSEL           dsPadding;
-    RTSEL           cs;
-    RTSEL           csPadding;
-    /** @} */
-
-#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    /** Control registers.
-     * @{ */
-    uint32_t        cr0;
-    //uint32_t        cr2; - scratch
-    uint32_t        cr3;
-    uint32_t        cr4;
-    /** @} */
-
-    /** Debug registers.
-     * @{ */
-    uint32_t        dr0;
-    uint32_t        dr1;
-    uint32_t        dr2;
-    uint32_t        dr3;
-    uint32_t        dr6;
-    uint32_t        dr7;
-    /** @} */
-
-    /** Global Descriptor Table register. */
-    X86XDTR32       gdtr;
-    uint16_t        gdtrPadding;
-    /** Interrupt Descriptor Table register. */
-    X86XDTR32       idtr;
-    uint16_t        idtrPadding;
-    /** The task register. */
-    RTSEL           ldtr;
-    RTSEL           ldtrPadding;
-    /** The task register. */
-    RTSEL           tr;
-    RTSEL           trPadding;
-    uint32_t        SysEnterPadding;
-
-    /** The sysenter msr registers.
-     * This member is not used by the hypervisor context. */
-    CPUMSYSENTER    SysEnter;
-
-    /** MSRs
-     * @{ */
-    uint64_t        efer;
-    /** @} */
-
-    /* padding to get 64byte aligned size */
-    uint8_t         auPadding[16+32];
-
-#elif HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-
-    /** Control registers.
-     * @{ */
-    uint64_t        cr0;
-    //uint64_t        cr2; - scratch
-    uint64_t        cr3;
-    uint64_t        cr4;
-    uint64_t        cr8;
-    /** @} */
-
-    /** Debug registers.
-     * @{ */
-    uint64_t        dr0;
-    uint64_t        dr1;
-    uint64_t        dr2;
-    uint64_t        dr3;
-    uint64_t        dr6;
-    uint64_t        dr7;
-    /** @} */
-
-    /** Global Descriptor Table register. */
-    X86XDTR64       gdtr;
-    uint16_t        gdtrPadding;
-    /** Interrupt Descriptor Table register. */
-    X86XDTR64       idtr;
-    uint16_t        idtrPadding;
-    /** The task register. */
-    RTSEL           ldtr;
-    RTSEL           ldtrPadding;
-    /** The task register. */
-    RTSEL           tr;
-    RTSEL           trPadding;
-
-    /** MSRs
-     * @{ */
-    CPUMSYSENTER    SysEnter;
-    uint64_t        FSbase;
-    uint64_t        GSbase;
-    uint64_t        efer;
-    /** @} */
-
-    /* padding to get 32byte aligned size */
-# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
-    uint8_t         auPadding[16];
-# else
-    uint8_t         auPadding[8+32];
-# endif
-
-#else
-# error HC_ARCH_BITS not defined
-#endif
-} CPUMHOSTCTX;
-/** Pointer to the saved host CPU state. */
-typedef CPUMHOSTCTX *PCPUMHOSTCTX;
-
-
-/**
- * CPUM Data (part of VM)
- */
-typedef struct CPUM
-{
-    /** Offset from CPUM to CPUMCPU for the first CPU. */
-    uint32_t                offCPUMCPU0;
-
-    /** Use flags.
-     * These flags indicates which CPU features the host uses.
-     */
-    uint32_t                fHostUseFlags;
-
-    /** Host CPU Features - ECX */
-    struct
-    {
-        /** edx part */
-        X86CPUIDFEATEDX     edx;
-        /** ecx part */
-        X86CPUIDFEATECX     ecx;
-    } CPUFeatures;
-    /** Host extended CPU features. */
-    struct
-    {
-        /** edx part */
-        uint32_t            edx;
-        /** ecx part */
-        uint32_t            ecx;
-    } CPUFeaturesExt;
-
-    /** Host CPU manufacturer. */
-    CPUMCPUVENDOR           enmHostCpuVendor;
-    /** Guest CPU manufacturer. */
-    CPUMCPUVENDOR           enmGuestCpuVendor;
-
-    /** CR4 mask */
-    struct
-    {
-        uint32_t            AndMask;
-        uint32_t            OrMask;
-    } CR4;
-
-    /** Synthetic CPU type? */
-    bool                    fSyntheticCpu;
-    /** The (more) portable CPUID level.  */
-    uint8_t                 u8PortableCpuIdLevel;
-    /** Indicates that a state restore is pending.
-     * This is used to verify load order dependencies (PGM). */
-    bool                    fPendingRestore;
-    uint8_t                 abPadding[HC_ARCH_BITS == 64 ? 5 : 1];
-
-    /** The standard set of CpuId leafs. */
-    CPUMCPUID               aGuestCpuIdStd[6];
-    /** The extended set of CpuId leafs. */
-    CPUMCPUID               aGuestCpuIdExt[10];
-    /** The centaur set of CpuId leafs. */
-    CPUMCPUID               aGuestCpuIdCentaur[4];
-    /** The default set of CpuId leafs. */
-    CPUMCPUID               GuestCpuIdDef;
-
-#if HC_ARCH_BITS == 32
-    uint8_t                 abPadding2[4];
-#endif
-
-#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
-    RTHCPTR                 pvApicBase;
-    uint32_t                fApicDisVectors;
-    uint8_t                 abPadding3[HC_ARCH_BITS == 32 ? 56 : 52];
-#endif
-
-    /**
-     * Guest context on raw mode entry. 64-byte aligned!
-     * This a debug feature, see CPUMR3SaveEntryCtx.
-     */
-    CPUMCTX                 GuestEntry;
-} CPUM;
-/** Pointer to the CPUM instance data residing in the shared VM structure. */
-typedef CPUM *PCPUM;
-
-/**
- * CPUM Data (part of VMCPU)
- */
-typedef struct CPUMCPU
-{
-    /**
-     * Hypervisor context.
-     * Aligned on a 64-byte boundary.
-     */
-    CPUMCTX                 Hyper;
-
-    /**
-     * Saved host context. Only valid while inside GC.
-     * Aligned on a 64-byte boundary.
-     */
-    CPUMHOSTCTX             Host;
-
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-    uint8_t                 aMagic[56];
-    uint64_t                uMagic;
-#endif
-
-    /**
-     * Guest context.
-     * Aligned on a 64-byte boundary.
-     */
-    CPUMCTX                 Guest;
-
-    /**
-     * Guest context - misc MSRs
-     * Aligned on a 64-byte boundary.
-     */
-    CPUMCTXMSR              GuestMsr;
-
-    /** Pointer to the current hypervisor core context - R3Ptr. */
-    R3PTRTYPE(PCPUMCTXCORE) pHyperCoreR3;
-    /** Pointer to the current hypervisor core context - R0Ptr. */
-    R0PTRTYPE(PCPUMCTXCORE) pHyperCoreR0;
-    /** Pointer to the current hypervisor core context - RCPtr. */
-    RCPTRTYPE(PCPUMCTXCORE) pHyperCoreRC;
-
-    /** Use flags.
-     * These flags indicates both what is to be used and what has been used.
-     */
-    uint32_t                fUseFlags;
-
-    /** Changed flags.
-     * These flags indicates to REM (and others) which important guest
-     * registers which has been changed since last time the flags were cleared.
-     * See the CPUM_CHANGED_* defines for what we keep track of.
-     */
-    uint32_t                fChanged;
-
-    /** Offset from CPUM to CPUMCPU. */
-    uint32_t                offCPUM;
-
-    /** Temporary storage for the return code of the function called in the
-     * 32-64 switcher. */
-    uint32_t                u32RetCode;
-
-    /** Have we entered raw-mode? */
-    bool                    fRawEntered;
-    /** Have we entered the recompiler? */
-    bool                    fRemEntered;
-
-    /** Align the structure on a 64-byte boundary. */
-    uint8_t                 abPadding2[HC_ARCH_BITS == 32 ? 34 : 26];
-} CPUMCPU, *PCPUMCPU;
-/** Pointer to the CPUMCPU instance data residing in the shared VMCPU structure. */
-typedef CPUMCPU *PCPUMCPU;
-
-RT_C_DECLS_BEGIN
-
-DECLASM(int)      cpumHandleLazyFPUAsm(PCPUMCPU pCPUM);
-
-#ifdef IN_RING0
-DECLASM(int)      cpumR0SaveHostRestoreGuestFPUState(PCPUMCPU pCPUM);
-DECLASM(int)      cpumR0SaveGuestRestoreHostFPUState(PCPUMCPU pCPUM);
-DECLASM(int)      cpumR0SaveHostFPUState(PCPUMCPU pCPUM);
-DECLASM(int)      cpumR0RestoreHostFPUState(PCPUMCPU pCPUM);
-DECLASM(void)     cpumR0LoadFPU(PCPUMCTX pCtx);
-DECLASM(void)     cpumR0SaveFPU(PCPUMCTX pCtx);
-DECLASM(void)     cpumR0LoadXMM(PCPUMCTX pCtx);
-DECLASM(void)     cpumR0SaveXMM(PCPUMCTX pCtx);
-DECLASM(void)     cpumR0SetFCW(uint16_t u16FCW);
-DECLASM(uint16_t) cpumR0GetFCW(void);
-DECLASM(void)     cpumR0SetMXCSR(uint32_t u32MXCSR);
-DECLASM(uint32_t) cpumR0GetMXCSR(void);
-DECLASM(void)     cpumR0LoadDRx(uint64_t const *pa4Regs);
-DECLASM(void)     cpumR0SaveDRx(uint64_t *pa4Regs);
-#endif
-
-RT_C_DECLS_END
-
-/** @} */
-
-#endif
-
Index: unk/src/VBox/VMM/CPUMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/CPUMInternal.mac	(revision 35332)
+++ 	(revision )
@@ -1,468 +1,0 @@
-; $Id$
-;; @file
-; CPUM - Internal header file (asm).
-;
-
-;
-; Copyright (C) 2006-2010 Oracle Corporation
-;
-; This file is part of VirtualBox Open Source Edition (OSE), as
-; available from http://www.virtualbox.org. This file is free software;
-; you can redistribute it and/or modify it under the terms of the GNU
-; General Public License (GPL) as published by the Free Software
-; Foundation, in version 2 as it comes in the "COPYING" file of the
-; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
-; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
-;
-
-%include "VBox/asmdefs.mac"
-
-%define CPUM_USED_FPU                   RT_BIT(0)
-%define CPUM_USED_FPU_SINCE_REM         RT_BIT(1)
-%define CPUM_USE_SYSENTER               RT_BIT(2)
-%define CPUM_USE_SYSCALL                RT_BIT(3)
-%define CPUM_USE_DEBUG_REGS_HOST        RT_BIT(4)
-%define CPUM_USE_DEBUG_REGS             RT_BIT(5)
-%define CPUM_SYNC_FPU_STATE             RT_BIT(7)
-%define CPUM_SYNC_DEBUG_STATE           RT_BIT(8)
-
-%define CPUM_HANDLER_DS                 1
-%define CPUM_HANDLER_ES                 2
-%define CPUM_HANDLER_FS                 3
-%define CPUM_HANDLER_GS                 4
-%define CPUM_HANDLER_IRET               5
-%define CPUM_HANDLER_TYPEMASK           0ffh
-%define CPUM_HANDLER_CTXCORE_IN_EBP     RT_BIT(31)
-
-%define VMMGCRET_USED_FPU               040000000h
-
-%define FPUSTATE_SIZE                   512
-
-;; if anyone figures how to do %if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) in
-; nasm please tell / fix this hack.
-%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
- %define fVBOX_WITH_HYBRID_32BIT_KERNEL 1
-%else
- %define fVBOX_WITH_HYBRID_32BIT_KERNEL 0
-%endif
-
-struc CPUM
-    ;...
-    .offCPUMCPU0          resd    1
-    .fHostUseFlags        resd    1
-
-    ; CPUID eax=1
-    .CPUFeatures.edx      resd    1
-    .CPUFeatures.ecx      resd    1
-
-    ; CPUID eax=0x80000001
-    .CPUFeaturesExt.edx   resd    1
-    .CPUFeaturesExt.ecx   resd    1
-
-    .enmHostCpuVendor     resd    1
-    .enmGuestCpuVendor    resd    1
-
-    ; CR4 masks
-    .CR4.AndMask          resd    1
-    .CR4.OrMask           resd    1
-    ; entered rawmode?
-    .fSyntheticCpu        resb    1
-    .u8PortableCpuIdLevel resb    1
-    .fPendingRestore      resb    1
-%if RTHCPTR_CB == 8
-    .abPadding            resb    5
-%else
-    .abPadding            resb    1
-%endif
-
-    ; CPUID leafs
-    .aGuestCpuIdStd       resb    16*6
-    .aGuestCpuIdExt       resb    16*10
-    .aGuestCpuIdCentaur   resb    16*4
-    .GuestCpuIdDef        resb    16
-
-%if HC_ARCH_BITS == 32
-    .abPadding2           resb    4
-%endif
-
-%ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
-    .pvApicBase           RTR0PTR_RES 1
-    .fApicDisVectors      resd    1
-%endif
-
-    alignb 64
-    ; CPUMCTX debug stuff...
-    .GuestEntry           resb   1024
-endstruc
-
-struc CPUMCPU
-    ;
-    ; Hypervisor Context.
-    ;
-    alignb 64                           ; the padding
-    .Hyper.fpu            resb    FPUSTATE_SIZE
-
-    .Hyper.edi            resq    1
-    .Hyper.esi            resq    1
-    .Hyper.ebp            resq    1
-    .Hyper.eax            resq    1
-    .Hyper.ebx            resq    1
-    .Hyper.edx            resq    1
-    .Hyper.ecx            resq    1
-    .Hyper.esp            resq    1
-    .Hyper.lss_esp        resd    1
-    .Hyper.ss             resw    1
-    .Hyper.ssPadding      resw    1
-    .Hyper.gs             resw    1
-    .Hyper.gsPadding      resw    1
-    .Hyper.fs             resw    1
-    .Hyper.fsPadding      resw    1
-    .Hyper.es             resw    1
-    .Hyper.esPadding      resw    1
-    .Hyper.ds             resw    1
-    .Hyper.dsPadding      resw    1
-    .Hyper.cs             resw    1
-    .Hyper.csPadding      resw    3
-    .Hyper.eflags         resq    1
-    .Hyper.eip            resq    1
-    .Hyper.r8             resq    1
-    .Hyper.r9             resq    1
-    .Hyper.r10            resq    1
-    .Hyper.r11            resq    1
-    .Hyper.r12            resq    1
-    .Hyper.r13            resq    1
-    .Hyper.r14            resq    1
-    .Hyper.r15            resq    1
-
-    .Hyper.esHid.u64Base      resq    1
-    .Hyper.esHid.u32Limit     resd    1
-    .Hyper.esHid.Attr         resd    1
-
-    .Hyper.csHid.u64Base      resq    1
-    .Hyper.csHid.u32Limit     resd    1
-    .Hyper.csHid.Attr         resd    1
-
-    .Hyper.ssHid.u64Base      resq    1
-    .Hyper.ssHid.u32Limit     resd    1
-    .Hyper.ssHid.Attr         resd    1
-
-    .Hyper.dsHid.u64Base      resq    1
-    .Hyper.dsHid.u32Limit     resd    1
-    .Hyper.dsHid.Attr         resd    1
-
-    .Hyper.fsHid.u64Base      resq    1
-    .Hyper.fsHid.u32Limit     resd    1
-    .Hyper.fsHid.Attr         resd    1
-
-    .Hyper.gsHid.u64Base      resq    1
-    .Hyper.gsHid.u32Limit     resd    1
-    .Hyper.gsHid.Attr         resd    1
-
-    .Hyper.cr0            resq    1
-    .Hyper.cr2            resq    1
-    .Hyper.cr3            resq    1
-    .Hyper.cr4            resq    1
-
-    .Hyper.dr             resq    8
-
-    .Hyper.gdtr           resb    10          ; GDT limit + linear address
-    .Hyper.gdtrPadding    resw    1
-    .Hyper.idtr           resb    10          ; IDT limit + linear address
-    .Hyper.idtrPadding    resw    1
-    .Hyper.ldtr           resw    1
-    .Hyper.ldtrPadding    resw    1
-    .Hyper.tr             resw    1
-    .Hyper.trPadding      resw    1
-
-    .Hyper.SysEnter.cs        resb    8
-    .Hyper.SysEnter.eip       resb    8
-    .Hyper.SysEnter.esp       resb    8
-
-    .Hyper.msrEFER            resb    8
-    .Hyper.msrSTAR            resb    8
-    .Hyper.msrPAT             resb    8
-    .Hyper.msrLSTAR           resb    8
-    .Hyper.msrCSTAR           resb    8
-    .Hyper.msrSFMASK          resb    8
-    .Hyper.msrKERNELGSBASE    resb    8
-
-    .Hyper.ldtrHid.u64Base    resq    1
-    .Hyper.ldtrHid.u32Limit   resd    1
-    .Hyper.ldtrHid.Attr       resd    1
-
-    .Hyper.trHid.u64Base      resq    1
-    .Hyper.trHid.u32Limit     resd    1
-    .Hyper.trHid.Attr         resd    1
-
-    ;
-    ; Host context state
-    ;
-    .Host.fpu            resb    FPUSTATE_SIZE
-
-%if HC_ARCH_BITS == 64 || fVBOX_WITH_HYBRID_32BIT_KERNEL
-    ;.Host.rax            resq    1 - scratch
-    .Host.rbx            resq    1
-    ;.Host.rcx            resq    1 - scratch
-    ;.Host.rdx            resq    1 - scratch
-    .Host.rdi            resq    1
-    .Host.rsi            resq    1
-    .Host.rbp            resq    1
-    .Host.rsp            resq    1
-    ;.Host.r8             resq    1 - scratch
-    ;.Host.r9             resq    1 - scratch
-    .Host.r10            resq    1
-    .Host.r11            resq    1
-    .Host.r12            resq    1
-    .Host.r13            resq    1
-    .Host.r14            resq    1
-    .Host.r15            resq    1
-    ;.Host.rip            resd    1 - scratch
-    .Host.rflags         resq    1
-%endif
-%if HC_ARCH_BITS == 32
-    ;.Host.eax            resd    1 - scratch
-    .Host.ebx            resd    1
-    ;.Host.edx            resd    1 - scratch
-    ;.Host.ecx            resd    1 - scratch
-    .Host.edi            resd    1
-    .Host.esi            resd    1
-    .Host.ebp            resd    1
-    .Host.eflags         resd    1
-    ;.Host.eip            resd    1 - scratch
-    ; lss pair!
-    .Host.esp            resd    1
-%endif
-    .Host.ss             resw    1
-    .Host.ssPadding      resw    1
-    .Host.gs             resw    1
-    .Host.gsPadding      resw    1
-    .Host.fs             resw    1
-    .Host.fsPadding      resw    1
-    .Host.es             resw    1
-    .Host.esPadding      resw    1
-    .Host.ds             resw    1
-    .Host.dsPadding      resw    1
-    .Host.cs             resw    1
-    .Host.csPadding      resw    1
-
-%if HC_ARCH_BITS == 32 && fVBOX_WITH_HYBRID_32BIT_KERNEL == 0
-    .Host.cr0            resd    1
-    ;.Host.cr2            resd    1 - scratch
-    .Host.cr3            resd    1
-    .Host.cr4            resd    1
-
-    .Host.dr0            resd    1
-    .Host.dr1            resd    1
-    .Host.dr2            resd    1
-    .Host.dr3            resd    1
-    .Host.dr6            resd    1
-    .Host.dr7            resd    1
-
-    .Host.gdtr           resb    6           ; GDT limit + linear address
-    .Host.gdtrPadding    resw    1
-    .Host.idtr           resb    6           ; IDT limit + linear address
-    .Host.idtrPadding    resw    1
-    .Host.ldtr           resw    1
-    .Host.ldtrPadding    resw    1
-    .Host.tr             resw    1
-    .Host.trPadding      resw    1
-
-    .Host.SysEnterPadding resd   1
-    .Host.SysEnter.cs    resq    1
-    .Host.SysEnter.eip   resq    1
-    .Host.SysEnter.esp   resq    1
-    .Host.efer           resq    1
-
-%else ; 64-bit
-
-    .Host.cr0            resq    1
-    ;.Host.cr2            resq    1 - scratch
-    .Host.cr3            resq    1
-    .Host.cr4            resq    1
-    .Host.cr8            resq    1
-
-    .Host.dr0            resq    1
-    .Host.dr1            resq    1
-    .Host.dr2            resq    1
-    .Host.dr3            resq    1
-    .Host.dr6            resq    1
-    .Host.dr7            resq    1
-
-    .Host.gdtr           resb    10         ; GDT limit + linear address
-    .Host.gdtrPadding    resw    1
-    .Host.idtr           resb    10         ; IDT limit + linear address
-    .Host.idtrPadding    resw    1
-    .Host.ldtr           resw    1
-    .Host.ldtrPadding    resw    1
-    .Host.tr             resw    1
-    .Host.trPadding      resw    1
-
-    .Host.SysEnter.cs    resq    1
-    .Host.SysEnter.eip   resq    1
-    .Host.SysEnter.esp   resq    1
-    .Host.FSbase         resq    1
-    .Host.GSbase         resq    1
-    .Host.efer           resq    1
-%endif ; 64-bit
-
-%ifdef VBOX_WITH_CRASHDUMP_MAGIC
-    .aMagic              resb    56
-    .uMagic              resq    1
-%endif
-    ;
-    ; Guest context state
-    ; (Identical to the .Hyper chunk above.)
-    ;
-    alignb 64
-    .Guest.fpu            resb    FPUSTATE_SIZE
-
-    .Guest.edi            resq    1
-    .Guest.esi            resq    1
-    .Guest.ebp            resq    1
-    .Guest.eax            resq    1
-    .Guest.ebx            resq    1
-    .Guest.edx            resq    1
-    .Guest.ecx            resq    1
-    .Guest.esp            resq    1
-    .Guest.lss_esp        resd    1
-    .Guest.ss             resw    1
-    .Guest.ssPadding      resw    1
-    .Guest.gs             resw    1
-    .Guest.gsPadding      resw    1
-    .Guest.fs             resw    1
-    .Guest.fsPadding      resw    1
-    .Guest.es             resw    1
-    .Guest.esPadding      resw    1
-    .Guest.ds             resw    1
-    .Guest.dsPadding      resw    1
-    .Guest.cs             resw    1
-    .Guest.csPadding      resw    3
-    .Guest.eflags         resq    1
-    .Guest.eip            resq    1
-    .Guest.r8             resq    1
-    .Guest.r9             resq    1
-    .Guest.r10            resq    1
-    .Guest.r11            resq    1
-    .Guest.r12            resq    1
-    .Guest.r13            resq    1
-    .Guest.r14            resq    1
-    .Guest.r15            resq    1
-
-    .Guest.esHid.u64Base      resq    1
-    .Guest.esHid.u32Limit     resd    1
-    .Guest.esHid.Attr         resd    1
-
-    .Guest.csHid.u64Base      resq    1
-    .Guest.csHid.u32Limit     resd    1
-    .Guest.csHid.Attr         resd    1
-
-    .Guest.ssHid.u64Base      resq    1
-    .Guest.ssHid.u32Limit     resd    1
-    .Guest.ssHid.Attr         resd    1
-
-    .Guest.dsHid.u64Base      resq    1
-    .Guest.dsHid.u32Limit     resd    1
-    .Guest.dsHid.Attr         resd    1
-
-    .Guest.fsHid.u64Base      resq    1
-    .Guest.fsHid.u32Limit     resd    1
-    .Guest.fsHid.Attr         resd    1
-
-    .Guest.gsHid.u64Base      resq    1
-    .Guest.gsHid.u32Limit     resd    1
-    .Guest.gsHid.Attr         resd    1
-
-    .Guest.cr0            resq    1
-    .Guest.cr2            resq    1
-    .Guest.cr3            resq    1
-    .Guest.cr4            resq    1
-
-    .Guest.dr             resq    8
-
-    .Guest.gdtr.cbGdt     resw    1
-    .Guest.gdtr.pGdt      resq    1
-    .Guest.gdtrPadding    resw    1
-    .Guest.idtr.cbIdt     resw    1
-    .Guest.idtr.pIdt      resq    1
-    .Guest.idtrPadding    resw    1
-    .Guest.ldtr           resw    1
-    .Guest.ldtrPadding    resw    1
-    .Guest.tr             resw    1
-    .Guest.trPadding      resw    1
-
-    .Guest.SysEnter.cs        resb    8
-    .Guest.SysEnter.eip       resb    8
-    .Guest.SysEnter.esp       resb    8
-
-    .Guest.msrEFER            resb    8
-    .Guest.msrSTAR            resb    8
-    .Guest.msrPAT             resb    8
-    .Guest.msrLSTAR           resb    8
-    .Guest.msrCSTAR           resb    8
-    .Guest.msrSFMASK          resb    8
-    .Guest.msrKERNELGSBASE    resb    8
-
-    .Guest.ldtrHid.u64Base    resq    1
-    .Guest.ldtrHid.u32Limit   resd    1
-    .Guest.ldtrHid.Attr       resd    1
-
-    .Guest.trHid.u64Base      resq    1
-    .Guest.trHid.u32Limit     resd    1
-    .Guest.trHid.Attr         resd    1
-
-    .GuestMsr.au64            resq    64
-
-    ;
-    ; Other stuff.
-    ;
-    alignb 64
-    ; hypervisor core context.
-    .pHyperCoreR3         RTR3PTR_RES   1
-    .pHyperCoreR0         RTR0PTR_RES   1
-    .pHyperCoreRC         RTRCPTR_RES   1
-
-    .fUseFlags            resd    1
-    .fChanged             resd    1
-    .offCPUM              resd    1
-    .u32RetCode           resd    1
-    .fRawEntered          resb    1
-    .fRemEntered          resb    1
-
-%if RTHCPTR_CB == 8
-    .abPadding2           resb    26
-%else
-    .abPadding2           resb    34
-%endif
-
-endstruc
-
-
-;;
-; Converts the CPUM pointer to CPUMCPU
-; @param   %1   register name
-%macro CPUMCPU_FROM_CPUM 1
-    add     %1, dword [%1 + CPUM.offCPUMCPU0]
-%endmacro
-
-;;
-; Converts the CPUM pointer to CPUMCPU
-; @param   %1   register name (PVM)
-; @param   %2   register name (CPUMCPU offset)
-%macro CPUMCPU_FROM_CPUM_WITH_OFFSET 2
-    add     %1, %2
-%endmacro
-
-;;
-; Converts the CPUMCPU pointer to CPUM
-; @param   %1   register name
-%macro CPUM_FROM_CPUMCPU 1
-    sub     %1, dword [%1 + CPUMCPU.offCPUM]
-%endmacro
-
-;;
-; Converts the CPUMCPU pointer to CPUM
-; @param   %1   register name (PVM)
-; @param   %2   register name (CPUMCPU offset)
-%macro CPUM_FROM_CPUMCPU_WITH_OFFSET 2
-    sub     %1, %2
-%endmacro
Index: unk/src/VBox/VMM/DBGF.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGF.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1118 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page   pg_dbgf     DBGF - The Debugger Facility
- *
- * The purpose of the DBGF is to provide an interface for debuggers to
- * manipulate the VMM without having to mess up the source code for each of
- * them. The DBGF is always built in and will always work when a debugger
- * attaches to the VM. The DBGF provides the basic debugger features, such as
- * halting execution, handling breakpoints, single step execution, instruction
- * disassembly, info querying, OS specific diggers, symbol and module
- * management.
- *
- * The interface is working in a manner similar to the win32, linux and os2
- * debugger interfaces. The interface has an asynchronous nature. This comes
- * from the fact that the VMM and the Debugger are running in different threads.
- * They are referred to as the "emulation thread" and the "debugger thread", or
- * as the "ping thread" and the "pong thread, respectivly. (The last set of
- * names comes from the use of the Ping-Pong synchronization construct from the
- * RTSem API.)
- *
- * @see grp_dbgf
- *
- *
- * @section sec_dbgf_scenario   Usage Scenario
- *
- * The debugger starts by attaching to the VM. For practical reasons we limit the
- * number of concurrently attached debuggers to 1 per VM. The action of
- * attaching to the VM causes the VM to check and generate debug events.
- *
- * The debugger then will wait/poll for debug events and issue commands.
- *
- * The waiting and polling is done by the DBGFEventWait() function. It will wait
- * for the emulation thread to send a ping, thus indicating that there is an
- * event waiting to be processed.
- *
- * An event can be a response to a command issued previously, the hitting of a
- * breakpoint, or running into a bad/fatal VMM condition. The debugger now has
- * the ping and must respond to the event at hand - the VMM is waiting. This
- * usually means that the user of the debugger must do something, but it doesn't
- * have to. The debugger is free to call any DBGF function (nearly at least)
- * while processing the event.
- *
- * Typically the user will issue a request for the execution to be resumed, so
- * the debugger calls DBGFResume() and goes back to waiting/polling for events.
- *
- * When the user eventually terminates the debugging session or selects another
- * VM, the debugger detaches from the VM. This means that breakpoints are
- * disabled and that the emulation thread no longer polls for debugger commands.
- *
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/selm.h>
-#include <VBox/rem.h>
-#include <VBox/em.h>
-#include <VBox/hwaccm.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/semaphore.h>
-#include <iprt/thread.h>
-#include <iprt/asm.h>
-#include <iprt/time.h>
-#include <iprt/assert.h>
-#include <iprt/stream.h>
-#include <iprt/env.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int dbgfR3VMMWait(PVM pVM);
-static int dbgfR3VMMCmd(PVM pVM, DBGFCMD enmCmd, PDBGFCMDDATA pCmdData, bool *pfResumeExecution);
-static DECLCALLBACK(int) dbgfR3Attach(PVM pVM);
-
-
-/**
- * Sets the VMM Debug Command variable.
- *
- * @returns Previous command.
- * @param   pVM     VM Handle.
- * @param   enmCmd  The command.
- */
-DECLINLINE(DBGFCMD) dbgfR3SetCmd(PVM pVM, DBGFCMD enmCmd)
-{
-    DBGFCMD rc;
-    if (enmCmd == DBGFCMD_NO_COMMAND)
-    {
-        Log2(("DBGF: Setting command to %d (DBGFCMD_NO_COMMAND)\n", enmCmd));
-        rc = (DBGFCMD)ASMAtomicXchgU32((uint32_t volatile *)(void *)&pVM->dbgf.s.enmVMMCmd, enmCmd);
-        VM_FF_CLEAR(pVM, VM_FF_DBGF);
-    }
-    else
-    {
-        Log2(("DBGF: Setting command to %d\n", enmCmd));
-        AssertMsg(pVM->dbgf.s.enmVMMCmd == DBGFCMD_NO_COMMAND, ("enmCmd=%d enmVMMCmd=%d\n", enmCmd, pVM->dbgf.s.enmVMMCmd));
-        rc = (DBGFCMD)ASMAtomicXchgU32((uint32_t volatile *)(void *)&pVM->dbgf.s.enmVMMCmd, enmCmd);
-        VM_FF_SET(pVM, VM_FF_DBGF);
-        VMR3NotifyGlobalFFU(pVM->pUVM, 0 /* didn't notify REM */);
-    }
-    return rc;
-}
-
-
-/**
- * Initializes the DBGF.
- *
- * @returns VBox status code.
- * @param   pVM     VM handle.
- */
-VMMR3DECL(int) DBGFR3Init(PVM pVM)
-{
-    int rc = dbgfR3InfoInit(pVM);
-    if (RT_SUCCESS(rc))
-        rc = dbgfR3AsInit(pVM);
-    if (RT_SUCCESS(rc))
-        rc = dbgfR3SymInit(pVM);
-    if (RT_SUCCESS(rc))
-        rc = dbgfR3BpInit(pVM);
-    return rc;
-}
-
-
-/**
- * Terminates and cleans up resources allocated by the DBGF.
- *
- * @returns VBox status code.
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(int) DBGFR3Term(PVM pVM)
-{
-    int rc;
-
-    /*
-     * Send a termination event to any attached debugger.
-     */
-    /* wait to become the speaker (we should already be that). */
-    if (    pVM->dbgf.s.fAttached
-        &&  RTSemPingShouldWait(&pVM->dbgf.s.PingPong))
-        RTSemPingWait(&pVM->dbgf.s.PingPong, 5000);
-
-    /* now, send the event if we're the speaker. */
-    if (    pVM->dbgf.s.fAttached
-        &&  RTSemPingIsSpeaker(&pVM->dbgf.s.PingPong))
-    {
-        DBGFCMD enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
-        if (enmCmd == DBGFCMD_DETACH_DEBUGGER)
-            /* the debugger beat us to initiating the detaching. */
-            rc = VINF_SUCCESS;
-        else
-        {
-            /* ignore the command (if any). */
-            enmCmd = DBGFCMD_NO_COMMAND;
-            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_TERMINATING;
-            pVM->dbgf.s.DbgEvent.enmCtx  = DBGFEVENTCTX_OTHER;
-            rc = RTSemPing(&pVM->dbgf.s.PingPong);
-        }
-
-        /*
-         * Process commands until we get a detached command.
-         */
-        while (RT_SUCCESS(rc) && enmCmd != DBGFCMD_DETACHED_DEBUGGER)
-        {
-            if (enmCmd != DBGFCMD_NO_COMMAND)
-            {
-                /* process command */
-                bool fResumeExecution;
-                DBGFCMDDATA CmdData = pVM->dbgf.s.VMMCmdData;
-                rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
-                enmCmd = DBGFCMD_NO_COMMAND;
-            }
-            else
-            {
-                /* wait for new command. */
-                rc = RTSemPingWait(&pVM->dbgf.s.PingPong, RT_INDEFINITE_WAIT);
-                if (RT_SUCCESS(rc))
-                    enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
-            }
-        }
-    }
-
-    /*
-     * Terminate the other bits.
-     */
-    dbgfR3OSTerm(pVM);
-    dbgfR3AsTerm(pVM);
-    dbgfR3InfoTerm(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM         VM handle.
- * @param   offDelta    Relocation delta relative to old location.
- */
-VMMR3DECL(void) DBGFR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    dbgfR3AsRelocate(pVM, offDelta);
-}
-
-
-/**
- * Waits a little while for a debuggger to attach.
- *
- * @returns True is a debugger have attached.
- * @param   pVM         VM handle.
- * @param   enmEvent    Event.
- */
-bool dbgfR3WaitForAttach(PVM pVM, DBGFEVENTTYPE enmEvent)
-{
-    /*
-     * First a message.
-     */
-#ifndef RT_OS_L4
-
-# if !defined(DEBUG) || defined(DEBUG_sandervl) || defined(DEBUG_frank)
-    int cWait = 10;
-# else
-    int cWait = HWACCMIsEnabled(pVM)
-             && (   enmEvent == DBGFEVENT_ASSERTION_HYPER
-                 || enmEvent == DBGFEVENT_FATAL_ERROR)
-             && !RTEnvExist("VBOX_DBGF_WAIT_FOR_ATTACH")
-              ? 10
-              : 150;
-# endif
-    RTStrmPrintf(g_pStdErr, "DBGF: No debugger attached, waiting %d second%s for one to attach (event=%d)\n",
-                 cWait / 10, cWait != 10 ? "s" : "", enmEvent);
-    RTStrmFlush(g_pStdErr);
-    while (cWait > 0)
-    {
-        RTThreadSleep(100);
-        if (pVM->dbgf.s.fAttached)
-        {
-            RTStrmPrintf(g_pStdErr, "Attached!\n");
-            RTStrmFlush(g_pStdErr);
-            return true;
-        }
-
-        /* next */
-        if (!(cWait % 10))
-        {
-            RTStrmPrintf(g_pStdErr, "%d.", cWait / 10);
-            RTStrmFlush(g_pStdErr);
-        }
-        cWait--;
-    }
-#endif
-
-    RTStrmPrintf(g_pStdErr, "Stopping the VM!\n");
-    RTStrmFlush(g_pStdErr);
-    return false;
-}
-
-
-/**
- * Forced action callback.
- * The VMM will call this from it's main loop when VM_FF_DBGF is set.
- *
- * The function checks and executes pending commands from the debugger.
- *
- * @returns VINF_SUCCESS normally.
- * @returns VERR_DBGF_RAISE_FATAL_ERROR to pretend a fatal error happened.
- * @param   pVM         VM Handle.
- */
-VMMR3DECL(int) DBGFR3VMMForcedAction(PVM pVM)
-{
-    int rc = VINF_SUCCESS;
-
-    if (VM_FF_TESTANDCLEAR(pVM, VM_FF_DBGF))
-    {
-        PVMCPU pVCpu = VMMGetCpu(pVM);
-
-        /*
-         * Commands?
-         */
-        if (pVM->dbgf.s.enmVMMCmd != DBGFCMD_NO_COMMAND)
-        {
-            /** @todo stupid GDT/LDT sync hack. go away! */
-            SELMR3UpdateFromCPUM(pVM, pVCpu);
-
-            /*
-             * Process the command.
-             */
-            bool            fResumeExecution;
-            DBGFCMDDATA     CmdData = pVM->dbgf.s.VMMCmdData;
-            DBGFCMD         enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
-            rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
-            if (!fResumeExecution)
-                rc = dbgfR3VMMWait(pVM);
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Flag whether the event implies that we're stopped in the hypervisor code
- * and have to block certain operations.
- *
- * @param   pVM         The VM handle.
- * @param   enmEvent    The event.
- */
-static void dbgfR3EventSetStoppedInHyperFlag(PVM pVM, DBGFEVENTTYPE enmEvent)
-{
-    switch (enmEvent)
-    {
-        case DBGFEVENT_STEPPED_HYPER:
-        case DBGFEVENT_ASSERTION_HYPER:
-        case DBGFEVENT_BREAKPOINT_HYPER:
-            pVM->dbgf.s.fStoppedInHyper = true;
-            break;
-        default:
-            pVM->dbgf.s.fStoppedInHyper = false;
-            break;
-    }
-}
-
-
-/**
- * Try to determine the event context.
- *
- * @returns debug event context.
- * @param   pVM         The VM handle.
- */
-static DBGFEVENTCTX dbgfR3FigureEventCtx(PVM pVM)
-{
-    /** @todo SMP support! */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    switch (EMGetState(pVCpu))
-    {
-        case EMSTATE_RAW:
-        case EMSTATE_DEBUG_GUEST_RAW:
-            return DBGFEVENTCTX_RAW;
-
-        case EMSTATE_REM:
-        case EMSTATE_DEBUG_GUEST_REM:
-            return DBGFEVENTCTX_REM;
-
-        case EMSTATE_DEBUG_HYPER:
-        case EMSTATE_GURU_MEDITATION:
-            return DBGFEVENTCTX_HYPER;
-
-        default:
-            return DBGFEVENTCTX_OTHER;
-    }
-}
-
-/**
- * The common event prologue code.
- * It will set the 'stopped-in-hyper' flag, make sure someone is attached,
- * and perhaps process any high priority pending actions (none yet).
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   enmEvent    The event to be sent.
- */
-static int dbgfR3EventPrologue(PVM pVM, DBGFEVENTTYPE enmEvent)
-{
-    /** @todo SMP */
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    /*
-     * Check if a debugger is attached.
-     */
-    if (    !pVM->dbgf.s.fAttached
-        &&  !dbgfR3WaitForAttach(pVM, enmEvent))
-    {
-        Log(("DBGFR3VMMEventSrc: enmEvent=%d - debugger not attached\n", enmEvent));
-        return VERR_DBGF_NOT_ATTACHED;
-    }
-
-    /*
-     * Sync back the state from the REM.
-     */
-    dbgfR3EventSetStoppedInHyperFlag(pVM, enmEvent);
-    if (!pVM->dbgf.s.fStoppedInHyper)
-        REMR3StateUpdate(pVM, pVCpu);
-
-    /*
-     * Look thru pending commands and finish those which make sense now.
-     */
-    /** @todo Process/purge pending commands. */
-    //int rc = DBGFR3VMMForcedAction(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Sends the event in the event buffer.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-static int dbgfR3SendEvent(PVM pVM)
-{
-    int rc = RTSemPing(&pVM->dbgf.s.PingPong);
-    if (RT_SUCCESS(rc))
-        rc = dbgfR3VMMWait(pVM);
-
-    pVM->dbgf.s.fStoppedInHyper = false;
-    /** @todo sync VMM -> REM after exitting the debugger. everything may change while in the debugger! */
-    return rc;
-}
-
-
-/**
- * Send a generic debugger event which takes no data.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   enmEvent    The event to send.
- */
-VMMR3DECL(int) DBGFR3Event(PVM pVM, DBGFEVENTTYPE enmEvent)
-{
-    int rc = dbgfR3EventPrologue(pVM, enmEvent);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Send the event and process the reply communication.
-     */
-    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
-    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
-    return dbgfR3SendEvent(pVM);
-}
-
-
-/**
- * Send a debugger event which takes the full source file location.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   enmEvent    The event to send.
- * @param   pszFile     Source file.
- * @param   uLine       Line number in source file.
- * @param   pszFunction Function name.
- * @param   pszFormat   Message which accompanies the event.
- * @param   ...         Message arguments.
- */
-VMMR3DECL(int) DBGFR3EventSrc(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszFile, unsigned uLine, const char *pszFunction, const char *pszFormat, ...)
-{
-    va_list args;
-    va_start(args, pszFormat);
-    int rc = DBGFR3EventSrcV(pVM, enmEvent, pszFile, uLine, pszFunction, pszFormat, args);
-    va_end(args);
-    return rc;
-}
-
-
-/**
- * Send a debugger event which takes the full source file location.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   enmEvent    The event to send.
- * @param   pszFile     Source file.
- * @param   uLine       Line number in source file.
- * @param   pszFunction Function name.
- * @param   pszFormat   Message which accompanies the event.
- * @param   args        Message arguments.
- */
-VMMR3DECL(int) DBGFR3EventSrcV(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszFile, unsigned uLine, const char *pszFunction, const char *pszFormat, va_list args)
-{
-    int rc = dbgfR3EventPrologue(pVM, enmEvent);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Format the message.
-     */
-    char   *pszMessage = NULL;
-    char    szMessage[8192];
-    if (pszFormat && *pszFormat)
-    {
-        pszMessage = &szMessage[0];
-        RTStrPrintfV(szMessage, sizeof(szMessage), pszFormat, args);
-    }
-
-    /*
-     * Send the event and process the reply communication.
-     */
-    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
-    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
-    pVM->dbgf.s.DbgEvent.u.Src.pszFile      = pszFile;
-    pVM->dbgf.s.DbgEvent.u.Src.uLine        = uLine;
-    pVM->dbgf.s.DbgEvent.u.Src.pszFunction  = pszFunction;
-    pVM->dbgf.s.DbgEvent.u.Src.pszMessage   = pszMessage;
-    return dbgfR3SendEvent(pVM);
-}
-
-
-/**
- * Send a debugger event which takes the two assertion messages.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   enmEvent    The event to send.
- * @param   pszMsg1     First assertion message.
- * @param   pszMsg2     Second assertion message.
- */
-VMMR3DECL(int) DBGFR3EventAssertion(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszMsg1, const char *pszMsg2)
-{
-    int rc = dbgfR3EventPrologue(pVM, enmEvent);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Send the event and process the reply communication.
-     */
-    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
-    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
-    pVM->dbgf.s.DbgEvent.u.Assert.pszMsg1 = pszMsg1;
-    pVM->dbgf.s.DbgEvent.u.Assert.pszMsg2 = pszMsg2;
-    return dbgfR3SendEvent(pVM);
-}
-
-
-/**
- * Breakpoint was hit somewhere.
- * Figure out which breakpoint it is and notify the debugger.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   enmEvent    DBGFEVENT_BREAKPOINT_HYPER or DBGFEVENT_BREAKPOINT.
- */
-VMMR3DECL(int) DBGFR3EventBreakpoint(PVM pVM, DBGFEVENTTYPE enmEvent)
-{
-    int rc = dbgfR3EventPrologue(pVM, enmEvent);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Send the event and process the reply communication.
-     */
-    /** @todo SMP */
-    PVMCPU pVCpu = VMMGetCpu0(pVM);
-
-    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
-    RTUINT iBp = pVM->dbgf.s.DbgEvent.u.Bp.iBp = pVCpu->dbgf.s.iActiveBp;
-    pVCpu->dbgf.s.iActiveBp = ~0U;
-    if (iBp != ~0U)
-        pVM->dbgf.s.DbgEvent.enmCtx = DBGFEVENTCTX_RAW;
-    else
-    {
-        /* REM breakpoints has be been searched for. */
-#if 0   /** @todo get flat PC api! */
-        uint32_t eip = CPUMGetGuestEIP(pVM);
-#else
-        /* @todo SMP support!! */
-        PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(VMMGetCpu(pVM));
-        RTGCPTR  eip = pCtx->rip + pCtx->csHid.u64Base;
-#endif
-        for (iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); iBp++)
-            if (    pVM->dbgf.s.aBreakpoints[iBp].enmType == DBGFBPTYPE_REM
-                &&  pVM->dbgf.s.aBreakpoints[iBp].GCPtr == eip)
-            {
-                pVM->dbgf.s.DbgEvent.u.Bp.iBp = iBp;
-                break;
-            }
-        AssertMsg(pVM->dbgf.s.DbgEvent.u.Bp.iBp != ~0U, ("eip=%08x\n", eip));
-        pVM->dbgf.s.DbgEvent.enmCtx = DBGFEVENTCTX_REM;
-    }
-    return dbgfR3SendEvent(pVM);
-}
-
-
-/**
- * Waits for the debugger to respond.
- *
- * @returns VBox status. (clearify)
- * @param   pVM     VM handle.
- */
-static int dbgfR3VMMWait(PVM pVM)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    LogFlow(("dbgfR3VMMWait:\n"));
-
-    /** @todo stupid GDT/LDT sync hack. go away! */
-    SELMR3UpdateFromCPUM(pVM, pVCpu);
-    int rcRet = VINF_SUCCESS;
-
-    /*
-     * Waits for the debugger to reply (i.e. issue an command).
-     */
-    for (;;)
-    {
-        /*
-         * Wait.
-         */
-        uint32_t cPollHack = 1; /** @todo this interface is horrible now that we're using lots of VMR3ReqCall stuff all over DBGF. */
-        for (;;)
-        {
-            int rc;
-            if (    !VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_REQUEST)
-                &&  !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
-            {
-                rc = RTSemPingWait(&pVM->dbgf.s.PingPong, cPollHack);
-                if (RT_SUCCESS(rc))
-                    break;
-                if (rc != VERR_TIMEOUT)
-                {
-                    LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
-                    return rc;
-                }
-            }
-
-            if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
-            {
-                rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
-                cPollHack = 1;
-            }
-            else if (   VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
-                     || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
-            {
-                LogFlow(("dbgfR3VMMWait: Processes requests...\n"));
-                rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
-                if (rc == VINF_SUCCESS)
-                    rc = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu);
-                LogFlow(("dbgfR3VMMWait: VMR3ReqProcess -> %Rrc rcRet=%Rrc\n", rc, rcRet));
-                cPollHack = 1;
-            }
-            else
-            {
-                rc = VINF_SUCCESS;
-                if (cPollHack < 120)
-                    cPollHack++;
-            }
-
-            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
-            {
-                switch (rc)
-                {
-                    case VINF_EM_DBG_BREAKPOINT:
-                    case VINF_EM_DBG_STEPPED:
-                    case VINF_EM_DBG_STEP:
-                    case VINF_EM_DBG_STOP:
-                        AssertMsgFailed(("rc=%Rrc\n", rc));
-                        break;
-
-                    /* return straight away */
-                    case VINF_EM_TERMINATE:
-                    case VINF_EM_OFF:
-                        LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
-                        return rc;
-
-                    /* remember return code. */
-                    default:
-                        AssertReleaseMsgFailed(("rc=%Rrc is not in the switch!\n", rc));
-                    case VINF_EM_RESET:
-                    case VINF_EM_SUSPEND:
-                    case VINF_EM_HALT:
-                    case VINF_EM_RESUME:
-                    case VINF_EM_RESCHEDULE:
-                    case VINF_EM_RESCHEDULE_REM:
-                    case VINF_EM_RESCHEDULE_RAW:
-                        if (rc < rcRet || rcRet == VINF_SUCCESS)
-                            rcRet = rc;
-                        break;
-                }
-            }
-            else if (RT_FAILURE(rc))
-            {
-                LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
-                return rc;
-            }
-        }
-
-        /*
-         * Process the command.
-         */
-        bool            fResumeExecution;
-        DBGFCMDDATA     CmdData = pVM->dbgf.s.VMMCmdData;
-        DBGFCMD         enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
-        int rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
-        if (fResumeExecution)
-        {
-            if (RT_FAILURE(rc))
-                rcRet = rc;
-            else if (    rc >= VINF_EM_FIRST
-                     &&  rc <= VINF_EM_LAST
-                     &&  (rc < rcRet || rcRet == VINF_SUCCESS))
-                rcRet = rc;
-            LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rcRet));
-            return rcRet;
-        }
-    }
-}
-
-
-/**
- * Executes command from debugger.
- * The caller is responsible for waiting or resuming execution based on the
- * value returned in the *pfResumeExecution indicator.
- *
- * @returns VBox status. (clearify!)
- * @param   pVM                 VM Handle.
- * @param   enmCmd              The command in question.
- * @param   pCmdData            Pointer to the command data.
- * @param   pfResumeExecution   Where to store the resume execution / continue waiting indicator.
- */
-static int dbgfR3VMMCmd(PVM pVM, DBGFCMD enmCmd, PDBGFCMDDATA pCmdData, bool *pfResumeExecution)
-{
-    bool    fSendEvent;
-    bool    fResume;
-    int     rc = VINF_SUCCESS;
-
-    switch (enmCmd)
-    {
-        /*
-         * Halt is answered by an event say that we've halted.
-         */
-        case DBGFCMD_HALT:
-        {
-            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_HALT_DONE;
-            pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
-            fSendEvent = true;
-            fResume = false;
-            break;
-        }
-
-
-        /*
-         * Resume is not answered we'll just resume execution.
-         */
-        case DBGFCMD_GO:
-        {
-            fSendEvent = false;
-            fResume = true;
-            break;
-        }
-
-        /** @todo implement (and define) the rest of the commands. */
-
-        /*
-         * Disable breakpoints and stuff.
-         * Send an everythings cool event to the debugger thread and resume execution.
-         */
-        case DBGFCMD_DETACH_DEBUGGER:
-        {
-            ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, false);
-            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_DETACH_DONE;
-            pVM->dbgf.s.DbgEvent.enmCtx  = DBGFEVENTCTX_OTHER;
-            fSendEvent = true;
-            fResume = true;
-            break;
-        }
-
-        /*
-         * The debugger has detached successfully.
-         * There is no reply to this event.
-         */
-        case DBGFCMD_DETACHED_DEBUGGER:
-        {
-            fSendEvent = false;
-            fResume = true;
-            break;
-        }
-
-        /*
-         * Single step, with trace into.
-         */
-        case DBGFCMD_SINGLE_STEP:
-        {
-            Log2(("Single step\n"));
-            rc = VINF_EM_DBG_STEP;
-            /** @todo SMP */
-            PVMCPU pVCpu = VMMGetCpu0(pVM);
-            pVCpu->dbgf.s.fSingleSteppingRaw = true;
-            fSendEvent = false;
-            fResume = true;
-            break;
-        }
-
-        /*
-         * Default is to send an invalid command event.
-         */
-        default:
-        {
-            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_INVALID_COMMAND;
-            pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
-            fSendEvent = true;
-            fResume = false;
-            break;
-        }
-    }
-
-    /*
-     * Send pending event.
-     */
-    if (fSendEvent)
-    {
-        Log2(("DBGF: Emulation thread: sending event %d\n", pVM->dbgf.s.DbgEvent.enmType));
-        int rc2 = RTSemPing(&pVM->dbgf.s.PingPong);
-        if (RT_FAILURE(rc2))
-        {
-            AssertRC(rc2);
-            *pfResumeExecution = true;
-            return rc2;
-        }
-    }
-
-    /*
-     * Return.
-     */
-    *pfResumeExecution = fResume;
-    return rc;
-}
-
-
-/**
- * Attaches a debugger to the specified VM.
- *
- * Only one debugger at a time.
- *
- * @returns VBox status code.
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(int) DBGFR3Attach(PVM pVM)
-{
-    /*
-     * Some validations first.
-     */
-    if (!VALID_PTR(pVM))
-    {
-        Log(("DBGFR3Attach: bad VM handle: %p\n", pVM));
-        return VERR_INVALID_HANDLE;
-    }
-    VMSTATE enmVMState = pVM->enmVMState;
-    if (    enmVMState >= VMSTATE_DESTROYING
-        ||  enmVMState <  VMSTATE_CREATING)
-    {
-        Log(("DBGFR3Attach: Invalid VM state: %s\n", VMGetStateName(enmVMState)));
-        return VERR_INVALID_HANDLE;
-    }
-
-    /*
-     * Call the VM, use EMT for serialization.
-     */
-    /** @todo SMP */
-    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3Attach, 1, pVM);
-}
-
-
-/**
- * EMT worker for DBGFR3Attach.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- */
-static DECLCALLBACK(int) dbgfR3Attach(PVM pVM)
-{
-    if (pVM->dbgf.s.fAttached)
-    {
-        Log(("dbgR3Attach: Debugger already attached\n"));
-        return VERR_DBGF_ALREADY_ATTACHED;
-    }
-
-    /*
-     * Create the Ping-Pong structure.
-     */
-    int rc = RTSemPingPongInit(&pVM->dbgf.s.PingPong);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Set the attached flag.
-     */
-    ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, true);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Detaches a debugger from the specified VM.
- *
- * Caller must be attached to the VM.
- *
- * @returns VBox status code.
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(int) DBGFR3Detach(PVM pVM)
-{
-    LogFlow(("DBGFR3Detach:\n"));
-    int rc;
-
-    /*
-     * Check if attached.
-     */
-    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
-
-    /*
-     * Try send the detach command.
-     * Keep in mind that we might be racing EMT, so, be extra careful.
-     */
-    DBGFCMD enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_DETACH_DEBUGGER);
-    if (RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong))
-    {
-        rc = RTSemPong(&pVM->dbgf.s.PingPong);
-        AssertMsgRCReturn(rc, ("Failed to signal emulation thread. rc=%Rrc\n", rc), rc);
-        LogRel(("DBGFR3Detach: enmCmd=%d (pong -> ping)\n", enmCmd));
-    }
-
-    /*
-     * Wait for the OK event.
-     */
-    rc = RTSemPongWait(&pVM->dbgf.s.PingPong, RT_INDEFINITE_WAIT);
-    AssertLogRelMsgRCReturn(rc, ("Wait on detach command failed, rc=%Rrc\n", rc), rc);
-
-    /*
-     * Send the notification command indicating that we're really done.
-     */
-    enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_DETACHED_DEBUGGER);
-    rc = RTSemPong(&pVM->dbgf.s.PingPong);
-    AssertMsgRCReturn(rc, ("Failed to signal emulation thread. rc=%Rrc\n", rc), rc);
-
-    LogFlowFunc(("returns VINF_SUCCESS\n"));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Wait for a debug event.
- *
- * @returns VBox status. Will not return VBOX_INTERRUPTED.
- * @param   pVM         VM handle.
- * @param   cMillies    Number of millis to wait.
- * @param   ppEvent     Where to store the event pointer.
- */
-VMMR3DECL(int) DBGFR3EventWait(PVM pVM, RTMSINTERVAL cMillies, PCDBGFEVENT *ppEvent)
-{
-    /*
-     * Check state.
-     */
-    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
-    *ppEvent = NULL;
-
-    /*
-     * Wait.
-     */
-    int rc = RTSemPongWait(&pVM->dbgf.s.PingPong, cMillies);
-    if (RT_SUCCESS(rc))
-    {
-        *ppEvent = &pVM->dbgf.s.DbgEvent;
-        Log2(("DBGF: Debugger thread: receiving event %d\n", (*ppEvent)->enmType));
-        return VINF_SUCCESS;
-    }
-
-    return rc;
-}
-
-
-/**
- * Halts VM execution.
- *
- * After calling this the VM isn't actually halted till an DBGFEVENT_HALT_DONE
- * arrives. Until that time it's not possible to issue any new commands.
- *
- * @returns VBox status.
- * @param   pVM     VM handle.
- */
-VMMR3DECL(int) DBGFR3Halt(PVM pVM)
-{
-    /*
-     * Check state.
-     */
-    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
-    RTPINGPONGSPEAKER enmSpeaker = pVM->dbgf.s.PingPong.enmSpeaker;
-    if (   enmSpeaker == RTPINGPONGSPEAKER_PONG
-        || enmSpeaker == RTPINGPONGSPEAKER_PONG_SIGNALED)
-        return VWRN_DBGF_ALREADY_HALTED;
-
-    /*
-     * Send command.
-     */
-    dbgfR3SetCmd(pVM, DBGFCMD_HALT);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Checks if the VM is halted by the debugger.
- *
- * @returns True if halted.
- * @returns False if not halted.
- * @param   pVM     VM handle.
- */
-VMMR3DECL(bool) DBGFR3IsHalted(PVM pVM)
-{
-    AssertReturn(pVM->dbgf.s.fAttached, false);
-    RTPINGPONGSPEAKER enmSpeaker = pVM->dbgf.s.PingPong.enmSpeaker;
-    return enmSpeaker == RTPINGPONGSPEAKER_PONG_SIGNALED
-        || enmSpeaker == RTPINGPONGSPEAKER_PONG;
-}
-
-
-/**
- * Checks if the debugger can wait for events or not.
- *
- * This function is only used by lazy, multiplexing debuggers. :-)
- *
- * @returns True if waitable.
- * @returns False if not waitable.
- * @param   pVM     VM handle.
- */
-VMMR3DECL(bool) DBGFR3CanWait(PVM pVM)
-{
-    AssertReturn(pVM->dbgf.s.fAttached, false);
-    return RTSemPongShouldWait(&pVM->dbgf.s.PingPong);
-}
-
-
-/**
- * Resumes VM execution.
- *
- * There is no receipt event on this command.
- *
- * @returns VBox status.
- * @param   pVM     VM handle.
- */
-VMMR3DECL(int) DBGFR3Resume(PVM pVM)
-{
-    /*
-     * Check state.
-     */
-    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
-    AssertReturn(RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong), VERR_SEM_OUT_OF_TURN);
-
-    /*
-     * Send the ping back to the emulation thread telling it to run.
-     */
-    dbgfR3SetCmd(pVM, DBGFCMD_GO);
-    int rc = RTSemPong(&pVM->dbgf.s.PingPong);
-    AssertRC(rc);
-
-    return rc;
-}
-
-
-/**
- * Step Into.
- *
- * A single step event is generated from this command.
- * The current implementation is not reliable, so don't rely on the event coming.
- *
- * @returns VBox status.
- * @param   pVM     VM handle.
- * @param   idCpu   The ID of the CPU to single step on.
- */
-VMMR3DECL(int) DBGFR3Step(PVM pVM, VMCPUID idCpu)
-{
-    /*
-     * Check state.
-     */
-    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
-    AssertReturn(RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong), VERR_SEM_OUT_OF_TURN);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
-
-    /*
-     * Send the ping back to the emulation thread telling it to run.
-     */
-/** @todo SMP (idCpu) */
-    dbgfR3SetCmd(pVM, DBGFCMD_SINGLE_STEP);
-    int rc = RTSemPong(&pVM->dbgf.s.PingPong);
-    AssertRC(rc);
-    return rc;
-}
-
-
-/**
- * Call this to single step programmatically.
- *
- * You must pass down the return code to the EM loop! That's
- * where the actual single stepping take place (at least in the
- * current implementation).
- *
- * @returns VINF_EM_DBG_STEP
- *
- * @param   pVCpu       The virtual CPU handle.
- *
- * @thread  VCpu EMT
- */
-VMMR3DECL(int) DBGFR3PrgStep(PVMCPU pVCpu)
-{
-    VMCPU_ASSERT_EMT(pVCpu);
-
-    pVCpu->dbgf.s.fSingleSteppingRaw = true;
-    return VINF_EM_DBG_STEP;
-}
-
Index: unk/src/VBox/VMM/DBGFAddr.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFAddr.cpp	(revision 35332)
+++ 	(revision )
@@ -1,492 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Mixed Address Methods.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/selm.h>
-#include <VBox/mm.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <include/internal/pgm.h>
-
-
-
-/**
- * Checks if an address is in the HMA or not.
- * @returns true if it's inside the HMA.
- * @returns flase if it's not inside the HMA.
- * @param   pVM         The VM handle.
- * @param   FlatPtr     The address in question.
- */
-DECLINLINE(bool) dbgfR3IsHMA(PVM pVM, RTGCUINTPTR FlatPtr)
-{
-    return MMHyperIsInsideArea(pVM, FlatPtr);
-}
-
-
-/**
- * Common worker for DBGFR3AddrFromSelOff and DBGFR3AddrFromSelInfoOff.
- */
-static int dbgfR3AddrFromSelInfoOffWorker(PDBGFADDRESS pAddress, PCDBGFSELINFO pSelInfo, RTUINTPTR off)
-{
-    if (pSelInfo->fFlags & (DBGFSELINFO_FLAGS_INVALID | DBGFSELINFO_FLAGS_NOT_PRESENT))
-        return pSelInfo->fFlags & DBGFSELINFO_FLAGS_NOT_PRESENT
-             ? VERR_SELECTOR_NOT_PRESENT
-             : VERR_INVALID_SELECTOR;
-
-    /** @todo This all goes voodoo in long mode. */
-    /* check limit. */
-    if (DBGFSelInfoIsExpandDown(pSelInfo))
-    {
-        if (    !pSelInfo->u.Raw.Gen.u1Granularity
-            &&  off > UINT32_C(0xffff))
-            return VERR_OUT_OF_SELECTOR_BOUNDS;
-        if (off <= pSelInfo->cbLimit)
-            return VERR_OUT_OF_SELECTOR_BOUNDS;
-    }
-    else if (off > pSelInfo->cbLimit)
-        return VERR_OUT_OF_SELECTOR_BOUNDS;
-
-    pAddress->FlatPtr = pSelInfo->GCPtrBase + off;
-
-    /** @todo fix all these selector tests! */
-    if (    !pSelInfo->GCPtrBase
-        &&  pSelInfo->u.Raw.Gen.u1Granularity
-        &&  pSelInfo->u.Raw.Gen.u1DefBig)
-        pAddress->fFlags = DBGFADDRESS_FLAGS_FLAT;
-    else if (pSelInfo->cbLimit <= UINT32_C(0xffff))
-        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR16;
-    else if (pSelInfo->cbLimit <= UINT32_C(0xffffffff))
-        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR32;
-    else
-        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR64;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Creates a mixed address from a Sel:off pair.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   idCpu       The CPU ID.
- * @param   pAddress    Where to store the mixed address.
- * @param   Sel         The selector part.
- * @param   off         The offset part.
- */
-VMMR3DECL(int) DBGFR3AddrFromSelOff(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, RTSEL Sel, RTUINTPTR off)
-{
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
-
-    pAddress->Sel = Sel;
-    pAddress->off = off;
-    if (Sel != DBGF_SEL_FLAT)
-    {
-        DBGFSELINFO SelInfo;
-        int rc = DBGFR3SelQueryInfo(pVM, idCpu, Sel, DBGFSELQI_FLAGS_DT_GUEST | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE, &SelInfo);
-        if (RT_FAILURE(rc))
-            return rc;
-        rc = dbgfR3AddrFromSelInfoOffWorker(pAddress, &SelInfo, off);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-    else
-    {
-        pAddress->FlatPtr = off;
-        pAddress->fFlags = DBGFADDRESS_FLAGS_FLAT;
-    }
-    pAddress->fFlags |= DBGFADDRESS_FLAGS_VALID;
-    if (dbgfR3IsHMA(pVM, pAddress->FlatPtr))
-        pAddress->fFlags |= DBGFADDRESS_FLAGS_HMA;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Creates a mixed address from selector info and an offset into the segment
- * described by it.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   idCpu       The CPU ID.
- * @param   pAddress    Where to store the mixed address.
- * @param   pSelInfo    The selector info.
- * @param   off         The offset part.
- */
-VMMR3DECL(int) DBGFR3AddrFromSelInfoOff(PVM pVM, PDBGFADDRESS pAddress, PCDBGFSELINFO pSelInfo, RTUINTPTR off)
-{
-    pAddress->Sel = pSelInfo->Sel;
-    pAddress->off = off;
-    int rc = dbgfR3AddrFromSelInfoOffWorker(pAddress, pSelInfo, off);
-    if (RT_FAILURE(rc))
-        return rc;
-    pAddress->fFlags |= DBGFADDRESS_FLAGS_VALID;
-    if (dbgfR3IsHMA(pVM, pAddress->FlatPtr))
-        pAddress->fFlags |= DBGFADDRESS_FLAGS_HMA;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Creates a mixed address from a flat address.
- *
- * @returns pAddress.
- * @param   pVM         The VM handle.
- * @param   pAddress    Where to store the mixed address.
- * @param   FlatPtr     The flat pointer.
- */
-VMMR3DECL(PDBGFADDRESS) DBGFR3AddrFromFlat(PVM pVM, PDBGFADDRESS pAddress, RTGCUINTPTR FlatPtr)
-{
-    pAddress->Sel     = DBGF_SEL_FLAT;
-    pAddress->off     = FlatPtr;
-    pAddress->FlatPtr = FlatPtr;
-    pAddress->fFlags  = DBGFADDRESS_FLAGS_FLAT | DBGFADDRESS_FLAGS_VALID;
-    if (dbgfR3IsHMA(pVM, pAddress->FlatPtr))
-        pAddress->fFlags |= DBGFADDRESS_FLAGS_HMA;
-    return pAddress;
-}
-
-
-/**
- * Creates a mixed address from a guest physical address.
- *
- * @returns pAddress.
- * @param   pVM         The VM handle.
- * @param   pAddress    Where to store the mixed address.
- * @param   PhysAddr    The guest physical address.
- */
-VMMR3DECL(PDBGFADDRESS) DBGFR3AddrFromPhys(PVM pVM, PDBGFADDRESS pAddress, RTGCPHYS PhysAddr)
-{
-    pAddress->Sel     = DBGF_SEL_FLAT;
-    pAddress->off     = PhysAddr;
-    pAddress->FlatPtr = PhysAddr;
-    pAddress->fFlags  = DBGFADDRESS_FLAGS_PHYS | DBGFADDRESS_FLAGS_VALID;
-    return pAddress;
-}
-
-
-/**
- * Checks if the specified address is valid (checks the structure pointer too).
- *
- * @returns true if valid.
- * @returns false if invalid.
- * @param   pVM         The VM handle.
- * @param   pAddress    The address to validate.
- */
-VMMR3DECL(bool) DBGFR3AddrIsValid(PVM pVM, PCDBGFADDRESS pAddress)
-{
-    if (!VALID_PTR(pAddress))
-        return false;
-    if (!DBGFADDRESS_IS_VALID(pAddress))
-        return false;
-    /* more? */
-    return true;
-}
-
-
-/**
- * Called on the EMT for the VCpu.
- *
- * @returns VBox status code.
- * @param   pVCpu           The virtual CPU handle.
- * @param   pAddress        The address.
- * @param   pGCPhys         Where to return the physical address.
- */
-static DECLCALLBACK(int) dbgfR3AddrToPhysOnVCpu(PVMCPU pVCpu, PDBGFADDRESS pAddress, PRTGCPHYS pGCPhys)
-{
-    VMCPU_ASSERT_EMT(pVCpu);
-    /* This is just a wrapper because we cannot pass FlatPtr thru VMR3ReqCall directly. */
-    return PGMGstGetPage(pVCpu, pAddress->FlatPtr, NULL, pGCPhys);
-}
-
-
-/**
- * Converts an address to a guest physical address.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_PARAMETER if the address is invalid.
- * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
- *          CPU handle is invalid.
- * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
- * @retval  VERR_PAGE_NOT_PRESENT
- * @retval  VERR_PAGE_TABLE_NOT_PRESENT
- * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
- * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
- *
- * @param   pVM             The VM handle.
- * @param   idCpu           The ID of the CPU context to convert virtual
- *                          addresses.
- * @param   pAddress        The address.
- * @param   pGCPhys         Where to return the physical address.
- */
-VMMR3DECL(int)  DBGFR3AddrToPhys(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, PRTGCPHYS pGCPhys)
-{
-    /*
-     * Parameter validation.
-     */
-    AssertPtr(pGCPhys);
-    *pGCPhys = NIL_RTGCPHYS;
-    AssertPtr(pAddress);
-    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
-
-    /*
-     * Convert by address type.
-     */
-    int rc;
-    if (pAddress->fFlags & DBGFADDRESS_FLAGS_HMA)
-        rc = VERR_NOT_SUPPORTED;
-    else if (pAddress->fFlags & DBGFADDRESS_FLAGS_PHYS)
-    {
-        *pGCPhys = pAddress->FlatPtr;
-        rc = VINF_SUCCESS;
-    }
-    else
-    {
-        PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-        if (VMCPU_IS_EMT(pVCpu))
-            rc = dbgfR3AddrToPhysOnVCpu(pVCpu, pAddress, pGCPhys);
-        else
-            rc = VMR3ReqCallWait(pVCpu->pVMR3, pVCpu->idCpu,
-                                 (PFNRT)dbgfR3AddrToPhysOnVCpu, 3, pVCpu, pAddress, pGCPhys);
-    }
-    return rc;
-}
-
-
-/**
- * Converts an address to a host physical address.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_PARAMETER if the address is invalid.
- * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
- *          CPU handle is invalid.
- * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
- * @retval  VERR_PAGE_NOT_PRESENT
- * @retval  VERR_PAGE_TABLE_NOT_PRESENT
- * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
- * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
- * @retval  VERR_PGM_PHYS_PAGE_RESERVED
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS
- *
- * @param   pVM             The VM handle.
- * @param   idCpu           The ID of the CPU context to convert virtual
- *                          addresses.
- * @param   pAddress        The address.
- * @param   pHCPhys         Where to return the physical address.
- */
-VMMR3DECL(int)  DBGFR3AddrToHostPhys(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, PRTHCPHYS pHCPhys)
-{
-    /*
-     * Parameter validation.
-     */
-    AssertPtr(pHCPhys);
-    *pHCPhys = NIL_RTHCPHYS;
-    AssertPtr(pAddress);
-    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
-
-    /*
-     * Convert it if we can.
-     */
-    int rc;
-    if (pAddress->fFlags & DBGFADDRESS_FLAGS_HMA)
-        rc = VERR_NOT_SUPPORTED; /** @todo implement this */
-    else
-    {
-        RTGCPHYS GCPhys;
-        rc = DBGFR3AddrToPhys(pVM, idCpu, pAddress, &GCPhys);
-        if (RT_SUCCESS(rc))
-            rc = PGMPhysGCPhys2HCPhys(pVM, pAddress->FlatPtr, pHCPhys);
-    }
-    return rc;
-}
-
-
-/**
- * Called on the EMT for the VCpu.
- *
- * @returns VBox status code.
- *
- * @param   pVM             The VM handle.
- * @param   idCpu           The ID of the CPU context.
- * @param   pAddress        The address.
- * @param   fReadOnly       Whether returning a read-only page is fine or not.
- * @param   ppvR3Ptr        Where to return the address.
- */
-static DECLCALLBACK(int) dbgfR3AddrToVolatileR3PtrOnVCpu(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, bool fReadOnly, void **ppvR3Ptr)
-{
-    Assert(idCpu == VMMGetCpuId(pVM));
-
-    int rc;
-    if (pAddress->fFlags & DBGFADDRESS_FLAGS_HMA)
-    {
-        rc = VERR_NOT_SUPPORTED; /** @todo create some dedicated errors for this stuff. */
-        /** @todo this may assert, create a debug version of this which doesn't. */
-        if (MMHyperIsInsideArea(pVM, pAddress->FlatPtr))
-        {
-            void *pv = MMHyperRCToCC(pVM, (RTRCPTR)pAddress->FlatPtr);
-            if (pv)
-            {
-                *ppvR3Ptr = pv;
-                rc = VINF_SUCCESS;
-            }
-        }
-    }
-    else
-    {
-        /*
-         * This is a tad ugly, but it gets the job done.
-         */
-        PGMPAGEMAPLOCK Lock;
-        if (pAddress->fFlags & DBGFADDRESS_FLAGS_PHYS)
-        {
-            if (fReadOnly)
-                rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, pAddress->FlatPtr, (void const **)ppvR3Ptr, &Lock);
-            else
-                rc = PGMPhysGCPhys2CCPtr(pVM, pAddress->FlatPtr, ppvR3Ptr, &Lock);
-        }
-        else
-        {
-            PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-            if (fReadOnly)
-                rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, pAddress->FlatPtr, (void const **)ppvR3Ptr, &Lock);
-            else
-                rc = PGMPhysGCPtr2CCPtr(pVCpu, pAddress->FlatPtr, ppvR3Ptr, &Lock);
-        }
-        if (RT_SUCCESS(rc))
-            PGMPhysReleasePageMappingLock(pVM, &Lock);
-    }
-    return rc;
-}
-
-
-
-
-/**
- * Converts an address to a volatile host virtual address.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_PARAMETER if the address is invalid.
- * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
- *          CPU handle is invalid.
- * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
- * @retval  VERR_PAGE_NOT_PRESENT
- * @retval  VERR_PAGE_TABLE_NOT_PRESENT
- * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
- * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
- * @retval  VERR_PGM_PHYS_PAGE_RESERVED
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS
- *
- * @param   pVM             The VM handle.
- * @param   idCpu           The ID of the CPU context to convert virtual
- *                          addresses.
- * @param   pAddress        The address.
- * @param   fReadOnly       Whether returning a read-only page is fine or not.
- *                          If set to thru the page may have to be made writable
- *                          before we return.
- * @param   ppvR3Ptr        Where to return the address.
- */
-VMMR3DECL(int)  DBGFR3AddrToVolatileR3Ptr(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, bool fReadOnly, void **ppvR3Ptr)
-{
-    /*
-     * Parameter validation.
-     */
-    AssertPtr(ppvR3Ptr);
-    *ppvR3Ptr = NULL;
-    AssertPtr(pAddress);
-    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
-
-    /*
-     * Convert it.
-     */
-    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3AddrToVolatileR3PtrOnVCpu, 5, pVM, idCpu, pAddress, fReadOnly, ppvR3Ptr);
-}
-
-
-/**
- * Adds an offset to an address.
- *
- * @returns pAddress.
- *
- * @param   pAddress        The address.
- * @param   uAddend         How much to add.
- *
- * @remarks No address space or segment limit checks are performed,
- */
-VMMR3DECL(PDBGFADDRESS) DBGFR3AddrAdd(PDBGFADDRESS pAddress, RTGCUINTPTR uAddend)
-{
-    /*
-     * Parameter validation.
-     */
-    AssertPtrReturn(pAddress, NULL);
-    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), NULL);
-
-    /*
-     * Add the stuff.
-     */
-    pAddress->off     += uAddend;
-    pAddress->FlatPtr += uAddend;
-
-    return pAddress;
-}
-
-
-/**
- * Subtracts an offset from an address.
- *
- * @returns VINF_SUCCESS on success.
- *
- * @param   pAddress        The address.
- * @param   uSubtrahend     How much to subtract.
- *
- * @remarks No address space or segment limit checks are performed,
- */
-VMMR3DECL(PDBGFADDRESS) DBGFR3AddrSub(PDBGFADDRESS pAddress, RTGCUINTPTR uSubtrahend)
-{
-    /*
-     * Parameter validation.
-     */
-    AssertPtrReturn(pAddress, NULL);
-    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), NULL);
-
-    /*
-     * Add the stuff.
-     */
-    pAddress->off     -= uSubtrahend;
-    pAddress->FlatPtr -= uSubtrahend;
-
-    return pAddress;
-}
-
Index: unk/src/VBox/VMM/DBGFAddrSpace.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFAddrSpace.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1114 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Address Space Management.
- */
-
-/*
- * Copyright (C) 2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page pg_dbgf_addr_space     DBGFAddrSpace - Address Space Management
- *
- * What's an address space? It's mainly a convenient way of stuffing
- * module segments and ad-hoc symbols together. It will also help out
- * when the debugger gets extended to deal with user processes later.
- *
- * There are two standard address spaces that will always be present:
- *   - The physical address space.
- *   - The global virtual address space.
- *
- * Additional address spaces will be added and removed at runtime for
- * guest processes. The global virtual address space will be used to
- * track the kernel parts of the OS, or at least the bits of the kernel
- * that is part of all address spaces (mac os x and 4G/4G patched linux).
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/pdmapi.h>
-#include <VBox/mm.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/ctype.h>
-#include <iprt/env.h>
-#include <iprt/path.h>
-#include <iprt/param.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Address space database node.
- */
-typedef struct DBGFASDBNODE
-{
-    /** The node core for DBGF::AsHandleTree, the key is the address space handle. */
-    AVLPVNODECORE   HandleCore;
-    /** The node core for DBGF::AsPidTree, the key is the process id. */
-    AVLU32NODECORE  PidCore;
-    /** The node core for DBGF::AsNameSpace, the string is the address space name. */
-    RTSTRSPACECORE  NameCore;
-
-} DBGFASDBNODE;
-/** Pointer to an address space database node. */
-typedef DBGFASDBNODE *PDBGFASDBNODE;
-
-
-/**
- * For dbgfR3AsLoadImageOpenData and dbgfR3AsLoadMapOpenData.
- */
-typedef struct DBGFR3ASLOADOPENDATA
-{
-    const char *pszModName;
-    RTGCUINTPTR uSubtrahend;
-    uint32_t fFlags;
-    RTDBGMOD hMod;
-} DBGFR3ASLOADOPENDATA;
-
-/**
- * Callback for dbgfR3AsSearchPath and dbgfR3AsSearchEnvPath.
- *
- * @returns VBox status code. If success, then the search is completed.
- * @param   pszFilename     The file name under evaluation.
- * @param   pvUser          The user argument.
- */
-typedef int FNDBGFR3ASSEARCHOPEN(const char *pszFilename, void *pvUser);
-/** Pointer to a FNDBGFR3ASSEARCHOPEN. */
-typedef FNDBGFR3ASSEARCHOPEN *PFNDBGFR3ASSEARCHOPEN;
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** Locks the address space database for writing. */
-#define DBGF_AS_DB_LOCK_WRITE(pVM) \
-    do { \
-        int rcSem = RTSemRWRequestWrite((pVM)->dbgf.s.hAsDbLock, RT_INDEFINITE_WAIT); \
-        AssertRC(rcSem); \
-    } while (0)
-
-/** Unlocks the address space database after writing. */
-#define DBGF_AS_DB_UNLOCK_WRITE(pVM) \
-    do { \
-        int rcSem = RTSemRWReleaseWrite((pVM)->dbgf.s.hAsDbLock); \
-        AssertRC(rcSem); \
-    } while (0)
-
-/** Locks the address space database for reading. */
-#define DBGF_AS_DB_LOCK_READ(pVM) \
-    do { \
-        int rcSem = RTSemRWRequestRead((pVM)->dbgf.s.hAsDbLock, RT_INDEFINITE_WAIT); \
-        AssertRC(rcSem); \
-    } while (0)
-
-/** Unlocks the address space database after reading. */
-#define DBGF_AS_DB_UNLOCK_READ(pVM) \
-    do { \
-        int rcSem = RTSemRWReleaseRead((pVM)->dbgf.s.hAsDbLock); \
-        AssertRC(rcSem); \
-    } while (0)
-
-
-
-/**
- * Initializes the address space parts of DBGF.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- */
-int dbgfR3AsInit(PVM pVM)
-{
-    /*
-     * Create the semaphore.
-     */
-    int rc = RTSemRWCreate(&pVM->dbgf.s.hAsDbLock);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Create the standard address spaces.
-     */
-    RTDBGAS hDbgAs;
-    rc = RTDbgAsCreate(&hDbgAs, 0, RTGCPTR_MAX, "Global");
-    AssertRCReturn(rc, rc);
-    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
-    AssertRCReturn(rc, rc);
-    RTDbgAsRetain(hDbgAs);
-    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_GLOBAL)] = hDbgAs;
-
-    RTDbgAsRetain(hDbgAs);
-    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_KERNEL)] = hDbgAs;
-
-    rc = RTDbgAsCreate(&hDbgAs, 0, RTGCPHYS_MAX, "Physical");
-    AssertRCReturn(rc, rc);
-    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
-    AssertRCReturn(rc, rc);
-    RTDbgAsRetain(hDbgAs);
-    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_PHYS)] = hDbgAs;
-
-    rc = RTDbgAsCreate(&hDbgAs, 0, RTRCPTR_MAX, "HyperRawMode");
-    AssertRCReturn(rc, rc);
-    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
-    AssertRCReturn(rc, rc);
-    RTDbgAsRetain(hDbgAs);
-    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC)] = hDbgAs;
-    RTDbgAsRetain(hDbgAs);
-    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC_AND_GC_GLOBAL)] = hDbgAs;
-
-    rc = RTDbgAsCreate(&hDbgAs, 0, RTR0PTR_MAX, "HyperRing0");
-    AssertRCReturn(rc, rc);
-    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
-    AssertRCReturn(rc, rc);
-    RTDbgAsRetain(hDbgAs);
-    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_R0)] = hDbgAs;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Callback used by dbgfR3AsTerm / RTAvlPVDestroy to release an address space.
- *
- * @returns 0.
- * @param   pNode           The address space database node.
- * @param   pvIgnore        NULL.
- */
-static DECLCALLBACK(int) dbgfR3AsTermDestroyNode(PAVLPVNODECORE pNode, void *pvIgnore)
-{
-    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)pNode;
-    RTDbgAsRelease((RTDBGAS)pDbNode->HandleCore.Key);
-    pDbNode->HandleCore.Key = NIL_RTDBGAS;
-    /* Don't bother freeing it here as MM will free it soon and MM is much at
-       it when doing it wholesale instead of piecemeal. */
-    NOREF(pvIgnore);
-    return 0;
-}
-
-
-/**
- * Terminates the address space parts of DBGF.
- *
- * @param   pVM             The VM handle.
- */
-void dbgfR3AsTerm(PVM pVM)
-{
-    /*
-     * Create the semaphore.
-     */
-    int rc = RTSemRWDestroy(pVM->dbgf.s.hAsDbLock);
-    AssertRC(rc);
-    pVM->dbgf.s.hAsDbLock = NIL_RTSEMRW;
-
-    /*
-     * Release all the address spaces.
-     */
-    RTAvlPVDestroy(&pVM->dbgf.s.AsHandleTree, dbgfR3AsTermDestroyNode, NULL);
-    for (size_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.ahAsAliases); i++)
-    {
-        RTDbgAsRelease(pVM->dbgf.s.ahAsAliases[i]);
-        pVM->dbgf.s.ahAsAliases[i] = NIL_RTDBGAS;
-    }
-}
-
-
-/**
- * Relocates the RC address space.
- *
- * @param   pVM             The VM handle.
- * @param   offDelta        The relocation delta.
- */
-void dbgfR3AsRelocate(PVM pVM, RTGCUINTPTR offDelta)
-{
-    /** @todo */
-}
-
-
-/**
- * Adds the address space to the database.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   hDbgAs          The address space handle. The reference of the
- *                          caller will NOT be consumed.
- * @param   ProcId          The process id or NIL_RTPROCESS.
- */
-VMMR3DECL(int) DBGFR3AsAdd(PVM pVM, RTDBGAS hDbgAs, RTPROCESS ProcId)
-{
-    /*
-     * Input validation.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    const char *pszName = RTDbgAsName(hDbgAs);
-    if (!pszName)
-        return VERR_INVALID_HANDLE;
-    uint32_t cRefs = RTDbgAsRetain(hDbgAs);
-    if (cRefs == UINT32_MAX)
-        return VERR_INVALID_HANDLE;
-
-    /*
-     * Allocate a tracking node.
-     */
-    int rc = VERR_NO_MEMORY;
-    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)MMR3HeapAlloc(pVM, MM_TAG_DBGF_AS, sizeof(*pDbNode));
-    if (pDbNode)
-    {
-        pDbNode->HandleCore.Key = hDbgAs;
-        pDbNode->PidCore.Key = NIL_RTPROCESS;
-        pDbNode->NameCore.pszString = pszName;
-        pDbNode->NameCore.cchString = strlen(pszName);
-        DBGF_AS_DB_LOCK_WRITE(pVM);
-        if (RTStrSpaceInsert(&pVM->dbgf.s.AsNameSpace, &pDbNode->NameCore))
-        {
-            if (RTAvlPVInsert(&pVM->dbgf.s.AsHandleTree, &pDbNode->HandleCore))
-            {
-                DBGF_AS_DB_UNLOCK_WRITE(pVM);
-                return VINF_SUCCESS;
-            }
-
-            /* bail out */
-            RTStrSpaceRemove(&pVM->dbgf.s.AsNameSpace, pszName);
-        }
-        DBGF_AS_DB_UNLOCK_WRITE(pVM);
-        MMR3HeapFree(pDbNode);
-    }
-    RTDbgAsRelease(hDbgAs);
-    return rc;
-}
-
-
-/**
- * Delete an address space from the database.
- *
- * The address space must not be engaged as any of the standard aliases.
- *
- * @returns VBox status code.
- * @retval  VERR_SHARING_VIOLATION if in use as an alias.
- * @retval  VERR_NOT_FOUND if not found in the address space database.
- *
- * @param   pVM             The VM handle.
- * @param   hDbgAs          The address space handle. Aliases are not allowed.
- */
-VMMR3DECL(int) DBGFR3AsDelete(PVM pVM, RTDBGAS hDbgAs)
-{
-    /*
-     * Input validation. Retain the address space so it can be released outside
-     * the lock as well as validated.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    if (hDbgAs == NIL_RTDBGAS)
-        return VINF_SUCCESS;
-    uint32_t cRefs = RTDbgAsRetain(hDbgAs);
-    if (cRefs == UINT32_MAX)
-        return VERR_INVALID_HANDLE;
-    RTDbgAsRelease(hDbgAs);
-
-    DBGF_AS_DB_LOCK_WRITE(pVM);
-
-    /*
-     * You cannot delete any of the aliases.
-     */
-    for (size_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.ahAsAliases); i++)
-        if (pVM->dbgf.s.ahAsAliases[i] == hDbgAs)
-        {
-            DBGF_AS_DB_UNLOCK_WRITE(pVM);
-            return VERR_SHARING_VIOLATION;
-        }
-
-    /*
-     * Ok, try remove it from the database.
-     */
-    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)RTAvlPVRemove(&pVM->dbgf.s.AsHandleTree, hDbgAs);
-    if (!pDbNode)
-    {
-        DBGF_AS_DB_UNLOCK_WRITE(pVM);
-        return VERR_NOT_FOUND;
-    }
-    RTStrSpaceRemove(&pVM->dbgf.s.AsNameSpace, pDbNode->NameCore.pszString);
-    if (pDbNode->PidCore.Key != NIL_RTPROCESS)
-        RTAvlU32Remove(&pVM->dbgf.s.AsPidTree, pDbNode->PidCore.Key);
-
-    DBGF_AS_DB_UNLOCK_WRITE(pVM);
-
-    /*
-     * Free the resources.
-     */
-    RTDbgAsRelease(hDbgAs);
-    MMR3HeapFree(pDbNode);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Changes an alias to point to a new address space.
- *
- * Not all the aliases can be changed, currently it's only DBGF_AS_GLOBAL
- * and DBGF_AS_KERNEL.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   hAlias          The alias to change.
- * @param   hAliasFor       The address space hAlias should be an alias for.
- *                          This can be an alias. The caller's reference to
- *                          this address space will NOT be consumed.
- */
-VMMR3DECL(int) DBGFR3AsSetAlias(PVM pVM, RTDBGAS hAlias, RTDBGAS hAliasFor)
-{
-    /*
-     * Input validation.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertMsgReturn(DBGF_AS_IS_ALIAS(hAlias), ("%p\n", hAlias), VERR_INVALID_PARAMETER);
-    AssertMsgReturn(!DBGF_AS_IS_FIXED_ALIAS(hAlias), ("%p\n", hAlias), VERR_INVALID_PARAMETER);
-    RTDBGAS hRealAliasFor = DBGFR3AsResolveAndRetain(pVM, hAlias);
-    if (hRealAliasFor == NIL_RTDBGAS)
-        return VERR_INVALID_HANDLE;
-
-    /*
-     * Make sure the handle has is already in the database.
-     */
-    int rc = VERR_NOT_FOUND;
-    DBGF_AS_DB_LOCK_WRITE(pVM);
-    if (RTAvlPVGet(&pVM->dbgf.s.AsHandleTree, hRealAliasFor))
-    {
-        /*
-         * Update the alias table and release the current address space.
-         */
-        RTDBGAS hAsOld;
-        ASMAtomicXchgHandle(&pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(hAlias)], hRealAliasFor, &hAsOld);
-        uint32_t cRefs = RTDbgAsRelease(hAsOld);
-        Assert(cRefs > 0);
-        Assert(cRefs != UINT32_MAX);
-        rc = VINF_SUCCESS;
-    }
-    DBGF_AS_DB_UNLOCK_WRITE(pVM);
-
-    return rc;
-}
-
-
-/**
- * @callback_method_impl{FNPDMR3ENUM}
- */
-static DECLCALLBACK(int) dbgfR3AsLazyPopulateR0Callback(PVM pVM, const char *pszFilename, const char *pszName,
-                                                        RTUINTPTR ImageBase, size_t cbImage, bool fRC, void *pvArg)
-{
-    /* Only ring-0 modules. */
-    if (!fRC)
-    {
-        RTDBGMOD hDbgMod;
-        int rc = RTDbgModCreateFromImage(&hDbgMod, pszFilename, pszName, 0 /*fFlags*/);
-        if (RT_SUCCESS(rc))
-        {
-            rc = RTDbgAsModuleLink((RTDBGAS)pvArg, hDbgMod, ImageBase, 0 /*fFlags*/);
-            if (RT_FAILURE(rc))
-                LogRel(("DBGF: Failed to link module \"%s\" into DBGF_AS_R0 at %RTptr: %Rrc\n",
-                        pszName, ImageBase, rc));
-        }
-        else
-            LogRel(("DBGF: RTDbgModCreateFromImage failed with rc=%Rrc for module \"%s\" (%s)\n",
-                    rc, pszName, pszFilename));
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Lazily populates the specified address space.
- *
- * @param   pVM                 The VM handle.
- * @param   hAlias              The alias.
- */
-static void dbgfR3AsLazyPopulate(PVM pVM, RTDBGAS hAlias)
-{
-    DBGF_AS_DB_LOCK_WRITE(pVM);
-    uintptr_t iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
-    if (!pVM->dbgf.s.afAsAliasPopuplated[iAlias])
-    {
-        RTDBGAS hAs = pVM->dbgf.s.ahAsAliases[iAlias];
-        if (hAlias == DBGF_AS_R0)
-            PDMR3LdrEnumModules(pVM, dbgfR3AsLazyPopulateR0Callback, hAs);
-        /** @todo what do we do about DBGF_AS_RC?  */
-
-        pVM->dbgf.s.afAsAliasPopuplated[iAlias] = true;
-    }
-    DBGF_AS_DB_UNLOCK_WRITE(pVM);
-}
-
-
-/**
- * Resolves the address space handle into a real handle if it's an alias.
- *
- * @returns Real address space handle. NIL_RTDBGAS if invalid handle.
- *
- * @param   pVM             The VM handle.
- * @param   hAlias          The possibly address space alias.
- *
- * @remarks Doesn't take any locks.
- */
-VMMR3DECL(RTDBGAS) DBGFR3AsResolve(PVM pVM, RTDBGAS hAlias)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, NULL);
-    AssertCompileNS(NIL_RTDBGAS == (RTDBGAS)0);
-
-    uintptr_t   iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
-    if (iAlias < DBGF_AS_COUNT)
-        ASMAtomicReadHandle(&pVM->dbgf.s.ahAsAliases[iAlias], &hAlias);
-    return hAlias;
-}
-
-
-/**
- * Resolves the address space handle into a real handle if it's an alias,
- * and retains whatever it is.
- *
- * @returns Real address space handle. NIL_RTDBGAS if invalid handle.
- *
- * @param   pVM             The VM handle.
- * @param   hAlias          The possibly address space alias.
- */
-VMMR3DECL(RTDBGAS) DBGFR3AsResolveAndRetain(PVM pVM, RTDBGAS hAlias)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, NULL);
-    AssertCompileNS(NIL_RTDBGAS == (RTDBGAS)0);
-
-    uint32_t    cRefs;
-    uintptr_t   iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
-    if (iAlias < DBGF_AS_COUNT)
-    {
-        if (DBGF_AS_IS_FIXED_ALIAS(hAlias))
-        {
-            /* Perform lazy address space population. */
-            if (!pVM->dbgf.s.afAsAliasPopuplated[iAlias])
-                dbgfR3AsLazyPopulate(pVM, hAlias);
-
-            /* Won't ever change, no need to grab the lock. */
-            hAlias = pVM->dbgf.s.ahAsAliases[iAlias];
-            cRefs = RTDbgAsRetain(hAlias);
-        }
-        else
-        {
-            /* May change, grab the lock so we can read it safely. */
-            DBGF_AS_DB_LOCK_READ(pVM);
-            hAlias = pVM->dbgf.s.ahAsAliases[iAlias];
-            cRefs = RTDbgAsRetain(hAlias);
-            DBGF_AS_DB_UNLOCK_READ(pVM);
-        }
-    }
-    else
-        /* Not an alias, just retain it. */
-        cRefs = RTDbgAsRetain(hAlias);
-
-    return cRefs != UINT32_MAX ? hAlias : NIL_RTDBGAS;
-}
-
-
-/**
- * Query an address space by name.
- *
- * @returns Retained address space handle if found, NIL_RTDBGAS if not.
- *
- * @param   pVM         The VM handle.
- * @param   pszName     The name.
- */
-VMMR3DECL(RTDBGAS) DBGFR3AsQueryByName(PVM pVM, const char *pszName)
-{
-    /*
-     * Validate the input.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, NIL_RTDBGAS);
-    AssertPtrReturn(pszName, NIL_RTDBGAS);
-    AssertReturn(*pszName, NIL_RTDBGAS);
-
-    /*
-     * Look it up in the string space and retain the result.
-     */
-    RTDBGAS hDbgAs = NIL_RTDBGAS;
-    DBGF_AS_DB_LOCK_READ(pVM);
-
-    PRTSTRSPACECORE pNode = RTStrSpaceGet(&pVM->dbgf.s.AsNameSpace, pszName);
-    if (pNode)
-    {
-        PDBGFASDBNODE pDbNode = RT_FROM_MEMBER(pNode, DBGFASDBNODE, NameCore);
-        hDbgAs = (RTDBGAS)pDbNode->HandleCore.Key;
-        uint32_t cRefs = RTDbgAsRetain(hDbgAs);
-        if (RT_UNLIKELY(cRefs == UINT32_MAX))
-            hDbgAs = NIL_RTDBGAS;
-    }
-    DBGF_AS_DB_UNLOCK_READ(pVM);
-
-    return hDbgAs;
-}
-
-
-/**
- * Query an address space by process ID.
- *
- * @returns Retained address space handle if found, NIL_RTDBGAS if not.
- *
- * @param   pVM         The VM handle.
- * @param   ProcId      The process ID.
- */
-VMMR3DECL(RTDBGAS) DBGFR3AsQueryByPid(PVM pVM, RTPROCESS ProcId)
-{
-    /*
-     * Validate the input.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, NIL_RTDBGAS);
-    AssertReturn(ProcId != NIL_RTPROCESS, NIL_RTDBGAS);
-
-    /*
-     * Look it up in the PID tree and retain the result.
-     */
-    RTDBGAS hDbgAs = NIL_RTDBGAS;
-    DBGF_AS_DB_LOCK_READ(pVM);
-
-    PAVLU32NODECORE pNode = RTAvlU32Get(&pVM->dbgf.s.AsPidTree, ProcId);
-    if (pNode)
-    {
-        PDBGFASDBNODE pDbNode = RT_FROM_MEMBER(pNode, DBGFASDBNODE, PidCore);
-        hDbgAs = (RTDBGAS)pDbNode->HandleCore.Key;
-        uint32_t cRefs = RTDbgAsRetain(hDbgAs);
-        if (RT_UNLIKELY(cRefs == UINT32_MAX))
-            hDbgAs = NIL_RTDBGAS;
-    }
-    DBGF_AS_DB_UNLOCK_READ(pVM);
-
-    return hDbgAs;
-}
-
-
-/**
- * Searches for the file in the path.
- *
- * The file is first tested without any path modification, then we walk the path
- * looking in each directory.
- *
- * @returns VBox status code.
- * @param   pszFilename     The file to search for.
- * @param   pszPath         The search path.
- * @param   pfnOpen         The open callback function.
- * @param   pvUser          User argument for the callback.
- */
-static int dbgfR3AsSearchPath(const char *pszFilename, const char *pszPath, PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
-{
-    char szFound[RTPATH_MAX];
-
-    /* Check the filename length. */
-    size_t const    cchFilename = strlen(pszFilename);
-    if (cchFilename >= sizeof(szFound))
-        return VERR_FILENAME_TOO_LONG;
-    const char     *pszName = RTPathFilename(pszFilename);
-    if (!pszName)
-        return VERR_IS_A_DIRECTORY;
-    size_t const    cchName = strlen(pszName);
-
-    /*
-     * Try default location first.
-     */
-    memcpy(szFound, pszFilename, cchFilename + 1);
-    int rc = pfnOpen(szFound, pvUser);
-    if (RT_SUCCESS(rc))
-        return rc;
-
-    /*
-     * Walk the search path.
-     */
-    const char *psz = pszPath;
-    while (*psz)
-    {
-        /* Skip leading blanks - no directories with leading spaces, thank you. */
-        while (RT_C_IS_BLANK(*psz))
-            psz++;
-
-        /* Find the end of this element. */
-        const char *pszNext;
-        const char *pszEnd = strchr(psz, ';');
-        if (!pszEnd)
-            pszEnd = pszNext = strchr(psz, '\0');
-        else
-            pszNext = pszEnd + 1;
-        if (pszEnd != psz)
-        {
-            size_t const cch = pszEnd - psz;
-            if (cch + 1 + cchName < sizeof(szFound))
-            {
-                /** @todo RTPathCompose, RTPathComposeN(). This code isn't right
-                 * for 'E:' on DOS systems. It may also create unwanted double slashes. */
-                memcpy(szFound, psz, cch);
-                szFound[cch] = '/';
-                memcpy(szFound + cch + 1, pszName, cchName + 1);
-                int rc2 = pfnOpen(szFound, pvUser);
-                if (RT_SUCCESS(rc2))
-                    return rc2;
-                if (    rc2 != rc
-                    &&  (   rc == VERR_FILE_NOT_FOUND
-                         || rc == VERR_OPEN_FAILED))
-                    rc = rc2;
-            }
-        }
-
-        /* advance */
-        psz = pszNext;
-    }
-
-    /*
-     * Walk the path once again, this time do a depth search.
-     */
-    /** @todo do a depth search using the specified path. */
-
-    /* failed */
-    return rc;
-}
-
-
-/**
- * Same as dbgfR3AsSearchEnv, except that the path is taken from the environment.
- *
- * It the environment variable doesn't exist, the current directory is searched instead.
- *
- * @returns VBox status code.
- * @param   pszFilename     The filename.
- * @param   pszEnvVar       The environment variable name.
- * @param   pfnOpen         The open callback function.
- * @param   pvUser          User argument for the callback.
- */
-static int dbgfR3AsSearchEnvPath(const char *pszFilename, const char *pszEnvVar, PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
-{
-    int     rc;
-    char   *pszPath = RTEnvDupEx(RTENV_DEFAULT, pszEnvVar);
-    if (pszPath)
-    {
-        rc = dbgfR3AsSearchPath(pszFilename, pszPath, pfnOpen, pvUser);
-        RTStrFree(pszPath);
-    }
-    else
-        rc = dbgfR3AsSearchPath(pszFilename, ".", pfnOpen, pvUser);
-    return rc;
-}
-
-
-/**
- * Callback function used by DBGFR3AsLoadImage.
- *
- * @returns VBox status code.
- * @param   pszFilename     The filename under evaluation.
- * @param   pvUser          Use arguments (DBGFR3ASLOADOPENDATA).
- */
-static DECLCALLBACK(int) dbgfR3AsLoadImageOpen(const char *pszFilename, void *pvUser)
-{
-    DBGFR3ASLOADOPENDATA *pData = (DBGFR3ASLOADOPENDATA *)pvUser;
-    return RTDbgModCreateFromImage(&pData->hMod, pszFilename, pData->pszModName, pData->fFlags);
-}
-
-
-/**
- * Load symbols from an executable module into the specified address space.
- *
- * If an module exist at the specified address it will be replaced by this
- * call, otherwise a new module is created.
- *
- * @returns VBox status code.
- *
- * @param   pVM             The VM handle.
- * @param   hDbgAs          The address space.
- * @param   pszFilename     The filename of the executable module.
- * @param   pszModName      The module name. If NULL, then then the file name
- *                          base is used (no extension or nothing).
- * @param   pModAddress     The load address of the module.
- * @param   iModSeg         The segment to load, pass NIL_RTDBGSEGIDX to load
- *                          the whole image.
- * @param   fFlags          Flags reserved for future extensions, must be 0.
- */
-VMMR3DECL(int) DBGFR3AsLoadImage(PVM pVM, RTDBGAS hDbgAs, const char *pszFilename, const char *pszModName, PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, uint32_t fFlags)
-{
-    /*
-     * Validate input
-     */
-    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
-    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
-    AssertReturn(DBGFR3AddrIsValid(pVM, pModAddress), VERR_INVALID_PARAMETER);
-    AssertReturn(fFlags == 0, VERR_INVALID_PARAMETER);
-    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
-    if (hRealAS == NIL_RTDBGAS)
-        return VERR_INVALID_HANDLE;
-
-    /*
-     * Do the work.
-     */
-    DBGFR3ASLOADOPENDATA Data;
-    Data.pszModName = pszModName;
-    Data.uSubtrahend = 0;
-    Data.fFlags = 0;
-    Data.hMod = NIL_RTDBGMOD;
-    int rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_IMAGE_PATH", dbgfR3AsLoadImageOpen, &Data);
-    if (RT_FAILURE(rc))
-        rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_PATH", dbgfR3AsLoadImageOpen, &Data);
-    if (RT_SUCCESS(rc))
-    {
-        rc = DBGFR3AsLinkModule(pVM, hRealAS, Data.hMod, pModAddress, iModSeg, 0);
-        if (RT_FAILURE(rc))
-            RTDbgModRelease(Data.hMod);
-    }
-
-    RTDbgAsRelease(hRealAS);
-    return rc;
-}
-
-
-/**
- * Callback function used by DBGFR3AsLoadMap.
- *
- * @returns VBox status code.
- * @param   pszFilename     The filename under evaluation.
- * @param   pvUser          Use arguments (DBGFR3ASLOADOPENDATA).
- */
-static DECLCALLBACK(int) dbgfR3AsLoadMapOpen(const char *pszFilename, void *pvUser)
-{
-    DBGFR3ASLOADOPENDATA *pData = (DBGFR3ASLOADOPENDATA *)pvUser;
-    return RTDbgModCreateFromMap(&pData->hMod, pszFilename, pData->pszModName, pData->uSubtrahend, pData->fFlags);
-}
-
-
-/**
- * Load symbols from a map file into a module at the specified address space.
- *
- * If an module exist at the specified address it will be replaced by this
- * call, otherwise a new module is created.
- *
- * @returns VBox status code.
- *
- * @param   pVM             The VM handle.
- * @param   hDbgAs          The address space.
- * @param   pszFilename     The map file.
- * @param   pszModName      The module name. If NULL, then then the file name
- *                          base is used (no extension or nothing).
- * @param   pModAddress     The load address of the module.
- * @param   iModSeg         The segment to load, pass NIL_RTDBGSEGIDX to load
- *                          the whole image.
- * @param   uSubtrahend     Value to to subtract from the symbols in the map
- *                          file. This is useful for the linux System.map and
- *                          /proc/kallsyms.
- * @param   fFlags          Flags reserved for future extensions, must be 0.
- */
-VMMR3DECL(int) DBGFR3AsLoadMap(PVM pVM, RTDBGAS hDbgAs, const char *pszFilename, const char *pszModName,
-                               PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, RTGCUINTPTR uSubtrahend, uint32_t fFlags)
-{
-    /*
-     * Validate input
-     */
-    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
-    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
-    AssertReturn(DBGFR3AddrIsValid(pVM, pModAddress), VERR_INVALID_PARAMETER);
-    AssertReturn(fFlags == 0, VERR_INVALID_PARAMETER);
-    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
-    if (hRealAS == NIL_RTDBGAS)
-        return VERR_INVALID_HANDLE;
-
-    /*
-     * Do the work.
-     */
-    DBGFR3ASLOADOPENDATA Data;
-    Data.pszModName = pszModName;
-    Data.uSubtrahend = uSubtrahend;
-    Data.fFlags = 0;
-    Data.hMod = NIL_RTDBGMOD;
-    int rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_MAP_PATH", dbgfR3AsLoadMapOpen, &Data);
-    if (RT_FAILURE(rc))
-        rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_PATH", dbgfR3AsLoadMapOpen, &Data);
-    if (RT_SUCCESS(rc))
-    {
-        rc = DBGFR3AsLinkModule(pVM, hRealAS, Data.hMod, pModAddress, iModSeg, 0);
-        if (RT_FAILURE(rc))
-            RTDbgModRelease(Data.hMod);
-    }
-
-    RTDbgAsRelease(hRealAS);
-    return rc;
-}
-
-
-/**
- * Wrapper around RTDbgAsModuleLink, RTDbgAsModuleLinkSeg and DBGFR3AsResolve.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   hDbgAs          The address space handle.
- * @param   hMod            The module handle.
- * @param   pModAddress     The link address.
- * @param   iModSeg         The segment to link, NIL_RTDBGSEGIDX for the entire image.
- * @param   fFlags          Flags to pass to the link functions, see RTDBGASLINK_FLAGS_*.
- */
-VMMR3DECL(int) DBGFR3AsLinkModule(PVM pVM, RTDBGAS hDbgAs, RTDBGMOD hMod, PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, uint32_t fFlags)
-{
-    /*
-     * Input validation.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(DBGFR3AddrIsValid(pVM, pModAddress), VERR_INVALID_PARAMETER);
-    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
-    if (hRealAS == NIL_RTDBGAS)
-        return VERR_INVALID_HANDLE;
-
-    /*
-     * Do the job.
-     */
-    int rc;
-    if (iModSeg == NIL_RTDBGSEGIDX)
-        rc = RTDbgAsModuleLink(hRealAS, hMod, pModAddress->FlatPtr, fFlags);
-    else
-        rc = RTDbgAsModuleLinkSeg(hRealAS, hMod, iModSeg, pModAddress->FlatPtr, fFlags);
-
-    RTDbgAsRelease(hRealAS);
-    return rc;
-}
-
-
-/**
- * Adds the module name to the symbol name.
- *
- * @param   pSymbol         The symbol info (in/out).
- * @param   hMod            The module handle.
- */
-static void dbgfR3AsSymbolJoinNames(PRTDBGSYMBOL pSymbol, RTDBGMOD hMod)
-{
-    /* Figure the lengths, adjust them if the result is too long. */
-    const char *pszModName = RTDbgModName(hMod);
-    size_t      cchModName = strlen(pszModName);
-    size_t      cchSymbol  = strlen(pSymbol->szName);
-    if (cchModName + 1 + cchSymbol >= sizeof(pSymbol->szName))
-    {
-        if (cchModName >= sizeof(pSymbol->szName) / 4)
-            cchModName = sizeof(pSymbol->szName) / 4;
-        if (cchModName + 1 + cchSymbol >= sizeof(pSymbol->szName))
-            cchSymbol = sizeof(pSymbol->szName) - cchModName - 2;
-        Assert(cchModName + 1 + cchSymbol < sizeof(pSymbol->szName));
-    }
-
-    /* Do the moving and copying. */
-    memmove(&pSymbol->szName[cchModName + 1], &pSymbol->szName[0], cchSymbol + 1);
-    memcpy(&pSymbol->szName[0], pszModName, cchModName);
-    pSymbol->szName[cchModName] = '!';
-}
-
-
-/** Temporary symbol conversion function. */
-static void dbgfR3AsSymbolConvert(PRTDBGSYMBOL pSymbol, PCDBGFSYMBOL pDbgfSym)
-{
-    pSymbol->offSeg = pSymbol->Value = pDbgfSym->Value;
-    pSymbol->cb = pDbgfSym->cb;
-    pSymbol->iSeg = 0;
-    pSymbol->fFlags = 0;
-    pSymbol->iOrdinal = UINT32_MAX;
-    strcpy(pSymbol->szName, pDbgfSym->szName);
-}
-
-
-/**
- * Query a symbol by address.
- *
- * The returned symbol is the one we consider closes to the specified address.
- *
- * @returns VBox status code. See RTDbgAsSymbolByAddr.
- *
- * @param   pVM                 The VM handle.
- * @param   hDbgAs              The address space handle.
- * @param   pAddress            The address to lookup.
- * @param   poffDisp            Where to return the distance between the
- *                              returned symbol and pAddress. Optional.
- * @param   pSymbol             Where to return the symbol information.
- *                              The returned symbol name will be prefixed by
- *                              the module name as far as space allows.
- * @param   phMod               Where to return the module handle. Optional.
- */
-VMMR3DECL(int) DBGFR3AsSymbolByAddr(PVM pVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress,
-                                    PRTGCINTPTR poffDisp, PRTDBGSYMBOL pSymbol, PRTDBGMOD phMod)
-{
-    /*
-     * Implement the special address space aliases the lazy way.
-     */
-    if (hDbgAs == DBGF_AS_RC_AND_GC_GLOBAL)
-    {
-        int rc = DBGFR3AsSymbolByAddr(pVM, DBGF_AS_RC, pAddress, poffDisp, pSymbol, phMod);
-        if (RT_FAILURE(rc))
-            rc = DBGFR3AsSymbolByAddr(pVM, DBGF_AS_GLOBAL, pAddress, poffDisp, pSymbol, phMod);
-        return rc;
-    }
-
-    /*
-     * Input validation.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(DBGFR3AddrIsValid(pVM, pAddress), VERR_INVALID_PARAMETER);
-    AssertPtrNullReturn(poffDisp, VERR_INVALID_POINTER);
-    AssertPtrReturn(pSymbol, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(phMod, VERR_INVALID_POINTER);
-    if (poffDisp)
-        *poffDisp = 0;
-    if (phMod)
-        *phMod = NIL_RTDBGMOD;
-    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
-    if (hRealAS == NIL_RTDBGAS)
-        return VERR_INVALID_HANDLE;
-
-    /*
-     * Do the lookup.
-     */
-    RTDBGMOD hMod;
-    int rc = RTDbgAsSymbolByAddr(hRealAS, pAddress->FlatPtr, poffDisp, pSymbol, &hMod);
-    if (RT_SUCCESS(rc))
-    {
-        dbgfR3AsSymbolJoinNames(pSymbol, hMod);
-        if (!phMod)
-            RTDbgModRelease(hMod);
-    }
-    /* Temporary conversions. */
-    else if (hDbgAs == DBGF_AS_GLOBAL)
-    {
-        DBGFSYMBOL DbgfSym;
-        rc = DBGFR3SymbolByAddr(pVM, pAddress->FlatPtr, poffDisp, &DbgfSym);
-        if (RT_SUCCESS(rc))
-            dbgfR3AsSymbolConvert(pSymbol, &DbgfSym);
-    }
-    else if (hDbgAs == DBGF_AS_R0)
-    {
-        RTR0PTR     R0PtrMod;
-        char        szNearSym[260];
-        RTR0PTR     R0PtrNearSym;
-        RTR0PTR     R0PtrNearSym2;
-        rc = PDMR3LdrQueryR0ModFromPC(pVM, pAddress->FlatPtr,
-                                      pSymbol->szName, sizeof(pSymbol->szName) / 2, &R0PtrMod,
-                                      &szNearSym[0],   sizeof(szNearSym),           &R0PtrNearSym,
-                                      NULL,            0,                           &R0PtrNearSym2);
-        if (RT_SUCCESS(rc))
-        {
-            pSymbol->offSeg     = pSymbol->Value = R0PtrNearSym;
-            pSymbol->cb         = R0PtrNearSym2 > R0PtrNearSym ? R0PtrNearSym2 - R0PtrNearSym : 0;
-            pSymbol->iSeg       = 0;
-            pSymbol->fFlags     = 0;
-            pSymbol->iOrdinal   = UINT32_MAX;
-            size_t offName = strlen(pSymbol->szName);
-            pSymbol->szName[offName++] = '!';
-            size_t cchNearSym = strlen(szNearSym);
-            if (cchNearSym + offName >= sizeof(pSymbol->szName))
-                cchNearSym = sizeof(pSymbol->szName) - offName - 1;
-            strncpy(&pSymbol->szName[offName], szNearSym, cchNearSym);
-            pSymbol->szName[offName + cchNearSym] = '\0';
-            if (poffDisp)
-                *poffDisp = pAddress->FlatPtr - pSymbol->Value;
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * Convenience function that combines RTDbgSymbolDup and DBGFR3AsSymbolByAddr.
- *
- * @returns Pointer to the symbol on success. This must be free using
- *          RTDbgSymbolFree(). NULL is returned if not found or any error
- *          occurs.
- *
- * @param   pVM                 The VM handle.
- * @param   hDbgAs              See DBGFR3AsSymbolByAddr.
- * @param   pAddress            See DBGFR3AsSymbolByAddr.
- * @param   poffDisp            See DBGFR3AsSymbolByAddr.
- * @param   phMod               See DBGFR3AsSymbolByAddr.
- */
-VMMR3DECL(PRTDBGSYMBOL) DBGFR3AsSymbolByAddrA(PVM pVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress, PRTGCINTPTR poffDisp, PRTDBGMOD phMod)
-{
-    RTDBGSYMBOL SymInfo;
-    int rc = DBGFR3AsSymbolByAddr(pVM, hDbgAs, pAddress, poffDisp, &SymInfo, phMod);
-    if (RT_SUCCESS(rc))
-        return RTDbgSymbolDup(&SymInfo);
-    return NULL;
-}
-
-
-/**
- * Query a symbol by name.
- *
- * The symbol can be prefixed by a module name pattern to scope the search. The
- * pattern is a simple string pattern with '*' and '?' as wild chars. See
- * RTStrSimplePatternMatch().
- *
- * @returns VBox status code. See RTDbgAsSymbolByAddr.
- *
- * @param   pVM                 The VM handle.
- * @param   hDbgAs              The address space handle.
- * @param   pszSymbol           The symbol to search for, maybe prefixed by a
- *                              module pattern.
- * @param   pSymbol             Where to return the symbol information.
- *                              The returned symbol name will be prefixed by
- *                              the module name as far as space allows.
- * @param   phMod               Where to return the module handle. Optional.
- */
-VMMR3DECL(int) DBGFR3AsSymbolByName(PVM pVM, RTDBGAS hDbgAs, const char *pszSymbol,
-                                    PRTDBGSYMBOL pSymbol, PRTDBGMOD phMod)
-{
-    /*
-     * Implement the special address space aliases the lazy way.
-     */
-    if (hDbgAs == DBGF_AS_RC_AND_GC_GLOBAL)
-    {
-        int rc = DBGFR3AsSymbolByName(pVM, DBGF_AS_RC, pszSymbol, pSymbol, phMod);
-        if (RT_FAILURE(rc))
-            rc = DBGFR3AsSymbolByName(pVM, DBGF_AS_GLOBAL, pszSymbol, pSymbol, phMod);
-        return rc;
-    }
-
-    /*
-     * Input validation.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertPtrReturn(pSymbol, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(phMod, VERR_INVALID_POINTER);
-    if (phMod)
-        *phMod = NIL_RTDBGMOD;
-    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
-    if (hRealAS == NIL_RTDBGAS)
-        return VERR_INVALID_HANDLE;
-
-
-    /*
-     * Do the lookup.
-     */
-    RTDBGMOD hMod;
-    int rc = RTDbgAsSymbolByName(hRealAS, pszSymbol, pSymbol, &hMod);
-    if (RT_SUCCESS(rc))
-    {
-        dbgfR3AsSymbolJoinNames(pSymbol, hMod);
-        if (!phMod)
-            RTDbgModRelease(hMod);
-    }
-    /* Temporary conversion. */
-    else if (hDbgAs == DBGF_AS_GLOBAL)
-    {
-        DBGFSYMBOL DbgfSym;
-        rc = DBGFR3SymbolByName(pVM, pszSymbol, &DbgfSym);
-        if (RT_SUCCESS(rc))
-            dbgfR3AsSymbolConvert(pSymbol, &DbgfSym);
-    }
-
-    return rc;
-}
-
Index: unk/src/VBox/VMM/DBGFBp.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFBp.cpp	(revision 35332)
+++ 	(revision )
@@ -1,968 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Breakpoint Management.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/selm.h>
-#include <VBox/rem.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/mm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-RT_C_DECLS_BEGIN
-static DECLCALLBACK(int) dbgfR3BpSetReg(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable,
-                                        uint8_t u8Type, uint8_t cb, PRTUINT piBp);
-static DECLCALLBACK(int) dbgfR3BpSetInt3(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp);
-static DECLCALLBACK(int) dbgfR3BpSetREM(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp);
-static DECLCALLBACK(int) dbgfR3BpClear(PVM pVM, RTUINT iBp);
-static DECLCALLBACK(int) dbgfR3BpEnable(PVM pVM, RTUINT iBp);
-static DECLCALLBACK(int) dbgfR3BpDisable(PVM pVM, RTUINT iBp);
-static DECLCALLBACK(int) dbgfR3BpEnum(PVM pVM, PFNDBGFBPENUM pfnCallback, void *pvUser);
-static int dbgfR3BpRegArm(PVM pVM, PDBGFBP pBp);
-static int dbgfR3BpRegDisarm(PVM pVM, PDBGFBP pBp);
-static int dbgfR3BpInt3Arm(PVM pVM, PDBGFBP pBp);
-static int dbgfR3BpInt3Disarm(PVM pVM, PDBGFBP pBp);
-RT_C_DECLS_END
-
-
-
-/**
- * Initialize the breakpoint stuff.
- *
- * @returns VINF_SUCCESS
- * @param   pVM     The VM handle.
- */
-int dbgfR3BpInit(PVM pVM)
-{
-    /*
-     * Init structures.
-     */
-    unsigned i;
-    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++)
-    {
-        pVM->dbgf.s.aHwBreakpoints[i].iBp = i;
-        pVM->dbgf.s.aHwBreakpoints[i].enmType = DBGFBPTYPE_FREE;
-        pVM->dbgf.s.aHwBreakpoints[i].u.Reg.iReg = i;
-    }
-
-    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
-    {
-        pVM->dbgf.s.aBreakpoints[i].iBp = i + RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
-        pVM->dbgf.s.aBreakpoints[i].enmType = DBGFBPTYPE_FREE;
-    }
-
-    /*
-     * Register saved state.
-     */
-    /** @todo */
-
-    return VINF_SUCCESS;
-}
-
-
-
-/**
- * Allocate a breakpoint.
- *
- * @returns Pointer to the allocated breakpoint.
- * @returns NULL if we're out of breakpoints.
- * @param   pVM     The VM handle.
- * @param   enmType The type to allocate.
- */
-static PDBGFBP dbgfR3BpAlloc(PVM pVM, DBGFBPTYPE enmType)
-{
-    /*
-     * Determine which array to search.
-     */
-    unsigned cBps;
-    PRTUINT  pcBpsCur;
-    PDBGFBP  paBps;
-    switch (enmType)
-    {
-        case DBGFBPTYPE_REG:
-            cBps = RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
-            paBps = &pVM->dbgf.s.aHwBreakpoints[0];
-            pcBpsCur = &pVM->dbgf.s.cHwBreakpoints;
-            break;
-
-        case DBGFBPTYPE_INT3:
-        case DBGFBPTYPE_REM:
-            cBps = RT_ELEMENTS(pVM->dbgf.s.aBreakpoints);
-            paBps = &pVM->dbgf.s.aBreakpoints[0];
-            pcBpsCur = &pVM->dbgf.s.cBreakpoints;
-            break;
-
-        default:
-            AssertMsgFailed(("enmType=%d\n", enmType));
-            return NULL;
-    }
-
-    /*
-     * Search.
-     */
-    for (unsigned iBp = 0; iBp < cBps; iBp++)
-        if (paBps[iBp].enmType == DBGFBPTYPE_FREE)
-        {
-            ++*pcBpsCur;
-            paBps[iBp].cHits   = 0;
-            paBps[iBp].enmType = enmType;
-            return &paBps[iBp];
-        }
-
-    LogFlow(("dbgfR3BpAlloc: returns NULL - we're out of breakpoint slots! %u/%u\n", *pcBpsCur, cBps));
-    return NULL;
-}
-
-
-/**
- * Get a breakpoint give by breakpoint id.
- *
- * @returns Pointer to the allocated breakpoint.
- * @returns NULL if the breakpoint is invalid.
- * @param   pVM     The VM handle.
- * @param   iBp     The breakpoint id.
- */
-static PDBGFBP dbgfR3BpGet(PVM pVM, RTUINT iBp)
-{
-    /* Find it. */
-    PDBGFBP pBp;
-    if (iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints))
-        pBp = &pVM->dbgf.s.aHwBreakpoints[iBp];
-    else
-    {
-        iBp -= RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
-        if (iBp >= RT_ELEMENTS(pVM->dbgf.s.aBreakpoints))
-            return NULL;
-        pBp = &pVM->dbgf.s.aBreakpoints[iBp];
-    }
-
-    /* check if it's valid. */
-    switch (pBp->enmType)
-    {
-        case DBGFBPTYPE_FREE:
-            return NULL;
-
-        case DBGFBPTYPE_REG:
-        case DBGFBPTYPE_INT3:
-        case DBGFBPTYPE_REM:
-            break;
-
-        default:
-            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
-            return NULL;
-    }
-
-    return pBp;
-}
-
-
-/**
- * Get a breakpoint give by address.
- *
- * @returns Pointer to the allocated breakpoint.
- * @returns NULL if the breakpoint is invalid.
- * @param   pVM     The VM handle.
- * @param   enmType The breakpoint type.
- * @param   GCPtr   The breakpoint address.
- */
-static PDBGFBP dbgfR3BpGetByAddr(PVM pVM, DBGFBPTYPE enmType, RTGCUINTPTR GCPtr)
-{
-    /*
-     * Determine which array to search.
-     */
-    unsigned cBps;
-    PDBGFBP  paBps;
-    switch (enmType)
-    {
-        case DBGFBPTYPE_REG:
-            cBps = RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
-            paBps = &pVM->dbgf.s.aHwBreakpoints[0];
-            break;
-
-        case DBGFBPTYPE_INT3:
-        case DBGFBPTYPE_REM:
-            cBps = RT_ELEMENTS(pVM->dbgf.s.aBreakpoints);
-            paBps = &pVM->dbgf.s.aBreakpoints[0];
-            break;
-
-        default:
-            AssertMsgFailed(("enmType=%d\n", enmType));
-            return NULL;
-    }
-
-    /*
-     * Search.
-     */
-    for (unsigned iBp = 0; iBp < cBps; iBp++)
-    {
-        if (    paBps[iBp].enmType == enmType
-            &&  paBps[iBp].GCPtr == GCPtr)
-            return &paBps[iBp];
-    }
-
-    return NULL;
-}
-
-
-/**
- * Frees a breakpoint.
- *
- * @param   pVM     The VM handle.
- * @param   pBp     The breakpoint to free.
- */
-static void dbgfR3BpFree(PVM pVM, PDBGFBP pBp)
-{
-    switch (pBp->enmType)
-    {
-        case DBGFBPTYPE_FREE:
-            AssertMsgFailed(("Already freed!\n"));
-            return;
-
-        case DBGFBPTYPE_REG:
-            Assert(pVM->dbgf.s.cHwBreakpoints > 0);
-            pVM->dbgf.s.cHwBreakpoints--;
-            break;
-
-        case DBGFBPTYPE_INT3:
-        case DBGFBPTYPE_REM:
-            Assert(pVM->dbgf.s.cBreakpoints > 0);
-            pVM->dbgf.s.cBreakpoints--;
-            break;
-
-        default:
-            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
-            return;
-
-    }
-    pBp->enmType = DBGFBPTYPE_FREE;
-}
-
-
-/**
- * Sets a breakpoint (int 3 based).
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pAddress    The address of the breakpoint.
- * @param   iHitTrigger The hit count at which the breakpoint start triggering.
- *                      Use 0 (or 1) if it's gonna trigger at once.
- * @param   iHitDisable The hit count which disables the breakpoint.
- *                      Use ~(uint64_t) if it's never gonna be disabled.
- * @param   piBp        Where to store the breakpoint id. (optional)
- * @thread  Any thread.
- */
-VMMR3DECL(int) DBGFR3BpSet(PVM pVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable, PRTUINT piBp)
-{
-    /*
-     * This must be done in EMT.
-     */
-    /** @todo SMP? */
-    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpSetInt3, 5, pVM, pAddress, &iHitTrigger, &iHitDisable, piBp);
-    LogFlow(("DBGFR3BpSet: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Sets a breakpoint (int 3 based).
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pAddress        The address of the breakpoint.
- * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
- *                          Use 0 (or 1) if it's gonna trigger at once.
- * @param   piHitDisable    The hit count which disables the breakpoint.
- *                          Use ~(uint64_t) if it's never gonna be disabled.
- * @param   piBp            Where to store the breakpoint id. (optional)
- * @thread  Any thread.
- */
-static DECLCALLBACK(int) dbgfR3BpSetInt3(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp)
-{
-    /*
-     * Validate input.
-     */
-    if (!DBGFR3AddrIsValid(pVM, pAddress))
-        return VERR_INVALID_PARAMETER;
-    if (*piHitTrigger > *piHitDisable)
-        return VERR_INVALID_PARAMETER;
-    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
-    if (piBp)
-        *piBp = ~0;
-
-    /*
-     * Check if the breakpoint already exists.
-     */
-    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_INT3, pAddress->FlatPtr);
-    if (pBp)
-    {
-        int rc = VINF_SUCCESS;
-        if (!pBp->fEnabled)
-            rc = dbgfR3BpInt3Arm(pVM, pBp);
-        if (RT_SUCCESS(rc))
-        {
-            rc = VINF_DBGF_BP_ALREADY_EXIST;
-            if (piBp)
-                *piBp = pBp->iBp;
-        }
-        return rc;
-    }
-
-    /*
-     * Allocate and initialize the bp.
-     */
-    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_INT3);
-    if (!pBp)
-        return VERR_DBGF_NO_MORE_BP_SLOTS;
-    pBp->GCPtr       = pAddress->FlatPtr;
-    pBp->iHitTrigger = *piHitTrigger;
-    pBp->iHitDisable = *piHitDisable;
-    pBp->fEnabled    = true;
-
-    /*
-     * Now ask REM to set the breakpoint.
-     */
-    int rc = dbgfR3BpInt3Arm(pVM, pBp);
-    if (RT_SUCCESS(rc))
-    {
-        if (piBp)
-            *piBp = pBp->iBp;
-    }
-    else
-        dbgfR3BpFree(pVM, pBp);
-
-    return rc;
-}
-
-
-/**
- * Arms an int 3 breakpoint.
- * This is used to implement both DBGFR3BpSetReg() and DBGFR3BpEnable().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pBp         The breakpoint.
- */
-static int dbgfR3BpInt3Arm(PVM pVM, PDBGFBP pBp)
-{
-    /** @todo should actually use physical address here! */
-
-    /* @todo SMP support! */
-    VMCPUID idCpu = 0;
-
-    /*
-     * Save current byte and write int3 instruction.
-     */
-    DBGFADDRESS Addr;
-    DBGFR3AddrFromFlat(pVM, &Addr, pBp->GCPtr);
-    int rc = DBGFR3MemRead(pVM, idCpu, &Addr, &pBp->u.Int3.bOrg, 1);
-    if (RT_SUCCESS(rc))
-    {
-        static const uint8_t s_bInt3 = 0xcc;
-        rc = DBGFR3MemWrite(pVM, idCpu, &Addr, &s_bInt3, 1);
-    }
-    return rc;
-}
-
-
-/**
- * Disarms an int 3 breakpoint.
- * This is used to implement both DBGFR3BpClear() and DBGFR3BpDisable().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pBp         The breakpoint.
- */
-static int dbgfR3BpInt3Disarm(PVM pVM, PDBGFBP pBp)
-{
-    /* @todo SMP support! */
-    VMCPUID idCpu = 0;
-
-    /*
-     * Check that the current byte is the int3 instruction, and restore the original one.
-     * We currently ignore invalid bytes.
-     */
-    DBGFADDRESS     Addr;
-    DBGFR3AddrFromFlat(pVM, &Addr, pBp->GCPtr);
-    uint8_t         bCurrent;
-    int rc = DBGFR3MemRead(pVM, idCpu, &Addr, &bCurrent, 1);
-    if (bCurrent == 0xcc)
-        rc = DBGFR3MemWrite(pVM, idCpu, &Addr, &pBp->u.Int3.bOrg, 1);
-    return rc;
-}
-
-
-/**
- * Sets a register breakpoint.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pAddress    The address of the breakpoint.
- * @param   iHitTrigger The hit count at which the breakpoint start triggering.
- *                      Use 0 (or 1) if it's gonna trigger at once.
- * @param   iHitDisable The hit count which disables the breakpoint.
- *                      Use ~(uint64_t) if it's never gonna be disabled.
- * @param   fType       The access type (one of the X86_DR7_RW_* defines).
- * @param   cb          The access size - 1,2,4 or 8 (the latter is AMD64 long mode only.
- *                      Must be 1 if fType is X86_DR7_RW_EO.
- * @param   piBp        Where to store the breakpoint id. (optional)
- * @thread  Any thread.
- */
-VMMR3DECL(int) DBGFR3BpSetReg(PVM pVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable,
-                              uint8_t fType, uint8_t cb, PRTUINT piBp)
-{
-    /** @todo SMP - broadcast, VT-x/AMD-V. */
-    /*
-     * This must be done in EMT.
-     */
-    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpSetReg, 7, pVM, pAddress, &iHitTrigger, &iHitDisable, fType, cb, piBp);
-    LogFlow(("DBGFR3BpSetReg: returns %Rrc\n", rc));
-    return rc;
-
-}
-
-
-/**
- * Sets a register breakpoint.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pAddress        The address of the breakpoint.
- * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
- *                          Use 0 (or 1) if it's gonna trigger at once.
- * @param   piHitDisable    The hit count which disables the breakpoint.
- *                          Use ~(uint64_t) if it's never gonna be disabled.
- * @param   fType           The access type (one of the X86_DR7_RW_* defines).
- * @param   cb              The access size - 1,2,4 or 8 (the latter is AMD64 long mode only.
- *                          Must be 1 if fType is X86_DR7_RW_EO.
- * @param   piBp            Where to store the breakpoint id. (optional)
- * @thread  EMT
- * @internal
- */
-static DECLCALLBACK(int) dbgfR3BpSetReg(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable,
-                                        uint8_t fType, uint8_t cb, PRTUINT piBp)
-{
-    /*
-     * Validate input.
-     */
-    if (!DBGFR3AddrIsValid(pVM, pAddress))
-        return VERR_INVALID_PARAMETER;
-    if (*piHitTrigger > *piHitDisable)
-        return VERR_INVALID_PARAMETER;
-    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
-    if (piBp)
-        *piBp = ~0;
-    switch (fType)
-    {
-        case X86_DR7_RW_EO:
-            if (cb == 1)
-                break;
-            AssertMsgFailed(("fType=%#x cb=%d != 1\n", fType, cb));
-            return VERR_INVALID_PARAMETER;
-        case X86_DR7_RW_IO:
-        case X86_DR7_RW_RW:
-        case X86_DR7_RW_WO:
-            break;
-        default:
-            AssertMsgFailed(("fType=%#x\n", fType));
-            return VERR_INVALID_PARAMETER;
-    }
-    switch (cb)
-    {
-        case 1:
-        case 2:
-        case 4:
-            break;
-        default:
-            AssertMsgFailed(("cb=%#x\n", cb));
-            return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Check if the breakpoint already exists.
-     */
-    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_REG, pAddress->FlatPtr);
-    if (    pBp
-        &&  pBp->u.Reg.cb == cb
-        &&  pBp->u.Reg.fType == fType)
-    {
-        int rc = VINF_SUCCESS;
-        if (!pBp->fEnabled)
-            rc = dbgfR3BpRegArm(pVM, pBp);
-        if (RT_SUCCESS(rc))
-        {
-            rc = VINF_DBGF_BP_ALREADY_EXIST;
-            if (piBp)
-                *piBp = pBp->iBp;
-        }
-        return rc;
-    }
-
-    /*
-     * Allocate and initialize the bp.
-     */
-    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_REG);
-    if (!pBp)
-        return VERR_DBGF_NO_MORE_BP_SLOTS;
-    pBp->GCPtr       = pAddress->FlatPtr;
-    pBp->iHitTrigger = *piHitTrigger;
-    pBp->iHitDisable = *piHitDisable;
-    pBp->fEnabled    = true;
-    Assert(pBp->iBp == pBp->u.Reg.iReg);
-    pBp->u.Reg.fType = fType;
-    pBp->u.Reg.cb    = cb;
-
-    /*
-     * Arm the breakpoint.
-     */
-    int rc = dbgfR3BpRegArm(pVM, pBp);
-    if (RT_SUCCESS(rc))
-    {
-        if (piBp)
-            *piBp = pBp->iBp;
-    }
-    else
-        dbgfR3BpFree(pVM, pBp);
-
-    return rc;
-}
-
-
-/**
- * Arms a debug register breakpoint.
- * This is used to implement both DBGFR3BpSetReg() and DBGFR3BpEnable().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pBp         The breakpoint.
- */
-static int dbgfR3BpRegArm(PVM pVM, PDBGFBP pBp)
-{
-    /* @todo SMP support! */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    Assert(pBp->fEnabled);
-    return CPUMRecalcHyperDRx(pVCpu);
-}
-
-
-/**
- * Disarms a debug register breakpoint.
- * This is used to implement both DBGFR3BpClear() and DBGFR3BpDisable().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pBp         The breakpoint.
- */
-static int dbgfR3BpRegDisarm(PVM pVM, PDBGFBP pBp)
-{
-    /** @todo SMP support! */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    Assert(!pBp->fEnabled);
-    return CPUMRecalcHyperDRx(pVCpu);
-}
-
-
-/**
- * Sets a recompiler breakpoint.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pAddress    The address of the breakpoint.
- * @param   iHitTrigger The hit count at which the breakpoint start triggering.
- *                      Use 0 (or 1) if it's gonna trigger at once.
- * @param   iHitDisable The hit count which disables the breakpoint.
- *                      Use ~(uint64_t) if it's never gonna be disabled.
- * @param   piBp        Where to store the breakpoint id. (optional)
- * @thread  Any thread.
- */
-VMMR3DECL(int) DBGFR3BpSetREM(PVM pVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable, PRTUINT piBp)
-{
-    /*
-     * This must be done in EMT.
-     */
-    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpSetREM, 5, pVM, pAddress, &iHitTrigger, &iHitDisable, piBp);
-    LogFlow(("DBGFR3BpSetREM: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * EMT worker for DBGFR3BpSetREM().
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pAddress        The address of the breakpoint.
- * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
- *                          Use 0 (or 1) if it's gonna trigger at once.
- * @param   piHitDisable    The hit count which disables the breakpoint.
- *                          Use ~(uint64_t) if it's never gonna be disabled.
- * @param   piBp            Where to store the breakpoint id. (optional)
- * @thread  EMT
- * @internal
- */
-static DECLCALLBACK(int) dbgfR3BpSetREM(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp)
-{
-    /*
-     * Validate input.
-     */
-    if (!DBGFR3AddrIsValid(pVM, pAddress))
-        return VERR_INVALID_PARAMETER;
-    if (*piHitTrigger > *piHitDisable)
-        return VERR_INVALID_PARAMETER;
-    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
-    if (piBp)
-        *piBp = ~0;
-
-
-    /*
-     * Check if the breakpoint already exists.
-     */
-    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_REM, pAddress->FlatPtr);
-    if (pBp)
-    {
-        int rc = VINF_SUCCESS;
-        if (!pBp->fEnabled)
-            rc = REMR3BreakpointSet(pVM, pBp->GCPtr);
-        if (RT_SUCCESS(rc))
-        {
-            rc = VINF_DBGF_BP_ALREADY_EXIST;
-            if (piBp)
-                *piBp = pBp->iBp;
-        }
-        return rc;
-    }
-
-    /*
-     * Allocate and initialize the bp.
-     */
-    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_REM);
-    if (!pBp)
-        return VERR_DBGF_NO_MORE_BP_SLOTS;
-    pBp->GCPtr       = pAddress->FlatPtr;
-    pBp->iHitTrigger = *piHitTrigger;
-    pBp->iHitDisable = *piHitDisable;
-    pBp->fEnabled    = true;
-
-    /*
-     * Now ask REM to set the breakpoint.
-     */
-    int rc = REMR3BreakpointSet(pVM, pAddress->FlatPtr);
-    if (RT_SUCCESS(rc))
-    {
-        if (piBp)
-            *piBp = pBp->iBp;
-    }
-    else
-        dbgfR3BpFree(pVM, pBp);
-
-    return rc;
-}
-
-
-/**
- * Clears a breakpoint.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   iBp         The id of the breakpoint which should be removed (cleared).
- * @thread  Any thread.
- */
-VMMR3DECL(int) DBGFR3BpClear(PVM pVM, RTUINT iBp)
-{
-    /*
-     * This must be done in EMT.
-     */
-    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpClear, 2, pVM, iBp);
-    LogFlow(("DBGFR3BpClear: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * EMT worker for DBGFR3BpClear().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   iBp         The id of the breakpoint which should be removed (cleared).
- * @thread  EMT
- * @internal
- */
-static DECLCALLBACK(int) dbgfR3BpClear(PVM pVM, RTUINT iBp)
-{
-    /*
-     * Validate input.
-     */
-    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
-    if (!pBp)
-        return VERR_DBGF_BP_NOT_FOUND;
-
-    /*
-     * Disarm the breakpoint if it's enabled.
-     */
-    if (pBp->fEnabled)
-    {
-        pBp->fEnabled = false;
-        int rc;
-        switch (pBp->enmType)
-        {
-            case DBGFBPTYPE_REG:
-                rc = dbgfR3BpRegDisarm(pVM, pBp);
-                break;
-
-            case DBGFBPTYPE_INT3:
-                rc = dbgfR3BpInt3Disarm(pVM, pBp);
-                break;
-
-            case DBGFBPTYPE_REM:
-                rc = REMR3BreakpointClear(pVM, pBp->GCPtr);
-                break;
-
-            default:
-                AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
-                return VERR_INTERNAL_ERROR;
-        }
-        AssertRCReturn(rc, rc);
-    }
-
-    /*
-     * Free the breakpoint.
-     */
-    dbgfR3BpFree(pVM, pBp);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Enables a breakpoint.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   iBp         The id of the breakpoint which should be enabled.
- * @thread  Any thread.
- */
-VMMR3DECL(int) DBGFR3BpEnable(PVM pVM, RTUINT iBp)
-{
-    /*
-     * This must be done in EMT.
-     */
-    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpEnable, 2, pVM, iBp);
-    LogFlow(("DBGFR3BpEnable: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * EMT worker for DBGFR3BpEnable().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   iBp         The id of the breakpoint which should be enabled.
- * @thread  EMT
- * @internal
- */
-static DECLCALLBACK(int) dbgfR3BpEnable(PVM pVM, RTUINT iBp)
-{
-    /*
-     * Validate input.
-     */
-    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
-    if (!pBp)
-        return VERR_DBGF_BP_NOT_FOUND;
-
-    /*
-     * Already enabled?
-     */
-    if (pBp->fEnabled)
-        return VINF_DBGF_BP_ALREADY_ENABLED;
-
-    /*
-     * Remove the breakpoint.
-     */
-    int rc;
-    pBp->fEnabled = true;
-    switch (pBp->enmType)
-    {
-        case DBGFBPTYPE_REG:
-            rc = dbgfR3BpRegArm(pVM, pBp);
-            break;
-
-        case DBGFBPTYPE_INT3:
-            rc = dbgfR3BpInt3Arm(pVM, pBp);
-            break;
-
-        case DBGFBPTYPE_REM:
-            rc = REMR3BreakpointSet(pVM, pBp->GCPtr);
-            break;
-
-        default:
-            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
-            return VERR_INTERNAL_ERROR;
-    }
-    if (RT_FAILURE(rc))
-        pBp->fEnabled = false;
-
-    return rc;
-}
-
-
-/**
- * Disables a breakpoint.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   iBp         The id of the breakpoint which should be disabled.
- * @thread  Any thread.
- */
-VMMR3DECL(int) DBGFR3BpDisable(PVM pVM, RTUINT iBp)
-{
-    /*
-     * This must be done in EMT.
-     */
-    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpDisable, 2, pVM, iBp);
-    LogFlow(("DBGFR3BpDisable: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * EMT worker for DBGFR3BpDisable().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   iBp         The id of the breakpoint which should be disabled.
- * @thread  EMT
- * @internal
- */
-static DECLCALLBACK(int) dbgfR3BpDisable(PVM pVM, RTUINT iBp)
-{
-    /*
-     * Validate input.
-     */
-    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
-    if (!pBp)
-        return VERR_DBGF_BP_NOT_FOUND;
-
-    /*
-     * Already enabled?
-     */
-    if (!pBp->fEnabled)
-        return VINF_DBGF_BP_ALREADY_DISABLED;
-
-    /*
-     * Remove the breakpoint.
-     */
-    pBp->fEnabled = false;
-    int rc;
-    switch (pBp->enmType)
-    {
-        case DBGFBPTYPE_REG:
-            rc = dbgfR3BpRegDisarm(pVM, pBp);
-            break;
-
-        case DBGFBPTYPE_INT3:
-            rc = dbgfR3BpInt3Disarm(pVM, pBp);
-            break;
-
-        case DBGFBPTYPE_REM:
-            rc = REMR3BreakpointClear(pVM, pBp->GCPtr);
-            break;
-
-        default:
-            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
-            return VERR_INTERNAL_ERROR;
-    }
-
-    return rc;
-}
-
-
-/**
- * Enumerate the breakpoints.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pfnCallback The callback function.
- * @param   pvUser      The user argument to pass to the callback.
- * @thread  Any thread but the callback will be called from EMT.
- */
-VMMR3DECL(int) DBGFR3BpEnum(PVM pVM, PFNDBGFBPENUM pfnCallback, void *pvUser)
-{
-    /*
-     * This must be done in EMT.
-     */
-    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpEnum, 3, pVM, pfnCallback, pvUser);
-    LogFlow(("DBGFR3BpClear: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * EMT worker for DBGFR3BpEnum().
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pfnCallback The callback function.
- * @param   pvUser      The user argument to pass to the callback.
- * @thread  EMT
- * @internal
- */
-static DECLCALLBACK(int) dbgfR3BpEnum(PVM pVM, PFNDBGFBPENUM pfnCallback, void *pvUser)
-{
-    /*
-     * Validate input.
-     */
-    AssertMsgReturn(VALID_PTR(pfnCallback), ("pfnCallback=%p\n", pfnCallback), VERR_INVALID_POINTER);
-
-    /*
-     * Enumerate the hardware breakpoints.
-     */
-    unsigned i;
-    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++)
-        if (pVM->dbgf.s.aHwBreakpoints[i].enmType != DBGFBPTYPE_FREE)
-        {
-            int rc = pfnCallback(pVM, pvUser, &pVM->dbgf.s.aHwBreakpoints[i]);
-            if (RT_FAILURE(rc))
-                return rc;
-        }
-
-    /*
-     * Enumerate the other breakpoints.
-     */
-    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
-        if (pVM->dbgf.s.aBreakpoints[i].enmType != DBGFBPTYPE_FREE)
-        {
-            int rc = pfnCallback(pVM, pvUser, &pVM->dbgf.s.aBreakpoints[i]);
-            if (RT_FAILURE(rc))
-                return rc;
-        }
-
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/DBGFCoreWrite.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFCoreWrite.cpp	(revision 35332)
+++ 	(revision )
@@ -1,570 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Guest Core Dump.
- */
-
-/*
- * Copyright (C) 2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_dbgf_vmcore    VMCore Format
- *
- * The VirtualBox VMCore Format:
- * [ ELF 64 Header]  -- Only 1
- *
- * [ PT_NOTE ]       -- Only 1
- *    - Offset into CoreDescriptor followed by list of Notes (Note Hdr + data) of VBox CPUs.
- *    - (Any Additional custom Note sections).
- *
- * [ PT_LOAD ]       -- One for each contiguous memory chunk
- *    - Memory offset (physical).
- *    - File offset.
- *
- * CoreDescriptor
- *    - Magic, VBox version.
- *    - Number of CPus.
- *
- * Per-CPU register dump
- *    - CPU 1 Note Hdr + Data.
- *    - CPU 2 Note Hdr + Data.
- *    ...
- * (Additional custom notes Hdr+data)
- *    - VBox 1 Note Hdr + Data.
- *    - VBox 2 Note Hdr + Data.
- *    ...
- * Memory dump
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <iprt/param.h>
-#include <iprt/file.h>
-
-#include "DBGFInternal.h"
-
-#include <VBox/cpum.h>
-#include "CPUMInternal.h"
-#include <VBox/dbgf.h>
-#include <VBox/dbgfcorefmt.h>
-#include <VBox/vm.h>
-#include <VBox/pgm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <VBox/mm.h>
-#include <VBox/version.h>
-
-#include "../Runtime/include/internal/ldrELF64.h"
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-#ifdef DEBUG_ramshankar
-# undef Log
-# define Log LogRel
-#endif
-#define DBGFLOG_NAME           "DBGFCoreWrite"
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-static const int s_NoteAlign  = 8;
-static const int s_cbNoteName = 16;
-
-/* These strings *HAVE* to be 8-byte aligned */
-static const char *s_pcszCoreVBoxCore = "VBCORE";
-static const char *s_pcszCoreVBoxCpu  = "VBCPU";
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Guest core writer data.
- *
- * Used to pass parameters from DBGFR3CoreWrite to dbgfR3CoreWriteRendezvous.
- */
-typedef struct DBGFCOREDATA
-{
-    /** The name of the file to write the file to. */
-    const char *pszFilename;
-    /** Whether to replace (/overwrite) any existing file. */
-    bool        fReplaceFile;
-} DBGFCOREDATA;
-/** Pointer to the guest core writer data.  */
-typedef DBGFCOREDATA *PDBGFCOREDATA;
-
-
-
-/**
- * ELF function to write 64-bit ELF header.
- *
- * @param hFile             The file to write to.
- * @param cProgHdrs         Number of program headers.
- * @param cSecHdrs          Number of section headers.
- *
- * @return IPRT status code.
- */
-static int Elf64WriteElfHdr(RTFILE hFile, uint16_t cProgHdrs, uint16_t cSecHdrs)
-{
-    Elf64_Ehdr ElfHdr;
-    RT_ZERO(ElfHdr);
-    ElfHdr.e_ident[EI_MAG0]  = ELFMAG0;
-    ElfHdr.e_ident[EI_MAG1]  = ELFMAG1;
-    ElfHdr.e_ident[EI_MAG2]  = ELFMAG2;
-    ElfHdr.e_ident[EI_MAG3]  = ELFMAG3;
-    ElfHdr.e_ident[EI_DATA]  = ELFDATA2LSB;
-    ElfHdr.e_type            = ET_CORE;
-    ElfHdr.e_version         = EV_CURRENT;
-    ElfHdr.e_ident[EI_CLASS] = ELFCLASS64;
-    /* 32-bit builds will produce cores with e_machine EM_386. */
-#ifdef RT_ARCH_AMD64
-    ElfHdr.e_machine         = EM_X86_64;
-#else
-    ElfHdr.e_machine         = EM_386;
-#endif
-    ElfHdr.e_phnum           = cProgHdrs;
-    ElfHdr.e_shnum           = cSecHdrs;
-    ElfHdr.e_ehsize          = sizeof(ElfHdr);
-    ElfHdr.e_phoff           = sizeof(ElfHdr);
-    ElfHdr.e_phentsize       = sizeof(Elf64_Phdr);
-    ElfHdr.e_shentsize       = sizeof(Elf64_Shdr);
-
-    return RTFileWrite(hFile, &ElfHdr, sizeof(ElfHdr), NULL /* all */);
-}
-
-
-/**
- * ELF function to write 64-bit program header.
- *
- * @param hFile             The file to write to.
- * @param Type              Type of program header (PT_*).
- * @param fFlags            Flags (access permissions, PF_*).
- * @param offFileData       File offset of contents.
- * @param cbFileData        Size of contents in the file.
- * @param cbMemData         Size of contents in memory.
- * @param Phys              Physical address, pass zero if not applicable.
- *
- * @return IPRT status code.
- */
-static int Elf64WriteProgHdr(RTFILE hFile, uint32_t Type, uint32_t fFlags, uint64_t offFileData, uint64_t cbFileData,
-                             uint64_t cbMemData, RTGCPHYS Phys)
-{
-    Elf64_Phdr ProgHdr;
-    RT_ZERO(ProgHdr);
-    ProgHdr.p_type          = Type;
-    ProgHdr.p_flags         = fFlags;
-    ProgHdr.p_offset        = offFileData;
-    ProgHdr.p_filesz        = cbFileData;
-    ProgHdr.p_memsz         = cbMemData;
-    ProgHdr.p_paddr         = Phys;
-
-    return RTFileWrite(hFile, &ProgHdr, sizeof(ProgHdr), NULL /* all */);
-}
-
-
-/**
- * Returns the size of the NOTE section given the name and size of the data.
- *
- * @param pszName           Name of the note section.
- * @param cb                Size of the data portion of the note section.
- *
- * @return The size of the NOTE section as rounded to the file alignment.
- */
-static uint64_t Elf64NoteSectionSize(const char *pszName, uint64_t cbData)
-{
-    uint64_t cbNote = sizeof(Elf64_Nhdr);
-
-    size_t cchName      = strlen(pszName) + 1;
-    size_t cchNameAlign = RT_ALIGN_Z(cchName, s_NoteAlign);
-
-    cbNote += cchNameAlign;
-    cbNote += RT_ALIGN_64(cbData, s_NoteAlign);
-    return cbNote;
-}
-
-
-/**
- * Elf function to write 64-bit note header.
- *
- * @param hFile             The file to write to.
- * @param Type              Type of this section.
- * @param pszName           Name of this section.
- * @param pcv               Opaque pointer to the data, if NULL only computes size.
- * @param cbData            Size of the data.
- *
- * @return IPRT status code.
- */
-static int Elf64WriteNoteHdr(RTFILE hFile, uint16_t Type, const char *pszName, const void *pcvData, uint64_t cbData)
-{
-    AssertReturn(pcvData, VERR_INVALID_POINTER);
-    AssertReturn(cbData > 0, VERR_NO_DATA);
-
-    char szNoteName[s_cbNoteName];
-    RT_ZERO(szNoteName);
-    RTStrCopy(szNoteName, sizeof(szNoteName), pszName);
-
-    size_t cchName       = strlen(szNoteName) + 1;
-    size_t cchNameAlign  = RT_ALIGN_Z(cchName, s_NoteAlign);
-    uint64_t cbDataAlign = RT_ALIGN_64(cbData, s_NoteAlign);
-
-    /*
-     * Yell loudly and bail if we are going to be writing a core file that is not compatible with
-     * both Solaris and the 64-bit ELF spec. which dictates 8-byte alignment. See #5211 comment 3.
-     */
-    if (cchNameAlign - cchName > 3)
-    {
-        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr pszName=%s cchName=%u cchNameAlign=%u, cchName aligns to 4 not 8-bytes!\n", pszName, cchName,
-                cchNameAlign));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (cbDataAlign - cbData > 3)
-    {
-        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr pszName=%s cbData=%u cbDataAlign=%u, cbData aligns to 4 not 8-bytes!\n", pszName, cbData,
-                cbDataAlign));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    static const char s_achPad[7] = { 0, 0, 0, 0, 0, 0, 0 };
-    AssertCompile(sizeof(s_achPad) >= s_NoteAlign - 1);
-
-    Elf64_Nhdr ElfNoteHdr;
-    RT_ZERO(ElfNoteHdr);
-    ElfNoteHdr.n_namesz = (Elf64_Word)cchName - 1; /* Again a discrepancy between ELF-64 and Solaris (#5211 comment 3), we will follow ELF-64 */
-    ElfNoteHdr.n_type   = Type;
-    ElfNoteHdr.n_descsz = (Elf64_Word)cbDataAlign;
-
-    /*
-     * Write note header.
-     */
-    int rc = RTFileWrite(hFile, &ElfNoteHdr, sizeof(ElfNoteHdr), NULL /* all */);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Write note name.
-         */
-        rc = RTFileWrite(hFile, szNoteName, cchName, NULL /* all */);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Write note name padding if required.
-             */
-            if (cchNameAlign > cchName)
-                rc = RTFileWrite(hFile, s_achPad, cchNameAlign - cchName, NULL);
-
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Write note data.
-                 */
-                rc = RTFileWrite(hFile, pcvData, cbData, NULL /* all */);
-                if (RT_SUCCESS(rc))
-                {
-                    /*
-                     * Write note data padding if required.
-                     */
-                    if (cbDataAlign > cbData)
-                        rc = RTFileWrite(hFile, s_achPad, cbDataAlign - cbData, NULL /* all*/);
-                }
-            }
-        }
-    }
-
-    if (RT_FAILURE(rc))
-        LogRel((DBGFLOG_NAME ": RTFileWrite failed. rc=%Rrc pszName=%s cchName=%u cchNameAlign=%u cbData=%u cbDataAlign=%u\n",
-                rc, pszName, cchName, cchNameAlign, cbData, cbDataAlign));
-
-    return rc;
-}
-
-
-/**
- * Count the number of memory ranges that go into the core file.
- *
- * We cannot do a page-by-page dump of the entire guest memory as there will be
- * way too many program header entries. Also we don't want to dump MMIO regions
- * which means we cannot have a 1:1 mapping between core file offset and memory
- * offset. Instead we dump the memory in ranges. A memory range is a contiguous
- * memory area suitable for dumping to a core file.
- *
- * @param pVM               The VM handle.
- *
- * @return Number of memory ranges
- */
-static uint32_t dbgfR3GetRamRangeCount(PVM pVM)
-{
-    return PGMR3PhysGetRamRangeCount(pVM);
-}
-
-
-/**
- * Worker function for dbgfR3CoreWrite which does the writing.
- *
- * @returns VBox status code
- * @param   pVM                 The VM handle.
- * @param   hFile               The file to write to.  Caller closes this.
- */
-static int dbgfR3CoreWriteWorker(PVM pVM, RTFILE hFile)
-{
-    /*
-     * Collect core information.
-     */
-    uint32_t const cu32MemRanges = dbgfR3GetRamRangeCount(pVM);
-    uint16_t const cMemRanges    = cu32MemRanges < UINT16_MAX - 1 ? cu32MemRanges : UINT16_MAX - 1; /* One PT_NOTE Program header */
-    uint16_t const cProgHdrs     = cMemRanges + 1;
-
-    DBGFCOREDESCRIPTOR CoreDescriptor;
-    RT_ZERO(CoreDescriptor);
-    CoreDescriptor.u32Magic         = DBGFCORE_MAGIC;
-    CoreDescriptor.u32FmtVersion    = DBGFCORE_FMT_VERSION;
-    CoreDescriptor.cbSelf           = sizeof(CoreDescriptor);
-    CoreDescriptor.u32VBoxVersion   = VBOX_FULL_VERSION;
-    CoreDescriptor.u32VBoxRevision  = VMMGetSvnRev();
-    CoreDescriptor.cCpus            = pVM->cCpus;
-
-    Log((DBGFLOG_NAME ": CoreDescriptor Version=%u Revision=%u\n", CoreDescriptor.u32VBoxVersion, CoreDescriptor.u32VBoxRevision));
-
-    /*
-     * Compute the file layout (see pg_dbgf_vmcore).
-     */
-    uint64_t const offElfHdr        = RTFileTell(hFile);
-    uint64_t const offNoteSection   = offElfHdr         + sizeof(Elf64_Ehdr);
-    uint64_t const offLoadSections  = offNoteSection    + sizeof(Elf64_Phdr);
-    uint64_t const cbLoadSections   = cMemRanges * sizeof(Elf64_Phdr);
-    uint64_t const offCoreDescriptor= offLoadSections   + cbLoadSections;
-    uint64_t const cbCoreDescriptor = Elf64NoteSectionSize(s_pcszCoreVBoxCore, sizeof(CoreDescriptor));
-    uint64_t const offCpuDumps      = offCoreDescriptor + cbCoreDescriptor;
-    uint64_t const cbCpuDumps       = pVM->cCpus * Elf64NoteSectionSize(s_pcszCoreVBoxCpu, sizeof(CPUMCTX));
-    uint64_t const offMemory        = offCpuDumps       + cbCpuDumps;
-
-    uint64_t const offNoteSectionData = offCoreDescriptor;
-    uint64_t const cbNoteSectionData  = cbCoreDescriptor + cbCpuDumps;
-
-    /*
-     * Write ELF header.
-     */
-    int rc = Elf64WriteElfHdr(hFile, cProgHdrs, 0 /* cSecHdrs */);
-    if (RT_FAILURE(rc))
-    {
-        LogRel((DBGFLOG_NAME ": Elf64WriteElfHdr failed. rc=%Rrc\n", rc));
-        return rc;
-    }
-
-    /*
-     * Write PT_NOTE program header.
-     */
-    Assert(RTFileTell(hFile) == offNoteSection);
-    rc = Elf64WriteProgHdr(hFile, PT_NOTE, PF_R,
-                           offNoteSectionData,  /* file offset to contents */
-                           cbNoteSectionData,   /* size in core file */
-                           cbNoteSectionData,   /* size in memory */
-                           0);                  /* physical address */
-    if (RT_FAILURE(rc))
-    {
-        LogRel((DBGFLOG_NAME ": Elf64WritreProgHdr failed for PT_NOTE. rc=%Rrc\n", rc));
-        return rc;
-    }
-
-    /*
-     * Write PT_LOAD program header for each memory range.
-     */
-    Assert(RTFileTell(hFile) == offLoadSections);
-    uint64_t offMemRange = offMemory;
-    for (uint16_t iRange = 0; iRange < cMemRanges; iRange++)
-    {
-        RTGCPHYS    GCPhysStart;
-        RTGCPHYS    GCPhysEnd;
-        bool        fIsMmio;
-        rc = PGMR3PhysGetRange(pVM, iRange, &GCPhysStart, &GCPhysEnd, NULL /* pszDesc */, &fIsMmio);
-        if (RT_FAILURE(rc))
-        {
-            LogRel((DBGFLOG_NAME ": PGMR3PhysGetRange failed for iRange(%u) rc=%Rrc\n", iRange, rc));
-            return rc;
-        }
-
-        uint64_t cbMemRange  = GCPhysEnd - GCPhysStart + 1;
-        uint64_t cbFileRange = fIsMmio ? 0 : cbMemRange;
-
-        Log((DBGFLOG_NAME ": PGMR3PhysGetRange iRange=%u GCPhysStart=%#x GCPhysEnd=%#x cbMemRange=%u\n",
-             iRange, GCPhysStart, GCPhysEnd, cbMemRange));
-
-        rc = Elf64WriteProgHdr(hFile, PT_LOAD, PF_R,
-                               offMemRange,                         /* file offset to contents */
-                               cbFileRange,                         /* size in core file */
-                               cbMemRange,                          /* size in memory */
-                               GCPhysStart);                        /* physical address */
-        if (RT_FAILURE(rc))
-        {
-            LogRel((DBGFLOG_NAME ": Elf64WriteProgHdr failed for memory range(%u) cbFileRange=%u cbMemRange=%u rc=%Rrc\n",
-                    iRange, cbFileRange, cbMemRange, rc));
-            return rc;
-        }
-
-        offMemRange += cbFileRange;
-    }
-
-    /*
-     * Write the Core descriptor note header and data.
-     */
-    Assert(RTFileTell(hFile) == offCoreDescriptor);
-    rc = Elf64WriteNoteHdr(hFile, NT_VBOXCORE, s_pcszCoreVBoxCore, &CoreDescriptor, sizeof(CoreDescriptor));
-    if (RT_FAILURE(rc))
-    {
-        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr failed for Note '%s' rc=%Rrc\n", s_pcszCoreVBoxCore, rc));
-        return rc;
-    }
-
-    /*
-     * Write the CPU context note headers and data.
-     */
-    Assert(RTFileTell(hFile) == offCpuDumps);
-    for (uint32_t iCpu = 0; iCpu < pVM->cCpus; iCpu++)
-    {
-        PCPUMCTX pCpuCtx = &pVM->aCpus[iCpu].cpum.s.Guest;
-        rc = Elf64WriteNoteHdr(hFile, NT_VBOXCPU, s_pcszCoreVBoxCpu, pCpuCtx, sizeof(CPUMCTX));
-        if (RT_FAILURE(rc))
-        {
-            LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr failed for vCPU[%u] rc=%Rrc\n", iCpu, rc));
-            return rc;
-        }
-    }
-
-    /*
-     * Write memory ranges.
-     */
-    Assert(RTFileTell(hFile) == offMemory);
-    for (uint16_t iRange = 0; iRange < cMemRanges; iRange++)
-    {
-        RTGCPHYS GCPhysStart;
-        RTGCPHYS GCPhysEnd;
-        bool     fIsMmio;
-        rc = PGMR3PhysGetRange(pVM, iRange, &GCPhysStart, &GCPhysEnd, NULL /* pszDesc */, &fIsMmio);
-        if (RT_FAILURE(rc))
-        {
-            LogRel((DBGFLOG_NAME ": PGMR3PhysGetRange(2) failed for iRange(%u) rc=%Rrc\n", iRange, rc));
-            return rc;
-        }
-
-        if (fIsMmio)
-            continue;
-
-        /*
-         * Write page-by-page of this memory range.
-         *
-         * The read function may fail on MMIO ranges, we write these as zero
-         * pages for now (would be nice to have the VGA bits there though).
-         */
-        uint64_t cbMemRange  = GCPhysEnd - GCPhysStart + 1;
-        uint64_t cPages      = cbMemRange >> PAGE_SHIFT;
-        for (uint64_t iPage = 0; iPage < cPages; iPage++)
-        {
-            uint8_t abPage[PAGE_SIZE];
-            rc = PGMPhysSimpleReadGCPhys(pVM, abPage, GCPhysStart + (iPage << PAGE_SHIFT),  sizeof(abPage));
-            if (RT_FAILURE(rc))
-            {
-                if (rc != VERR_PGM_PHYS_PAGE_RESERVED)
-                    LogRel((DBGFLOG_NAME ": PGMPhysRead failed for iRange=%u iPage=%u. rc=%Rrc. Ignoring...\n", iRange, iPage, rc));
-                RT_ZERO(abPage);
-            }
-
-            rc = RTFileWrite(hFile, abPage, sizeof(abPage), NULL /* all */);
-            if (RT_FAILURE(rc))
-            {
-                LogRel((DBGFLOG_NAME ": RTFileWrite failed. iRange=%u iPage=%u rc=%Rrc\n", iRange, iPage, rc));
-                return rc;
-            }
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * EMT Rendezvous worker function for DBGFR3CoreWrite.
- *
- * @param   pVM              The VM handle.
- * @param   pVCpu            The handle of the calling VCPU.
- * @param   pvData           Opaque data.
- *
- * @return VBox status code.
- */
-static DECLCALLBACK(VBOXSTRICTRC) dbgfR3CoreWriteRendezvous(PVM pVM, PVMCPU pVCpu, void *pvData)
-{
-    /*
-     * Validate input.
-     */
-    AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(pVCpu, VERR_INVALID_VMCPU_HANDLE);
-    AssertReturn(pvData, VERR_INVALID_POINTER);
-
-    PDBGFCOREDATA pDbgfData = (PDBGFCOREDATA)pvData;
-
-    /*
-     * Create the core file.
-     */
-    uint32_t fFlags = (pDbgfData->fReplaceFile ? RTFILE_O_CREATE_REPLACE : RTFILE_O_CREATE)
-                    | RTFILE_O_WRITE
-                    | RTFILE_O_DENY_ALL
-                    | (0600 << RTFILE_O_CREATE_MODE_SHIFT);
-    RTFILE   hFile;
-    int rc = RTFileOpen(&hFile, pDbgfData->pszFilename, fFlags);
-    if (RT_SUCCESS(rc))
-    {
-        rc = dbgfR3CoreWriteWorker(pVM, hFile);
-        RTFileClose(hFile);
-    }
-    else
-        LogRel((DBGFLOG_NAME ": RTFileOpen failed for '%s' rc=%Rrc\n", pDbgfData->pszFilename, rc));
-    return rc;
-}
-
-
-/**
- * Write core dump of the guest.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszFilename         The name of the file to which the guest core
- *                              dump should be written.
- * @param   fReplaceFile        Whether to replace the file or not.
- *
- * @remarks The VM should be suspended before calling this function or DMA may
- *          interfer with the state.
- */
-VMMR3DECL(int) DBGFR3CoreWrite(PVM pVM, const char *pszFilename, bool fReplaceFile)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(pszFilename, VERR_INVALID_HANDLE);
-
-    /*
-     * Pass the core write request down to EMT rendezvous which makes sure
-     * other EMTs, if any, are not running. IO threads could still be running
-     * but we don't care about them.
-     */
-    DBGFCOREDATA CoreData;
-    RT_ZERO(CoreData);
-    CoreData.pszFilename  = pszFilename;
-    CoreData.fReplaceFile = fReplaceFile;
-
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, dbgfR3CoreWriteRendezvous, &CoreData);
-    if (RT_SUCCESS(rc))
-        LogRel((DBGFLOG_NAME ": Successfully wrote guest core dump '%s'\n", pszFilename));
-    else
-        LogRel((DBGFLOG_NAME ": Failed to write guest core dump '%s'. rc=%Rrc\n", pszFilename, rc));
-    return rc;
-}
-
Index: unk/src/VBox/VMM/DBGFCpu.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFCpu.cpp	(revision 35332)
+++ 	(revision )
@@ -1,68 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, CPU State Accessors.
- */
-
-/*
- * Copyright (C) 2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/cpum.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <VBox/param.h>
-#include <iprt/assert.h>
-
-
-/**
- * Wrapper around CPUMGetGuestMode.
- *
- * @returns VINF_SUCCESS.
- * @param   pVM                 The VM handle.
- * @param   idCpu               The current CPU ID.
- * @param   penmMode            Where to return the mode.
- */
-static DECLCALLBACK(int) dbgfR3CpuGetMode(PVM pVM, VMCPUID idCpu, CPUMMODE *penmMode)
-{
-    Assert(idCpu == VMMGetCpuId(pVM));
-    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-    *penmMode = CPUMGetGuestMode(pVCpu);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Get the current CPU mode.
- *
- * @returns The CPU mode on success, CPUMMODE_INVALID on failure.
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- */
-VMMR3DECL(CPUMMODE) DBGFR3CpuGetMode(PVM pVM, VMCPUID idCpu)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, CPUMMODE_INVALID);
-    AssertReturn(idCpu < pVM->cCpus, CPUMMODE_INVALID);
-
-    CPUMMODE enmMode;
-    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3CpuGetMode, 3, pVM, idCpu, &enmMode);
-    if (RT_FAILURE(rc))
-        return CPUMMODE_INVALID;
-    return enmMode;
-}
-
Index: unk/src/VBox/VMM/DBGFDisas.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFDisas.cpp	(revision 35332)
+++ 	(revision )
@@ -1,667 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Disassembler.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/selm.h>
-#include <VBox/mm.h>
-#include <VBox/pgm.h>
-#include <VBox/cpum.h>
-#include "DBGFInternal.h"
-#include <VBox/dis.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/vm.h>
-#include <include/internal/pgm.h>
-
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-#include <iprt/alloca.h>
-#include <iprt/ctype.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Structure used when disassembling and instructions in DBGF.
- * This is used so the reader function can get the stuff it needs.
- */
-typedef struct
-{
-    /** The core structure. */
-    DISCPUSTATE     Cpu;
-    /** The VM handle. */
-    PVM             pVM;
-    /** The VMCPU handle. */
-    PVMCPU          pVCpu;
-    /** The address space for resolving symbol. */
-    RTDBGAS         hAs;
-    /** Pointer to the first byte in the segment. */
-    RTGCUINTPTR     GCPtrSegBase;
-    /** Pointer to the byte after the end of the segment. (might have wrapped!) */
-    RTGCUINTPTR     GCPtrSegEnd;
-    /** The size of the segment minus 1. */
-    RTGCUINTPTR     cbSegLimit;
-    /** The guest paging mode. */
-    PGMMODE         enmMode;
-    /** Pointer to the current page - R3 Ptr. */
-    void const     *pvPageR3;
-    /** Pointer to the current page - GC Ptr. */
-    RTGCPTR         pvPageGC;
-    /** Pointer to the next instruction (relative to GCPtrSegBase). */
-    RTGCUINTPTR     GCPtrNext;
-    /** The lock information that PGMPhysReleasePageMappingLock needs. */
-    PGMPAGEMAPLOCK  PageMapLock;
-    /** Whether the PageMapLock is valid or not. */
-    bool            fLocked;
-    /** 64 bits mode or not. */
-    bool            f64Bits;
-} DBGFDISASSTATE, *PDBGFDISASSTATE;
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) dbgfR3DisasInstrRead(RTUINTPTR pSrc, uint8_t *pDest, uint32_t size, void *pvUserdata);
-
-
-
-/**
- * Calls the disassembler with the proper reader functions and such for disa
- *
- * @returns VBox status code.
- * @param   pVM         VM handle
- * @param   pVCpu       VMCPU handle
- * @param   pSelInfo    The selector info.
- * @param   enmMode     The guest paging mode.
- * @param   fFlags      DBGF_DISAS_FLAGS_XXX.
- * @param   GCPtr       The GC pointer (selector offset).
- * @param   pState      The disas CPU state.
- */
-static int dbgfR3DisasInstrFirst(PVM pVM, PVMCPU pVCpu, PDBGFSELINFO pSelInfo, PGMMODE enmMode,
-                                 RTGCPTR GCPtr, uint32_t fFlags, PDBGFDISASSTATE pState)
-{
-    pState->GCPtrSegBase    = pSelInfo->GCPtrBase;
-    pState->GCPtrSegEnd     = pSelInfo->cbLimit + 1 + (RTGCUINTPTR)pSelInfo->GCPtrBase;
-    pState->cbSegLimit      = pSelInfo->cbLimit;
-    pState->enmMode         = enmMode;
-    pState->pvPageGC        = 0;
-    pState->pvPageR3        = NULL;
-    pState->hAs             = pSelInfo->fFlags & DBGFSELINFO_FLAGS_HYPER /** @todo Deal more explicitly with RC in DBGFR3Disas*. */
-                            ? DBGF_AS_RC_AND_GC_GLOBAL
-                            : DBGF_AS_GLOBAL;
-    pState->pVM             = pVM;
-    pState->pVCpu           = pVCpu;
-    pState->fLocked         = false;
-    pState->f64Bits         = enmMode >= PGMMODE_AMD64 && pSelInfo->u.Raw.Gen.u1Long;
-
-    DISCPUMODE enmCpuMode;
-    switch (fFlags & DBGF_DISAS_FLAGS_MODE_MASK)
-    {
-        default:
-            AssertFailed();
-        case DBGF_DISAS_FLAGS_DEFAULT_MODE:
-            enmCpuMode   = pState->f64Bits
-                         ? CPUMODE_64BIT
-                         : pSelInfo->u.Raw.Gen.u1DefBig
-                         ? CPUMODE_32BIT
-                         : CPUMODE_16BIT;
-            break;
-        case DBGF_DISAS_FLAGS_16BIT_MODE:
-        case DBGF_DISAS_FLAGS_16BIT_REAL_MODE:
-            enmCpuMode = CPUMODE_16BIT;
-            break;
-        case DBGF_DISAS_FLAGS_32BIT_MODE:
-            enmCpuMode = CPUMODE_32BIT;
-            break;
-        case DBGF_DISAS_FLAGS_64BIT_MODE:
-            enmCpuMode = CPUMODE_64BIT;
-            break;
-    }
-
-    uint32_t cbInstr;
-    int rc = DISCoreOneEx(GCPtr,
-                          enmCpuMode,
-                          dbgfR3DisasInstrRead,
-                          &pState->Cpu,
-                          &pState->Cpu,
-                          &cbInstr);
-    if (RT_SUCCESS(rc))
-    {
-        pState->GCPtrNext = GCPtr + cbInstr;
-        return VINF_SUCCESS;
-    }
-
-    /* cleanup */
-    if (pState->fLocked)
-    {
-        PGMPhysReleasePageMappingLock(pVM, &pState->PageMapLock);
-        pState->fLocked = false;
-    }
-    return rc;
-}
-
-
-#if 0
-/**
- * Calls the disassembler for disassembling the next instruction.
- *
- * @returns VBox status code.
- * @param   pState      The disas CPU state.
- */
-static int dbgfR3DisasInstrNext(PDBGFDISASSTATE pState)
-{
-    uint32_t cbInstr;
-    int rc = DISInstr(&pState->Cpu, (void *)pState->GCPtrNext, 0, &cbInstr, NULL);
-    if (RT_SUCCESS(rc))
-    {
-        pState->GCPtrNext = GCPtr + cbInstr;
-        return VINF_SUCCESS;
-    }
-    return rc;
-}
-#endif
-
-
-/**
- * Done with the disassembler state, free associated resources.
- *
- * @param   pState      The disas CPU state ++.
- */
-static void dbgfR3DisasInstrDone(PDBGFDISASSTATE pState)
-{
-    if (pState->fLocked)
-    {
-        PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
-        pState->fLocked = false;
-    }
-}
-
-
-/**
- * Instruction reader.
- *
- * @returns VBox status code. (Why this is a int32_t and not just an int is also beyond me.)
- * @param   PtrSrc      Address to read from.
- *                      In our case this is relative to the selector pointed to by the 2nd user argument of uDisCpu.
- * @param   pu8Dst      Where to store the bytes.
- * @param   cbRead      Number of bytes to read.
- * @param   uDisCpu     Pointer to the disassembler cpu state. (Why this is a VBOXHUINTPTR is beyond me...)
- *                      In this context it's always pointer to the Core of a DBGFDISASSTATE.
- */
-static DECLCALLBACK(int) dbgfR3DisasInstrRead(RTUINTPTR PtrSrc, uint8_t *pu8Dst, uint32_t cbRead, void *pvDisCpu)
-{
-    PDBGFDISASSTATE pState = (PDBGFDISASSTATE)pvDisCpu;
-    Assert(cbRead > 0);
-    for (;;)
-    {
-        RTGCUINTPTR GCPtr = PtrSrc + pState->GCPtrSegBase;
-
-        /* Need to update the page translation? */
-        if (    !pState->pvPageR3
-            ||  (GCPtr >> PAGE_SHIFT) != (pState->pvPageGC >> PAGE_SHIFT))
-        {
-            int rc = VINF_SUCCESS;
-
-            /* translate the address */
-            pState->pvPageGC = GCPtr & PAGE_BASE_GC_MASK;
-            if (MMHyperIsInsideArea(pState->pVM, pState->pvPageGC))
-            {
-                pState->pvPageR3 = MMHyperRCToR3(pState->pVM, (RTRCPTR)pState->pvPageGC);
-                if (!pState->pvPageR3)
-                    rc = VERR_INVALID_POINTER;
-            }
-            else
-            {
-                if (pState->fLocked)
-                    PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
-
-                if (pState->enmMode <= PGMMODE_PROTECTED)
-                    rc = PGMPhysGCPhys2CCPtrReadOnly(pState->pVM, pState->pvPageGC, &pState->pvPageR3, &pState->PageMapLock);
-                else
-                    rc = PGMPhysGCPtr2CCPtrReadOnly(pState->pVCpu, pState->pvPageGC, &pState->pvPageR3, &pState->PageMapLock);
-                pState->fLocked = RT_SUCCESS_NP(rc);
-            }
-            if (RT_FAILURE(rc))
-            {
-                pState->pvPageR3 = NULL;
-                return rc;
-            }
-        }
-
-        /* check the segment limit */
-        if (!pState->f64Bits && PtrSrc > pState->cbSegLimit)
-            return VERR_OUT_OF_SELECTOR_BOUNDS;
-
-        /* calc how much we can read */
-        uint32_t cb = PAGE_SIZE - (GCPtr & PAGE_OFFSET_MASK);
-        if (!pState->f64Bits)
-        {
-            RTGCUINTPTR cbSeg = pState->GCPtrSegEnd - GCPtr;
-            if (cb > cbSeg && cbSeg)
-                cb = cbSeg;
-        }
-        if (cb > cbRead)
-            cb = cbRead;
-
-        /* read and advance */
-        memcpy(pu8Dst, (char *)pState->pvPageR3 + (GCPtr & PAGE_OFFSET_MASK), cb);
-        cbRead -= cb;
-        if (!cbRead)
-            return VINF_SUCCESS;
-        pu8Dst += cb;
-        PtrSrc += cb;
-    }
-}
-
-
-/**
- * @copydoc FNDISGETSYMBOL
- */
-static DECLCALLBACK(int) dbgfR3DisasGetSymbol(PCDISCPUSTATE pCpu, uint32_t u32Sel, RTUINTPTR uAddress, char *pszBuf, size_t cchBuf, RTINTPTR *poff, void *pvUser)
-{
-    PDBGFDISASSTATE pState   = (PDBGFDISASSTATE)pCpu;
-    PCDBGFSELINFO   pSelInfo = (PCDBGFSELINFO)pvUser;
-    DBGFADDRESS     Addr;
-    RTDBGSYMBOL     Sym;
-    RTGCINTPTR      off;
-    int             rc;
-
-    if (   DIS_FMT_SEL_IS_REG(u32Sel)
-        ?  DIS_FMT_SEL_GET_REG(u32Sel) == DIS_SELREG_CS
-        :  pSelInfo->Sel == DIS_FMT_SEL_GET_VALUE(u32Sel))
-    {
-        rc = DBGFR3AddrFromSelInfoOff(pState->pVM, &Addr, pSelInfo, uAddress);
-        if (RT_SUCCESS(rc))
-            rc = DBGFR3AsSymbolByAddr(pState->pVM, pState->hAs, &Addr, &off, &Sym, NULL /*phMod*/);
-    }
-    else
-        rc = VERR_SYMBOL_NOT_FOUND; /** @todo implement this */
-    if (RT_SUCCESS(rc))
-    {
-        size_t cchName = strlen(Sym.szName);
-        if (cchName >= cchBuf)
-            cchName = cchBuf - 1;
-        memcpy(pszBuf, Sym.szName, cchName);
-        pszBuf[cchName] = '\0';
-
-        *poff = off;
-    }
-
-    return rc;
-}
-
-
-/**
- * Disassembles the one instruction according to the specified flags and
- * address, internal worker executing on the EMT of the specified virtual CPU.
- *
- * @returns VBox status code.
- * @param       pVM             The VM handle.
- * @param       pVCpu           The virtual CPU handle.
- * @param       Sel             The code selector. This used to determine the 32/16 bit ness and
- *                              calculation of the actual instruction address.
- * @param       pGCPtr          Pointer to the variable holding the code address
- *                              relative to the base of Sel.
- * @param       fFlags          Flags controlling where to start and how to format.
- *                              A combination of the DBGF_DISAS_FLAGS_* \#defines.
- * @param       pszOutput       Output buffer.
- * @param       cbOutput        Size of the output buffer.
- * @param       pcbInstr        Where to return the size of the instruction.
- */
-static DECLCALLBACK(int)
-dbgfR3DisasInstrExOnVCpu(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PRTGCPTR pGCPtr, uint32_t fFlags,
-                         char *pszOutput, uint32_t cbOutput, uint32_t *pcbInstr)
-{
-    VMCPU_ASSERT_EMT(pVCpu);
-    RTGCPTR GCPtr = *pGCPtr;
-
-    /*
-     * Get the Sel and GCPtr if fFlags requests that.
-     */
-    PCCPUMCTXCORE  pCtxCore   = NULL;
-    PCPUMSELREGHID pHiddenSel = NULL;
-    int rc;
-    if (fFlags & (DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_CURRENT_HYPER))
-    {
-        if (fFlags & DBGF_DISAS_FLAGS_CURRENT_GUEST)
-            pCtxCore = CPUMGetGuestCtxCore(pVCpu);
-        else
-            pCtxCore = CPUMGetHyperCtxCore(pVCpu);
-        Sel        = pCtxCore->cs;
-        pHiddenSel = (CPUMSELREGHID *)&pCtxCore->csHid;
-        GCPtr      = pCtxCore->rip;
-    }
-
-    /*
-     * Read the selector info - assume no stale selectors and nasty stuff like that.
-     * Since the selector flags in the CPUMCTX structures aren't up to date unless
-     * we recently visited REM, we'll not search for the selector there.
-     */
-    DBGFSELINFO     SelInfo;
-    const PGMMODE   enmMode          = PGMGetGuestMode(pVCpu);
-    bool            fRealModeAddress = false;
-
-    if (    pHiddenSel
-        &&  (   (fFlags & DBGF_DISAS_FLAGS_HID_SEL_REGS_VALID)
-             || CPUMAreHiddenSelRegsValid(pVCpu)))
-    {
-        SelInfo.Sel                     = Sel;
-        SelInfo.SelGate                 = 0;
-        SelInfo.GCPtrBase               = pHiddenSel->u64Base;
-        SelInfo.cbLimit                 = pHiddenSel->u32Limit;
-        SelInfo.fFlags                  = PGMMODE_IS_LONG_MODE(enmMode)
-                                        ? DBGFSELINFO_FLAGS_LONG_MODE
-                                        : enmMode != PGMMODE_REAL && (!pCtxCore || !pCtxCore->eflags.Bits.u1VM)
-                                        ? DBGFSELINFO_FLAGS_PROT_MODE
-                                        : DBGFSELINFO_FLAGS_REAL_MODE;
-
-        SelInfo.u.Raw.au32[0]           = 0;
-        SelInfo.u.Raw.au32[1]           = 0;
-        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
-        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
-        SelInfo.u.Raw.Gen.u1Present     = pHiddenSel->Attr.n.u1Present;
-        SelInfo.u.Raw.Gen.u1Granularity = pHiddenSel->Attr.n.u1Granularity;;
-        SelInfo.u.Raw.Gen.u1DefBig      = pHiddenSel->Attr.n.u1DefBig;
-        SelInfo.u.Raw.Gen.u1Long        = pHiddenSel->Attr.n.u1Long;
-        SelInfo.u.Raw.Gen.u1DescType    = pHiddenSel->Attr.n.u1DescType;
-        SelInfo.u.Raw.Gen.u4Type        = pHiddenSel->Attr.n.u4Type;
-        fRealModeAddress                = !!(SelInfo.fFlags & DBGFSELINFO_FLAGS_REAL_MODE);
-    }
-    else if (Sel == DBGF_SEL_FLAT)
-    {
-        SelInfo.Sel                     = Sel;
-        SelInfo.SelGate                 = 0;
-        SelInfo.GCPtrBase               = 0;
-        SelInfo.cbLimit                 = ~0;
-        SelInfo.fFlags                  = PGMMODE_IS_LONG_MODE(enmMode)
-                                        ? DBGFSELINFO_FLAGS_LONG_MODE
-                                        : enmMode != PGMMODE_REAL
-                                        ? DBGFSELINFO_FLAGS_PROT_MODE
-                                        : DBGFSELINFO_FLAGS_REAL_MODE;
-        SelInfo.u.Raw.au32[0]           = 0;
-        SelInfo.u.Raw.au32[1]           = 0;
-        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
-        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
-
-        if (   (fFlags & DBGF_DISAS_FLAGS_HID_SEL_REGS_VALID)
-            || CPUMAreHiddenSelRegsValid(pVCpu))
-        {   /* Assume the current CS defines the execution mode. */
-            pCtxCore   = CPUMGetGuestCtxCore(pVCpu);
-            pHiddenSel = (CPUMSELREGHID *)&pCtxCore->csHid;
-
-            SelInfo.u.Raw.Gen.u1Present     = pHiddenSel->Attr.n.u1Present;
-            SelInfo.u.Raw.Gen.u1Granularity = pHiddenSel->Attr.n.u1Granularity;;
-            SelInfo.u.Raw.Gen.u1DefBig      = pHiddenSel->Attr.n.u1DefBig;
-            SelInfo.u.Raw.Gen.u1Long        = pHiddenSel->Attr.n.u1Long;
-            SelInfo.u.Raw.Gen.u1DescType    = pHiddenSel->Attr.n.u1DescType;
-            SelInfo.u.Raw.Gen.u4Type        = pHiddenSel->Attr.n.u4Type;
-        }
-        else
-        {
-            SelInfo.u.Raw.Gen.u1Present     = 1;
-            SelInfo.u.Raw.Gen.u1Granularity = 1;
-            SelInfo.u.Raw.Gen.u1DefBig      = 1;
-            SelInfo.u.Raw.Gen.u1DescType    = 1;
-            SelInfo.u.Raw.Gen.u4Type        = X86_SEL_TYPE_EO;
-        }
-    }
-    else if (   !(fFlags & DBGF_DISAS_FLAGS_CURRENT_HYPER)
-             && (   (pCtxCore && pCtxCore->eflags.Bits.u1VM)
-                 || enmMode == PGMMODE_REAL
-                 || (fFlags & DBGF_DISAS_FLAGS_MODE_MASK) == DBGF_DISAS_FLAGS_16BIT_REAL_MODE
-                )
-            )
-    {   /* V86 mode or real mode - real mode addressing */
-        SelInfo.Sel                     = Sel;
-        SelInfo.SelGate                 = 0;
-        SelInfo.GCPtrBase               = Sel * 16;
-        SelInfo.cbLimit                 = ~0;
-        SelInfo.fFlags                  = DBGFSELINFO_FLAGS_REAL_MODE;
-        SelInfo.u.Raw.au32[0]           = 0;
-        SelInfo.u.Raw.au32[1]           = 0;
-        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
-        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
-        SelInfo.u.Raw.Gen.u1Present     = 1;
-        SelInfo.u.Raw.Gen.u1Granularity = 1;
-        SelInfo.u.Raw.Gen.u1DefBig      = 0; /* 16 bits */
-        SelInfo.u.Raw.Gen.u1DescType    = 1;
-        SelInfo.u.Raw.Gen.u4Type        = X86_SEL_TYPE_EO;
-        fRealModeAddress                = true;
-    }
-    else
-    {
-        rc = SELMR3GetSelectorInfo(pVM, pVCpu, Sel, &SelInfo);
-        if (RT_FAILURE(rc))
-        {
-            RTStrPrintf(pszOutput, cbOutput, "Sel=%04x -> %Rrc\n", Sel, rc);
-            return rc;
-        }
-    }
-
-    /*
-     * Disassemble it.
-     */
-    DBGFDISASSTATE State;
-    rc = dbgfR3DisasInstrFirst(pVM, pVCpu, &SelInfo, enmMode, GCPtr, fFlags, &State);
-    if (RT_FAILURE(rc))
-    {
-        RTStrPrintf(pszOutput, cbOutput, "Disas -> %Rrc\n", rc);
-        return rc;
-    }
-
-    /*
-     * Format it.
-     */
-    char szBuf[512];
-    DISFormatYasmEx(&State.Cpu, szBuf, sizeof(szBuf),
-                    DIS_FMT_FLAGS_RELATIVE_BRANCH,
-                    fFlags & DBGF_DISAS_FLAGS_NO_SYMBOLS ? NULL : dbgfR3DisasGetSymbol,
-                    &SelInfo);
-
-    /*
-     * Print it to the user specified buffer.
-     */
-    if (fFlags & DBGF_DISAS_FLAGS_NO_BYTES)
-    {
-        if (fFlags & DBGF_DISAS_FLAGS_NO_ADDRESS)
-            RTStrPrintf(pszOutput, cbOutput, "%s", szBuf);
-        else if (fRealModeAddress)
-            RTStrPrintf(pszOutput, cbOutput, "%04x:%04x  %s", Sel, (unsigned)GCPtr, szBuf);
-        else if (Sel == DBGF_SEL_FLAT)
-        {
-            if (enmMode >= PGMMODE_AMD64)
-                RTStrPrintf(pszOutput, cbOutput, "%RGv  %s", GCPtr, szBuf);
-            else
-                RTStrPrintf(pszOutput, cbOutput, "%08RX32  %s", (uint32_t)GCPtr, szBuf);
-        }
-        else
-        {
-            if (enmMode >= PGMMODE_AMD64)
-                RTStrPrintf(pszOutput, cbOutput, "%04x:%RGv  %s", Sel, GCPtr, szBuf);
-            else
-                RTStrPrintf(pszOutput, cbOutput, "%04x:%08RX32  %s", Sel, (uint32_t)GCPtr, szBuf);
-        }
-    }
-    else
-    {
-        uint32_t cbBits = State.Cpu.opsize;
-        uint8_t *pau8Bits = (uint8_t *)alloca(cbBits);
-        rc = dbgfR3DisasInstrRead(GCPtr, pau8Bits, cbBits, &State);
-        AssertRC(rc);
-        if (fFlags & DBGF_DISAS_FLAGS_NO_ADDRESS)
-            RTStrPrintf(pszOutput, cbOutput, "%.*Rhxs%*s %s",
-                        cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
-                        szBuf);
-        else if (fRealModeAddress)
-            RTStrPrintf(pszOutput, cbOutput, "%04x:%04x %.*Rhxs%*s %s",
-                        Sel, (unsigned)GCPtr,
-                        cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
-                        szBuf);
-        else if (Sel == DBGF_SEL_FLAT)
-        {
-            if (enmMode >= PGMMODE_AMD64)
-                RTStrPrintf(pszOutput, cbOutput, "%RGv %.*Rhxs%*s %s",
-                            GCPtr,
-                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
-                            szBuf);
-            else
-                RTStrPrintf(pszOutput, cbOutput, "%08RX32 %.*Rhxs%*s %s",
-                            (uint32_t)GCPtr,
-                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
-                            szBuf);
-        }
-        else
-        {
-            if (enmMode >= PGMMODE_AMD64)
-                RTStrPrintf(pszOutput, cbOutput, "%04x:%RGv %.*Rhxs%*s %s",
-                            Sel, GCPtr,
-                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
-                            szBuf);
-            else
-                RTStrPrintf(pszOutput, cbOutput, "%04x:%08RX32 %.*Rhxs%*s %s",
-                            Sel, (uint32_t)GCPtr,
-                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
-                            szBuf);
-        }
-    }
-
-    if (pcbInstr)
-        *pcbInstr = State.Cpu.opsize;
-
-    dbgfR3DisasInstrDone(&State);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Disassembles the one instruction according to the specified flags and address.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   idCpu           The ID of virtual CPU.
- * @param   Sel             The code selector. This used to determine the 32/16 bit ness and
- *                          calculation of the actual instruction address.
- * @param   GCPtr           The code address relative to the base of Sel.
- * @param   fFlags          Flags controlling where to start and how to format.
- *                          A combination of the DBGF_DISAS_FLAGS_* \#defines.
- * @param   pszOutput       Output buffer.  This will always be properly
- *                          terminated if @a cbOutput is greater than zero.
- * @param   cbOutput        Size of the output buffer.
- * @param   pcbInstr        Where to return the size of the instruction.
- *
- * @remarks May have to switch to the EMT of the virtual CPU in order to do
- *          address conversion.
- */
-VMMR3DECL(int) DBGFR3DisasInstrEx(PVM pVM, VMCPUID idCpu, RTSEL Sel, RTGCPTR GCPtr, uint32_t fFlags,
-                                  char *pszOutput, uint32_t cbOutput, uint32_t *pcbInstr)
-{
-    AssertReturn(cbOutput > 0, VERR_INVALID_PARAMETER);
-    *pszOutput = '\0';
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    AssertReturn(!(fFlags & ~DBGF_DISAS_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn((fFlags & DBGF_DISAS_FLAGS_MODE_MASK) <= DBGF_DISAS_FLAGS_64BIT_MODE, VERR_INVALID_PARAMETER);
-
-    /*
-     * Optimize the common case where we're called on the EMT of idCpu since
-     * we're using this all the time when logging.
-     */
-    int     rc;
-    PVMCPU  pVCpu = VMMGetCpu(pVM);
-    if (    pVCpu
-        &&  pVCpu->idCpu == idCpu)
-        rc = dbgfR3DisasInstrExOnVCpu(pVM, pVCpu, Sel, &GCPtr, fFlags, pszOutput, cbOutput, pcbInstr);
-    else
-        rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3DisasInstrExOnVCpu, 8,
-                             pVM, VMMGetCpuById(pVM, idCpu), Sel, &GCPtr, fFlags, pszOutput, cbOutput, pcbInstr);
-    return rc;
-}
-
-
-/**
- * Disassembles the current guest context instruction.
- * All registers and data will be displayed. Addresses will be attempted resolved to symbols.
- *
- * @returns VBox status code.
- * @param   pVCpu           VMCPU handle.
- * @param   pszOutput       Output buffer.  This will always be properly
- *                          terminated if @a cbOutput is greater than zero.
- * @param   cbOutput        Size of the output buffer.
- */
-VMMR3DECL(int) DBGFR3DisasInstrCurrent(PVMCPU pVCpu, char *pszOutput, uint32_t cbOutput)
-{
-    AssertReturn(cbOutput > 0, VERR_INVALID_PARAMETER);
-    *pszOutput = '\0';
-    AssertReturn(pVCpu, VERR_INVALID_CONTEXT);
-    return DBGFR3DisasInstrEx(pVCpu->pVMR3, pVCpu->idCpu, 0, 0,
-                              DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                              pszOutput, cbOutput, NULL);
-}
-
-
-/**
- * Disassembles the current guest context instruction and writes it to the log.
- * All registers and data will be displayed. Addresses will be attempted resolved to symbols.
- *
- * @returns VBox status code.
- * @param   pVCpu           VMCPU handle.
- * @param   pszPrefix       Short prefix string to the disassembly string. (optional)
- */
-VMMR3DECL(int) DBGFR3DisasInstrCurrentLogInternal(PVMCPU pVCpu, const char *pszPrefix)
-{
-    char szBuf[256];
-    szBuf[0] = '\0';
-    int rc = DBGFR3DisasInstrCurrent(pVCpu, &szBuf[0], sizeof(szBuf));
-    if (RT_FAILURE(rc))
-        RTStrPrintf(szBuf, sizeof(szBuf), "DBGFR3DisasInstrCurrentLog failed with rc=%Rrc\n", rc);
-    if (pszPrefix && *pszPrefix)
-        RTLogPrintf("%s-CPU%d: %s\n", pszPrefix, pVCpu->idCpu, szBuf);
-    else
-        RTLogPrintf("%s\n", szBuf);
-    return rc;
-}
-
-
-
-/**
- * Disassembles the specified guest context instruction and writes it to the log.
- * Addresses will be attempted resolved to symbols.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pVCpu           The virtual CPU handle, defaults to CPU 0 if NULL.
- * @param   Sel             The code selector. This used to determine the 32/16 bit-ness and
- *                          calculation of the actual instruction address.
- * @param   GCPtr           The code address relative to the base of Sel.
- */
-VMMR3DECL(int) DBGFR3DisasInstrLogInternal(PVMCPU pVCpu, RTSEL Sel, RTGCPTR GCPtr)
-{
-    char szBuf[256];
-    int rc = DBGFR3DisasInstrEx(pVCpu->pVMR3, pVCpu->idCpu, Sel, GCPtr, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                                &szBuf[0], sizeof(szBuf), NULL);
-    if (RT_FAILURE(rc))
-        RTStrPrintf(szBuf, sizeof(szBuf), "DBGFR3DisasInstrLog(, %RTsel, %RGv) failed with rc=%Rrc\n", Sel, GCPtr, rc);
-    RTLogPrintf("%s\n", szBuf);
-    return rc;
-}
-
Index: unk/src/VBox/VMM/DBGFInfo.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFInfo.cpp	(revision 35332)
+++ 	(revision )
@@ -1,994 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Info.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF_INFO
-#include <VBox/dbgf.h>
-
-#include <VBox/mm.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-
-#include <iprt/assert.h>
-#include <iprt/ctype.h>
-#include <iprt/semaphore.h>
-#include <iprt/stream.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(void) dbgfR3InfoLog_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
-static DECLCALLBACK(void) dbgfR3InfoLog_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
-static DECLCALLBACK(void) dbgfR3InfoLogRel_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
-static DECLCALLBACK(void) dbgfR3InfoLogRel_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
-static DECLCALLBACK(void) dbgfR3InfoStdErr_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
-static DECLCALLBACK(void) dbgfR3InfoStdErr_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
-static DECLCALLBACK(void) dbgfR3InfoHelp(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-/** Logger output. */
-static const DBGFINFOHLP g_dbgfR3InfoLogHlp =
-{
-    dbgfR3InfoLog_Printf,
-    dbgfR3InfoLog_PrintfV
-};
-
-/** Release logger output. */
-static const DBGFINFOHLP g_dbgfR3InfoLogRelHlp =
-{
-    dbgfR3InfoLogRel_Printf,
-    dbgfR3InfoLogRel_PrintfV
-};
-
-/** Standard error output. */
-static const DBGFINFOHLP g_dbgfR3InfoStdErrHlp =
-{
-    dbgfR3InfoStdErr_Printf,
-    dbgfR3InfoStdErr_PrintfV
-};
-
-
-/**
- * Initialize the info handlers.
- *
- * @returns VBox status code.
- * @param   pVM     VM handle.
- */
-int dbgfR3InfoInit(PVM pVM)
-{
-    /*
-     * Make sure we already didn't initialized in the lazy manner.
-     */
-    if (RTCritSectIsInitialized(&pVM->dbgf.s.InfoCritSect))
-        return VINF_SUCCESS;
-
-    /*
-     * Initialize the crit sect.
-     */
-    int rc = RTCritSectInit(&pVM->dbgf.s.InfoCritSect);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Register the 'info help' item.
-     */
-    rc = DBGFR3InfoRegisterInternal(pVM, "help", "List of info items.", dbgfR3InfoHelp);
-    AssertRCReturn(rc, rc);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Terminate the info handlers.
- *
- * @returns VBox status code.
- * @param   pVM     VM handle.
- */
-int dbgfR3InfoTerm(PVM pVM)
-{
-    /*
-     * Delete the crit sect.
-     */
-    int rc = RTCritSectDelete(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-    return rc;
-}
-
-
-/** Logger output.
- * @copydoc DBGFINFOHLP::pfnPrintf */
-static DECLCALLBACK(void) dbgfR3InfoLog_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
-{
-    va_list args;
-    va_start(args, pszFormat);
-    RTLogPrintfV(pszFormat, args);
-    va_end(args);
-}
-
-/** Logger output.
- * @copydoc DBGFINFOHLP::pfnPrintfV */
-static DECLCALLBACK(void) dbgfR3InfoLog_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
-{
-    RTLogPrintfV(pszFormat, args);
-}
-
-
-/**
- * Gets the logger info helper.
- * The returned info helper will unconditionally write all output to the log.
- *
- * @returns Pointer to the logger info helper.
- */
-VMMR3DECL(PCDBGFINFOHLP) DBGFR3InfoLogHlp(void)
-{
-    return &g_dbgfR3InfoLogHlp;
-}
-
-
-/** Release logger output.
- * @copydoc DBGFINFOHLP::pfnPrintf */
-static DECLCALLBACK(void) dbgfR3InfoLogRel_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
-{
-    va_list args;
-    va_start(args, pszFormat);
-    RTLogRelPrintfV(pszFormat, args);
-    va_end(args);
-}
-
-/** Release logger output.
- * @copydoc DBGFINFOHLP::pfnPrintfV */
-static DECLCALLBACK(void) dbgfR3InfoLogRel_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
-{
-    RTLogRelPrintfV(pszFormat, args);
-}
-
-
-/** Standard error output.
- * @copydoc DBGFINFOHLP::pfnPrintf */
-static DECLCALLBACK(void) dbgfR3InfoStdErr_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
-{
-    va_list args;
-    va_start(args, pszFormat);
-    RTStrmPrintfV(g_pStdErr, pszFormat, args);
-    va_end(args);
-}
-
-/** Standard error output.
- * @copydoc DBGFINFOHLP::pfnPrintfV */
-static DECLCALLBACK(void) dbgfR3InfoStdErr_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
-{
-    RTStrmPrintfV(g_pStdErr, pszFormat, args);
-}
-
-
-/**
- * Gets the release logger info helper.
- * The returned info helper will unconditionally write all output to the release log.
- *
- * @returns Pointer to the release logger info helper.
- */
-VMMR3DECL(PCDBGFINFOHLP) DBGFR3InfoLogRelHlp(void)
-{
-    return &g_dbgfR3InfoLogRelHlp;
-}
-
-
-/**
- * Handle registration worker.
- * This allocates the structure, initializes the common fields and inserts into the list.
- * Upon successful return the we're inside the crit sect and the caller must leave it.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pszName     The identifier of the info.
- * @param   pszDesc     The description of the info and any arguments the handler may take.
- * @param   fFlags      The flags.
- * @param   ppInfo      Where to store the created
- */
-static int dbgfR3InfoRegister(PVM pVM, const char *pszName, const char *pszDesc, uint32_t fFlags, PDBGFINFO *ppInfo)
-{
-    /*
-     * Validate.
-     */
-    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
-    AssertReturn(*pszName, VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
-    AssertMsgReturn(!(fFlags & ~(DBGFINFO_FLAGS_RUN_ON_EMT)), ("fFlags=%#x\n", fFlags), VERR_INVALID_PARAMETER);
-
-    /*
-     * Allocate and initialize.
-     */
-    int rc;
-    size_t cchName = strlen(pszName) + 1;
-    PDBGFINFO pInfo = (PDBGFINFO)MMR3HeapAlloc(pVM, MM_TAG_DBGF_INFO, RT_OFFSETOF(DBGFINFO, szName[cchName]));
-    if (pInfo)
-    {
-        pInfo->enmType = DBGFINFOTYPE_INVALID;
-        pInfo->fFlags = fFlags;
-        pInfo->pszDesc = pszDesc;
-        pInfo->cchName = cchName - 1;
-        memcpy(pInfo->szName, pszName, cchName);
-
-        /* lazy init */
-        rc = VINF_SUCCESS;
-        if (!RTCritSectIsInitialized(&pVM->dbgf.s.InfoCritSect))
-            rc = dbgfR3InfoInit(pVM);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Insert in alphabetical order.
-             */
-            rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-            AssertRC(rc);
-            PDBGFINFO pPrev = NULL;
-            PDBGFINFO pCur;
-            for (pCur = pVM->dbgf.s.pInfoFirst; pCur; pPrev = pCur, pCur = pCur->pNext)
-                if (strcmp(pszName, pCur->szName) < 0)
-                    break;
-            pInfo->pNext = pCur;
-            if (pPrev)
-                pPrev->pNext = pInfo;
-            else
-                pVM->dbgf.s.pInfoFirst = pInfo;
-
-            *ppInfo = pInfo;
-            return VINF_SUCCESS;
-        }
-        MMR3HeapFree(pInfo);
-    }
-    else
-        rc = VERR_NO_MEMORY;
-    return rc;
-}
-
-
-/**
- * Register a info handler owned by a device.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pszName     The identifier of the info.
- * @param   pszDesc     The description of the info and any arguments the handler may take.
- * @param   pfnHandler  The handler function to be called to display the info.
- * @param   pDevIns     The device instance owning the info.
- */
-VMMR3DECL(int) DBGFR3InfoRegisterDevice(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDEV pfnHandler, PPDMDEVINS pDevIns)
-{
-    LogFlow(("DBGFR3InfoRegisterDevice: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDevIns=%p\n",
-             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDevIns));
-
-    /*
-     * Validate the specific stuff.
-     */
-    if (!pfnHandler)
-    {
-        AssertMsgFailed(("No handler\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!pDevIns)
-    {
-        AssertMsgFailed(("No pDevIns\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Register
-     */
-    PDBGFINFO pInfo;
-    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, 0, &pInfo);
-    if (RT_SUCCESS(rc))
-    {
-        pInfo->enmType = DBGFINFOTYPE_DEV;
-        pInfo->u.Dev.pfnHandler = pfnHandler;
-        pInfo->u.Dev.pDevIns = pDevIns;
-        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    }
-
-    return rc;
-}
-
-
-/**
- * Register a info handler owned by a driver.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pszName     The identifier of the info.
- * @param   pszDesc     The description of the info and any arguments the handler may take.
- * @param   pfnHandler  The handler function to be called to display the info.
- * @param   pDrvIns     The driver instance owning the info.
- */
-VMMR3DECL(int) DBGFR3InfoRegisterDriver(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDRV pfnHandler, PPDMDRVINS pDrvIns)
-{
-    LogFlow(("DBGFR3InfoRegisterDriver: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDrvIns=%p\n",
-             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDrvIns));
-
-    /*
-     * Validate the specific stuff.
-     */
-    if (!pfnHandler)
-    {
-        AssertMsgFailed(("No handler\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!pDrvIns)
-    {
-        AssertMsgFailed(("No pDrvIns\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Register
-     */
-    PDBGFINFO pInfo;
-    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, 0, &pInfo);
-    if (RT_SUCCESS(rc))
-    {
-        pInfo->enmType = DBGFINFOTYPE_DRV;
-        pInfo->u.Drv.pfnHandler = pfnHandler;
-        pInfo->u.Drv.pDrvIns = pDrvIns;
-        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    }
-
-    return rc;
-}
-
-
-/**
- * Register a info handler owned by an internal component.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pszName     The identifier of the info.
- * @param   pszDesc     The description of the info and any arguments the handler may take.
- * @param   pfnHandler  The handler function to be called to display the info.
- */
-VMMR3DECL(int) DBGFR3InfoRegisterInternal(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERINT pfnHandler)
-{
-    return DBGFR3InfoRegisterInternalEx(pVM, pszName, pszDesc, pfnHandler, 0);
-}
-
-
-/**
- * Register a info handler owned by an internal component.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pszName     The identifier of the info.
- * @param   pszDesc     The description of the info and any arguments the handler may take.
- * @param   pfnHandler  The handler function to be called to display the info.
- * @param   fFlags      Flags, see the DBGFINFO_FLAGS_*.
- */
-VMMR3DECL(int) DBGFR3InfoRegisterInternalEx(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERINT pfnHandler, uint32_t fFlags)
-{
-    LogFlow(("DBGFR3InfoRegisterInternal: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p fFlags=%x\n",
-             pszName, pszName, pszDesc, pszDesc, pfnHandler, fFlags));
-
-    /*
-     * Validate the specific stuff.
-     */
-    if (!pfnHandler)
-    {
-        AssertMsgFailed(("No handler\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Register
-     */
-    PDBGFINFO pInfo;
-    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, fFlags, &pInfo);
-    if (RT_SUCCESS(rc))
-    {
-        pInfo->enmType = DBGFINFOTYPE_INT;
-        pInfo->u.Int.pfnHandler = pfnHandler;
-        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    }
-
-    return rc;
-}
-
-
-/**
- * Register a info handler owned by an external component.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pszName     The identifier of the info.
- * @param   pszDesc     The description of the info and any arguments the handler may take.
- * @param   pfnHandler  The handler function to be called to display the info.
- * @param   pvUser      User argument to be passed to the handler.
- */
-VMMR3DECL(int) DBGFR3InfoRegisterExternal(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLEREXT pfnHandler, void *pvUser)
-{
-    LogFlow(("DBGFR3InfoRegisterExternal: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pvUser=%p\n",
-             pszName, pszName, pszDesc, pszDesc, pfnHandler, pvUser));
-
-    /*
-     * Validate the specific stuff.
-     */
-    if (!pfnHandler)
-    {
-        AssertMsgFailed(("No handler\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Register
-     */
-    PDBGFINFO pInfo;
-    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, 0, &pInfo);
-    if (RT_SUCCESS(rc))
-    {
-        pInfo->enmType = DBGFINFOTYPE_EXT;
-        pInfo->u.Ext.pfnHandler = pfnHandler;
-        pInfo->u.Ext.pvUser = pvUser;
-        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    }
-
-    return rc;
-}
-
-
-/**
- * Deregister one(/all) info handler(s) owned by a device.
- *
- * @returns VBox status code.
- * @param   pVM         VM Handle.
- * @param   pDevIns     Device instance.
- * @param   pszName     The identifier of the info. If NULL all owned by the device.
- */
-VMMR3DECL(int) DBGFR3InfoDeregisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName)
-{
-    LogFlow(("DBGFR3InfoDeregisterDevice: pDevIns=%p pszName=%p:{%s}\n", pDevIns, pszName, pszName));
-
-    /*
-     * Validate input.
-     */
-    if (!pDevIns)
-    {
-        AssertMsgFailed(("!pDevIns\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-    size_t cchName = pszName ? strlen(pszName) : 0;
-
-    /*
-     * Enumerate the info handlers and free the requested entries.
-     */
-    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-    rc = VERR_FILE_NOT_FOUND;
-    PDBGFINFO pPrev = NULL;
-    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
-    if (pszName)
-    {
-        /*
-         * Free a specific one.
-         */
-        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
-            if (    pInfo->enmType == DBGFINFOTYPE_DEV
-                &&  pInfo->u.Dev.pDevIns == pDevIns
-                &&  pInfo->cchName == cchName
-                &&  !strcmp(pInfo->szName, pszName))
-            {
-                if (pPrev)
-                    pPrev->pNext = pInfo->pNext;
-                else
-                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
-                MMR3HeapFree(pInfo);
-                rc = VINF_SUCCESS;
-                break;
-            }
-    }
-    else
-    {
-        /*
-         * Free all owned by the driver.
-         */
-        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
-            if (    pInfo->enmType == DBGFINFOTYPE_DEV
-                &&  pInfo->u.Dev.pDevIns == pDevIns)
-            {
-                if (pPrev)
-                    pPrev->pNext = pInfo->pNext;
-                else
-                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
-                MMR3HeapFree(pInfo);
-                pInfo = pPrev;
-            }
-        rc = VINF_SUCCESS;
-    }
-    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc2);
-    AssertRC(rc);
-    LogFlow(("DBGFR3InfoDeregisterDevice: returns %Rrc\n", rc));
-    return rc;
-}
-
-/**
- * Deregister one(/all) info handler(s) owned by a driver.
- *
- * @returns VBox status code.
- * @param   pVM         VM Handle.
- * @param   pDrvIns     Driver instance.
- * @param   pszName     The identifier of the info. If NULL all owned by the driver.
- */
-VMMR3DECL(int) DBGFR3InfoDeregisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName)
-{
-    LogFlow(("DBGFR3InfoDeregisterDriver: pDrvIns=%p pszName=%p:{%s}\n", pDrvIns, pszName, pszName));
-
-    /*
-     * Validate input.
-     */
-    if (!pDrvIns)
-    {
-        AssertMsgFailed(("!pDrvIns\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-    size_t cchName = pszName ? strlen(pszName) : 0;
-
-    /*
-     * Enumerate the info handlers and free the requested entries.
-     */
-    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-    rc = VERR_FILE_NOT_FOUND;
-    PDBGFINFO pPrev = NULL;
-    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
-    if (pszName)
-    {
-        /*
-         * Free a specific one.
-         */
-        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
-            if (    pInfo->enmType == DBGFINFOTYPE_DRV
-                &&  pInfo->u.Drv.pDrvIns == pDrvIns
-                &&  pInfo->cchName == cchName
-                &&  !strcmp(pInfo->szName, pszName))
-            {
-                if (pPrev)
-                    pPrev->pNext = pInfo->pNext;
-                else
-                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
-                MMR3HeapFree(pInfo);
-                rc = VINF_SUCCESS;
-                break;
-            }
-    }
-    else
-    {
-        /*
-         * Free all owned by the driver.
-         */
-        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
-            if (    pInfo->enmType == DBGFINFOTYPE_DRV
-                &&  pInfo->u.Drv.pDrvIns == pDrvIns)
-            {
-                if (pPrev)
-                    pPrev->pNext = pInfo->pNext;
-                else
-                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
-                MMR3HeapFree(pInfo);
-                pInfo = pPrev;
-            }
-        rc = VINF_SUCCESS;
-    }
-    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc2);
-    AssertRC(rc);
-    LogFlow(("DBGFR3InfoDeregisterDriver: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Internal deregistration helper.
- *
- * @returns VBox status code.
- * @param   pVM         VM Handle.
- * @param   pszName     The identifier of the info.
- * @param   enmType     The info owner type.
- */
-static int dbgfR3InfoDeregister(PVM pVM, const char *pszName, DBGFINFOTYPE enmType)
-{
-    /*
-     * Validate input.
-     */
-    if (!pszName)
-    {
-        AssertMsgFailed(("!pszName\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Find the info handler.
-     */
-    size_t cchName = strlen(pszName);
-    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-    rc = VERR_FILE_NOT_FOUND;
-    PDBGFINFO pPrev = NULL;
-    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
-    for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
-        if (    pInfo->cchName == cchName
-            &&  !strcmp(pInfo->szName, pszName)
-            &&  pInfo->enmType == enmType)
-        {
-            if (pPrev)
-                pPrev->pNext = pInfo->pNext;
-            else
-                pVM->dbgf.s.pInfoFirst = pInfo->pNext;
-            MMR3HeapFree(pInfo);
-            rc = VINF_SUCCESS;
-            break;
-        }
-    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc2);
-    AssertRC(rc);
-    LogFlow(("dbgfR3InfoDeregister: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Deregister a info handler owned by an internal component.
- *
- * @returns VBox status code.
- * @param   pVM         VM Handle.
- * @param   pszName     The identifier of the info. If NULL all owned by the device.
- */
-VMMR3DECL(int) DBGFR3InfoDeregisterInternal(PVM pVM, const char *pszName)
-{
-    LogFlow(("DBGFR3InfoDeregisterInternal: pszName=%p:{%s}\n", pszName, pszName));
-    return dbgfR3InfoDeregister(pVM, pszName, DBGFINFOTYPE_INT);
-}
-
-
-/**
- * Deregister a info handler owned by an external component.
- *
- * @returns VBox status code.
- * @param   pVM         VM Handle.
- * @param   pszName     The identifier of the info. If NULL all owned by the device.
- */
-VMMR3DECL(int) DBGFR3InfoDeregisterExternal(PVM pVM, const char *pszName)
-{
-    LogFlow(("DBGFR3InfoDeregisterExternal: pszName=%p:{%s}\n", pszName, pszName));
-    return dbgfR3InfoDeregister(pVM, pszName, DBGFINFOTYPE_EXT);
-}
-
-
-/**
- * Display a piece of info writing to the supplied handler.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pszName     The identifier of the info to display.
- * @param   pszArgs     Arguments to the info handler.
- * @param   pHlp        The output helper functions. If NULL the logger will be used.
- */
-VMMR3DECL(int) DBGFR3Info(PVM pVM, const char *pszName, const char *pszArgs, PCDBGFINFOHLP pHlp)
-{
-    /*
-     * Validate input.
-     */
-    if (!pszName)
-    {
-        AssertMsgFailed(("!pszName\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (pHlp)
-    {
-        if (    !pHlp->pfnPrintf
-            ||  !pHlp->pfnPrintfV)
-        {
-            AssertMsgFailed(("A pHlp member is missing!\n"));
-            return VERR_INVALID_PARAMETER;
-        }
-    }
-    else
-        pHlp = &g_dbgfR3InfoLogHlp;
-
-    /*
-     * Find the info handler.
-     */
-    size_t cchName = strlen(pszName);
-    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
-    for (; pInfo; pInfo = pInfo->pNext)
-        if (    pInfo->cchName == cchName
-            &&  !memcmp(pInfo->szName, pszName, cchName))
-            break;
-    if (pInfo)
-    {
-        /*
-         * Found it.
-         *      Make a copy of it on the stack so we can leave the crit sect.
-         *      Switch on the type and invoke the handler.
-         */
-        DBGFINFO Info = *pInfo;
-        rc = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-        AssertRC(rc);
-        rc = VINF_SUCCESS;
-        switch (Info.enmType)
-        {
-            case DBGFINFOTYPE_DEV:
-                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Dev.pfnHandler, 3, Info.u.Dev.pDevIns, pHlp, pszArgs);
-                else
-                    Info.u.Dev.pfnHandler(Info.u.Dev.pDevIns, pHlp, pszArgs);
-                break;
-
-            case DBGFINFOTYPE_DRV:
-                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Drv.pfnHandler, 3, Info.u.Drv.pDrvIns, pHlp, pszArgs);
-                else
-                    Info.u.Drv.pfnHandler(Info.u.Drv.pDrvIns, pHlp, pszArgs);
-                break;
-
-            case DBGFINFOTYPE_INT:
-                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Int.pfnHandler, 3, pVM, pHlp, pszArgs);
-                else
-                    Info.u.Int.pfnHandler(pVM, pHlp, pszArgs);
-                break;
-
-            case DBGFINFOTYPE_EXT:
-                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Ext.pfnHandler, 3, Info.u.Ext.pvUser, pHlp, pszArgs);
-                else
-                    Info.u.Ext.pfnHandler(Info.u.Ext.pvUser, pHlp, pszArgs);
-                break;
-
-            default:
-                AssertMsgFailed(("Invalid info type enmType=%d\n", Info.enmType));
-                rc = VERR_INTERNAL_ERROR;
-                break;
-        }
-    }
-    else
-    {
-        rc = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-        AssertRC(rc);
-        rc = VERR_FILE_NOT_FOUND;
-    }
-    return rc;
-}
-
-
-/**
- * Wrapper for DBGFR3Info that outputs to the release log.
- *
- * @returns See DBGFR3Info.
- * @param   pVM                 The VM handle.
- * @param   pszName             See DBGFR3Info.
- * @param   pszArgs             See DBGFR3Info.
- */
-VMMR3DECL(int) DBGFR3InfoLogRel(PVM pVM, const char *pszName, const char *pszArgs)
-{
-    return DBGFR3Info(pVM, pszName, pszArgs, &g_dbgfR3InfoLogRelHlp);
-}
-
-
-/**
- * Wrapper for DBGFR3Info that outputs to standard error.
- *
- * @returns See DBGFR3Info.
- * @param   pVM                 The VM handle.
- * @param   pszName             See DBGFR3Info.
- * @param   pszArgs             See DBGFR3Info.
- */
-VMMR3DECL(int) DBGFR3InfoStdErr(PVM pVM, const char *pszName, const char *pszArgs)
-{
-    return DBGFR3Info(pVM, pszName, pszArgs, &g_dbgfR3InfoStdErrHlp);
-}
-
-
-/**
- * Display several info items.
- *
- * This is intended used by the fatal error dump only.
- *
- * @returns
- * @param   pVM             The VM handle.
- * @param   pszIncludePat   Simple string pattern of info items to include.
- * @param   pszExcludePat   Simple string pattern of info items to exclude.
- * @param   pszSepFmt       Item separator format string.  The item name will be
- *                          given as parameter.
- * @param   pHlp            The output helper functions.  If NULL the logger
- *                          will be used.
- *
- * @threads EMT
- */
-VMMR3DECL(int) DBGFR3InfoMulti(PVM pVM, const char *pszIncludePat, const char *pszExcludePat, const char *pszSepFmt,
-                               PCDBGFINFOHLP pHlp)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-    AssertPtrReturn(pszIncludePat, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszExcludePat, VERR_INVALID_POINTER);
-    if (pHlp)
-    {
-        AssertPtrReturn(pHlp->pfnPrintf, VERR_INVALID_POINTER);
-        AssertPtrReturn(pHlp->pfnPrintfV, VERR_INVALID_POINTER);
-    }
-    else
-        pHlp = &g_dbgfR3InfoLogHlp;
-
-    size_t const cchIncludePat = strlen(pszIncludePat);
-    size_t const cchExcludePat = strlen(pszExcludePat);
-    const char *pszArgs = "";
-
-    /*
-     * Enumerate the info handlers and call the ones matching.
-     * Note! We won't leave the critical section here...
-     */
-    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-    rc = VWRN_NOT_FOUND;
-    for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
-    {
-        if (   RTStrSimplePatternMultiMatch(pszIncludePat, cchIncludePat, pInfo->szName, pInfo->cchName, NULL)
-            && !RTStrSimplePatternMultiMatch(pszExcludePat, cchExcludePat, pInfo->szName, pInfo->cchName, NULL))
-        {
-            pHlp->pfnPrintf(pHlp, pszSepFmt, pInfo->szName);
-            rc = VINF_SUCCESS;
-            switch (pInfo->enmType)
-            {
-                case DBGFINFOTYPE_DEV:
-                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Dev.pfnHandler, 3, pInfo->u.Dev.pDevIns, pHlp, pszArgs);
-                    else
-                        pInfo->u.Dev.pfnHandler(pInfo->u.Dev.pDevIns, pHlp, pszArgs);
-                    break;
-
-                case DBGFINFOTYPE_DRV:
-                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Drv.pfnHandler, 3, pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
-                    else
-                        pInfo->u.Drv.pfnHandler(pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
-                    break;
-
-                case DBGFINFOTYPE_INT:
-                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Int.pfnHandler, 3, pVM, pHlp, pszArgs);
-                    else
-                        pInfo->u.Int.pfnHandler(pVM, pHlp, pszArgs);
-                    break;
-
-                case DBGFINFOTYPE_EXT:
-                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
-                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Ext.pfnHandler, 3, pInfo->u.Ext.pvUser, pHlp, pszArgs);
-                    else
-                        pInfo->u.Ext.pfnHandler(pInfo->u.Ext.pvUser, pHlp, pszArgs);
-                    break;
-
-                default:
-                    AssertMsgFailed(("Invalid info type enmType=%d\n", pInfo->enmType));
-                    rc = VERR_INTERNAL_ERROR;
-                    break;
-            }
-        }
-    }
-    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc2);
-
-    return rc;
-}
-
-
-/**
- * Enumerate all the register info handlers.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pfnCallback     Pointer to callback function.
- * @param   pvUser          User argument to pass to the callback.
- */
-VMMR3DECL(int) DBGFR3InfoEnum(PVM pVM, PFNDBGFINFOENUM pfnCallback, void *pvUser)
-{
-    LogFlow(("DBGFR3InfoLog: pfnCallback=%p pvUser=%p\n", pfnCallback, pvUser));
-
-    /*
-     * Validate input.
-     */
-    if (!pfnCallback)
-    {
-        AssertMsgFailed(("!pfnCallback\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Enter and enumerate.
-     */
-    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-
-    rc = VINF_SUCCESS;
-    for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; RT_SUCCESS(rc) && pInfo; pInfo = pInfo->pNext)
-        rc = pfnCallback(pVM, pInfo->szName, pInfo->pszDesc, pvUser);
-
-    /*
-     * Leave and exit.
-     */
-    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc2);
-
-    LogFlow(("DBGFR3InfoLog: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Info handler, internal version.
- *
- * @param   pVM         The VM handle.
- * @param   pHlp        Callback functions for doing output.
- * @param   pszArgs     Argument string. Optional and specific to the handler.
- */
-static DECLCALLBACK(void) dbgfR3InfoHelp(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    LogFlow(("dbgfR3InfoHelp: pszArgs=%s\n", pszArgs));
-
-    /*
-     * Enter and enumerate.
-     */
-    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-
-    if (pszArgs && *pszArgs)
-    {
-        for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
-        {
-            const char *psz = strstr(pszArgs, pInfo->szName);
-            if (    psz
-                &&  (   psz == pszArgs
-                     || RT_C_IS_SPACE(psz[-1]))
-                &&  (   !psz[pInfo->cchName]
-                     || RT_C_IS_SPACE(psz[pInfo->cchName])))
-                pHlp->pfnPrintf(pHlp, "%-16s  %s\n",
-                                pInfo->szName, pInfo->pszDesc);
-        }
-    }
-    else
-    {
-        for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
-            pHlp->pfnPrintf(pHlp, "%-16s  %s\n",
-                            pInfo->szName, pInfo->pszDesc);
-    }
-
-    /*
-     * Leave and exit.
-     */
-    rc = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
-    AssertRC(rc);
-}
-
Index: unk/src/VBox/VMM/DBGFInternal.h
===================================================================
--- /trunk/src/VBox/VMM/DBGFInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,331 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___DBGFInternal_h
-#define ___DBGFInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <iprt/semaphore.h>
-#include <iprt/critsect.h>
-#include <iprt/string.h>
-#include <iprt/avl.h>
-#include <VBox/dbgf.h>
-
-
-
-/** @defgroup grp_dbgf_int   Internals
- * @ingroup grp_dbgf
- * @internal
- * @{
- */
-
-
-/** VMM Debugger Command. */
-typedef enum DBGFCMD
-{
-    /** No command.
-     * This is assigned to the field by the emulation thread after
-     * a command has been completed. */
-    DBGFCMD_NO_COMMAND = 0,
-    /** Halt the VM. */
-    DBGFCMD_HALT,
-    /** Resume execution. */
-    DBGFCMD_GO,
-    /** Single step execution - stepping into calls. */
-    DBGFCMD_SINGLE_STEP,
-    /** Set a breakpoint. */
-    DBGFCMD_BREAKPOINT_SET,
-    /** Set a access breakpoint. */
-    DBGFCMD_BREAKPOINT_SET_ACCESS,
-    /** Set a REM breakpoint. */
-    DBGFCMD_BREAKPOINT_SET_REM,
-    /** Clear a breakpoint. */
-    DBGFCMD_BREAKPOINT_CLEAR,
-    /** Enable a breakpoint. */
-    DBGFCMD_BREAKPOINT_ENABLE,
-    /** Disable a breakpoint. */
-    DBGFCMD_BREAKPOINT_DISABLE,
-    /** List breakpoints. */
-    DBGFCMD_BREAKPOINT_LIST,
-
-    /** Detaches the debugger.
-     * Disabling all breakpoints, watch points and the like. */
-    DBGFCMD_DETACH_DEBUGGER = 0x7ffffffe,
-    /** Detached the debugger.
-     * The isn't a command as such, it's just that it's necessary for the
-     * detaching protocol to be racefree. */
-    DBGFCMD_DETACHED_DEBUGGER = 0x7fffffff
-} DBGFCMD;
-
-/**
- * VMM Debugger Command.
- */
-typedef union DBGFCMDDATA
-{
-    uint32_t    uDummy;
-} DBGFCMDDATA;
-/** Pointer to DBGF Command Data. */
-typedef DBGFCMDDATA *PDBGFCMDDATA;
-
-/**
- * Info type.
- */
-typedef enum DBGFINFOTYPE
-{
-    /** Invalid. */
-    DBGFINFOTYPE_INVALID = 0,
-    /** Device owner. */
-    DBGFINFOTYPE_DEV,
-    /** Driver owner. */
-    DBGFINFOTYPE_DRV,
-    /** Internal owner. */
-    DBGFINFOTYPE_INT,
-    /** External owner. */
-    DBGFINFOTYPE_EXT
-} DBGFINFOTYPE;
-
-
-/** Pointer to info structure. */
-typedef struct DBGFINFO *PDBGFINFO;
-
-/**
- * Info structure.
- */
-typedef struct DBGFINFO
-{
-    /** The flags. */
-    uint32_t        fFlags;
-    /** Owner type. */
-    DBGFINFOTYPE    enmType;
-    /** Per type data. */
-    union
-    {
-        /** DBGFINFOTYPE_DEV */
-        struct
-        {
-            /** Device info handler function. */
-            PFNDBGFHANDLERDEV   pfnHandler;
-            /** The device instance. */
-            PPDMDEVINS          pDevIns;
-        } Dev;
-
-        /** DBGFINFOTYPE_DRV */
-        struct
-        {
-            /** Driver info handler function. */
-            PFNDBGFHANDLERDRV   pfnHandler;
-            /** The driver instance. */
-            PPDMDRVINS          pDrvIns;
-        } Drv;
-
-        /** DBGFINFOTYPE_INT */
-        struct
-        {
-            /** Internal info handler function. */
-            PFNDBGFHANDLERINT   pfnHandler;
-        } Int;
-
-        /** DBGFINFOTYPE_EXT */
-        struct
-        {
-            /** External info handler function. */
-            PFNDBGFHANDLEREXT   pfnHandler;
-            /** The user argument. */
-            void               *pvUser;
-        } Ext;
-    } u;
-
-    /** Pointer to the description. */
-    const char     *pszDesc;
-    /** Pointer to the next info structure. */
-    PDBGFINFO       pNext;
-    /** The identifier name length. */
-    size_t          cchName;
-    /** The identifier name. (Extends 'beyond' the struct as usual.) */
-    char            szName[1];
-} DBGFINFO;
-
-
-/**
- * Guest OS digger instance.
- */
-typedef struct DBGFOS
-{
-    /** Pointer to the registration record. */
-    PCDBGFOSREG pReg;
-    /** Pointer to the next OS we've registered. */
-    struct DBGFOS *pNext;
-    /** The instance data (variable size). */
-    uint8_t abData[16];
-} DBGFOS;
-/** Pointer to guest OS digger instance. */
-typedef DBGFOS *PDBGFOS;
-/** Pointer to const guest OS digger instance. */
-typedef DBGFOS const *PCDBGFOS;
-
-
-/**
- * Converts a DBGF pointer into a VM pointer.
- * @returns Pointer to the VM structure the CPUM is part of.
- * @param   pDBGF   Pointer to DBGF instance data.
- */
-#define DBGF2VM(pDBGF)  ( (PVM)((char*)pDBGF - pDBGF->offVM) )
-
-
-/**
- * DBGF Data (part of VM)
- */
-typedef struct DBGF
-{
-    /** Offset to the VM structure. */
-    RTINT                   offVM;
-
-    /** Debugger Attached flag.
-     * Set if a debugger is attached, elsewise it's clear.
-     */
-    bool volatile           fAttached;
-
-    /** Stopped in the Hypervisor.
-     * Set if we're stopped on a trace, breakpoint or assertion inside
-     * the hypervisor and have to restrict the available operations.
-     */
-    bool volatile           fStoppedInHyper;
-
-    /**
-     * Ping-Pong construct where the Ping side is the VMM and the Pong side
-     * the Debugger.
-     */
-    RTPINGPONG              PingPong;
-
-    /** The Event to the debugger.
-     * The VMM will ping the debugger when the event is ready. The event is
-     * either a response to a command or to a break/watch point issued
-     * previously.
-     */
-    DBGFEVENT               DbgEvent;
-
-    /** The Command to the VMM.
-     * Operated in an atomic fashion since the VMM will poll on this.
-     * This means that a the command data must be written before this member
-     * is set. The VMM will reset this member to the no-command state
-     * when it have processed it.
-     */
-    DBGFCMD volatile        enmVMMCmd;
-    /** The Command data.
-     * Not all commands take data. */
-    DBGFCMDDATA             VMMCmdData;
-
-    /** List of registered info handlers. */
-    R3PTRTYPE(PDBGFINFO)    pInfoFirst;
-    /** Critical section protecting the above list. */
-    RTCRITSECT              InfoCritSect;
-
-    /** Range tree containing the loaded symbols of the a VM.
-     * This tree will never have blind spots. */
-    R3PTRTYPE(AVLRGCPTRTREE) SymbolTree;
-    /** Symbol name space. */
-    R3PTRTYPE(PRTSTRSPACE)  pSymbolSpace;
-    /** Indicates whether DBGFSym.cpp is initialized or not.
-     * This part is initialized in a lazy manner for performance reasons. */
-    bool                    fSymInited;
-    /** Alignment padding. */
-    RTUINT                  uAlignment0;
-
-    /** The number of hardware breakpoints. */
-    RTUINT                  cHwBreakpoints;
-    /** The number of active breakpoints. */
-    RTUINT                  cBreakpoints;
-    /** Array of hardware breakpoints. (0..3)
-     * This is shared among all the CPUs because life is much simpler that way. */
-    DBGFBP                  aHwBreakpoints[4];
-    /** Array of int 3 and REM breakpoints. (4..)
-     * @remark This is currently a fixed size array for reasons of simplicity. */
-    DBGFBP                  aBreakpoints[32];
-
-    /** The address space database lock. */
-    RTSEMRW                 hAsDbLock;
-    /** The address space handle database.      (Protected by hAsDbLock.) */
-    R3PTRTYPE(AVLPVTREE)    AsHandleTree;
-    /** The address space process id database.  (Protected by hAsDbLock.) */
-    R3PTRTYPE(AVLU32TREE)   AsPidTree;
-    /** The address space name database.        (Protected by hAsDbLock.) */
-    R3PTRTYPE(RTSTRSPACE)   AsNameSpace;
-    /** Special address space aliases.          (Protected by hAsDbLock.) */
-    RTDBGAS volatile        ahAsAliases[DBGF_AS_COUNT];
-    /** For lazily populating the aliased address spaces. */
-    bool volatile           afAsAliasPopuplated[DBGF_AS_COUNT];
-    /** Alignment padding. */
-    bool                    afAlignment[2];
-
-    /** The current Guest OS digger. */
-    R3PTRTYPE(PDBGFOS)      pCurOS;
-    /** The head of the Guest OS digger instances. */
-    R3PTRTYPE(PDBGFOS)      pOSHead;
-} DBGF;
-/** Pointer to DBGF Data. */
-typedef DBGF *PDBGF;
-
-
-/** Converts a DBGFCPU pointer into a VM pointer. */
-#define DBGFCPU_2_VM(pDbgfCpu) ((PVM)((uint8_t *)(pDbgfCpu) + (pDbgfCpu)->offVM))
-
-/**
- * The per CPU data for DBGF.
- */
-typedef struct DBGFCPU
-{
-    /** The offset into the VM structure.
-     * @see DBGFCPU_2_VM(). */
-    uint32_t                offVM;
-
-    /** Current active breakpoint (id).
-     * This is ~0U if not active. It is set when a execution engine
-     * encounters a breakpoint and returns VINF_EM_DBG_BREAKPOINT. This is
-     * currently not used for REM breakpoints because of the lazy coupling
-     * between VBox and REM. */
-    uint32_t                iActiveBp;
-    /** Set if we're singlestepping in raw mode.
-     * This is checked and cleared in the \#DB handler. */
-    bool                    fSingleSteppingRaw;
-
-    /** Padding the structure to 16 bytes. */
-    uint8_t                 abReserved[3];
-} DBGFCPU;
-/** Pointer to DBGFCPU data. */
-typedef DBGFCPU *PDBGFCPU;
-
-
-int  dbgfR3AsInit(PVM pVM);
-void dbgfR3AsTerm(PVM pVM);
-void dbgfR3AsRelocate(PVM pVM, RTGCUINTPTR offDelta);
-int  dbgfR3InfoInit(PVM pVM);
-int  dbgfR3InfoTerm(PVM pVM);
-void dbgfR3OSTerm(PVM pVM);
-int  dbgfR3SymInit(PVM pVM);
-int  dbgfR3SymTerm(PVM pVM);
-int  dbgfR3BpInit(PVM pVM);
-
-
-
-#ifdef IN_RING3
-
-#endif
-
-/** @} */
-
-#endif
Index: unk/src/VBox/VMM/DBGFLog.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFLog.cpp	(revision 35332)
+++ 	(revision )
@@ -1,132 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Log Manager.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#include <VBox/vmapi.h>
-#include <VBox/vmm.h>
-#include <VBox/dbgf.h>
-#include <VBox/log.h>
-#include <VBox/err.h>
-#include <iprt/assert.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) dbgfR3LogModifyGroups(PVM pVM, const char *pszGroupSettings);
-static DECLCALLBACK(int) dbgfR3LogModifyFlags(PVM pVM, const char *pszFlagSettings);
-static DECLCALLBACK(int) dbgfR3LogModifyDestinations(PVM pVM, const char *pszDestSettings);
-
-
-/**
- * Changes the logger group settings.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszGroupSettings    The group settings string. (VBOX_LOG)
- */
-VMMR3DECL(int) DBGFR3LogModifyGroups(PVM pVM, const char *pszGroupSettings)
-{
-    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszGroupSettings, VERR_INVALID_POINTER);
-
-    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyGroups, 2, pVM, pszGroupSettings);
-}
-
-
-/**
- * EMT worker for DBGFR3LogModifyGroups.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszGroupSettings    The group settings string. (VBOX_LOG)
- */
-static DECLCALLBACK(int) dbgfR3LogModifyGroups(PVM pVM, const char *pszGroupSettings)
-{
-    int rc = RTLogGroupSettings(NULL, pszGroupSettings);
-    if (RT_SUCCESS(rc))
-        rc = VMMR3UpdateLoggers(pVM);
-    return rc;
-}
-
-
-/**
- * Changes the logger flag settings.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszFlagSettings     The group settings string. (VBOX_LOG_FLAGS)
- */
-VMMR3DECL(int) DBGFR3LogModifyFlags(PVM pVM, const char *pszFlagSettings)
-{
-    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszFlagSettings, VERR_INVALID_POINTER);
-
-    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyFlags, 2, pVM, pszFlagSettings);
-}
-
-
-/**
- * EMT worker for DBGFR3LogModifyFlags.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszFlagSettings     The group settings string. (VBOX_LOG_FLAGS)
- */
-static DECLCALLBACK(int) dbgfR3LogModifyFlags(PVM pVM, const char *pszFlagSettings)
-{
-    int rc = RTLogFlags(NULL, pszFlagSettings);
-    if (RT_SUCCESS(rc))
-        rc = VMMR3UpdateLoggers(pVM);
-    return rc;
-}
-
-
-/**
- * Changes the logger destination settings.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszDestSettings     The destination settings string. (VBOX_LOG_DEST)
- */
-VMMR3DECL(int) DBGFR3LogModifyDestinations(PVM pVM, const char *pszDestSettings)
-{
-    AssertReturn(VALID_PTR(pVM), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(pszDestSettings), VERR_INVALID_POINTER);
-
-    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyDestinations, 2, pVM, pszDestSettings);
-}
-
-
-/**
- * EMT worker for DBGFR3LogModifyFlags.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszDestSettings     The destination settings string. (VBOX_LOG_DEST)
- */
-static DECLCALLBACK(int) dbgfR3LogModifyDestinations(PVM pVM, const char *pszDestSettings)
-{
-    int rc = RTLogDestinations(NULL, pszDestSettings);
-    if (RT_SUCCESS(rc))
-        rc = VMMR3UpdateLoggers(pVM);
-    return rc;
-}
-
Index: unk/src/VBox/VMM/DBGFMem.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFMem.cpp	(revision 35332)
+++ 	(revision )
@@ -1,662 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Memory Methods.
- */
-
-/*
- * Copyright (C) 2007-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/selm.h>
-#include <VBox/hwaccm.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <VBox/mm.h>
-
-
-
-/**
- * Scan guest memory for an exact byte string.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   idCpu       The ID of the CPU context to search in.
- * @param   pAddress    Where to store the mixed address.
- * @param   puAlign     The alignment restriction imposed on the search result.
- * @param   pcbRange    The number of bytes to scan. Passed as a pointer because
- *                      it may be 64-bit.
- * @param   pabNeedle   What to search for - exact search.
- * @param   cbNeedle    Size of the search byte string.
- * @param   pHitAddress Where to put the address of the first hit.
- */
-static DECLCALLBACK(int) dbgfR3MemScan(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, PCRTGCUINTPTR pcbRange, RTGCUINTPTR *puAlign,
-                                       const uint8_t *pabNeedle, size_t cbNeedle, PDBGFADDRESS pHitAddress)
-{
-    Assert(idCpu == VMMGetCpuId(pVM));
-
-    /*
-     * Validate the input we use, PGM does the rest.
-     */
-    RTGCUINTPTR cbRange = *pcbRange;
-    if (!DBGFR3AddrIsValid(pVM, pAddress))
-        return VERR_INVALID_POINTER;
-    if (!VALID_PTR(pHitAddress))
-        return VERR_INVALID_POINTER;
-    if (DBGFADDRESS_IS_HMA(pAddress))
-        return VERR_INVALID_POINTER;
-
-    /*
-     * Select DBGF worker by addressing mode.
-     */
-    int     rc;
-    PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
-    PGMMODE enmMode = PGMGetGuestMode(pVCpu);
-    if (    enmMode == PGMMODE_REAL
-        ||  enmMode == PGMMODE_PROTECTED
-        ||  DBGFADDRESS_IS_PHYS(pAddress)
-        )
-    {
-        RTGCPHYS GCPhysAlign = *puAlign;
-        if (GCPhysAlign != *puAlign)
-            return VERR_OUT_OF_RANGE;
-        RTGCPHYS PhysHit;
-        rc = PGMR3DbgScanPhysical(pVM, pAddress->FlatPtr, cbRange, GCPhysAlign, pabNeedle, cbNeedle, &PhysHit);
-        if (RT_SUCCESS(rc))
-            DBGFR3AddrFromPhys(pVM, pHitAddress, PhysHit);
-    }
-    else
-    {
-#if GC_ARCH_BITS > 32
-        if (    (   pAddress->FlatPtr >= _4G
-                 || pAddress->FlatPtr + cbRange > _4G)
-            &&  enmMode != PGMMODE_AMD64
-            &&  enmMode != PGMMODE_AMD64_NX)
-            return VERR_DBGF_MEM_NOT_FOUND;
-#endif
-        RTGCUINTPTR GCPtrHit;
-        rc = PGMR3DbgScanVirtual(pVM, pVCpu, pAddress->FlatPtr, cbRange, *puAlign, pabNeedle, cbNeedle, &GCPtrHit);
-        if (RT_SUCCESS(rc))
-            DBGFR3AddrFromFlat(pVM, pHitAddress, GCPtrHit);
-    }
-
-    return rc;
-}
-
-
-/**
- * Scan guest memory for an exact byte string.
- *
- * @returns VBox status codes:
- * @retval  VINF_SUCCESS and *pGCPtrHit on success.
- * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
- * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
- * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
- *
- * @param   pVM         The VM handle.
- * @param   idCpu       The ID of the CPU context to search in.
- * @param   pAddress    Where to store the mixed address.
- * @param   cbRange     The number of bytes to scan.
- * @param   uAlign      The alignment restriction imposed on the result.
- *                      Usually set to 1.
- * @param   pvNeedle    What to search for - exact search.
- * @param   cbNeedle    Size of the search byte string.
- * @param   pHitAddress Where to put the address of the first hit.
- *
- * @thread  Any thread.
- */
-VMMR3DECL(int) DBGFR3MemScan(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, RTGCUINTPTR cbRange, RTGCUINTPTR uAlign,
-                             const void *pvNeedle, size_t cbNeedle, PDBGFADDRESS pHitAddress)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3MemScan, 8,
-                           pVM, idCpu, pAddress, &cbRange, &uAlign, pvNeedle, cbNeedle, pHitAddress);
-
-}
-
-
-/**
- * Read guest memory.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pAddress        Where to start reading.
- * @param   pvBuf           Where to store the data we've read.
- * @param   cbRead          The number of bytes to read.
- */
-static DECLCALLBACK(int) dbgfR3MemRead(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void *pvBuf, size_t cbRead)
-{
-    Assert(idCpu == VMMGetCpuId(pVM));
-
-    /*
-     * Validate the input we use, PGM does the rest.
-     */
-    if (!DBGFR3AddrIsValid(pVM, pAddress))
-        return VERR_INVALID_POINTER;
-    if (!VALID_PTR(pvBuf))
-        return VERR_INVALID_POINTER;
-
-    /*
-     * HMA is special
-     */
-    int rc;
-    if (DBGFADDRESS_IS_HMA(pAddress))
-    {
-        if (DBGFADDRESS_IS_PHYS(pAddress))
-            rc = VERR_INVALID_POINTER;
-        else
-            rc = MMR3HyperReadGCVirt(pVM, pvBuf, pAddress->FlatPtr, cbRead);
-    }
-    else
-    {
-        /*
-         * Select DBGF worker by addressing mode.
-         */
-        PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
-        PGMMODE enmMode = PGMGetGuestMode(pVCpu);
-        if (    enmMode == PGMMODE_REAL
-            ||  enmMode == PGMMODE_PROTECTED
-            ||  DBGFADDRESS_IS_PHYS(pAddress) )
-            rc = PGMPhysSimpleReadGCPhys(pVM, pvBuf, pAddress->FlatPtr, cbRead);
-        else
-        {
-#if GC_ARCH_BITS > 32
-            if (    (   pAddress->FlatPtr >= _4G
-                     || pAddress->FlatPtr + cbRead > _4G)
-                &&  enmMode != PGMMODE_AMD64
-                &&  enmMode != PGMMODE_AMD64_NX)
-                return VERR_PAGE_TABLE_NOT_PRESENT;
-#endif
-            rc = PGMPhysSimpleReadGCPtr(pVCpu, pvBuf, pAddress->FlatPtr, cbRead);
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Read guest memory.
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idCpu           The ID of the source CPU context (for the address).
- * @param   pAddress        Where to start reading.
- * @param   pvBuf           Where to store the data we've read.
- * @param   cbRead          The number of bytes to read.
- */
-VMMR3DECL(int) DBGFR3MemRead(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void *pvBuf, size_t cbRead)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    if ((pAddress->fFlags & DBGFADDRESS_FLAGS_TYPE_MASK) == DBGFADDRESS_FLAGS_RING0)
-    {
-        AssertCompile(sizeof(RTHCUINTPTR) <= sizeof(pAddress->FlatPtr));
-        return VMMR3ReadR0Stack(pVM, idCpu, (RTHCUINTPTR)pAddress->FlatPtr, pvBuf, cbRead);
-    }
-    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3MemRead, 5, pVM, idCpu, pAddress, pvBuf, cbRead);
-}
-
-
-/**
- * Read a zero terminated string from guest memory.
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idCpu           The ID of the source CPU context (for the address).
- * @param   pAddress        Where to start reading.
- * @param   pszBuf          Where to store the string.
- * @param   cchBuf          The size of the buffer.
- */
-static DECLCALLBACK(int) dbgfR3MemReadString(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, char *pszBuf, size_t cchBuf)
-{
-    /*
-     * Validate the input we use, PGM does the rest.
-     */
-    if (!DBGFR3AddrIsValid(pVM, pAddress))
-        return VERR_INVALID_POINTER;
-    if (!VALID_PTR(pszBuf))
-        return VERR_INVALID_POINTER;
-
-    /*
-     * Let dbgfR3MemRead do the job.
-     */
-    int rc = dbgfR3MemRead(pVM, idCpu, pAddress, pszBuf, cchBuf);
-
-    /*
-     * Make sure the result is terminated and that overflow is signaled.
-     * This may look a bit reckless with the rc but, it should be fine.
-     */
-    if (!RTStrEnd(pszBuf, cchBuf))
-    {
-        pszBuf[cchBuf - 1] = '\0';
-        rc = VINF_BUFFER_OVERFLOW;
-    }
-    /*
-     * Handle partial reads (not perfect).
-     */
-    else if (RT_FAILURE(rc))
-    {
-        if (pszBuf[0])
-            rc = VINF_SUCCESS;
-    }
-
-    return rc;
-}
-
-
-/**
- * Read a zero terminated string from guest memory.
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idCpu           The ID of the source CPU context (for the address).
- * @param   pAddress        Where to start reading.
- * @param   pszBuf          Where to store the string.
- * @param   cchBuf          The size of the buffer.
- */
-VMMR3DECL(int) DBGFR3MemReadString(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, char *pszBuf, size_t cchBuf)
-{
-    /*
-     * Validate and zero output.
-     */
-    if (!VALID_PTR(pszBuf))
-        return VERR_INVALID_POINTER;
-    if (cchBuf <= 0)
-        return VERR_INVALID_PARAMETER;
-    memset(pszBuf, 0, cchBuf);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-
-    /*
-     * Pass it on to the EMT.
-     */
-    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3MemReadString, 5, pVM, idCpu, pAddress, pszBuf, cchBuf);
-}
-
-
-/**
- * Writes guest memory.
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idCpu           The ID of the target CPU context (for the address).
- * @param   pAddress        Where to start writing.
- * @param   pvBuf           The data to write.
- * @param   cbWrite         The number of bytes to write.
- */
-static DECLCALLBACK(int) dbgfR3MemWrite(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void const *pvBuf, size_t cbWrite)
-{
-    /*
-     * Validate the input we use, PGM does the rest.
-     */
-    if (!DBGFR3AddrIsValid(pVM, pAddress))
-        return VERR_INVALID_POINTER;
-    if (!VALID_PTR(pvBuf))
-        return VERR_INVALID_POINTER;
-
-    /*
-     * HMA is always special.
-     */
-    int rc;
-    if (DBGFADDRESS_IS_HMA(pAddress))
-    {
-        /** @todo write to HMA. */
-        rc = VERR_ACCESS_DENIED;
-    }
-    else
-    {
-        /*
-         * Select PGM function by addressing mode.
-         */
-        PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
-        PGMMODE enmMode = PGMGetGuestMode(pVCpu);
-        if (    enmMode == PGMMODE_REAL
-            ||  enmMode == PGMMODE_PROTECTED
-            ||  DBGFADDRESS_IS_PHYS(pAddress) )
-            rc = PGMPhysSimpleWriteGCPhys(pVM, pAddress->FlatPtr, pvBuf, cbWrite);
-        else
-        {
-#if GC_ARCH_BITS > 32
-            if (    (   pAddress->FlatPtr >= _4G
-                     || pAddress->FlatPtr + cbWrite > _4G)
-                &&  enmMode != PGMMODE_AMD64
-                &&  enmMode != PGMMODE_AMD64_NX)
-                return VERR_PAGE_TABLE_NOT_PRESENT;
-#endif
-            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pAddress->FlatPtr, pvBuf, cbWrite);
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Read guest memory.
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idCpu           The ID of the target CPU context (for the address).
- * @param   pAddress        Where to start writing.
- * @param   pvBuf           The data to write.
- * @param   cbRead          The number of bytes to write.
- */
-VMMR3DECL(int) DBGFR3MemWrite(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void const *pvBuf, size_t cbWrite)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3MemWrite, 5, pVM, idCpu, pAddress, pvBuf, cbWrite);
-}
-
-
-/**
- * Worker for DBGFR3SelQueryInfo that calls into SELM.
- */
-static DECLCALLBACK(int) dbgfR3SelQueryInfo(PVM pVM, VMCPUID idCpu, RTSEL Sel, uint32_t fFlags, PDBGFSELINFO pSelInfo)
-{
-    /*
-     * Make the query.
-     */
-    int rc;
-    if (!(fFlags & DBGFSELQI_FLAGS_DT_SHADOW))
-    {
-        PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-        VMCPU_ASSERT_EMT(pVCpu);
-        rc = SELMR3GetSelectorInfo(pVM, pVCpu, Sel, pSelInfo);
-
-        /*
-         * 64-bit mode HACKS for making data and stack selectors wide open when
-         * queried. This is voodoo magic.
-         */
-        if (fFlags & DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE)
-        {
-            /* Expand 64-bit data and stack selectors. The check is a bit bogus... */
-            if (    RT_SUCCESS(rc)
-                &&  (pSelInfo->fFlags & (  DBGFSELINFO_FLAGS_LONG_MODE | DBGFSELINFO_FLAGS_REAL_MODE | DBGFSELINFO_FLAGS_PROT_MODE
-                                         | DBGFSELINFO_FLAGS_GATE      | DBGFSELINFO_FLAGS_HYPER
-                                         | DBGFSELINFO_FLAGS_INVALID   | DBGFSELINFO_FLAGS_NOT_PRESENT))
-                     == DBGFSELINFO_FLAGS_LONG_MODE
-                &&  pSelInfo->cbLimit != ~(RTGCPTR)0
-                &&  CPUMIsGuestIn64BitCode(pVCpu, CPUMGetGuestCtxCore(pVCpu)) )
-            {
-                pSelInfo->GCPtrBase = 0;
-                pSelInfo->cbLimit   = ~(RTGCPTR)0;
-            }
-            else if (   Sel == 0
-                     && CPUMIsGuestIn64BitCode(pVCpu, CPUMGetGuestCtxCore(pVCpu)))
-            {
-                pSelInfo->GCPtrBase = 0;
-                pSelInfo->cbLimit   = ~(RTGCPTR)0;
-                pSelInfo->Sel       = 0;
-                pSelInfo->SelGate   = 0;
-                pSelInfo->fFlags    = DBGFSELINFO_FLAGS_LONG_MODE;
-                pSelInfo->u.Raw64.Gen.u1Present  = 1;
-                pSelInfo->u.Raw64.Gen.u1Long     = 1;
-                pSelInfo->u.Raw64.Gen.u1DescType = 1;
-                rc = VINF_SUCCESS;
-            }
-        }
-    }
-    else
-    {
-        if (HWACCMIsEnabled(pVM))
-            rc = VERR_INVALID_STATE;
-        else
-            rc = SELMR3GetShadowSelectorInfo(pVM, Sel, pSelInfo);
-    }
-    return rc;
-}
-
-
-/**
- * Gets information about a selector.
- *
- * Intended for the debugger mostly and will prefer the guest
- * descriptor tables over the shadow ones.
- *
- * @returns VBox status code, the following are the common ones.
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
- *          descriptor table.
- * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
- *          is not returned if the selector itself isn't present, you have to
- *          check that for yourself (see DBGFSELINFO::fFlags).
- * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
- *          pagetable or page backing the selector table wasn't present.
- *
- * @param   pVM         VM handle.
- * @param   idCpu       The ID of the virtual CPU context.
- * @param   Sel         The selector to get info about.
- * @param   fFlags      Flags, see DBGFQSEL_FLAGS_*.
- * @param   pSelInfo    Where to store the information. This will always be
- *                      updated.
- *
- * @remarks This is a wrapper around SELMR3GetSelectorInfo and
- *          SELMR3GetShadowSelectorInfo.
- */
-VMMR3DECL(int) DBGFR3SelQueryInfo(PVM pVM, VMCPUID idCpu, RTSEL Sel, uint32_t fFlags, PDBGFSELINFO pSelInfo)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    AssertReturn(!(fFlags & ~(DBGFSELQI_FLAGS_DT_GUEST | DBGFSELQI_FLAGS_DT_SHADOW | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE)), VERR_INVALID_PARAMETER);
-    AssertReturn(    (fFlags & (DBGFSELQI_FLAGS_DT_SHADOW | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE))
-                  !=           (DBGFSELQI_FLAGS_DT_SHADOW | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE), VERR_INVALID_PARAMETER);
-
-    /* Clear the return data here on this thread. */
-    memset(pSelInfo, 0, sizeof(*pSelInfo));
-
-    /*
-     * Dispatch the request to a worker running on the target CPU.
-     */
-    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3SelQueryInfo, 5, pVM, idCpu, Sel, fFlags, pSelInfo);
-}
-
-
-/**
- * Validates a CS selector.
- *
- * @returns VBox status code.
- * @param   pSelInfo    Pointer to the selector information for the CS selector.
- * @param   SelCPL      The selector defining the CPL (SS).
- */
-VMMDECL(int) DBGFR3SelInfoValidateCS(PCDBGFSELINFO pSelInfo, RTSEL SelCPL)
-{
-    /*
-     * Check if present.
-     */
-    if (pSelInfo->u.Raw.Gen.u1Present)
-    {
-        /*
-         * Type check.
-         */
-        if (    pSelInfo->u.Raw.Gen.u1DescType == 1
-            &&  (pSelInfo->u.Raw.Gen.u4Type & X86_SEL_TYPE_CODE))
-        {
-            /*
-             * Check level.
-             */
-            unsigned uLevel = RT_MAX(SelCPL & X86_SEL_RPL, pSelInfo->Sel & X86_SEL_RPL);
-            if (    !(pSelInfo->u.Raw.Gen.u4Type & X86_SEL_TYPE_CONF)
-                ?   uLevel <= pSelInfo->u.Raw.Gen.u2Dpl
-                :   uLevel >= pSelInfo->u.Raw.Gen.u2Dpl /* hope I got this right now... */
-                    )
-                return VINF_SUCCESS;
-            return VERR_INVALID_RPL;
-        }
-        return VERR_NOT_CODE_SELECTOR;
-    }
-    return VERR_SELECTOR_NOT_PRESENT;
-}
-
-
-/**
- * Converts a PGM paging mode to a set of DBGFPGDMP_XXX flags.
- *
- * @returns Flags. UINT32_MAX if the mode is invalid (asserted).
- * @param   enmMode             The mode.
- */
-static uint32_t dbgfR3PagingDumpModeToFlags(PGMMODE enmMode)
-{
-    switch (enmMode)
-    {
-        case PGMMODE_32_BIT:
-            return DBGFPGDMP_FLAGS_PSE;
-        case PGMMODE_PAE:
-            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE;
-        case PGMMODE_PAE_NX:
-            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_NXE;
-        case PGMMODE_AMD64:
-            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME;
-        case PGMMODE_AMD64_NX:
-            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME | DBGFPGDMP_FLAGS_NXE;
-        case PGMMODE_NESTED:
-            return DBGFPGDMP_FLAGS_NP;
-        case PGMMODE_EPT:
-            return DBGFPGDMP_FLAGS_EPT;
-        default:
-            AssertFailedReturn(UINT32_MAX);
-    }
-}
-
-
-/**
- * EMT worker for DBGFR3PagingDumpEx.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   idCpu           The current CPU ID.
- * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.  Valid.
- * @param   pcr3            The CR3 to use (unless we're getting the current
- *                          state, see @a fFlags).
- * @param   pu64FirstAddr   The first address.
- * @param   pu64LastAddr    The last address.
- * @param   cMaxDepth       The depth.
- * @param   pHlp            The output callbacks.
- */
-static DECLCALLBACK(int) dbgfR3PagingDumpEx(PVM pVM, VMCPUID idCpu, uint32_t fFlags, uint64_t *pcr3,
-                                            uint64_t *pu64FirstAddr, uint64_t *pu64LastAddr,
-                                            uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
-{
-    /*
-     * Implement dumping both context by means of recursion.
-     */
-    if ((fFlags & (DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_SHADOW)) == (DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_SHADOW))
-    {
-        int rc1 = dbgfR3PagingDumpEx(pVM, idCpu, fFlags & ~DBGFPGDMP_FLAGS_GUEST,
-                                     pcr3, pu64FirstAddr, pu64LastAddr, cMaxDepth, pHlp);
-        int rc2 = dbgfR3PagingDumpEx(pVM, idCpu, fFlags & ~DBGFPGDMP_FLAGS_SHADOW,
-                                     pcr3, pu64FirstAddr, pu64LastAddr, cMaxDepth, pHlp);
-        return RT_FAILURE(rc1) ? rc1 : rc2;
-    }
-
-    /*
-     * Get the current CR3/mode if required.
-     */
-    uint64_t cr3 = *pcr3;
-    if (fFlags & (DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE))
-    {
-        PVMCPU pVCpu = &pVM->aCpus[idCpu];
-        if (fFlags & DBGFPGDMP_FLAGS_SHADOW)
-        {
-            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_CR3)
-                cr3 = PGMGetHyperCR3(pVCpu);
-            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE)
-            {
-                fFlags |= dbgfR3PagingDumpModeToFlags(PGMGetShadowMode(pVCpu));
-                if (fFlags & DBGFPGDMP_FLAGS_NP)
-                {
-                    fFlags |= dbgfR3PagingDumpModeToFlags(PGMGetHostMode(pVM));
-                    if (HC_ARCH_BITS == 32 && CPUMIsGuestInLongMode(pVCpu))
-                        fFlags |= DBGFPGDMP_FLAGS_LME;
-                }
-            }
-        }
-        else
-        {
-            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_CR3)
-                cr3 = CPUMGetGuestCR3(pVCpu);
-            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE)
-            {
-                AssertCompile(DBGFPGDMP_FLAGS_PSE == X86_CR4_PSE);      AssertCompile(DBGFPGDMP_FLAGS_PAE == X86_CR4_PAE);
-                fFlags |= CPUMGetGuestCR4(pVCpu)  & (X86_CR4_PSE | X86_CR4_PAE);
-                AssertCompile(DBGFPGDMP_FLAGS_LME == MSR_K6_EFER_LME);  AssertCompile(DBGFPGDMP_FLAGS_NXE == MSR_K6_EFER_NXE);
-                fFlags |= CPUMGetGuestEFER(pVCpu) & (MSR_K6_EFER_LME | MSR_K6_EFER_NXE);
-            }
-        }
-    }
-    fFlags &= ~(DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3);
-
-    /*
-     * Call PGM to do the real work.
-     */
-    int rc;
-    if (fFlags & DBGFPGDMP_FLAGS_SHADOW)
-        rc = PGMR3DumpHierarchyShw(pVM, cr3, fFlags, *pu64FirstAddr, *pu64LastAddr, cMaxDepth, pHlp);
-    else
-        rc = PGMR3DumpHierarchyGst(pVM, cr3, fFlags, *pu64FirstAddr, *pu64LastAddr, cMaxDepth, pHlp);
-    return rc;
-}
-
-
-/**
- * Dump paging structures.
- *
- * This API can be used to dump both guest and shadow structures.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   idCpu           The current CPU ID.
- * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
- * @param   cr3             The CR3 to use (unless we're getting the current
- *                          state, see @a fFlags).
- * @param   u64FirstAddr    The address to start dumping at.
- * @param   u64LastAddr     The address to end dumping after.
- * @param   cMaxDepth       The depth.
- * @param   pHlp            The output callbacks.  Defaults to the debug log if
- *                          NULL.
- */
-VMMDECL(int) DBGFR3PagingDumpEx(PVM pVM, VMCPUID idCpu, uint32_t fFlags, uint64_t cr3, uint64_t u64FirstAddr,
-                                uint64_t u64LastAddr, uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
-{
-    /*
-     * Input validation.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    AssertReturn(!(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(fFlags & (DBGFPGDMP_FLAGS_SHADOW | DBGFPGDMP_FLAGS_GUEST), VERR_INVALID_PARAMETER);
-    AssertReturn((fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE) || !(fFlags & DBGFPGDMP_FLAGS_MODE_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(   !(fFlags & DBGFPGDMP_FLAGS_EPT)
-                 || !(fFlags & (DBGFPGDMP_FLAGS_LME | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_NXE))
-                 , VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pHlp, VERR_INVALID_POINTER);
-    AssertReturn(cMaxDepth, VERR_INVALID_PARAMETER);
-
-    /*
-     * Forward the request to the target CPU.
-     */
-    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3PagingDumpEx, 8,
-                            pVM, idCpu, fFlags, &cr3, &u64FirstAddr, &u64LastAddr, cMaxDepth, pHlp);
-}
-
Index: unk/src/VBox/VMM/DBGFModule.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFModule.cpp	(revision 35332)
+++ 	(revision )
@@ -1,290 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Module & Segment Management.
- */
-
-/*
- * Copyright (C) 2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page pg_dbgf_module    DBGFModule - Module & Segment Management
- *
- * A module is our representation of an executable binary. It's main purpose
- * is to provide segments that can be mapped into address spaces and thereby
- * provide debug info for those parts for the guest code or data.
- *
- * This module will not deal directly with debug info, it will only serve
- * as an interface between the debugger / symbol lookup and the debug info
- * readers.
- *
- * An executable binary doesn't need to have a file, or that is, we don't
- * need the file to create a module for it. There will be interfaces for
- * ROMs to register themselves so we can get to their symbols, and there
- * will be interfaces for the guest OS plugins (@see pg_dbgf_os) to
- * register kernel, drivers and other global modules.
- */
-
-#if 0
-#include <VBox/dbgf.h>
-
-
-/** Special segment number that indicates that the offset is a relative
- * virtual address (RVA). I.e. an offset from the start of the module. */
-#define DBGF_SEG_RVA UINT32_C(0xfffffff0)
-
-/** @defgroup grp_dbgf_dbginfo Debug Info Types
- * @{ */
-/** Other format. */
-#define DBGF_DBGINFO_OTHER      RT_BIT_32(0)
-/** Stabs. */
-#define DBGF_DBGINFO_STABS      RT_BIT_32(1)
-/** Debug With Arbitrary Record Format (DWARF). */
-#define DBGF_DBGINFO_DWARF      RT_BIT_32(2)
-/** Microsoft Codeview debug info. */
-#define DBGF_DBGINFO_CODEVIEW   RT_BIT_32(3)
-/** Watcom debug info. */
-#define DBGF_DBGINFO_WATCOM     RT_BIT_32(4)
-/** IBM High Level Language debug info. */
-#define DBGF_DBGINFO_HLL        RT_BIT_32(5)
-/** Old OS/2 and Windows symbol file. */
-#define DBGF_DBGINFO_SYM        RT_BIT_32(6)
-/** Map file. */
-#define DBGF_DBGINFO_MAP        RT_BIT_32(7)
-/** @} */
-
-/** @defgroup grp_dbgf_exeimg Executable Image Types
- * @{ */
-/** Some other format. */
-#define DBGF_EXEIMG_OTHER       RT_BIT_32(0)
-/** Portable Executable. */
-#define DBGF_EXEIMG_PE          RT_BIT_32(1)
-/** Linear eXecutable. */
-#define DBGF_EXEIMG_LX          RT_BIT_32(2)
-/** Linear Executable. */
-#define DBGF_EXEIMG_LE          RT_BIT_32(3)
-/** New Executable. */
-#define DBGF_EXEIMG_NE          RT_BIT_32(4)
-/** DOS Executable (Mark Zbikowski). */
-#define DBGF_EXEIMG_MZ          RT_BIT_32(5)
-/** COM Executable. */
-#define DBGF_EXEIMG_COM         RT_BIT_32(6)
-/** a.out Executable. */
-#define DBGF_EXEIMG_AOUT        RT_BIT_32(7)
-/** Executable and Linkable Format. */
-#define DBGF_EXEIMG_ELF         RT_BIT_32(8)
-/** Mach-O Executable (including FAT ones). */
-#define DBGF_EXEIMG_MACHO       RT_BIT_32(9)
-/** @} */
-
-/** Pointer to a module. */
-typedef struct DBGFMOD *PDBGFMOD;
-
-
-/**
- * Virtual method table for executable image interpreters.
- */
-typedef struct DBGFMODVTIMG
-{
-    /** Magic number (DBGFMODVTIMG_MAGIC). */
-    uint32_t    u32Magic;
-    /** Mask of supported debug info types, see grp_dbgf_exeimg.
-     * Used to speed up the search for a suitable interpreter. */
-    uint32_t    fSupports;
-    /** The name of the interpreter. */
-    const char *pszName;
-
-    /**
-     * Try open the image.
-     *
-     * This combines probing and opening.
-     *
-     * @returns VBox status code. No informational returns defined.
-     *
-     * @param   pMod        Pointer to the module that is being opened.
-     *
-     *                      The DBGFMOD::pszDbgFile member will point to
-     *                      the filename of any debug info we're aware of
-     *                      on input. Also, or alternatively, it is expected
-     *                      that the interpreter will look for debug info in
-     *                      the executable image file when present and that it
-     *                      may ask the image interpreter for this when it's
-     *                      around.
-     *
-     *                      Upon successful return the method is expected to
-     *                      initialize pDbgOps and pvDbgPriv.
-     */
-    DECLCALLBACKMEMBER(int, pfnTryOpen)(PDBGFMOD pMod);
-
-    /**
-     * Close the interpreter, freeing all associated resources.
-     *
-     * The caller sets the pDbgOps and pvDbgPriv DBGFMOD members
-     * to NULL upon return.
-     *
-     * @param   pMod        Pointer to the module structure.
-     */
-    DECLCALLBACKMEMBER(int, pfnClose)(PDBGFMOD pMod);
-
-} DBGFMODVTIMG
-
-/**
- * Virtual method table for debug info interpreters.
- */
-typedef struct DBGFMODVTDBG
-{
-    /** Magic number (DBGFMODVTDBG_MAGIC). */
-    uint32_t    u32Magic;
-    /** Mask of supported debug info types, see grp_dbgf_dbginfo.
-     * Used to speed up the search for a suitable interpreter. */
-    uint32_t    fSupports;
-    /** The name of the interpreter. */
-    const char *pszName;
-
-    /**
-     * Try open the image.
-     *
-     * This combines probing and opening.
-     *
-     * @returns VBox status code. No informational returns defined.
-     *
-     * @param   pMod        Pointer to the module that is being opened.
-     *
-     *                      The DBGFMOD::pszDbgFile member will point to
-     *                      the filename of any debug info we're aware of
-     *                      on input. Also, or alternatively, it is expected
-     *                      that the interpreter will look for debug info in
-     *                      the executable image file when present and that it
-     *                      may ask the image interpreter for this when it's
-     *                      around.
-     *
-     *                      Upon successful return the method is expected to
-     *                      initialize pDbgOps and pvDbgPriv.
-     */
-    DECLCALLBACKMEMBER(int, pfnTryOpen)(PDBGFMOD pMod);
-
-    /**
-     * Close the interpreter, freeing all associated resources.
-     *
-     * The caller sets the pDbgOps and pvDbgPriv DBGFMOD members
-     * to NULL upon return.
-     *
-     * @param   pMod        Pointer to the module structure.
-     */
-    DECLCALLBACKMEMBER(int, pfnClose)(PDBGFMOD pMod);
-
-    /**
-     * Queries symbol information by symbol name.
-     *
-     * @returns VBox status code.
-     * @retval  VINF_SUCCESS on success, no informational status code.
-     * @retval  VERR_DBGF_NO_SYMBOLS if there aren't any symbols.
-     * @retval  VERR_SYMBOL_NOT_FOUND if no suitable symbol was found.
-     *
-     * @param   pMod        Pointer to the module structure.
-     * @param   pszSymbol   The symbol name.
-     * @para    pSymbol     Where to store the symbol information.
-     */
-    DECLCALLBACKMEMBER(int, pfnSymbolByName)(PDBGFMOD pMod, const char *pszSymbol, PDBGFSYMBOL pSymbol);
-
-    /**
-     * Queries symbol information by address.
-     *
-     * The returned symbol is what the debug info interpreter considers the symbol
-     * most applicable to the specified address. This usually means a symbol with an
-     * address equal or lower than the requested.
-     *
-     * @returns VBox status code.
-     * @retval  VINF_SUCCESS on success, no informational status code.
-     * @retval  VERR_DBGF_NO_SYMBOLS if there aren't any symbols.
-     * @retval  VERR_SYMBOL_NOT_FOUND if no suitable symbol was found.
-     *
-     * @param   pMod        Pointer to the module structure.
-     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
-     * @param   off         The offset into the segment.
-     * @param   poffDisp    Where to store the distance between the specified address
-     *                      and the returned symbol. Optional.
-     * @param   pSymbol     Where to store the symbol information.
-     */
-    DECLCALLBACKMEMBER(int, pfnSymbolByAddr)(PDBGFMOD pMod, uint32_t iSeg, RTGCUINTPTR off, PRTGCINTPTR poffDisp, PDBGFSYMBOL pSymbol);
-
-    /**
-     * Queries line number information by address.
-     *
-     * @returns VBox status code.
-     * @retval  VINF_SUCCESS on success, no informational status code.
-     * @retval  VERR_DBGF_NO_LINE_NUMBERS if there aren't any line numbers.
-     * @retval  VERR_DBGF_LINE_NOT_FOUND if no suitable line number was found.
-     *
-     * @param   pMod        Pointer to the module structure.
-     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
-     * @param   off         The offset into the segment.
-     * @param   poffDisp    Where to store the distance between the specified address
-     *                      and the returned line number. Optional.
-     * @param   pLine       Where to store the information about the closest line number.
-     */
-    DECLCALLBACKMEMBER(int, pfnLineByAddr)(PDBGFMOD pMod, uint32_t iSeg, RTGCUINTPTR off, PRTGCINTPTR poffDisp, PDBGFLINE pLine);
-
-    /**
-     * Adds a symbol to the module (optional).
-     *
-     * This method is used to implement DBGFR3SymbolAdd.
-     *
-     * @returns VBox status code.
-     * @retval  VERR_NOT_SUPPORTED if the interpreter doesn't support this feature.
-     *
-     * @param   pMod        Pointer to the module structure.
-     * @param   pszSymbol   The symbol name.
-     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
-     * @param   off         The offset into the segment.
-     * @param   cbSymbol    The area covered by the symbol. 0 is fine.
-     */
-    DECLCALLBACKMEMBER(int, pfnSymbolAdd)(PDBGFMOD pMod, const char *pszSymbol, uint32_t iSeg, RTGCUINTPTR off, RTUINT cbSymbol);
-
-    /** For catching initialization errors (DBGFMODVTDBG_MAGIC). */
-    uint32_t    u32EndMagic;
-} DBGFMODVTDBG;
-
-#define DBGFMODVTDBG_MAGIC 123
-
-/**
- * Module.
- */
-typedef struct DBGFMOD
-{
-    /** Magic value (DBGFMOD_MAGIC). */
-    uint32_t        u32Magic;
-    /** The number of address spaces this module is currently linked into.
-     * This is used to perform automatic cleanup and sharing. */
-    uint32_t        cLinks;
-    /** The module name (short). */
-    const char     *pszName;
-    /** The module filename. Can be NULL. */
-    const char     *pszImgFile;
-    /** The debug info file (if external). Can be NULL. */
-    const char     *pszDbgFile;
-
-    /** The method table for the executable image interpreter. */
-    PCDBGFMODVTIMG  pImgVt;
-    /** Pointer to the private data of the executable image interpreter. */
-    void           *pvImgPriv;
-
-    /** The method table for the debug info interpreter. */
-    PCDBGFMODVTDBG  pDbgVt;
-    /** Pointer to the private data of the debug info interpreter. */
-    void           *pvDbgPriv;
-
-} DBGFMOD;
-
-#define DBGFMOD_MAGIC   0x12345678
-
-#endif
-
Index: unk/src/VBox/VMM/DBGFOS.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFOS.cpp	(revision 35332)
+++ 	(revision )
@@ -1,447 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Guest OS Diggers.
- */
-
-/*
- * Copyright (C) 2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/mm.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-#include <iprt/param.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-#define DBGF_OS_READ_LOCK(pVM)      do { } while (0)
-#define DBGF_OS_READ_UNLOCK(pVM)    do { } while (0)
-
-#define DBGF_OS_WRITE_LOCK(pVM)     do { } while (0)
-#define DBGF_OS_WRITE_UNLOCK(pVM)   do { } while (0)
-
-
-/**
- * Internal cleanup routine called by DBGFR3Term().
- *
- * @param   pVM     Pointer to the shared VM structure.
- */
-void dbgfR3OSTerm(PVM pVM)
-{
-    /*
-     * Terminate the current one.
-     */
-    if (pVM->dbgf.s.pCurOS)
-    {
-        pVM->dbgf.s.pCurOS->pReg->pfnTerm(pVM, pVM->dbgf.s.pCurOS->abData);
-        pVM->dbgf.s.pCurOS = NULL;
-    }
-
-    /*
-     * Destroy all the instances.
-     */
-    while (pVM->dbgf.s.pOSHead)
-    {
-        PDBGFOS pOS = pVM->dbgf.s.pOSHead;
-        pVM->dbgf.s.pOSHead = pOS->pNext;
-        if (pOS->pReg->pfnDestruct)
-            pOS->pReg->pfnDestruct(pVM, pOS->abData);
-        MMR3HeapFree(pOS);
-    }
-}
-
-
-/**
- * EMT worker function for DBGFR3OSRegister.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- * @param   pReg    The registration structure.
- */
-static DECLCALLBACK(int) dbgfR3OSRegister(PVM pVM, PDBGFOSREG pReg)
-{
-    /* more validations. */
-    DBGF_OS_READ_LOCK(pVM);
-    PDBGFOS pOS;
-    for (pOS = pVM->dbgf.s.pOSHead; pOS; pOS = pOS->pNext)
-        if (!strcmp(pOS->pReg->szName, pReg->szName))
-        {
-            DBGF_OS_READ_UNLOCK(pVM);
-            Log(("dbgfR3OSRegister: %s -> VERR_ALREADY_LOADED\n", pReg->szName));
-            return VERR_ALREADY_LOADED;
-        }
-
-    /*
-     * Allocate a new structure, call the constructor and link it into the list.
-     */
-    pOS = (PDBGFOS)MMR3HeapAllocZ(pVM, MM_TAG_DBGF_OS, RT_OFFSETOF(DBGFOS, abData[pReg->cbData]));
-    AssertReturn(pOS, VERR_NO_MEMORY);
-    pOS->pReg = pReg;
-
-    int rc = pOS->pReg->pfnConstruct(pVM, pOS->abData);
-    if (RT_SUCCESS(rc))
-    {
-        DBGF_OS_WRITE_LOCK(pVM);
-        pOS->pNext = pVM->dbgf.s.pOSHead;
-        pVM->dbgf.s.pOSHead = pOS;
-        DBGF_OS_WRITE_UNLOCK(pVM);
-    }
-    else
-    {
-        if (pOS->pReg->pfnDestruct)
-            pOS->pReg->pfnDestruct(pVM, pOS->abData);
-        MMR3HeapFree(pOS);
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Registers a guest OS digger.
- *
- * This will instantiate an instance of the digger and add it
- * to the list for us in the next call to DBGFR3OSDetect().
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- * @param   pReg    The registration structure.
- * @thread  Any.
- */
-VMMR3DECL(int) DBGFR3OSRegister(PVM pVM, PCDBGFOSREG pReg)
-{
-    /*
-     * Validate intput.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
-    AssertReturn(pReg->u32Magic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
-    AssertReturn(pReg->u32EndMagic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
-    AssertReturn(!pReg->fFlags, VERR_INVALID_PARAMETER);
-    AssertReturn(pReg->cbData < _2G, VERR_INVALID_PARAMETER);
-    AssertReturn(pReg->szName[0], VERR_INVALID_NAME);
-    AssertReturn(RTStrEnd(&pReg->szName[0], sizeof(pReg->szName)), VERR_INVALID_NAME);
-    AssertPtrReturn(pReg->pfnConstruct, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(pReg->pfnDestruct, VERR_INVALID_POINTER);
-    AssertPtrReturn(pReg->pfnProbe, VERR_INVALID_POINTER);
-    AssertPtrReturn(pReg->pfnInit, VERR_INVALID_POINTER);
-    AssertPtrReturn(pReg->pfnRefresh, VERR_INVALID_POINTER);
-    AssertPtrReturn(pReg->pfnTerm, VERR_INVALID_POINTER);
-    AssertPtrReturn(pReg->pfnQueryVersion, VERR_INVALID_POINTER);
-    AssertPtrReturn(pReg->pfnQueryInterface, VERR_INVALID_POINTER);
-
-    /*
-     * Pass it on to EMT(0).
-     */
-    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSRegister, 2, pVM, pReg);
-}
-
-
-/**
- * EMT worker function for DBGFR3OSDeregister.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- * @param   pReg    The registration structure.
- */
-static DECLCALLBACK(int) dbgfR3OSDeregister(PVM pVM, PDBGFOSREG pReg)
-{
-    /*
-     * Unlink it.
-     */
-    bool    fWasCurOS = false;
-    PDBGFOS pOSPrev   = NULL;
-    PDBGFOS pOS;
-    DBGF_OS_WRITE_LOCK(pVM);
-    for (pOS = pVM->dbgf.s.pOSHead; pOS; pOSPrev = pOS, pOS = pOS->pNext)
-        if (pOS->pReg == pReg)
-        {
-            if (pOSPrev)
-                pOSPrev->pNext = pOS->pNext;
-            else
-                pVM->dbgf.s.pOSHead = pOS->pNext;
-            if (pVM->dbgf.s.pCurOS == pOS)
-            {
-                pVM->dbgf.s.pCurOS = NULL;
-                fWasCurOS = true;
-            }
-            break;
-        }
-    DBGF_OS_WRITE_UNLOCK(pVM);
-    if (!pOS)
-    {
-        Log(("DBGFR3OSDeregister: %s -> VERR_NOT_FOUND\n", pReg->szName));
-        return VERR_NOT_FOUND;
-    }
-
-    /*
-     * Terminate it if it was the current OS, then invoke the
-     * destructor and clean up.
-     */
-    if (fWasCurOS)
-        pOS->pReg->pfnTerm(pVM, pOS->abData);
-    if (pOS->pReg->pfnDestruct)
-        pOS->pReg->pfnDestruct(pVM, pOS->abData);
-    MMR3HeapFree(pOS);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Deregisters a guest OS digger previously registered by DBGFR3OSRegister.
- *
- * @returns VBox status code.
- *
- * @param   pVM     Pointer to the shared VM structure.
- * @param   pReg    The registration structure.
- * @thread  Any.
- */
-VMMR3DECL(int)  DBGFR3OSDeregister(PVM pVM, PCDBGFOSREG pReg)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
-    AssertReturn(pReg->u32Magic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
-    AssertReturn(pReg->u32EndMagic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
-    AssertReturn(RTStrEnd(&pReg->szName[0], sizeof(pReg->szName)), VERR_INVALID_NAME);
-
-    DBGF_OS_READ_LOCK(pVM);
-    PDBGFOS pOS;
-    for (pOS = pVM->dbgf.s.pOSHead; pOS; pOS = pOS->pNext)
-        if (pOS->pReg == pReg)
-            break;
-    DBGF_OS_READ_LOCK(pVM);
-
-    if (!pOS)
-    {
-        Log(("DBGFR3OSDeregister: %s -> VERR_NOT_FOUND\n", pReg->szName));
-        return VERR_NOT_FOUND;
-    }
-
-    /*
-     * Pass it on to EMT(0).
-     */
-    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSDeregister, 2, pVM, pReg);
-}
-
-
-/**
- * EMT worker function for DBGFR3OSDetect.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS if successfully detected.
- * @retval  VINF_DBGF_OS_NOT_DETCTED if we cannot figure it out.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pszName     Where to store the OS name. Empty string if not detected.
- * @param   cchName     Size of the buffer.
- */
-static DECLCALLBACK(int) dbgfR3OSDetect(PVM pVM, char *pszName, size_t cchName)
-{
-    /*
-     * Cycle thru the detection routines.
-     */
-    PDBGFOS const pOldOS = pVM->dbgf.s.pCurOS;
-    pVM->dbgf.s.pCurOS = NULL;
-
-    for (PDBGFOS pNewOS = pVM->dbgf.s.pOSHead; pNewOS; pNewOS = pNewOS->pNext)
-        if (pNewOS->pReg->pfnProbe(pVM, pNewOS->abData))
-        {
-            int rc;
-            pVM->dbgf.s.pCurOS = pNewOS;
-            if (pOldOS == pNewOS)
-                rc = pNewOS->pReg->pfnRefresh(pVM, pNewOS->abData);
-            else
-            {
-                if (pOldOS)
-                    pOldOS->pReg->pfnTerm(pVM, pNewOS->abData);
-                rc = pNewOS->pReg->pfnInit(pVM, pNewOS->abData);
-            }
-            if (pszName && cchName)
-                strncat(pszName, pNewOS->pReg->szName, cchName);
-            return rc;
-        }
-
-    /* not found */
-    if (pOldOS)
-        pOldOS->pReg->pfnTerm(pVM, pOldOS->abData);
-    return VINF_DBGF_OS_NOT_DETCTED;
-}
-
-
-/**
- * Detects the guest OS and try dig out symbols and useful stuff.
- *
- * When called the 2nd time, symbols will be updated that if the OS
- * is the same.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS if successfully detected.
- * @retval  VINF_DBGF_OS_NOT_DETCTED if we cannot figure it out.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pszName     Where to store the OS name. Empty string if not detected.
- * @param   cchName     Size of the buffer.
- * @thread  Any.
- */
-VMMR3DECL(int) DBGFR3OSDetect(PVM pVM, char *pszName, size_t cchName)
-{
-    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
-    if (pszName && cchName)
-        *pszName = '\0';
-
-    /*
-     * Pass it on to EMT(0).
-     */
-    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSDetect, 3, pVM, pszName, cchName);
-}
-
-
-/**
- * EMT worker function for DBGFR3OSQueryNameAndVersion
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pszName         Where to store the OS name. Optional.
- * @param   cchName         The size of the name buffer.
- * @param   pszVersion      Where to store the version string. Optional.
- * @param   cchVersion      The size of the version buffer.
- */
-static DECLCALLBACK(int) dbgfR3OSQueryNameAndVersion(PVM pVM, char *pszName, size_t cchName, char *pszVersion, size_t cchVersion)
-{
-    /*
-     * Any known OS?
-     */
-    if (pVM->dbgf.s.pCurOS)
-    {
-        int rc = VINF_SUCCESS;
-        if (pszName && cchName)
-        {
-            size_t cch = strlen(pVM->dbgf.s.pCurOS->pReg->szName);
-            if (cchName > cch)
-                memcpy(pszName, pVM->dbgf.s.pCurOS->pReg->szName, cch + 1);
-            else
-            {
-                memcpy(pszName, pVM->dbgf.s.pCurOS->pReg->szName, cchName - 1);
-                pszName[cchName - 1] = '\0';
-                rc = VINF_BUFFER_OVERFLOW;
-            }
-        }
-
-        if (pszVersion && cchVersion)
-        {
-            int rc2 = pVM->dbgf.s.pCurOS->pReg->pfnQueryVersion(pVM, pVM->dbgf.s.pCurOS->abData, pszVersion, cchVersion);
-            if (RT_FAILURE(rc2) || rc == VINF_SUCCESS)
-                rc = rc2;
-        }
-        return rc;
-    }
-
-    return VERR_DBGF_OS_NOT_DETCTED;
-}
-
-
-/**
- * Queries the name and/or version string for the guest OS.
- *
- * It goes without saying that this querying is done using the current
- * guest OS digger and not additions or user configuration.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pszName         Where to store the OS name. Optional.
- * @param   cchName         The size of the name buffer.
- * @param   pszVersion      Where to store the version string. Optional.
- * @param   cchVersion      The size of the version buffer.
- * @thread  Any.
- */
-VMMR3DECL(int) DBGFR3OSQueryNameAndVersion(PVM pVM, char *pszName, size_t cchName, char *pszVersion, size_t cchVersion)
-{
-    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(pszVersion, VERR_INVALID_POINTER);
-
-    /*
-     * Initialize the output up front.
-     */
-    if (pszName && cchName)
-        *pszName = '\0';
-    if (pszVersion && cchVersion)
-        *pszVersion = '\0';
-
-    /*
-     * Pass it on to EMT(0).
-     */
-    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/,
-                            (PFNRT)dbgfR3OSQueryNameAndVersion, 5, pVM, pszName, cchName, pszVersion, cchVersion);
-}
-
-
-/**
- * EMT worker for DBGFR3OSQueryInterface.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   enmIf       The interface identifier.
- * @param   ppvIf       Where to store the interface pointer on success.
- */
-static DECLCALLBACK(void) dbgfR3OSQueryInterface(PVM pVM, DBGFOSINTERFACE enmIf, void **ppvIf)
-{
-    if (pVM->dbgf.s.pCurOS)
-    {
-        *ppvIf = pVM->dbgf.s.pCurOS->pReg->pfnQueryInterface(pVM, pVM->dbgf.s.pCurOS->abData, enmIf);
-        if (*ppvIf)
-        {
-            /** @todo Create EMT wrapper for the returned interface once we've defined one...
-             * Just keep a list of wrapper together with the OS instance. */
-        }
-    }
-    else
-        *ppvIf = NULL;
-}
-
-
-/**
- * Query an optional digger interface.
- *
- * @returns Pointer to the digger interface on success, NULL if the interfaces isn't
- *          available or no active guest OS digger.
- * @param   pVM         Pointer to the shared VM structure.
- * @param   enmIf       The interface identifier.
- * @thread  Any.
- */
-VMMR3DECL(void *) DBGFR3OSQueryInterface(PVM pVM, DBGFOSINTERFACE enmIf)
-{
-    AssertMsgReturn(enmIf > DBGFOSINTERFACE_INVALID && enmIf < DBGFOSINTERFACE_END, ("%d\n", enmIf), NULL);
-
-    /*
-     * Pass it on to an EMT.
-     */
-    void *pvIf = NULL;
-    VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3OSQueryInterface, 3, pVM, enmIf, &pvIf);
-    return pvIf;
-}
-
Index: unk/src/VBox/VMM/DBGFReg.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFReg.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1198 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Register Methods.
- */
-
-/*
- * Copyright (C) 2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** @name Register and value sizes used by dbgfR3RegQueryWorker and
- *        dbgfR3RegSetWorker.
- * @{ */
-#define R_SZ_8          RT_BIT(0)
-#define R_SZ_16         RT_BIT(1)
-#define R_SZ_32         RT_BIT(2)
-#define R_SZ_64         RT_BIT(3)
-#define R_SZ_64_16      RT_BIT(4)
-#define R_SZ_80         RT_BIT(5)
-#define R_SZ_128        RT_BIT(6)
-#define R_SZ_8_TO_64    (R_SZ_8 | R_SZ_16 | R_SZ_32 | R_SZ_64)
-#define R_SZ_16_TO_64   (R_SZ_16 | R_SZ_32 | R_SZ_64)
-#define R_SZ_32_OR_64   (R_SZ_32 | R_SZ_64)
-/** @}  */
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Register sub-field descriptor.
- */
-typedef struct DBGFREGSUBFIELD
-{
-    /** The name of the sub-field.  NULL is used to terminate the array. */
-    const char     *pszName;
-    /** The index of the first bit. */
-    uint8_t         iFirstBit;
-    /** The number of bits. */
-    uint8_t         cBits;
-    /** The shift count. */
-    int8_t          cShift;
-    /** @todo getter/setter callbacks? */
-} DBGFREGSUBFIELD;
-/** Pointer to a const register sub-field descriptor. */
-typedef DBGFREGSUBFIELD const *PCDBGFREGSUBFIELD;
-
-/**
- * Register alias descriptor.
- */
-typedef struct DBGFREGALIAS
-{
-    /** The alias name.  NULL is used to terminate the array. */
-    const char     *pszName;
-    /** Set to a valid type if the alias has a different type. */
-    DBGFREGVALTYPE  enmType;
-} DBGFREGALIAS;
-/** Pointer to a const register alias descriptor. */
-typedef DBGFREGALIAS const *PCDBGFREGALIAS;
-
-/**
- * Register descriptor.
- */
-typedef struct DBGFREGDESC
-{
-    /** The normal register name. */
-    const char             *pszName;
-    /** The register identifier (same as the table index). */
-    DBGFREG                 enmReg;
-    /** The default register type. */
-    DBGFREGVALTYPE          enmType;
-    /** The offset info the CPUMCTX structure. ~(size_t)0 if not applicable. */
-    size_t                  offCtx;
-    /** Getter (optional). */
-    DECLCALLBACKMEMBER(int, pfnGet)(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-    /** Setter (optional). */
-    DECLCALLBACKMEMBER(int, pfnSet)(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-    /** Aliases (optional). */
-    PCDBGFREGALIAS          paAliases;
-    /** Sub fields (optional). */
-    PCDBGFREGSUBFIELD       paSubFields;
-} DBGFREGDESC;
-/** Pointer to a const register descriptor. */
-typedef DBGFREGDESC const *PCDBGFREGDESC;
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) dbgfR3RegSet_seg(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-static DECLCALLBACK(int) dbgfR3RegGet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-static DECLCALLBACK(int) dbgfR3RegSet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-static DECLCALLBACK(int) dbgfR3RegGet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-static DECLCALLBACK(int) dbgfR3RegSet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-static DECLCALLBACK(int) dbgfR3RegGet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-static DECLCALLBACK(int) dbgfR3RegSet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-static DECLCALLBACK(int) dbgfR3RegGet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-static DECLCALLBACK(int) dbgfR3RegSet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-static DECLCALLBACK(int) dbgfR3RegGet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-static DECLCALLBACK(int) dbgfR3RegSet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-static DECLCALLBACK(int) dbgfR3RegGet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-static DECLCALLBACK(int) dbgfR3RegSet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-static DECLCALLBACK(int) dbgfR3RegGet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
-static DECLCALLBACK(int) dbgfR3RegSet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-/*
- * Set up aliases.
- */
-#define DBGFREGALIAS_STD(Name, psz32, psz16, psz8)  \
-    static DBGFREGALIAS const g_aDbgfRegAliases_##Name[] = \
-    { \
-        { psz32, DBGFREGVALTYPE_U32    }, \
-        { psz16, DBGFREGVALTYPE_U16    }, \
-        { psz8,  DBGFREGVALTYPE_U8     }, \
-        { NULL, DBGFREGVALTYPE_INVALID } \
-    }
-DBGFREGALIAS_STD(rax,  "eax",   "ax",   "al");
-DBGFREGALIAS_STD(rcx,  "ecx",   "cx",   "cl");
-DBGFREGALIAS_STD(rdx,  "edx",   "dx",   "dl");
-DBGFREGALIAS_STD(rbx,  "ebx",   "bx",   "bl");
-DBGFREGALIAS_STD(rsp,  "esp",   "sp",   NULL);
-DBGFREGALIAS_STD(rbp,  "ebp",   "bp",   NULL);
-DBGFREGALIAS_STD(rsi,  "esi",   "si",  "sil");
-DBGFREGALIAS_STD(rdi,  "edi",   "di",  "dil");
-DBGFREGALIAS_STD(r8,   "r8d",  "r8w",  "r8b");
-DBGFREGALIAS_STD(r9,   "r9d",  "r9w",  "r9b");
-DBGFREGALIAS_STD(r10, "r10d", "r10w", "r10b");
-DBGFREGALIAS_STD(r11, "r11d", "r11w", "r11b");
-DBGFREGALIAS_STD(r12, "r12d", "r12w", "r12b");
-DBGFREGALIAS_STD(r13, "r13d", "r13w", "r13b");
-DBGFREGALIAS_STD(r14, "r14d", "r14w", "r14b");
-DBGFREGALIAS_STD(r15, "r15d", "r15w", "r15b");
-DBGFREGALIAS_STD(rip, "eip",   "ip",    NULL);
-DBGFREGALIAS_STD(rflags, "eflags", "flags", NULL);
-#undef DBGFREGALIAS_STD
-
-static DBGFREGALIAS const g_aDbgfRegAliases_fpuip[] =
-{
-    { "fpuip", DBGFREGVALTYPE_U16  },
-    { NULL, DBGFREGVALTYPE_INVALID }
-};
-
-static DBGFREGALIAS const g_aDbgfRegAliases_fpudp[] =
-{
-    { "fpudp", DBGFREGVALTYPE_U16  },
-    { NULL, DBGFREGVALTYPE_INVALID }
-};
-
-static DBGFREGALIAS const g_aDbgfRegAliases_cr0[] =
-{
-    { "msw", DBGFREGVALTYPE_U16  },
-    { NULL, DBGFREGVALTYPE_INVALID }
-};
-
-/*
- * Sub fields.
- */
-/** Sub-fields for the (hidden) segment attribute register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_seg[] =
-{
-    { "type",   0,   4,  0 },
-    { "s",      4,   1,  0 },
-    { "dpl",    5,   2,  0 },
-    { "p",      7,   1,  0 },
-    { "avl",   12,   1,  0 },
-    { "l",     13,   1,  0 },
-    { "d",     14,   1,  0 },
-    { "g",     15,   1,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the flags register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_rflags[] =
-{
-    { "cf",     0,   1,  0 },
-    { "pf",     2,   1,  0 },
-    { "af",     4,   1,  0 },
-    { "zf",     6,   1,  0 },
-    { "sf",     7,   1,  0 },
-    { "tf",     8,   1,  0 },
-    { "if",     9,   1,  0 },
-    { "df",    10,   1,  0 },
-    { "of",    11,   1,  0 },
-    { "iopl",  12,   2,  0 },
-    { "nt",    14,   1,  0 },
-    { "rf",    16,   1,  0 },
-    { "vm",    17,   1,  0 },
-    { "ac",    18,   1,  0 },
-    { "vif",   19,   1,  0 },
-    { "vip",   20,   1,  0 },
-    { "id",    21,   1,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the FPU control word register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_fcw[] =
-{
-    { "im",     1,   1,  0 },
-    { "dm",     2,   1,  0 },
-    { "zm",     3,   1,  0 },
-    { "om",     4,   1,  0 },
-    { "um",     5,   1,  0 },
-    { "pm",     6,   1,  0 },
-    { "pc",     8,   2,  0 },
-    { "rc",    10,   2,  0 },
-    { "x",     12,   1,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the FPU status word register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_fsw[] =
-{
-    { "ie",     0,   1,  0 },
-    { "de",     1,   1,  0 },
-    { "ze",     2,   1,  0 },
-    { "oe",     3,   1,  0 },
-    { "ue",     4,   1,  0 },
-    { "pe",     5,   1,  0 },
-    { "se",     6,   1,  0 },
-    { "es",     7,   1,  0 },
-    { "c0",     8,   1,  0 },
-    { "c1",     9,   1,  0 },
-    { "c2",    10,   1,  0 },
-    { "top",   11,   3,  0 },
-    { "c3",    14,   1,  0 },
-    { "b",     15,   1,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the FPU tag word register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_ftw[] =
-{
-    { "tag0",   0,   2,  0 },
-    { "tag1",   2,   2,  0 },
-    { "tag2",   4,   2,  0 },
-    { "tag3",   6,   2,  0 },
-    { "tag4",   8,   2,  0 },
-    { "tag5",  10,   2,  0 },
-    { "tag6",  12,   2,  0 },
-    { "tag7",  14,   2,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the Multimedia Extensions Control and Status Register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_mxcsr[] =
-{
-    { "ie",     0,   1,  0 },
-    { "de",     1,   1,  0 },
-    { "ze",     2,   1,  0 },
-    { "oe",     3,   1,  0 },
-    { "ue",     4,   1,  0 },
-    { "pe",     5,   1,  0 },
-    { "daz",    6,   1,  0 },
-    { "im",     7,   1,  0 },
-    { "dm",     8,   1,  0 },
-    { "zm",     9,   1,  0 },
-    { "om",    10,   1,  0 },
-    { "um",    11,   1,  0 },
-    { "pm",    12,   1,  0 },
-    { "rc",    13,   2,  0 },
-    { "fz",    14,   1,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the FPU tag word register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_stN[] =
-{
-    { "man",    0,  64,  0 },
-    { "exp",   64,  15,  0 },
-    { "sig",   79,   1,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the MMX registers. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_mmN[] =
-{
-    { "dw0",    0,  32,  0 },
-    { "dw1",   32,  32,  0 },
-    { "w0",     0,  16,  0 },
-    { "w1",    16,  16,  0 },
-    { "w2",    32,  16,  0 },
-    { "w3",    48,  16,  0 },
-    { "b0",     0,   8,  0 },
-    { "b1",     8,   8,  0 },
-    { "b2",    16,   8,  0 },
-    { "b3",    24,   8,  0 },
-    { "b4",    32,   8,  0 },
-    { "b5",    40,   8,  0 },
-    { "b6",    48,   8,  0 },
-    { "b7",    56,   8,  0 },
-    { NULL,     0,   0,  0 }
-};
-
-/** Sub-fields for the XMM registers. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_xmmN[] =
-{
-    { "r0",      0,     32,  0 },
-    { "r0.man",  0+ 0,  23,  0 },
-    { "r0.exp",  0+23,   8,  0 },
-    { "r0.sig",  0+31,   1,  0 },
-    { "r1",     32,     32,  0 },
-    { "r1.man", 32+ 0,  23,  0 },
-    { "r1.exp", 32+23,   8,  0 },
-    { "r1.sig", 32+31,   1,  0 },
-    { "r2",     64,     32,  0 },
-    { "r2.man", 64+ 0,  23,  0 },
-    { "r2.exp", 64+23,   8,  0 },
-    { "r2.sig", 64+31,   1,  0 },
-    { "r3",     96,     32,  0 },
-    { "r3.man", 96+ 0,  23,  0 },
-    { "r3.exp", 96+23,   8,  0 },
-    { "r3.sig", 96+31,   1,  0 },
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the CR0 register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_cr0[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the CR3 register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_cr3[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the CR4 register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_cr4[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the DR6 register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_dr6[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the DR7 register. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_dr7[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the CR_PAT MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_apic_base[] =
-{
-    { "bsp",     8,      1,  0 },
-    { "ge",      9,      1,  0 },
-    { "base",    12,    20, 12 },
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the CR_PAT MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_cr_pat[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the PERF_STATUS MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_perf_status[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the EFER MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_efer[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the STAR MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_star[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the CSTAR MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_cstar[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the LSTAR MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_lstar[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-/** Sub-fields for the SF_MASK MSR. */
-static DBGFREGSUBFIELD const g_aDbgfRegFields_sf_mask[] =
-{
-    /** @todo  */
-    { NULL,      0,      0,  0 }
-};
-
-
-
-/**
- * The register descriptors.
- */
-static DBGFREGDESC const g_aDbgfRegDescs[] =
-{
-#define DBGFREGDESC_REG(UName, LName) \
-    { #LName, DBGFREG_##UName, DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, LName), NULL, NULL, g_aDbgfRegAliases_##LName, NULL }
-    DBGFREGDESC_REG(RAX, rax),
-    DBGFREGDESC_REG(RCX, rcx),
-    DBGFREGDESC_REG(RDX, rdx),
-    DBGFREGDESC_REG(RSP, rsp),
-    DBGFREGDESC_REG(RBP, rbp),
-    DBGFREGDESC_REG(RSI, rsi),
-    DBGFREGDESC_REG(RDI, rdi),
-    DBGFREGDESC_REG(R8,   r8),
-    DBGFREGDESC_REG(R9,   r9),
-    DBGFREGDESC_REG(R10, r10),
-    DBGFREGDESC_REG(R11, r11),
-    DBGFREGDESC_REG(R12, r12),
-    DBGFREGDESC_REG(R13, r13),
-    DBGFREGDESC_REG(R14, r14),
-    DBGFREGDESC_REG(R15, r15),
-#define DBGFREGDESC_SEG(UName, LName) \
-    { #LName,         DBGFREG_##UName,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, LName),               NULL, dbgfR3RegSet_seg, NULL, NULL },  \
-    { #LName "_attr", DBGFREG_##UName##_ATTR, DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, LName##Hid.Attr.u),   NULL, NULL, NULL, g_aDbgfRegFields_seg }, \
-    { #LName "_base", DBGFREG_##UName##_ATTR, DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, LName##Hid.u64Base),  NULL, NULL, NULL, NULL }, \
-    { #LName "_lim",  DBGFREG_##UName##_ATTR, DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, LName##Hid.u32Limit), NULL, NULL, NULL, NULL }
-    DBGFREGDESC_SEG(CS, cs),
-    DBGFREGDESC_SEG(DS, ds),
-    DBGFREGDESC_SEG(ES, es),
-    DBGFREGDESC_SEG(FS, fs),
-    DBGFREGDESC_SEG(GS, gs),
-    DBGFREGDESC_SEG(SS, ss),
-    DBGFREGDESC_REG(RIP, rip),
-    { "rflags",     DBGFREG_RFLAGS,     DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, rflags),           NULL, NULL, g_aDbgfRegAliases_rflags, g_aDbgfRegFields_rflags },
-    { "fcw",        DBGFREG_FCW,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FCW),          NULL, NULL, NULL, g_aDbgfRegFields_fcw },
-    { "fsw",        DBGFREG_FSW,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FSW),          NULL, NULL, NULL, g_aDbgfRegFields_fsw },
-    { "ftw",        DBGFREG_FTW,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FTW), dbgfR3RegGet_ftw, dbgfR3RegSet_ftw, NULL, g_aDbgfRegFields_ftw },
-    { "fop",        DBGFREG_FOP,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FOP),          NULL, NULL, NULL, NULL },
-    { "fpuip",      DBGFREG_FPUIP,      DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.FPUIP),        NULL, NULL, g_aDbgfRegAliases_fpuip, NULL },
-    { "fpucs",      DBGFREG_FPUCS,      DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.CS),           NULL, NULL, NULL, NULL },
-    { "fpudp",      DBGFREG_FPUDP,      DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.FPUDP),        NULL, NULL, g_aDbgfRegAliases_fpudp, NULL },
-    { "fpuds",      DBGFREG_FPUDS,      DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.DS),           NULL, NULL, NULL, NULL },
-    { "mxcsr",      DBGFREG_MXCSR,      DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.MXCSR),        NULL, NULL, NULL, g_aDbgfRegFields_mxcsr },
-    { "mxcsr_mask", DBGFREG_MXCSR_MASK, DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.MXCSR_MASK),   NULL, NULL, NULL, g_aDbgfRegFields_mxcsr },
-#define DBGFREGDESC_ST(n) \
-    { "st" #n,      DBGFREG_ST##n,      DBGFREGVALTYPE_80, ~(size_t)0, dbgfR3RegGet_stN, dbgfR3RegSet_stN, NULL, g_aDbgfRegFields_stN }
-    DBGFREGDESC_ST(0),
-    DBGFREGDESC_ST(1),
-    DBGFREGDESC_ST(2),
-    DBGFREGDESC_ST(3),
-    DBGFREGDESC_ST(4),
-    DBGFREGDESC_ST(5),
-    DBGFREGDESC_ST(6),
-    DBGFREGDESC_ST(7),
-#define DBGFREGDESC_MM(n) \
-    { "mm" #n,      DBGFREG_MM##n,      DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, fpu.aRegs[n].mmx), NULL, NULL, NULL, g_aDbgfRegFields_mmN }
-    DBGFREGDESC_MM(0),
-    DBGFREGDESC_MM(1),
-    DBGFREGDESC_MM(2),
-    DBGFREGDESC_MM(3),
-    DBGFREGDESC_MM(4),
-    DBGFREGDESC_MM(5),
-    DBGFREGDESC_MM(6),
-    DBGFREGDESC_MM(7),
-#define DBGFREGDESC_XMM(n) \
-    { "xmm" #n,     DBGFREG_XMM##n,     DBGFREGVALTYPE_U128, RT_OFFSETOF(CPUMCTX, fpu.aXMM[n].xmm), NULL, NULL, NULL, g_aDbgfRegFields_xmmN }
-    DBGFREGDESC_XMM(0),
-    DBGFREGDESC_XMM(1),
-    DBGFREGDESC_XMM(2),
-    DBGFREGDESC_XMM(3),
-    DBGFREGDESC_XMM(4),
-    DBGFREGDESC_XMM(5),
-    DBGFREGDESC_XMM(6),
-    DBGFREGDESC_XMM(7),
-    DBGFREGDESC_XMM(8),
-    DBGFREGDESC_XMM(9),
-    DBGFREGDESC_XMM(10),
-    DBGFREGDESC_XMM(11),
-    DBGFREGDESC_XMM(12),
-    DBGFREGDESC_XMM(13),
-    DBGFREGDESC_XMM(14),
-    DBGFREGDESC_XMM(15),
-    { "gdtr_base",  DBGFREG_GDTR_BASE,      DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, gdtr.pGdt),           NULL, NULL, NULL, NULL },
-    { "gdtr_limit", DBGFREG_GDTR_LIMIT,     DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, gdtr.cbGdt),          NULL, NULL, NULL, NULL },
-    { "idtr_base",  DBGFREG_IDTR_BASE,      DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, idtr.pIdt),           NULL, NULL, NULL, NULL },
-    { "idtr_limit", DBGFREG_IDTR_LIMIT,     DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, idtr.cbIdt),          NULL, NULL, NULL, NULL },
-    DBGFREGDESC_SEG(LDTR, ldtr),
-    DBGFREGDESC_SEG(TR, tr),
-    { "cr0",        DBGFREG_CR0,       DBGFREGVALTYPE_U32, 0, dbgfR3RegGet_crX, dbgfR3RegSet_crX, g_aDbgfRegAliases_cr0, g_aDbgfRegFields_cr0 },
-    { "cr2",        DBGFREG_CR2,       DBGFREGVALTYPE_U64, 2, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, NULL },
-    { "cr3",        DBGFREG_CR3,       DBGFREGVALTYPE_U64, 3, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, g_aDbgfRegFields_cr3 },
-    { "cr4",        DBGFREG_CR4,       DBGFREGVALTYPE_U32, 4, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, g_aDbgfRegFields_cr4 },
-    { "cr8",        DBGFREG_CR8,       DBGFREGVALTYPE_U32, 8, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, NULL },
-    { "dr0",        DBGFREG_DR0,       DBGFREGVALTYPE_U64, 0, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
-    { "dr1",        DBGFREG_DR1,       DBGFREGVALTYPE_U64, 1, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
-    { "dr2",        DBGFREG_DR2,       DBGFREGVALTYPE_U64, 2, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
-    { "dr3",        DBGFREG_DR3,       DBGFREGVALTYPE_U64, 3, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
-    { "dr6",        DBGFREG_DR6,       DBGFREGVALTYPE_U32, 6, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, g_aDbgfRegFields_dr6 },
-    { "dr7",        DBGFREG_DR7,       DBGFREGVALTYPE_U32, 7, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, g_aDbgfRegFields_dr7 },
-    { "apic_base",    DBGFREG_MSR_IA32_APICBASE,      DBGFREGVALTYPE_U32, MSR_IA32_APICBASE,      dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_apic_base },
-    { "pat",          DBGFREG_MSR_IA32_CR_PAT,        DBGFREGVALTYPE_U64, MSR_IA32_CR_PAT,        dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_cr_pat },
-    { "perf_status",  DBGFREG_MSR_IA32_PERF_STATUS,   DBGFREGVALTYPE_U64, MSR_IA32_PERF_STATUS,   dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_perf_status },
-    { "sysenter_cs",  DBGFREG_MSR_IA32_SYSENTER_CS,   DBGFREGVALTYPE_U16, MSR_IA32_SYSENTER_CS,   dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "sysenter_eip", DBGFREG_MSR_IA32_SYSENTER_EIP,  DBGFREGVALTYPE_U32, MSR_IA32_SYSENTER_EIP,  dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "sysenter_esp", DBGFREG_MSR_IA32_SYSENTER_ESP,  DBGFREGVALTYPE_U32, MSR_IA32_SYSENTER_ESP,  dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "tsc",          DBGFREG_MSR_IA32_TSC,           DBGFREGVALTYPE_U32, MSR_IA32_TSC,           dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "efer",         DBGFREG_MSR_K6_EFER,            DBGFREGVALTYPE_U32, MSR_K6_EFER,            dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_efer },
-    { "star",         DBGFREG_MSR_K6_STAR,            DBGFREGVALTYPE_U64, MSR_K6_STAR,            dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_star },
-    { "cstar",        DBGFREG_MSR_K8_CSTAR,           DBGFREGVALTYPE_U64, MSR_K8_CSTAR,           dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_cstar },
-    { "msr_fs_base",  DBGFREG_MSR_K8_FS_BASE,         DBGFREGVALTYPE_U64, MSR_K8_FS_BASE,         dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "msr_gs_base",  DBGFREG_MSR_K8_GS_BASE,         DBGFREGVALTYPE_U64, MSR_K8_GS_BASE,         dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "krnl_gs_base", DBGFREG_MSR_K8_KERNEL_GS_BASE,  DBGFREGVALTYPE_U64, MSR_K8_KERNEL_GS_BASE,  dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "lstar",        DBGFREG_MSR_K8_LSTAR,           DBGFREGVALTYPE_U64, MSR_K8_LSTAR,           dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_lstar },
-    { "tsc_aux",      DBGFREG_MSR_K8_TSC_AUX,         DBGFREGVALTYPE_U64, MSR_K8_TSC_AUX,         dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
-    { "ah",           DBGFREG_AH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rax) + 1,  NULL, NULL, NULL, NULL },
-    { "ch",           DBGFREG_CH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rcx) + 1,  NULL, NULL, NULL, NULL },
-    { "dh",           DBGFREG_DH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rdx) + 1,  NULL, NULL, NULL, NULL },
-    { "bh",           DBGFREG_BH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rbx) + 1,  NULL, NULL, NULL, NULL },
-    { "gdtr",         DBGFREG_GDTR,     DBGFREGVALTYPE_DTR, ~(size_t)0, dbgfR3RegGet_gdtr, dbgfR3RegSet_gdtr, NULL, NULL },
-    { "idtr",         DBGFREG_IDTR,     DBGFREGVALTYPE_DTR, ~(size_t)0, dbgfR3RegGet_idtr, dbgfR3RegSet_idtr, NULL, NULL },
-#undef DBGFREGDESC_REG
-#undef DBGFREGDESC_SEG
-#undef DBGFREGDESC_ST
-#undef DBGFREGDESC_MM
-#undef DBGFREGDESC_XMM
-};
-
-
-static DECLCALLBACK(int) dbgfR3RegSet_seg(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegGet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegSet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegGet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegSet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegGet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegSet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegGet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegSet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegGet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegSet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegGet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegSet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegGet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-static DECLCALLBACK(int) dbgfR3RegSet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-
-
-/**
- * Wrapper around CPUMQueryGuestMsr for dbgfR3RegQueryWorker.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_DBGF_INVALID_REGISTER
- *
- * @param   pVCpu               The current CPU.
- * @param   pu64                Where to store the register value.
- * @param   pfRegSizes          Where to store the register sizes.
- * @param   idMsr               The MSR to get.
- */
-static uint64_t dbgfR3RegGetMsr(PVMCPU pVCpu, uint64_t *pu64, uint32_t *pfRegSizes, uint32_t idMsr)
-{
-    *pfRegSizes = R_SZ_64;
-    int rc = CPUMQueryGuestMsr(pVCpu, idMsr, pu64);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsg(rc == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", rc));
-        *pu64 = 0;
-    }
-    return VINF_SUCCESS;
-}
-
-/**
- * Worker for DBGFR3RegQueryU8, DBGFR3RegQueryU16, DBGFR3RegQueryU32 and
- * DBGFR3RegQueryU64.
- *
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- * @param   enmReg              The register that's being queried.
- * @param   pu64                Where to store the register value.
- * @param   pfRegSizes          Where to store the register sizes.
- */
-static DECLCALLBACK(int) dbgfR3RegQueryWorker(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint64_t *pu64, uint32_t *pfRegSizes)
-{
-/** @todo rewrite this to use the register descriptors. */
-    PVMCPU    pVCpu = &pVM->aCpus[idCpu];
-    PCCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
-    switch (enmReg)
-    {
-        case DBGFREG_RAX:       *pu64 = pCtx->rax;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_RCX:       *pu64 = pCtx->rcx;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_RDX:       *pu64 = pCtx->rdx;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_RBX:       *pu64 = pCtx->rbx;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_RSP:       *pu64 = pCtx->rsp;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_RBP:       *pu64 = pCtx->rbp;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_RSI:       *pu64 = pCtx->rsi;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_RDI:       *pu64 = pCtx->rdi;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R8:        *pu64 = pCtx->r8;   *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R9:        *pu64 = pCtx->r9;   *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R10:       *pu64 = pCtx->r10;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R11:       *pu64 = pCtx->r11;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R12:       *pu64 = pCtx->r12;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R13:       *pu64 = pCtx->r13;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R14:       *pu64 = pCtx->r14;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-        case DBGFREG_R15:       *pu64 = pCtx->r15;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
-
-        case DBGFREG_CS:        *pu64 = pCtx->cs;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
-        case DBGFREG_DS:        *pu64 = pCtx->ds;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
-        case DBGFREG_ES:        *pu64 = pCtx->es;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
-        case DBGFREG_FS:        *pu64 = pCtx->fs;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
-        case DBGFREG_GS:        *pu64 = pCtx->gs;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
-        case DBGFREG_SS:        *pu64 = pCtx->ss;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
-
-        case DBGFREG_CS_ATTR:   *pu64 = pCtx->csHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
-        case DBGFREG_DS_ATTR:   *pu64 = pCtx->dsHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
-        case DBGFREG_ES_ATTR:   *pu64 = pCtx->esHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
-        case DBGFREG_FS_ATTR:   *pu64 = pCtx->fsHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
-        case DBGFREG_GS_ATTR:   *pu64 = pCtx->gsHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
-        case DBGFREG_SS_ATTR:   *pu64 = pCtx->ssHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
-
-        case DBGFREG_CS_BASE:   *pu64 = pCtx->csHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
-        case DBGFREG_DS_BASE:   *pu64 = pCtx->dsHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
-        case DBGFREG_ES_BASE:   *pu64 = pCtx->esHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
-        case DBGFREG_FS_BASE:   *pu64 = pCtx->fsHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
-        case DBGFREG_GS_BASE:   *pu64 = pCtx->gsHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
-        case DBGFREG_SS_BASE:   *pu64 = pCtx->ssHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
-
-        case DBGFREG_CS_LIMIT:  *pu64 = pCtx->csHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
-        case DBGFREG_DS_LIMIT:  *pu64 = pCtx->dsHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
-        case DBGFREG_ES_LIMIT:  *pu64 = pCtx->esHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
-        case DBGFREG_FS_LIMIT:  *pu64 = pCtx->fsHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
-        case DBGFREG_GS_LIMIT:  *pu64 = pCtx->gsHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
-        case DBGFREG_SS_LIMIT:  *pu64 = pCtx->ssHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
-
-        case DBGFREG_RIP:       *pu64 = pCtx->rip;      *pfRegSizes = R_SZ_16_TO_64;    return VINF_SUCCESS;
-        case DBGFREG_FLAGS:     *pu64 = pCtx->rflags.u; *pfRegSizes = R_SZ_16_TO_64;    return VINF_SUCCESS;
-
-        /** @todo check fxsave/fsave format? */
-        case DBGFREG_FCW:       *pu64 = pCtx->fpu.FCW;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_FSW:       *pu64 = pCtx->fpu.FSW;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_FTW:       *pu64 = pCtx->fpu.FTW;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_FOP:       *pu64 = pCtx->fpu.FOP;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_FPUIP:     *pu64 = pCtx->fpu.FPUIP;    *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_FPUCS:     *pu64 = pCtx->fpu.CS;       *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_FPUDP:     *pu64 = pCtx->fpu.FPUDP;    *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_FPUDS:     *pu64 = pCtx->fpu.DS;       *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_MXCSR:     *pu64 = pCtx->fpu.MXCSR;    *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_MXCSR_MASK:*pu64 = pCtx->fpu.MXCSR_MASK; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
-
-        case DBGFREG_ST0:
-        case DBGFREG_ST1:
-        case DBGFREG_ST2:
-        case DBGFREG_ST3:
-        case DBGFREG_ST4:
-        case DBGFREG_ST5:
-        case DBGFREG_ST6:
-        case DBGFREG_ST7:
-        {
-            /** @todo check tags and fxsave/fsave format. */
-            unsigned iReg = (pCtx->fpu.FSW >> 11) & 7;
-            iReg += enmReg - DBGFREG_ST0;
-            iReg &= 7;
-
-            *pu64 = pCtx->fpu.aRegs[iReg].au64[0];
-            *pfRegSizes = R_SZ_80;
-            return VINF_SUCCESS;
-        }
-
-        case DBGFREG_MM0:
-        case DBGFREG_MM1:
-        case DBGFREG_MM2:
-        case DBGFREG_MM3:
-        case DBGFREG_MM4:
-        case DBGFREG_MM5:
-        case DBGFREG_MM6:
-        case DBGFREG_MM7:
-            *pu64 = pCtx->fpu.aRegs[enmReg - DBGFREG_MM0].mmx;
-            *pfRegSizes = R_SZ_64;
-            return VINF_SUCCESS;
-
-        case DBGFREG_XMM0:
-        case DBGFREG_XMM1:
-        case DBGFREG_XMM2:
-        case DBGFREG_XMM3:
-        case DBGFREG_XMM4:
-        case DBGFREG_XMM5:
-        case DBGFREG_XMM6:
-        case DBGFREG_XMM7:
-        case DBGFREG_XMM8:
-        case DBGFREG_XMM9:
-        case DBGFREG_XMM10:
-        case DBGFREG_XMM11:
-        case DBGFREG_XMM12:
-        case DBGFREG_XMM13:
-        case DBGFREG_XMM14:
-        case DBGFREG_XMM15:
-            *pu64 = pCtx->fpu.aRegs[enmReg - DBGFREG_XMM0].au64[0];
-            *pfRegSizes = R_SZ_128;
-            return VINF_SUCCESS;
-
-        case DBGFREG_GDTR_BASE:     *pu64 = pCtx->gdtr.pGdt;        *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
-        case DBGFREG_GDTR_LIMIT:    *pu64 = pCtx->gdtr.cbGdt;       *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_IDTR_BASE:     *pu64 = pCtx->idtr.pIdt;        *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
-        case DBGFREG_IDTR_LIMIT:    *pu64 = pCtx->idtr.cbIdt;       *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_LDTR:          *pu64 = pCtx->ldtr;             *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_LDTR_ATTR:     *pu64 = pCtx->ldtrHid.Attr.u;   *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_LDTR_BASE:     *pu64 = pCtx->ldtrHid.u64Base;  *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
-        case DBGFREG_LDTR_LIMIT:    *pu64 = pCtx->ldtrHid.u32Limit; *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_TR:            *pu64 = pCtx->tr;               *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
-        case DBGFREG_TR_ATTR:       *pu64 = pCtx->trHid.Attr.u;     *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-        case DBGFREG_TR_BASE:       *pu64 = pCtx->trHid.u64Base;    *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
-        case DBGFREG_TR_LIMIT:      *pu64 = pCtx->trHid.u32Limit;   *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
-
-        case DBGFREG_CR0:           *pu64 = CPUMGetGuestCR0(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_CR2:           *pu64 = CPUMGetGuestCR2(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_CR3:           *pu64 = CPUMGetGuestCR3(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_CR4:           *pu64 = CPUMGetGuestCR4(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_CR8:           *pu64 = CPUMGetGuestCR8(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-
-        case DBGFREG_DR0:           *pu64 = CPUMGetGuestDR0(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_DR1:           *pu64 = CPUMGetGuestDR1(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_DR2:           *pu64 = CPUMGetGuestDR2(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_DR3:           *pu64 = CPUMGetGuestDR3(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_DR6:           *pu64 = CPUMGetGuestDR6(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-        case DBGFREG_DR7:           *pu64 = CPUMGetGuestDR7(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
-
-        case DBGFREG_MSR_IA32_APICBASE:     return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_APICBASE);
-        case DBGFREG_MSR_IA32_CR_PAT:       return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_CR_PAT);
-        case DBGFREG_MSR_IA32_PERF_STATUS:  return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_PERF_STATUS);
-        case DBGFREG_MSR_IA32_SYSENTER_CS:  return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_SYSENTER_CS);
-        case DBGFREG_MSR_IA32_SYSENTER_EIP: return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_SYSENTER_EIP);
-        case DBGFREG_MSR_IA32_SYSENTER_ESP: return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_SYSENTER_ESP);
-        case DBGFREG_MSR_IA32_TSC:          return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_TSC);
-        case DBGFREG_MSR_K6_EFER:           return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K6_EFER);
-        case DBGFREG_MSR_K6_STAR:           return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K6_STAR);
-        case DBGFREG_MSR_K8_CSTAR:          return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_CSTAR);
-        case DBGFREG_MSR_K8_FS_BASE:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_FS_BASE);
-        case DBGFREG_MSR_K8_GS_BASE:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_GS_BASE);
-        case DBGFREG_MSR_K8_KERNEL_GS_BASE: return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_KERNEL_GS_BASE);
-        case DBGFREG_MSR_K8_LSTAR:          return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_LSTAR);
-        case DBGFREG_MSR_K8_SF_MASK:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_SF_MASK);
-        case DBGFREG_MSR_K8_TSC_AUX:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_TSC_AUX);
-
-        case DBGFREG_AH:        *pu64 = RT_BYTE2(pCtx->ax); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
-        case DBGFREG_CH:        *pu64 = RT_BYTE2(pCtx->cx); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
-        case DBGFREG_DH:        *pu64 = RT_BYTE2(pCtx->dx); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
-        case DBGFREG_BH:        *pu64 = RT_BYTE2(pCtx->bx); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
-        case DBGFREG_GDTR:      *pu64 = pCtx->gdtr.pGdt;    *pfRegSizes = R_SZ_64_16;   return VINF_SUCCESS;
-        case DBGFREG_IDTR:      *pu64 = pCtx->idtr.pIdt;    *pfRegSizes = R_SZ_64_16;   return VINF_SUCCESS;
-
-        case DBGFREG_END:
-        case DBGFREG_32BIT_HACK:
-            /* no default! We want GCC warnings. */
-            break;
-    }
-
-    AssertMsgFailed(("%d (%#x)\n", enmReg, enmReg));
-    return VERR_DBGF_INVALID_REGISTER;
-}
-
-
-/**
- * Queries a 8-bit register value.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_VM_HANDLE
- * @retval  VERR_INVALID_CPU_ID
- * @retval  VERR_DBGF_INVALID_REGISTER
- * @retval  VINF_DBGF_TRUNCATED_REGISTER
- *
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- * @param   enmReg              The register that's being queried.
- * @param   pu8                 Where to store the register value.
- */
-VMMR3DECL(int) DBGFR3RegQueryU8(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint8_t *pu8)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-
-    uint64_t u64Value;
-    uint32_t fRegSizes;
-    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
-    if (RT_SUCCESS(rc))
-    {
-        *pu8 = (uint8_t)u64Value;
-        if (R_SZ_8 & fRegSizes)
-            rc = VINF_SUCCESS;
-        else
-            rc = VINF_DBGF_TRUNCATED_REGISTER;
-    }
-    else
-        *pu8 = 0;
-    return rc;
-}
-
-
-/**
- * Queries a 16-bit register value.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_VM_HANDLE
- * @retval  VERR_INVALID_CPU_ID
- * @retval  VERR_DBGF_INVALID_REGISTER
- * @retval  VINF_DBGF_TRUNCATED_REGISTER
- * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
- *
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- * @param   enmReg              The register that's being queried.
- * @param   pu16                Where to store the register value.
- */
-VMMR3DECL(int) DBGFR3RegQueryU16(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint16_t *pu16)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-
-    uint64_t u64Value;
-    uint32_t fRegSizes;
-    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
-    if (RT_SUCCESS(rc))
-    {
-        *pu16 = (uint16_t)u64Value;
-        if (R_SZ_16 & fRegSizes)
-            rc = VINF_SUCCESS;
-        else if (~(R_SZ_8 | R_SZ_16) & fRegSizes)
-            rc = VINF_DBGF_TRUNCATED_REGISTER;
-        else
-            rc = VINF_DBGF_ZERO_EXTENDED_REGISTER;
-    }
-    else
-        *pu16 = 0;
-    return rc;
-}
-
-
-/**
- * Queries a 32-bit register value.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_VM_HANDLE
- * @retval  VERR_INVALID_CPU_ID
- * @retval  VERR_DBGF_INVALID_REGISTER
- * @retval  VINF_DBGF_TRUNCATED_REGISTER
- * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
- *
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- * @param   enmReg              The register that's being queried.
- * @param   pu32                Where to store the register value.
- */
-VMMR3DECL(int) DBGFR3RegQueryU32(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint32_t *pu32)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-
-    uint64_t u64Value;
-    uint32_t fRegSizes;
-    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
-    if (RT_SUCCESS(rc))
-    {
-        *pu32 = (uint32_t)u64Value;
-        if (R_SZ_32 & fRegSizes)
-            rc = VINF_SUCCESS;
-        else if (~(R_SZ_8 | R_SZ_16 | R_SZ_32) & fRegSizes)
-            rc = VINF_DBGF_TRUNCATED_REGISTER;
-        else
-            rc = VINF_DBGF_ZERO_EXTENDED_REGISTER;
-    }
-    else
-        *pu32 = 0;
-    return rc;
-}
-
-
-/**
- * Queries a 64-bit register value.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_VM_HANDLE
- * @retval  VERR_INVALID_CPU_ID
- * @retval  VERR_DBGF_INVALID_REGISTER
- * @retval  VINF_DBGF_TRUNCATED_REGISTER
- * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
- *
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- * @param   enmReg              The register that's being queried.
- * @param   pu64                Where to store the register value.
- */
-VMMR3DECL(int) DBGFR3RegQueryU64(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint64_t *pu64)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-
-    uint64_t u64Value;
-    uint32_t fRegSizes;
-    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
-    if (RT_SUCCESS(rc))
-    {
-        *pu64 = u64Value;
-        if (R_SZ_64 & fRegSizes)
-            rc = VINF_SUCCESS;
-        else if (~(R_SZ_8 | R_SZ_16 | R_SZ_32 | R_SZ_64) & fRegSizes)
-            rc = VINF_DBGF_TRUNCATED_REGISTER;
-        else
-            rc = VINF_DBGF_ZERO_EXTENDED_REGISTER;
-    }
-    else
-        *pu64 = 0;
-    return rc;
-}
-
-
-/**
- * Wrapper around CPUMQueryGuestMsr for dbgfR3RegQueryBatchWorker.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_DBGF_INVALID_REGISTER
- *
- * @param   pVCpu               The current CPU.
- * @param   pReg                The where to store the register value and
- *                              size.
- * @param   idMsr               The MSR to get.
- */
-static void dbgfR3RegGetMsrBatch(PVMCPU pVCpu, PDBGFREGENTRY pReg, uint32_t idMsr)
-{
-    pReg->enmType = DBGFREGVALTYPE_U64;
-    int rc = CPUMQueryGuestMsr(pVCpu, idMsr, &pReg->Val.u64);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsg(rc == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", rc));
-        pReg->Val.u64 = 0;
-    }
-}
-
-
-static DECLCALLBACK(int) dbgfR3RegQueryBatchWorker(PVM pVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
-{
-    PVMCPU    pVCpu = &pVM->aCpus[idCpu];
-    PCCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
-
-    PDBGFREGENTRY pReg = paRegs - 1;
-    while (cRegs-- > 0)
-    {
-        pReg++;
-        pReg->Val.au64[0] = 0;
-        pReg->Val.au64[1] = 0;
-
-        DBGFREG const enmReg = pReg->enmReg;
-        AssertMsgReturn(enmReg >= 0 && enmReg <= DBGFREG_END, ("%d (%#x)\n", enmReg, enmReg), VERR_DBGF_INVALID_REGISTER);
-        if (enmReg != DBGFREG_END)
-        {
-            PCDBGFREGDESC pDesc = &g_aDbgfRegDescs[enmReg];
-            if (!pDesc->pfnGet)
-            {
-                PCRTUINT128U pu = (PCRTUINT128U)((uintptr_t)pCtx + pDesc->offCtx);
-                pReg->enmType = pDesc->enmType;
-                switch (pDesc->enmType)
-                {
-                    case DBGFREGVALTYPE_U8:     pReg->Val.u8   = pu->au8[0];   break;
-                    case DBGFREGVALTYPE_U16:    pReg->Val.u16  = pu->au16[0];  break;
-                    case DBGFREGVALTYPE_U32:    pReg->Val.u32  = pu->au32[0];  break;
-                    case DBGFREGVALTYPE_U64:    pReg->Val.u64  = pu->au64[0];  break;
-                    case DBGFREGVALTYPE_U128:
-                        pReg->Val.au64[0] = pu->au64[0];
-                        pReg->Val.au64[1] = pu->au64[1];
-                        break;
-                    case DBGFREGVALTYPE_LRD:
-                        pReg->Val.au64[0] = pu->au64[0];
-                        pReg->Val.au16[5] = pu->au16[5];
-                        break;
-                    default:
-                        AssertMsgFailedReturn(("%s %d\n", pDesc->pszName, pDesc->enmType), VERR_INTERNAL_ERROR_3);
-                }
-            }
-            else
-            {
-                int rc = pDesc->pfnGet(pVCpu, pDesc, pCtx, &pReg->Val.u);
-                if (RT_FAILURE(rc))
-                    return rc;
-            }
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Query a batch of registers.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_VM_HANDLE
- * @retval  VERR_INVALID_CPU_ID
- * @retval  VERR_DBGF_INVALID_REGISTER
- *
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- * @param   paRegs              Pointer to an array of @a cRegs elements.  On
- *                              input the enmReg members indicates which
- *                              registers to query.  On successful return the
- *                              other members are set.  DBGFREG_END can be used
- *                              as a filler.
- * @param   cRegs               The number of entries in @a paRegs.
- */
-VMMR3DECL(int) DBGFR3RegQueryBatch(PVM pVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    if (!cRegs)
-        return VINF_SUCCESS;
-    AssertReturn(cRegs < _1M, VERR_OUT_OF_RANGE);
-    AssertPtrReturn(paRegs, VERR_INVALID_POINTER);
-    size_t iReg = cRegs;
-    while (iReg-- > 0)
-    {
-        DBGFREG enmReg = paRegs[iReg].enmReg;
-        AssertMsgReturn(enmReg < DBGFREG_END && enmReg >= DBGFREG_AL, ("%d (%#x)", enmReg, enmReg), VERR_DBGF_INVALID_REGISTER);
-    }
-
-    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryBatchWorker, 4, pVM, idCpu, paRegs, cRegs);
-}
-
-
-/**
- * Query a all registers for a Virtual CPU.
- *
- * @retval  VINF_SUCCESS
- * @retval  VERR_INVALID_VM_HANDLE
- * @retval  VERR_INVALID_CPU_ID
- *
- * @param   pVM                 The VM handle.
- * @param   idCpu               The target CPU ID.
- * @param   paRegs              Pointer to an array of @a cRegs elements.
- *                              These will be filled with the CPU register
- *                              values. Overflowing entries will be set to
- *                              DBGFREG_END.  The returned registers can be
- *                              accessed by using the DBGFREG values as index.
- * @param   cRegs               The number of entries in @a paRegs.  The
- *                              recommended value is DBGFREG_ALL_COUNT.
- */
-VMMR3DECL(int) DBGFR3RegQueryAll(PVM pVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    if (!cRegs)
-        return VINF_SUCCESS;
-    AssertReturn(cRegs < _1M, VERR_OUT_OF_RANGE);
-    AssertPtrReturn(paRegs, VERR_INVALID_POINTER);
-
-    /*
-     * Convert it into a batch query (lazy bird).
-     */
-    unsigned iReg = 0;
-    while (iReg < cRegs && iReg < DBGFREG_ALL_COUNT)
-    {
-        paRegs[iReg].enmReg = (DBGFREG)iReg;
-        iReg++;
-    }
-    while (iReg < cRegs)
-        paRegs[iReg++].enmReg = DBGFREG_END;
-
-    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryBatchWorker, 4, pVM, idCpu, paRegs, cRegs);
-}
-
-
-
-/**
- * Gets the name of a register.
- *
- * @returns Pointer to read-only register name (lower case).  NULL if the
- *          parameters are invalid.
- * @param   enmReg              The register identifier.
- * @param   enmType             The register type.  This is for sort out
- *                              aliases.  Pass DBGFREGVALTYPE_INVALID to get
- *                              the standard name.
- */
-VMMR3DECL(const char *) DBGFR3RegName(DBGFREG enmReg, DBGFREGVALTYPE enmType)
-{
-    AssertReturn(enmReg >= DBGFREG_AL && enmReg < DBGFREG_END, NULL);
-    AssertReturn(enmType >= DBGFREGVALTYPE_INVALID && enmType < DBGFREGVALTYPE_END, NULL);
-
-    PCDBGFREGDESC   pDesc  = &g_aDbgfRegDescs[enmReg];
-    PCDBGFREGALIAS  pAlias = pDesc->paAliases;
-    if (   pAlias
-        && pDesc->enmType != enmType
-        && enmType != DBGFREGVALTYPE_INVALID)
-    {
-        while (pAlias->pszName)
-        {
-            if (pAlias->enmType == enmType)
-                return pAlias->pszName;
-            pAlias++;
-        }
-    }
-
-    return pDesc->pszName;
-}
-
-
Index: unk/src/VBox/VMM/DBGFStack.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFStack.cpp	(revision 35332)
+++ 	(revision )
@@ -1,585 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Call Stack Analyser.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#include <VBox/dbgf.h>
-#include <VBox/selm.h>
-#include <VBox/mm.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <iprt/param.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-#include <iprt/alloca.h>
-
-
-
-/**
- * Read stack memory.
- */
-DECLINLINE(int) dbgfR3Read(PVM pVM, VMCPUID idCpu, void *pvBuf, PCDBGFADDRESS pSrcAddr, size_t cb, size_t *pcbRead)
-{
-    int rc = DBGFR3MemRead(pVM, idCpu, pSrcAddr, pvBuf, cb);
-    if (RT_FAILURE(rc))
-    {
-        /* fallback: byte by byte and zero the ones we fail to read. */
-        size_t cbRead;
-        for (cbRead = 0; cbRead < cb; cbRead++)
-        {
-            DBGFADDRESS Addr = *pSrcAddr;
-            rc = DBGFR3MemRead(pVM, idCpu, DBGFR3AddrAdd(&Addr, cbRead), (uint8_t *)pvBuf + cbRead, 1);
-            if (RT_FAILURE(rc))
-                break;
-        }
-        if (cbRead)
-            rc = VINF_SUCCESS;
-        memset((char *)pvBuf + cbRead, 0, cb - cbRead);
-        *pcbRead = cbRead;
-    }
-    else
-        *pcbRead = cb;
-    return rc;
-}
-
-
-/**
- * Internal worker routine.
- *
- * On x86 the typical stack frame layout is like this:
- *     ..  ..
- *     16  parameter 2
- *     12  parameter 1
- *      8  parameter 0
- *      4  return address
- *      0  old ebp; current ebp points here
- *
- * @todo Add AMD64 support (needs teaming up with the module management for
- *       unwind tables).
- */
-static int dbgfR3StackWalk(PVM pVM, VMCPUID idCpu, RTDBGAS hAs, PDBGFSTACKFRAME pFrame)
-{
-    /*
-     * Stop if we got a read error in the previous run.
-     */
-    if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_LAST)
-        return VERR_NO_MORE_FILES;
-
-    /*
-     * Read the raw frame data.
-     */
-    const DBGFADDRESS AddrOldPC = pFrame->AddrPC;
-    const unsigned cbRetAddr = DBGFReturnTypeSize(pFrame->enmReturnType);
-    unsigned cbStackItem;
-    switch (AddrOldPC.fFlags & DBGFADDRESS_FLAGS_TYPE_MASK)
-    {
-        case DBGFADDRESS_FLAGS_FAR16: cbStackItem = 2; break;
-        case DBGFADDRESS_FLAGS_FAR32: cbStackItem = 4; break;
-        case DBGFADDRESS_FLAGS_FAR64: cbStackItem = 8; break;
-        case DBGFADDRESS_FLAGS_RING0: cbStackItem = sizeof(RTHCUINTPTR); break;
-        default:
-            switch (pFrame->enmReturnType)
-            {
-                case DBGFRETURNTYPE_FAR16:
-                case DBGFRETURNTYPE_IRET16:
-                case DBGFRETURNTYPE_IRET32_V86:
-                case DBGFRETURNTYPE_NEAR16: cbStackItem = 2; break;
-
-                case DBGFRETURNTYPE_FAR32:
-                case DBGFRETURNTYPE_IRET32:
-                case DBGFRETURNTYPE_IRET32_PRIV:
-                case DBGFRETURNTYPE_NEAR32: cbStackItem = 4; break;
-
-                case DBGFRETURNTYPE_FAR64:
-                case DBGFRETURNTYPE_IRET64:
-                case DBGFRETURNTYPE_NEAR64: cbStackItem = 8; break;
-
-                default:
-                    AssertMsgFailed(("%d\n", pFrame->enmReturnType));
-                    cbStackItem = 4;
-                    break;
-            }
-    }
-
-    union
-    {
-        uint64_t *pu64;
-        uint32_t *pu32;
-        uint16_t *pu16;
-        uint8_t  *pb;
-        void     *pv;
-    } u, uRet, uArgs, uBp;
-    size_t cbRead = cbRetAddr + cbStackItem + sizeof(pFrame->Args);
-    u.pv = alloca(cbRead);
-    uBp = u;
-    uRet.pb = u.pb + cbStackItem;
-    uArgs.pb = u.pb + cbStackItem + cbRetAddr;
-
-    Assert(DBGFADDRESS_IS_VALID(&pFrame->AddrFrame));
-    int rc = dbgfR3Read(pVM, idCpu, u.pv,
-                        pFrame->fFlags & DBGFSTACKFRAME_FLAGS_ALL_VALID
-                        ? &pFrame->AddrReturnFrame
-                        : &pFrame->AddrFrame,
-                        cbRead, &cbRead);
-    if (    RT_FAILURE(rc)
-        ||  cbRead < cbRetAddr + cbStackItem)
-        pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_LAST;
-
-    /*
-     * The first step is taken in a different way than the others.
-     */
-    if (!(pFrame->fFlags & DBGFSTACKFRAME_FLAGS_ALL_VALID))
-    {
-        pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_ALL_VALID;
-        pFrame->iFrame = 0;
-
-        /* Current PC - set by caller, just find symbol & line. */
-        if (DBGFADDRESS_IS_VALID(&pFrame->AddrPC))
-        {
-            pFrame->pSymPC  = DBGFR3AsSymbolByAddrA(pVM, hAs, &pFrame->AddrPC, NULL /*offDisp*/, NULL /*phMod*/);
-            pFrame->pLinePC = DBGFR3LineByAddrAlloc(pVM, pFrame->AddrPC.FlatPtr, NULL);
-        }
-    }
-    else /* 2nd and subsequent steps */
-    {
-        /* frame, pc and stack is taken from the existing frames return members. */
-        pFrame->AddrFrame = pFrame->AddrReturnFrame;
-        pFrame->AddrPC    = pFrame->AddrReturnPC;
-        pFrame->pSymPC    = pFrame->pSymReturnPC;
-        pFrame->pLinePC   = pFrame->pLineReturnPC;
-
-        /* increment the frame number. */
-        pFrame->iFrame++;
-    }
-
-    /*
-     * Return Frame address.
-     */
-    pFrame->AddrReturnFrame = pFrame->AddrFrame;
-    switch (cbStackItem)
-    {
-        case 2:    pFrame->AddrReturnFrame.off = *uBp.pu16; break;
-        case 4:    pFrame->AddrReturnFrame.off = *uBp.pu32; break;
-        case 8:    pFrame->AddrReturnFrame.off = *uBp.pu64; break;
-        default:    AssertMsgFailed(("cbStackItem=%d\n", cbStackItem)); return VERR_INTERNAL_ERROR;
-    }
-    pFrame->AddrReturnFrame.FlatPtr += pFrame->AddrReturnFrame.off - pFrame->AddrFrame.off;
-
-    /*
-     * Return PC and Stack Addresses.
-     */
-    /** @todo AddrReturnStack is not correct for stdcall and pascal. (requires scope info) */
-    pFrame->AddrReturnStack          = pFrame->AddrFrame;
-    pFrame->AddrReturnStack.off     += cbStackItem + cbRetAddr;
-    pFrame->AddrReturnStack.FlatPtr += cbStackItem + cbRetAddr;
-
-    pFrame->AddrReturnPC             = pFrame->AddrPC;
-    switch (pFrame->enmReturnType)
-    {
-        case DBGFRETURNTYPE_NEAR16:
-            if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
-            {
-                pFrame->AddrReturnPC.FlatPtr += *uRet.pu16 - pFrame->AddrReturnPC.off;
-                pFrame->AddrReturnPC.off      = *uRet.pu16;
-            }
-            else
-                DBGFR3AddrFromFlat(pVM, &pFrame->AddrReturnPC, *uRet.pu16);
-            break;
-        case DBGFRETURNTYPE_NEAR32:
-            if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
-            {
-                pFrame->AddrReturnPC.FlatPtr += *uRet.pu32 - pFrame->AddrReturnPC.off;
-                pFrame->AddrReturnPC.off      = *uRet.pu32;
-            }
-            else
-                DBGFR3AddrFromFlat(pVM, &pFrame->AddrReturnPC, *uRet.pu32);
-            break;
-        case DBGFRETURNTYPE_NEAR64:
-            if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
-            {
-                pFrame->AddrReturnPC.FlatPtr += *uRet.pu64 - pFrame->AddrReturnPC.off;
-                pFrame->AddrReturnPC.off      = *uRet.pu64;
-            }
-            else
-                DBGFR3AddrFromFlat(pVM, &pFrame->AddrReturnPC, *uRet.pu64);
-            break;
-        case DBGFRETURNTYPE_FAR16:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[1], uRet.pu16[0]);
-            break;
-        case DBGFRETURNTYPE_FAR32:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
-            break;
-        case DBGFRETURNTYPE_FAR64:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[4], uRet.pu64[0]);
-            break;
-        case DBGFRETURNTYPE_IRET16:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[1], uRet.pu16[0]);
-            break;
-        case DBGFRETURNTYPE_IRET32:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
-            break;
-        case DBGFRETURNTYPE_IRET32_PRIV:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
-            break;
-        case DBGFRETURNTYPE_IRET32_V86:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
-            break;
-        case DBGFRETURNTYPE_IRET64:
-            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[4], uRet.pu64[0]);
-            break;
-        default:
-            AssertMsgFailed(("enmReturnType=%d\n", pFrame->enmReturnType));
-            return VERR_INVALID_PARAMETER;
-    }
-
-    pFrame->pSymReturnPC  = DBGFR3AsSymbolByAddrA(pVM, hAs, &pFrame->AddrReturnPC, NULL /*offDisp*/, NULL /*phMod*/);
-    pFrame->pLineReturnPC = DBGFR3LineByAddrAlloc(pVM, pFrame->AddrReturnPC.FlatPtr, NULL);
-
-    /*
-     * Frame bitness flag.
-     */
-    switch (cbStackItem)
-    {
-        case 2: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_16BIT; break;
-        case 4: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_32BIT; break;
-        case 8: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_64BIT; break;
-        default:    AssertMsgFailed(("cbStackItem=%d\n", cbStackItem)); return VERR_INTERNAL_ERROR;
-    }
-
-    /*
-     * The arguments.
-     */
-    memcpy(&pFrame->Args, uArgs.pv, sizeof(pFrame->Args));
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Walks the entire stack allocating memory as we walk.
- */
-static DECLCALLBACK(int) dbgfR3StackWalkCtxFull(PVM pVM, VMCPUID idCpu, PCCPUMCTXCORE pCtxCore, RTDBGAS hAs,
-                                                DBGFCODETYPE enmCodeType,
-                                                PCDBGFADDRESS pAddrFrame,
-                                                PCDBGFADDRESS pAddrStack,
-                                                PCDBGFADDRESS pAddrPC,
-                                                DBGFRETURNTYPE enmReturnType,
-                                                PCDBGFSTACKFRAME *ppFirstFrame)
-{
-    /* alloc first frame. */
-    PDBGFSTACKFRAME pCur = (PDBGFSTACKFRAME)MMR3HeapAllocZ(pVM, MM_TAG_DBGF_STACK, sizeof(*pCur));
-    if (!pCur)
-        return VERR_NO_MEMORY;
-
-    /*
-     * Initialize the frame.
-     */
-    pCur->pNextInternal = NULL;
-    pCur->pFirstInternal = pCur;
-
-    int rc = VINF_SUCCESS;
-    if (pAddrPC)
-        pCur->AddrPC = *pAddrPC;
-    else
-        rc = DBGFR3AddrFromSelOff(pVM, idCpu, &pCur->AddrPC, pCtxCore->cs, pCtxCore->rip);
-    if (RT_SUCCESS(rc))
-    {
-        if (enmReturnType == DBGFRETURNTYPE_INVALID)
-            switch (pCur->AddrPC.fFlags & DBGFADDRESS_FLAGS_TYPE_MASK)
-            {
-                case DBGFADDRESS_FLAGS_FAR16: pCur->enmReturnType = DBGFRETURNTYPE_NEAR16; break;
-                case DBGFADDRESS_FLAGS_FAR32: pCur->enmReturnType = DBGFRETURNTYPE_NEAR32; break;
-                case DBGFADDRESS_FLAGS_FAR64: pCur->enmReturnType = DBGFRETURNTYPE_NEAR64; break;
-                case DBGFADDRESS_FLAGS_RING0: pCur->enmReturnType = HC_ARCH_BITS == 64 ? DBGFRETURNTYPE_NEAR64 : DBGFRETURNTYPE_NEAR32; break;
-                default:                      pCur->enmReturnType = DBGFRETURNTYPE_NEAR32; break; /// @todo 64-bit guests
-            }
-
-        uint64_t fAddrMask;
-        if (enmCodeType == DBGFCODETYPE_RING0)
-            fAddrMask = HC_ARCH_BITS == 64 ? UINT64_MAX : UINT32_MAX;
-        else if (enmCodeType == DBGFCODETYPE_HYPER)
-            fAddrMask = UINT32_MAX;
-        else if (DBGFADDRESS_IS_FAR16(&pCur->AddrPC))
-            fAddrMask = UINT16_MAX;
-        else if (DBGFADDRESS_IS_FAR32(&pCur->AddrPC))
-            fAddrMask = UINT32_MAX;
-        else if (DBGFADDRESS_IS_FAR64(&pCur->AddrPC))
-            fAddrMask = UINT64_MAX;
-        else
-        {
-            PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-            CPUMMODE CpuMode = CPUMGetGuestMode(pVCpu);
-            if (CpuMode == CPUMMODE_REAL)
-                fAddrMask = UINT16_MAX;
-            else if (   CpuMode == CPUMMODE_PROTECTED
-                     || !CPUMIsGuestIn64BitCode(pVCpu, pCtxCore))
-                fAddrMask = UINT32_MAX;
-            else
-                fAddrMask = UINT64_MAX;
-        }
-
-        if (pAddrStack)
-            pCur->AddrStack = *pAddrStack;
-        else
-            rc = DBGFR3AddrFromSelOff(pVM, idCpu, &pCur->AddrStack, pCtxCore->ss, pCtxCore->rsp & fAddrMask);
-
-        if (pAddrFrame)
-            pCur->AddrFrame = *pAddrFrame;
-        else if (RT_SUCCESS(rc))
-            rc = DBGFR3AddrFromSelOff(pVM, idCpu, &pCur->AddrFrame, pCtxCore->ss, pCtxCore->rbp & fAddrMask);
-    }
-    else
-        pCur->enmReturnType = enmReturnType;
-
-    /*
-     * The first frame.
-     */
-    if (RT_SUCCESS(rc))
-        rc = dbgfR3StackWalk(pVM, idCpu, hAs, pCur);
-    if (RT_FAILURE(rc))
-    {
-        DBGFR3StackWalkEnd(pCur);
-        return rc;
-    }
-
-    /*
-     * The other frames.
-     */
-    DBGFSTACKFRAME Next = *pCur;
-    while (!(pCur->fFlags & (DBGFSTACKFRAME_FLAGS_LAST | DBGFSTACKFRAME_FLAGS_MAX_DEPTH | DBGFSTACKFRAME_FLAGS_LOOP)))
-    {
-        /* try walk. */
-        rc = dbgfR3StackWalk(pVM, idCpu, hAs, &Next);
-        if (RT_FAILURE(rc))
-            break;
-
-        /* add the next frame to the chain. */
-        PDBGFSTACKFRAME pNext = (PDBGFSTACKFRAME)MMR3HeapAlloc(pVM, MM_TAG_DBGF_STACK, sizeof(*pNext));
-        if (!pNext)
-        {
-            DBGFR3StackWalkEnd(pCur);
-            return VERR_NO_MEMORY;
-        }
-        *pNext = Next;
-        pCur->pNextInternal = pNext;
-        pCur = pNext;
-        Assert(pCur->pNextInternal == NULL);
-
-        /* check for loop */
-        for (PCDBGFSTACKFRAME pLoop = pCur->pFirstInternal;
-             pLoop && pLoop != pCur;
-             pLoop = pLoop->pNextInternal)
-            if (pLoop->AddrFrame.FlatPtr == pCur->AddrFrame.FlatPtr)
-            {
-                pCur->fFlags |= DBGFSTACKFRAME_FLAGS_LOOP;
-                break;
-            }
-
-        /* check for insane recursion */
-        if (pCur->iFrame >= 2048)
-            pCur->fFlags |= DBGFSTACKFRAME_FLAGS_MAX_DEPTH;
-    }
-
-    *ppFirstFrame = pCur->pFirstInternal;
-    return rc;
-}
-
-
-/**
- * Common worker for DBGFR3StackWalkBeginGuestEx, DBGFR3StackWalkBeginHyperEx,
- * DBGFR3StackWalkBeginGuest and DBGFR3StackWalkBeginHyper.
- */
-static int dbgfR3StackWalkBeginCommon(PVM pVM,
-                                      VMCPUID idCpu,
-                                      DBGFCODETYPE enmCodeType,
-                                      PCDBGFADDRESS pAddrFrame,
-                                      PCDBGFADDRESS pAddrStack,
-                                      PCDBGFADDRESS pAddrPC,
-                                      DBGFRETURNTYPE enmReturnType,
-                                      PCDBGFSTACKFRAME *ppFirstFrame)
-{
-    /*
-     * Validate parameters.
-     */
-    *ppFirstFrame = NULL;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-    if (pAddrFrame)
-        AssertReturn(DBGFR3AddrIsValid(pVM, pAddrFrame), VERR_INVALID_PARAMETER);
-    if (pAddrStack)
-        AssertReturn(DBGFR3AddrIsValid(pVM, pAddrStack), VERR_INVALID_PARAMETER);
-    if (pAddrPC)
-        AssertReturn(DBGFR3AddrIsValid(pVM, pAddrPC), VERR_INVALID_PARAMETER);
-    AssertReturn(enmReturnType >= DBGFRETURNTYPE_INVALID && enmReturnType < DBGFRETURNTYPE_END, VERR_INVALID_PARAMETER);
-
-    /*
-     * Get the CPUM context pointer and pass it on the specified EMT.
-     */
-    RTDBGAS         hAs;
-    PCCPUMCTXCORE   pCtxCore;
-    switch (enmCodeType)
-    {
-        case DBGFCODETYPE_GUEST:
-            pCtxCore = CPUMGetGuestCtxCore(VMMGetCpuById(pVM, idCpu));
-            hAs = DBGF_AS_GLOBAL;
-            break;
-        case DBGFCODETYPE_HYPER:
-            pCtxCore = CPUMGetHyperCtxCore(VMMGetCpuById(pVM, idCpu));
-            hAs = DBGF_AS_RC_AND_GC_GLOBAL;
-            break;
-        case DBGFCODETYPE_RING0:
-            pCtxCore = NULL;    /* No valid context present. */
-            hAs = DBGF_AS_R0;
-            break;
-        default:
-            AssertFailedReturn(VERR_INVALID_PARAMETER);
-    }
-    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3StackWalkCtxFull, 10,
-                           pVM, idCpu, pCtxCore, hAs, enmCodeType,
-                           pAddrFrame, pAddrStack, pAddrPC, enmReturnType, ppFirstFrame);
-}
-
-
-/**
- * Begins a guest stack walk, extended version.
- *
- * This will walk the current stack, constructing a list of info frames which is
- * returned to the caller. The caller uses DBGFR3StackWalkNext to traverse the
- * list and DBGFR3StackWalkEnd to release it.
- *
- * @returns VINF_SUCCESS on success.
- * @returns VERR_NO_MEMORY if we're out of memory.
- *
- * @param   pVM             The VM handle.
- * @param   idCpu           The ID of the virtual CPU which stack we want to walk.
- * @param   enmCodeType     Code type
- * @param   pAddrFrame      Frame address to start at. (Optional)
- * @param   pAddrStack      Stack address to start at. (Optional)
- * @param   pAddrPC         Program counter to start at. (Optional)
- * @param   enmReturnType   The return address type. (Optional)
- * @param   ppFirstFrame    Where to return the pointer to the first info frame.
- */
-VMMR3DECL(int) DBGFR3StackWalkBeginEx(PVM pVM,
-                                      VMCPUID idCpu,
-                                      DBGFCODETYPE enmCodeType,
-                                      PCDBGFADDRESS pAddrFrame,
-                                      PCDBGFADDRESS pAddrStack,
-                                      PCDBGFADDRESS pAddrPC,
-                                      DBGFRETURNTYPE enmReturnType,
-                                      PCDBGFSTACKFRAME *ppFirstFrame)
-{
-    return dbgfR3StackWalkBeginCommon(pVM, idCpu, enmCodeType, pAddrFrame, pAddrStack, pAddrPC, enmReturnType, ppFirstFrame);
-}
-
-
-/**
- * Begins a guest stack walk.
- *
- * This will walk the current stack, constructing a list of info frames which is
- * returned to the caller. The caller uses DBGFR3StackWalkNext to traverse the
- * list and DBGFR3StackWalkEnd to release it.
- *
- * @returns VINF_SUCCESS on success.
- * @returns VERR_NO_MEMORY if we're out of memory.
- *
- * @param   pVM             The VM handle.
- * @param   idCpu           The ID of the virtual CPU which stack we want to walk.
- * @param   enmCodeType     Code type
- * @param   ppFirstFrame    Where to return the pointer to the first info frame.
- */
-VMMR3DECL(int) DBGFR3StackWalkBegin(PVM pVM, VMCPUID idCpu, DBGFCODETYPE enmCodeType, PCDBGFSTACKFRAME *ppFirstFrame)
-{
-    return dbgfR3StackWalkBeginCommon(pVM, idCpu, enmCodeType, NULL, NULL, NULL, DBGFRETURNTYPE_INVALID, ppFirstFrame);
-}
-
-/**
- * Gets the next stack frame.
- *
- * @returns Pointer to the info for the next stack frame.
- *          NULL if no more frames.
- *
- * @param   pCurrent    Pointer to the current stack frame.
- *
- */
-VMMR3DECL(PCDBGFSTACKFRAME) DBGFR3StackWalkNext(PCDBGFSTACKFRAME pCurrent)
-{
-    return pCurrent
-         ? pCurrent->pNextInternal
-         : NULL;
-}
-
-
-/**
- * Ends a stack walk process.
- *
- * This *must* be called after a successful first call to any of the stack
- * walker functions. If not called we will leak memory or other resources.
- *
- * @param   pFirstFrame     The frame returned by one of the the begin
- *                          functions.
- */
-VMMR3DECL(void) DBGFR3StackWalkEnd(PCDBGFSTACKFRAME pFirstFrame)
-{
-    if (    !pFirstFrame
-        ||  !pFirstFrame->pFirstInternal)
-        return;
-
-    PDBGFSTACKFRAME pFrame = (PDBGFSTACKFRAME)pFirstFrame->pFirstInternal;
-    while (pFrame)
-    {
-        PDBGFSTACKFRAME pCur = pFrame;
-        pFrame = (PDBGFSTACKFRAME)pCur->pNextInternal;
-        if (pFrame)
-        {
-            if (pCur->pSymReturnPC == pFrame->pSymPC)
-                pFrame->pSymPC = NULL;
-            if (pCur->pSymReturnPC == pFrame->pSymReturnPC)
-                pFrame->pSymReturnPC = NULL;
-
-            if (pCur->pSymPC == pFrame->pSymPC)
-                pFrame->pSymPC = NULL;
-            if (pCur->pSymPC == pFrame->pSymReturnPC)
-                pFrame->pSymReturnPC = NULL;
-
-            if (pCur->pLineReturnPC == pFrame->pLinePC)
-                pFrame->pLinePC = NULL;
-            if (pCur->pLineReturnPC == pFrame->pLineReturnPC)
-                pFrame->pLineReturnPC = NULL;
-
-            if (pCur->pLinePC == pFrame->pLinePC)
-                pFrame->pLinePC = NULL;
-            if (pCur->pLinePC == pFrame->pLineReturnPC)
-                pFrame->pLineReturnPC = NULL;
-        }
-
-        RTDbgSymbolFree(pCur->pSymPC);
-        RTDbgSymbolFree(pCur->pSymReturnPC);
-        DBGFR3LineFree(pCur->pLinePC);
-        DBGFR3LineFree(pCur->pLineReturnPC);
-
-        pCur->pNextInternal = NULL;
-        pCur->pFirstInternal = NULL;
-        pCur->fFlags = 0;
-        MMR3HeapFree(pCur);
-    }
-}
-
Index: unk/src/VBox/VMM/DBGFSym.cpp
===================================================================
--- /trunk/src/VBox/VMM/DBGFSym.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1066 +1,0 @@
-/* $Id$ */
-/** @file
- * DBGF - Debugger Facility, Symbol Management.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_DBGF
-#if defined(RT_OS_WINDOWS) && 0 //defined(DEBUG_bird) // enabled this is you want to debug win32 guests, the hypervisor of EFI.
-# include <Windows.h>
-# define _IMAGEHLP64
-# include <DbgHelp.h>
-# define HAVE_DBGHELP /* if doing guest stuff, this can be nice. */
-#endif
-/** @todo Only use DBGHELP for reading modules since it doesn't do all we want (relocations), or is way to slow in some cases (add symbol)! */
-#include <VBox/dbgf.h>
-#include <VBox/mm.h>
-#include <VBox/pdmapi.h>
-#include "DBGFInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-
-#include <iprt/assert.h>
-#include <iprt/path.h>
-#include <iprt/ctype.h>
-#include <iprt/env.h>
-#include <iprt/param.h>
-#ifndef HAVE_DBGHELP
-# include <iprt/avl.h>
-# include <iprt/string.h>
-#endif
-
-#include <stdio.h> /* for fopen(). */ /** @todo use iprt/stream.h! */
-#include <stdlib.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-#ifdef HAVE_DBGHELP
-static DECLCALLBACK(int) dbgfR3EnumModules(PVM pVM, const char *pszFilename, const char *pszName,
-                                           RTUINTPTR ImageBase, size_t cbImage, bool fRC, void *pvArg);
-static int win32Error(PVM pVM);
-#endif
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-#ifndef HAVE_DBGHELP
-/* later */
-typedef struct DBGFMOD *PDBGFMOD;
-
-/**
- * Internal representation of a symbol.
- */
-typedef struct DBGFSYM
-{
-    /** Node core with the symbol address range. */
-    AVLRGCPTRNODECORE       Core;
-    /** Pointer to the module this symbol is associated with. */
-    PDBGFMOD                pModule;
-    /** Pointer to the next symbol in with this name. */
-    struct DBGFSYM         *pNext;
-    /** Symbol name. */
-    char                    szName[1];
-} DBGFSYM, *PDBGFSYM;
-
-/**
- * Symbol name space node.
- */
-typedef struct DBGFSYMSPACE
-{
-    /** Node core with the symbol name.
-     * (it's allocated in the same block as this struct) */
-    RTSTRSPACECORE          Core;
-    /** Pointer to the first symbol with this name (LIFO). */
-    PDBGFSYM                pSym;
-} DBGFSYMSPACE, *PDBGFSYMSPACE;
-
-#endif
-
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-#ifndef HAVE_DBGHELP
-
-/**
- * Initializes the symbol tree.
- */
-static int dbgfR3SymbolInit(PVM pVM)
-{
-    PDBGFSYM pSym = (PDBGFSYM)MMR3HeapAlloc(pVM, MM_TAG_DBGF_SYMBOL, sizeof(*pSym));
-    if (pSym)
-    {
-        pSym->Core.Key = 0;
-        pSym->Core.KeyLast = ~0;
-        pSym->pModule = NULL;
-        pSym->szName[0] = '\0';
-        if (RTAvlrGCPtrInsert(&pVM->dbgf.s.SymbolTree, &pSym->Core))
-            return VINF_SUCCESS;
-        AssertReleaseMsgFailed(("Failed to insert %RGv-%RGv!\n", pSym->Core.Key, pSym->Core.KeyLast));
-        return VERR_INTERNAL_ERROR;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Insert a record into the symbol tree.
- */
-static int dbgfR3SymbolInsert(PVM pVM, const char *pszName, RTGCPTR Address, size_t cb, PDBGFMOD pModule)
-{
-    /*
-     * Make the address space node.
-     */
-    size_t      cchName = strlen(pszName) + 1;
-    PDBGFSYM pSym = (PDBGFSYM)MMR3HeapAlloc(pVM, MM_TAG_DBGF_SYMBOL, RT_OFFSETOF(DBGFSYM, szName[cchName]));
-    if (pSym)
-    {
-        pSym->Core.Key = Address;
-        pSym->Core.KeyLast = Address + cb;
-        pSym->pModule = pModule;
-        memcpy(pSym->szName, pszName, cchName);
-
-        PDBGFSYM pOld = (PDBGFSYM)RTAvlrGCPtrRangeGet(&pVM->dbgf.s.SymbolTree, (RTGCPTR)Address);
-        if (pOld)
-        {
-            pSym->Core.KeyLast = pOld->Core.KeyLast;
-            if (pOld->Core.Key == pSym->Core.Key)
-            {
-                pOld = (PDBGFSYM)RTAvlrGCPtrRemove(&pVM->dbgf.s.SymbolTree, (RTGCPTR)Address);
-                AssertRelease(pOld);
-                MMR3HeapFree(pOld);
-            }
-            else
-                pOld->Core.KeyLast = Address - 1;
-            if (RTAvlrGCPtrInsert(&pVM->dbgf.s.SymbolTree, &pSym->Core))
-            {
-                /*
-                 * Make the name space node.
-                 */
-                PDBGFSYMSPACE pName = (PDBGFSYMSPACE)RTStrSpaceGet(pVM->dbgf.s.pSymbolSpace, pszName);
-                if (!pName)
-                {
-                    /* make new symbol space node. */
-                    pName = (PDBGFSYMSPACE)MMR3HeapAlloc(pVM, MM_TAG_DBGF_SYMBOL, sizeof(*pName) + cchName);
-                    if (pName)
-                    {
-                        pName->Core.pszString = (char *)memcpy(pName + 1, pszName, cchName);
-                        pName->pSym = pSym;
-                        if (RTStrSpaceInsert(pVM->dbgf.s.pSymbolSpace, &pName->Core))
-                            return VINF_SUCCESS;
-                    }
-                    else
-                        return VINF_SUCCESS;
-                }
-                else
-                {
-                    /* Add to existing symbol name. */
-                    pSym->pNext = pName->pSym;
-                    pName->pSym = pSym;
-                    return VINF_SUCCESS;
-                }
-            }
-            AssertReleaseMsgFailed(("Failed to insert %RGv-%RGv!\n", pSym->Core.Key, pSym->Core.KeyLast));
-        }
-        else
-            AssertMsgFailed(("pOld! %RGv %s\n", pSym->Core.Key, pszName));
-        return VERR_INTERNAL_ERROR;
-
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Get nearest symbol.
- * @returns NULL if no symbol was the for that address.
- */
-static PDBGFSYM dbgfR3SymbolGetAddr(PVM pVM, RTGCPTR Address)
-{
-    PDBGFSYM pSym = (PDBGFSYM)RTAvlrGCPtrRangeGet(&pVM->dbgf.s.SymbolTree, Address);
-    Assert(pSym);
-    if (pSym && pSym->szName[0])
-        return pSym;
-    return NULL;
-}
-
-
-/**
- * Get first symbol.
- * @returns NULL if no symbol by that name.
- */
-static PDBGFSYM dbgfR3SymbolGetName(PVM pVM, const char *pszSymbol)
-{
-    PDBGFSYMSPACE pName = (PDBGFSYMSPACE)RTStrSpaceGet(pVM->dbgf.s.pSymbolSpace, pszSymbol);
-    if (pName)
-        return pName->pSym;
-    return NULL;
-}
-
-#endif
-
-
-/**
- * Strips all kind of spaces from head and tail of a string.
- */
-static char *dbgfR3Strip(char *psz)
-{
-    while (*psz && RT_C_IS_SPACE(*psz))
-        psz++;
-    char *psz2 = strchr(psz, '\0') - 1;
-    while (psz2 >= psz && RT_C_IS_SPACE(*psz2))
-        *psz2-- = '\0';
-    return psz;
-}
-
-
-/**
- * Initialize the debug info for a VM.
- *
- * This will check the CFGM for any symbols or symbol files
- * which needs loading.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-int dbgfR3SymInit(PVM pVM)
-{
-    int rc;
-
-    /*
-     * Initialize the symbol table.
-     */
-    pVM->dbgf.s.pSymbolSpace = (PRTSTRSPACE)MMR3HeapAllocZ(pVM, MM_TAG_DBGF_SYMBOL, sizeof(*pVM->dbgf.s.pSymbolSpace));
-    AssertReturn(pVM->dbgf.s.pSymbolSpace, VERR_NO_MEMORY);
-
-#ifndef HAVE_DBGHELP
-    /* modules & lines later */
-    rc = dbgfR3SymbolInit(pVM);
-    if (RT_FAILURE(rc))
-        return rc;
-    pVM->dbgf.s.fSymInited = true;
-#endif
-
-    /*
-     * Check if there are 'loadsyms' commands in the configuration.
-     */
-    PCFGMNODE pNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/DBGF/loadsyms/");
-    if (pNode)
-    {
-        /*
-         * Enumerate the commands.
-         */
-        for (PCFGMNODE pCmdNode = CFGMR3GetFirstChild(pNode);
-             pCmdNode;
-             pCmdNode = CFGMR3GetNextChild(pCmdNode))
-        {
-            char szCmdName[128];
-            CFGMR3GetName(pCmdNode, &szCmdName[0], sizeof(szCmdName));
-
-            /* File */
-            char *pszFilename;
-            rc = CFGMR3QueryStringAlloc(pCmdNode, "Filename", &pszFilename);
-            AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'File' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
-
-            /* Delta (optional) */
-            RTGCINTPTR offDelta;
-            rc = CFGMR3QueryGCPtrS(pNode, "Delta", &offDelta);
-            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-                offDelta = 0;
-            else
-                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'Delta' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
-
-            /* Module (optional) */
-            char *pszModule;
-            rc = CFGMR3QueryStringAlloc(pCmdNode, "Module", &pszModule);
-            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-                pszModule = NULL;
-            else
-                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'Module' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
-
-            /* Module (optional) */
-            RTGCUINTPTR ModuleAddress;
-            rc = CFGMR3QueryGCPtrU(pNode, "ModuleAddress", &ModuleAddress);
-            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-                ModuleAddress = 0;
-            else
-                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'ModuleAddress' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
-
-            /* Image size (optional) */
-            RTGCUINTPTR cbModule;
-            rc = CFGMR3QueryGCPtrU(pNode, "ModuleSize", &cbModule);
-            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-                cbModule = 0;
-            else
-                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'ModuleAddress' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
-
-
-            /*
-             * Execute the command.
-             */
-            rc = DBGFR3ModuleLoad(pVM, pszFilename, offDelta, pszModule, ModuleAddress, cbModule);
-            AssertMsgRCReturn(rc, ("pszFilename=%s offDelta=%RGv pszModule=%s ModuleAddress=%RGv cbModule=%RGv\n",
-                                   pszFilename, offDelta, pszModule, ModuleAddress, cbModule), rc);
-
-            MMR3HeapFree(pszModule);
-            MMR3HeapFree(pszFilename);
-        }
-    }
-
-    /*
-     * Check if there are any 'symadd' commands in the configuration.
-     */
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * We delay certain
- * Initialize the debug info for a VM.
- */
-int dbgfR3SymLazyInit(PVM pVM)
-{
-    if (pVM->dbgf.s.fSymInited)
-        return VINF_SUCCESS;
-#ifdef HAVE_DBGHELP
-    if (SymInitialize(pVM, NULL, FALSE))
-    {
-        pVM->dbgf.s.fSymInited = true;
-        SymSetOptions(SYMOPT_LOAD_LINES | SYMOPT_AUTO_PUBLICS | SYMOPT_ALLOW_ABSOLUTE_SYMBOLS);
-
-        /*
-         * Enumerate all modules loaded by PDM and add them to the symbol database.
-         */
-        PDMR3LdrEnumModules(pVM, dbgfR3EnumModules, NULL);
-        return VINF_SUCCESS;
-    }
-    return win32Error(pVM);
-#else
-    return VINF_SUCCESS;
-#endif
-}
-
-
-#ifdef HAVE_DBGHELP
-/**
- * Module enumeration callback function.
- *
- * @returns VBox status.
- *          Failure will stop the search and return the return code.
- *          Warnings will be ignored and not returned.
- * @param   pVM             VM Handle.
- * @param   pszFilename     Module filename.
- * @param   pszName         Module name. (short and unique)
- * @param   ImageBase       Address where to executable image is loaded.
- * @param   cbImage         Size of the executable image.
- * @param   fRC             Set if guest context, clear if host context.
- * @param   pvArg           User argument.
- */
-static DECLCALLBACK(int) dbgfR3EnumModules(PVM pVM, const char *pszFilename, const char *pszName,
-                                           RTUINTPTR ImageBase, size_t cbImage, bool fRC, void *pvArg)
-{
-    DWORD64 LoadedImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)pszFilename,
-                                                (char *)(void *)pszName, ImageBase, (DWORD)cbImage);
-    if (!LoadedImageBase)
-        Log(("SymLoadModule64(,,%s,,) -> lasterr=%d\n", pszFilename, GetLastError()));
-    else
-        Log(("Loaded debuginfo for %s - %s %llx\n", pszName, pszFilename, LoadedImageBase));
-
-    return VINF_SUCCESS;
-}
-#endif
-
-
-/**
- * Terminate the debug info repository for the specified VM.
- *
- * @returns VBox status.
- * @param   pVM             VM Handle.
- */
-int dbgfR3SymTerm(PVM pVM)
-{
-#ifdef HAVE_DBGHELP
-    if (pVM->dbgf.s.fSymInited)
-        SymCleanup(pVM);
-    pVM->dbgf.s.fSymInited = false;
-    return VINF_SUCCESS;
-#else
-    pVM->dbgf.s.SymbolTree = 0; /* MM cleans up allocations */
-    pVM->dbgf.s.fSymInited = false;
-    return VINF_SUCCESS;
-#endif
-}
-
-
-/** Symbol file type.. */
-typedef enum SYMFILETYPE
-{
-    SYMFILETYPE_UNKNOWN,
-    SYMFILETYPE_LD_MAP,
-    SYMFILETYPE_MS_MAP,
-    SYMFILETYPE_OBJDUMP,
-    SYMFILETYPE_LINUX_SYSTEM_MAP,
-    SYMFILETYPE_PDB,
-    SYMFILETYPE_DBG,
-    SYMFILETYPE_MZ,
-    SYMFILETYPE_ELF
-} SYMFILETYPE, *PSYMFILETYPE;
-
-
-
-/**
- * Probe the type of a symbol information file.
- *
- * @returns The file type.
- * @param   pFile   File handle.
- */
-SYMFILETYPE dbgfR3ModuleProbe(FILE *pFile)
-{
-    char szHead[4096];
-    size_t cchHead = fread(szHead, 1, sizeof(szHead) - 1, pFile);
-    if (cchHead > 0)
-    {
-        szHead[cchHead] = '\0';
-        if (strstr(szHead, "Preferred load address is"))
-            return SYMFILETYPE_MS_MAP;
-
-        if (    strstr(szHead, "Archive member included because of")
-            ||  strstr(szHead, "Memory Configuration")
-            ||  strstr(szHead, "Linker script and memory map"))
-            return SYMFILETYPE_LD_MAP;
-
-        if (   RT_C_IS_XDIGIT(szHead[0])
-            && RT_C_IS_XDIGIT(szHead[1])
-            && RT_C_IS_XDIGIT(szHead[2])
-            && RT_C_IS_XDIGIT(szHead[3])
-            && RT_C_IS_XDIGIT(szHead[4])
-            && RT_C_IS_XDIGIT(szHead[5])
-            && RT_C_IS_XDIGIT(szHead[6])
-            && RT_C_IS_XDIGIT(szHead[7])
-            && szHead[8] == ' '
-            && RT_C_IS_ALPHA(szHead[9])
-            && szHead[10] == ' '
-            && (RT_C_IS_ALPHA(szHead[11]) || szHead[11] == '_' || szHead[11] == '$')
-            )
-            return SYMFILETYPE_LINUX_SYSTEM_MAP;
-
-        if (   RT_C_IS_XDIGIT(szHead[0])
-            && RT_C_IS_XDIGIT(szHead[1])
-            && RT_C_IS_XDIGIT(szHead[2])
-            && RT_C_IS_XDIGIT(szHead[3])
-            && RT_C_IS_XDIGIT(szHead[4])
-            && RT_C_IS_XDIGIT(szHead[5])
-            && RT_C_IS_XDIGIT(szHead[6])
-            && RT_C_IS_XDIGIT(szHead[7])
-            && RT_C_IS_XDIGIT(szHead[8])
-            && RT_C_IS_XDIGIT(szHead[9])
-            && RT_C_IS_XDIGIT(szHead[10])
-            && RT_C_IS_XDIGIT(szHead[11])
-            && RT_C_IS_XDIGIT(szHead[12])
-            && RT_C_IS_XDIGIT(szHead[13])
-            && RT_C_IS_XDIGIT(szHead[14])
-            && RT_C_IS_XDIGIT(szHead[15])
-            && szHead[16] == ' '
-            && RT_C_IS_ALPHA(szHead[17])
-            && szHead[18] == ' '
-            && (RT_C_IS_ALPHA(szHead[19]) || szHead[19] == '_' || szHead[19] == '$')
-            )
-            return SYMFILETYPE_LINUX_SYSTEM_MAP;
-
-        if (strstr(szHead, "Microsoft C/C++ MSF") == szHead)
-            return SYMFILETYPE_PDB;
-
-        if (strstr(szHead, "ELF") == szHead + 1)
-            return SYMFILETYPE_ELF;
-
-        if (   strstr(szHead, "MZ") == szHead
-            || strstr(szHead, "PE") == szHead
-            || strstr(szHead, "LE") == szHead
-            || strstr(szHead, "LX") == szHead
-            || strstr(szHead, "NE") == szHead)
-            return SYMFILETYPE_MZ;
-
-
-        if (strstr(szHead, "file format"))
-            return SYMFILETYPE_OBJDUMP;
-    }
-
-    return SYMFILETYPE_UNKNOWN;
-}
-
-
-static int dbgfR3LoadLinuxSystemMap(PVM pVM, FILE *pFile, RTGCUINTPTR ModuleAddress, RTGCUINTPTR AddressDelta)
-{
-    char szLine[4096];
-    while (fgets(szLine, sizeof(szLine), pFile))
-    {
-        /* parse the line: <address> <type> <name> */
-        const char *psz = dbgfR3Strip(szLine);
-        char *pszEnd = NULL;
-        uint64_t u64Address;
-        int rc = RTStrToUInt64Ex(psz, &pszEnd, 16, &u64Address);
-        RTGCUINTPTR Address = u64Address;
-        if (    RT_SUCCESS(rc)
-            &&  (*pszEnd == ' ' || *pszEnd == '\t')
-            &&  Address == u64Address
-            &&  u64Address != 0
-            &&  u64Address != (RTGCUINTPTR)~0)
-        {
-            pszEnd++;
-            if (    RT_C_IS_ALPHA(*pszEnd)
-                &&  (pszEnd[1] == ' ' || pszEnd[1] == '\t'))
-            {
-                psz = dbgfR3Strip(pszEnd + 2);
-                if (*psz)
-                {
-                    int rc2 = DBGFR3SymbolAdd(pVM, ModuleAddress, Address + AddressDelta, 0, psz);
-                    if (RT_FAILURE(rc2))
-                        Log2(("DBGFR3SymbolAdd(,, %RGv, 0, '%s') -> %Rrc\n", Address, psz, rc2));
-                }
-            }
-        }
-    }
-    return VINF_SUCCESS;
-}
-
-/**
- * Tries to open the file using the image search paths.
- *
- * This is currently a quick hack and the only way to specifying the path is by setting
- * VBOXDBG_IMAGE_PATH in the environment. It uses semicolon as separator everywhere.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pszFilename     The name of the file to locate and open.
- * @param   pszFound        Where to return the actual filename.
- * @param   cchFound        The buffer size.
- * @param   ppFile          Where to return the opened file.
- */
-int dbgfR3ModuleLocateAndOpen(PVM pVM, const char *pszFilename, char *pszFound, size_t cchFound, FILE **ppFile)
-{
-    /* Check the filename length. */
-    size_t const    cchFilename = strlen(pszFilename);
-    if (cchFilename >= cchFound)
-        return VERR_FILENAME_TOO_LONG;
-    const char     *pszName = RTPathFilename(pszFilename);
-    if (!pszName)
-        return VERR_IS_A_DIRECTORY;
-    size_t const    cchName = strlen(pszName);
-
-    /*
-     * Try default location first.
-     */
-    memcpy(pszFound, pszFilename, cchFilename + 1);
-    FILE *pFile = *ppFile = fopen(pszFound, "rb");
-    if (pFile)
-        return VINF_SUCCESS;
-
-    /*
-     * Walk the search path.
-     */
-    char *pszFreeMe = RTEnvDupEx(RTENV_DEFAULT, "VBOXDBG_IMAGE_PATH");
-    const char *psz = pszFreeMe ? pszFreeMe : ".";
-    while (*psz)
-    {
-        /* Skip leading blanks - no directories with leading spaces, thank you. */
-        while (RT_C_IS_BLANK(*psz))
-            psz++;
-
-        /* Fine the end of this element. */
-        const char *pszNext;
-        const char *pszEnd = strchr(psz, ';');
-        if (!pszEnd)
-            pszEnd = pszNext = strchr(psz, '\0');
-        else
-            pszNext = pszEnd + 1;
-        if (pszEnd != psz)
-        {
-            size_t const cch = pszEnd - psz;
-            if (cch + 1 + cchName < cchFound)
-            {
-                /** @todo RTPathCompose, RTPathComposeN(). This code isn't right
-                 * for 'E:' on DOS systems. It may also create unwanted double slashes. */
-                memcpy(pszFound, psz, cch);
-                pszFound[cch] = '/';
-                memcpy(pszFound + cch + 1, pszName, cchName + 1);
-                *ppFile = pFile = fopen(pszFound, "rb");
-                if (pFile)
-                {
-                    RTStrFree(pszFreeMe);
-                    return VINF_SUCCESS;
-                }
-            }
-
-            /** @todo do a depth search using the specified path. */
-        }
-
-        /* advance */
-        psz = pszNext;
-    }
-
-    /* not found */
-    RTStrFree(pszFreeMe);
-    return VERR_OPEN_FAILED;
-}
-
-
-/**
- * Load debug info, optionally related to a specific module.
- *
- * @returns VBox status.
- * @param   pVM             VM Handle.
- * @param   pszFilename     Path to the file containing the symbol information.
- *                          This can be the executable image, a flat symbol file of some kind or stripped debug info.
- * @param   AddressDelta    The value to add to the loaded symbols.
- * @param   pszName         Short hand name for the module. If not related to a module specify NULL.
- * @param   ModuleAddress   Address which the image is loaded at. This will be used to reference the module other places in the api.
- *                          Ignored when pszName is NULL.
- * @param   cbImage         Size of the image.
- *                          Ignored when pszName is NULL.
- */
-VMMR3DECL(int) DBGFR3ModuleLoad(PVM pVM, const char *pszFilename, RTGCUINTPTR AddressDelta, const char *pszName, RTGCUINTPTR ModuleAddress, unsigned cbImage)
-{
-    /*
-     * Lazy init.
-     */
-    if (!pVM->dbgf.s.fSymInited)
-    {
-        int rc = dbgfR3SymLazyInit(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Open the load file.
-     */
-    FILE *pFile = NULL;
-    char szFoundFile[RTPATH_MAX];
-    int rc = dbgfR3ModuleLocateAndOpen(pVM, pszFilename, szFoundFile, sizeof(szFoundFile), &pFile);
-    if (pFile)
-    {
-        /*
-         * Probe the file type.
-         */
-        SYMFILETYPE enmType = dbgfR3ModuleProbe(pFile);
-        if (enmType != SYMFILETYPE_UNKNOWN)
-        {
-            /*
-             * Add the module.
-             */
-            if (pszName)
-            {
-                #ifdef HAVE_DBGHELP
-                /** @todo arg! checkout the inserting of modules and then loading them again.... Or just the module representation.... */
-                DWORD64 ImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)szFoundFile, (char *)(void *)pszName, ModuleAddress, cbImage);
-                if (!ImageBase)
-                    ImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)pszName, (char *)(void *)pszName, ModuleAddress, cbImage);
-                if (ImageBase)
-                {
-                    AssertMsg(ModuleAddress == 0 || ModuleAddress == ImageBase, ("ModuleAddres=%RGv ImageBase=%llx\n", ModuleAddress, ImageBase));
-                    ModuleAddress = ImageBase;
-                }
-                else
-                    rc = win32Error(pVM);
-                #else
-                rc = VERR_NOT_IMPLEMENTED;
-                #endif
-            }
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Seek to the start of the file.
-                 */
-                rc = fseek(pFile, 0, SEEK_SET);
-                Assert(!rc);
-
-                /*
-                 * Process the specific.
-                 */
-                switch (enmType)
-                {
-                    case SYMFILETYPE_LINUX_SYSTEM_MAP:
-                        rc = dbgfR3LoadLinuxSystemMap(pVM, pFile, ModuleAddress, AddressDelta);
-                        break;
-
-                    case SYMFILETYPE_PDB:
-                    case SYMFILETYPE_DBG:
-                    case SYMFILETYPE_MZ:
-                #ifdef HAVE_DBGHELP
-                        /* done it all above! */
-                        break;
-                #endif
-                    case SYMFILETYPE_LD_MAP:
-                    case SYMFILETYPE_MS_MAP:
-                    case SYMFILETYPE_OBJDUMP:
-                    case SYMFILETYPE_ELF:
-                        rc = VERR_NOT_SUPPORTED;
-                        break;
-
-                    default:
-                        AssertFailed();
-                        rc = VERR_INTERNAL_ERROR;
-                        break;
-                } /* file switch. */
-            } /* module added successfully. */
-        } /* format identified */
-        else
-            rc = VERR_NOT_SUPPORTED;
-        /** @todo check for read errors */
-        fclose(pFile);
-    }
-    return rc;
-}
-
-
-/**
- * Interface used by PDMR3LdrRelocate for telling us that a GC module has been relocated.
- *
- * @param   pVM             The VM handle.
- * @param   OldImageBase    The old image base.
- * @param   NewImageBase    The new image base.
- * @param   cbImage         The image size.
- * @param   pszFilename     The image filename.
- * @param   pszName         The module name.
- */
-VMMR3DECL(void) DBGFR3ModuleRelocate(PVM pVM, RTGCUINTPTR OldImageBase, RTGCUINTPTR NewImageBase, RTGCUINTPTR cbImage,
-                                      const char *pszFilename, const char *pszName)
-{
-#ifdef HAVE_DBGHELP
-    if (pVM->dbgf.s.fSymInited)
-    {
-        if (!SymUnloadModule64(pVM, OldImageBase))
-            Log(("SymUnloadModule64(,%RGv) failed, lasterr=%d\n", OldImageBase, GetLastError()));
-
-        DWORD ImageSize = (DWORD)cbImage; Assert(ImageSize == cbImage);
-        DWORD64 LoadedImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)pszFilename, (char *)(void *)pszName, NewImageBase, ImageSize);
-        if (!LoadedImageBase)
-            Log(("SymLoadModule64(,,%s,,) -> lasterr=%d (relocate)\n", pszFilename, GetLastError()));
-        else
-            Log(("Reloaded debuginfo for %s - %s %llx\n", pszName, pszFilename, LoadedImageBase));
-    }
-#else
-
-#endif
-}
-
-
-/**
- * Adds a symbol to the debug info manager.
- *
- * @returns VBox status.
- * @param   pVM             VM Handle.
- * @param   ModuleAddress   Module address. Use 0 if no module.
- * @param   SymbolAddress   Symbol address
- * @param   cbSymbol        Size of the symbol. Use 0 if info not available.
- * @param   pszSymbol       Symbol name.
- */
-VMMR3DECL(int) DBGFR3SymbolAdd(PVM pVM, RTGCUINTPTR ModuleAddress, RTGCUINTPTR SymbolAddress, RTUINT cbSymbol, const char *pszSymbol)
-{
-    /*
-     * Validate.
-     */
-    if (!pszSymbol || !*pszSymbol)
-    {
-        AssertMsgFailed(("No symbol name!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Lazy init.
-     */
-    if (!pVM->dbgf.s.fSymInited)
-    {
-        int rc = dbgfR3SymLazyInit(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-#ifdef HAVE_DBGHELP
-    if (SymAddSymbol(pVM, ModuleAddress, (char *)(void *)pszSymbol, SymbolAddress, cbSymbol, 0))
-        return VINF_SUCCESS;
-    return win32Error(pVM);
-#else
-    /** @todo module lookup. */
-    return dbgfR3SymbolInsert(pVM, pszSymbol, SymbolAddress, cbSymbol, NULL);
-#endif
-}
-
-
-/**
- * Find symbol by address (nearest).
- *
- * @returns VBox status.
- * @param   pVM                 VM handle.
- * @param   Address             Address.
- * @param   poffDisplacement    Where to store the symbol displacement from Address.
- * @param   pSymbol             Where to store the symbol info.
- */
-VMMR3DECL(int) DBGFR3SymbolByAddr(PVM pVM, RTGCUINTPTR Address, PRTGCINTPTR poffDisplacement, PDBGFSYMBOL pSymbol)
-{
-    /*
-     * Lazy init.
-     */
-    if (!pVM->dbgf.s.fSymInited)
-    {
-        int rc = dbgfR3SymLazyInit(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Look it up.
-     */
-#ifdef HAVE_DBGHELP
-    char                achBuffer[sizeof(IMAGEHLP_SYMBOL64) + DBGF_SYMBOL_NAME_LENGTH * sizeof(TCHAR) + sizeof(ULONG64)];
-    PIMAGEHLP_SYMBOL64  pSym = (PIMAGEHLP_SYMBOL64)&achBuffer[0];
-    pSym->SizeOfStruct      = sizeof(IMAGEHLP_SYMBOL64);
-    pSym->MaxNameLength     = DBGF_SYMBOL_NAME_LENGTH;
-
-    if (SymGetSymFromAddr64(pVM, Address, (PDWORD64)poffDisplacement, pSym))
-    {
-        pSymbol->Value  = (RTGCUINTPTR)pSym->Address;
-        pSymbol->cb     = pSym->Size;
-        pSymbol->fFlags = pSym->Flags;
-        strcpy(pSymbol->szName, pSym->Name);
-        return VINF_SUCCESS;
-    }
-    //return win32Error(pVM);
-
-#else
-
-    PDBGFSYM pSym = dbgfR3SymbolGetAddr(pVM, Address);
-    if (pSym)
-    {
-        pSymbol->Value = pSym->Core.Key;
-        pSymbol->cb = pSym->Core.KeyLast - pSym->Core.Key + 1;
-        pSymbol->fFlags = 0;
-        pSymbol->szName[0] = '\0';
-        strncat(pSymbol->szName, pSym->szName,  sizeof(pSymbol->szName) - 1);
-        if (poffDisplacement)
-            *poffDisplacement = Address - pSymbol->Value;
-        return VINF_SUCCESS;
-    }
-
-#endif
-
-    /*
-     * Try PDM.
-     */
-    if (MMHyperIsInsideArea(pVM, Address))
-    {
-        char        szModName[64];
-        RTRCPTR     RCPtrMod;
-        char        szNearSym1[260];
-        RTRCPTR     RCPtrNearSym1;
-        char        szNearSym2[260];
-        RTRCPTR     RCPtrNearSym2;
-        int rc = PDMR3LdrQueryRCModFromPC(pVM, Address,
-                                          &szModName[0],  sizeof(szModName),  &RCPtrMod,
-                                          &szNearSym1[0], sizeof(szNearSym1), &RCPtrNearSym1,
-                                          &szNearSym2[0], sizeof(szNearSym2), &RCPtrNearSym2);
-        if (RT_SUCCESS(rc) && szNearSym1[0])
-        {
-            pSymbol->Value = RCPtrNearSym1;
-            pSymbol->cb = RCPtrNearSym2 > RCPtrNearSym1 ? RCPtrNearSym2 - RCPtrNearSym1 : 0;
-            pSymbol->fFlags = 0;
-            pSymbol->szName[0] = '\0';
-            strncat(pSymbol->szName, szNearSym1,  sizeof(pSymbol->szName) - 1);
-            if (poffDisplacement)
-                *poffDisplacement = Address - pSymbol->Value;
-            return VINF_SUCCESS;
-        }
-    }
-
-    return VERR_SYMBOL_NOT_FOUND;
-}
-
-
-/**
- * Find symbol by name (first).
- *
- * @returns VBox status.
- * @param   pVM                 VM handle.
- * @param   pszSymbol           Symbol name.
- * @param   pSymbol             Where to store the symbol info.
- */
-VMMR3DECL(int) DBGFR3SymbolByName(PVM pVM, const char *pszSymbol, PDBGFSYMBOL pSymbol)
-{
-    /*
-     * Lazy init.
-     */
-    if (!pVM->dbgf.s.fSymInited)
-    {
-        int rc = dbgfR3SymLazyInit(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Look it up.
-     */
-#ifdef HAVE_DBGHELP
-    char            achBuffer[sizeof(IMAGEHLP_SYMBOL64) + DBGF_SYMBOL_NAME_LENGTH * sizeof(TCHAR) + sizeof(ULONG64)];
-    PIMAGEHLP_SYMBOL64 pSym = (PIMAGEHLP_SYMBOL64)&achBuffer[0];
-    pSym->SizeOfStruct      = sizeof(IMAGEHLP_SYMBOL64);
-    pSym->MaxNameLength     = DBGF_SYMBOL_NAME_LENGTH;
-
-    if (SymGetSymFromName64(pVM, (char *)(void *)pszSymbol, pSym))
-    {
-        pSymbol->Value  = (RTGCUINTPTR)pSym->Address;
-        pSymbol->cb     = pSym->Size;
-        pSymbol->fFlags = pSym->Flags;
-        strcpy(pSymbol->szName, pSym->Name);
-        return VINF_SUCCESS;
-    }
-    return win32Error(pVM);
-#else
-
-    PDBGFSYM pSym = dbgfR3SymbolGetName(pVM, pszSymbol);
-    if (pSym)
-    {
-        pSymbol->Value = pSym->Core.Key;
-        pSymbol->cb = pSym->Core.KeyLast - pSym->Core.Key + 1;
-        pSymbol->fFlags = 0;
-        pSymbol->szName[0] = '\0';
-        strncat(pSymbol->szName, pSym->szName, sizeof(pSymbol->szName) - 1);
-        return VINF_SUCCESS;
-    }
-
-    return VERR_SYMBOL_NOT_FOUND;
-#endif
-}
-
-
-/**
- * Find line by address (nearest).
- *
- * @returns VBox status.
- * @param   pVM                 VM handle.
- * @param   Address             Address.
- * @param   poffDisplacement    Where to store the line displacement from Address.
- * @param   pLine               Where to store the line info.
- */
-VMMR3DECL(int) DBGFR3LineByAddr(PVM pVM, RTGCUINTPTR Address, PRTGCINTPTR poffDisplacement, PDBGFLINE pLine)
-{
-    /*
-     * Lazy init.
-     */
-    if (!pVM->dbgf.s.fSymInited)
-    {
-        int rc = dbgfR3SymLazyInit(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Look it up.
-     */
-#ifdef HAVE_DBGHELP
-    IMAGEHLP_LINE64     Line = {0};
-    DWORD               off = 0;
-    Line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
-    if (SymGetLineFromAddr64(pVM, Address, &off, &Line))
-    {
-        if (poffDisplacement)
-            *poffDisplacement = (long)off;
-        pLine->Address      = (RTGCUINTPTR)Line.Address;
-        pLine->uLineNo      = Line.LineNumber;
-        pLine->szFilename[0] = '\0';
-        strncat(pLine->szFilename, Line.FileName, sizeof(pLine->szFilename));
-        return VINF_SUCCESS;
-    }
-    return win32Error(pVM);
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif
-}
-
-
-/**
- * Duplicates a line.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pLine           The line to duplicate.
- */
-static PDBGFLINE dbgfR3LineDup(PVM pVM, PCDBGFLINE pLine)
-{
-    size_t cb = strlen(pLine->szFilename) + RT_OFFSETOF(DBGFLINE, szFilename[1]);
-    PDBGFLINE pDup = (PDBGFLINE)MMR3HeapAlloc(pVM, MM_TAG_DBGF_LINE_DUP, cb);
-    if (pDup)
-        memcpy(pDup, pLine, cb);
-    return pDup;
-}
-
-
-/**
- * Find line by address (nearest), allocate return buffer.
- *
- * @returns Pointer to the line. Must be freed using DBGFR3LineFree().
- * @returns NULL if the line was not found or if we're out of memory.
- * @param   pVM                 VM handle.
- * @param   Address             Address.
- * @param   poffDisplacement    Where to store the line displacement from Address.
- */
-VMMR3DECL(PDBGFLINE) DBGFR3LineByAddrAlloc(PVM pVM, RTGCUINTPTR Address, PRTGCINTPTR poffDisplacement)
-{
-    DBGFLINE Line;
-    int rc = DBGFR3LineByAddr(pVM, Address, poffDisplacement, &Line);
-    if (RT_FAILURE(rc))
-        return NULL;
-    return dbgfR3LineDup(pVM, &Line);
-}
-
-
-/**
- * Frees a line returned by DBGFR3LineByAddressAlloc().
- *
- * @param   pLine           Pointer to the line.
- */
-VMMR3DECL(void) DBGFR3LineFree(PDBGFLINE pLine)
-{
-    if (pLine)
-        MMR3HeapFree(pLine);
-}
-
-
-#ifdef HAVE_DBGHELP
-
-//static BOOL CALLBACK win32EnumModulesCallback(PSTR ModuleName, DWORD64 BaseOfDll, PVOID UserContext)
-//{
-//    Log(("dbg: module: %08llx %s\n", ModuleName, BaseOfDll));
-//    return TRUE;
-//}
-
-static int win32Error(PVM pVM)
-{
-    int rc = GetLastError();
-    Log(("Lasterror=%d\n", rc));
-
-    //SymEnumerateModules64(pVM, win32EnumModulesCallback, NULL);
-
-    return VERR_GENERAL_FAILURE;
-}
-#endif
-
Index: unk/src/VBox/VMM/EM.cpp
===================================================================
--- /trunk/src/VBox/VMM/EM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2149 +1,0 @@
-/* $Id$ */
-/** @file
- * EM - Execution Monitor / Manager.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_em         EM - The Execution Monitor / Manager
- *
- * The Execution Monitor/Manager is responsible for running the VM, scheduling
- * the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
- * Interpreted), and keeping the CPU states in sync. The function
- * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
- * modes has different inner loops (emR3RawExecute, emR3HwAccExecute, and
- * emR3RemExecute).
- *
- * The interpreted execution is only used to avoid switching between
- * raw-mode/hwaccm and the recompiler when fielding virtualization traps/faults.
- * The interpretation is thus implemented as part of EM.
- *
- * @see grp_em
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_EM
-#include <VBox/em.h>
-#include <VBox/vmm.h>
-#include <VBox/patm.h>
-#include <VBox/csam.h>
-#include <VBox/selm.h>
-#include <VBox/trpm.h>
-#include <VBox/iom.h>
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/rem.h>
-#include <VBox/tm.h>
-#include <VBox/mm.h>
-#include <VBox/ssm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/pdmqueue.h>
-#include <VBox/hwaccm.h>
-#include <VBox/patm.h>
-#include "EMInternal.h"
-#include "include/internal/em.h"
-#include <VBox/vm.h>
-#include <VBox/cpumdis.h>
-#include <VBox/dis.h>
-#include <VBox/disopcode.h>
-#include <VBox/dbgf.h>
-
-#include <iprt/asm.h>
-#include <iprt/string.h>
-#include <iprt/stream.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-#if 0 /* Disabled till after 2.1.0 when we've time to test it. */
-#define EM_NOTIFY_HWACCM
-#endif
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static const char *emR3GetStateName(EMSTATE enmState);
-static int emR3Debug(PVM pVM, PVMCPU pVCpu, int rc);
-static int emR3RemStep(PVM pVM, PVMCPU pVCpu);
-static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
-int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
-
-
-/**
- * Initializes the EM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) EMR3Init(PVM pVM)
-{
-    LogFlow(("EMR3Init\n"));
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompileMemberAlignment(VM, em.s, 32);
-    AssertCompile(sizeof(pVM->em.s) <= sizeof(pVM->em.padding));
-    AssertCompile(sizeof(pVM->aCpus[0].em.s.u.FatalLongJump) <= sizeof(pVM->aCpus[0].em.s.u.achPaddingFatalLongJump));
-    AssertCompileMemberAlignment(EM, CritSectREM, sizeof(uintptr_t));
-
-    /*
-     * Init the structure.
-     */
-    pVM->em.s.offVM = RT_OFFSETOF(VM, em.s);
-    int rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "RawR3Enabled", &pVM->fRawR3Enabled);
-    if (RT_FAILURE(rc))
-        pVM->fRawR3Enabled = true;
-    rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "RawR0Enabled", &pVM->fRawR0Enabled);
-    if (RT_FAILURE(rc))
-        pVM->fRawR0Enabled = true;
-    Log(("EMR3Init: fRawR3Enabled=%d fRawR0Enabled=%d\n", pVM->fRawR3Enabled, pVM->fRawR0Enabled));
-
-    /*
-     * Initialize the REM critical section.
-     */
-    rc = PDMR3CritSectInit(pVM, &pVM->em.s.CritSectREM, RT_SRC_POS, "EM-REM");
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Saved state.
-     */
-    rc = SSMR3RegisterInternal(pVM, "em", 0, EM_SAVED_STATE_VERSION, 16,
-                               NULL, NULL, NULL,
-                               NULL, emR3Save, NULL,
-                               NULL, emR3Load, NULL);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        pVCpu->em.s.offVMCPU = RT_OFFSETOF(VMCPU, em.s);
-
-        pVCpu->em.s.enmState     = (i == 0) ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
-        pVCpu->em.s.enmPrevState = EMSTATE_NONE;
-        pVCpu->em.s.fForceRAW    = false;
-
-        pVCpu->em.s.pCtx         = CPUMQueryGuestCtxPtr(pVCpu);
-        pVCpu->em.s.pPatmGCState = PATMR3QueryGCStateHC(pVM);
-        AssertMsg(pVCpu->em.s.pPatmGCState, ("PATMR3QueryGCStateHC failed!\n"));
-
-        /* Force reset of the time slice. */
-        pVCpu->em.s.u64TimeSliceStart = 0;
-
-# define EM_REG_COUNTER(a, b, c) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, i); \
-        AssertRC(rc);
-
-# define EM_REG_COUNTER_USED(a, b, c) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, c, b, i); \
-        AssertRC(rc);
-
-# define EM_REG_PROFILE(a, b, c) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
-        AssertRC(rc);
-
-# define EM_REG_PROFILE_ADV(a, b, c) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
-        AssertRC(rc);
-
-        /*
-         * Statistics.
-         */
-#ifdef VBOX_WITH_STATISTICS
-        PEMSTATS pStats;
-        rc = MMHyperAlloc(pVM, sizeof(*pStats), 0, MM_TAG_EM, (void **)&pStats);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        pVCpu->em.s.pStatsR3 = pStats;
-        pVCpu->em.s.pStatsR0 = MMHyperR3ToR0(pVM, pStats);
-        pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pStats);
-
-        EM_REG_PROFILE(&pStats->StatRZEmulate,               "/EM/CPU%d/RZ/Interpret",                   "Profiling of EMInterpretInstruction.");
-        EM_REG_PROFILE(&pStats->StatR3Emulate,               "/EM/CPU%d/R3/Interpret",                   "Profiling of EMInterpretInstruction.");
-
-        EM_REG_PROFILE(&pStats->StatRZInterpretSucceeded,    "/EM/CPU%d/RZ/Interpret/Success",           "The number of times an instruction was successfully interpreted.");
-        EM_REG_PROFILE(&pStats->StatR3InterpretSucceeded,    "/EM/CPU%d/R3/Interpret/Success",           "The number of times an instruction was successfully interpreted.");
-
-        EM_REG_COUNTER_USED(&pStats->StatRZAnd,                  "/EM/CPU%d/RZ/Interpret/Success/And",       "The number of times AND was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3And,                  "/EM/CPU%d/R3/Interpret/Success/And",       "The number of times AND was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZAdd,                  "/EM/CPU%d/RZ/Interpret/Success/Add",       "The number of times ADD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Add,                  "/EM/CPU%d/R3/Interpret/Success/Add",       "The number of times ADD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZAdc,                  "/EM/CPU%d/RZ/Interpret/Success/Adc",       "The number of times ADC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Adc,                  "/EM/CPU%d/R3/Interpret/Success/Adc",       "The number of times ADC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZSub,                  "/EM/CPU%d/RZ/Interpret/Success/Sub",       "The number of times SUB was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Sub,                  "/EM/CPU%d/R3/Interpret/Success/Sub",       "The number of times SUB was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZCpuId,                "/EM/CPU%d/RZ/Interpret/Success/CpuId",     "The number of times CPUID was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3CpuId,                "/EM/CPU%d/R3/Interpret/Success/CpuId",     "The number of times CPUID was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZDec,                  "/EM/CPU%d/RZ/Interpret/Success/Dec",       "The number of times DEC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Dec,                  "/EM/CPU%d/R3/Interpret/Success/Dec",       "The number of times DEC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZHlt,                  "/EM/CPU%d/RZ/Interpret/Success/Hlt",       "The number of times HLT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Hlt,                  "/EM/CPU%d/R3/Interpret/Success/Hlt",       "The number of times HLT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZInc,                  "/EM/CPU%d/RZ/Interpret/Success/Inc",       "The number of times INC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Inc,                  "/EM/CPU%d/R3/Interpret/Success/Inc",       "The number of times INC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZInvlPg,               "/EM/CPU%d/RZ/Interpret/Success/Invlpg",    "The number of times INVLPG was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3InvlPg,               "/EM/CPU%d/R3/Interpret/Success/Invlpg",    "The number of times INVLPG was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZIret,                 "/EM/CPU%d/RZ/Interpret/Success/Iret",      "The number of times IRET was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Iret,                 "/EM/CPU%d/R3/Interpret/Success/Iret",      "The number of times IRET was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZLLdt,                 "/EM/CPU%d/RZ/Interpret/Success/LLdt",      "The number of times LLDT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3LLdt,                 "/EM/CPU%d/R3/Interpret/Success/LLdt",      "The number of times LLDT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZLIdt,                 "/EM/CPU%d/RZ/Interpret/Success/LIdt",      "The number of times LIDT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3LIdt,                 "/EM/CPU%d/R3/Interpret/Success/LIdt",      "The number of times LIDT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZLGdt,                 "/EM/CPU%d/RZ/Interpret/Success/LGdt",      "The number of times LGDT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3LGdt,                 "/EM/CPU%d/R3/Interpret/Success/LGdt",      "The number of times LGDT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZMov,                  "/EM/CPU%d/RZ/Interpret/Success/Mov",       "The number of times MOV was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Mov,                  "/EM/CPU%d/R3/Interpret/Success/Mov",       "The number of times MOV was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZMovCRx,               "/EM/CPU%d/RZ/Interpret/Success/MovCRx",    "The number of times MOV CRx was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3MovCRx,               "/EM/CPU%d/R3/Interpret/Success/MovCRx",    "The number of times MOV CRx was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZMovDRx,               "/EM/CPU%d/RZ/Interpret/Success/MovDRx",    "The number of times MOV DRx was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3MovDRx,               "/EM/CPU%d/R3/Interpret/Success/MovDRx",    "The number of times MOV DRx was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZOr,                   "/EM/CPU%d/RZ/Interpret/Success/Or",        "The number of times OR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Or,                   "/EM/CPU%d/R3/Interpret/Success/Or",        "The number of times OR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZPop,                  "/EM/CPU%d/RZ/Interpret/Success/Pop",       "The number of times POP was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Pop,                  "/EM/CPU%d/R3/Interpret/Success/Pop",       "The number of times POP was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZRdtsc,                "/EM/CPU%d/RZ/Interpret/Success/Rdtsc",     "The number of times RDTSC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Rdtsc,                "/EM/CPU%d/R3/Interpret/Success/Rdtsc",     "The number of times RDTSC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZRdpmc,                "/EM/CPU%d/RZ/Interpret/Success/Rdpmc",     "The number of times RDPMC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Rdpmc,                "/EM/CPU%d/R3/Interpret/Success/Rdpmc",     "The number of times RDPMC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZSti,                  "/EM/CPU%d/RZ/Interpret/Success/Sti",       "The number of times STI was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Sti,                  "/EM/CPU%d/R3/Interpret/Success/Sti",       "The number of times STI was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZXchg,                 "/EM/CPU%d/RZ/Interpret/Success/Xchg",      "The number of times XCHG was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Xchg,                 "/EM/CPU%d/R3/Interpret/Success/Xchg",      "The number of times XCHG was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZXor,                  "/EM/CPU%d/RZ/Interpret/Success/Xor",       "The number of times XOR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Xor,                  "/EM/CPU%d/R3/Interpret/Success/Xor",       "The number of times XOR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZMonitor,              "/EM/CPU%d/RZ/Interpret/Success/Monitor",   "The number of times MONITOR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Monitor,              "/EM/CPU%d/R3/Interpret/Success/Monitor",   "The number of times MONITOR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZMWait,                "/EM/CPU%d/RZ/Interpret/Success/MWait",     "The number of times MWAIT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3MWait,                "/EM/CPU%d/R3/Interpret/Success/MWait",     "The number of times MWAIT was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZBtr,                  "/EM/CPU%d/RZ/Interpret/Success/Btr",       "The number of times BTR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Btr,                  "/EM/CPU%d/R3/Interpret/Success/Btr",       "The number of times BTR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZBts,                  "/EM/CPU%d/RZ/Interpret/Success/Bts",       "The number of times BTS was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Bts,                  "/EM/CPU%d/R3/Interpret/Success/Bts",       "The number of times BTS was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZBtc,                  "/EM/CPU%d/RZ/Interpret/Success/Btc",       "The number of times BTC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Btc,                  "/EM/CPU%d/R3/Interpret/Success/Btc",       "The number of times BTC was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg,              "/EM/CPU%d/RZ/Interpret/Success/CmpXchg",   "The number of times CMPXCHG was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg,              "/EM/CPU%d/R3/Interpret/Success/CmpXchg",   "The number of times CMPXCHG was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg8b,            "/EM/CPU%d/RZ/Interpret/Success/CmpXchg8b",   "The number of times CMPXCHG8B was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg8b,            "/EM/CPU%d/R3/Interpret/Success/CmpXchg8b",   "The number of times CMPXCHG8B was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZXAdd,                 "/EM/CPU%d/RZ/Interpret/Success/XAdd",      "The number of times XADD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3XAdd,                 "/EM/CPU%d/R3/Interpret/Success/XAdd",      "The number of times XADD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Rdmsr,                "/EM/CPU%d/R3/Interpret/Success/Rdmsr",      "The number of times RDMSR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZRdmsr,                "/EM/CPU%d/RZ/Interpret/Success/Rdmsr",      "The number of times RDMSR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Wrmsr,                "/EM/CPU%d/R3/Interpret/Success/Wrmsr",      "The number of times WRMSR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZWrmsr,                "/EM/CPU%d/RZ/Interpret/Success/Wrmsr",      "The number of times WRMSR was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3StosWD,               "/EM/CPU%d/R3/Interpret/Success/Stoswd",     "The number of times STOSWD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZStosWD,               "/EM/CPU%d/RZ/Interpret/Success/Stoswd",     "The number of times STOSWD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZWbInvd,               "/EM/CPU%d/RZ/Interpret/Success/WbInvd",     "The number of times WBINVD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3WbInvd,               "/EM/CPU%d/R3/Interpret/Success/WbInvd",     "The number of times WBINVD was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZLmsw,                 "/EM/CPU%d/RZ/Interpret/Success/Lmsw",       "The number of times LMSW was successfully interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3Lmsw,                 "/EM/CPU%d/R3/Interpret/Success/Lmsw",       "The number of times LMSW was successfully interpreted.");
-
-        EM_REG_COUNTER(&pStats->StatRZInterpretFailed,           "/EM/CPU%d/RZ/Interpret/Failed",            "The number of times an instruction was not interpreted.");
-        EM_REG_COUNTER(&pStats->StatR3InterpretFailed,           "/EM/CPU%d/R3/Interpret/Failed",            "The number of times an instruction was not interpreted.");
-
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedAnd,            "/EM/CPU%d/RZ/Interpret/Failed/And",        "The number of times AND was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedAnd,            "/EM/CPU%d/R3/Interpret/Failed/And",        "The number of times AND was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedCpuId,          "/EM/CPU%d/RZ/Interpret/Failed/CpuId",      "The number of times CPUID was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedCpuId,          "/EM/CPU%d/R3/Interpret/Failed/CpuId",      "The number of times CPUID was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedDec,            "/EM/CPU%d/RZ/Interpret/Failed/Dec",        "The number of times DEC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedDec,            "/EM/CPU%d/R3/Interpret/Failed/Dec",        "The number of times DEC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedHlt,            "/EM/CPU%d/RZ/Interpret/Failed/Hlt",        "The number of times HLT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedHlt,            "/EM/CPU%d/R3/Interpret/Failed/Hlt",        "The number of times HLT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedInc,            "/EM/CPU%d/RZ/Interpret/Failed/Inc",        "The number of times INC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedInc,            "/EM/CPU%d/R3/Interpret/Failed/Inc",        "The number of times INC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedInvlPg,         "/EM/CPU%d/RZ/Interpret/Failed/InvlPg",     "The number of times INVLPG was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedInvlPg,         "/EM/CPU%d/R3/Interpret/Failed/InvlPg",     "The number of times INVLPG was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedIret,           "/EM/CPU%d/RZ/Interpret/Failed/Iret",       "The number of times IRET was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedIret,           "/EM/CPU%d/R3/Interpret/Failed/Iret",       "The number of times IRET was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedLLdt,           "/EM/CPU%d/RZ/Interpret/Failed/LLdt",       "The number of times LLDT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedLLdt,           "/EM/CPU%d/R3/Interpret/Failed/LLdt",       "The number of times LLDT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedLIdt,           "/EM/CPU%d/RZ/Interpret/Failed/LIdt",       "The number of times LIDT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedLIdt,           "/EM/CPU%d/R3/Interpret/Failed/LIdt",       "The number of times LIDT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedLGdt,           "/EM/CPU%d/RZ/Interpret/Failed/LGdt",       "The number of times LGDT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedLGdt,           "/EM/CPU%d/R3/Interpret/Failed/LGdt",       "The number of times LGDT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedMov,            "/EM/CPU%d/RZ/Interpret/Failed/Mov",        "The number of times MOV was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedMov,            "/EM/CPU%d/R3/Interpret/Failed/Mov",        "The number of times MOV was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedMovCRx,         "/EM/CPU%d/RZ/Interpret/Failed/MovCRx",     "The number of times MOV CRx was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedMovCRx,         "/EM/CPU%d/R3/Interpret/Failed/MovCRx",     "The number of times MOV CRx was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedMovDRx,         "/EM/CPU%d/RZ/Interpret/Failed/MovDRx",     "The number of times MOV DRx was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedMovDRx,         "/EM/CPU%d/R3/Interpret/Failed/MovDRx",     "The number of times MOV DRx was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedOr,             "/EM/CPU%d/RZ/Interpret/Failed/Or",         "The number of times OR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedOr,             "/EM/CPU%d/R3/Interpret/Failed/Or",         "The number of times OR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedPop,            "/EM/CPU%d/RZ/Interpret/Failed/Pop",        "The number of times POP was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedPop,            "/EM/CPU%d/R3/Interpret/Failed/Pop",        "The number of times POP was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedSti,            "/EM/CPU%d/RZ/Interpret/Failed/Sti",        "The number of times STI was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedSti,            "/EM/CPU%d/R3/Interpret/Failed/Sti",        "The number of times STI was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedXchg,           "/EM/CPU%d/RZ/Interpret/Failed/Xchg",       "The number of times XCHG was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedXchg,           "/EM/CPU%d/R3/Interpret/Failed/Xchg",       "The number of times XCHG was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedXor,            "/EM/CPU%d/RZ/Interpret/Failed/Xor",        "The number of times XOR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedXor,            "/EM/CPU%d/R3/Interpret/Failed/Xor",        "The number of times XOR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedMonitor,        "/EM/CPU%d/RZ/Interpret/Failed/Monitor",    "The number of times MONITOR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedMonitor,        "/EM/CPU%d/R3/Interpret/Failed/Monitor",    "The number of times MONITOR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedMWait,          "/EM/CPU%d/RZ/Interpret/Failed/MWait",      "The number of times MWAIT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedMWait,          "/EM/CPU%d/R3/Interpret/Failed/MWait",      "The number of times MWAIT was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedRdtsc,          "/EM/CPU%d/RZ/Interpret/Failed/Rdtsc",      "The number of times RDTSC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedRdtsc,          "/EM/CPU%d/R3/Interpret/Failed/Rdtsc",      "The number of times RDTSC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedRdpmc,          "/EM/CPU%d/RZ/Interpret/Failed/Rdpmc",      "The number of times RDPMC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedRdpmc,          "/EM/CPU%d/R3/Interpret/Failed/Rdpmc",      "The number of times RDPMC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedRdmsr,          "/EM/CPU%d/RZ/Interpret/Failed/Rdmsr",      "The number of times RDMSR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedRdmsr,          "/EM/CPU%d/R3/Interpret/Failed/Rdmsr",      "The number of times RDMSR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedWrmsr,          "/EM/CPU%d/RZ/Interpret/Failed/Wrmsr",      "The number of times WRMSR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedWrmsr,          "/EM/CPU%d/R3/Interpret/Failed/Wrmsr",      "The number of times WRMSR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedLmsw,           "/EM/CPU%d/RZ/Interpret/Failed/Lmsw",       "The number of times LMSW was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedLmsw,           "/EM/CPU%d/R3/Interpret/Failed/Lmsw",       "The number of times LMSW was not interpreted.");
-
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedMisc,           "/EM/CPU%d/RZ/Interpret/Failed/Misc",       "The number of times some misc instruction was encountered.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedMisc,           "/EM/CPU%d/R3/Interpret/Failed/Misc",       "The number of times some misc instruction was encountered.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedAdd,            "/EM/CPU%d/RZ/Interpret/Failed/Add",        "The number of times ADD was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedAdd,            "/EM/CPU%d/R3/Interpret/Failed/Add",        "The number of times ADD was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedAdc,            "/EM/CPU%d/RZ/Interpret/Failed/Adc",        "The number of times ADC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedAdc,            "/EM/CPU%d/R3/Interpret/Failed/Adc",        "The number of times ADC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedBtr,            "/EM/CPU%d/RZ/Interpret/Failed/Btr",        "The number of times BTR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedBtr,            "/EM/CPU%d/R3/Interpret/Failed/Btr",        "The number of times BTR was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedBts,            "/EM/CPU%d/RZ/Interpret/Failed/Bts",        "The number of times BTS was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedBts,            "/EM/CPU%d/R3/Interpret/Failed/Bts",        "The number of times BTS was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedBtc,            "/EM/CPU%d/RZ/Interpret/Failed/Btc",        "The number of times BTC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedBtc,            "/EM/CPU%d/R3/Interpret/Failed/Btc",        "The number of times BTC was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedCli,            "/EM/CPU%d/RZ/Interpret/Failed/Cli",        "The number of times CLI was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedCli,            "/EM/CPU%d/R3/Interpret/Failed/Cli",        "The number of times CLI was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg,        "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg",    "The number of times CMPXCHG was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg,        "/EM/CPU%d/R3/Interpret/Failed/CmpXchg",    "The number of times CMPXCHG was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg8b,      "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg8b",  "The number of times CMPXCHG8B was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg8b,      "/EM/CPU%d/R3/Interpret/Failed/CmpXchg8b",  "The number of times CMPXCHG8B was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedXAdd,           "/EM/CPU%d/RZ/Interpret/Failed/XAdd",       "The number of times XADD was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedXAdd,           "/EM/CPU%d/R3/Interpret/Failed/XAdd",       "The number of times XADD was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedMovNTPS,        "/EM/CPU%d/RZ/Interpret/Failed/MovNTPS",    "The number of times MOVNTPS was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedMovNTPS,        "/EM/CPU%d/R3/Interpret/Failed/MovNTPS",    "The number of times MOVNTPS was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedStosWD,         "/EM/CPU%d/RZ/Interpret/Failed/StosWD",     "The number of times STOSWD was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedStosWD,         "/EM/CPU%d/R3/Interpret/Failed/StosWD",     "The number of times STOSWD was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedSub,            "/EM/CPU%d/RZ/Interpret/Failed/Sub",        "The number of times SUB was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedSub,            "/EM/CPU%d/R3/Interpret/Failed/Sub",        "The number of times SUB was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedWbInvd,         "/EM/CPU%d/RZ/Interpret/Failed/WbInvd",     "The number of times WBINVD was not interpreted.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedWbInvd,         "/EM/CPU%d/R3/Interpret/Failed/WbInvd",     "The number of times WBINVD was not interpreted.");
-
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedUserMode,       "/EM/CPU%d/RZ/Interpret/Failed/UserMode",   "The number of rejections because of CPL.");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedUserMode,       "/EM/CPU%d/R3/Interpret/Failed/UserMode",   "The number of rejections because of CPL.");
-        EM_REG_COUNTER_USED(&pStats->StatRZFailedPrefix,         "/EM/CPU%d/RZ/Interpret/Failed/Prefix",     "The number of rejections because of prefix .");
-        EM_REG_COUNTER_USED(&pStats->StatR3FailedPrefix,         "/EM/CPU%d/R3/Interpret/Failed/Prefix",     "The number of rejections because of prefix .");
-
-        EM_REG_COUNTER_USED(&pStats->StatCli,                    "/EM/CPU%d/R3/PrivInst/Cli",                "Number of cli instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatSti,                    "/EM/CPU%d/R3/PrivInst/Sti",                "Number of sli instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatIn,                     "/EM/CPU%d/R3/PrivInst/In",                 "Number of in instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatOut,                    "/EM/CPU%d/R3/PrivInst/Out",                "Number of out instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatIoRestarted,            "/EM/CPU%d/R3/PrivInst/IoRestarted",        "Number of restarted i/o instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatHlt,                    "/EM/CPU%d/R3/PrivInst/Hlt",                "Number of hlt instructions not handled in GC because of PATM.");
-        EM_REG_COUNTER_USED(&pStats->StatInvlpg,                 "/EM/CPU%d/R3/PrivInst/Invlpg",             "Number of invlpg instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMisc,                   "/EM/CPU%d/R3/PrivInst/Misc",               "Number of misc. instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[0],          "/EM/CPU%d/R3/PrivInst/Mov CR0, X",         "Number of mov CR0 read instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[1],          "/EM/CPU%d/R3/PrivInst/Mov CR1, X",         "Number of mov CR1 read instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[2],          "/EM/CPU%d/R3/PrivInst/Mov CR2, X",         "Number of mov CR2 read instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[3],          "/EM/CPU%d/R3/PrivInst/Mov CR3, X",         "Number of mov CR3 read instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[4],          "/EM/CPU%d/R3/PrivInst/Mov CR4, X",         "Number of mov CR4 read instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[0],           "/EM/CPU%d/R3/PrivInst/Mov X, CR0",         "Number of mov CR0 write instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[1],           "/EM/CPU%d/R3/PrivInst/Mov X, CR1",         "Number of mov CR1 write instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[2],           "/EM/CPU%d/R3/PrivInst/Mov X, CR2",         "Number of mov CR2 write instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[3],           "/EM/CPU%d/R3/PrivInst/Mov X, CR3",         "Number of mov CR3 write instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[4],           "/EM/CPU%d/R3/PrivInst/Mov X, CR4",         "Number of mov CR4 write instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovDRx,                 "/EM/CPU%d/R3/PrivInst/MovDRx",             "Number of mov DRx instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatIret,                   "/EM/CPU%d/R3/PrivInst/Iret",               "Number of iret instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovLgdt,                "/EM/CPU%d/R3/PrivInst/Lgdt",               "Number of lgdt instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovLidt,                "/EM/CPU%d/R3/PrivInst/Lidt",               "Number of lidt instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatMovLldt,                "/EM/CPU%d/R3/PrivInst/Lldt",               "Number of lldt instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatSysEnter,               "/EM/CPU%d/R3/PrivInst/Sysenter",           "Number of sysenter instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatSysExit,                "/EM/CPU%d/R3/PrivInst/Sysexit",            "Number of sysexit instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatSysCall,                "/EM/CPU%d/R3/PrivInst/Syscall",            "Number of syscall instructions.");
-        EM_REG_COUNTER_USED(&pStats->StatSysRet,                 "/EM/CPU%d/R3/PrivInst/Sysret",             "Number of sysret instructions.");
-
-        EM_REG_COUNTER(&pVCpu->em.s.StatTotalClis,               "/EM/CPU%d/Cli/Total",                      "Total number of cli instructions executed.");
-        pVCpu->em.s.pCliStatTree = 0;
-
-        /* these should be considered for release statistics. */
-        EM_REG_COUNTER(&pVCpu->em.s.StatIOEmu,                 "/PROF/CPU%d/EM/Emulation/IO",      "Profiling of emR3RawExecuteIOInstruction.");
-        EM_REG_COUNTER(&pVCpu->em.s.StatPrivEmu,               "/PROF/CPU%d/EM/Emulation/Priv",    "Profiling of emR3RawPrivileged.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatHwAccEntry,           "/PROF/CPU%d/EM/HwAccEnter",        "Profiling Hardware Accelerated Mode entry overhead.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatHwAccExec,            "/PROF/CPU%d/EM/HwAccExec",         "Profiling Hardware Accelerated Mode execution.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatREMEmu,               "/PROF/CPU%d/EM/REMEmuSingle",      "Profiling single instruction REM execution.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatREMExec,              "/PROF/CPU%d/EM/REMExec",           "Profiling REM execution.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatREMSync,              "/PROF/CPU%d/EM/REMSync",           "Profiling REM context syncing.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatRAWEntry,             "/PROF/CPU%d/EM/RAWEnter",          "Profiling Raw Mode entry overhead.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatRAWExec,              "/PROF/CPU%d/EM/RAWExec",           "Profiling Raw Mode execution.");
-        EM_REG_PROFILE(&pVCpu->em.s.StatRAWTail,              "/PROF/CPU%d/EM/RAWTail",           "Profiling Raw Mode tail overhead.");
-
-#endif /* VBOX_WITH_STATISTICS */
-
-        EM_REG_COUNTER(&pVCpu->em.s.StatForcedActions,     "/PROF/CPU%d/EM/ForcedActions",     "Profiling forced action execution.");
-        EM_REG_COUNTER(&pVCpu->em.s.StatHalted,            "/PROF/CPU%d/EM/Halted",            "Profiling halted state (VMR3WaitHalted).");
-        EM_REG_PROFILE_ADV(&pVCpu->em.s.StatCapped,        "/PROF/CPU%d/EM/Capped",            "Profiling capped state (sleep).");
-        EM_REG_COUNTER(&pVCpu->em.s.StatREMTotal,          "/PROF/CPU%d/EM/REMTotal",          "Profiling emR3RemExecute (excluding FFs).");
-        EM_REG_COUNTER(&pVCpu->em.s.StatRAWTotal,          "/PROF/CPU%d/EM/RAWTotal",          "Profiling emR3RawExecute (excluding FFs).");
-
-        EM_REG_PROFILE_ADV(&pVCpu->em.s.StatTotal,         "/PROF/CPU%d/EM/Total",             "Profiling EMR3ExecuteVM.");
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM     The VM.
- */
-VMMR3DECL(void) EMR3Relocate(PVM pVM)
-{
-    LogFlow(("EMR3Relocate\n"));
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-        if (pVCpu->em.s.pStatsR3)
-            pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pVCpu->em.s.pStatsR3);
-    }
-}
-
-
-/**
- * Reset the EM state for a CPU.
- *
- * Called by EMR3Reset and hot plugging.
- *
- * @param   pVCpu               The virtual CPU.
- */
-VMMR3DECL(void) EMR3ResetCpu(PVMCPU pVCpu)
-{
-    pVCpu->em.s.fForceRAW = false;
-
-    /* VMR3Reset may return VINF_EM_RESET or VINF_EM_SUSPEND, so transition
-       out of the HALTED state here so that enmPrevState doesn't end up as
-       HALTED when EMR3Execute returns. */
-    if (pVCpu->em.s.enmState == EMSTATE_HALTED)
-    {
-        Log(("EMR3ResetCpu: Cpu#%u %s -> %s\n", pVCpu->idCpu, emR3GetStateName(pVCpu->em.s.enmState), pVCpu->idCpu == 0 ? "EMSTATE_NONE" : "EMSTATE_WAIT_SIPI"));
-        pVCpu->em.s.enmState = pVCpu->idCpu == 0 ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
-    }
-}
-
-
-/**
- * Reset notification.
- *
- * @param   pVM                 The VM handle.
- */
-VMMR3DECL(void) EMR3Reset(PVM pVM)
-{
-    Log(("EMR3Reset: \n"));
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        EMR3ResetCpu(&pVM->aCpus[i]);
-}
-
-
-/**
- * Terminates the EM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) EMR3Term(PVM pVM)
-{
-    AssertMsg(pVM->em.s.offVM, ("bad init order!\n"));
-
-    PDMR3CritSectDelete(&pVM->em.s.CritSectREM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM)
-{
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        int rc = SSMR3PutBool(pSSM, pVCpu->em.s.fForceRAW);
-        AssertRCReturn(rc, rc);
-
-        Assert(pVCpu->em.s.enmState     == EMSTATE_SUSPENDED);
-        Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
-        rc = SSMR3PutU32(pSSM, pVCpu->em.s.enmPrevState);
-        AssertRCReturn(rc, rc);
-
-        /* Save mwait state. */
-        rc = SSMR3PutU32(pSSM, pVCpu->em.s.mwait.fWait);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMWaitEAX);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMWaitECX);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMonitorEAX);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMonitorECX);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMonitorEDX);
-        AssertRCReturn(rc, rc);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    /*
-     * Validate version.
-     */
-    if (    uVersion != EM_SAVED_STATE_VERSION
-        &&  uVersion != EM_SAVED_STATE_VERSION_PRE_MWAIT
-        &&  uVersion != EM_SAVED_STATE_VERSION_PRE_SMP)
-    {
-        AssertMsgFailed(("emR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, EM_SAVED_STATE_VERSION));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
-
-    /*
-     * Load the saved state.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        int rc = SSMR3GetBool(pSSM, &pVCpu->em.s.fForceRAW);
-        if (RT_FAILURE(rc))
-            pVCpu->em.s.fForceRAW = false;
-        AssertRCReturn(rc, rc);
-
-        if (uVersion > EM_SAVED_STATE_VERSION_PRE_SMP)
-        {
-            AssertCompile(sizeof(pVCpu->em.s.enmPrevState) == sizeof(uint32_t));
-            rc = SSMR3GetU32(pSSM, (uint32_t *)&pVCpu->em.s.enmPrevState);
-            AssertRCReturn(rc, rc);
-            Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
-
-            pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
-        }
-        if (uVersion > EM_SAVED_STATE_VERSION_PRE_MWAIT)
-        {
-            /* Load mwait state. */
-            rc = SSMR3GetU32(pSSM, &pVCpu->em.s.mwait.fWait);
-            AssertRCReturn(rc, rc);
-            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMWaitEAX);
-            AssertRCReturn(rc, rc);
-            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMWaitECX);
-            AssertRCReturn(rc, rc);
-            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMonitorEAX);
-            AssertRCReturn(rc, rc);
-            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMonitorECX);
-            AssertRCReturn(rc, rc);
-            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMonitorEDX);
-            AssertRCReturn(rc, rc);
-        }
-
-        Assert(!pVCpu->em.s.pCliStatTree);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Raise a fatal error.
- *
- * Safely terminate the VM with full state report and stuff. This function
- * will naturally never return.
- *
- * @param   pVCpu       VMCPU handle.
- * @param   rc          VBox status code.
- */
-VMMR3DECL(void) EMR3FatalError(PVMCPU pVCpu, int rc)
-{
-    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
-    longjmp(pVCpu->em.s.u.FatalLongJump, rc);
-    AssertReleaseMsgFailed(("longjmp returned!\n"));
-}
-
-
-/**
- * Gets the EM state name.
- *
- * @returns pointer to read only state name,
- * @param   enmState    The state.
- */
-static const char *emR3GetStateName(EMSTATE enmState)
-{
-    switch (enmState)
-    {
-        case EMSTATE_NONE:              return "EMSTATE_NONE";
-        case EMSTATE_RAW:               return "EMSTATE_RAW";
-        case EMSTATE_HWACC:             return "EMSTATE_HWACC";
-        case EMSTATE_REM:               return "EMSTATE_REM";
-        case EMSTATE_PARAV:             return "EMSTATE_PARAV";
-        case EMSTATE_HALTED:            return "EMSTATE_HALTED";
-        case EMSTATE_WAIT_SIPI:         return "EMSTATE_WAIT_SIPI";
-        case EMSTATE_SUSPENDED:         return "EMSTATE_SUSPENDED";
-        case EMSTATE_TERMINATING:       return "EMSTATE_TERMINATING";
-        case EMSTATE_DEBUG_GUEST_RAW:   return "EMSTATE_DEBUG_GUEST_RAW";
-        case EMSTATE_DEBUG_GUEST_REM:   return "EMSTATE_DEBUG_GUEST_REM";
-        case EMSTATE_DEBUG_HYPER:       return "EMSTATE_DEBUG_HYPER";
-        case EMSTATE_GURU_MEDITATION:   return "EMSTATE_GURU_MEDITATION";
-        default:                        return "Unknown!";
-    }
-}
-
-
-/**
- * Debug loop.
- *
- * @returns VBox status code for EM.
- * @param   pVM     VM handle.
- * @param   pVCpu   VMCPU handle.
- * @param   rc      Current EM VBox status code..
- */
-static int emR3Debug(PVM pVM, PVMCPU pVCpu, int rc)
-{
-    for (;;)
-    {
-        Log(("emR3Debug: rc=%Rrc\n", rc));
-        const int rcLast = rc;
-
-        /*
-         * Debug related RC.
-         */
-        switch (rc)
-        {
-            /*
-             * Single step an instruction.
-             */
-            case VINF_EM_DBG_STEP:
-                if (    pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
-                    ||  pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER
-                    ||  pVCpu->em.s.fForceRAW /* paranoia */)
-                    rc = emR3RawStep(pVM, pVCpu);
-                else
-                {
-                    Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM);
-                    rc = emR3RemStep(pVM, pVCpu);
-                }
-                break;
-
-            /*
-             * Simple events: stepped, breakpoint, stop/assertion.
-             */
-            case VINF_EM_DBG_STEPPED:
-                rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED);
-                break;
-
-            case VINF_EM_DBG_BREAKPOINT:
-                rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT);
-                break;
-
-            case VINF_EM_DBG_STOP:
-                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, NULL, 0, NULL, NULL);
-                break;
-
-            case VINF_EM_DBG_HYPER_STEPPED:
-                rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED_HYPER);
-                break;
-
-            case VINF_EM_DBG_HYPER_BREAKPOINT:
-                rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT_HYPER);
-                break;
-
-            case VINF_EM_DBG_HYPER_ASSERTION:
-                RTPrintf("\nVINF_EM_DBG_HYPER_ASSERTION:\n%s%s\n", VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
-                rc = DBGFR3EventAssertion(pVM, DBGFEVENT_ASSERTION_HYPER, VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
-                break;
-
-            /*
-             * Guru meditation.
-             */
-            case VERR_VMM_RING0_ASSERTION: /** @todo Make a guru meditation event! */
-                rc = DBGFR3EventSrc(pVM, DBGFEVENT_FATAL_ERROR, "VERR_VMM_RING0_ASSERTION", 0, NULL, NULL);
-                break;
-            case VERR_REM_TOO_MANY_TRAPS: /** @todo Make a guru meditation event! */
-                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VERR_REM_TOO_MANY_TRAPS", 0, NULL, NULL);
-                break;
-
-            default: /** @todo don't use default for guru, but make special errors code! */
-                rc = DBGFR3Event(pVM, DBGFEVENT_FATAL_ERROR);
-                break;
-        }
-
-        /*
-         * Process the result.
-         */
-        do
-        {
-            switch (rc)
-            {
-                /*
-                 * Continue the debugging loop.
-                 */
-                case VINF_EM_DBG_STEP:
-                case VINF_EM_DBG_STOP:
-                case VINF_EM_DBG_STEPPED:
-                case VINF_EM_DBG_BREAKPOINT:
-                case VINF_EM_DBG_HYPER_STEPPED:
-                case VINF_EM_DBG_HYPER_BREAKPOINT:
-                case VINF_EM_DBG_HYPER_ASSERTION:
-                    break;
-
-                /*
-                 * Resuming execution (in some form) has to be done here if we got
-                 * a hypervisor debug event.
-                 */
-                case VINF_SUCCESS:
-                case VINF_EM_RESUME:
-                case VINF_EM_SUSPEND:
-                case VINF_EM_RESCHEDULE:
-                case VINF_EM_RESCHEDULE_RAW:
-                case VINF_EM_RESCHEDULE_REM:
-                case VINF_EM_HALT:
-                    if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
-                    {
-                        rc = emR3RawResumeHyper(pVM, pVCpu);
-                        if (rc != VINF_SUCCESS && RT_SUCCESS(rc))
-                            continue;
-                    }
-                    if (rc == VINF_SUCCESS)
-                        rc = VINF_EM_RESCHEDULE;
-                    return rc;
-
-                /*
-                 * The debugger isn't attached.
-                 * We'll simply turn the thing off since that's the easiest thing to do.
-                 */
-                case VERR_DBGF_NOT_ATTACHED:
-                    switch (rcLast)
-                    {
-                        case VINF_EM_DBG_HYPER_STEPPED:
-                        case VINF_EM_DBG_HYPER_BREAKPOINT:
-                        case VINF_EM_DBG_HYPER_ASSERTION:
-                        case VERR_TRPM_PANIC:
-                        case VERR_TRPM_DONT_PANIC:
-                        case VERR_VMM_RING0_ASSERTION:
-                        case VERR_VMM_HYPER_CR3_MISMATCH:
-                        case VERR_VMM_RING3_CALL_DISABLED:
-                            return rcLast;
-                    }
-                    return VINF_EM_OFF;
-
-                /*
-                 * Status codes terminating the VM in one or another sense.
-                 */
-                case VINF_EM_TERMINATE:
-                case VINF_EM_OFF:
-                case VINF_EM_RESET:
-                case VINF_EM_NO_MEMORY:
-                case VINF_EM_RAW_STALE_SELECTOR:
-                case VINF_EM_RAW_IRET_TRAP:
-                case VERR_TRPM_PANIC:
-                case VERR_TRPM_DONT_PANIC:
-                case VERR_VMM_RING0_ASSERTION:
-                case VERR_VMM_HYPER_CR3_MISMATCH:
-                case VERR_VMM_RING3_CALL_DISABLED:
-                case VERR_INTERNAL_ERROR:
-                case VERR_INTERNAL_ERROR_2:
-                case VERR_INTERNAL_ERROR_3:
-                case VERR_INTERNAL_ERROR_4:
-                case VERR_INTERNAL_ERROR_5:
-                case VERR_IPE_UNEXPECTED_STATUS:
-                case VERR_IPE_UNEXPECTED_INFO_STATUS:
-                case VERR_IPE_UNEXPECTED_ERROR_STATUS:
-                    return rc;
-
-                /*
-                 * The rest is unexpected, and will keep us here.
-                 */
-                default:
-                    AssertMsgFailed(("Unexpected rc %Rrc!\n", rc));
-                    break;
-            }
-        } while (false);
-    } /* debug for ever */
-}
-
-/**
- * Steps recompiled code.
- *
- * @returns VBox status code. The most important ones are: VINF_EM_STEP_EVENT,
- *          VINF_EM_RESCHEDULE, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- */
-static int emR3RemStep(PVM pVM, PVMCPU pVCpu)
-{
-    LogFlow(("emR3RemStep: cs:eip=%04x:%08x\n", CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
-
-    EMRemLock(pVM);
-
-    /*
-     * Switch to REM, step instruction, switch back.
-     */
-    int rc = REMR3State(pVM, pVCpu);
-    if (RT_SUCCESS(rc))
-    {
-        rc = REMR3Step(pVM, pVCpu);
-        REMR3StateBack(pVM, pVCpu);
-    }
-    EMRemUnlock(pVM);
-
-    LogFlow(("emR3RemStep: returns %Rrc cs:eip=%04x:%08x\n", rc, CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
-    return rc;
-}
-
-
-/**
- * emR3RemExecute helper that syncs the state back from REM and leave the REM
- * critical section.
- *
- * @returns false - new fInREMState value.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The virtual CPU handle.
- */
-DECLINLINE(bool) emR3RemExecuteSyncBack(PVM pVM, PVMCPU pVCpu)
-{
-    STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, a);
-    REMR3StateBack(pVM, pVCpu);
-    STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, a);
-
-    EMRemUnlock(pVM);
-    return false;
-}
-
-
-/**
- * Executes recompiled code.
- *
- * This function contains the recompiler version of the inner
- * execution loop (the outer loop being in EMR3ExecuteVM()).
- *
- * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
- *          VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   pfFFDone    Where to store an indicator telling whether or not
- *                      FFs were done before returning.
- *
- */
-static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
-{
-#ifdef LOG_ENABLED
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-    uint32_t cpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
-
-    if (pCtx->eflags.Bits.u1VM)
-        Log(("EMV86: %04X:%08X IF=%d\n", pCtx->cs, pCtx->eip, pCtx->eflags.Bits.u1IF));
-    else
-        Log(("EMR%d: %04X:%08X ESP=%08X IF=%d CR0=%x eflags=%x\n", cpl, pCtx->cs, pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, (uint32_t)pCtx->cr0, pCtx->eflags.u));
-#endif
-    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatREMTotal, a);
-
-#if defined(VBOX_STRICT) && defined(DEBUG_bird)
-    AssertMsg(   VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
-              || !MMHyperIsInsideArea(pVM, CPUMGetGuestEIP(pVCpu)),  /** @todo #1419 - get flat address. */
-              ("cs:eip=%RX16:%RX32\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
-#endif
-
-    /*
-     * Spin till we get a forced action which returns anything but VINF_SUCCESS
-     * or the REM suggests raw-mode execution.
-     */
-    *pfFFDone = false;
-    bool    fInREMState = false;
-    int     rc          = VINF_SUCCESS;
-    for (;;)
-    {
-        /*
-         * Lock REM and update the state if not already in sync.
-         *
-         * Note! Big lock, but you are not supposed to own any lock when
-         *       coming in here.
-         */
-        if (!fInREMState)
-        {
-            EMRemLock(pVM);
-            STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, b);
-
-            /* Flush the recompiler translation blocks if the VCPU has changed,
-               also force a full CPU state resync. */
-            if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
-            {
-                REMFlushTBs(pVM);
-                CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
-            }
-            pVM->em.s.idLastRemCpu = pVCpu->idCpu;
-
-            rc = REMR3State(pVM, pVCpu);
-
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, b);
-            if (RT_FAILURE(rc))
-                break;
-            fInREMState = true;
-
-            /*
-             * We might have missed the raising of VMREQ, TIMER and some other
-             * important FFs while we were busy switching the state. So, check again.
-             */
-            if (    VM_FF_ISPENDING(pVM, VM_FF_REQUEST | VM_FF_PDM_QUEUES | VM_FF_DBGF | VM_FF_CHECK_VM_STATE | VM_FF_RESET)
-                ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TIMER | VMCPU_FF_REQUEST))
-            {
-                LogFlow(("emR3RemExecute: Skipping run, because FF is set. %#x\n", pVM->fGlobalForcedActions));
-                goto l_REMDoForcedActions;
-            }
-        }
-
-
-        /*
-         * Execute REM.
-         */
-        if (RT_LIKELY(EMR3IsExecutionAllowed(pVM, pVCpu)))
-        {
-            STAM_PROFILE_START(&pVCpu->em.s.StatREMExec, c);
-            rc = REMR3Run(pVM, pVCpu);
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatREMExec, c);
-        }
-        else
-        {
-            /* Give up this time slice; virtual time continues */
-            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
-            RTThreadSleep(5);
-            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
-            rc = VINF_SUCCESS;
-        }
-
-        /*
-         * Deal with high priority post execution FFs before doing anything
-         * else.  Sync back the state and leave the lock to be on the safe side.
-         */
-        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
-        {
-            fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
-            rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
-        }
-
-        /*
-         * Process the returned status code.
-         */
-        if (rc != VINF_SUCCESS)
-        {
-            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
-                break;
-            if (rc != VINF_REM_INTERRUPED_FF)
-            {
-                /*
-                 * Anything which is not known to us means an internal error
-                 * and the termination of the VM!
-                 */
-                AssertMsg(rc == VERR_REM_TOO_MANY_TRAPS, ("Unknown GC return code: %Rra\n", rc));
-                break;
-            }
-        }
-
-
-        /*
-         * Check and execute forced actions.
-         *
-         * Sync back the VM state and leave the lock  before calling any of
-         * these, you never know what's going to happen here.
-         */
-#ifdef VBOX_HIGH_RES_TIMERS_HACK
-        TMTimerPollVoid(pVM, pVCpu);
-#endif
-        AssertCompile((VMCPU_FF_ALL_BUT_RAW_MASK & ~(VMCPU_FF_CSAM_PENDING_ACTION | VMCPU_FF_CSAM_SCAN_PAGE)) & VMCPU_FF_TIMER);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_ALL_BUT_RAW_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_BUT_RAW_MASK & ~(VMCPU_FF_CSAM_PENDING_ACTION | VMCPU_FF_CSAM_SCAN_PAGE)))
-        {
-l_REMDoForcedActions:
-            if (fInREMState)
-                fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
-            STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatREMTotal, a);
-            rc = emR3ForcedActions(pVM, pVCpu, rc);
-            STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatREMTotal, a);
-            if (    rc != VINF_SUCCESS
-                &&  rc != VINF_EM_RESCHEDULE_REM)
-            {
-                *pfFFDone = true;
-                break;
-            }
-        }
-
-    } /* The Inner Loop, recompiled execution mode version. */
-
-
-    /*
-     * Returning. Sync back the VM state if required.
-     */
-    if (fInREMState)
-        fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
-
-    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatREMTotal, a);
-    return rc;
-}
-
-
-#ifdef DEBUG
-
-int emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
-{
-    EMSTATE  enmOldState = pVCpu->em.s.enmState;
-
-    pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
-
-    Log(("Single step BEGIN:\n"));
-    for (uint32_t i = 0; i < cIterations; i++)
-    {
-        DBGFR3PrgStep(pVCpu);
-        DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
-        emR3RemStep(pVM, pVCpu);
-        if (emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx) != EMSTATE_REM)
-            break;
-    }
-    Log(("Single step END:\n"));
-    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
-    pVCpu->em.s.enmState = enmOldState;
-    return VINF_EM_RESCHEDULE;
-}
-
-#endif /* DEBUG */
-
-
-/**
- * Decides whether to execute RAW, HWACC or REM.
- *
- * @returns new EM state
- * @param   pVM     The VM.
- * @param   pVCpu   The VMCPU handle.
- * @param   pCtx    The CPU context.
- */
-EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
-{
-    /*
-     * When forcing raw-mode execution, things are simple.
-     */
-    if (pVCpu->em.s.fForceRAW)
-        return EMSTATE_RAW;
-
-    /*
-     * We stay in the wait for SIPI state unless explicitly told otherwise.
-     */
-    if (pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI)
-        return EMSTATE_WAIT_SIPI;
-
-    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
-    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
-    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
-
-    X86EFLAGS EFlags = pCtx->eflags;
-    if (HWACCMIsEnabled(pVM))
-    {
-        /* Hardware accelerated raw-mode:
-         *
-         * Typically only 32-bits protected mode, with paging enabled, code is allowed here.
-         */
-        if (HWACCMR3CanExecuteGuest(pVM, pCtx) == true)
-            return EMSTATE_HWACC;
-
-        /* Note: Raw mode and hw accelerated mode are incompatible. The latter turns
-         * off monitoring features essential for raw mode! */
-        return EMSTATE_REM;
-    }
-
-    /*
-     * Standard raw-mode:
-     *
-     * Here we only support 16 & 32 bits protected mode ring 3 code that has no IO privileges
-     * or 32 bits protected mode ring 0 code
-     *
-     * The tests are ordered by the likelihood of being true during normal execution.
-     */
-    if (EFlags.u32 & (X86_EFL_TF /* | HF_INHIBIT_IRQ_MASK*/))
-    {
-        Log2(("raw mode refused: EFlags=%#x\n", EFlags.u32));
-        return EMSTATE_REM;
-    }
-
-#ifndef VBOX_RAW_V86
-    if (EFlags.u32 & X86_EFL_VM) {
-        Log2(("raw mode refused: VM_MASK\n"));
-        return EMSTATE_REM;
-    }
-#endif
-
-    /** @todo check up the X86_CR0_AM flag in respect to raw mode!!! We're probably not emulating it right! */
-    uint32_t u32CR0 = pCtx->cr0;
-    if ((u32CR0 & (X86_CR0_PG | X86_CR0_PE)) != (X86_CR0_PG | X86_CR0_PE))
-    {
-        //Log2(("raw mode refused: %s%s%s\n", (u32CR0 & X86_CR0_PG) ? "" : " !PG", (u32CR0 & X86_CR0_PE) ? "" : " !PE", (u32CR0 & X86_CR0_AM) ? "" : " !AM"));
-        return EMSTATE_REM;
-    }
-
-    if (pCtx->cr4 & X86_CR4_PAE)
-    {
-        uint32_t u32Dummy, u32Features;
-
-        CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
-        if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
-            return EMSTATE_REM;
-    }
-
-    unsigned uSS = pCtx->ss;
-    if (    pCtx->eflags.Bits.u1VM
-        ||  (uSS & X86_SEL_RPL) == 3)
-    {
-        if (!EMIsRawRing3Enabled(pVM))
-            return EMSTATE_REM;
-
-        if (!(EFlags.u32 & X86_EFL_IF))
-        {
-            Log2(("raw mode refused: IF (RawR3)\n"));
-            return EMSTATE_REM;
-        }
-
-        if (!(u32CR0 & X86_CR0_WP) && EMIsRawRing0Enabled(pVM))
-        {
-            Log2(("raw mode refused: CR0.WP + RawR0\n"));
-            return EMSTATE_REM;
-        }
-    }
-    else
-    {
-        if (!EMIsRawRing0Enabled(pVM))
-            return EMSTATE_REM;
-
-        /* Only ring 0 supervisor code. */
-        if ((uSS & X86_SEL_RPL) != 0)
-        {
-            Log2(("raw r0 mode refused: CPL %d\n", uSS & X86_SEL_RPL));
-            return EMSTATE_REM;
-        }
-
-        // Let's start with pure 32 bits ring 0 code first
-        /** @todo What's pure 32-bit mode? flat? */
-        if (    !(pCtx->ssHid.Attr.n.u1DefBig)
-            ||  !(pCtx->csHid.Attr.n.u1DefBig))
-        {
-            Log2(("raw r0 mode refused: SS/CS not 32bit\n"));
-            return EMSTATE_REM;
-        }
-
-        /* Write protection must be turned on, or else the guest can overwrite our hypervisor code and data. */
-        if (!(u32CR0 & X86_CR0_WP))
-        {
-            Log2(("raw r0 mode refused: CR0.WP=0!\n"));
-            return EMSTATE_REM;
-        }
-
-        if (PATMShouldUseRawMode(pVM, (RTGCPTR)pCtx->eip))
-        {
-            Log2(("raw r0 mode forced: patch code\n"));
-            return EMSTATE_RAW;
-        }
-
-#if !defined(VBOX_ALLOW_IF0) && !defined(VBOX_RUN_INTERRUPT_GATE_HANDLERS)
-        if (!(EFlags.u32 & X86_EFL_IF))
-        {
-            ////Log2(("R0: IF=0 VIF=%d %08X\n", eip, pVMeflags));
-            //Log2(("RR0: Interrupts turned off; fall back to emulation\n"));
-            return EMSTATE_REM;
-        }
-#endif
-
-        /** @todo still necessary??? */
-        if (EFlags.Bits.u2IOPL != 0)
-        {
-            Log2(("raw r0 mode refused: IOPL %d\n", EFlags.Bits.u2IOPL));
-            return EMSTATE_REM;
-        }
-    }
-
-    Assert(PGMPhysIsA20Enabled(pVCpu));
-    return EMSTATE_RAW;
-}
-
-
-/**
- * Executes all high priority post execution force actions.
- *
- * @returns rc or a fatal status code.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   rc          The current rc.
- */
-int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
-{
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
-        PDMCritSectFF(pVCpu);
-
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_CSAM_PENDING_ACTION))
-        CSAMR3DoPendingAction(pVM, pVCpu);
-
-    if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
-    {
-        if (    rc > VINF_EM_NO_MEMORY
-            &&  rc <= VINF_EM_LAST)
-            rc = VINF_EM_NO_MEMORY;
-    }
-
-    return rc;
-}
-
-
-/**
- * Executes all pending forced actions.
- *
- * Forced actions can cause execution delays and execution
- * rescheduling. The first we deal with using action priority, so
- * that for instance pending timers aren't scheduled and ran until
- * right before execution. The rescheduling we deal with using
- * return codes. The same goes for VM termination, only in that case
- * we exit everything.
- *
- * @returns VBox status code of equal or greater importance/severity than rc.
- *          The most important ones are: VINF_EM_RESCHEDULE,
- *          VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   rc          The current rc.
- *
- */
-int emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
-{
-    STAM_REL_PROFILE_START(&pVCpu->em.s.StatForcedActions, a);
-#ifdef VBOX_STRICT
-    int rcIrq = VINF_SUCCESS;
-#endif
-    int rc2;
-#define UPDATE_RC() \
-        do { \
-            AssertMsg(rc2 <= 0 || (rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST), ("Invalid FF return code: %Rra\n", rc2)); \
-            if (rc2 == VINF_SUCCESS || rc < VINF_SUCCESS) \
-                break; \
-            if (!rc || rc2 < rc) \
-                rc = rc2; \
-        } while (0)
-
-    /*
-     * Post execution chunk first.
-     */
-    if (    VM_FF_ISPENDING(pVM, VM_FF_NORMAL_PRIORITY_POST_MASK)
-        ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_POST_MASK))
-    {
-        /*
-         * EMT Rendezvous (must be serviced before termination).
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
-        {
-            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
-            UPDATE_RC();
-            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
-             * stopped/reset before the next VM state change is made. We need a better
-             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
-             * && rc >= VINF_EM_SUSPEND). */
-            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
-            {
-                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
-                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                return rc;
-            }
-        }
-
-        /*
-         * State change request (cleared by vmR3SetStateLocked).
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_CHECK_VM_STATE))
-        {
-            VMSTATE enmState = VMR3GetState(pVM);
-            switch (enmState)
-            {
-                case VMSTATE_FATAL_ERROR:
-                case VMSTATE_FATAL_ERROR_LS:
-                    Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
-                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                    return VINF_EM_SUSPEND;
-
-                case VMSTATE_DESTROYING:
-                    Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
-                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                    return VINF_EM_TERMINATE;
-
-                default:
-                    AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
-            }
-        }
-
-        /*
-         * Debugger Facility polling.
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_DBGF))
-        {
-            rc2 = DBGFR3VMMForcedAction(pVM);
-            UPDATE_RC();
-        }
-
-        /*
-         * Postponed reset request.
-         */
-        if (VM_FF_TESTANDCLEAR(pVM, VM_FF_RESET))
-        {
-            rc2 = VMR3Reset(pVM);
-            UPDATE_RC();
-        }
-
-        /*
-         * CSAM page scanning.
-         */
-        if (    !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
-            &&  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE))
-        {
-            PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-
-            /** @todo: check for 16 or 32 bits code! (D bit in the code selector) */
-            Log(("Forced action VMCPU_FF_CSAM_SCAN_PAGE\n"));
-
-            CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
-            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE);
-        }
-
-        /*
-         * Out of memory? Putting this after CSAM as it may in theory cause us to run out of memory.
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
-        {
-            rc2 = PGMR3PhysAllocateHandyPages(pVM);
-            UPDATE_RC();
-            if (rc == VINF_EM_NO_MEMORY)
-                return rc;
-        }
-
-        /* check that we got them all  */
-        AssertCompile(VM_FF_NORMAL_PRIORITY_POST_MASK == (VM_FF_CHECK_VM_STATE | VM_FF_DBGF | VM_FF_RESET | VM_FF_PGM_NO_MEMORY | VM_FF_EMT_RENDEZVOUS));
-        AssertCompile(VMCPU_FF_NORMAL_PRIORITY_POST_MASK == VMCPU_FF_CSAM_SCAN_PAGE);
-    }
-
-    /*
-     * Normal priority then.
-     * (Executed in no particular order.)
-     */
-    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_NORMAL_PRIORITY_MASK, VM_FF_PGM_NO_MEMORY))
-    {
-        /*
-         * PDM Queues are pending.
-         */
-        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_QUEUES, VM_FF_PGM_NO_MEMORY))
-            PDMR3QueueFlushAll(pVM);
-
-        /*
-         * PDM DMA transfers are pending.
-         */
-        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_DMA, VM_FF_PGM_NO_MEMORY))
-            PDMR3DmaRun(pVM);
-
-        /*
-         * EMT Rendezvous (make sure they are handled before the requests).
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
-        {
-            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
-            UPDATE_RC();
-            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
-             * stopped/reset before the next VM state change is made. We need a better
-             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
-             * && rc >= VINF_EM_SUSPEND). */
-            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
-            {
-                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
-                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                return rc;
-            }
-        }
-
-        /*
-         * Requests from other threads.
-         */
-        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REQUEST, VM_FF_PGM_NO_MEMORY))
-        {
-            rc2 = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
-            if (rc2 == VINF_EM_OFF || rc2 == VINF_EM_TERMINATE) /** @todo this shouldn't be necessary */
-            {
-                Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
-                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                return rc2;
-            }
-            UPDATE_RC();
-            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
-             * stopped/reset before the next VM state change is made. We need a better
-             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
-             * && rc >= VINF_EM_SUSPEND). */
-            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
-            {
-                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
-                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                return rc;
-            }
-        }
-
-        /* Replay the handler notification changes. */
-        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REM_HANDLER_NOTIFY, VM_FF_PGM_NO_MEMORY))
-        {
-            /* Try not to cause deadlocks. */
-            if (    pVM->cCpus == 1
-                ||  (   !PGMIsLockOwner(pVM)
-                     && !IOMIsLockOwner(pVM))
-               )
-            {
-                EMRemLock(pVM);
-                REMR3ReplayHandlerNotifications(pVM);
-                EMRemUnlock(pVM);
-            }
-        }
-
-        /* check that we got them all  */
-        AssertCompile(VM_FF_NORMAL_PRIORITY_MASK == (VM_FF_REQUEST | VM_FF_PDM_QUEUES | VM_FF_PDM_DMA | VM_FF_REM_HANDLER_NOTIFY | VM_FF_EMT_RENDEZVOUS));
-    }
-
-    /*
-     * Normal priority then. (per-VCPU)
-     * (Executed in no particular order.)
-     */
-    if (    !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
-        &&  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_MASK))
-    {
-        /*
-         * Requests from other threads.
-         */
-        if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
-        {
-            rc2 = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu);
-            if (rc2 == VINF_EM_OFF || rc2 == VINF_EM_TERMINATE || rc2 == VINF_EM_RESET)
-            {
-                Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
-                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                return rc2;
-            }
-            UPDATE_RC();
-            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
-             * stopped/reset before the next VM state change is made. We need a better
-             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
-             * && rc >= VINF_EM_SUSPEND). */
-            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
-            {
-                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
-                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                return rc;
-            }
-        }
-
-        /* check that we got them all  */
-        Assert(!(VMCPU_FF_NORMAL_PRIORITY_MASK & ~(VMCPU_FF_REQUEST)));
-    }
-
-    /*
-     * High priority pre execution chunk last.
-     * (Executed in ascending priority order.)
-     */
-    if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_MASK)
-        ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_MASK))
-    {
-        /*
-         * Timers before interrupts.
-         */
-        if (    VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TIMER)
-            &&  !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
-            TMR3TimerQueuesDo(pVM);
-
-        /*
-         * The instruction following an emulated STI should *always* be executed!
-         */
-        if (    VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
-            &&  !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
-        {
-            Log(("VM_FF_EMULATED_STI at %RGv successor %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu), EMGetInhibitInterruptsPC(pVCpu)));
-            if (CPUMGetGuestEIP(pVCpu) != EMGetInhibitInterruptsPC(pVCpu))
-            {
-                /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here if the eip is the same as the inhibited instr address.
-                 *  Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
-                 *  force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
-                 *  break the guest. Sounds very unlikely, but such timing sensitive problem are not as rare as you might think.
-                 */
-                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
-            }
-            if (HWACCMR3IsActive(pVCpu))
-                rc2 = VINF_EM_RESCHEDULE_HWACC;
-            else
-                rc2 = PATMAreInterruptsEnabled(pVM) ? VINF_EM_RESCHEDULE_RAW : VINF_EM_RESCHEDULE_REM;
-
-            UPDATE_RC();
-        }
-
-        /*
-         * Interrupts.
-         */
-        if (    !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
-            &&  !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
-            &&  (!rc || rc >= VINF_EM_RESCHEDULE_HWACC)
-            &&  !TRPMHasTrap(pVCpu) /* an interrupt could already be scheduled for dispatching in the recompiler. */
-            &&  PATMAreInterruptsEnabled(pVM)
-            &&  !HWACCMR3IsEventPending(pVCpu))
-        {
-            Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
-            if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
-            {
-                /* Note: it's important to make sure the return code from TRPMR3InjectEvent isn't ignored! */
-                /** @todo this really isn't nice, should properly handle this */
-                rc2 = TRPMR3InjectEvent(pVM, pVCpu, TRPM_HARDWARE_INT);
-#ifdef VBOX_STRICT
-                rcIrq = rc2;
-#endif
-                UPDATE_RC();
-            }
-            /** @todo really ugly; if we entered the hlt state when exiting the recompiler and an interrupt was pending, we previously got stuck in the halted state. */
-            else if (REMR3QueryPendingInterrupt(pVM, pVCpu) != REM_NO_PENDING_IRQ)
-            {
-                rc2 = VINF_EM_RESCHEDULE_REM;
-                UPDATE_RC();
-            }
-        }
-
-        /*
-         * Allocate handy pages.
-         */
-        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
-        {
-            rc2 = PGMR3PhysAllocateHandyPages(pVM);
-            UPDATE_RC();
-        }
-
-        /*
-         * Debugger Facility request.
-         */
-        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_DBGF, VM_FF_PGM_NO_MEMORY))
-        {
-            rc2 = DBGFR3VMMForcedAction(pVM);
-            UPDATE_RC();
-        }
-
-        /*
-         * EMT Rendezvous (must be serviced before termination).
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
-        {
-            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
-            UPDATE_RC();
-            /** @todo HACK ALERT! The following test is to make sure EM+TM thinks the VM is
-             * stopped/reset before the next VM state change is made. We need a better
-             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
-             * && rc >= VINF_EM_SUSPEND). */
-            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
-            {
-                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
-                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                return rc;
-            }
-        }
-
-        /*
-         * State change request (cleared by vmR3SetStateLocked).
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_CHECK_VM_STATE))
-        {
-            VMSTATE enmState = VMR3GetState(pVM);
-            switch (enmState)
-            {
-                case VMSTATE_FATAL_ERROR:
-                case VMSTATE_FATAL_ERROR_LS:
-                    Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
-                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                    return VINF_EM_SUSPEND;
-
-                case VMSTATE_DESTROYING:
-                    Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
-                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-                    return VINF_EM_TERMINATE;
-
-                default:
-                    AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
-            }
-        }
-
-        /*
-         * Out of memory? Since most of our fellow high priority actions may cause us
-         * to run out of memory, we're employing VM_FF_IS_PENDING_EXCEPT and putting this
-         * at the end rather than the start. Also, VM_FF_TERMINATE has higher priority
-         * than us since we can terminate without allocating more memory.
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
-        {
-            rc2 = PGMR3PhysAllocateHandyPages(pVM);
-            UPDATE_RC();
-            if (rc == VINF_EM_NO_MEMORY)
-                return rc;
-        }
-
-        /*
-         * If the virtual sync clock is still stopped, make TM restart it.
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_TM_VIRTUAL_SYNC))
-            TMR3VirtualSyncFF(pVM, pVCpu);
-
-#ifdef DEBUG
-        /*
-         * Debug, pause the VM.
-         */
-        if (VM_FF_ISPENDING(pVM, VM_FF_DEBUG_SUSPEND))
-        {
-            VM_FF_CLEAR(pVM, VM_FF_DEBUG_SUSPEND);
-            Log(("emR3ForcedActions: returns VINF_EM_SUSPEND\n"));
-            return VINF_EM_SUSPEND;
-        }
-#endif
-
-        /* check that we got them all  */
-        AssertCompile(VM_FF_HIGH_PRIORITY_PRE_MASK == (VM_FF_TM_VIRTUAL_SYNC | VM_FF_DBGF | VM_FF_CHECK_VM_STATE | VM_FF_DEBUG_SUSPEND | VM_FF_PGM_NEED_HANDY_PAGES | VM_FF_PGM_NO_MEMORY | VM_FF_EMT_RENDEZVOUS));
-        AssertCompile(VMCPU_FF_HIGH_PRIORITY_PRE_MASK == (VMCPU_FF_TIMER | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_SELM_SYNC_TSS | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_INHIBIT_INTERRUPTS));
-    }
-
-#undef UPDATE_RC
-    Log2(("emR3ForcedActions: returns %Rrc\n", rc));
-    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
-    Assert(rcIrq == VINF_SUCCESS || rcIrq == rc);
-    return rc;
-}
-
-
-/**
- * Check if the preset execution time cap restricts guest execution scheduling.
- *
- * @returns true if allowed, false otherwise
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VMCPU to operate on.
- *
- */
-VMMR3DECL(bool) EMR3IsExecutionAllowed(PVM pVM, PVMCPU pVCpu)
-{
-    uint64_t u64UserTime, u64KernelTime;
-
-    if (    pVM->uCpuExecutionCap != 100
-        &&  RT_SUCCESS(RTThreadGetExecutionTimeMilli(&u64KernelTime, &u64UserTime)))
-    {
-        uint64_t u64TimeNow = RTTimeMilliTS();
-        if (pVCpu->em.s.u64TimeSliceStart + EM_TIME_SLICE < u64TimeNow)
-        {
-            /* New time slice. */
-            pVCpu->em.s.u64TimeSliceStart     = u64TimeNow;
-            pVCpu->em.s.u64TimeSliceStartExec = u64KernelTime + u64UserTime;
-            pVCpu->em.s.u64TimeSliceExec      = 0;
-        }
-        pVCpu->em.s.u64TimeSliceExec = u64KernelTime + u64UserTime - pVCpu->em.s.u64TimeSliceStartExec;
-
-        Log2(("emR3IsExecutionAllowed: start=%RX64 startexec=%RX64 exec=%RX64 (cap=%x)\n", pVCpu->em.s.u64TimeSliceStart, pVCpu->em.s.u64TimeSliceStartExec, pVCpu->em.s.u64TimeSliceExec, (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100));
-        if (pVCpu->em.s.u64TimeSliceExec >= (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100)
-            return false;
-    }
-    return true;
-}
-
-
-/**
- * Execute VM.
- *
- * This function is the main loop of the VM. The emulation thread
- * calls this function when the VM has been successfully constructed
- * and we're ready for executing the VM.
- *
- * Returning from this function means that the VM is turned off or
- * suspended (state already saved) and deconstruction in next in line.
- *
- * All interaction from other thread are done using forced actions
- * and signaling of the wait object.
- *
- * @returns VBox status code, informational status codes may indicate failure.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VMCPU to operate on.
- */
-VMMR3DECL(int) EMR3ExecuteVM(PVM pVM, PVMCPU pVCpu)
-{
-    Log(("EMR3ExecuteVM: pVM=%p enmVMState=%d (%s)  enmState=%d (%s) enmPrevState=%d (%s) fForceRAW=%RTbool\n",
-         pVM,
-         pVM->enmVMState,          VMR3GetStateName(pVM->enmVMState),
-         pVCpu->em.s.enmState,     emR3GetStateName(pVCpu->em.s.enmState),
-         pVCpu->em.s.enmPrevState, emR3GetStateName(pVCpu->em.s.enmPrevState),
-         pVCpu->em.s.fForceRAW));
-    VM_ASSERT_EMT(pVM);
-    AssertMsg(   pVCpu->em.s.enmState == EMSTATE_NONE
-              || pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI
-              || pVCpu->em.s.enmState == EMSTATE_SUSPENDED,
-              ("%s\n", emR3GetStateName(pVCpu->em.s.enmState)));
-
-    int rc = setjmp(pVCpu->em.s.u.FatalLongJump);
-    if (rc == 0)
-    {
-        /*
-         * Start the virtual time.
-         */
-        TMR3NotifyResume(pVM, pVCpu);
-
-        /*
-         * The Outer Main Loop.
-         */
-        bool fFFDone = false;
-
-        /* Reschedule right away to start in the right state. */
-        rc = VINF_SUCCESS;
-
-        /* If resuming after a pause or a state load, restore the previous
-           state or else we'll start executing code. Else, just reschedule. */
-        if (    pVCpu->em.s.enmState == EMSTATE_SUSPENDED
-            &&  (   pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
-                 || pVCpu->em.s.enmPrevState == EMSTATE_HALTED))
-            pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
-        else
-            pVCpu->em.s.enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
-
-        STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
-        for (;;)
-        {
-            /*
-             * Before we can schedule anything (we're here because
-             * scheduling is required) we must service any pending
-             * forced actions to avoid any pending action causing
-             * immediate rescheduling upon entering an inner loop
-             *
-             * Do forced actions.
-             */
-            if (   !fFFDone
-                && rc != VINF_EM_TERMINATE
-                && rc != VINF_EM_OFF
-                && (   VM_FF_ISPENDING(pVM, VM_FF_ALL_BUT_RAW_MASK)
-                    || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_BUT_RAW_MASK)))
-            {
-                rc = emR3ForcedActions(pVM, pVCpu, rc);
-                if (    (   rc == VINF_EM_RESCHEDULE_REM
-                         || rc == VINF_EM_RESCHEDULE_HWACC)
-                    &&  pVCpu->em.s.fForceRAW)
-                    rc = VINF_EM_RESCHEDULE_RAW;
-            }
-            else if (fFFDone)
-                fFFDone = false;
-
-            /*
-             * Now what to do?
-             */
-            Log2(("EMR3ExecuteVM: rc=%Rrc\n", rc));
-            switch (rc)
-            {
-                /*
-                 * Keep doing what we're currently doing.
-                 */
-                case VINF_SUCCESS:
-                    break;
-
-                /*
-                 * Reschedule - to raw-mode execution.
-                 */
-                case VINF_EM_RESCHEDULE_RAW:
-                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_RAW: %d -> %d (EMSTATE_RAW)\n", pVCpu->em.s.enmState, EMSTATE_RAW));
-                    pVCpu->em.s.enmState = EMSTATE_RAW;
-                    break;
-
-                /*
-                 * Reschedule - to hardware accelerated raw-mode execution.
-                 */
-                case VINF_EM_RESCHEDULE_HWACC:
-                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_HWACC: %d -> %d (EMSTATE_HWACC)\n", pVCpu->em.s.enmState, EMSTATE_HWACC));
-                    Assert(!pVCpu->em.s.fForceRAW);
-                    pVCpu->em.s.enmState = EMSTATE_HWACC;
-                    break;
-
-                /*
-                 * Reschedule - to recompiled execution.
-                 */
-                case VINF_EM_RESCHEDULE_REM:
-                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_REM: %d -> %d (EMSTATE_REM)\n", pVCpu->em.s.enmState, EMSTATE_REM));
-                    pVCpu->em.s.enmState = EMSTATE_REM;
-                    break;
-
-                /*
-                 * Resume.
-                 */
-                case VINF_EM_RESUME:
-                    Log2(("EMR3ExecuteVM: VINF_EM_RESUME: %d -> VINF_EM_RESCHEDULE\n", pVCpu->em.s.enmState));
-                    /* Don't reschedule in the halted or wait for SIPI case. */
-                    if (    pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
-                        ||  pVCpu->em.s.enmPrevState == EMSTATE_HALTED)
-                        break;
-                    /* fall through and get scheduled. */
-
-                /*
-                 * Reschedule.
-                 */
-                case VINF_EM_RESCHEDULE:
-                {
-                    EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
-                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE: %d -> %d (%s)\n", pVCpu->em.s.enmState, enmState, emR3GetStateName(enmState)));
-                    pVCpu->em.s.enmState = enmState;
-                    break;
-                }
-
-                /*
-                 * Halted.
-                 */
-                case VINF_EM_HALT:
-                    Log2(("EMR3ExecuteVM: VINF_EM_HALT: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_HALTED));
-                    pVCpu->em.s.enmState = EMSTATE_HALTED;
-                    break;
-
-                /*
-                 * Switch to the wait for SIPI state (application processor only)
-                 */
-                case VINF_EM_WAIT_SIPI:
-                    Assert(pVCpu->idCpu != 0);
-                    Log2(("EMR3ExecuteVM: VINF_EM_WAIT_SIPI: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_WAIT_SIPI));
-                    pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
-                    break;
-
-
-                /*
-                 * Suspend.
-                 */
-                case VINF_EM_SUSPEND:
-                    Log2(("EMR3ExecuteVM: VINF_EM_SUSPEND: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_SUSPENDED));
-                    pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
-                    pVCpu->em.s.enmState     = EMSTATE_SUSPENDED;
-                    break;
-
-                /*
-                 * Reset.
-                 * We might end up doing a double reset for now, we'll have to clean up the mess later.
-                 */
-                case VINF_EM_RESET:
-                {
-                    if (pVCpu->idCpu == 0)
-                    {
-                        EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
-                        Log2(("EMR3ExecuteVM: VINF_EM_RESET: %d -> %d (%s)\n", pVCpu->em.s.enmState, enmState, emR3GetStateName(enmState)));
-                        pVCpu->em.s.enmState = enmState;
-                    }
-                    else
-                    {
-                        /* All other VCPUs go into the wait for SIPI state. */
-                        pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
-                    }
-                    break;
-                }
-
-                /*
-                 * Power Off.
-                 */
-                case VINF_EM_OFF:
-                    pVCpu->em.s.enmState = EMSTATE_TERMINATING;
-                    Log2(("EMR3ExecuteVM: returns VINF_EM_OFF (%d -> %d)\n", pVCpu->em.s.enmState, EMSTATE_TERMINATING));
-                    TMR3NotifySuspend(pVM, pVCpu);
-                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-                    return rc;
-
-                /*
-                 * Terminate the VM.
-                 */
-                case VINF_EM_TERMINATE:
-                    pVCpu->em.s.enmState = EMSTATE_TERMINATING;
-                    Log(("EMR3ExecuteVM returns VINF_EM_TERMINATE (%d -> %d)\n", pVCpu->em.s.enmState, EMSTATE_TERMINATING));
-                    if (pVM->enmVMState < VMSTATE_DESTROYING) /* ugly */
-                        TMR3NotifySuspend(pVM, pVCpu);
-                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-                    return rc;
-
-
-                /*
-                 * Out of memory, suspend the VM and stuff.
-                 */
-                case VINF_EM_NO_MEMORY:
-                    Log2(("EMR3ExecuteVM: VINF_EM_NO_MEMORY: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_SUSPENDED));
-                    pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
-                    pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
-                    TMR3NotifySuspend(pVM, pVCpu);
-                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-
-                    rc = VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_SUSPEND, "HostMemoryLow",
-                                           N_("Unable to allocate and lock memory. The virtual machine will be paused. Please close applications to free up memory or close the VM"));
-                    if (rc != VINF_EM_SUSPEND)
-                    {
-                        if (RT_SUCCESS_NP(rc))
-                        {
-                            AssertLogRelMsgFailed(("%Rrc\n", rc));
-                            rc = VERR_EM_INTERNAL_ERROR;
-                        }
-                        pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
-                    }
-                    return rc;
-
-                /*
-                 * Guest debug events.
-                 */
-                case VINF_EM_DBG_STEPPED:
-                    AssertMsgFailed(("VINF_EM_DBG_STEPPED cannot be here!"));
-                case VINF_EM_DBG_STOP:
-                case VINF_EM_DBG_BREAKPOINT:
-                case VINF_EM_DBG_STEP:
-                    if (pVCpu->em.s.enmState == EMSTATE_RAW)
-                    {
-                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_GUEST_RAW));
-                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RAW;
-                    }
-                    else
-                    {
-                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_GUEST_REM));
-                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
-                    }
-                    break;
-
-                /*
-                 * Hypervisor debug events.
-                 */
-                case VINF_EM_DBG_HYPER_STEPPED:
-                case VINF_EM_DBG_HYPER_BREAKPOINT:
-                case VINF_EM_DBG_HYPER_ASSERTION:
-                    Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_HYPER));
-                    pVCpu->em.s.enmState = EMSTATE_DEBUG_HYPER;
-                    break;
-
-                /*
-                 * Guru mediations.
-                 */
-                case VERR_VMM_RING0_ASSERTION:
-                    Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, pVCpu->em.s.enmState, EMSTATE_GURU_MEDITATION));
-                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
-                    break;
-
-                /*
-                 * Any error code showing up here other than the ones we
-                 * know and process above are considered to be FATAL.
-                 *
-                 * Unknown warnings and informational status codes are also
-                 * included in this.
-                 */
-                default:
-                    if (RT_SUCCESS_NP(rc))
-                    {
-                        AssertMsgFailed(("Unexpected warning or informational status code %Rra!\n", rc));
-                        rc = VERR_EM_INTERNAL_ERROR;
-                    }
-                    Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, pVCpu->em.s.enmState, EMSTATE_GURU_MEDITATION));
-                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
-                    break;
-            }
-
-            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x); /* (skip this in release) */
-            STAM_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
-
-            /*
-             * Act on the state.
-             */
-            switch (pVCpu->em.s.enmState)
-            {
-                /*
-                 * Execute raw.
-                 */
-                case EMSTATE_RAW:
-                    rc = emR3RawExecute(pVM, pVCpu, &fFFDone);
-                    break;
-
-                /*
-                 * Execute hardware accelerated raw.
-                 */
-                case EMSTATE_HWACC:
-                    rc = emR3HwAccExecute(pVM, pVCpu, &fFFDone);
-                    break;
-
-                /*
-                 * Execute recompiled.
-                 */
-                case EMSTATE_REM:
-                    rc = emR3RemExecute(pVM, pVCpu, &fFFDone);
-                    Log2(("EMR3ExecuteVM: emR3RemExecute -> %Rrc\n", rc));
-                    break;
-
-                /*
-                 * Application processor execution halted until SIPI.
-                 */
-                case EMSTATE_WAIT_SIPI:
-                    /* no break */
-                /*
-                 * hlt - execution halted until interrupt.
-                 */
-                case EMSTATE_HALTED:
-                {
-                    STAM_REL_PROFILE_START(&pVCpu->em.s.StatHalted, y);
-                    if (pVCpu->em.s.mwait.fWait & EMMWAIT_FLAG_ACTIVE)
-                    {
-                        /* mwait has a special extension where it's woken up when an interrupt is pending even when IF=0. */
-                        rc = VMR3WaitHalted(pVM, pVCpu, !(pVCpu->em.s.mwait.fWait & EMMWAIT_FLAG_BREAKIRQIF0) && !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF));
-                        pVCpu->em.s.mwait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0);
-                    }
-                    else
-                        rc = VMR3WaitHalted(pVM, pVCpu, !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF));
-
-                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHalted, y);
-                    break;
-                }
-
-                /*
-                 * Suspended - return to VM.cpp.
-                 */
-                case EMSTATE_SUSPENDED:
-                    TMR3NotifySuspend(pVM, pVCpu);
-                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
-                    return VINF_EM_SUSPEND;
-
-                /*
-                 * Debugging in the guest.
-                 */
-                case EMSTATE_DEBUG_GUEST_REM:
-                case EMSTATE_DEBUG_GUEST_RAW:
-                    TMR3NotifySuspend(pVM, pVCpu);
-                    rc = emR3Debug(pVM, pVCpu, rc);
-                    TMR3NotifyResume(pVM, pVCpu);
-                    Log2(("EMR3ExecuteVM: enmr3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
-                    break;
-
-                /*
-                 * Debugging in the hypervisor.
-                 */
-                case EMSTATE_DEBUG_HYPER:
-                {
-                    TMR3NotifySuspend(pVM, pVCpu);
-                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-
-                    rc = emR3Debug(pVM, pVCpu, rc);
-                    Log2(("EMR3ExecuteVM: enmr3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
-                    if (rc != VINF_SUCCESS)
-                    {
-                        /* switch to guru meditation mode */
-                        pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
-                        VMMR3FatalDump(pVM, pVCpu, rc);
-                        Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
-                        return rc;
-                    }
-
-                    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
-                    TMR3NotifyResume(pVM, pVCpu);
-                    break;
-                }
-
-                /*
-                 * Guru meditation takes place in the debugger.
-                 */
-                case EMSTATE_GURU_MEDITATION:
-                {
-                    TMR3NotifySuspend(pVM, pVCpu);
-                    VMMR3FatalDump(pVM, pVCpu, rc);
-                    emR3Debug(pVM, pVCpu, rc);
-                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
-                    return rc;
-                }
-
-                /*
-                 * The states we don't expect here.
-                 */
-                case EMSTATE_NONE:
-                case EMSTATE_TERMINATING:
-                default:
-                    AssertMsgFailed(("EMR3ExecuteVM: Invalid state %d!\n", pVCpu->em.s.enmState));
-                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
-                    TMR3NotifySuspend(pVM, pVCpu);
-                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
-                    return VERR_EM_INTERNAL_ERROR;
-            }
-        } /* The Outer Main Loop */
-    }
-    else
-    {
-        /*
-         * Fatal error.
-         */
-        Log(("EMR3ExecuteVM: returns %Rrc because of longjmp / fatal error; (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
-        TMR3NotifySuspend(pVM, pVCpu);
-        VMMR3FatalDump(pVM, pVCpu, rc);
-        emR3Debug(pVM, pVCpu, rc);
-        STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
-        /** @todo change the VM state! */
-        return rc;
-    }
-
-    /* (won't ever get here). */
-    AssertFailed();
-}
-
-/**
- * Notify EM of a state change (used by FTM)
- *
- * @param   pVM             VM Handle.
- */
-VMMR3DECL(int) EMR3NotifySuspend(PVM pVM)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    TMR3NotifySuspend(pVM, pVCpu);  /* Stop the virtual time. */
-    pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
-    pVCpu->em.s.enmState     = EMSTATE_SUSPENDED;
-    return VINF_SUCCESS;
-}
-
-/**
- * Notify EM of a state change (used by FTM)
- *
- * @param   pVM             VM Handle.
- */
-VMMR3DECL(int) EMR3NotifyResume(PVM pVM)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    EMSTATE enmCurState = pVCpu->em.s.enmState;
-
-    TMR3NotifyResume(pVM, pVCpu);  /* Resume the virtual time. */
-    pVCpu->em.s.enmState     = pVCpu->em.s.enmPrevState;
-    pVCpu->em.s.enmPrevState = enmCurState;
-    return VINF_SUCCESS;
-}
Index: unk/src/VBox/VMM/EMHandleRCTmpl.h
===================================================================
--- /trunk/src/VBox/VMM/EMHandleRCTmpl.h	(revision 35332)
+++ 	(revision )
@@ -1,359 +1,0 @@
-/* $Id$ */
-/** @file
- * EM - emR3[Raw|Hwaccm]HandleRC template.
- */
-
-/*
- * Copyright (C) 2006-2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___EMHandleRCTmpl_h
-#define ___EMHandleRCTmpl_h
-
-/**
- * Process a subset of the raw-mode return code.
- *
- * Since we have to share this with raw-mode single stepping, this inline
- * function has been created to avoid code duplication.
- *
- * @returns VINF_SUCCESS if it's ok to continue raw mode.
- * @returns VBox status code to return to the EM main loop.
- *
- * @param   pVM     The VM handle
- * @param   pVCpu   The VMCPU handle
- * @param   rc      The return code.
- * @param   pCtx    The guest cpu context.
- */
-#ifdef EMHANDLERC_WITH_PATM
-int emR3RawHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
-#elif defined(EMHANDLERC_WITH_HWACCM)
-int emR3HwaccmHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
-#endif
-{
-    switch (rc)
-    {
-        /*
-         * Common & simple ones.
-         */
-        case VINF_SUCCESS:
-            break;
-        case VINF_EM_RESCHEDULE_RAW:
-        case VINF_EM_RESCHEDULE_HWACC:
-        case VINF_EM_RAW_INTERRUPT:
-        case VINF_EM_RAW_TO_R3:
-        case VINF_EM_RAW_TIMER_PENDING:
-        case VINF_EM_PENDING_REQUEST:
-            rc = VINF_SUCCESS;
-            break;
-
-#ifdef EMHANDLERC_WITH_PATM
-        /*
-         * Privileged instruction.
-         */
-        case VINF_EM_RAW_EXCEPTION_PRIVILEGED:
-        case VINF_PATM_PATCH_TRAP_GP:
-            rc = emR3RawPrivileged(pVM, pVCpu);
-            break;
-
-        case VINF_EM_RAW_GUEST_TRAP:
-            /*
-             * Got a trap which needs dispatching.
-             */
-            if (PATMR3IsInsidePatchJump(pVM, pCtx->eip, NULL))
-            {
-                AssertReleaseMsgFailed(("FATAL ERROR: executing random instruction inside generated patch jump %08X\n", CPUMGetGuestEIP(pVCpu)));
-                rc = VERR_EM_RAW_PATCH_CONFLICT;
-                break;
-            }
-            rc = emR3RawGuestTrap(pVM, pVCpu);
-            break;
-
-        /*
-         * Trap in patch code.
-         */
-        case VINF_PATM_PATCH_TRAP_PF:
-        case VINF_PATM_PATCH_INT3:
-            rc = emR3PatchTrap(pVM, pVCpu, pCtx, rc);
-            break;
-
-        case VINF_PATM_DUPLICATE_FUNCTION:
-            Assert(PATMIsPatchGCAddr(pVM, pCtx->eip));
-            rc = PATMR3DuplicateFunctionRequest(pVM, pCtx);
-            AssertRC(rc);
-            rc = VINF_SUCCESS;
-            break;
-
-        case VINF_PATM_CHECK_PATCH_PAGE:
-            rc = PATMR3HandleMonitoredPage(pVM);
-            AssertRC(rc);
-            rc = VINF_SUCCESS;
-            break;
-
-        /*
-         * Patch manager.
-         */
-        case VERR_EM_RAW_PATCH_CONFLICT:
-            AssertReleaseMsgFailed(("%Rrc handling is not yet implemented\n", rc));
-            break;
-#endif /* EMHANDLERC_WITH_PATM */
-
-#ifdef EMHANDLERC_WITH_PATM
-        /*
-         * Memory mapped I/O access - attempt to patch the instruction
-         */
-        case VINF_PATM_HC_MMIO_PATCH_READ:
-            rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
-                                    PATMFL_MMIO_ACCESS | ((SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0));
-            if (RT_FAILURE(rc))
-                rc = emR3ExecuteInstruction(pVM, pVCpu, "MMIO");
-            break;
-
-        case VINF_PATM_HC_MMIO_PATCH_WRITE:
-            AssertFailed(); /* not yet implemented. */
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "MMIO");
-            break;
-#endif /* EMHANDLERC_WITH_PATM */
-
-        /*
-         * Conflict or out of page tables.
-         *
-         * VM_FF_PGM_SYNC_CR3 is set by the hypervisor and all we need to
-         * do here is to execute the pending forced actions.
-         */
-        case VINF_PGM_SYNC_CR3:
-            AssertMsg(VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL),
-                      ("VINF_PGM_SYNC_CR3 and no VMCPU_FF_PGM_SYNC_CR3*!\n"));
-            rc = VINF_SUCCESS;
-            break;
-
-        /*
-         * PGM pool flush pending (guest SMP only).
-         */
-        /** @todo jumping back and forth between ring 0 and 3 can burn a lot of cycles
-         * if the EMT thread that's supposed to handle the flush is currently not active
-         * (e.g. waiting to be scheduled) -> fix this properly!
-         *
-         * bird: Since the clearing is global and done via a rendezvous any CPU can do
-         *       it. They would have to choose who to call VMMR3EmtRendezvous and send
-         *       the rest to VMMR3EmtRendezvousFF ... Hmm ... that's not going to work
-         *       all that well since the the latter will race the setup done by the
-         *       first.  Guess that means we need some new magic in that area for
-         *       handling this case. :/
-         */
-        case VINF_PGM_POOL_FLUSH_PENDING:
-            rc = VINF_SUCCESS;
-            break;
-
-        /*
-         * Paging mode change.
-         */
-        case VINF_PGM_CHANGE_MODE:
-            rc = PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
-            if (rc == VINF_SUCCESS)
-                rc = VINF_EM_RESCHEDULE;
-            AssertMsg(RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST), ("%Rrc\n", rc));
-            break;
-
-#ifdef EMHANDLERC_WITH_PATM
-        /*
-         * CSAM wants to perform a task in ring-3. It has set an FF action flag.
-         */
-        case VINF_CSAM_PENDING_ACTION:
-            rc = VINF_SUCCESS;
-            break;
-
-        /*
-         * Invoked Interrupt gate - must directly (!) go to the recompiler.
-         */
-        case VINF_EM_RAW_INTERRUPT_PENDING:
-        case VINF_EM_RAW_RING_SWITCH_INT:
-            Assert(TRPMHasTrap(pVCpu));
-            Assert(!PATMIsPatchGCAddr(pVM, pCtx->eip));
-
-            if (TRPMHasTrap(pVCpu))
-            {
-                /* If the guest gate is marked unpatched, then we will check again if we can patch it. */
-                uint8_t u8Interrupt = TRPMGetTrapNo(pVCpu);
-                if (TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) == TRPM_INVALID_HANDLER)
-                {
-                    CSAMR3CheckGates(pVM, u8Interrupt, 1);
-                    Log(("emR3RawHandleRC: recheck gate %x -> valid=%d\n", u8Interrupt, TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) != TRPM_INVALID_HANDLER));
-                    /* Note: If it was successful, then we could go back to raw mode, but let's keep things simple for now. */
-                }
-            }
-            rc = VINF_EM_RESCHEDULE_REM;
-            break;
-
-        /*
-         * Other ring switch types.
-         */
-        case VINF_EM_RAW_RING_SWITCH:
-            rc = emR3RawRingSwitch(pVM, pVCpu);
-            break;
-#endif  /* EMHANDLERC_WITH_PATM */
-
-        /*
-         * I/O Port access - emulate the instruction.
-         */
-        case VINF_IOM_HC_IOPORT_READ:
-        case VINF_IOM_HC_IOPORT_WRITE:
-            rc = emR3ExecuteIOInstruction(pVM, pVCpu);
-            break;
-
-        /*
-         * Memory mapped I/O access - emulate the instruction.
-         */
-        case VINF_IOM_HC_MMIO_READ:
-        case VINF_IOM_HC_MMIO_WRITE:
-        case VINF_IOM_HC_MMIO_READ_WRITE:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "MMIO");
-            break;
-
-#ifdef EMHANDLERC_WITH_HWACCM
-        /*
-         * (MM)IO intensive code block detected; fall back to the recompiler for better performance
-         */
-        case VINF_EM_RAW_EMULATE_IO_BLOCK:
-            rc = HWACCMR3EmulateIoBlock(pVM, pCtx);
-            break;
-
-        case VINF_EM_HWACCM_PATCH_TPR_INSTR:
-            rc = HWACCMR3PatchTprInstr(pVM, pVCpu, pCtx);
-            break;
-#endif
-
-#ifdef EMHANDLERC_WITH_PATM
-        /*
-         * Execute instruction.
-         */
-        case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "LDT FAULT: ");
-            break;
-        case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "GDT FAULT: ");
-            break;
-        case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "IDT FAULT: ");
-            break;
-        case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "TSS FAULT: ");
-            break;
-        case VINF_EM_RAW_EMULATE_INSTR_PD_FAULT:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "PD FAULT: ");
-            break;
-        case VINF_EM_RAW_EMULATE_INSTR_HLT:
-            /** @todo skip instruction and go directly to the halt state. (see REM for implementation details) */
-            rc = emR3RawPrivileged(pVM, pVCpu);
-            break;
-#endif
-
-#ifdef EMHANDLERC_WITH_PATM
-        case VINF_PATM_PENDING_IRQ_AFTER_IRET:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "EMUL: ", VINF_PATM_PENDING_IRQ_AFTER_IRET);
-            break;
-
-        case VINF_PATCH_EMULATE_INSTR:
-#else
-        case VINF_EM_RAW_GUEST_TRAP:
-#endif
-        case VINF_EM_RAW_EMULATE_INSTR:
-            rc = emR3ExecuteInstruction(pVM, pVCpu, "EMUL: ");
-            break;
-
-#ifdef EMHANDLERC_WITH_PATM
-        /*
-         * Stale selector and iret traps => REM.
-         */
-        case VINF_EM_RAW_STALE_SELECTOR:
-        case VINF_EM_RAW_IRET_TRAP:
-            /* We will not go to the recompiler if EIP points to patch code. */
-            if (PATMIsPatchGCAddr(pVM, pCtx->eip))
-            {
-                pCtx->eip = PATMR3PatchToGCPtr(pVM, (RTGCPTR)pCtx->eip, 0);
-            }
-            LogFlow(("emR3RawHandleRC: %Rrc -> %Rrc\n", rc, VINF_EM_RESCHEDULE_REM));
-            rc = VINF_EM_RESCHEDULE_REM;
-            break;
-#endif
-
-        /*
-         * Up a level.
-         */
-        case VINF_EM_TERMINATE:
-        case VINF_EM_OFF:
-        case VINF_EM_RESET:
-        case VINF_EM_SUSPEND:
-        case VINF_EM_HALT:
-        case VINF_EM_RESUME:
-        case VINF_EM_NO_MEMORY:
-        case VINF_EM_RESCHEDULE:
-        case VINF_EM_RESCHEDULE_REM:
-        case VINF_EM_WAIT_SIPI:
-            break;
-
-        /*
-         * Up a level and invoke the debugger.
-         */
-        case VINF_EM_DBG_STEPPED:
-        case VINF_EM_DBG_BREAKPOINT:
-        case VINF_EM_DBG_STEP:
-        case VINF_EM_DBG_HYPER_BREAKPOINT:
-        case VINF_EM_DBG_HYPER_STEPPED:
-        case VINF_EM_DBG_HYPER_ASSERTION:
-        case VINF_EM_DBG_STOP:
-            break;
-
-        /*
-         * Up a level, dump and debug.
-         */
-        case VERR_TRPM_DONT_PANIC:
-        case VERR_TRPM_PANIC:
-        case VERR_VMM_RING0_ASSERTION:
-        case VERR_VMM_HYPER_CR3_MISMATCH:
-        case VERR_VMM_RING3_CALL_DISABLED:
-            break;
-
-#ifdef EMHANDLERC_WITH_HWACCM
-        /*
-         * Up a level, after HwAccM have done some release logging.
-         */
-        case VERR_VMX_INVALID_VMCS_FIELD:
-        case VERR_VMX_INVALID_VMCS_PTR:
-        case VERR_VMX_INVALID_VMXON_PTR:
-        case VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE:
-        case VERR_VMX_UNEXPECTED_EXCEPTION:
-        case VERR_VMX_UNEXPECTED_EXIT_CODE:
-        case VERR_VMX_INVALID_GUEST_STATE:
-        case VERR_VMX_UNABLE_TO_START_VM:
-        case VERR_VMX_UNABLE_TO_RESUME_VM:
-            HWACCMR3CheckError(pVM, rc);
-            break;
-
-        /* Up a level; fatal */
-        case VERR_VMX_IN_VMX_ROOT_MODE:
-        case VERR_SVM_IN_USE:
-        case VERR_SVM_UNABLE_TO_START_VM:
-            break;
-#endif
-
-        /*
-         * Anything which is not known to us means an internal error
-         * and the termination of the VM!
-         */
-        default:
-            AssertMsgFailed(("Unknown GC return code: %Rra\n", rc));
-            break;
-    }
-    return rc;
-}
-
-#endif
-
Index: unk/src/VBox/VMM/EMHwaccm.cpp
===================================================================
--- /trunk/src/VBox/VMM/EMHwaccm.cpp	(revision 35332)
+++ 	(revision )
@@ -1,609 +1,0 @@
-/* $Id$ */
-/** @file
- * EM - Execution Monitor / Manager - hardware virtualization
- */
-
-/*
- * Copyright (C) 2006-2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_em         EM - The Execution Monitor / Manager
- *
- * The Execution Monitor/Manager is responsible for running the VM, scheduling
- * the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
- * Interpreted), and keeping the CPU states in sync. The function
- * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
- * modes has different inner loops (emR3RawExecute, emR3HwAccExecute, and
- * emR3RemExecute).
- *
- * The interpreted execution is only used to avoid switching between
- * raw-mode/hwaccm and the recompiler when fielding virtualization traps/faults.
- * The interpretation is thus implemented as part of EM.
- *
- * @see grp_em
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_EM
-#include <VBox/em.h>
-#include <VBox/vmm.h>
-#include <VBox/csam.h>
-#include <VBox/selm.h>
-#include <VBox/trpm.h>
-#include <VBox/iom.h>
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/rem.h>
-#include <VBox/tm.h>
-#include <VBox/mm.h>
-#include <VBox/ssm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/pdmqueue.h>
-#include <VBox/hwaccm.h>
-#include "EMInternal.h"
-#include "include/internal/em.h"
-#include <VBox/vm.h>
-#include <VBox/cpumdis.h>
-#include <VBox/dis.h>
-#include <VBox/disopcode.h>
-#include <VBox/dbgf.h>
-
-#include <iprt/asm.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-#if 0 /* Disabled till after 2.1.0 when we've time to test it. */
-#define EM_NOTIFY_HWACCM
-#endif
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC = VINF_SUCCESS);
-static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu);
-static int emR3HwaccmForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
-
-#define EMHANDLERC_WITH_HWACCM
-#include "EMHandleRCTmpl.h"
-
-
-#ifdef DEBUG
-
-/**
- * Steps hardware accelerated mode.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU handle.
- */
-static int emR3HwAccStep(PVM pVM, PVMCPU pVCpu)
-{
-    Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC);
-
-    int         rc;
-    PCPUMCTX    pCtx   = pVCpu->em.s.pCtx;
-    VMCPU_FF_CLEAR(pVCpu, (VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_TSS));
-
-    /*
-     * Check vital forced actions, but ignore pending interrupts and timers.
-     */
-    if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
-        ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
-    {
-        rc = emR3HwaccmForcedActions(pVM, pVCpu, pCtx);
-        if (rc != VINF_SUCCESS)
-            return rc;
-    }
-    /*
-     * Set flags for single stepping.
-     */
-    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
-
-    /*
-     * Single step.
-     * We do not start time or anything, if anything we should just do a few nanoseconds.
-     */
-    do
-    {
-        rc = VMMR3HwAccRunGC(pVM, pVCpu);
-    } while (   rc == VINF_SUCCESS
-             || rc == VINF_EM_RAW_INTERRUPT);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
-
-    /*
-     * Make sure the trap flag is cleared.
-     * (Too bad if the guest is trying to single step too.)
-     */
-    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
-
-    /*
-     * Deal with the return codes.
-     */
-    rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
-    rc = emR3HwaccmHandleRC(pVM, pVCpu, pCtx, rc);
-    return rc;
-}
-
-
-static int emR3SingleStepExecHwAcc(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
-{
-    int     rc          = VINF_SUCCESS;
-    EMSTATE enmOldState = pVCpu->em.s.enmState;
-    pVCpu->em.s.enmState  = EMSTATE_DEBUG_GUEST_HWACC;
-
-    Log(("Single step BEGIN:\n"));
-    for (uint32_t i = 0; i < cIterations; i++)
-    {
-        DBGFR3PrgStep(pVCpu);
-        DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
-        rc = emR3HwAccStep(pVM, pVCpu);
-        if (    rc != VINF_SUCCESS
-            ||  !HWACCMR3CanExecuteGuest(pVM, pVCpu->em.s.pCtx))
-            break;
-    }
-    Log(("Single step END: rc=%Rrc\n", rc));
-    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
-    pVCpu->em.s.enmState = enmOldState;
-    return rc == VINF_SUCCESS ? VINF_EM_RESCHEDULE_REM : rc;
-}
-
-#endif /* DEBUG */
-
-
-/**
- * Executes one (or perhaps a few more) instruction(s).
- *
- * @returns VBox status code suitable for EM.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle
- * @param   rcGC        GC return code
- * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
- *                      instruction and prefix the log output with this text.
- */
-#ifdef LOG_ENABLED
-static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC, const char *pszPrefix)
-#else
-static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC)
-#endif
-{
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-    int      rc;
-
-    /*
-     *
-     * The simple solution is to use the recompiler.
-     * The better solution is to disassemble the current instruction and
-     * try handle as many as possible without using REM.
-     *
-     */
-
-#ifdef LOG_ENABLED
-    /*
-     * Disassemble the instruction if requested.
-     */
-    if (pszPrefix)
-    {
-        DBGFR3InfoLog(pVM, "cpumguest", pszPrefix);
-        DBGFR3DisasInstrCurrentLog(pVCpu, pszPrefix);
-    }
-#endif /* LOG_ENABLED */
-
-#if 0
-    /* Try our own instruction emulator before falling back to the recompiler. */
-    DISCPUSTATE Cpu;
-    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "GEN EMU");
-    if (RT_SUCCESS(rc))
-    {
-        uint32_t size;
-
-        switch (Cpu.pCurInstr->opcode)
-        {
-        /* @todo we can do more now */
-        case OP_MOV:
-        case OP_AND:
-        case OP_OR:
-        case OP_XOR:
-        case OP_POP:
-        case OP_INC:
-        case OP_DEC:
-        case OP_XCHG:
-            STAM_PROFILE_START(&pVCpu->em.s.StatMiscEmu, a);
-            rc = EMInterpretInstructionCPU(pVM, pVCpu, &Cpu, CPUMCTX2CORE(pCtx), 0, &size);
-            if (RT_SUCCESS(rc))
-            {
-                pCtx->rip += Cpu.opsize;
-#ifdef EM_NOTIFY_HWACCM
-                if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
-                    HWACCMR3NotifyEmulated(pVCpu);
-#endif
-                STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
-                return rc;
-            }
-            if (rc != VERR_EM_INTERPRETER)
-                AssertMsgFailedReturn(("rc=%Rrc\n", rc), rc);
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
-            break;
-        }
-    }
-#endif /* 0 */
-    STAM_PROFILE_START(&pVCpu->em.s.StatREMEmu, a);
-    Log(("EMINS: %04x:%RGv RSP=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip, (RTGCPTR)pCtx->rsp));
-    EMRemLock(pVM);
-    /* Flush the recompiler TLB if the VCPU has changed. */
-    if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
-        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
-    pVM->em.s.idLastRemCpu = pVCpu->idCpu;
-
-    rc = REMR3EmulateInstruction(pVM, pVCpu);
-    EMRemUnlock(pVM);
-    STAM_PROFILE_STOP(&pVCpu->em.s.StatREMEmu, a);
-
-#ifdef EM_NOTIFY_HWACCM
-    if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
-        HWACCMR3NotifyEmulated(pVCpu);
-#endif
-    return rc;
-}
-
-
-/**
- * Executes one (or perhaps a few more) instruction(s).
- * This is just a wrapper for discarding pszPrefix in non-logging builds.
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
- *                      instruction and prefix the log output with this text.
- * @param   rcGC        GC return code
- */
-DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC)
-{
-#ifdef LOG_ENABLED
-    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC, pszPrefix);
-#else
-    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC);
-#endif
-}
-
-/**
- * Executes one (or perhaps a few more) IO instruction(s).
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- */
-static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu)
-{
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-
-    STAM_PROFILE_START(&pVCpu->em.s.StatIOEmu, a);
-
-    /* Try to restart the io instruction that was refused in ring-0. */
-    VBOXSTRICTRC rcStrict = HWACCMR3RestartPendingIOInstr(pVM, pVCpu, pCtx);
-    if (IOM_SUCCESS(rcStrict))
-    {
-        STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIoRestarted);
-        STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-        return VBOXSTRICTRC_TODO(rcStrict);     /* rip already updated. */
-    }
-    AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
-                    RT_SUCCESS_NP(rcStrict) ? VERR_INTERNAL_ERROR_5 : VBOXSTRICTRC_TODO(rcStrict));
-
-    /** @todo probably we should fall back to the recompiler; otherwise we'll go back and forth between HC & GC
-     *   as io instructions tend to come in packages of more than one
-     */
-    DISCPUSTATE Cpu;
-    int rc2 = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "IO EMU");
-    if (RT_SUCCESS(rc2))
-    {
-        rcStrict = VINF_EM_RAW_EMULATE_INSTR;
-
-        if (!(Cpu.prefix & (PREFIX_REP | PREFIX_REPNE)))
-        {
-            switch (Cpu.pCurInstr->opcode)
-            {
-                case OP_IN:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
-                    rcStrict = IOMInterpretIN(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-
-                case OP_OUT:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
-                    rcStrict = IOMInterpretOUT(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-            }
-        }
-        else if (Cpu.prefix & PREFIX_REP)
-        {
-            switch (Cpu.pCurInstr->opcode)
-            {
-                case OP_INSB:
-                case OP_INSWD:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
-                    rcStrict = IOMInterpretINS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-
-                case OP_OUTSB:
-                case OP_OUTSWD:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
-                    rcStrict = IOMInterpretOUTS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-            }
-        }
-
-        /*
-         * Handled the I/O return codes.
-         * (The unhandled cases end up with rcStrict == VINF_EM_RAW_EMULATE_INSTR.)
-         */
-        if (IOM_SUCCESS(rcStrict))
-        {
-            pCtx->rip += Cpu.opsize;
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-            return VBOXSTRICTRC_TODO(rcStrict);
-        }
-
-        if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
-        {
-            /* The active trap will be dispatched. */
-            Assert(TRPMHasTrap(pVCpu));
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-            return VINF_SUCCESS;
-        }
-        AssertMsg(rcStrict != VINF_TRPM_XCPT_DISPATCHED, ("Handle VINF_TRPM_XCPT_DISPATCHED\n"));
-
-        if (RT_FAILURE(rcStrict))
-        {
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-            return VBOXSTRICTRC_TODO(rcStrict);
-        }
-        AssertMsg(rcStrict == VINF_EM_RAW_EMULATE_INSTR || rcStrict == VINF_EM_RESCHEDULE_REM, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
-    }
-
-    STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-    return emR3ExecuteInstruction(pVM, pVCpu, "IO: ");
-}
-
-
-/**
- * Process raw-mode specific forced actions.
- *
- * This function is called when any FFs in the VM_FF_HIGH_PRIORITY_PRE_RAW_MASK is pending.
- *
- * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
- *          EM statuses.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU handle.
- * @param   pCtx        The guest CPUM register context.
- */
-static int emR3HwaccmForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
-{
-    /*
-     * Sync page directory.
-     */
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
-    {
-        Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
-        int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
-        if (RT_FAILURE(rc))
-            return rc;
-
-        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
-
-        /* Prefetch pages for EIP and ESP. */
-        /** @todo This is rather expensive. Should investigate if it really helps at all. */
-        rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->rip));
-        if (rc == VINF_SUCCESS)
-            rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->rsp));
-        if (rc != VINF_SUCCESS)
-        {
-            if (rc != VINF_PGM_SYNC_CR3)
-            {
-                AssertLogRelMsgReturn(RT_FAILURE(rc), ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
-                return rc;
-            }
-            rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
-            if (RT_FAILURE(rc))
-                return rc;
-        }
-        /** @todo maybe prefetch the supervisor stack page as well */
-        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
-    }
-
-    /*
-     * Allocate handy pages (just in case the above actions have consumed some pages).
-     */
-    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
-    {
-        int rc = PGMR3PhysAllocateHandyPages(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Check whether we're out of memory now.
-     *
-     * This may stem from some of the above actions or operations that has been executed
-     * since we ran FFs. The allocate handy pages must for instance always be followed by
-     * this check.
-     */
-    if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
-        return VINF_EM_NO_MEMORY;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Executes hardware accelerated raw code. (Intel VT-x & AMD-V)
- *
- * This function contains the raw-mode version of the inner
- * execution loop (the outer loop being in EMR3ExecuteVM()).
- *
- * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE, VINF_EM_RESCHEDULE_RAW,
- *          VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   pfFFDone    Where to store an indicator telling whether or not
- *                      FFs were done before returning.
- */
-int emR3HwAccExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
-{
-    int      rc = VERR_INTERNAL_ERROR;
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-
-    LogFlow(("emR3HwAccExecute%d: (cs:eip=%04x:%RGv)\n", pVCpu->idCpu, pCtx->cs, (RTGCPTR)pCtx->rip));
-    *pfFFDone = false;
-
-    STAM_COUNTER_INC(&pVCpu->em.s.StatHwAccExecuteEntry);
-
-#ifdef EM_NOTIFY_HWACCM
-    HWACCMR3NotifyScheduled(pVCpu);
-#endif
-
-    /*
-     * Spin till we get a forced action which returns anything but VINF_SUCCESS.
-     */
-    for (;;)
-    {
-        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatHwAccEntry, a);
-
-        /* Check if a forced reschedule is pending. */
-        if (HWACCMR3IsRescheduleRequired(pVM, pCtx))
-        {
-            rc = VINF_EM_RESCHEDULE;
-            break;
-        }
-
-        /*
-         * Process high priority pre-execution raw-mode FFs.
-         */
-        VMCPU_FF_CLEAR(pVCpu, (VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_TSS)); /* not relevant in HWACCM mode; shouldn't be set really. */
-        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
-        {
-            rc = emR3HwaccmForcedActions(pVM, pVCpu, pCtx);
-            if (rc != VINF_SUCCESS)
-                break;
-        }
-
-#ifdef LOG_ENABLED
-        /*
-         * Log important stuff before entering GC.
-         */
-        if (TRPMHasTrap(pVCpu))
-            Log(("CPU%d: Pending hardware interrupt=0x%x cs:rip=%04X:%RGv\n", pVCpu->idCpu, TRPMGetTrapNo(pVCpu), pCtx->cs, (RTGCPTR)pCtx->rip));
-
-        uint32_t cpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
-
-        if (pVM->cCpus == 1)
-        {
-            if (pCtx->eflags.Bits.u1VM)
-                Log(("HWV86: %08X IF=%d\n", pCtx->eip, pCtx->eflags.Bits.u1IF));
-            else if (CPUMIsGuestIn64BitCodeEx(pCtx))
-                Log(("HWR%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->rsp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
-            else
-                Log(("HWR%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pCtx->cs,          pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
-        }
-        else
-        {
-            if (pCtx->eflags.Bits.u1VM)
-                Log(("HWV86-CPU%d: %08X IF=%d\n", pVCpu->idCpu, pCtx->eip, pCtx->eflags.Bits.u1IF));
-            else if (CPUMIsGuestIn64BitCodeEx(pCtx))
-                Log(("HWR%d-CPU%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->rsp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
-            else
-                Log(("HWR%d-CPU%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pCtx->cs,          pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
-        }
-#endif /* LOG_ENABLED */
-
-        /*
-         * Execute the code.
-         */
-        STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatHwAccEntry, a);
-
-        if (RT_LIKELY(EMR3IsExecutionAllowed(pVM, pVCpu)))
-        {
-            STAM_PROFILE_START(&pVCpu->em.s.StatHwAccExec, x);
-            rc = VMMR3HwAccRunGC(pVM, pVCpu);
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatHwAccExec, x);
-        }
-        else
-        {
-            /* Give up this time slice; virtual time continues */
-            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
-            RTThreadSleep(5);
-            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
-            rc = VINF_SUCCESS;
-        }
-
-
-        /*
-         * Deal with high priority post execution FFs before doing anything else.
-         */
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
-            rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
-
-        /*
-         * Process the returned status code.
-         */
-        if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
-            break;
-
-        rc = emR3HwaccmHandleRC(pVM, pVCpu, pCtx, rc);
-        if (rc != VINF_SUCCESS)
-            break;
-
-        /*
-         * Check and execute forced actions.
-         */
-#ifdef VBOX_HIGH_RES_TIMERS_HACK
-        TMTimerPollVoid(pVM, pVCpu);
-#endif
-        if (    VM_FF_ISPENDING(pVM, VM_FF_ALL_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_MASK))
-        {
-            rc = emR3ForcedActions(pVM, pVCpu, rc);
-            if (    rc != VINF_SUCCESS
-                &&  rc != VINF_EM_RESCHEDULE_HWACC)
-            {
-                *pfFFDone = true;
-                break;
-            }
-        }
-    }
-
-    /*
-     * Return to outer loop.
-     */
-#if defined(LOG_ENABLED) && defined(DEBUG)
-    RTLogFlush(NULL);
-#endif
-    return rc;
-}
-
Index: unk/src/VBox/VMM/EMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/EMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,463 +1,0 @@
-/* $Id$ */
-/** @file
- * EM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___EMInternal_h
-#define ___EMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/em.h>
-#include <VBox/stam.h>
-#include <VBox/patm.h>
-#include <VBox/dis.h>
-#include <VBox/pdmcritsect.h>
-#include <iprt/avl.h>
-#include <setjmp.h>
-
-RT_C_DECLS_BEGIN
-
-
-/** @defgroup grp_em_int       Internal
- * @ingroup grp_em
- * @internal
- * @{
- */
-
-/** The saved state version. */
-#define EM_SAVED_STATE_VERSION                          4
-#define EM_SAVED_STATE_VERSION_PRE_MWAIT                3
-#define EM_SAVED_STATE_VERSION_PRE_SMP                  2
-
-
-/**
- * MWait state flags.
- */
-/* MWait activated. */
-#define EMMWAIT_FLAG_ACTIVE             RT_BIT(0)
-/* MWait will continue when an interrupt is pending even when IF=0. */
-#define EMMWAIT_FLAG_BREAKIRQIF0        RT_BIT(1)
-/* Monitor instruction was executed previously. */
-#define EMMWAIT_FLAG_MONITOR_ACTIVE     RT_BIT(2)
-
-/** EM time slice in ms; used for capping execution time. */
-#define EM_TIME_SLICE                   100
-
-/**
- * Cli node structure
- */
-typedef struct CLISTAT
-{
-    /** The key is the cli address. */
-    AVLGCPTRNODECORE        Core;
-#if HC_ARCH_BITS == 32 && !defined(RT_OS_WINDOWS)
-    /** Padding. */
-    uint32_t                u32Padding;
-#endif
-    /** Occurrences. */
-    STAMCOUNTER             Counter;
-} CLISTAT, *PCLISTAT;
-#ifdef IN_RING3
-AssertCompileMemberAlignment(CLISTAT, Counter, 8);
-#endif
-
-
-/**
- * Excessive EM statistics.
- */
-typedef struct EMSTATS
-{
-    /** GC: Profiling of EMInterpretInstruction(). */
-    STAMPROFILE             StatRZEmulate;
-    /** HC: Profiling of EMInterpretInstruction(). */
-    STAMPROFILE             StatR3Emulate;
-
-    /** @name Interpreter Instruction statistics.
-     * @{
-     */
-    STAMCOUNTER             StatRZInterpretSucceeded;
-    STAMCOUNTER             StatR3InterpretSucceeded;
-
-    STAMCOUNTER             StatRZAnd;
-    STAMCOUNTER             StatR3And;
-    STAMCOUNTER             StatRZCpuId;
-    STAMCOUNTER             StatR3CpuId;
-    STAMCOUNTER             StatRZDec;
-    STAMCOUNTER             StatR3Dec;
-    STAMCOUNTER             StatRZHlt;
-    STAMCOUNTER             StatR3Hlt;
-    STAMCOUNTER             StatRZInc;
-    STAMCOUNTER             StatR3Inc;
-    STAMCOUNTER             StatRZInvlPg;
-    STAMCOUNTER             StatR3InvlPg;
-    STAMCOUNTER             StatRZIret;
-    STAMCOUNTER             StatR3Iret;
-    STAMCOUNTER             StatRZLLdt;
-    STAMCOUNTER             StatR3LLdt;
-    STAMCOUNTER             StatRZLIdt;
-    STAMCOUNTER             StatR3LIdt;
-    STAMCOUNTER             StatRZLGdt;
-    STAMCOUNTER             StatR3LGdt;
-    STAMCOUNTER             StatRZMov;
-    STAMCOUNTER             StatR3Mov;
-    STAMCOUNTER             StatRZMovCRx;
-    STAMCOUNTER             StatR3MovCRx;
-    STAMCOUNTER             StatRZMovDRx;
-    STAMCOUNTER             StatR3MovDRx;
-    STAMCOUNTER             StatRZOr;
-    STAMCOUNTER             StatR3Or;
-    STAMCOUNTER             StatRZPop;
-    STAMCOUNTER             StatR3Pop;
-    STAMCOUNTER             StatRZSti;
-    STAMCOUNTER             StatR3Sti;
-    STAMCOUNTER             StatRZXchg;
-    STAMCOUNTER             StatR3Xchg;
-    STAMCOUNTER             StatRZXor;
-    STAMCOUNTER             StatR3Xor;
-    STAMCOUNTER             StatRZMonitor;
-    STAMCOUNTER             StatR3Monitor;
-    STAMCOUNTER             StatRZMWait;
-    STAMCOUNTER             StatR3MWait;
-    STAMCOUNTER             StatRZAdd;
-    STAMCOUNTER             StatR3Add;
-    STAMCOUNTER             StatRZSub;
-    STAMCOUNTER             StatR3Sub;
-    STAMCOUNTER             StatRZAdc;
-    STAMCOUNTER             StatR3Adc;
-    STAMCOUNTER             StatRZRdtsc;
-    STAMCOUNTER             StatR3Rdtsc;
-    STAMCOUNTER             StatRZRdpmc;
-    STAMCOUNTER             StatR3Rdpmc;
-    STAMCOUNTER             StatRZBtr;
-    STAMCOUNTER             StatR3Btr;
-    STAMCOUNTER             StatRZBts;
-    STAMCOUNTER             StatR3Bts;
-    STAMCOUNTER             StatRZBtc;
-    STAMCOUNTER             StatR3Btc;
-    STAMCOUNTER             StatRZCmpXchg;
-    STAMCOUNTER             StatR3CmpXchg;
-    STAMCOUNTER             StatRZCmpXchg8b;
-    STAMCOUNTER             StatR3CmpXchg8b;
-    STAMCOUNTER             StatRZXAdd;
-    STAMCOUNTER             StatR3XAdd;
-    STAMCOUNTER             StatRZClts;
-    STAMCOUNTER             StatR3Clts;
-    STAMCOUNTER             StatRZStosWD;
-    STAMCOUNTER             StatR3StosWD;
-    STAMCOUNTER             StatR3Rdmsr;
-    STAMCOUNTER             StatR3Wrmsr;
-    STAMCOUNTER             StatRZRdmsr;
-    STAMCOUNTER             StatRZWrmsr;
-    STAMCOUNTER             StatRZWbInvd;
-    STAMCOUNTER             StatR3WbInvd;
-    STAMCOUNTER             StatRZLmsw;
-    STAMCOUNTER             StatR3Lmsw;
-    STAMCOUNTER             StatRZSmsw;
-    STAMCOUNTER             StatR3Smsw;
-
-    STAMCOUNTER             StatRZInterpretFailed;
-    STAMCOUNTER             StatR3InterpretFailed;
-
-    STAMCOUNTER             StatRZFailedAnd;
-    STAMCOUNTER             StatR3FailedAnd;
-    STAMCOUNTER             StatRZFailedCpuId;
-    STAMCOUNTER             StatR3FailedCpuId;
-    STAMCOUNTER             StatRZFailedDec;
-    STAMCOUNTER             StatR3FailedDec;
-    STAMCOUNTER             StatRZFailedHlt;
-    STAMCOUNTER             StatR3FailedHlt;
-    STAMCOUNTER             StatRZFailedInc;
-    STAMCOUNTER             StatR3FailedInc;
-    STAMCOUNTER             StatRZFailedInvlPg;
-    STAMCOUNTER             StatR3FailedInvlPg;
-    STAMCOUNTER             StatRZFailedIret;
-    STAMCOUNTER             StatR3FailedIret;
-    STAMCOUNTER             StatRZFailedLLdt;
-    STAMCOUNTER             StatR3FailedLLdt;
-    STAMCOUNTER             StatRZFailedLGdt;
-    STAMCOUNTER             StatR3FailedLGdt;
-    STAMCOUNTER             StatRZFailedLIdt;
-    STAMCOUNTER             StatR3FailedLIdt;
-    STAMCOUNTER             StatRZFailedMisc;
-    STAMCOUNTER             StatR3FailedMisc;
-    STAMCOUNTER             StatRZFailedMov;
-    STAMCOUNTER             StatR3FailedMov;
-    STAMCOUNTER             StatRZFailedMovCRx;
-    STAMCOUNTER             StatR3FailedMovCRx;
-    STAMCOUNTER             StatRZFailedMovDRx;
-    STAMCOUNTER             StatR3FailedMovDRx;
-    STAMCOUNTER             StatRZFailedOr;
-    STAMCOUNTER             StatR3FailedOr;
-    STAMCOUNTER             StatRZFailedPop;
-    STAMCOUNTER             StatR3FailedPop;
-    STAMCOUNTER             StatRZFailedSti;
-    STAMCOUNTER             StatR3FailedSti;
-    STAMCOUNTER             StatRZFailedXchg;
-    STAMCOUNTER             StatR3FailedXchg;
-    STAMCOUNTER             StatRZFailedXor;
-    STAMCOUNTER             StatR3FailedXor;
-    STAMCOUNTER             StatRZFailedMonitor;
-    STAMCOUNTER             StatR3FailedMonitor;
-    STAMCOUNTER             StatRZFailedMWait;
-    STAMCOUNTER             StatR3FailedMWait;
-    STAMCOUNTER             StatR3FailedRdmsr;
-    STAMCOUNTER             StatR3FailedWrmsr;
-    STAMCOUNTER             StatRZFailedRdmsr;
-    STAMCOUNTER             StatRZFailedWrmsr;
-    STAMCOUNTER             StatRZFailedLmsw;
-    STAMCOUNTER             StatR3FailedLmsw;
-    STAMCOUNTER             StatRZFailedSmsw;
-    STAMCOUNTER             StatR3FailedSmsw;
-
-    STAMCOUNTER             StatRZFailedAdd;
-    STAMCOUNTER             StatR3FailedAdd;
-    STAMCOUNTER             StatRZFailedAdc;
-    STAMCOUNTER             StatR3FailedAdc;
-    STAMCOUNTER             StatRZFailedBtr;
-    STAMCOUNTER             StatR3FailedBtr;
-    STAMCOUNTER             StatRZFailedBts;
-    STAMCOUNTER             StatR3FailedBts;
-    STAMCOUNTER             StatRZFailedBtc;
-    STAMCOUNTER             StatR3FailedBtc;
-    STAMCOUNTER             StatRZFailedCli;
-    STAMCOUNTER             StatR3FailedCli;
-    STAMCOUNTER             StatRZFailedCmpXchg;
-    STAMCOUNTER             StatR3FailedCmpXchg;
-    STAMCOUNTER             StatRZFailedCmpXchg8b;
-    STAMCOUNTER             StatR3FailedCmpXchg8b;
-    STAMCOUNTER             StatRZFailedXAdd;
-    STAMCOUNTER             StatR3FailedXAdd;
-    STAMCOUNTER             StatR3FailedMovNTPS;
-    STAMCOUNTER             StatRZFailedMovNTPS;
-    STAMCOUNTER             StatRZFailedStosWD;
-    STAMCOUNTER             StatR3FailedStosWD;
-    STAMCOUNTER             StatRZFailedSub;
-    STAMCOUNTER             StatR3FailedSub;
-    STAMCOUNTER             StatRZFailedWbInvd;
-    STAMCOUNTER             StatR3FailedWbInvd;
-    STAMCOUNTER             StatRZFailedRdtsc;
-    STAMCOUNTER             StatR3FailedRdtsc;
-    STAMCOUNTER             StatRZFailedRdpmc;
-    STAMCOUNTER             StatR3FailedRdpmc;
-    STAMCOUNTER             StatRZFailedClts;
-    STAMCOUNTER             StatR3FailedClts;
-
-    STAMCOUNTER             StatRZFailedUserMode;
-    STAMCOUNTER             StatR3FailedUserMode;
-    STAMCOUNTER             StatRZFailedPrefix;
-    STAMCOUNTER             StatR3FailedPrefix;
-    /** @} */
-
-    /** @name Privileged Instructions Ending Up In HC.
-     * @{ */
-    STAMCOUNTER             StatCli;
-    STAMCOUNTER             StatSti;
-    STAMCOUNTER             StatIn;
-    STAMCOUNTER             StatIoRestarted;
-    STAMCOUNTER             StatOut;
-    STAMCOUNTER             StatInvlpg;
-    STAMCOUNTER             StatHlt;
-    STAMCOUNTER             StatMovReadCR[USE_REG_CR4 + 1];
-    STAMCOUNTER             StatMovWriteCR[USE_REG_CR4 + 1];
-    STAMCOUNTER             StatMovDRx;
-    STAMCOUNTER             StatIret;
-    STAMCOUNTER             StatMovLgdt;
-    STAMCOUNTER             StatMovLldt;
-    STAMCOUNTER             StatMovLidt;
-    STAMCOUNTER             StatMisc;
-    STAMCOUNTER             StatSysEnter;
-    STAMCOUNTER             StatSysExit;
-    STAMCOUNTER             StatSysCall;
-    STAMCOUNTER             StatSysRet;
-    /** @} */
-
-} EMSTATS;
-/** Pointer to the excessive EM statistics. */
-typedef EMSTATS *PEMSTATS;
-
-
-/**
- * Converts a EM pointer into a VM pointer.
- * @returns Pointer to the VM structure the EM is part of.
- * @param   pEM   Pointer to EM instance data.
- */
-#define EM2VM(pEM)  ( (PVM)((char*)pEM - pEM->offVM) )
-
-/**
- * EM VM Instance data.
- * Changes to this must checked against the padding of the cfgm union in VM!
- */
-typedef struct EM
-{
-    /** Offset to the VM structure.
-     * See EM2VM(). */
-    RTUINT                  offVM;
-
-    /** Id of the VCPU that last executed code in the recompiler. */
-    VMCPUID                 idLastRemCpu;
-
-    /** REM critical section.
-     * This protects recompiler usage
-     */
-    PDMCRITSECT             CritSectREM;
-} EM;
-/** Pointer to EM VM instance data. */
-typedef EM *PEM;
-
-
-/**
- * EM VMCPU Instance data.
- */
-typedef struct EMCPU
-{
-    /** Offset to the VM structure.
-     * See EMCPU2VM(). */
-    RTUINT                  offVMCPU;
-
-    /** Execution Manager State. */
-    EMSTATE volatile        enmState;
-
-    /** Previous Execution Manager State. */
-    EMSTATE                 enmPrevState;
-
-    /** Force raw-mode execution.
-     * This is used to prevent REM from trying to execute patch code.
-     * The flag is cleared upon entering emR3RawExecute() and updated in certain return paths. */
-    bool                    fForceRAW;
-
-    uint8_t                 u8Padding[3];
-
-    /** Inhibit interrupts for this instruction. Valid only when VM_FF_INHIBIT_INTERRUPTS is set. */
-    RTGCUINTPTR             GCPtrInhibitInterrupts;
-
-    /** Pointer to the PATM status structure. (R3 Ptr) */
-    R3PTRTYPE(PPATMGCSTATE) pPatmGCState;
-
-    /** Pointer to the guest CPUM state. (R3 Ptr) */
-    R3PTRTYPE(PCPUMCTX)     pCtx;
-
-#if GC_ARCH_BITS == 64
-    RTGCPTR                 aPadding1;
-#endif
-
-    /** Start of the current time slice in ms. */
-    uint64_t                u64TimeSliceStart;
-    /** Start of the current time slice in thread execution time (ms). */
-    uint64_t                u64TimeSliceStartExec;
-    /** Current time slice value. */
-    uint64_t                u64TimeSliceExec;
-    uint64_t                u64Alignment;
-
-    /* MWait halt state. */
-    struct
-    {
-        uint32_t            fWait;          /* type of mwait; see EMMWAIT_FLAG_* */
-        uint32_t            a32Padding[1];
-        RTGCPTR             uMWaitEAX;      /* mwait hints */
-        RTGCPTR             uMWaitECX;      /* mwait extensions */
-        RTGCPTR             uMonitorEAX;    /* monitored address. */
-        RTGCPTR             uMonitorECX;    /* monitor extension. */
-        RTGCPTR             uMonitorEDX;    /* monitor hint. */
-    } mwait;
-
-    union
-    {
-        /** Padding used in the other rings.
-         * This must be larger than jmp_buf on any supported platform. */
-        char                achPaddingFatalLongJump[HC_ARCH_BITS == 32 ? 176 : 256];
-#ifdef IN_RING3
-        /** Long buffer jump for fatal VM errors.
-         * It will jump to before the outer EM loop is entered. */
-        jmp_buf             FatalLongJump;
-#endif
-    } u;
-
-    /** For saving stack space, the disassembler state is allocated here instead of
-     * on the stack.
-     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
-    union
-    {
-        /** The disassembler scratch space. */
-        DISCPUSTATE         DisState;
-        /** Padding. */
-        uint8_t             abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
-    };
-
-    /** @name Execution profiling.
-     * @{ */
-    STAMPROFILE             StatForcedActions;
-    STAMPROFILE             StatHalted;
-    STAMPROFILEADV          StatCapped;
-    STAMPROFILEADV          StatHwAccEntry;
-    STAMPROFILE             StatHwAccExec;
-    STAMPROFILE             StatREMEmu;
-    STAMPROFILE             StatREMExec;
-    STAMPROFILE             StatREMSync;
-    STAMPROFILEADV          StatREMTotal;
-    STAMPROFILE             StatRAWExec;
-    STAMPROFILEADV          StatRAWEntry;
-    STAMPROFILEADV          StatRAWTail;
-    STAMPROFILEADV          StatRAWTotal;
-    STAMPROFILEADV          StatTotal;
-    /** @} */
-
-    /** R3: Profiling of emR3RawExecuteIOInstruction. */
-    STAMPROFILE             StatIOEmu;
-    /** R3: Profiling of emR3RawPrivileged. */
-    STAMPROFILE             StatPrivEmu;
-    /** R3: Number of time emR3HwAccExecute is called. */
-    STAMCOUNTER             StatHwAccExecuteEntry;
-
-    /** More statistics (R3). */
-    R3PTRTYPE(PEMSTATS)     pStatsR3;
-    /** More statistics (R0). */
-    R0PTRTYPE(PEMSTATS)     pStatsR0;
-    /** More statistics (RC). */
-    RCPTRTYPE(PEMSTATS)     pStatsRC;
-#if HC_ARCH_BITS == 64
-    RTRCPTR                 padding0;
-#endif
-
-    /** Tree for keeping track of cli occurrences (debug only). */
-    R3PTRTYPE(PAVLGCPTRNODECORE) pCliStatTree;
-    STAMCOUNTER             StatTotalClis;
-#if 0
-    /** 64-bit Visual C++ rounds the struct size up to 16 byte. */
-    uint64_t                padding1;
-#endif
-} EMCPU;
-/** Pointer to EM VM instance data. */
-typedef EMCPU *PEMCPU;
-
-/** @} */
-
-
-int     emR3HwAccExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
-int     emR3RawExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
-int     emR3RawHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
-int     emR3HwaccmHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
-EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
-int     emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
-int     emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
-int     emR3RawUpdateForceFlag(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
-int     emR3RawResumeHyper(PVM pVM, PVMCPU pVCpu);
-int     emR3RawStep(PVM pVM, PVMCPU pVCpu);
-int     emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations);
-
-RT_C_DECLS_END
-
-#endif
-
Index: unk/src/VBox/VMM/EMRaw.cpp
===================================================================
--- /trunk/src/VBox/VMM/EMRaw.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1655 +1,0 @@
-/* $Id$ */
-/** @file
- * EM - Execution Monitor / Manager - software virtualization
- */
-
-/*
- * Copyright (C) 2006-2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_em         EM - The Execution Monitor / Manager
- *
- * The Execution Monitor/Manager is responsible for running the VM, scheduling
- * the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
- * Interpreted), and keeping the CPU states in sync. The function
- * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
- * modes has different inner loops (emR3RawExecute, emR3HwAccExecute, and
- * emR3RemExecute).
- *
- * The interpreted execution is only used to avoid switching between
- * raw-mode/hwaccm and the recompiler when fielding virtualization traps/faults.
- * The interpretation is thus implemented as part of EM.
- *
- * @see grp_em
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_EM
-#include <VBox/em.h>
-#include <VBox/vmm.h>
-#include <VBox/patm.h>
-#include <VBox/csam.h>
-#include <VBox/selm.h>
-#include <VBox/trpm.h>
-#include <VBox/iom.h>
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/rem.h>
-#include <VBox/tm.h>
-#include <VBox/mm.h>
-#include <VBox/ssm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/pdmqueue.h>
-#include <VBox/patm.h>
-#include "EMInternal.h"
-#include "include/internal/em.h"
-#include <VBox/vm.h>
-#include <VBox/cpumdis.h>
-#include <VBox/dis.h>
-#include <VBox/disopcode.h>
-#include <VBox/dbgf.h>
-
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/string.h>
-#include <iprt/stream.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int emR3RawForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
-DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC = VINF_SUCCESS);
-static int emR3RawGuestTrap(PVM pVM, PVMCPU pVCpu);
-static int emR3PatchTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int gcret);
-static int emR3SingleStepExecRem(PVM pVM, uint32_t cIterations);
-static int emR3RawPrivileged(PVM pVM, PVMCPU pVCpu);
-static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu);
-static int emR3RawRingSwitch(PVM pVM, PVMCPU pVCpu);
-
-#define EMHANDLERC_WITH_PATM
-#include "EMHandleRCTmpl.h"
-
-/**
- * Enables or disables a set of raw-mode execution modes.
- *
- * @returns VINF_SUCCESS on success.
- * @returns VINF_RESCHEDULE if a rescheduling might be required.
- * @returns VERR_INVALID_PARAMETER on an invalid enmMode value.
- *
- * @param   pVM         The VM to operate on.
- * @param   enmMode     The execution mode change.
- * @thread  The emulation thread.
- */
-VMMR3DECL(int) EMR3RawSetMode(PVM pVM, EMRAWMODE enmMode)
-{
-    switch (enmMode)
-    {
-        case EMRAW_NONE:
-            pVM->fRawR3Enabled = false;
-            pVM->fRawR0Enabled = false;
-            break;
-        case EMRAW_RING3_ENABLE:
-            pVM->fRawR3Enabled = true;
-            break;
-        case EMRAW_RING3_DISABLE:
-            pVM->fRawR3Enabled = false;
-            break;
-        case EMRAW_RING0_ENABLE:
-            pVM->fRawR0Enabled = true;
-            break;
-        case EMRAW_RING0_DISABLE:
-            pVM->fRawR0Enabled = false;
-            break;
-        default:
-            AssertMsgFailed(("Invalid enmMode=%d\n", enmMode));
-            return VERR_INVALID_PARAMETER;
-    }
-    Log(("EMR3SetRawMode: fRawR3Enabled=%RTbool fRawR0Enabled=%RTbool\n",
-          pVM->fRawR3Enabled, pVM->fRawR0Enabled));
-    return pVM->aCpus[0].em.s.enmState == EMSTATE_RAW ? VINF_EM_RESCHEDULE : VINF_SUCCESS;
-}
-
-
-
-#ifdef VBOX_WITH_STATISTICS
-/**
- * Just a braindead function to keep track of cli addresses.
- * @param   pVM         VM handle.
- * @param   pVMCPU      VMCPU handle.
- * @param   GCPtrInstr  The EIP of the cli instruction.
- */
-static void emR3RecordCli(PVM pVM, PVMCPU pVCpu, RTGCPTR GCPtrInstr)
-{
-    PCLISTAT pRec;
-
-    pRec = (PCLISTAT)RTAvlGCPtrGet(&pVCpu->em.s.pCliStatTree, GCPtrInstr);
-    if (!pRec)
-    {
-        /* New cli instruction; insert into the tree. */
-        pRec = (PCLISTAT)MMR3HeapAllocZ(pVM, MM_TAG_EM, sizeof(*pRec));
-        Assert(pRec);
-        if (!pRec)
-            return;
-        pRec->Core.Key = GCPtrInstr;
-
-        char szCliStatName[32];
-        RTStrPrintf(szCliStatName, sizeof(szCliStatName), "/EM/Cli/0x%RGv", GCPtrInstr);
-        STAM_REG(pVM, &pRec->Counter, STAMTYPE_COUNTER, szCliStatName, STAMUNIT_OCCURENCES, "Number of times cli was executed.");
-
-        bool fRc = RTAvlGCPtrInsert(&pVCpu->em.s.pCliStatTree, &pRec->Core);
-        Assert(fRc); NOREF(fRc);
-    }
-    STAM_COUNTER_INC(&pRec->Counter);
-    STAM_COUNTER_INC(&pVCpu->em.s.StatTotalClis);
-}
-#endif /* VBOX_WITH_STATISTICS */
-
-
-
-/**
- * Resumes executing hypervisor after a debug event.
- *
- * This is kind of special since our current guest state is
- * potentially out of sync.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU handle.
- */
-int emR3RawResumeHyper(PVM pVM, PVMCPU pVCpu)
-{
-    int         rc;
-    PCPUMCTX    pCtx = pVCpu->em.s.pCtx;
-    Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER);
-    Log(("emR3RawResumeHyper: cs:eip=%RTsel:%RGr efl=%RGr\n", pCtx->cs, pCtx->eip, pCtx->eflags));
-
-    /*
-     * Resume execution.
-     */
-    CPUMR3RawEnter(pVCpu, NULL);
-    CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_RF);
-    rc = VMMR3ResumeHyper(pVM, pVCpu);
-    Log(("emR3RawResumeHyper: cs:eip=%RTsel:%RGr efl=%RGr - returned from GC with rc=%Rrc\n", pCtx->cs, pCtx->eip, pCtx->eflags, rc));
-    rc = CPUMR3RawLeave(pVCpu, NULL, rc);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
-
-    /*
-     * Deal with the return code.
-     */
-    rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
-    rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
-    rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
-    return rc;
-}
-
-
-/**
- * Steps rawmode.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU handle.
- */
-int emR3RawStep(PVM pVM, PVMCPU pVCpu)
-{
-    Assert(   pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER
-           || pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
-           || pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM);
-    int         rc;
-    PCPUMCTX    pCtx   = pVCpu->em.s.pCtx;
-    bool        fGuest = pVCpu->em.s.enmState != EMSTATE_DEBUG_HYPER;
-#ifndef DEBUG_sandervl
-    Log(("emR3RawStep: cs:eip=%RTsel:%RGr efl=%RGr\n", fGuest ? CPUMGetGuestCS(pVCpu) : CPUMGetHyperCS(pVCpu),
-         fGuest ? CPUMGetGuestEIP(pVCpu) : CPUMGetHyperEIP(pVCpu), fGuest ? CPUMGetGuestEFlags(pVCpu) : CPUMGetHyperEFlags(pVCpu)));
-#endif
-    if (fGuest)
-    {
-        /*
-         * Check vital forced actions, but ignore pending interrupts and timers.
-         */
-        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
-        {
-            rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
-            if (rc != VINF_SUCCESS)
-                return rc;
-        }
-
-        /*
-         * Set flags for single stepping.
-         */
-        CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
-    }
-    else
-        CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
-
-    /*
-     * Single step.
-     * We do not start time or anything, if anything we should just do a few nanoseconds.
-     */
-    CPUMR3RawEnter(pVCpu, NULL);
-    do
-    {
-        if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
-            rc = VMMR3ResumeHyper(pVM, pVCpu);
-        else
-            rc = VMMR3RawRunGC(pVM, pVCpu);
-#ifndef DEBUG_sandervl
-        Log(("emR3RawStep: cs:eip=%RTsel:%RGr efl=%RGr - GC rc %Rrc\n", fGuest ? CPUMGetGuestCS(pVCpu) : CPUMGetHyperCS(pVCpu),
-             fGuest ? CPUMGetGuestEIP(pVCpu) : CPUMGetHyperEIP(pVCpu), fGuest ? CPUMGetGuestEFlags(pVCpu) : CPUMGetHyperEFlags(pVCpu), rc));
-#endif
-    } while (   rc == VINF_SUCCESS
-             || rc == VINF_EM_RAW_INTERRUPT);
-    rc = CPUMR3RawLeave(pVCpu, NULL, rc);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
-
-    /*
-     * Make sure the trap flag is cleared.
-     * (Too bad if the guest is trying to single step too.)
-     */
-    if (fGuest)
-        CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
-    else
-        CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) & ~X86_EFL_TF);
-
-    /*
-     * Deal with the return codes.
-     */
-    rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
-    rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
-    rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
-    return rc;
-}
-
-
-#ifdef DEBUG
-
-
-int emR3SingleStepExecRaw(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
-{
-    int     rc          = VINF_SUCCESS;
-    EMSTATE enmOldState = pVCpu->em.s.enmState;
-    pVCpu->em.s.enmState  = EMSTATE_DEBUG_GUEST_RAW;
-
-    Log(("Single step BEGIN:\n"));
-    for (uint32_t i = 0; i < cIterations; i++)
-    {
-        DBGFR3PrgStep(pVCpu);
-        DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
-        rc = emR3RawStep(pVM, pVCpu);
-        if (rc != VINF_SUCCESS)
-            break;
-    }
-    Log(("Single step END: rc=%Rrc\n", rc));
-    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
-    pVCpu->em.s.enmState = enmOldState;
-    return rc;
-}
-
-#endif /* DEBUG */
-
-
-/**
- * Executes one (or perhaps a few more) instruction(s).
- *
- * @returns VBox status code suitable for EM.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle
- * @param   rcGC        GC return code
- * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
- *                      instruction and prefix the log output with this text.
- */
-#ifdef LOG_ENABLED
-static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC, const char *pszPrefix)
-#else
-static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC)
-#endif
-{
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-    int      rc;
-
-    /*
-     *
-     * The simple solution is to use the recompiler.
-     * The better solution is to disassemble the current instruction and
-     * try handle as many as possible without using REM.
-     *
-     */
-
-#ifdef LOG_ENABLED
-    /*
-     * Disassemble the instruction if requested.
-     */
-    if (pszPrefix)
-    {
-        DBGFR3InfoLog(pVM, "cpumguest", pszPrefix);
-        DBGFR3DisasInstrCurrentLog(pVCpu, pszPrefix);
-    }
-#endif /* LOG_ENABLED */
-
-    /*
-     * PATM is making life more interesting.
-     * We cannot hand anything to REM which has an EIP inside patch code. So, we'll
-     * tell PATM there is a trap in this code and have it take the appropriate actions
-     * to allow us execute the code in REM.
-     */
-    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
-    {
-        Log(("emR3ExecuteInstruction: In patch block. eip=%RRv\n", (RTRCPTR)pCtx->eip));
-
-        RTGCPTR pNewEip;
-        rc = PATMR3HandleTrap(pVM, pCtx, pCtx->eip, &pNewEip);
-        switch (rc)
-        {
-            /*
-             * It's not very useful to emulate a single instruction and then go back to raw
-             * mode; just execute the whole block until IF is set again.
-             */
-            case VINF_SUCCESS:
-                Log(("emR3ExecuteInstruction: Executing instruction starting at new address %RGv IF=%d VMIF=%x\n",
-                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
-                pCtx->eip = pNewEip;
-                Assert(pCtx->eip);
-
-                if (pCtx->eflags.Bits.u1IF)
-                {
-                    /*
-                     * The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
-                     */
-                    Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
-                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
-                }
-                else if (rcGC == VINF_PATM_PENDING_IRQ_AFTER_IRET)
-                {
-                    /* special case: iret, that sets IF,  detected a pending irq/event */
-                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIRET");
-                }
-                return VINF_EM_RESCHEDULE_REM;
-
-            /*
-             * One instruction.
-             */
-            case VINF_PATCH_EMULATE_INSTR:
-                Log(("emR3ExecuteInstruction: Emulate patched instruction at %RGv IF=%d VMIF=%x\n",
-                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
-                pCtx->eip = pNewEip;
-                return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
-
-            /*
-             * The patch was disabled, hand it to the REM.
-             */
-            case VERR_PATCH_DISABLED:
-                Log(("emR3ExecuteInstruction: Disabled patch -> new eip %RGv IF=%d VMIF=%x\n",
-                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
-                pCtx->eip = pNewEip;
-                if (pCtx->eflags.Bits.u1IF)
-                {
-                    /*
-                     * The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
-                     */
-                    Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
-                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
-                }
-                return VINF_EM_RESCHEDULE_REM;
-
-            /* Force continued patch exection; usually due to write monitored stack. */
-            case VINF_PATCH_CONTINUE:
-                return VINF_SUCCESS;
-
-            default:
-                AssertReleaseMsgFailed(("Unknown return code %Rrc from PATMR3HandleTrap\n", rc));
-                return VERR_IPE_UNEXPECTED_STATUS;
-        }
-    }
-
-#if 0
-    /* Try our own instruction emulator before falling back to the recompiler. */
-    DISCPUSTATE Cpu;
-    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "GEN EMU");
-    if (RT_SUCCESS(rc))
-    {
-        uint32_t size;
-
-        switch (Cpu.pCurInstr->opcode)
-        {
-        /* @todo we can do more now */
-        case OP_MOV:
-        case OP_AND:
-        case OP_OR:
-        case OP_XOR:
-        case OP_POP:
-        case OP_INC:
-        case OP_DEC:
-        case OP_XCHG:
-            STAM_PROFILE_START(&pVCpu->em.s.StatMiscEmu, a);
-            rc = EMInterpretInstructionCPU(pVM, &Cpu, CPUMCTX2CORE(pCtx), 0, &size);
-            if (RT_SUCCESS(rc))
-            {
-                pCtx->rip += Cpu.opsize;
-                STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
-                return rc;
-            }
-            if (rc != VERR_EM_INTERPRETER)
-                AssertMsgFailedReturn(("rc=%Rrc\n", rc), rc);
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
-            break;
-        }
-    }
-#endif /* 0 */
-    STAM_PROFILE_START(&pVCpu->em.s.StatREMEmu, a);
-    Log(("EMINS: %04x:%RGv RSP=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip, (RTGCPTR)pCtx->rsp));
-    EMRemLock(pVM);
-    /* Flush the recompiler TLB if the VCPU has changed. */
-    if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
-        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
-    pVM->em.s.idLastRemCpu = pVCpu->idCpu;
-
-    rc = REMR3EmulateInstruction(pVM, pVCpu);
-    EMRemUnlock(pVM);
-    STAM_PROFILE_STOP(&pVCpu->em.s.StatREMEmu, a);
-
-    return rc;
-}
-
-
-/**
- * Executes one (or perhaps a few more) instruction(s).
- * This is just a wrapper for discarding pszPrefix in non-logging builds.
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
- *                      instruction and prefix the log output with this text.
- * @param   rcGC        GC return code
- */
-DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC)
-{
-#ifdef LOG_ENABLED
-    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC, pszPrefix);
-#else
-    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC);
-#endif
-}
-
-/**
- * Executes one (or perhaps a few more) IO instruction(s).
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- */
-static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu)
-{
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-
-    STAM_PROFILE_START(&pVCpu->em.s.StatIOEmu, a);
-
-    /** @todo probably we should fall back to the recompiler; otherwise we'll go back and forth between HC & GC
-     *   as io instructions tend to come in packages of more than one
-     */
-    DISCPUSTATE Cpu;
-    int rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "IO EMU");
-    if (RT_SUCCESS(rc))
-    {
-        VBOXSTRICTRC rcStrict = VINF_EM_RAW_EMULATE_INSTR;
-
-        if (!(Cpu.prefix & (PREFIX_REP | PREFIX_REPNE)))
-        {
-            switch (Cpu.pCurInstr->opcode)
-            {
-                case OP_IN:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
-                    rcStrict = IOMInterpretIN(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-
-                case OP_OUT:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
-                    rcStrict = IOMInterpretOUT(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-            }
-        }
-        else if (Cpu.prefix & PREFIX_REP)
-        {
-            switch (Cpu.pCurInstr->opcode)
-            {
-                case OP_INSB:
-                case OP_INSWD:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
-                    rcStrict = IOMInterpretINS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-
-                case OP_OUTSB:
-                case OP_OUTSWD:
-                {
-                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
-                    rcStrict = IOMInterpretOUTS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
-                    break;
-                }
-            }
-        }
-
-        /*
-         * Handled the I/O return codes.
-         * (The unhandled cases end up with rcStrict == VINF_EM_RAW_EMULATE_INSTR.)
-         */
-        if (IOM_SUCCESS(rcStrict))
-        {
-            pCtx->rip += Cpu.opsize;
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-            return VBOXSTRICTRC_TODO(rcStrict);
-        }
-
-        if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
-        {
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-            rcStrict = emR3RawGuestTrap(pVM, pVCpu);
-            return VBOXSTRICTRC_TODO(rcStrict);
-        }
-        AssertMsg(rcStrict != VINF_TRPM_XCPT_DISPATCHED, ("Handle VINF_TRPM_XCPT_DISPATCHED\n"));
-
-        if (RT_FAILURE(rcStrict))
-        {
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-            return VBOXSTRICTRC_TODO(rcStrict);
-        }
-        AssertMsg(rcStrict == VINF_EM_RAW_EMULATE_INSTR || rcStrict == VINF_EM_RESCHEDULE_REM, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
-    }
-    STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
-    return emR3ExecuteInstruction(pVM, pVCpu, "IO: ");
-}
-
-
-/**
- * Handle a guest context trap.
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- */
-static int emR3RawGuestTrap(PVM pVM, PVMCPU pVCpu)
-{
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-
-    /*
-     * Get the trap info.
-     */
-    uint8_t         u8TrapNo;
-    TRPMEVENT       enmType;
-    RTGCUINT        uErrorCode;
-    RTGCUINTPTR     uCR2;
-    int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
-    if (RT_FAILURE(rc))
-    {
-        AssertReleaseMsgFailed(("No trap! (rc=%Rrc)\n", rc));
-        return rc;
-    }
-
-
-#if 1 /* Experimental: Review, disable if it causes trouble. */
-    /*
-     * Handle traps in patch code first.
-     *
-     * We catch a few of these cases in RC before returning to R3 (#PF, #GP, #BP)
-     * but several traps isn't handled specially by TRPM in RC and we end up here
-     * instead. One example is #DE.
-     */
-    uint32_t uCpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
-    if (    uCpl == 0
-        &&  PATMIsPatchGCAddr(pVM, pCtx->eip))
-    {
-        LogFlow(("emR3RawGuestTrap: trap %#x in patch code; eip=%08x\n", u8TrapNo, pCtx->eip));
-        return emR3PatchTrap(pVM, pVCpu, pCtx, rc);
-    }
-#endif
-
-    /*
-     * If the guest gate is marked unpatched, then we will check again if we can patch it.
-     * (This assumes that we've already tried and failed to dispatch the trap in
-     * RC for the gates that already has been patched. Which is true for most high
-     * volume traps, because these are handled specially, but not for odd ones like #DE.)
-     */
-    if (TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) == TRPM_INVALID_HANDLER)
-    {
-        CSAMR3CheckGates(pVM, u8TrapNo, 1);
-        Log(("emR3RawHandleRC: recheck gate %x -> valid=%d\n", u8TrapNo, TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) != TRPM_INVALID_HANDLER));
-
-        /* If it was successful, then we could go back to raw mode. */
-        if (TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) != TRPM_INVALID_HANDLER)
-        {
-            /* Must check pending forced actions as our IDT or GDT might be out of sync. */
-            rc = EMR3CheckRawForcedActions(pVM, pVCpu);
-            AssertRCReturn(rc, rc);
-
-            TRPMERRORCODE enmError = uErrorCode != ~0U
-                                   ? TRPM_TRAP_HAS_ERRORCODE
-                                   : TRPM_TRAP_NO_ERRORCODE;
-            rc = TRPMForwardTrap(pVCpu, CPUMCTX2CORE(pCtx), u8TrapNo, uErrorCode, enmError, TRPM_TRAP, -1);
-            if (rc == VINF_SUCCESS /* Don't use RT_SUCCESS */)
-            {
-                TRPMResetTrap(pVCpu);
-                return VINF_EM_RESCHEDULE_RAW;
-            }
-            AssertMsg(rc == VINF_EM_RAW_GUEST_TRAP, ("%Rrc\n", rc));
-        }
-    }
-
-    /*
-     * Scan kernel code that traps; we might not get another chance.
-     */
-    /** @todo move this up before the dispatching? */
-    if (    (pCtx->ss & X86_SEL_RPL) <= 1
-        &&  !pCtx->eflags.Bits.u1VM)
-    {
-        Assert(!PATMIsPatchGCAddr(pVM, pCtx->eip));
-        CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
-    }
-
-    /*
-     * Trap specific handling.
-     */
-    if (u8TrapNo == 6) /* (#UD) Invalid opcode. */
-    {
-        /*
-         * If MONITOR & MWAIT are supported, then interpret them here.
-         */
-        DISCPUSTATE cpu;
-        rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &cpu, "Guest Trap (#UD): ");
-        if (    RT_SUCCESS(rc)
-            && (cpu.pCurInstr->opcode == OP_MONITOR || cpu.pCurInstr->opcode == OP_MWAIT))
-        {
-            uint32_t u32Dummy, u32Features, u32ExtFeatures;
-            CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32ExtFeatures, &u32Features);
-            if (u32ExtFeatures & X86_CPUID_FEATURE_ECX_MONITOR)
-            {
-                rc = TRPMResetTrap(pVCpu);
-                AssertRC(rc);
-
-                uint32_t opsize;
-                rc = VBOXSTRICTRC_TODO(EMInterpretInstructionCPU(pVM, pVCpu, &cpu, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &opsize));
-                if (RT_SUCCESS(rc))
-                {
-                    pCtx->rip += cpu.opsize;
-                    return rc;
-                }
-                return emR3ExecuteInstruction(pVM, pVCpu, "Monitor: ");
-            }
-        }
-    }
-    else if (u8TrapNo == 13) /* (#GP) Privileged exception */
-    {
-        /*
-         * Handle I/O bitmap?
-         */
-        /** @todo We're not supposed to be here with a false guest trap concerning
-         *        I/O access. We can easily handle those in RC.  */
-        DISCPUSTATE cpu;
-        rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &cpu, "Guest Trap: ");
-        if (    RT_SUCCESS(rc)
-            &&  (cpu.pCurInstr->optype & OPTYPE_PORTIO))
-        {
-            /*
-             * We should really check the TSS for the IO bitmap, but it's not like this
-             * lazy approach really makes things worse.
-             */
-            rc = TRPMResetTrap(pVCpu);
-            AssertRC(rc);
-            return emR3ExecuteInstruction(pVM, pVCpu, "IO Guest Trap: ");
-        }
-    }
-
-#ifdef LOG_ENABLED
-    DBGFR3InfoLog(pVM, "cpumguest", "Guest trap");
-    DBGFR3DisasInstrCurrentLog(pVCpu, "Guest trap");
-
-    /* Get guest page information. */
-    uint64_t    fFlags = 0;
-    RTGCPHYS    GCPhys = 0;
-    int rc2 = PGMGstGetPage(pVCpu, uCR2, &fFlags, &GCPhys);
-    Log(("emR3RawGuestTrap: cs:eip=%04x:%08x: trap=%02x err=%08x cr2=%08x cr0=%08x%s: Phys=%RGp fFlags=%08llx %s %s %s%s rc2=%d\n",
-         pCtx->cs, pCtx->eip, u8TrapNo, uErrorCode, uCR2, (uint32_t)pCtx->cr0, (enmType == TRPM_SOFTWARE_INT) ? " software" : "",  GCPhys, fFlags,
-         fFlags & X86_PTE_P  ? "P " : "NP", fFlags & X86_PTE_US ? "U"  : "S",
-         fFlags & X86_PTE_RW ? "RW" : "R0", fFlags & X86_PTE_G  ? " G" : "", rc2));
-#endif
-
-    /*
-     * #PG has CR2.
-     * (Because of stuff like above we must set CR2 in a delayed fashion.)
-     */
-    if (u8TrapNo == 14 /* #PG */)
-        pCtx->cr2 = uCR2;
-
-    return VINF_EM_RESCHEDULE_REM;
-}
-
-
-/**
- * Handle a ring switch trap.
- * Need to do statistics and to install patches. The result is going to REM.
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM     VM handle.
- * @param   pVCpu       VMCPU handle.
- */
-static int emR3RawRingSwitch(PVM pVM, PVMCPU pVCpu)
-{
-    int         rc;
-    DISCPUSTATE Cpu;
-    PCPUMCTX    pCtx = pVCpu->em.s.pCtx;
-
-    /*
-     * sysenter, syscall & callgate
-     */
-    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "RSWITCH: ");
-    if (RT_SUCCESS(rc))
-    {
-        if (Cpu.pCurInstr->opcode == OP_SYSENTER)
-        {
-            if (pCtx->SysEnter.cs != 0)
-            {
-                rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
-                                        (SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0);
-                if (RT_SUCCESS(rc))
-                {
-                    DBGFR3DisasInstrCurrentLog(pVCpu, "Patched sysenter instruction");
-                    return VINF_EM_RESCHEDULE_RAW;
-                }
-            }
-        }
-
-#ifdef VBOX_WITH_STATISTICS
-        switch (Cpu.pCurInstr->opcode)
-        {
-            case OP_SYSENTER:
-                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysEnter);
-                break;
-            case OP_SYSEXIT:
-                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysExit);
-                break;
-            case OP_SYSCALL:
-                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysCall);
-                break;
-            case OP_SYSRET:
-                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysRet);
-                break;
-        }
-#endif
-    }
-    else
-        AssertRC(rc);
-
-    /* go to the REM to emulate a single instruction */
-    return emR3ExecuteInstruction(pVM, pVCpu, "RSWITCH: ");
-}
-
-
-/**
- * Handle a trap (\#PF or \#GP) in patch code
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM     VM handle.
- * @param   pVCpu   VMCPU handle.
- * @param   pCtx    CPU context
- * @param   gcret   GC return code
- */
-static int emR3PatchTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int gcret)
-{
-    uint8_t         u8TrapNo;
-    int             rc;
-    TRPMEVENT       enmType;
-    RTGCUINT        uErrorCode;
-    RTGCUINTPTR     uCR2;
-
-    Assert(PATMIsPatchGCAddr(pVM, pCtx->eip));
-
-    if (gcret == VINF_PATM_PATCH_INT3)
-    {
-        u8TrapNo   = 3;
-        uCR2       = 0;
-        uErrorCode = 0;
-    }
-    else if (gcret == VINF_PATM_PATCH_TRAP_GP)
-    {
-        /* No active trap in this case. Kind of ugly. */
-        u8TrapNo   = X86_XCPT_GP;
-        uCR2       = 0;
-        uErrorCode = 0;
-    }
-    else
-    {
-        rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
-        if (RT_FAILURE(rc))
-        {
-            AssertReleaseMsgFailed(("emR3PatchTrap: no trap! (rc=%Rrc) gcret=%Rrc\n", rc, gcret));
-            return rc;
-        }
-        /* Reset the trap as we'll execute the original instruction again. */
-        TRPMResetTrap(pVCpu);
-    }
-
-    /*
-     * Deal with traps inside patch code.
-     * (This code won't run outside GC.)
-     */
-    if (u8TrapNo != 1)
-    {
-#ifdef LOG_ENABLED
-        DBGFR3InfoLog(pVM, "cpumguest", "Trap in patch code");
-        DBGFR3DisasInstrCurrentLog(pVCpu, "Patch code");
-
-        DISCPUSTATE Cpu;
-        rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->eip, &Cpu, "Patch code: ");
-        if (    RT_SUCCESS(rc)
-            &&  Cpu.pCurInstr->opcode == OP_IRET)
-        {
-            uint32_t eip, selCS, uEFlags;
-
-            /* Iret crashes are bad as we have already changed the flags on the stack */
-            rc  = PGMPhysSimpleReadGCPtr(pVCpu, &eip,     pCtx->esp, 4);
-            rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selCS,   pCtx->esp+4, 4);
-            rc |= PGMPhysSimpleReadGCPtr(pVCpu, &uEFlags, pCtx->esp+8, 4);
-            if (rc == VINF_SUCCESS)
-            {
-                if (    (uEFlags & X86_EFL_VM)
-                    ||  (selCS & X86_SEL_RPL) == 3)
-                {
-                    uint32_t selSS, esp;
-
-                    rc |= PGMPhysSimpleReadGCPtr(pVCpu, &esp,     pCtx->esp + 12, 4);
-                    rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selSS,   pCtx->esp + 16, 4);
-
-                    if (uEFlags & X86_EFL_VM)
-                    {
-                        uint32_t selDS, selES, selFS, selGS;
-                        rc  = PGMPhysSimpleReadGCPtr(pVCpu, &selES,   pCtx->esp + 20, 4);
-                        rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selDS,   pCtx->esp + 24, 4);
-                        rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selFS,   pCtx->esp + 28, 4);
-                        rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selGS,   pCtx->esp + 32, 4);
-                        if (rc == VINF_SUCCESS)
-                        {
-                            Log(("Patch code: IRET->VM stack frame: return address %04X:%08RX32 eflags=%08x ss:esp=%04X:%08RX32\n", selCS, eip, uEFlags, selSS, esp));
-                            Log(("Patch code: IRET->VM stack frame: DS=%04X ES=%04X FS=%04X GS=%04X\n", selDS, selES, selFS, selGS));
-                        }
-                    }
-                    else
-                        Log(("Patch code: IRET stack frame: return address %04X:%08RX32 eflags=%08x ss:esp=%04X:%08RX32\n", selCS, eip, uEFlags, selSS, esp));
-                }
-                else
-                    Log(("Patch code: IRET stack frame: return address %04X:%08RX32 eflags=%08x\n", selCS, eip, uEFlags));
-            }
-        }
-#endif /* LOG_ENABLED */
-        Log(("emR3PatchTrap: in patch: eip=%08x: trap=%02x err=%08x cr2=%08x cr0=%08x\n",
-             pCtx->eip, u8TrapNo, uErrorCode, uCR2, (uint32_t)pCtx->cr0));
-
-        RTGCPTR pNewEip;
-        rc = PATMR3HandleTrap(pVM, pCtx, pCtx->eip, &pNewEip);
-        switch (rc)
-        {
-            /*
-             * Execute the faulting instruction.
-             */
-            case VINF_SUCCESS:
-            {
-                /** @todo execute a whole block */
-                Log(("emR3PatchTrap: Executing faulting instruction at new address %RGv\n", pNewEip));
-                if (!(pVCpu->em.s.pPatmGCState->uVMFlags & X86_EFL_IF))
-                    Log(("emR3PatchTrap: Virtual IF flag disabled!!\n"));
-
-                pCtx->eip = pNewEip;
-                AssertRelease(pCtx->eip);
-
-                if (pCtx->eflags.Bits.u1IF)
-                {
-                    /* Windows XP lets irets fault intentionally and then takes action based on the opcode; an
-                     * int3 patch overwrites it and leads to blue screens. Remove the patch in this case.
-                     */
-                    if (    u8TrapNo == X86_XCPT_GP
-                        &&  PATMIsInt3Patch(pVM, pCtx->eip, NULL, NULL))
-                    {
-                        /** @todo move to PATMR3HandleTrap */
-                        Log(("Possible Windows XP iret fault at %08RX32\n", pCtx->eip));
-                        PATMR3RemovePatch(pVM, pCtx->eip);
-                    }
-
-                    /** @todo Knoppix 5 regression when returning VINF_SUCCESS here and going back to raw mode. */
-                    /* Note: possibly because a reschedule is required (e.g. iret to V86 code) */
-
-                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
-                    /* Interrupts are enabled; just go back to the original instruction.
-                    return VINF_SUCCESS; */
-                }
-                return VINF_EM_RESCHEDULE_REM;
-            }
-
-            /*
-             * One instruction.
-             */
-            case VINF_PATCH_EMULATE_INSTR:
-                Log(("emR3PatchTrap: Emulate patched instruction at %RGv IF=%d VMIF=%x\n",
-                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
-                pCtx->eip = pNewEip;
-                AssertRelease(pCtx->eip);
-                return emR3ExecuteInstruction(pVM, pVCpu, "PATCHEMUL: ");
-
-            /*
-             * The patch was disabled, hand it to the REM.
-             */
-            case VERR_PATCH_DISABLED:
-                if (!(pVCpu->em.s.pPatmGCState->uVMFlags & X86_EFL_IF))
-                    Log(("emR3PatchTrap: Virtual IF flag disabled!!\n"));
-                pCtx->eip = pNewEip;
-                AssertRelease(pCtx->eip);
-
-                if (pCtx->eflags.Bits.u1IF)
-                {
-                    /*
-                     * The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
-                     */
-                    Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
-                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
-                }
-                return VINF_EM_RESCHEDULE_REM;
-
-            /* Force continued patch exection; usually due to write monitored stack. */
-            case VINF_PATCH_CONTINUE:
-                return VINF_SUCCESS;
-
-            /*
-             * Anything else is *fatal*.
-             */
-            default:
-                AssertReleaseMsgFailed(("Unknown return code %Rrc from PATMR3HandleTrap!\n", rc));
-                return VERR_IPE_UNEXPECTED_STATUS;
-        }
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Handle a privileged instruction.
- *
- * @returns VBox status code suitable for EM.
- * @param   pVM     VM handle.
- * @param   pVCpu   VMCPU handle;
- */
-static int emR3RawPrivileged(PVM pVM, PVMCPU pVCpu)
-{
-    PCPUMCTX    pCtx = pVCpu->em.s.pCtx;
-
-    Assert(!pCtx->eflags.Bits.u1VM);
-
-    if (PATMIsEnabled(pVM))
-    {
-        /*
-         * Check if in patch code.
-         */
-        if (PATMR3IsInsidePatchJump(pVM, pCtx->eip, NULL))
-        {
-#ifdef LOG_ENABLED
-            DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
-#endif
-            AssertMsgFailed(("FATAL ERROR: executing random instruction inside generated patch jump %08X\n", pCtx->eip));
-            return VERR_EM_RAW_PATCH_CONFLICT;
-        }
-        if (   (pCtx->ss & X86_SEL_RPL) == 0
-            && !pCtx->eflags.Bits.u1VM
-            && !PATMIsPatchGCAddr(pVM, pCtx->eip))
-        {
-            int rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
-                                        (SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0);
-            if (RT_SUCCESS(rc))
-            {
-#ifdef LOG_ENABLED
-                DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
-#endif
-                DBGFR3DisasInstrCurrentLog(pVCpu, "Patched privileged instruction");
-                return VINF_SUCCESS;
-            }
-        }
-    }
-
-#ifdef LOG_ENABLED
-    if (!PATMIsPatchGCAddr(pVM, pCtx->eip))
-    {
-        DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
-        DBGFR3DisasInstrCurrentLog(pVCpu, "Privileged instr: ");
-    }
-#endif
-
-    /*
-     * Instruction statistics and logging.
-     */
-    DISCPUSTATE Cpu;
-    int         rc;
-
-    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "PRIV: ");
-    if (RT_SUCCESS(rc))
-    {
-#ifdef VBOX_WITH_STATISTICS
-        PEMSTATS pStats = pVCpu->em.s.CTX_SUFF(pStats);
-        switch (Cpu.pCurInstr->opcode)
-        {
-            case OP_INVLPG:
-                STAM_COUNTER_INC(&pStats->StatInvlpg);
-                break;
-            case OP_IRET:
-                STAM_COUNTER_INC(&pStats->StatIret);
-                break;
-            case OP_CLI:
-                STAM_COUNTER_INC(&pStats->StatCli);
-                emR3RecordCli(pVM, pVCpu, pCtx->rip);
-                break;
-            case OP_STI:
-                STAM_COUNTER_INC(&pStats->StatSti);
-                break;
-            case OP_INSB:
-            case OP_INSWD:
-            case OP_IN:
-            case OP_OUTSB:
-            case OP_OUTSWD:
-            case OP_OUT:
-                AssertMsgFailed(("Unexpected privileged exception due to port IO\n"));
-                break;
-
-            case OP_MOV_CR:
-                if (Cpu.param1.flags & USE_REG_GEN32)
-                {
-                    //read
-                    Assert(Cpu.param2.flags & USE_REG_CR);
-                    Assert(Cpu.param2.base.reg_ctrl <= USE_REG_CR4);
-                    STAM_COUNTER_INC(&pStats->StatMovReadCR[Cpu.param2.base.reg_ctrl]);
-                }
-                else
-                {
-                    //write
-                    Assert(Cpu.param1.flags & USE_REG_CR);
-                    Assert(Cpu.param1.base.reg_ctrl <= USE_REG_CR4);
-                    STAM_COUNTER_INC(&pStats->StatMovWriteCR[Cpu.param1.base.reg_ctrl]);
-                }
-                break;
-
-            case OP_MOV_DR:
-                STAM_COUNTER_INC(&pStats->StatMovDRx);
-                break;
-            case OP_LLDT:
-                STAM_COUNTER_INC(&pStats->StatMovLldt);
-                break;
-            case OP_LIDT:
-                STAM_COUNTER_INC(&pStats->StatMovLidt);
-                break;
-            case OP_LGDT:
-                STAM_COUNTER_INC(&pStats->StatMovLgdt);
-                break;
-            case OP_SYSENTER:
-                STAM_COUNTER_INC(&pStats->StatSysEnter);
-                break;
-            case OP_SYSEXIT:
-                STAM_COUNTER_INC(&pStats->StatSysExit);
-                break;
-            case OP_SYSCALL:
-                STAM_COUNTER_INC(&pStats->StatSysCall);
-                break;
-            case OP_SYSRET:
-                STAM_COUNTER_INC(&pStats->StatSysRet);
-                break;
-            case OP_HLT:
-                STAM_COUNTER_INC(&pStats->StatHlt);
-                break;
-            default:
-                STAM_COUNTER_INC(&pStats->StatMisc);
-                Log4(("emR3RawPrivileged: opcode=%d\n", Cpu.pCurInstr->opcode));
-                break;
-        }
-#endif /* VBOX_WITH_STATISTICS */
-        if (    (pCtx->ss & X86_SEL_RPL) == 0
-            &&  !pCtx->eflags.Bits.u1VM
-            &&  SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT)
-        {
-            uint32_t size;
-
-            STAM_PROFILE_START(&pVCpu->em.s.StatPrivEmu, a);
-            switch (Cpu.pCurInstr->opcode)
-            {
-                case OP_CLI:
-                    pCtx->eflags.u32 &= ~X86_EFL_IF;
-                    Assert(Cpu.opsize == 1);
-                    pCtx->rip += Cpu.opsize;
-                    STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
-                    return VINF_EM_RESCHEDULE_REM; /* must go to the recompiler now! */
-
-                case OP_STI:
-                    pCtx->eflags.u32 |= X86_EFL_IF;
-                    EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + Cpu.opsize);
-                    Assert(Cpu.opsize == 1);
-                    pCtx->rip += Cpu.opsize;
-                    STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
-                    return VINF_SUCCESS;
-
-                case OP_HLT:
-                    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
-                    {
-                        PATMTRANSSTATE  enmState;
-                        RTGCPTR         pOrgInstrGC = PATMR3PatchToGCPtr(pVM, pCtx->eip, &enmState);
-
-                        if (enmState == PATMTRANS_OVERWRITTEN)
-                        {
-                            rc = PATMR3DetectConflict(pVM, pOrgInstrGC, pOrgInstrGC);
-                            Assert(rc == VERR_PATCH_DISABLED);
-                            /* Conflict detected, patch disabled */
-                            Log(("emR3RawPrivileged: detected conflict -> disabled patch at %08RX32\n", pCtx->eip));
-
-                            enmState = PATMTRANS_SAFE;
-                        }
-
-                        /* The translation had better be successful. Otherwise we can't recover. */
-                        AssertReleaseMsg(pOrgInstrGC && enmState != PATMTRANS_OVERWRITTEN, ("Unable to translate instruction address at %08RX32\n", pCtx->eip));
-                        if (enmState != PATMTRANS_OVERWRITTEN)
-                            pCtx->eip = pOrgInstrGC;
-                    }
-                    /* no break; we could just return VINF_EM_HALT here */
-
-                case OP_MOV_CR:
-                case OP_MOV_DR:
-#ifdef LOG_ENABLED
-                    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
-                    {
-                        DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
-                        DBGFR3DisasInstrCurrentLog(pVCpu, "Privileged instr: ");
-                    }
-#endif
-
-                    rc = VBOXSTRICTRC_TODO(EMInterpretInstructionCPU(pVM, pVCpu, &Cpu, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &size));
-                    if (RT_SUCCESS(rc))
-                    {
-                        pCtx->rip += Cpu.opsize;
-                        STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
-
-                        if (    Cpu.pCurInstr->opcode == OP_MOV_CR
-                            &&  Cpu.param1.flags == USE_REG_CR /* write */
-                           )
-                        {
-                            /* Deal with CR0 updates inside patch code that force
-                             * us to go to the recompiler.
-                             */
-                            if (   PATMIsPatchGCAddr(pVM, pCtx->rip)
-                                && (pCtx->cr0 & (X86_CR0_WP|X86_CR0_PG|X86_CR0_PE)) != (X86_CR0_WP|X86_CR0_PG|X86_CR0_PE))
-                            {
-                                PATMTRANSSTATE  enmState;
-                                RTGCPTR         pOrgInstrGC = PATMR3PatchToGCPtr(pVM, pCtx->rip, &enmState);
-
-                                Log(("Force recompiler switch due to cr0 (%RGp) update rip=%RGv -> %RGv (enmState=%d)\n", pCtx->cr0, pCtx->rip, pOrgInstrGC, enmState));
-                                if (enmState == PATMTRANS_OVERWRITTEN)
-                                {
-                                    rc = PATMR3DetectConflict(pVM, pOrgInstrGC, pOrgInstrGC);
-                                    Assert(rc == VERR_PATCH_DISABLED);
-                                    /* Conflict detected, patch disabled */
-                                    Log(("emR3RawPrivileged: detected conflict -> disabled patch at %RGv\n", (RTGCPTR)pCtx->rip));
-                                    enmState = PATMTRANS_SAFE;
-                                }
-                                /* The translation had better be successful. Otherwise we can't recover. */
-                                AssertReleaseMsg(pOrgInstrGC && enmState != PATMTRANS_OVERWRITTEN, ("Unable to translate instruction address at %RGv\n", (RTGCPTR)pCtx->rip));
-                                if (enmState != PATMTRANS_OVERWRITTEN)
-                                    pCtx->rip = pOrgInstrGC;
-                            }
-
-                            /* Reschedule is necessary as the execution/paging mode might have changed. */
-                            return VINF_EM_RESCHEDULE;
-                        }
-                        return rc; /* can return VINF_EM_HALT as well. */
-                    }
-                    AssertMsgReturn(rc == VERR_EM_INTERPRETER, ("%Rrc\n", rc), rc);
-                    break; /* fall back to the recompiler */
-            }
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
-        }
-    }
-
-    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
-        return emR3PatchTrap(pVM, pVCpu, pCtx, VINF_PATM_PATCH_TRAP_GP);
-
-    return emR3ExecuteInstruction(pVM, pVCpu, "PRIV");
-}
-
-
-/**
- * Update the forced rawmode execution modifier.
- *
- * This function is called when we're returning from the raw-mode loop(s). If we're
- * in patch code, it will set a flag forcing execution to be resumed in raw-mode,
- * if not in patch code, the flag will be cleared.
- *
- * We should never interrupt patch code while it's being executed. Cli patches can
- * contain big code blocks, but they are always executed with IF=0. Other patches
- * replace single instructions and should be atomic.
- *
- * @returns Updated rc.
- *
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU handle.
- * @param   pCtx    The guest CPU context.
- * @param   rc      The result code.
- */
-int emR3RawUpdateForceFlag(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
-{
-    if (PATMIsPatchGCAddr(pVM, pCtx->eip)) /** @todo check cs selector base/type */
-    {
-        /* ignore reschedule attempts. */
-        switch (rc)
-        {
-            case VINF_EM_RESCHEDULE:
-            case VINF_EM_RESCHEDULE_REM:
-                LogFlow(("emR3RawUpdateForceFlag: patch address -> force raw reschedule\n"));
-                rc = VINF_SUCCESS;
-                break;
-        }
-        pVCpu->em.s.fForceRAW = true;
-    }
-    else
-        pVCpu->em.s.fForceRAW = false;
-    return rc;
-}
-
-
-/**
- * Check for pending raw actions
- *
- * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
- *          EM statuses.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VMCPU handle.
- */
-VMMR3DECL(int) EMR3CheckRawForcedActions(PVM pVM, PVMCPU pVCpu)
-{
-    return emR3RawForcedActions(pVM, pVCpu, pVCpu->em.s.pCtx);
-}
-
-
-/**
- * Process raw-mode specific forced actions.
- *
- * This function is called when any FFs in the VM_FF_HIGH_PRIORITY_PRE_RAW_MASK is pending.
- *
- * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
- *          EM statuses.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU handle.
- * @param   pCtx        The guest CPUM register context.
- */
-static int emR3RawForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
-{
-    /*
-     * Note that the order is *vitally* important!
-     * Also note that SELMR3UpdateFromCPUM may trigger VM_FF_SELM_SYNC_TSS.
-     */
-
-
-    /*
-     * Sync selector tables.
-     */
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT))
-    {
-        int rc = SELMR3UpdateFromCPUM(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Sync IDT.
-     *
-     * The CSAMR3CheckGates call in TRPMR3SyncIDT may call PGMPrefetchPage
-     * and PGMShwModifyPage, so we're in for trouble if for instance a
-     * PGMSyncCR3+pgmR3PoolClearAll is pending.
-     */
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TRPM_SYNC_IDT))
-    {
-        if (   VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
-            && EMIsRawRing0Enabled(pVM)
-            && CSAMIsEnabled(pVM))
-        {
-            int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
-            if (RT_FAILURE(rc))
-                return rc;
-        }
-
-        int rc = TRPMR3SyncIDT(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Sync TSS.
-     */
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_TSS))
-    {
-        int rc = SELMR3SyncTSS(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Sync page directory.
-     */
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
-    {
-        Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
-        int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
-        if (RT_FAILURE(rc))
-            return rc;
-
-        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
-
-        /* Prefetch pages for EIP and ESP. */
-        /** @todo This is rather expensive. Should investigate if it really helps at all. */
-        rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->rip));
-        if (rc == VINF_SUCCESS)
-            rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->rsp));
-        if (rc != VINF_SUCCESS)
-        {
-            if (rc != VINF_PGM_SYNC_CR3)
-            {
-                AssertLogRelMsgReturn(RT_FAILURE(rc), ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
-                return rc;
-            }
-            rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
-            if (RT_FAILURE(rc))
-                return rc;
-        }
-        /** @todo maybe prefetch the supervisor stack page as well */
-        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
-    }
-
-    /*
-     * Allocate handy pages (just in case the above actions have consumed some pages).
-     */
-    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
-    {
-        int rc = PGMR3PhysAllocateHandyPages(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Check whether we're out of memory now.
-     *
-     * This may stem from some of the above actions or operations that has been executed
-     * since we ran FFs. The allocate handy pages must for instance always be followed by
-     * this check.
-     */
-    if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
-        return VINF_EM_NO_MEMORY;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Executes raw code.
- *
- * This function contains the raw-mode version of the inner
- * execution loop (the outer loop being in EMR3ExecuteVM()).
- *
- * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
- *          VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   pfFFDone    Where to store an indicator telling whether or not
- *                      FFs were done before returning.
- */
-int emR3RawExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
-{
-    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatRAWTotal, a);
-
-    int      rc = VERR_INTERNAL_ERROR;
-    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
-    LogFlow(("emR3RawExecute: (cs:eip=%04x:%08x)\n", pCtx->cs, pCtx->eip));
-    pVCpu->em.s.fForceRAW = false;
-    *pfFFDone = false;
-
-
-    /*
-     *
-     * Spin till we get a forced action or raw mode status code resulting in
-     * in anything but VINF_SUCCESS or VINF_EM_RESCHEDULE_RAW.
-     *
-     */
-    for (;;)
-    {
-        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatRAWEntry, b);
-
-        /*
-         * Check various preconditions.
-         */
-#ifdef VBOX_STRICT
-        Assert(REMR3QueryPendingInterrupt(pVM, pVCpu) == REM_NO_PENDING_IRQ);
-        Assert(pCtx->eflags.Bits.u1VM || (pCtx->ss & X86_SEL_RPL) == 3 || (pCtx->ss & X86_SEL_RPL) == 0);
-        AssertMsg(   (pCtx->eflags.u32 & X86_EFL_IF)
-                  || PATMShouldUseRawMode(pVM, (RTGCPTR)pCtx->eip),
-                  ("Tried to execute code with IF at EIP=%08x!\n", pCtx->eip));
-        if (    !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
-            &&  PGMMapHasConflicts(pVM))
-        {
-            PGMMapCheck(pVM);
-            AssertMsgFailed(("We should not get conflicts any longer!!!\n"));
-            return VERR_INTERNAL_ERROR;
-        }
-#endif /* VBOX_STRICT */
-
-        /*
-         * Process high priority pre-execution raw-mode FFs.
-         */
-        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
-        {
-            rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
-            if (rc != VINF_SUCCESS)
-                break;
-        }
-
-        /*
-         * If we're going to execute ring-0 code, the guest state needs to
-         * be modified a bit and some of the state components (IF, SS/CS RPL,
-         * and perhaps EIP) needs to be stored with PATM.
-         */
-        rc = CPUMR3RawEnter(pVCpu, NULL);
-        if (rc != VINF_SUCCESS)
-        {
-            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWEntry, b);
-            break;
-        }
-
-        /*
-         * Scan code before executing it. Don't bother with user mode or V86 code
-         */
-        if (    (pCtx->ss & X86_SEL_RPL) <= 1
-            &&  !pCtx->eflags.Bits.u1VM
-            && !PATMIsPatchGCAddr(pVM, pCtx->eip))
-        {
-            STAM_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatRAWEntry, b);
-            CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
-            STAM_PROFILE_ADV_RESUME(&pVCpu->em.s.StatRAWEntry, b);
-            if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
-                ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
-            {
-                rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
-                if (rc != VINF_SUCCESS)
-                {
-                    rc = CPUMR3RawLeave(pVCpu, NULL, rc);
-                    break;
-                }
-            }
-        }
-
-#ifdef LOG_ENABLED
-        /*
-         * Log important stuff before entering GC.
-         */
-        PPATMGCSTATE pGCState = PATMR3QueryGCStateHC(pVM);
-        if (pCtx->eflags.Bits.u1VM)
-            Log(("RV86: %04X:%08X IF=%d VMFlags=%x\n", pCtx->cs, pCtx->eip, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags));
-        else if ((pCtx->ss & X86_SEL_RPL) == 1)
-        {
-            bool fCSAMScanned = CSAMIsPageScanned(pVM, (RTGCPTR)pCtx->eip);
-            Log(("RR0: %08X ESP=%08X IF=%d VMFlags=%x PIF=%d CPL=%d (Scanned=%d)\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags, pGCState->fPIF, (pCtx->ss & X86_SEL_RPL), fCSAMScanned));
-        }
-        else if ((pCtx->ss & X86_SEL_RPL) == 3)
-            Log(("RR3: %08X ESP=%08X IF=%d VMFlags=%x\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags));
-#endif /* LOG_ENABLED */
-
-
-
-        /*
-         * Execute the code.
-         */
-        STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWEntry, b);
-        if (RT_LIKELY(EMR3IsExecutionAllowed(pVM, pVCpu)))
-        {
-            STAM_PROFILE_START(&pVCpu->em.s.StatRAWExec, c);
-            rc = VMMR3RawRunGC(pVM, pVCpu);
-            STAM_PROFILE_STOP(&pVCpu->em.s.StatRAWExec, c);
-        }
-        else
-        {
-            /* Give up this time slice; virtual time continues */
-            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
-            RTThreadSleep(5);
-            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
-            rc = VINF_SUCCESS;
-        }
-        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatRAWTail, d);
-
-        LogFlow(("RR0-E: %08X ESP=%08X IF=%d VMFlags=%x PIF=%d CPL=%d\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags, pGCState->fPIF, (pCtx->ss & X86_SEL_RPL)));
-        LogFlow(("VMMR3RawRunGC returned %Rrc\n", rc));
-
-
-
-        /*
-         * Restore the real CPU state and deal with high priority post
-         * execution FFs before doing anything else.
-         */
-        rc = CPUMR3RawLeave(pVCpu, NULL, rc);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
-            rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
-
-#ifdef VBOX_STRICT
-        /*
-         * Assert TSS consistency & rc vs patch code.
-         */
-        if (   !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_TSS | VMCPU_FF_SELM_SYNC_GDT) /* GDT implies TSS at the moment. */
-            &&  EMIsRawRing0Enabled(pVM))
-            SELMR3CheckTSS(pVM);
-        switch (rc)
-        {
-            case VINF_SUCCESS:
-            case VINF_EM_RAW_INTERRUPT:
-            case VINF_PATM_PATCH_TRAP_PF:
-            case VINF_PATM_PATCH_TRAP_GP:
-            case VINF_PATM_PATCH_INT3:
-            case VINF_PATM_CHECK_PATCH_PAGE:
-            case VINF_EM_RAW_EXCEPTION_PRIVILEGED:
-            case VINF_EM_RAW_GUEST_TRAP:
-            case VINF_EM_RESCHEDULE_RAW:
-                break;
-
-            default:
-                if (PATMIsPatchGCAddr(pVM, pCtx->eip) && !(pCtx->eflags.u32 & X86_EFL_TF))
-                    LogIt(NULL, 0, LOG_GROUP_PATM, ("Patch code interrupted at %RRv for reason %Rrc\n", (RTRCPTR)CPUMGetGuestEIP(pVCpu), rc));
-                break;
-        }
-        /*
-         * Let's go paranoid!
-         */
-        if (    !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
-            &&  PGMMapHasConflicts(pVM))
-        {
-            PGMMapCheck(pVM);
-            AssertMsgFailed(("We should not get conflicts any longer!!! rc=%Rrc\n", rc));
-            return VERR_INTERNAL_ERROR;
-        }
-#endif /* VBOX_STRICT */
-
-        /*
-         * Process the returned status code.
-         */
-        if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
-        {
-            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
-            break;
-        }
-        rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
-        if (rc != VINF_SUCCESS)
-        {
-            rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
-            if (rc != VINF_SUCCESS)
-            {
-                STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
-                break;
-            }
-        }
-
-        /*
-         * Check and execute forced actions.
-         */
-#ifdef VBOX_HIGH_RES_TIMERS_HACK
-        TMTimerPollVoid(pVM, pVCpu);
-#endif
-        STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
-        if (    VM_FF_ISPENDING(pVM, ~VM_FF_HIGH_PRIORITY_PRE_RAW_MASK | VM_FF_PGM_NO_MEMORY)
-            ||  VMCPU_FF_ISPENDING(pVCpu, ~VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
-        {
-            Assert(pCtx->eflags.Bits.u1VM || (pCtx->ss & X86_SEL_RPL) != 1);
-
-            STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatRAWTotal, a);
-            rc = emR3ForcedActions(pVM, pVCpu, rc);
-            STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatRAWTotal, a);
-            if (    rc != VINF_SUCCESS
-                &&  rc != VINF_EM_RESCHEDULE_RAW)
-            {
-                rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
-                if (rc != VINF_SUCCESS)
-                {
-                    *pfFFDone = true;
-                    break;
-                }
-            }
-        }
-    }
-
-    /*
-     * Return to outer loop.
-     */
-#if defined(LOG_ENABLED) && defined(DEBUG)
-    RTLogFlush(NULL);
-#endif
-    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTotal, a);
-    return rc;
-}
-
Index: unk/src/VBox/VMM/FTM.cpp
===================================================================
--- /trunk/src/VBox/VMM/FTM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1351 +1,0 @@
-/* $Id$ */
-/** @file
- * FTM - Fault Tolerance Manager
- */
-
-/*
- * Copyright (C) 2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_FTM
-#include "FTMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/vmm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/ssm.h>
-#include <VBox/log.h>
-#include <VBox/pgm.h>
-#include <VBox/pdm.h>
-
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-#include <iprt/string.h>
-#include <iprt/mem.h>
-#include <iprt/tcp.h>
-#include <iprt/socket.h>
-#include <iprt/semaphore.h>
-#include <iprt/asm.h>
-
-#include <include/internal/vm.h>
-#include <include/internal/em.h>
-#include <include/internal/pgm.h>
-
-/*******************************************************************************
- * Structures and Typedefs                                                     *
- *******************************************************************************/
-
-/**
- * TCP stream header.
- *
- * This is an extra layer for fixing the problem with figuring out when the SSM
- * stream ends.
- */
-typedef struct FTMTCPHDR
-{
-    /** Magic value. */
-    uint32_t    u32Magic;
-    /** The size of the data block following this header.
-     * 0 indicates the end of the stream, while UINT32_MAX indicates
-     * cancelation. */
-    uint32_t    cb;
-} FTMTCPHDR;
-/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
-#define FTMTCPHDR_MAGIC       UINT32_C(0x19471205)
-/** The max block size. */
-#define FTMTCPHDR_MAX_SIZE    UINT32_C(0x00fffff8)
-
-/**
- * TCP stream header.
- *
- * This is an extra layer for fixing the problem with figuring out when the SSM
- * stream ends.
- */
-typedef struct FTMTCPHDRMEM
-{
-    /** Magic value. */
-    uint32_t    u32Magic;
-    /** Size (Uncompressed) of the pages following the header. */
-    uint32_t    cbPageRange;
-    /** GC Physical address of the page(s) to sync. */
-    RTGCPHYS    GCPhys;
-    /** The size of the data block following this header.
-     * 0 indicates the end of the stream, while UINT32_MAX indicates
-     * cancelation. */
-    uint32_t    cb;
-} FTMTCPHDRMEM;
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
-
-static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser);
-
-/**
- * Initializes the FTM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) FTMR3Init(PVM pVM)
-{
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
-    AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
-
-    /** @todo saved state for master nodes! */
-    pVM->ftm.s.pszAddress               = NULL;
-    pVM->ftm.s.pszPassword              = NULL;
-    pVM->fFaultTolerantMaster           = false;
-    pVM->ftm.s.fIsStandbyNode           = false;
-    pVM->ftm.s.standby.hServer          = NIL_RTTCPSERVER;
-    pVM->ftm.s.hShutdownEvent           = NIL_RTSEMEVENT;
-    pVM->ftm.s.hSocket                  = NIL_RTSOCKET;
-
-    /*
-     * Initialize the PGM critical section.
-     */
-    int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Register statistics.
-     */
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem,               STAMTYPE_COUNTER, "/FT/Received/Mem",                  STAMUNIT_BYTES, "The amount of memory pages that was received.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState,             STAMTYPE_COUNTER, "/FT/Received/State",                STAMUNIT_BYTES, "The amount of state information that was received.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem,                   STAMTYPE_COUNTER, "/FT/Sent/Mem",                      STAMUNIT_BYTES, "The amount of memory pages that was sent.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState,                 STAMTYPE_COUNTER, "/FT/Sent/State",                    STAMUNIT_BYTES, "The amount of state information that was sent.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM,                   STAMTYPE_COUNTER, "/FT/Sync/DeltaVM",                  STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync,                  STAMTYPE_COUNTER, "/FT/Sync/Full",                     STAMUNIT_OCCURENCES, "Number of full vm syncs.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem,                  STAMTYPE_COUNTER, "/FT/Sync/DeltaMem",                 STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage,         STAMTYPE_COUNTER, "/FT/Checkpoint/Storage",            STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
-    STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork,         STAMTYPE_COUNTER, "/FT/Checkpoint/Network",            STAMUNIT_OCCURENCES, "Number of network checkpoints.");
-#ifdef VBOX_WITH_STATISTICS
-    STAM_REG(pVM,     &pVM->ftm.s.StatCheckpoint,                STAMTYPE_PROFILE, "/FT/Checkpoint",                    STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
-    STAM_REG(pVM,     &pVM->ftm.s.StatCheckpointPause,           STAMTYPE_PROFILE, "/FT/Checkpoint/Pause",              STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
-    STAM_REG(pVM,     &pVM->ftm.s.StatCheckpointResume,          STAMTYPE_PROFILE, "/FT/Checkpoint/Resume",             STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
-    STAM_REG(pVM,     &pVM->ftm.s.StatSentMemRAM,                STAMTYPE_COUNTER, "/FT/Sent/Mem/RAM",                  STAMUNIT_BYTES, "The amount of memory pages that was sent.");
-    STAM_REG(pVM,     &pVM->ftm.s.StatSentMemMMIO2,              STAMTYPE_COUNTER, "/FT/Sent/Mem/MMIO2",                STAMUNIT_BYTES, "The amount of memory pages that was sent.");
-    STAM_REG(pVM,     &pVM->ftm.s.StatSentMemShwROM,             STAMTYPE_COUNTER, "/FT/Sent/Mem/ShwROM",               STAMUNIT_BYTES, "The amount of memory pages that was sent.");
-    STAM_REG(pVM,     &pVM->ftm.s.StatSentStateWrite,            STAMTYPE_COUNTER, "/FT/Sent/State/Writes",             STAMUNIT_BYTES, "The nr of write calls.");
-#endif
-    return VINF_SUCCESS;
-}
-
-/**
- * Terminates the FTM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM itself is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) FTMR3Term(PVM pVM)
-{
-    if (pVM->ftm.s.hShutdownEvent != NIL_RTSEMEVENT)
-    {
-        RTSemEventDestroy(pVM->ftm.s.hShutdownEvent);
-        pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
-    }
-    if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
-    {
-        RTTcpClientClose(pVM->ftm.s.hSocket);
-        pVM->ftm.s.hSocket = NIL_RTSOCKET;
-    }
-    if (pVM->ftm.s.standby.hServer)
-    {
-        RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
-        pVM->ftm.s.standby.hServer = NULL;
-    }
-    if (pVM->ftm.s.pszAddress)
-        RTMemFree(pVM->ftm.s.pszAddress);
-    if (pVM->ftm.s.pszPassword)
-        RTMemFree(pVM->ftm.s.pszPassword);
-
-    /* Remove all pending memory updates. */
-    if (pVM->ftm.s.standby.pPhysPageTree)
-    {
-        RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, NULL);
-        pVM->ftm.s.standby.pPhysPageTree = NULL;
-    }
-
-    pVM->ftm.s.pszAddress  = NULL;
-    pVM->ftm.s.pszPassword = NULL;
-
-    PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
-    return VINF_SUCCESS;
-}
-
-
-static int ftmR3TcpWriteACK(PVM pVM)
-{
-    int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
-    }
-    return rc;
-}
-
-
-static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
-{
-    char    szMsg[256];
-    size_t  cch;
-    if (pszMsgText && *pszMsgText)
-    {
-        cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
-        for (size_t off = 6; off + 1 < cch; off++)
-            if (szMsg[off] == '\n')
-                szMsg[off] = '\r';
-    }
-    else
-        cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
-    int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
-    if (RT_FAILURE(rc))
-        LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
-    return rc;
-}
-
-/**
- * Reads a string from the socket.
- *
- * @returns VBox status code.
- *
- * @param   pState      The teleporter state structure.
- * @param   pszBuf      The output buffer.
- * @param   cchBuf      The size of the output buffer.
- *
- */
-static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
-{
-    char       *pszStart = pszBuf;
-    RTSOCKET    Sock     = pVM->ftm.s.hSocket;
-
-    AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
-    *pszBuf = '\0';
-
-    /* dead simple approach. */
-    for (;;)
-    {
-        char ch;
-        int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
-            return rc;
-        }
-        if (    ch == '\n'
-            ||  ch == '\0')
-            return VINF_SUCCESS;
-        if (cchBuf <= 1)
-        {
-            LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
-            return VERR_BUFFER_OVERFLOW;
-        }
-        *pszBuf++ = ch;
-        *pszBuf = '\0';
-        cchBuf--;
-    }
-}
-
-/**
- * Reads an ACK or NACK.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM to operate on.
- * @param   pszWhich            Which ACK is this this?
- * @param   pszNAckMsg          Optional NACK message.
- */
-static int ftmR3TcpReadACK(PVM pVM, const char *pszWhich, const char *pszNAckMsg = NULL)
-{
-    char szMsg[256];
-    int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
-    if (RT_FAILURE(rc))
-        return rc;
-
-    if (!strcmp(szMsg, "ACK"))
-        return VINF_SUCCESS;
-
-    if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
-    {
-        char *pszMsgText = strchr(szMsg, ';');
-        if (pszMsgText)
-            *pszMsgText++ = '\0';
-
-        int32_t vrc2;
-        rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
-        if (rc == VINF_SUCCESS)
-        {
-            /*
-             * Well formed NACK, transform it into an error.
-             */
-            if (pszNAckMsg)
-            {
-                LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
-                return VERR_INTERNAL_ERROR;
-            }
-
-            if (pszMsgText)
-            {
-                pszMsgText = RTStrStrip(pszMsgText);
-                for (size_t off = 0; pszMsgText[off]; off++)
-                    if (pszMsgText[off] == '\r')
-                        pszMsgText[off] = '\n';
-
-                LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
-            }
-            return VERR_INTERNAL_ERROR_2;
-        }
-
-        if (pszMsgText)
-            pszMsgText[-1] = ';';
-    }
-    return VERR_INTERNAL_ERROR_3;
-}
-
-/**
- * Submitts a command to the destination and waits for the ACK.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM to operate on.
- * @param   pszCommand          The command.
- * @param   fWaitForAck         Whether to wait for the ACK.
- */
-static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
-{
-    int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (!fWaitForAck)
-        return VINF_SUCCESS;
-    return ftmR3TcpReadACK(pVM, pszCommand);
-}
-
-/**
- * @copydoc SSMSTRMOPS::pfnWrite
- */
-static DECLCALLBACK(int) ftmR3TcpOpWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
-{
-    PVM pVM = (PVM)pvUser;
-
-    AssertReturn(cbToWrite > 0, VINF_SUCCESS);
-    AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
-    AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
-
-    STAM_COUNTER_INC(&pVM->ftm.s.StatSentStateWrite);
-    for (;;)
-    {
-        FTMTCPHDR Hdr;
-        Hdr.u32Magic = FTMTCPHDR_MAGIC;
-        Hdr.cb       = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
-        int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
-            return rc;
-        }
-        pVM->ftm.s.StatSentState.c      += Hdr.cb + sizeof(Hdr);
-        pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
-        if (Hdr.cb == cbToWrite)
-            return VINF_SUCCESS;
-
-        /* advance */
-        cbToWrite -= Hdr.cb;
-        pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
-    }
-}
-
-
-/**
- * Selects and poll for close condition.
- *
- * We can use a relatively high poll timeout here since it's only used to get
- * us out of error paths.  In the normal cause of events, we'll get a
- * end-of-stream header.
- *
- * @returns VBox status code.
- *
- * @param   pState          The teleporter state data.
- */
-static int ftmR3TcpReadSelect(PVM pVM)
-{
-    int rc;
-    do
-    {
-        rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
-        if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
-        {
-            pVM->ftm.s.syncstate.fIOError = true;
-            LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
-            break;
-        }
-        if (pVM->ftm.s.syncstate.fStopReading)
-        {
-            rc = VERR_EOF;
-            break;
-        }
-    } while (rc == VERR_TIMEOUT);
-    return rc;
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnRead
- */
-static DECLCALLBACK(int) ftmR3TcpOpRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
-{
-    PVM pVM = (PVM)pvUser;
-    AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
-
-    for (;;)
-    {
-        int rc;
-
-        /*
-         * Check for various conditions and may have been signalled.
-         */
-        if (pVM->ftm.s.syncstate.fEndOfStream)
-            return VERR_EOF;
-        if (pVM->ftm.s.syncstate.fStopReading)
-            return VERR_EOF;
-        if (pVM->ftm.s.syncstate.fIOError)
-            return VERR_IO_GEN_FAILURE;
-
-        /*
-         * If there is no more data in the current block, read the next
-         * block header.
-         */
-        if (!pVM->ftm.s.syncstate.cbReadBlock)
-        {
-            rc = ftmR3TcpReadSelect(pVM);
-            if (RT_FAILURE(rc))
-                return rc;
-            FTMTCPHDR Hdr;
-            rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
-            if (RT_FAILURE(rc))
-            {
-                pVM->ftm.s.syncstate.fIOError = true;
-                LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
-                return rc;
-            }
-            pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
-
-            if (RT_UNLIKELY(   Hdr.u32Magic != FTMTCPHDR_MAGIC
-                            || Hdr.cb > FTMTCPHDR_MAX_SIZE
-                            || Hdr.cb == 0))
-            {
-                if (    Hdr.u32Magic == FTMTCPHDR_MAGIC
-                    &&  (   Hdr.cb == 0
-                         || Hdr.cb == UINT32_MAX)
-                   )
-                {
-                    pVM->ftm.s.syncstate.fEndOfStream = true;
-                    pVM->ftm.s.syncstate.cbReadBlock  = 0;
-                    return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
-                }
-                pVM->ftm.s.syncstate.fIOError = true;
-                LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
-                return VERR_IO_GEN_FAILURE;
-            }
-
-            pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
-            if (pVM->ftm.s.syncstate.fStopReading)
-                return VERR_EOF;
-        }
-
-        /*
-         * Read more data.
-         */
-        rc = ftmR3TcpReadSelect(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
-        rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
-        if (RT_FAILURE(rc))
-        {
-            pVM->ftm.s.syncstate.fIOError = true;
-            LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
-            return rc;
-        }
-        if (pcbRead)
-        {
-            cb = (uint32_t)*pcbRead;
-            pVM->ftm.s.StatReceivedState.c   += cb;
-            pVM->ftm.s.syncstate.uOffStream  += cb;
-            pVM->ftm.s.syncstate.cbReadBlock -= cb;
-            return VINF_SUCCESS;
-        }
-        pVM->ftm.s.StatReceivedState.c   += cb;
-        pVM->ftm.s.syncstate.uOffStream  += cb;
-        pVM->ftm.s.syncstate.cbReadBlock -= cb;
-        if (cbToRead == cb)
-            return VINF_SUCCESS;
-
-        /* Advance to the next block. */
-        cbToRead -= cb;
-        pvBuf = (uint8_t *)pvBuf + cb;
-    }
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnSeek
- */
-static DECLCALLBACK(int) ftmR3TcpOpSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
-{
-    return VERR_NOT_SUPPORTED;
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnTell
- */
-static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
-{
-    PVM pVM = (PVM)pvUser;
-    return pVM->ftm.s.syncstate.uOffStream;
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnSize
- */
-static DECLCALLBACK(int) ftmR3TcpOpSize(void *pvUser, uint64_t *pcb)
-{
-    return VERR_NOT_SUPPORTED;
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnIsOk
- */
-static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
-{
-    PVM pVM = (PVM)pvUser;
-
-    if (pVM->fFaultTolerantMaster)
-    {
-        /* Poll for incoming NACKs and errors from the other side */
-        int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
-        if (rc != VERR_TIMEOUT)
-        {
-            if (RT_SUCCESS(rc))
-            {
-                LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
-                rc = VERR_SSM_CANCELLED;
-            }
-            else
-                LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
-            return rc;
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnClose
- */
-static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
-{
-    PVM pVM = (PVM)pvUser;
-
-    if (pVM->fFaultTolerantMaster)
-    {
-        FTMTCPHDR EofHdr;
-        EofHdr.u32Magic = FTMTCPHDR_MAGIC;
-        EofHdr.cb       = fCanceled ? UINT32_MAX : 0;
-        int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
-            return rc;
-        }
-    }
-    else
-    {
-        ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Method table for a TCP based stream.
- */
-static SSMSTRMOPS const g_ftmR3TcpOps =
-{
-    SSMSTRMOPS_VERSION,
-    ftmR3TcpOpWrite,
-    ftmR3TcpOpRead,
-    ftmR3TcpOpSeek,
-    ftmR3TcpOpTell,
-    ftmR3TcpOpSize,
-    ftmR3TcpOpIsOk,
-    ftmR3TcpOpClose,
-    SSMSTRMOPS_VERSION
-};
-
-
-/**
- * VMR3ReqCallWait callback
- *
- * @param   pVM         The VM handle.
- *
- */
-static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
-{
-    int rc = PGMR3PhysWriteProtectRAM(pVM);
-    AssertRC(rc);
-}
-
-
-/**
- * Sync the VM state
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- */
-static int ftmR3PerformFullSync(PVM pVM)
-{
-    bool fSuspended = false;
-
-    int rc = VMR3Suspend(pVM);
-    AssertRCReturn(rc, rc);
-
-    STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
-
-    RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
-
-    /* Reset the sync state. */
-    pVM->ftm.s.syncstate.uOffStream   = 0;
-    pVM->ftm.s.syncstate.cbReadBlock  = 0;
-    pVM->ftm.s.syncstate.fStopReading = false;
-    pVM->ftm.s.syncstate.fIOError     = false;
-    pVM->ftm.s.syncstate.fEndOfStream = false;
-
-    rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
-    AssertRC(rc);
-
-    pVM->ftm.s.fDeltaLoadSaveActive = false;
-    rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
-    AssertRC(rc);
-
-    rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
-    AssertRC(rc);
-
-    RTSocketRelease(pVM->ftm.s.hSocket);
-
-    /* Write protect all memory. */
-    rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
-    AssertRCReturn(rc, rc);
-
-    rc = VMR3Resume(pVM);
-    AssertRC(rc);
-
-    return rc;
-}
-
-
-/**
- * PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
- *
- * @param   pVM             VM Handle.
- * @param   GCPhys          GC physical address
- * @param   pRange          HC virtual address of the page(s)
- * @param   cbRange         Size of the dirty range in bytes.
- * @param   pvUser          User argument
- */
-static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t *pRange, unsigned cbRange, void *pvUser)
-{
-    FTMTCPHDRMEM Hdr;
-    Hdr.u32Magic    = FTMTCPHDR_MAGIC;
-    Hdr.GCPhys      = GCPhys;
-    Hdr.cbPageRange = cbRange;
-    Hdr.cb          = cbRange;
-    /** @todo compress page(s). */
-    int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
-        return rc;
-    }
-    pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
-
-#ifdef VBOX_WITH_STATISTICS
-    switch (PGMPhysGetPageType(pVM, GCPhys))
-    {
-    case PGMPAGETYPE_RAM:
-        pVM->ftm.s.StatSentMemRAM.c    += Hdr.cb + sizeof(Hdr);
-        break;
-
-    case PGMPAGETYPE_MMIO2:
-        pVM->ftm.s.StatSentMemMMIO2.c  += Hdr.cb + sizeof(Hdr);
-        break;
-
-    case PGMPAGETYPE_ROM_SHADOW:
-        pVM->ftm.s.StatSentMemShwROM.c += Hdr.cb + sizeof(Hdr);
-        break;
-
-    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
-        AssertFailed();
-        break;
-
-    default:
-        AssertFailed();
-        break;
-    }
-#endif
-
-    return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
-}
-
-/**
- * Thread function which starts syncing process for this master VM
- *
- * @param   Thread      The thread id.
- * @param   pvUser      Not used
- * @return  VINF_SUCCESS (ignored).
- *
- */
-static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
-{
-    int rc  = VINF_SUCCESS;
-    PVM pVM = (PVM)pvUser;
-
-    for (;;)
-    {
-        /*
-         * Try connect to the standby machine.
-         */
-        Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
-        rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
-        if (RT_SUCCESS(rc))
-        {
-            Log(("ftmR3MasterThread: CONNECTED\n"));
-
-            /* Disable Nagle. */
-            rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /*fEnable*/);
-            AssertRC(rc);
-
-            /* Read and check the welcome message. */
-            char szLine[RT_MAX(128, sizeof(g_szWelcome))];
-            RT_ZERO(szLine);
-            rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
-            if (    RT_SUCCESS(rc)
-                &&  !strcmp(szLine, g_szWelcome))
-            {
-                /* password */
-                if (pVM->ftm.s.pszPassword)
-                    rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
-
-                if (RT_SUCCESS(rc))
-                {
-                    /* ACK */
-                    rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
-                    if (RT_SUCCESS(rc))
-                    {
-                        /** todo: verify VM config. */
-                        break;
-                    }
-                }
-            }
-            /* Failed, so don't bother anymore. */
-            return VINF_SUCCESS;
-        }
-        rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, 1000 /* 1 second */);
-        if (rc != VERR_TIMEOUT)
-            return VINF_SUCCESS;    /* told to quit */
-    }
-
-    /* Successfully initialized the connection to the standby node.
-     * Start the sync process.
-     */
-
-    /* First sync all memory and write protect everything so
-     * we can send changed pages later on.
-     */
-
-    rc = ftmR3PerformFullSync(pVM);
-
-    for (;;)
-    {
-        rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
-        if (rc != VERR_TIMEOUT)
-            break;    /* told to quit */
-
-        if (!pVM->ftm.s.fCheckpointingActive)
-        {
-            rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
-            AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-
-            rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
-            AssertRC(rc);
-
-            /* sync the changed memory with the standby node. */
-            /* Write protect all memory. */
-            if (!pVM->ftm.s.fCheckpointingActive)
-            {
-                rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
-                AssertRC(rc);
-            }
-
-            /* Enumerate all dirty pages and send them to the standby VM. */
-            if (!pVM->ftm.s.fCheckpointingActive)
-            {
-                rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
-                Assert(rc == VINF_SUCCESS || rc == VERR_INTERRUPTED);
-            }
-
-            /* Send last memory header to signal the end. */
-            FTMTCPHDRMEM Hdr;
-            Hdr.u32Magic    = FTMTCPHDR_MAGIC;
-            Hdr.GCPhys      = 0;
-            Hdr.cbPageRange = 0;
-            Hdr.cb          = 0;
-            rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
-            if (RT_FAILURE(rc))
-                LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
-
-            rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
-            AssertRC(rc);
-
-            PDMCritSectLeave(&pVM->ftm.s.CritSect);
-        }
-    }
-    return rc;
-}
-
-/**
- * Syncs memory from the master VM
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- */
-static int ftmR3SyncMem(PVM pVM)
-{
-    while (true)
-    {
-        FTMTCPHDRMEM Hdr;
-        RTGCPHYS GCPhys;
-
-        /* Read memory header. */
-        int rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
-        if (RT_FAILURE(rc))
-        {
-            Log(("RTTcpRead failed with %Rrc\n", rc));
-            break;
-        }
-        pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
-
-        if (Hdr.cb == 0)
-            break;  /* end of sync. */
-
-        Assert(Hdr.cb == Hdr.cbPageRange);  /** @todo uncompress */
-        GCPhys = Hdr.GCPhys;
-
-        /* Must be a multiple of PAGE_SIZE. */
-        Assert((Hdr.cbPageRange & 0xfff) == 0);
-
-        while (Hdr.cbPageRange)
-        {
-            PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)RTAvlGCPhysGet(&pVM->ftm.s.standby.pPhysPageTree, GCPhys);
-            if (!pNode)
-            {
-                /* Allocate memory for the node and page. */
-                pNode = (PFTMPHYSPAGETREENODE)RTMemAllocZ(sizeof(*pNode) + PAGE_SIZE);
-                AssertBreak(pNode);
-
-                /* Insert the node into the tree. */
-                pNode->Core.Key = GCPhys;
-                pNode->pPage = (void *)(pNode + 1);
-                bool fRet = RTAvlGCPhysInsert(&pVM->ftm.s.standby.pPhysPageTree, &pNode->Core);
-                Assert(fRet);
-            }
-
-            /* Fetch the page. */
-            rc = RTTcpRead(pVM->ftm.s.hSocket, pNode->pPage, PAGE_SIZE, NULL);
-            if (RT_FAILURE(rc))
-            {
-                Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
-                break;
-            }
-            pVM->ftm.s.StatReceivedMem.c += PAGE_SIZE;
-            Hdr.cbPageRange              -= PAGE_SIZE;
-            GCPhys                       += PAGE_SIZE;
-        }
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Callback handler for RTAvlGCPhysDestroy
- *
- * @returns 0 to continue, otherwise stop
- * @param   pBaseNode       Node to destroy
- * @param   pvUser          User parameter
- */
-static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser)
-{
-    PVM pVM = (PVM)pvUser;
-    PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)pBaseNode;
-
-    if (pVM)    /* NULL when the VM is destroyed. */
-    {
-        /* Update the guest memory of the standby VM. */
-        int rc = PGMR3PhysWriteExternal(pVM, pNode->Core.Key, pNode->pPage, PAGE_SIZE, "FTMemSync");
-        AssertRC(rc);
-    }
-    RTMemFree(pNode);
-    return 0;
-}
-
-/**
- * Thread function which monitors the health of the master VM
- *
- * @param   Thread      The thread id.
- * @param   pvUser      Not used
- * @return  VINF_SUCCESS (ignored).
- *
- */
-static DECLCALLBACK(int) ftmR3StandbyThread(RTTHREAD Thread, void *pvUser)
-{
-    PVM pVM = (PVM)pvUser;
-
-    for (;;)
-    {
-        uint64_t u64TimeNow;
-
-        int rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
-        if (rc != VERR_TIMEOUT)
-            break;    /* told to quit */
-
-        if (pVM->ftm.s.standby.u64LastHeartbeat)
-        {
-            u64TimeNow = RTTimeMilliTS();
-
-            if (u64TimeNow > pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 4)
-            {
-                /* Timeout; prepare to fallover. */
-                LogRel(("FTSync: TIMEOUT (%RX64 vs %RX64 ms): activate standby VM!\n", u64TimeNow, pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 2));
-
-                pVM->ftm.s.fActivateStandby = true;
-                /** todo: prevent split-brain. */
-                break;
-            }
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Listen for incoming traffic destined for the standby VM.
- *
- * @copydoc FNRTTCPSERVE
- *
- * @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
- */
-static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
-{
-    PVM pVM = (PVM)pvUser;
-
-    pVM->ftm.s.hSocket = Sock;
-
-    /*
-     * Disable Nagle.
-     */
-    int rc = RTTcpSetSendCoalescing(Sock, false /*fEnable*/);
-    AssertRC(rc);
-
-    /* Send the welcome message to the master node. */
-    rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Password.
-     */
-    const char *pszPassword = pVM->ftm.s.pszPassword;
-    if (pszPassword)
-    {
-        unsigned    off = 0;
-        while (pszPassword[off])
-        {
-            char ch;
-            rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
-            if (    RT_FAILURE(rc)
-                ||  pszPassword[off] != ch)
-            {
-                if (RT_FAILURE(rc))
-                    LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
-                else
-                    LogRel(("FTSync: Invalid password (off=%u)\n", off));
-                ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
-                return VINF_SUCCESS;
-            }
-            off++;
-        }
-    }
-    rc = ftmR3TcpWriteACK(pVM);
-    if (RT_FAILURE(rc))
-        return VINF_SUCCESS;
-
-    /** todo: verify VM config. */
-
-    /*
-     * Stop the server.
-     *
-     * Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
-     *       to it we must not return that value!
-     */
-    RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
-
-    /*
-     * Command processing loop.
-     */
-    bool fDone = false;
-    for (;;)
-    {
-        bool fFullSync = false;
-        char szCmd[128];
-
-        rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
-        if (RT_FAILURE(rc))
-            break;
-
-        pVM->ftm.s.standby.u64LastHeartbeat = RTTimeMilliTS();
-        if (!strcmp(szCmd, "mem-sync"))
-        {
-            rc = ftmR3TcpWriteACK(pVM);
-            AssertRC(rc);
-            if (RT_FAILURE(rc))
-                continue;
-
-            rc = ftmR3SyncMem(pVM);
-            AssertRC(rc);
-
-            rc = ftmR3TcpWriteACK(pVM);
-            AssertRC(rc);
-        }
-        else
-        if (    !strcmp(szCmd, "checkpoint")
-            ||  !strcmp(szCmd, "full-sync")
-            ||  (fFullSync = true))  /* intended assignment */
-        {
-            rc = ftmR3TcpWriteACK(pVM);
-            AssertRC(rc);
-            if (RT_FAILURE(rc))
-                continue;
-
-            /* Flush all pending memory updates. */
-            if (pVM->ftm.s.standby.pPhysPageTree)
-            {
-                RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, pVM);
-                pVM->ftm.s.standby.pPhysPageTree = NULL;
-            }
-
-            RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
-
-            /* Reset the sync state. */
-            pVM->ftm.s.syncstate.uOffStream   = 0;
-            pVM->ftm.s.syncstate.cbReadBlock  = 0;
-            pVM->ftm.s.syncstate.fStopReading = false;
-            pVM->ftm.s.syncstate.fIOError     = false;
-            pVM->ftm.s.syncstate.fEndOfStream = false;
-
-            pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
-            rc = VMR3LoadFromStreamFT(pVM, &g_ftmR3TcpOps, pVM);
-            pVM->ftm.s.fDeltaLoadSaveActive = false;
-            RTSocketRelease(pVM->ftm.s.hSocket);
-            AssertRC(rc);
-            if (RT_FAILURE(rc))
-            {
-                LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
-                ftmR3TcpWriteNACK(pVM, rc);
-                continue;
-            }
-
-            /* The EOS might not have been read, make sure it is. */
-            pVM->ftm.s.syncstate.fStopReading = false;
-            size_t cbRead;
-            rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
-            if (rc != VERR_EOF)
-            {
-                LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
-                ftmR3TcpWriteNACK(pVM, rc);
-                continue;
-            }
-
-            rc = ftmR3TcpWriteACK(pVM);
-            AssertRC(rc);
-        }
-    }
-    LogFlowFunc(("returns mRc=%Rrc\n", rc));
-    return VERR_TCP_SERVER_STOP;
-}
-
-/**
- * Powers on the fault tolerant virtual machine.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM to operate on.
- * @param   fMaster     FT master or standby
- * @param   uInterval   FT sync interval
- * @param   pszAddress  Standby VM address
- * @param   uPort       Standby VM port
- * @param   pszPassword FT password (NULL for none)
- *
- * @thread      Any thread.
- * @vmstate     Created
- * @vmstateto   PoweringOn+Running (master), PoweringOn+Running_FT (standby)
- */
-VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char *pszAddress, unsigned uPort, const char *pszPassword)
-{
-    int rc = VINF_SUCCESS;
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    AssertMsgReturn(enmVMState == VMSTATE_CREATED,
-                    ("%s\n", VMR3GetStateName(enmVMState)),
-                    VERR_INTERNAL_ERROR_4);
-    AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
-
-    if (pVM->ftm.s.uInterval)
-        pVM->ftm.s.uInterval    = uInterval;
-    else
-        pVM->ftm.s.uInterval    = 50;   /* standard sync interval of 50ms */
-
-    pVM->ftm.s.uPort            = uPort;
-    pVM->ftm.s.pszAddress       = RTStrDup(pszAddress);
-    if (pszPassword)
-        pVM->ftm.s.pszPassword  = RTStrDup(pszPassword);
-
-    rc = RTSemEventCreate(&pVM->ftm.s.hShutdownEvent);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    if (fMaster)
-    {
-        rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
-                            0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
-        if (RT_FAILURE(rc))
-            return rc;
-
-        pVM->fFaultTolerantMaster = true;
-        if (PGMIsUsingLargePages(pVM))
-        {
-            /* Must disable large page usage as 2 MB pages are too big to write monitor. */
-            LogRel(("FTSync: disabling large page usage.\n"));
-            PGMSetLargePageUsage(pVM, false);
-        }
-        /** @todo might need to disable page fusion as well */
-
-        return VMR3PowerOn(pVM);
-    }
-    else
-    {
-        /* standby */
-        rc = RTThreadCreate(NULL, ftmR3StandbyThread, pVM,
-                            0, RTTHREADTYPE_DEFAULT, 0, "ftmStandby");
-        if (RT_FAILURE(rc))
-            return rc;
-
-        rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
-        if (RT_FAILURE(rc))
-            return rc;
-        pVM->ftm.s.fIsStandbyNode = true;
-
-        rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
-        /** @todo deal with the exit code to check if we should activate this standby VM. */
-        if (pVM->ftm.s.fActivateStandby)
-        {
-            /** @todo fallover. */
-        }
-
-        if (pVM->ftm.s.standby.hServer)
-        {
-            RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
-            pVM->ftm.s.standby.hServer = NULL;
-        }
-        if (rc == VERR_TCP_SERVER_SHUTDOWN)
-            rc = VINF_SUCCESS;  /* ignore this error; the standby process was cancelled. */
-    }
-    return rc;
-}
-
-/**
- * Powers off the fault tolerant virtual machine (standby).
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
-{
-    AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
-    Assert(pVM->ftm.s.standby.hServer);
-
-    return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
-}
-
-/**
- * Rendezvous callback used by FTMR3SetCheckpoint
- * Sync state + changed memory with the standby node.
- *
- * This is only called on one of the EMTs while the other ones are waiting for
- * it to complete this function.
- *
- * @returns VINF_SUCCESS (VBox strict status code).
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
- * @param   pvUser      User parameter
- */
-static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-    bool fSuspended = false;
-
-    /** We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
-     *  is only a short suspend.
-     */
-    STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointPause, a);
-    PDMR3Suspend(pVM);
-
-    /** Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
-    EMR3NotifySuspend(pVM);
-    STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointPause, a);
-
-    STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
-
-    RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
-
-    /* Reset the sync state. */
-    pVM->ftm.s.syncstate.uOffStream   = 0;
-    pVM->ftm.s.syncstate.cbReadBlock  = 0;
-    pVM->ftm.s.syncstate.fStopReading = false;
-    pVM->ftm.s.syncstate.fIOError     = false;
-    pVM->ftm.s.syncstate.fEndOfStream = false;
-
-    rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
-    AssertRC(rc);
-
-    pVM->ftm.s.fDeltaLoadSaveActive = true;
-    rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
-    pVM->ftm.s.fDeltaLoadSaveActive = false;
-    AssertRC(rc);
-
-    rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
-    AssertRC(rc);
-
-    RTSocketRelease(pVM->ftm.s.hSocket);
-
-    /* Write protect all memory. */
-    rc = PGMR3PhysWriteProtectRAM(pVM);
-    AssertRC(rc);
-
-    /** We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
-     *  is only a short suspend.
-     */
-    STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointResume, b);
-    PGMR3ResetNoMorePhysWritesFlag(pVM);
-    PDMR3Resume(pVM);
-
-    /** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
-    EMR3NotifyResume(pVM);
-    STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointResume, b);
-
-    return rc;
-}
-
-/**
- * Performs a full sync to the standby node
- *
- * @returns VBox status code.
- *
- * @param   pVM             The VM to operate on.
- * @param   enmCheckpoint   Checkpoint type
- */
-VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
-{
-    int rc;
-
-    if (!pVM->fFaultTolerantMaster)
-        return VINF_SUCCESS;
-
-    switch (enmCheckpoint)
-    {
-    case FTMCHECKPOINTTYPE_NETWORK:
-        STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
-        break;
-
-    case FTMCHECKPOINTTYPE_STORAGE:
-        STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
-        break;
-
-    default:
-        break;
-    }
-    pVM->ftm.s.fCheckpointingActive = true;
-    if (VM_IS_EMT(pVM))
-    {
-        PVMCPU pVCpu = VMMGetCpu(pVM);
-
-        /* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
-        while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
-        {
-            if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
-            {
-                rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
-                AssertRC(rc);
-            }
-
-            if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
-            {
-                rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
-                AssertRC(rc);
-            }
-        }
-    }
-    else
-        rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
-
-    AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-
-    STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
-
-    rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
-
-    STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
-
-    PDMCritSectLeave(&pVM->ftm.s.CritSect);
-    pVM->ftm.s.fCheckpointingActive = false;
-
-    return rc;
-}
Index: unk/src/VBox/VMM/FTMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/FTMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,125 +1,0 @@
-/* $Id$ */
-/** @file
- * FTM - Internal header file.
- */
-
-/*
- * Copyright (C) 2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___FTMInternal_h
-#define ___FTMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/ftm.h>
-#include <VBox/stam.h>
-#include <VBox/pdmcritsect.h>
-#include <iprt/avl.h>
-
-/** @defgroup grp_ftm_int Internals.
- * @ingroup grp_ftm
- * @{
- */
-
-/** Physical page tree node. */
-typedef struct FTMPHYSPAGETREENODE
-{
-    AVLGCPHYSNODECORE   Core;
-    void               *pPage;
-} FTMPHYSPAGETREENODE;
-/** Pointer to FTMPHYSPAGETREENODE */
-typedef FTMPHYSPAGETREENODE *PFTMPHYSPAGETREENODE;
-
-/**
- * FTM VM Instance data.
- * Changes to this must checked against the padding of the ftm union in VM!
- */
-typedef struct FTM
-{
-    /** Address of the standby VM. */
-    R3PTRTYPE(char *)   pszAddress;
-    /** Password to access the syncing server of the standby VM. */
-    R3PTRTYPE(char *)   pszPassword;
-    /** Port of the standby VM. */
-    unsigned            uPort;
-    /** Syncing interval in ms. */
-    unsigned            uInterval;
-
-    /** Set when this VM is the standby FT node. */
-    bool                fIsStandbyNode;
-    /** Set when this master VM is busy with checkpointing. */
-    bool                fCheckpointingActive;
-    /** Set when VM save/restore should only include changed pages. */
-    bool                fDeltaLoadSaveActive;
-    /** Fallover to the standby VM. */
-    bool                fActivateStandby;
-    bool                fAlignment[4];
-
-    /** Current active socket. */
-    RTSOCKET            hSocket;
-
-    /* Shutdown event semaphore. */
-    RTSEMEVENT          hShutdownEvent;
-
-    /** State sync. */
-    struct
-    {
-        unsigned            uOffStream;
-        unsigned            cbReadBlock;
-        bool                fStopReading;
-        bool                fIOError;
-        bool                fEndOfStream;
-        bool                fAlignment[5];
-    } syncstate;
-
-    struct
-    {
-        R3PTRTYPE(PRTTCPSERVER)    hServer;
-        R3PTRTYPE(AVLGCPHYSTREE)   pPhysPageTree;
-        uint64_t                   u64LastHeartbeat;
-    } standby;
-
-    /*
-    struct
-    {
-    } master;
-    */
-
-    /** FTM critical section.
-     * This makes sure only the checkpoint or sync is active
-     */
-    PDMCRITSECT         CritSect;
-
-    STAMCOUNTER         StatReceivedMem;
-    STAMCOUNTER         StatReceivedState;
-    STAMCOUNTER         StatSentMem;
-    STAMCOUNTER         StatSentState;
-    STAMCOUNTER         StatDeltaMem;
-    STAMCOUNTER         StatDeltaVM;
-    STAMCOUNTER         StatFullSync;
-    STAMCOUNTER         StatCheckpointNetwork;
-    STAMCOUNTER         StatCheckpointStorage;
-#ifdef VBOX_WITH_STATISTICS
-    STAMPROFILE         StatCheckpoint;
-    STAMPROFILE         StatCheckpointResume;
-    STAMPROFILE         StatCheckpointPause;
-    STAMCOUNTER         StatSentMemRAM;
-    STAMCOUNTER         StatSentMemMMIO2;
-    STAMCOUNTER         StatSentMemShwROM;
-    STAMCOUNTER         StatSentStateWrite;
-#endif
-} FTM;
-AssertCompileMemberAlignment(FTM, CritSect, 8);
-
-/** @} */
-
-#endif
Index: unk/src/VBox/VMM/GMM.cpp
===================================================================
--- /trunk/src/VBox/VMM/GMM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,442 +1,0 @@
-/* $Id$ */
-/** @file
- * GMM - Global Memory Manager, ring-3 request wrappers.
- */
-
-/*
- * Copyright (C) 2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_GMM
-#include <VBox/gmm.h>
-#include <VBox/vmm.h>
-#include <VBox/vm.h>
-#include <VBox/sup.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-
-#include <iprt/assert.h>
-#include <VBox/log.h>
-#include <iprt/mem.h>
-#include <iprt/string.h>
-
-
-/**
- * @see GMMR0InitialReservation
- */
-GMMR3DECL(int)  GMMR3InitialReservation(PVM pVM, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages,
-                                        GMMOCPOLICY enmPolicy, GMMPRIORITY enmPriority)
-{
-    GMMINITIALRESERVATIONREQ Req;
-    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq = sizeof(Req);
-    Req.cBasePages = cBasePages;
-    Req.cShadowPages = cShadowPages;
-    Req.cFixedPages = cFixedPages;
-    Req.enmPolicy = enmPolicy;
-    Req.enmPriority = enmPriority;
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_INITIAL_RESERVATION, 0, &Req.Hdr);
-}
-
-
-/**
- * @see GMMR0UpdateReservation
- */
-GMMR3DECL(int)  GMMR3UpdateReservation(PVM pVM, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages)
-{
-    GMMUPDATERESERVATIONREQ Req;
-    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq = sizeof(Req);
-    Req.cBasePages = cBasePages;
-    Req.cShadowPages = cShadowPages;
-    Req.cFixedPages = cFixedPages;
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_UPDATE_RESERVATION, 0, &Req.Hdr);
-}
-
-
-/**
- * Prepares a GMMR0AllocatePages request.
- *
- * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
- * @param       pVM         Pointer to the shared VM structure.
- * @param[out]  ppReq       Where to store the pointer to the request packet.
- * @param       cPages      The number of pages that's to be allocated.
- * @param       enmAccount  The account to charge.
- */
-GMMR3DECL(int) GMMR3AllocatePagesPrepare(PVM pVM, PGMMALLOCATEPAGESREQ *ppReq, uint32_t cPages, GMMACCOUNT enmAccount)
-{
-    uint32_t cb = RT_OFFSETOF(GMMALLOCATEPAGESREQ, aPages[cPages]);
-    PGMMALLOCATEPAGESREQ pReq = (PGMMALLOCATEPAGESREQ)RTMemTmpAllocZ(cb);
-    if (!pReq)
-        return VERR_NO_TMP_MEMORY;
-
-    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    pReq->Hdr.cbReq = cb;
-    pReq->enmAccount = enmAccount;
-    pReq->cPages = cPages;
-    NOREF(pVM);
-    *ppReq = pReq;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Performs a GMMR0AllocatePages request.
- * This will call VMSetError on failure.
- *
- * @returns VBox status code.
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pReq        Pointer to the request (returned by GMMR3AllocatePagesPrepare).
- */
-GMMR3DECL(int) GMMR3AllocatePagesPerform(PVM pVM, PGMMALLOCATEPAGESREQ pReq)
-{
-    for (unsigned i = 0; ; i++)
-    {
-        int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_ALLOCATE_PAGES, 0, &pReq->Hdr);
-        if (RT_SUCCESS(rc))
-        {
-#ifdef LOG_ENABLED
-            for (uint32_t iPage = 0; iPage < pReq->cPages; iPage++)
-                Log3(("GMMR3AllocatePagesPerform: idPage=%#x HCPhys=%RHp\n",
-                      pReq->aPages[iPage].idPage, pReq->aPages[iPage].HCPhysGCPhys));
-#endif
-            return rc;
-        }
-        if (rc != VERR_GMM_SEED_ME)
-            return VMSetError(pVM, rc, RT_SRC_POS,
-                              N_("GMMR0AllocatePages failed to allocate %u pages"),
-                              pReq->cPages);
-        Assert(i < pReq->cPages);
-
-        /*
-         * Seed another chunk.
-         */
-        void *pvChunk;
-        rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
-        if (RT_FAILURE(rc))
-            return VMSetError(pVM, rc, RT_SRC_POS,
-                              N_("Out of memory (SUPR3PageAlloc) seeding a %u pages allocation request"),
-                              pReq->cPages);
-
-        rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
-        if (RT_FAILURE(rc))
-            return VMSetError(pVM, rc, RT_SRC_POS, N_("GMM seeding failed"));
-    }
-}
-
-
-/**
- * Cleans up a GMMR0AllocatePages request.
- * @param   pReq        Pointer to the request (returned by GMMR3AllocatePagesPrepare).
- */
-GMMR3DECL(void) GMMR3AllocatePagesCleanup(PGMMALLOCATEPAGESREQ pReq)
-{
-    RTMemTmpFree(pReq);
-}
-
-
-/**
- * Prepares a GMMR0FreePages request.
- *
- * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
- * @param       pVM         Pointer to the shared VM structure.
- * @param[out]  ppReq       Where to store the pointer to the request packet.
- * @param       cPages      The number of pages that's to be freed.
- * @param       enmAccount  The account to charge.
- */
-GMMR3DECL(int) GMMR3FreePagesPrepare(PVM pVM, PGMMFREEPAGESREQ *ppReq, uint32_t cPages, GMMACCOUNT enmAccount)
-{
-    uint32_t cb = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[cPages]);
-    PGMMFREEPAGESREQ pReq = (PGMMFREEPAGESREQ)RTMemTmpAllocZ(cb);
-    if (!pReq)
-        return VERR_NO_TMP_MEMORY;
-
-    pReq->Hdr.u32Magic  = SUPVMMR0REQHDR_MAGIC;
-    pReq->Hdr.cbReq     = cb;
-    pReq->enmAccount    = enmAccount;
-    pReq->cPages        = cPages;
-    NOREF(pVM);
-    *ppReq = pReq;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Re-prepares a GMMR0FreePages request.
- *
- * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
- * @param       pVM         Pointer to the shared VM structure.
- * @param       pReq        A request buffer previously returned by
- *                          GMMR3FreePagesPrepare().
- * @param       cPages      The number of pages originally passed to
- *                          GMMR3FreePagesPrepare().
- * @param       enmAccount  The account to charge.
- */
-GMMR3DECL(void) GMMR3FreePagesRePrep(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t cPages, GMMACCOUNT enmAccount)
-{
-    Assert(pReq->Hdr.u32Magic == SUPVMMR0REQHDR_MAGIC);
-    pReq->Hdr.cbReq     = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[cPages]);
-    pReq->enmAccount    = enmAccount;
-    pReq->cPages        = cPages;
-    NOREF(pVM);
-}
-
-
-/**
- * Performs a GMMR0FreePages request.
- * This will call VMSetError on failure.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pReq            Pointer to the request (returned by GMMR3FreePagesPrepare).
- * @param   cActualPages    The number of pages actually freed.
- */
-GMMR3DECL(int) GMMR3FreePagesPerform(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t cActualPages)
-{
-    /*
-     * Adjust the request if we ended up with fewer pages than anticipated.
-     */
-    if (cActualPages != pReq->cPages)
-    {
-        AssertReturn(cActualPages < pReq->cPages, VERR_INTERNAL_ERROR);
-        if (!cActualPages)
-            return VINF_SUCCESS;
-        pReq->cPages = cActualPages;
-        pReq->Hdr.cbReq = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[cActualPages]);
-    }
-
-    /*
-     * Do the job.
-     */
-    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_PAGES, 0, &pReq->Hdr);
-    if (RT_SUCCESS(rc))
-        return rc;
-    AssertRC(rc);
-    return VMSetError(pVM, rc, RT_SRC_POS,
-                      N_("GMMR0FreePages failed to free %u pages"),
-                      pReq->cPages);
-}
-
-
-/**
- * Cleans up a GMMR0FreePages request.
- * @param   pReq        Pointer to the request (returned by GMMR3FreePagesPrepare).
- */
-GMMR3DECL(void) GMMR3FreePagesCleanup(PGMMFREEPAGESREQ pReq)
-{
-    RTMemTmpFree(pReq);
-}
-
-
-/**
- * Frees allocated pages, for bailing out on failure.
- *
- * This will not call VMSetError on failure but will use AssertLogRel instead.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pAllocReq   The allocation request to undo.
- */
-GMMR3DECL(void) GMMR3FreeAllocatedPages(PVM pVM, GMMALLOCATEPAGESREQ const *pAllocReq)
-{
-    uint32_t cb = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[pAllocReq->cPages]);
-    PGMMFREEPAGESREQ pReq = (PGMMFREEPAGESREQ)RTMemTmpAllocZ(cb);
-    AssertLogRelReturnVoid(pReq);
-
-    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    pReq->Hdr.cbReq = cb;
-    pReq->enmAccount = pAllocReq->enmAccount;
-    pReq->cPages = pAllocReq->cPages;
-    uint32_t iPage = pAllocReq->cPages;
-    while (iPage-- > 0)
-    {
-        Assert(pAllocReq->aPages[iPage].idPage != NIL_GMM_PAGEID);
-        pReq->aPages[iPage].idPage = pAllocReq->aPages[iPage].idPage;
-    }
-
-    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_PAGES, 0, &pReq->Hdr);
-    AssertLogRelRC(rc);
-
-    RTMemTmpFree(pReq);
-}
-
-
-/**
- * @see GMMR0BalloonedPages
- */
-GMMR3DECL(int)  GMMR3BalloonedPages(PVM pVM, GMMBALLOONACTION enmAction, uint32_t cBalloonedPages)
-{
-    GMMBALLOONEDPAGESREQ Req;
-    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq = sizeof(Req);
-    Req.enmAction = enmAction;
-    Req.cBalloonedPages = cBalloonedPages;
-
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_BALLOONED_PAGES, 0, &Req.Hdr);
-}
-
-/**
- * @see GMMR0QueryVMMMemoryStatsReq
- */
-GMMR3DECL(int)  GMMR3QueryHypervisorMemoryStats(PVM pVM, uint64_t *pcTotalAllocPages, uint64_t *pcTotalFreePages, uint64_t *pcTotalBalloonPages, uint64_t *puTotalBalloonSize)
-{
-    GMMMEMSTATSREQ Req;
-    Req.Hdr.u32Magic     = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq        = sizeof(Req);
-    Req.cAllocPages      = 0;
-    Req.cFreePages       = 0;
-    Req.cBalloonedPages  = 0;
-    Req.cSharedPages     = 0;
-
-    *pcTotalAllocPages   = 0;
-    *pcTotalFreePages    = 0;
-    *pcTotalBalloonPages = 0;
-    *puTotalBalloonSize  = 0;
-
-    /* Must be callable from any thread, so can't use VMMR3CallR0. */
-    int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS, 0, &Req.Hdr);
-    if (rc == VINF_SUCCESS)
-    {
-        *pcTotalAllocPages   = Req.cAllocPages;
-        *pcTotalFreePages    = Req.cFreePages;
-        *pcTotalBalloonPages = Req.cBalloonedPages;
-        *puTotalBalloonSize  = Req.cSharedPages;
-    }
-    return rc;
-}
-
-/**
- * @see GMMR0QueryMemoryStatsReq
- */
-GMMR3DECL(int)  GMMR3QueryMemoryStats(PVM pVM, uint64_t *pcAllocPages, uint64_t *pcMaxPages, uint64_t *pcBalloonPages)
-{
-    GMMMEMSTATSREQ Req;
-    Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq       = sizeof(Req);
-    Req.cAllocPages     = 0;
-    Req.cFreePages      = 0;
-    Req.cBalloonedPages = 0;
-
-    *pcAllocPages      = 0;
-    *pcMaxPages         = 0;
-    *pcBalloonPages     = 0;
-
-    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_QUERY_MEM_STATS, 0, &Req.Hdr);
-    if (rc == VINF_SUCCESS)
-    {
-        *pcAllocPages   = Req.cAllocPages;
-        *pcMaxPages     = Req.cMaxPages;
-        *pcBalloonPages = Req.cBalloonedPages;
-    }
-    return rc;
-}
-
-/**
- * @see GMMR0MapUnmapChunk
- */
-GMMR3DECL(int)  GMMR3MapUnmapChunk(PVM pVM, uint32_t idChunkMap, uint32_t idChunkUnmap, PRTR3PTR ppvR3)
-{
-    GMMMAPUNMAPCHUNKREQ Req;
-    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq = sizeof(Req);
-    Req.idChunkMap = idChunkMap;
-    Req.idChunkUnmap = idChunkUnmap;
-    Req.pvR3 = NULL;
-    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
-    if (RT_SUCCESS(rc) && ppvR3)
-        *ppvR3 = Req.pvR3;
-    return rc;
-}
-
-/**
- * @see GMMR0FreeLargePage
- */
-GMMR3DECL(int)  GMMR3FreeLargePage(PVM pVM,  uint32_t idPage)
-{
-    GMMFREELARGEPAGEREQ Req;
-    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq = sizeof(Req);
-    Req.idPage = idPage;
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_LARGE_PAGE, 0, &Req.Hdr);
-}
-
-/**
- * @see GMMR0SeedChunk
- */
-GMMR3DECL(int)  GMMR3SeedChunk(PVM pVM, RTR3PTR pvR3)
-{
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvR3, NULL);
-}
-
-
-/**
- * @see GMMR0RegisterSharedModule
- */
-GMMR3DECL(int) GMMR3RegisterSharedModule(PVM pVM, PGMMREGISTERSHAREDMODULEREQ pReq)
-{
-    pReq->Hdr.u32Magic  = SUPVMMR0REQHDR_MAGIC;
-    pReq->Hdr.cbReq     = RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[pReq->cRegions]);
-    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_REGISTER_SHARED_MODULE, 0, &pReq->Hdr);
-    if (rc == VINF_SUCCESS)
-        rc = pReq->rc;
-    return rc;
-}
-
-/**
- * @see GMMR0RegisterSharedModule
- */
-GMMR3DECL(int) GMMR3UnregisterSharedModule(PVM pVM, PGMMUNREGISTERSHAREDMODULEREQ pReq)
-{
-    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    pReq->Hdr.cbReq = sizeof(*pReq);
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE, 0, &pReq->Hdr);
-}
-
-/**
- * @see GMMR0ResetSharedModules
- */
-GMMR3DECL(int) GMMR3ResetSharedModules(PVM pVM)
-{
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_RESET_SHARED_MODULES, 0, NULL);
-}
-
-/**
- * @see GMMR0CheckSharedModules
- */
-GMMR3DECL(int)  GMMR3CheckSharedModules(PVM pVM)
-{
-    return VMMR3CallR0(pVM, VMMR0_DO_GMM_CHECK_SHARED_MODULES, 0, NULL);
-}
-
-#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
-/**
- * @see GMMR0FindDuplicatePage
- */
-GMMR3DECL(bool) GMMR3IsDuplicatePage(PVM pVM, uint32_t idPage)
-{
-    GMMFINDDUPLICATEPAGEREQ Req;
-    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq    = sizeof(Req);
-    Req.idPage       = idPage;
-    Req.fDuplicate   = false;
-
-    /* Must be callable from any thread, so can't use VMMR3CallR0. */
-    int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_FIND_DUPLICATE_PAGE, 0, &Req.Hdr);
-    if (rc == VINF_SUCCESS)
-        return Req.fDuplicate;
-    else
-        return false;
-}
-#endif /* VBOX_STRICT && HC_ARCH_BITS == 64 */
-
Index: unk/src/VBox/VMM/HWACCM.cpp
===================================================================
--- /trunk/src/VBox/VMM/HWACCM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2821 +1,0 @@
-/* $Id$ */
-/** @file
- * HWACCM - Intel/AMD VM Hardware Support Manager
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_HWACCM
-#include <VBox/cpum.h>
-#include <VBox/stam.h>
-#include <VBox/mm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pgm.h>
-#include <VBox/ssm.h>
-#include <VBox/trpm.h>
-#include <VBox/dbgf.h>
-#include <VBox/iom.h>
-#include <VBox/patm.h>
-#include <VBox/csam.h>
-#include <VBox/selm.h>
-#include <VBox/rem.h>
-#include <VBox/hwacc_vmx.h>
-#include <VBox/hwacc_svm.h>
-#include "HWACCMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-
-#include <iprt/assert.h>
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/asm-amd64-x86.h>
-#include <iprt/string.h>
-#include <iprt/env.h>
-#include <iprt/thread.h>
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-#ifdef VBOX_WITH_STATISTICS
-# define EXIT_REASON(def, val, str) #def " - " #val " - " str
-# define EXIT_REASON_NIL() NULL
-/** Exit reason descriptions for VT-x, used to describe statistics. */
-static const char * const g_apszVTxExitReasons[MAX_EXITREASON_STAT] =
-{
-    EXIT_REASON(VMX_EXIT_EXCEPTION          ,  0, "Exception or non-maskable interrupt (NMI)."),
-    EXIT_REASON(VMX_EXIT_EXTERNAL_IRQ       ,  1, "External interrupt."),
-    EXIT_REASON(VMX_EXIT_TRIPLE_FAULT       ,  2, "Triple fault."),
-    EXIT_REASON(VMX_EXIT_INIT_SIGNAL        ,  3, "INIT signal."),
-    EXIT_REASON(VMX_EXIT_SIPI               ,  4, "Start-up IPI (SIPI)."),
-    EXIT_REASON(VMX_EXIT_IO_SMI_IRQ         ,  5, "I/O system-management interrupt (SMI)."),
-    EXIT_REASON(VMX_EXIT_SMI_IRQ            ,  6, "Other SMI."),
-    EXIT_REASON(VMX_EXIT_IRQ_WINDOW         ,  7, "Interrupt window."),
-    EXIT_REASON_NIL(),
-    EXIT_REASON(VMX_EXIT_TASK_SWITCH        ,  9, "Task switch."),
-    EXIT_REASON(VMX_EXIT_CPUID              , 10, "Guest software attempted to execute CPUID."),
-    EXIT_REASON_NIL(),
-    EXIT_REASON(VMX_EXIT_HLT                , 12, "Guest software attempted to execute HLT."),
-    EXIT_REASON(VMX_EXIT_INVD               , 13, "Guest software attempted to execute INVD."),
-    EXIT_REASON(VMX_EXIT_INVPG              , 14, "Guest software attempted to execute INVPG."),
-    EXIT_REASON(VMX_EXIT_RDPMC              , 15, "Guest software attempted to execute RDPMC."),
-    EXIT_REASON(VMX_EXIT_RDTSC              , 16, "Guest software attempted to execute RDTSC."),
-    EXIT_REASON(VMX_EXIT_RSM                , 17, "Guest software attempted to execute RSM in SMM."),
-    EXIT_REASON(VMX_EXIT_VMCALL             , 18, "Guest software executed VMCALL."),
-    EXIT_REASON(VMX_EXIT_VMCLEAR            , 19, "Guest software executed VMCLEAR."),
-    EXIT_REASON(VMX_EXIT_VMLAUNCH           , 20, "Guest software executed VMLAUNCH."),
-    EXIT_REASON(VMX_EXIT_VMPTRLD            , 21, "Guest software executed VMPTRLD."),
-    EXIT_REASON(VMX_EXIT_VMPTRST            , 22, "Guest software executed VMPTRST."),
-    EXIT_REASON(VMX_EXIT_VMREAD             , 23, "Guest software executed VMREAD."),
-    EXIT_REASON(VMX_EXIT_VMRESUME           , 24, "Guest software executed VMRESUME."),
-    EXIT_REASON(VMX_EXIT_VMWRITE            , 25, "Guest software executed VMWRITE."),
-    EXIT_REASON(VMX_EXIT_VMXOFF             , 26, "Guest software executed VMXOFF."),
-    EXIT_REASON(VMX_EXIT_VMXON              , 27, "Guest software executed VMXON."),
-    EXIT_REASON(VMX_EXIT_CRX_MOVE           , 28, "Control-register accesses."),
-    EXIT_REASON(VMX_EXIT_DRX_MOVE           , 29, "Debug-register accesses."),
-    EXIT_REASON(VMX_EXIT_PORT_IO            , 30, "I/O instruction."),
-    EXIT_REASON(VMX_EXIT_RDMSR              , 31, "RDMSR. Guest software attempted to execute RDMSR."),
-    EXIT_REASON(VMX_EXIT_WRMSR              , 32, "WRMSR. Guest software attempted to execute WRMSR."),
-    EXIT_REASON(VMX_EXIT_ERR_INVALID_GUEST_STATE,  33, "VM-entry failure due to invalid guest state."),
-    EXIT_REASON(VMX_EXIT_ERR_MSR_LOAD       , 34, "VM-entry failure due to MSR loading."),
-    EXIT_REASON_NIL(),
-    EXIT_REASON(VMX_EXIT_MWAIT              , 36, "Guest software executed MWAIT."),
-    EXIT_REASON_NIL(),
-    EXIT_REASON_NIL(),
-    EXIT_REASON(VMX_EXIT_MONITOR            , 39, "Guest software attempted to execute MONITOR."),
-    EXIT_REASON(VMX_EXIT_PAUSE              , 40, "Guest software attempted to execute PAUSE."),
-    EXIT_REASON(VMX_EXIT_ERR_MACHINE_CHECK  , 41, "VM-entry failure due to machine-check."),
-    EXIT_REASON_NIL(),
-    EXIT_REASON(VMX_EXIT_TPR                , 43, "TPR below threshold. Guest software executed MOV to CR8."),
-    EXIT_REASON(VMX_EXIT_APIC_ACCESS        , 44, "APIC access. Guest software attempted to access memory at a physical address on the APIC-access page."),
-    EXIT_REASON_NIL(),
-    EXIT_REASON(VMX_EXIT_XDTR_ACCESS        , 46, "Access to GDTR or IDTR. Guest software attempted to execute LGDT, LIDT, SGDT, or SIDT."),
-    EXIT_REASON(VMX_EXIT_TR_ACCESS          , 47, "Access to LDTR or TR. Guest software attempted to execute LLDT, LTR, SLDT, or STR."),
-    EXIT_REASON(VMX_EXIT_EPT_VIOLATION      , 48, "EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures."),
-    EXIT_REASON(VMX_EXIT_EPT_MISCONFIG      , 49, "EPT misconfiguration. An attempt to access memory with a guest-physical address encountered a misconfigured EPT paging-structure entry."),
-    EXIT_REASON(VMX_EXIT_INVEPT             , 50, "INVEPT. Guest software attempted to execute INVEPT."),
-    EXIT_REASON_NIL(),
-    EXIT_REASON(VMX_EXIT_PREEMPTION_TIMER   , 52, "VMX-preemption timer expired. The preemption timer counted down to zero."),
-    EXIT_REASON(VMX_EXIT_INVVPID            , 53, "INVVPID. Guest software attempted to execute INVVPID."),
-    EXIT_REASON(VMX_EXIT_WBINVD             , 54, "WBINVD. Guest software attempted to execute WBINVD."),
-    EXIT_REASON(VMX_EXIT_XSETBV             , 55, "XSETBV. Guest software attempted to execute XSETBV."),
-    EXIT_REASON_NIL()
-};
-/** Exit reason descriptions for AMD-V, used to describe statistics. */
-static const char * const g_apszAmdVExitReasons[MAX_EXITREASON_STAT] =
-{
-    EXIT_REASON(SVM_EXIT_READ_CR0                   ,  0, "Read CR0."),
-    EXIT_REASON(SVM_EXIT_READ_CR1                   ,  1, "Read CR1."),
-    EXIT_REASON(SVM_EXIT_READ_CR2                   ,  2, "Read CR2."),
-    EXIT_REASON(SVM_EXIT_READ_CR3                   ,  3, "Read CR3."),
-    EXIT_REASON(SVM_EXIT_READ_CR4                   ,  4, "Read CR4."),
-    EXIT_REASON(SVM_EXIT_READ_CR5                   ,  5, "Read CR5."),
-    EXIT_REASON(SVM_EXIT_READ_CR6                   ,  6, "Read CR6."),
-    EXIT_REASON(SVM_EXIT_READ_CR7                   ,  7, "Read CR7."),
-    EXIT_REASON(SVM_EXIT_READ_CR8                   ,  8, "Read CR8."),
-    EXIT_REASON(SVM_EXIT_READ_CR9                   ,  9, "Read CR9."),
-    EXIT_REASON(SVM_EXIT_READ_CR10                  , 10, "Read CR10."),
-    EXIT_REASON(SVM_EXIT_READ_CR11                  , 11, "Read CR11."),
-    EXIT_REASON(SVM_EXIT_READ_CR12                  , 12, "Read CR12."),
-    EXIT_REASON(SVM_EXIT_READ_CR13                  , 13, "Read CR13."),
-    EXIT_REASON(SVM_EXIT_READ_CR14                  , 14, "Read CR14."),
-    EXIT_REASON(SVM_EXIT_READ_CR15                  , 15, "Read CR15."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR0                  , 16, "Write CR0."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR1                  , 17, "Write CR1."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR2                  , 18, "Write CR2."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR3                  , 19, "Write CR3."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR4                  , 20, "Write CR4."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR5                  , 21, "Write CR5."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR6                  , 22, "Write CR6."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR7                  , 23, "Write CR7."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR8                  , 24, "Write CR8."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR9                  , 25, "Write CR9."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR10                 , 26, "Write CR10."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR11                 , 27, "Write CR11."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR12                 , 28, "Write CR12."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR13                 , 29, "Write CR13."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR14                 , 30, "Write CR14."),
-    EXIT_REASON(SVM_EXIT_WRITE_CR15                 , 31, "Write CR15."),
-    EXIT_REASON(SVM_EXIT_READ_DR0                   , 32, "Read DR0."),
-    EXIT_REASON(SVM_EXIT_READ_DR1                   , 33, "Read DR1."),
-    EXIT_REASON(SVM_EXIT_READ_DR2                   , 34, "Read DR2."),
-    EXIT_REASON(SVM_EXIT_READ_DR3                   , 35, "Read DR3."),
-    EXIT_REASON(SVM_EXIT_READ_DR4                   , 36, "Read DR4."),
-    EXIT_REASON(SVM_EXIT_READ_DR5                   , 37, "Read DR5."),
-    EXIT_REASON(SVM_EXIT_READ_DR6                   , 38, "Read DR6."),
-    EXIT_REASON(SVM_EXIT_READ_DR7                   , 39, "Read DR7."),
-    EXIT_REASON(SVM_EXIT_READ_DR8                   , 40, "Read DR8."),
-    EXIT_REASON(SVM_EXIT_READ_DR9                   , 41, "Read DR9."),
-    EXIT_REASON(SVM_EXIT_READ_DR10                  , 42, "Read DR10."),
-    EXIT_REASON(SVM_EXIT_READ_DR11                  , 43, "Read DR11"),
-    EXIT_REASON(SVM_EXIT_READ_DR12                  , 44, "Read DR12."),
-    EXIT_REASON(SVM_EXIT_READ_DR13                  , 45, "Read DR13."),
-    EXIT_REASON(SVM_EXIT_READ_DR14                  , 46, "Read DR14."),
-    EXIT_REASON(SVM_EXIT_READ_DR15                  , 47, "Read DR15."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR0                  , 48, "Write DR0."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR1                  , 49, "Write DR1."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR2                  , 50, "Write DR2."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR3                  , 51, "Write DR3."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR4                  , 52, "Write DR4."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR5                  , 53, "Write DR5."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR6                  , 54, "Write DR6."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR7                  , 55, "Write DR7."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR8                  , 56, "Write DR8."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR9                  , 57, "Write DR9."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR10                 , 58, "Write DR10."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR11                 , 59, "Write DR11."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR12                 , 60, "Write DR12."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR13                 , 61, "Write DR13."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR14                 , 62, "Write DR14."),
-    EXIT_REASON(SVM_EXIT_WRITE_DR15                 , 63, "Write DR15."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_0                , 64, "Exception Vector 0  (0x0)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_1                , 65, "Exception Vector 1  (0x1)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_2                , 66, "Exception Vector 2  (0x2)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_3                , 67, "Exception Vector 3  (0x3)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_4                , 68, "Exception Vector 4  (0x4)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_5                , 69, "Exception Vector 5  (0x5)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_6                , 70, "Exception Vector 6  (0x6)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_7                , 71, "Exception Vector 7  (0x7)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_8                , 72, "Exception Vector 8  (0x8)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_9                , 73, "Exception Vector 9  (0x9)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_A                , 74, "Exception Vector 10 (0xA)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_B                , 75, "Exception Vector 11 (0xB)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_C                , 76, "Exception Vector 12 (0xC)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_D                , 77, "Exception Vector 13 (0xD)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_E                , 78, "Exception Vector 14 (0xE)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_F                , 79, "Exception Vector 15 (0xF)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_10               , 80, "Exception Vector 16 (0x10)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_11               , 81, "Exception Vector 17 (0x11)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_12               , 82, "Exception Vector 18 (0x12)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_13               , 83, "Exception Vector 19 (0x13)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_14               , 84, "Exception Vector 20 (0x14)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_15               , 85, "Exception Vector 22 (0x15)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_16               , 86, "Exception Vector 22 (0x16)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_17               , 87, "Exception Vector 23 (0x17)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_18               , 88, "Exception Vector 24 (0x18)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_19               , 89, "Exception Vector 25 (0x19)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_1A               , 90, "Exception Vector 26 (0x1A)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_1B               , 91, "Exception Vector 27 (0x1B)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_1C               , 92, "Exception Vector 28 (0x1C)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_1D               , 93, "Exception Vector 29 (0x1D)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_1E               , 94, "Exception Vector 30 (0x1E)."),
-    EXIT_REASON(SVM_EXIT_EXCEPTION_1F               , 95, "Exception Vector 31 (0x1F)."),
-    EXIT_REASON(SVM_EXIT_INTR                       , 96, "Physical maskable interrupt."),
-    EXIT_REASON(SVM_EXIT_NMI                        , 97, "Physical non-maskable interrupt."),
-    EXIT_REASON(SVM_EXIT_SMI                        , 98, "System management interrupt."),
-    EXIT_REASON(SVM_EXIT_INIT                       , 99, "Physical INIT signal."),
-    EXIT_REASON(SVM_EXIT_VINTR                      ,100, "Virtual interrupt."),
-    EXIT_REASON(SVM_EXIT_CR0_SEL_WRITE              ,101, "Write to CR0 that changed any bits other than CR0.TS or CR0.MP."),
-    EXIT_REASON(SVM_EXIT_IDTR_READ                  ,102, "Read IDTR"),
-    EXIT_REASON(SVM_EXIT_GDTR_READ                  ,103, "Read GDTR"),
-    EXIT_REASON(SVM_EXIT_LDTR_READ                  ,104, "Read LDTR."),
-    EXIT_REASON(SVM_EXIT_TR_READ                    ,105, "Read TR."),
-    EXIT_REASON(SVM_EXIT_TR_READ                    ,106, "Write IDTR."),
-    EXIT_REASON(SVM_EXIT_TR_READ                    ,107, "Write GDTR."),
-    EXIT_REASON(SVM_EXIT_TR_READ                    ,108, "Write LDTR."),
-    EXIT_REASON(SVM_EXIT_TR_READ                    ,109, "Write TR."),
-    EXIT_REASON(SVM_EXIT_RDTSC                      ,110, "RDTSC instruction."),
-    EXIT_REASON(SVM_EXIT_RDPMC                      ,111, "RDPMC instruction."),
-    EXIT_REASON(SVM_EXIT_PUSHF                      ,112, "PUSHF instruction."),
-    EXIT_REASON(SVM_EXIT_POPF                       ,113, "POPF instruction."),
-    EXIT_REASON(SVM_EXIT_CPUID                      ,114, "CPUID instruction."),
-    EXIT_REASON(SVM_EXIT_RSM                        ,115, "RSM instruction."),
-    EXIT_REASON(SVM_EXIT_IRET                       ,116, "IRET instruction."),
-    EXIT_REASON(SVM_EXIT_SWINT                      ,117, "Software interrupt (INTn instructions)."),
-    EXIT_REASON(SVM_EXIT_INVD                       ,118, "INVD instruction."),
-    EXIT_REASON(SVM_EXIT_PAUSE                      ,119, "PAUSE instruction."),
-    EXIT_REASON(SVM_EXIT_HLT                        ,120, "HLT instruction."),
-    EXIT_REASON(SVM_EXIT_INVLPG                     ,121, "INVLPG instruction."),
-    EXIT_REASON(SVM_EXIT_INVLPGA                    ,122, "INVLPGA instruction."),
-    EXIT_REASON(SVM_EXIT_IOIO                       ,123, "IN/OUT accessing protected port (EXITINFO1 field provides more information)."),
-    EXIT_REASON(SVM_EXIT_MSR                        ,124, "RDMSR or WRMSR access to protected MSR."),
-    EXIT_REASON(SVM_EXIT_TASK_SWITCH                ,125, "Task switch."),
-    EXIT_REASON(SVM_EXIT_FERR_FREEZE                ,126, "FP legacy handling enabled, and processor is frozen in an x87/mmx instruction waiting for an interrupt"),
-    EXIT_REASON(SVM_EXIT_SHUTDOWN                   ,127, "Shutdown."),
-    EXIT_REASON(SVM_EXIT_VMRUN                      ,128, "VMRUN instruction."),
-    EXIT_REASON(SVM_EXIT_VMMCALL                    ,129, "VMCALL instruction."),
-    EXIT_REASON(SVM_EXIT_VMLOAD                     ,130, "VMLOAD instruction."),
-    EXIT_REASON(SVM_EXIT_VMSAVE                     ,131, "VMSAVE instruction."),
-    EXIT_REASON(SVM_EXIT_STGI                       ,132, "STGI instruction."),
-    EXIT_REASON(SVM_EXIT_CLGI                       ,133, "CLGI instruction."),
-    EXIT_REASON(SVM_EXIT_SKINIT                     ,134, "SKINIT instruction."),
-    EXIT_REASON(SVM_EXIT_RDTSCP                     ,135, "RDTSCP instruction."),
-    EXIT_REASON(SVM_EXIT_ICEBP                      ,136, "ICEBP instruction."),
-    EXIT_REASON(SVM_EXIT_WBINVD                     ,137, "WBINVD instruction."),
-    EXIT_REASON(SVM_EXIT_MONITOR                    ,138, "MONITOR instruction."),
-    EXIT_REASON(SVM_EXIT_MWAIT_UNCOND               ,139, "MWAIT instruction unconditional."),
-    EXIT_REASON(SVM_EXIT_MWAIT_ARMED                ,140, "MWAIT instruction when armed."),
-    EXIT_REASON(SVM_EXIT_NPF                        ,1024, "Nested paging: host-level page fault occurred (EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault)."),
-    EXIT_REASON_NIL()
-};
-# undef EXIT_REASON
-# undef EXIT_REASON_NIL
-#endif /* VBOX_WITH_STATISTICS */
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) hwaccmR3Save(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int) hwaccmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static int hwaccmR3InitCPU(PVM pVM);
-static int hwaccmR3TermCPU(PVM pVM);
-
-
-/**
- * Initializes the HWACCM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) HWACCMR3Init(PVM pVM)
-{
-    LogFlow(("HWACCMR3Init\n"));
-
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompileMemberAlignment(VM, hwaccm.s, 32);
-    AssertCompile(sizeof(pVM->hwaccm.s) <= sizeof(pVM->hwaccm.padding));
-
-    /* Some structure checks. */
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, ctrl.EventInject) == 0xA8, ("ctrl.EventInject offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.EventInject)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, ctrl.ExitIntInfo) == 0x88, ("ctrl.ExitIntInfo offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.ExitIntInfo)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, ctrl.TLBCtrl) == 0x58, ("ctrl.TLBCtrl offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.TLBCtrl)));
-
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest) == 0x400, ("guest offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.TR) == 0x490, ("guest.TR offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.TR)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u8CPL) == 0x4CB, ("guest.u8CPL offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u8CPL)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64EFER) == 0x4D0, ("guest.u64EFER offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64EFER)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64CR4) == 0x548, ("guest.u64CR4 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64CR4)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64RIP) == 0x578, ("guest.u64RIP offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64RIP)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64RSP) == 0x5D8, ("guest.u64RSP offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64RSP)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64CR2) == 0x640, ("guest.u64CR2 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64CR2)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64GPAT) == 0x668, ("guest.u64GPAT offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64GPAT)));
-    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64LASTEXCPTO) == 0x690, ("guest.u64LASTEXCPTO offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64LASTEXCPTO)));
-    AssertReleaseMsg(sizeof(SVM_VMCB) == 0x1000, ("SVM_VMCB size = %x\n", sizeof(SVM_VMCB)));
-
-
-    /*
-     * Register the saved state data unit.
-     */
-    int rc = SSMR3RegisterInternal(pVM, "HWACCM", 0, HWACCM_SSM_VERSION, sizeof(HWACCM),
-                                   NULL, NULL, NULL,
-                                   NULL, hwaccmR3Save, NULL,
-                                   NULL, hwaccmR3Load, NULL);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /* Misc initialisation. */
-    pVM->hwaccm.s.vmx.fSupported = false;
-    pVM->hwaccm.s.svm.fSupported = false;
-    pVM->hwaccm.s.vmx.fEnabled   = false;
-    pVM->hwaccm.s.svm.fEnabled   = false;
-
-    pVM->hwaccm.s.fNestedPaging  = false;
-    pVM->hwaccm.s.fLargePages    = false;
-
-    /* Disabled by default. */
-    pVM->fHWACCMEnabled = false;
-
-    /*
-     * Check CFGM options.
-     */
-    PCFGMNODE pRoot      = CFGMR3GetRoot(pVM);
-    PCFGMNODE pHWVirtExt = CFGMR3GetChild(pRoot, "HWVirtExt/");
-    /* Nested paging: disabled by default. */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "EnableNestedPaging", &pVM->hwaccm.s.fAllowNestedPaging, false);
-    AssertRC(rc);
-
-    /* Large pages: disabled by default. */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "EnableLargePages", &pVM->hwaccm.s.fLargePages, false);
-    AssertRC(rc);
-
-    /* VT-x VPID: disabled by default. */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "EnableVPID", &pVM->hwaccm.s.vmx.fAllowVPID, false);
-    AssertRC(rc);
-
-    /* HWACCM support must be explicitely enabled in the configuration file. */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "Enabled", &pVM->hwaccm.s.fAllowed, false);
-    AssertRC(rc);
-
-    /* TPR patching for 32 bits (Windows) guests with IO-APIC: disabled by default. */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "TPRPatchingEnabled", &pVM->hwaccm.s.fTRPPatchingAllowed, false);
-    AssertRC(rc);
-
-#ifdef RT_OS_DARWIN
-    if (VMMIsHwVirtExtForced(pVM) != pVM->hwaccm.s.fAllowed)
-#else
-    if (VMMIsHwVirtExtForced(pVM) && !pVM->hwaccm.s.fAllowed)
-#endif
-    {
-        AssertLogRelMsgFailed(("VMMIsHwVirtExtForced=%RTbool fAllowed=%RTbool\n",
-                               VMMIsHwVirtExtForced(pVM), pVM->hwaccm.s.fAllowed));
-        return VERR_HWACCM_CONFIG_MISMATCH;
-    }
-
-    if (VMMIsHwVirtExtForced(pVM))
-        pVM->fHWACCMEnabled = true;
-
-#if HC_ARCH_BITS == 32
-    /* 64-bit mode is configurable and it depends on both the kernel mode and VT-x.
-     * (To use the default, don't set 64bitEnabled in CFGM.) */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "64bitEnabled", &pVM->hwaccm.s.fAllow64BitGuests, false);
-    AssertLogRelRCReturn(rc, rc);
-    if (pVM->hwaccm.s.fAllow64BitGuests)
-    {
-# ifdef RT_OS_DARWIN
-        if (!VMMIsHwVirtExtForced(pVM))
-# else
-        if (!pVM->hwaccm.s.fAllowed)
-# endif
-            return VM_SET_ERROR(pVM, VERR_INVALID_PARAMETER, "64-bit guest support was requested without also enabling HWVirtEx (VT-x/AMD-V).");
-    }
-#else
-    /* On 64-bit hosts 64-bit guest support is enabled by default, but allow this to be overridden
-     * via VBoxInternal/HWVirtExt/64bitEnabled=0. (ConsoleImpl2.cpp doesn't set this to false for 64-bit.) */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "64bitEnabled", &pVM->hwaccm.s.fAllow64BitGuests, true);
-    AssertLogRelRCReturn(rc, rc);
-#endif
-
-
-    /** Determine the init method for AMD-V and VT-x; either one global init for each host CPU
-     *  or local init each time we wish to execute guest code.
-     *
-     *  Default false for Mac OS X and Windows due to the higher risk of conflicts with other hypervisors.
-     */
-    rc = CFGMR3QueryBoolDef(pHWVirtExt, "Exclusive", &pVM->hwaccm.s.fGlobalInit,
-#if defined(RT_OS_DARWIN) || defined(RT_OS_WINDOWS)
-                            false
-#else
-                            true
-#endif
-                           );
-
-    /* Max number of resume loops. */
-    rc = CFGMR3QueryU32Def(pHWVirtExt, "MaxResumeLoops", &pVM->hwaccm.s.cMaxResumeLoops, 0 /* set by R0 later */);
-    AssertRC(rc);
-
-    return rc;
-}
-
-/**
- * Initializes the per-VCPU HWACCM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-static int hwaccmR3InitCPU(PVM pVM)
-{
-    LogFlow(("HWACCMR3InitCPU\n"));
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        pVCpu->hwaccm.s.fActive = false;
-    }
-
-#ifdef VBOX_WITH_STATISTICS
-    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRPatchSuccess,   STAMTYPE_COUNTER, "/HWACCM/TPR/Patch/Success",  STAMUNIT_OCCURENCES,    "Number of times an instruction was successfully patched.");
-    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRPatchFailure,   STAMTYPE_COUNTER, "/HWACCM/TPR/Patch/Failed",   STAMUNIT_OCCURENCES,    "Number of unsuccessful patch attempts.");
-    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRReplaceSuccess, STAMTYPE_COUNTER, "/HWACCM/TPR/Replace/Success",STAMUNIT_OCCURENCES,    "Number of times an instruction was successfully patched.");
-    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRReplaceFailure, STAMTYPE_COUNTER, "/HWACCM/TPR/Replace/Failed", STAMUNIT_OCCURENCES,    "Number of unsuccessful patch attempts.");
-
-    /*
-     * Statistics.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-        int    rc;
-
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatPoke, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of RTMpPokeCpu",
-                             "/PROF/HWACCM/CPU%d/Poke", i);
-        AssertRC(rc);
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatSpinPoke, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of poke wait",
-                             "/PROF/HWACCM/CPU%d/PokeWait", i);
-        AssertRC(rc);
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatSpinPokeFailed, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of poke wait when RTMpPokeCpu fails",
-                             "/PROF/HWACCM/CPU%d/PokeWaitFailed", i);
-        AssertRC(rc);
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatEntry, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode entry",
-                             "/PROF/HWACCM/CPU%d/SwitchToGC", i);
-        AssertRC(rc);
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit1, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode exit part 1",
-                             "/PROF/HWACCM/CPU%d/SwitchFromGC_1", i);
-        AssertRC(rc);
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode exit part 2",
-                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2", i);
-        AssertRC(rc);
-# if 1 /* temporary for tracking down darwin holdup. */
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2Sub1, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Temporary - I/O",
-                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2/Sub1", i);
-        AssertRC(rc);
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2Sub2, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Temporary - CRx RWs",
-                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2/Sub2", i);
-        AssertRC(rc);
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2Sub3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Temporary - Exceptions",
-                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2/Sub3", i);
-        AssertRC(rc);
-# endif
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatInGC, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of vmlaunch",
-                             "/PROF/HWACCM/CPU%d/InGC", i);
-        AssertRC(rc);
-
-# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatWorldSwitch3264, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of the 32/64 switcher",
-                             "/PROF/HWACCM/CPU%d/Switcher3264", i);
-        AssertRC(rc);
-# endif
-
-# define HWACCM_REG_COUNTER(a, b) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Profiling of vmlaunch", b, i); \
-        AssertRC(rc);
-
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitShadowNM,           "/HWACCM/CPU%d/Exit/Trap/Shw/#NM");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestNM,            "/HWACCM/CPU%d/Exit/Trap/Gst/#NM");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitShadowPF,           "/HWACCM/CPU%d/Exit/Trap/Shw/#PF");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestPF,            "/HWACCM/CPU%d/Exit/Trap/Gst/#PF");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestUD,            "/HWACCM/CPU%d/Exit/Trap/Gst/#UD");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestSS,            "/HWACCM/CPU%d/Exit/Trap/Gst/#SS");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestNP,            "/HWACCM/CPU%d/Exit/Trap/Gst/#NP");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestGP,            "/HWACCM/CPU%d/Exit/Trap/Gst/#GP");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestMF,            "/HWACCM/CPU%d/Exit/Trap/Gst/#MF");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestDE,            "/HWACCM/CPU%d/Exit/Trap/Gst/#DE");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestDB,            "/HWACCM/CPU%d/Exit/Trap/Gst/#DB");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitInvpg,              "/HWACCM/CPU%d/Exit/Instr/Invlpg");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitInvd,               "/HWACCM/CPU%d/Exit/Instr/Invd");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitCpuid,              "/HWACCM/CPU%d/Exit/Instr/Cpuid");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitRdtsc,              "/HWACCM/CPU%d/Exit/Instr/Rdtsc");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitRdpmc,              "/HWACCM/CPU%d/Exit/Instr/Rdpmc");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitRdmsr,              "/HWACCM/CPU%d/Exit/Instr/Rdmsr");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitWrmsr,              "/HWACCM/CPU%d/Exit/Instr/Wrmsr");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitMwait,              "/HWACCM/CPU%d/Exit/Instr/Mwait");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitMonitor,            "/HWACCM/CPU%d/Exit/Instr/Monitor");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitDRxWrite,           "/HWACCM/CPU%d/Exit/Instr/DR/Write");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitDRxRead,            "/HWACCM/CPU%d/Exit/Instr/DR/Read");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitCLTS,               "/HWACCM/CPU%d/Exit/Instr/CLTS");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitLMSW,               "/HWACCM/CPU%d/Exit/Instr/LMSW");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitCli,                "/HWACCM/CPU%d/Exit/Instr/Cli");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitSti,                "/HWACCM/CPU%d/Exit/Instr/Sti");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitPushf,              "/HWACCM/CPU%d/Exit/Instr/Pushf");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitPopf,               "/HWACCM/CPU%d/Exit/Instr/Popf");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIret,               "/HWACCM/CPU%d/Exit/Instr/Iret");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitInt,                "/HWACCM/CPU%d/Exit/Instr/Int");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitHlt,                "/HWACCM/CPU%d/Exit/Instr/Hlt");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIOWrite,            "/HWACCM/CPU%d/Exit/IO/Write");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIORead,             "/HWACCM/CPU%d/Exit/IO/Read");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIOStringWrite,      "/HWACCM/CPU%d/Exit/IO/WriteString");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIOStringRead,       "/HWACCM/CPU%d/Exit/IO/ReadString");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIrqWindow,          "/HWACCM/CPU%d/Exit/IrqWindow");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitMaxResume,          "/HWACCM/CPU%d/Exit/MaxResume");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitPreemptPending,     "/HWACCM/CPU%d/Exit/PreemptPending");
-
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatSwitchGuestIrq,         "/HWACCM/CPU%d/Switch/IrqPending");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatSwitchToR3,             "/HWACCM/CPU%d/Switch/ToR3");
-
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatIntInject,              "/HWACCM/CPU%d/Irq/Inject");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatIntReinject,            "/HWACCM/CPU%d/Irq/Reinject");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatPendingHostIrq,         "/HWACCM/CPU%d/Irq/PendingOnHost");
-
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPage,              "/HWACCM/CPU%d/Flush/Page");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPageManual,        "/HWACCM/CPU%d/Flush/Page/Virt");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPhysPageManual,    "/HWACCM/CPU%d/Flush/Page/Phys");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLB,               "/HWACCM/CPU%d/Flush/TLB");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBManual,         "/HWACCM/CPU%d/Flush/TLB/Manual");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBCRxChange,      "/HWACCM/CPU%d/Flush/TLB/CRx");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPageInvlpg,        "/HWACCM/CPU%d/Flush/Page/Invlpg");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch,    "/HWACCM/CPU%d/Flush/TLB/Switch");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch,  "/HWACCM/CPU%d/Flush/TLB/Skipped");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushASID,              "/HWACCM/CPU%d/Flush/TLB/ASID");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBInvlpga,        "/HWACCM/CPU%d/Flush/TLB/PhysInvl");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTlbShootdown,           "/HWACCM/CPU%d/Flush/Shootdown/Page");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTlbShootdownFlush,      "/HWACCM/CPU%d/Flush/Shootdown/TLB");
-
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTSCOffset,              "/HWACCM/CPU%d/TSC/Offset");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTSCIntercept,           "/HWACCM/CPU%d/TSC/Intercept");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow,   "/HWACCM/CPU%d/TSC/InterceptOverflow");
-
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDRxArmed,               "/HWACCM/CPU%d/Debug/Armed");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDRxContextSwitch,       "/HWACCM/CPU%d/Debug/ContextSwitch");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDRxIOCheck,             "/HWACCM/CPU%d/Debug/IOCheck");
-
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatLoadMinimal,            "/HWACCM/CPU%d/Load/Minimal");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatLoadFull,               "/HWACCM/CPU%d/Load/Full");
-
-#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFpu64SwitchBack,        "/HWACCM/CPU%d/Switch64/Fpu");
-        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDebug64SwitchBack,      "/HWACCM/CPU%d/Switch64/Debug");
-#endif
-
-        for (unsigned j=0;j<RT_ELEMENTS(pVCpu->hwaccm.s.StatExitCRxWrite);j++)
-        {
-            rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExitCRxWrite[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Profiling of CRx writes",
-                                "/HWACCM/CPU%d/Exit/Instr/CR/Write/%x", i, j);
-            AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExitCRxRead[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Profiling of CRx reads",
-                                "/HWACCM/CPU%d/Exit/Instr/CR/Read/%x", i, j);
-            AssertRC(rc);
-        }
-
-#undef HWACCM_REG_COUNTER
-
-        pVCpu->hwaccm.s.paStatExitReason = NULL;
-
-        rc = MMHyperAlloc(pVM, MAX_EXITREASON_STAT*sizeof(*pVCpu->hwaccm.s.paStatExitReason), 0, MM_TAG_HWACCM, (void **)&pVCpu->hwaccm.s.paStatExitReason);
-        AssertRC(rc);
-        if (RT_SUCCESS(rc))
-        {
-            const char * const *papszDesc = ASMIsIntelCpu() ? &g_apszVTxExitReasons[0] : &g_apszAmdVExitReasons[0];
-            for (int j=0;j<MAX_EXITREASON_STAT;j++)
-            {
-                if (papszDesc[j])
-                {
-                    rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.paStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
-                                        papszDesc[j], "/HWACCM/CPU%d/Exit/Reason/%02x", i, j);
-                    AssertRC(rc);
-                }
-            }
-            rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExitReasonNPF, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Nested page fault", "/HWACCM/CPU%d/Exit/Reason/#NPF", i);
-            AssertRC(rc);
-        }
-        pVCpu->hwaccm.s.paStatExitReasonR0 = MMHyperR3ToR0(pVM, pVCpu->hwaccm.s.paStatExitReason);
-# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-        Assert(pVCpu->hwaccm.s.paStatExitReasonR0 != NIL_RTR0PTR || !VMMIsHwVirtExtForced(pVM));
-# else
-        Assert(pVCpu->hwaccm.s.paStatExitReasonR0 != NIL_RTR0PTR);
-# endif
-
-        rc = MMHyperAlloc(pVM, sizeof(STAMCOUNTER) * 256, 8, MM_TAG_HWACCM, (void **)&pVCpu->hwaccm.s.paStatInjectedIrqs);
-        AssertRCReturn(rc, rc);
-        pVCpu->hwaccm.s.paStatInjectedIrqsR0 = MMHyperR3ToR0(pVM, pVCpu->hwaccm.s.paStatInjectedIrqs);
-# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-        Assert(pVCpu->hwaccm.s.paStatInjectedIrqsR0 != NIL_RTR0PTR || !VMMIsHwVirtExtForced(pVM));
-# else
-        Assert(pVCpu->hwaccm.s.paStatInjectedIrqsR0 != NIL_RTR0PTR);
-# endif
-        for (unsigned j = 0; j < 255; j++)
-            STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.paStatInjectedIrqs[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Forwarded interrupts.",
-                            (j < 0x20) ? "/HWACCM/CPU%d/Interrupt/Trap/%02X" : "/HWACCM/CPU%d/Interrupt/IRQ/%02X", i, j);
-
-    }
-#endif /* VBOX_WITH_STATISTICS */
-
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-    /* Magic marker for searching in crash dumps. */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
-        strcpy((char *)pCache->aMagic, "VMCSCACHE Magic");
-        pCache->uMagic = UINT64_C(0xDEADBEEFDEADBEEF);
-    }
-#endif
-    return VINF_SUCCESS;
-}
-
-/**
- * Called when a init phase has completed.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM.
- * @param   enmWhat             The phase that completed.
- */
-VMMR3_INT_DECL(int) HWACCMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
-{
-    switch (enmWhat)
-    {
-    case VMINITCOMPLETED_RING3:
-        return hwaccmR3InitCPU(pVM);
-    default:
-        return VINF_SUCCESS;
-    }
-}
-
-/**
- * Turns off normal raw mode features
- *
- * @param   pVM         The VM to operate on.
- */
-static void hwaccmR3DisableRawMode(PVM pVM)
-{
-    /* Disable PATM & CSAM. */
-    PATMR3AllowPatching(pVM, false);
-    CSAMDisableScanning(pVM);
-
-    /* Turn off IDT/LDT/GDT and TSS monitoring and sycing. */
-    SELMR3DisableMonitoring(pVM);
-    TRPMR3DisableMonitoring(pVM);
-
-    /* Disable the switcher code (safety precaution). */
-    VMMR3DisableSwitcher(pVM);
-
-    /* Disable mapping of the hypervisor into the shadow page table. */
-    PGMR3MappingsDisable(pVM);
-
-    /* Disable the switcher */
-    VMMR3DisableSwitcher(pVM);
-
-    /* Reinit the paging mode to force the new shadow mode. */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
-    }
-}
-
-/**
- * Initialize VT-x or AMD-V.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(int) HWACCMR3InitFinalizeR0(PVM pVM)
-{
-    int rc;
-
-    /* Hack to allow users to work around broken BIOSes that incorrectly set EFER.SVME, which makes us believe somebody else
-     * is already using AMD-V.
-     */
-    if (    !pVM->hwaccm.s.vmx.fSupported
-        &&  !pVM->hwaccm.s.svm.fSupported
-        &&  pVM->hwaccm.s.lLastError == VERR_SVM_IN_USE /* implies functional AMD-V */
-        &&  RTEnvExist("VBOX_HWVIRTEX_IGNORE_SVM_IN_USE"))
-    {
-        LogRel(("HWACCM: VBOX_HWVIRTEX_IGNORE_SVM_IN_USE active!\n"));
-        pVM->hwaccm.s.svm.fSupported        = true;
-        pVM->hwaccm.s.svm.fIgnoreInUseError = true;
-    }
-    else
-    if (    !pVM->hwaccm.s.vmx.fSupported
-        &&  !pVM->hwaccm.s.svm.fSupported)
-    {
-        LogRel(("HWACCM: No VT-x or AMD-V CPU extension found. Reason %Rrc\n", pVM->hwaccm.s.lLastError));
-        LogRel(("HWACCM: VMX MSR_IA32_FEATURE_CONTROL=%RX64\n", pVM->hwaccm.s.vmx.msr.feature_ctrl));
-
-        if (VMMIsHwVirtExtForced(pVM))
-        {
-            switch (pVM->hwaccm.s.lLastError)
-            {
-            case VERR_VMX_NO_VMX:
-                return VM_SET_ERROR(pVM, VERR_VMX_NO_VMX, "VT-x is not available.");
-            case VERR_VMX_IN_VMX_ROOT_MODE:
-                return VM_SET_ERROR(pVM, VERR_VMX_IN_VMX_ROOT_MODE, "VT-x is being used by another hypervisor.");
-            case VERR_SVM_IN_USE:
-                return VM_SET_ERROR(pVM, VERR_SVM_IN_USE, "AMD-V is being used by another hypervisor.");
-            case VERR_SVM_NO_SVM:
-                return VM_SET_ERROR(pVM, VERR_SVM_NO_SVM, "AMD-V is not available.");
-            case VERR_SVM_DISABLED:
-                return VM_SET_ERROR(pVM, VERR_SVM_DISABLED, "AMD-V is disabled in the BIOS.");
-            default:
-                return pVM->hwaccm.s.lLastError;
-            }
-        }
-        return VINF_SUCCESS;
-    }
-
-    if (pVM->hwaccm.s.vmx.fSupported)
-    {
-        rc = SUPR3QueryVTxSupported();
-        if (RT_FAILURE(rc))
-        {
-#ifdef RT_OS_LINUX
-            LogRel(("HWACCM: The host kernel does not support VT-x -- Linux 2.6.13 or newer required!\n"));
-#else
-            LogRel(("HWACCM: The host kernel does not support VT-x!\n"));
-#endif
-            if (   pVM->cCpus > 1
-                || VMMIsHwVirtExtForced(pVM))
-                return rc;
-
-            /* silently fall back to raw mode */
-            return VINF_SUCCESS;
-        }
-    }
-
-    if (!pVM->hwaccm.s.fAllowed)
-        return VINF_SUCCESS;    /* nothing to do */
-
-    /* Enable VT-x or AMD-V on all host CPUs. */
-    rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_HWACC_ENABLE, 0, NULL);
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("HWACCMR3InitFinalize: SUPR3CallVMMR0Ex VMMR0_DO_HWACC_ENABLE failed with %Rrc\n", rc));
-        return rc;
-    }
-    Assert(!pVM->fHWACCMEnabled || VMMIsHwVirtExtForced(pVM));
-
-    pVM->hwaccm.s.fHasIoApic = PDMHasIoApic(pVM);
-    /* No TPR patching is required when the IO-APIC is not enabled for this VM. (Main should have taken care of this already) */
-    if (!pVM->hwaccm.s.fHasIoApic)
-    {
-        Assert(!pVM->hwaccm.s.fTRPPatchingAllowed); /* paranoia */
-        pVM->hwaccm.s.fTRPPatchingAllowed = false;
-    }
-
-    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
-    if (pVM->hwaccm.s.vmx.fSupported)
-    {
-        Log(("pVM->hwaccm.s.vmx.fSupported = %d\n", pVM->hwaccm.s.vmx.fSupported));
-
-        if (    pVM->hwaccm.s.fInitialized == false
-            &&  pVM->hwaccm.s.vmx.msr.feature_ctrl != 0)
-        {
-            uint64_t val;
-            RTGCPHYS GCPhys = 0;
-
-            LogRel(("HWACCM: Host CR4=%08X\n", pVM->hwaccm.s.vmx.hostCR4));
-            LogRel(("HWACCM: MSR_IA32_FEATURE_CONTROL      = %RX64\n", pVM->hwaccm.s.vmx.msr.feature_ctrl));
-            LogRel(("HWACCM: MSR_IA32_VMX_BASIC_INFO       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_basic_info));
-            LogRel(("HWACCM: VMCS id                       = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
-            LogRel(("HWACCM: VMCS size                     = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
-            LogRel(("HWACCM: VMCS physical address limit   = %s\n", MSR_IA32_VMX_BASIC_INFO_VMCS_PHYS_WIDTH(pVM->hwaccm.s.vmx.msr.vmx_basic_info) ? "< 4 GB" : "None"));
-            LogRel(("HWACCM: VMCS memory type              = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_MEM_TYPE(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
-            LogRel(("HWACCM: Dual monitor treatment        = %d\n", MSR_IA32_VMX_BASIC_INFO_VMCS_DUAL_MON(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
-
-            LogRel(("HWACCM: MSR_IA32_VMX_PINBASED_CTLS    = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.u));
-            val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI\n"));
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER\n"));
-            val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER *must* be set\n"));
-
-            LogRel(("HWACCM: MSR_IA32_VMX_PROCBASED_CTLS   = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.u));
-            val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL\n"));
-
-            val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL *must* be set\n"));
-
-            if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
-            {
-                LogRel(("HWACCM: MSR_IA32_VMX_PROCBASED_CTLS2  = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.u));
-                val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_EPT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_EPT\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_X2APIC)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_X2APIC\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VPID)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VPID\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT\n"));
-
-                val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_X2APIC)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_X2APIC *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_EPT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_EPT *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VPID)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VPID *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE *must* be set\n"));
-                if (val & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT)
-                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT *must* be set\n"));
-            }
-
-            LogRel(("HWACCM: MSR_IA32_VMX_ENTRY_CTLS       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_entry.u));
-            val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR\n"));
-            val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR *must* be set\n"));
-
-            LogRel(("HWACCM: MSR_IA32_VMX_EXIT_CTLS        = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_exit.u));
-            val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER\n"));
-            val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64 *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR *must* be set\n"));
-            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
-                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER *must* be set\n"));
-
-            if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps)
-            {
-                LogRel(("HWACCM: MSR_IA32_VMX_EPT_VPID_CAPS    = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_eptcaps));
-
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_RWX_X_ONLY)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_RWX_X_ONLY\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_RWX_W_ONLY)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_RWX_W_ONLY\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_RWX_WX_ONLY)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_RWX_WX_ONLY\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_21_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_21_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_30_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_30_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_39_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_39_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_48_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_48_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_57_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_57_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_UC)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_UC\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WC)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WC\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WT)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WT\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WP)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WP\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WB)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WB\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_21_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_21_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_30_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_30_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_39_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_39_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_48_BITS)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_48_BITS\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL\n"));
-                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT_GLOBAL)
-                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT_GLOBAL\n"));
-            }
-
-            LogRel(("HWACCM: MSR_IA32_VMX_MISC             = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_misc));
-            if (MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(pVM->hwaccm.s.vmx.msr.vmx_misc) == pVM->hwaccm.s.vmx.cPreemptTimerShift)
-                LogRel(("HWACCM:    MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT %x\n", MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(pVM->hwaccm.s.vmx.msr.vmx_misc)));
-            else
-                LogRel(("HWACCM:    MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT %x - erratum detected, using %x instead\n", MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(pVM->hwaccm.s.vmx.msr.vmx_misc), pVM->hwaccm.s.vmx.cPreemptTimerShift));
-            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_ACTIVITY_STATES %x\n", MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hwaccm.s.vmx.msr.vmx_misc)));
-            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_CR3_TARGET      %x\n", MSR_IA32_VMX_MISC_CR3_TARGET(pVM->hwaccm.s.vmx.msr.vmx_misc)));
-            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_MAX_MSR         %x\n", MSR_IA32_VMX_MISC_MAX_MSR(pVM->hwaccm.s.vmx.msr.vmx_misc)));
-            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_MSEG_ID         %x\n", MSR_IA32_VMX_MISC_MSEG_ID(pVM->hwaccm.s.vmx.msr.vmx_misc)));
-
-            LogRel(("HWACCM: MSR_IA32_VMX_CR0_FIXED0       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0));
-            LogRel(("HWACCM: MSR_IA32_VMX_CR0_FIXED1       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1));
-            LogRel(("HWACCM: MSR_IA32_VMX_CR4_FIXED0       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0));
-            LogRel(("HWACCM: MSR_IA32_VMX_CR4_FIXED1       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1));
-            LogRel(("HWACCM: MSR_IA32_VMX_VMCS_ENUM        = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_vmcs_enum));
-
-            LogRel(("HWACCM: TPR shadow physaddr           = %RHp\n", pVM->hwaccm.s.vmx.pAPICPhys));
-
-            /* Paranoia */
-            AssertRelease(MSR_IA32_VMX_MISC_MAX_MSR(pVM->hwaccm.s.vmx.msr.vmx_misc) >= 512);
-
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-            {
-                LogRel(("HWACCM: VCPU%d: MSR bitmap physaddr      = %RHp\n", i, pVM->aCpus[i].hwaccm.s.vmx.pMSRBitmapPhys));
-                LogRel(("HWACCM: VCPU%d: VMCS physaddr            = %RHp\n", i, pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys));
-            }
-
-#ifdef HWACCM_VTX_WITH_EPT
-            if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_EPT)
-                pVM->hwaccm.s.fNestedPaging = pVM->hwaccm.s.fAllowNestedPaging;
-#endif /* HWACCM_VTX_WITH_EPT */
-#ifdef HWACCM_VTX_WITH_VPID
-            if (    (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VPID)
-                &&  !pVM->hwaccm.s.fNestedPaging)    /* VPID and EPT are mutually exclusive. */
-                pVM->hwaccm.s.vmx.fVPID = pVM->hwaccm.s.vmx.fAllowVPID;
-#endif /* HWACCM_VTX_WITH_VPID */
-
-            /* Unrestricted guest execution relies on EPT. */
-            if (    pVM->hwaccm.s.fNestedPaging
-                &&  (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE))
-            {
-                pVM->hwaccm.s.vmx.fUnrestrictedGuest = true;
-            }
-
-            /* Only try once. */
-            pVM->hwaccm.s.fInitialized = true;
-
-            if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
-            {
-                /* Allocate three pages for the TSS we need for real mode emulation. (2 pages for the IO bitmap) */
-                rc = PDMR3VMMDevHeapAlloc(pVM, HWACCM_VTX_TOTAL_DEVHEAP_MEM, (RTR3PTR *)&pVM->hwaccm.s.vmx.pRealModeTSS);
-                if (RT_SUCCESS(rc))
-                {
-                    /* The I/O bitmap starts right after the virtual interrupt redirection bitmap. */
-                    ASMMemZero32(pVM->hwaccm.s.vmx.pRealModeTSS, sizeof(*pVM->hwaccm.s.vmx.pRealModeTSS));
-                    pVM->hwaccm.s.vmx.pRealModeTSS->offIoBitmap = sizeof(*pVM->hwaccm.s.vmx.pRealModeTSS);
-                    /* Bit set to 0 means redirection enabled. */
-                    memset(pVM->hwaccm.s.vmx.pRealModeTSS->IntRedirBitmap, 0x0, sizeof(pVM->hwaccm.s.vmx.pRealModeTSS->IntRedirBitmap));
-                    /* Allow all port IO, so the VT-x IO intercepts do their job. */
-                    memset(pVM->hwaccm.s.vmx.pRealModeTSS + 1, 0, PAGE_SIZE*2);
-                    *((unsigned char *)pVM->hwaccm.s.vmx.pRealModeTSS + HWACCM_VTX_TSS_SIZE - 2) = 0xff;
-
-                    /* Construct a 1024 element page directory with 4 MB pages for the identity mapped page table used in
-                     * real and protected mode without paging with EPT.
-                     */
-                    pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable = (PX86PD)((char *)pVM->hwaccm.s.vmx.pRealModeTSS + PAGE_SIZE * 3);
-                    for (unsigned i=0;i<X86_PG_ENTRIES;i++)
-                    {
-                        pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable->a[i].u  = _4M * i;
-                        pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable->a[i].u |= X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_G;
-                    }
-
-                    /* We convert it here every time as pci regions could be reconfigured. */
-                    rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
-                    AssertRC(rc);
-                    LogRel(("HWACCM: Real Mode TSS guest physaddr  = %RGp\n", GCPhys));
-
-                    rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
-                    AssertRC(rc);
-                    LogRel(("HWACCM: Non-Paging Mode EPT CR3       = %RGp\n", GCPhys));
-                }
-                else
-                {
-                    LogRel(("HWACCM: No real mode VT-x support (PDMR3VMMDevHeapAlloc returned %Rrc)\n", rc));
-                    pVM->hwaccm.s.vmx.pRealModeTSS = NULL;
-                    pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable = NULL;
-                }
-            }
-
-            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_HWACC_SETUP_VM, 0, NULL);
-            AssertRC(rc);
-            if (rc == VINF_SUCCESS)
-            {
-                pVM->fHWACCMEnabled = true;
-                pVM->hwaccm.s.vmx.fEnabled = true;
-                hwaccmR3DisableRawMode(pVM);
-
-                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
-#ifdef VBOX_ENABLE_64_BITS_GUESTS
-                if (pVM->hwaccm.s.fAllow64BitGuests)
-                {
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);            /* 64 bits only on Intel CPUs */
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LAHF);
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
-                }
-                else
-                /* Turn on NXE if PAE has been enabled *and* the host has turned on NXE (we reuse the host EFER in the switcher) */
-                /* Todo: this needs to be fixed properly!! */
-                if (    CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE)
-                    &&  (pVM->hwaccm.s.vmx.hostEFER & MSR_K6_EFER_NXE))
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
-
-                LogRel((pVM->hwaccm.s.fAllow64BitGuests
-                        ? "HWACCM: 32-bit and 64-bit guests supported.\n"
-                        : "HWACCM: 32-bit guests supported.\n"));
-#else
-                LogRel(("HWACCM: 32-bit guests supported.\n"));
-#endif
-                LogRel(("HWACCM: VMX enabled!\n"));
-                if (pVM->hwaccm.s.fNestedPaging)
-                {
-                    LogRel(("HWACCM: Enabled nested paging\n"));
-                    LogRel(("HWACCM: EPT root page                 = %RHp\n", PGMGetHyperCR3(VMMGetCpu(pVM))));
-                    if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
-                        LogRel(("HWACCM: Unrestricted guest execution enabled!\n"));
-
-#if HC_ARCH_BITS == 64
-                    if (pVM->hwaccm.s.fLargePages)
-                    {
-                        /* Use large (2 MB) pages for our EPT PDEs where possible. */
-                        PGMSetLargePageUsage(pVM, true);
-                        LogRel(("HWACCM: Large page support enabled!\n"));
-                    }
-#endif
-                }
-                else
-                    Assert(!pVM->hwaccm.s.vmx.fUnrestrictedGuest);
-
-                if (pVM->hwaccm.s.vmx.fVPID)
-                    LogRel(("HWACCM: Enabled VPID\n"));
-
-                if (   pVM->hwaccm.s.fNestedPaging
-                    || pVM->hwaccm.s.vmx.fVPID)
-                {
-                    LogRel(("HWACCM: enmFlushPage    %d\n", pVM->hwaccm.s.vmx.enmFlushPage));
-                    LogRel(("HWACCM: enmFlushContext %d\n", pVM->hwaccm.s.vmx.enmFlushContext));
-                }
-
-                /* TPR patching status logging. */
-                if (pVM->hwaccm.s.fTRPPatchingAllowed)
-                {
-                    if (    (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
-                        &&  (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
-                    {
-                        pVM->hwaccm.s.fTRPPatchingAllowed = false;  /* not necessary as we have a hardware solution. */
-                        LogRel(("HWACCM: TPR Patching not required (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).\n"));
-                    }
-                    else
-                    {
-                        uint32_t u32Eax, u32Dummy;
-
-                        /* TPR patching needs access to the MSR_K8_LSTAR msr. */
-                        ASMCpuId(0x80000000, &u32Eax, &u32Dummy, &u32Dummy, &u32Dummy);
-                        if (    u32Eax < 0x80000001
-                            ||  !(ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
-                        {
-                            pVM->hwaccm.s.fTRPPatchingAllowed = false;
-                            LogRel(("HWACCM: TPR patching disabled (long mode not supported).\n"));
-                        }
-                    }
-                }
-                LogRel(("HWACCM: TPR Patching %s.\n", (pVM->hwaccm.s.fTRPPatchingAllowed) ? "enabled" : "disabled"));
-
-                /*
-                 * Check for preemption timer config override and log the state of it.
-                 */
-                if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
-                {
-                    PCFGMNODE pCfgHwAccM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "HWACCM");
-                    int rc2 = CFGMR3QueryBoolDef(pCfgHwAccM, "UsePreemptTimer", &pVM->hwaccm.s.vmx.fUsePreemptTimer, true);
-                    AssertLogRelRC(rc2);
-                }
-                if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
-                    LogRel(("HWACCM: Using the VMX-preemption timer (cPreemptTimerShift=%u)\n", pVM->hwaccm.s.vmx.cPreemptTimerShift));
-            }
-            else
-            {
-                LogRel(("HWACCM: VMX setup failed with rc=%Rrc!\n", rc));
-                LogRel(("HWACCM: Last instruction error %x\n", pVM->aCpus[0].hwaccm.s.vmx.lasterror.ulInstrError));
-                pVM->fHWACCMEnabled = false;
-            }
-        }
-    }
-    else
-    if (pVM->hwaccm.s.svm.fSupported)
-    {
-        Log(("pVM->hwaccm.s.svm.fSupported = %d\n", pVM->hwaccm.s.svm.fSupported));
-
-        if (pVM->hwaccm.s.fInitialized == false)
-        {
-            /* Erratum 170 which requires a forced TLB flush for each world switch:
-             * See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
-             *
-             * All BH-G1/2 and DH-G1/2 models include a fix:
-             * Athlon X2:   0x6b 1/2
-             *              0x68 1/2
-             * Athlon 64:   0x7f 1
-             *              0x6f 2
-             * Sempron:     0x7f 1/2
-             *              0x6f 2
-             *              0x6c 2
-             *              0x7c 2
-             * Turion 64:   0x68 2
-             *
-             */
-            uint32_t u32Dummy;
-            uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
-            ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
-            u32BaseFamily= (u32Version >> 8) & 0xf;
-            u32Family    = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
-            u32Model     = ((u32Version >> 4) & 0xf);
-            u32Model     = u32Model | ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
-            u32Stepping  = u32Version & 0xf;
-            if (    u32Family == 0xf
-                &&  !((u32Model == 0x68 || u32Model == 0x6b || u32Model == 0x7f) &&  u32Stepping >= 1)
-                &&  !((u32Model == 0x6f || u32Model == 0x6c || u32Model == 0x7c) &&  u32Stepping >= 2))
-            {
-                LogRel(("HWACMM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
-            }
-
-            LogRel(("HWACMM: cpuid 0x80000001.u32AMDFeatureECX = %RX32\n", pVM->hwaccm.s.cpuid.u32AMDFeatureECX));
-            LogRel(("HWACMM: cpuid 0x80000001.u32AMDFeatureEDX = %RX32\n", pVM->hwaccm.s.cpuid.u32AMDFeatureEDX));
-            LogRel(("HWACCM: AMD HWCR MSR                      = %RX64\n", pVM->hwaccm.s.svm.msrHWCR));
-            LogRel(("HWACCM: AMD-V revision                    = %X\n", pVM->hwaccm.s.svm.u32Rev));
-            LogRel(("HWACCM: AMD-V max ASID                    = %d\n", pVM->hwaccm.s.uMaxASID));
-            LogRel(("HWACCM: AMD-V features                    = %X\n", pVM->hwaccm.s.svm.u32Features));
-
-            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_NESTED_PAGING)
-                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_NESTED_PAGING\n"));
-            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_LBR_VIRT)
-                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_LBR_VIRT\n"));
-            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_SVM_LOCK)
-                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_SVM_LOCK\n"));
-            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
-                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_NRIP_SAVE\n"));
-            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_SSE_3_5_DISABLE)
-                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_SSE_3_5_DISABLE\n"));
-            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER)
-                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER\n"));
-
-            /* Only try once. */
-            pVM->hwaccm.s.fInitialized = true;
-
-            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_NESTED_PAGING)
-                pVM->hwaccm.s.fNestedPaging = pVM->hwaccm.s.fAllowNestedPaging;
-
-            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_HWACC_SETUP_VM, 0, NULL);
-            AssertRC(rc);
-            if (rc == VINF_SUCCESS)
-            {
-                pVM->fHWACCMEnabled = true;
-                pVM->hwaccm.s.svm.fEnabled = true;
-
-                if (pVM->hwaccm.s.fNestedPaging)
-                {
-                    LogRel(("HWACCM:    Enabled nested paging\n"));
-#if HC_ARCH_BITS == 64
-                    if (pVM->hwaccm.s.fLargePages)
-                    {
-                        /* Use large (2 MB) pages for our nested paging PDEs where possible. */
-                        PGMSetLargePageUsage(pVM, true);
-                        LogRel(("HWACCM:    Large page support enabled!\n"));
-                    }
-#endif
-                }
-
-                hwaccmR3DisableRawMode(pVM);
-                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
-                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);
-                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP);
-#ifdef VBOX_ENABLE_64_BITS_GUESTS
-                if (pVM->hwaccm.s.fAllow64BitGuests)
-                {
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LAHF);
-                }
-                else
-                /* Turn on NXE if PAE has been enabled. */
-                if (CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE))
-                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
-#endif
-
-                LogRel((pVM->hwaccm.s.fAllow64BitGuests
-                        ? "HWACCM:    32-bit and 64-bit guest supported.\n"
-                        : "HWACCM:    32-bit guest supported.\n"));
-
-                LogRel(("HWACCM:    TPR Patching %s.\n", (pVM->hwaccm.s.fTRPPatchingAllowed) ? "enabled" : "disabled"));
-            }
-            else
-            {
-                pVM->fHWACCMEnabled = false;
-            }
-        }
-    }
-    if (pVM->fHWACCMEnabled)
-        LogRel(("HWACCM:    VT-x/AMD-V init method: %s\n", (pVM->hwaccm.s.fGlobalInit) ? "GLOBAL" : "LOCAL"));
-    RTLogRelSetBuffering(fOldBuffered);
-    return VINF_SUCCESS;
-}
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM     The VM.
- */
-VMMR3DECL(void) HWACCMR3Relocate(PVM pVM)
-{
-    Log(("HWACCMR3Relocate to %RGv\n", MMHyperGetArea(pVM, 0)));
-
-    /* Fetch the current paging mode during the relocate callback during state loading. */
-    if (VMR3GetState(pVM) == VMSTATE_LOADING)
-    {
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            PVMCPU pVCpu = &pVM->aCpus[i];
-
-            pVCpu->hwaccm.s.enmShadowMode            = PGMGetShadowMode(pVCpu);
-            Assert(pVCpu->hwaccm.s.vmx.enmCurrGuestMode == PGMGetGuestMode(pVCpu));
-            pVCpu->hwaccm.s.vmx.enmCurrGuestMode     = PGMGetGuestMode(pVCpu);
-        }
-    }
-#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    if (pVM->fHWACCMEnabled)
-    {
-        int rc;
-
-        switch(PGMGetHostMode(pVM))
-        {
-        case PGMMODE_32_BIT:
-            pVM->hwaccm.s.pfnHost32ToGuest64R0 = VMMR3GetHostToGuestSwitcher(pVM, VMMSWITCHER_32_TO_AMD64);
-            break;
-
-        case PGMMODE_PAE:
-        case PGMMODE_PAE_NX:
-            pVM->hwaccm.s.pfnHost32ToGuest64R0 = VMMR3GetHostToGuestSwitcher(pVM, VMMSWITCHER_PAE_TO_AMD64);
-            break;
-
-        default:
-            AssertFailed();
-            break;
-        }
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "VMXGCStartVM64", &pVM->hwaccm.s.pfnVMXGCStartVM64);
-        AssertReleaseMsgRC(rc, ("VMXGCStartVM64 -> rc=%Rrc\n", rc));
-
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "SVMGCVMRun64",   &pVM->hwaccm.s.pfnSVMGCVMRun64);
-        AssertReleaseMsgRC(rc, ("SVMGCVMRun64 -> rc=%Rrc\n", rc));
-
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "HWACCMSaveGuestFPU64",   &pVM->hwaccm.s.pfnSaveGuestFPU64);
-        AssertReleaseMsgRC(rc, ("HWACCMSetupFPU64 -> rc=%Rrc\n", rc));
-
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "HWACCMSaveGuestDebug64",   &pVM->hwaccm.s.pfnSaveGuestDebug64);
-        AssertReleaseMsgRC(rc, ("HWACCMSetupDebug64 -> rc=%Rrc\n", rc));
-
-# ifdef DEBUG
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "HWACCMTestSwitcher64",   &pVM->hwaccm.s.pfnTest64);
-        AssertReleaseMsgRC(rc, ("HWACCMTestSwitcher64 -> rc=%Rrc\n", rc));
-# endif
-    }
-#endif
-    return;
-}
-
-/**
- * Checks hardware accelerated raw mode is allowed.
- *
- * @returns boolean
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(bool) HWACCMR3IsAllowed(PVM pVM)
-{
-    return pVM->hwaccm.s.fAllowed;
-}
-
-/**
- * Notification callback which is called whenever there is a chance that a CR3
- * value might have changed.
- *
- * This is called by PGM.
- *
- * @param   pVM            The VM to operate on.
- * @param   pVCpu          The VMCPU to operate on.
- * @param   enmShadowMode  New shadow paging mode.
- * @param   enmGuestMode   New guest paging mode.
- */
-VMMR3DECL(void) HWACCMR3PagingModeChanged(PVM pVM, PVMCPU pVCpu, PGMMODE enmShadowMode, PGMMODE enmGuestMode)
-{
-    /* Ignore page mode changes during state loading. */
-    if (VMR3GetState(pVCpu->pVMR3) == VMSTATE_LOADING)
-        return;
-
-    pVCpu->hwaccm.s.enmShadowMode = enmShadowMode;
-
-    if (   pVM->hwaccm.s.vmx.fEnabled
-        && pVM->fHWACCMEnabled)
-    {
-        if (    pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
-            &&  enmGuestMode >= PGMMODE_PROTECTED)
-        {
-            PCPUMCTX pCtx;
-
-            pCtx = CPUMQueryGuestCtxPtr(pVCpu);
-
-            /* After a real mode switch to protected mode we must force
-             * CPL to 0. Our real mode emulation had to set it to 3.
-             */
-            pCtx->ssHid.Attr.n.u2Dpl  = 0;
-        }
-    }
-
-    if (pVCpu->hwaccm.s.vmx.enmCurrGuestMode != enmGuestMode)
-    {
-        /* Keep track of paging mode changes. */
-        pVCpu->hwaccm.s.vmx.enmPrevGuestMode = pVCpu->hwaccm.s.vmx.enmCurrGuestMode;
-        pVCpu->hwaccm.s.vmx.enmCurrGuestMode = enmGuestMode;
-
-        /* Did we miss a change, because all code was executed in the recompiler? */
-        if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == enmGuestMode)
-        {
-            Log(("HWACCMR3PagingModeChanged missed %s->%s transition (prev %s)\n", PGMGetModeName(pVCpu->hwaccm.s.vmx.enmPrevGuestMode), PGMGetModeName(pVCpu->hwaccm.s.vmx.enmCurrGuestMode), PGMGetModeName(pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode)));
-            pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = pVCpu->hwaccm.s.vmx.enmPrevGuestMode;
-        }
-    }
-
-    /* Reset the contents of the read cache. */
-    PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
-    for (unsigned j=0;j<pCache->Read.cValidEntries;j++)
-        pCache->Read.aFieldVal[j] = 0;
-}
-
-/**
- * Terminates the HWACCM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) HWACCMR3Term(PVM pVM)
-{
-    if (pVM->hwaccm.s.vmx.pRealModeTSS)
-    {
-        PDMR3VMMDevHeapFree(pVM, pVM->hwaccm.s.vmx.pRealModeTSS);
-        pVM->hwaccm.s.vmx.pRealModeTSS       = 0;
-    }
-    hwaccmR3TermCPU(pVM);
-    return 0;
-}
-
-/**
- * Terminates the per-VCPU HWACCM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-static int hwaccmR3TermCPU(PVM pVM)
-{
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-#ifdef VBOX_WITH_STATISTICS
-        if (pVCpu->hwaccm.s.paStatExitReason)
-        {
-            MMHyperFree(pVM, pVCpu->hwaccm.s.paStatExitReason);
-            pVCpu->hwaccm.s.paStatExitReason   = NULL;
-            pVCpu->hwaccm.s.paStatExitReasonR0 = NIL_RTR0PTR;
-        }
-        if (pVCpu->hwaccm.s.paStatInjectedIrqs)
-        {
-            MMHyperFree(pVM, pVCpu->hwaccm.s.paStatInjectedIrqs);
-            pVCpu->hwaccm.s.paStatInjectedIrqs   = NULL;
-            pVCpu->hwaccm.s.paStatInjectedIrqsR0 = NIL_RTR0PTR;
-        }
-#endif
-
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-        memset(pVCpu->hwaccm.s.vmx.VMCSCache.aMagic, 0, sizeof(pVCpu->hwaccm.s.vmx.VMCSCache.aMagic));
-        pVCpu->hwaccm.s.vmx.VMCSCache.uMagic = 0;
-        pVCpu->hwaccm.s.vmx.VMCSCache.uPos = 0xffffffff;
-#endif
-    }
-    return 0;
-}
-
-/**
- * Resets a virtual CPU.
- *
- * Used by HWACCMR3Reset and CPU hot plugging.
- *
- * @param   pVCpu   The CPU to reset.
- */
-VMMR3DECL(void) HWACCMR3ResetCpu(PVMCPU pVCpu)
-{
-    /* On first entry we'll sync everything. */
-    pVCpu->hwaccm.s.fContextUseFlags = HWACCM_CHANGED_ALL;
-
-    pVCpu->hwaccm.s.vmx.cr0_mask = 0;
-    pVCpu->hwaccm.s.vmx.cr4_mask = 0;
-
-    pVCpu->hwaccm.s.fActive        = false;
-    pVCpu->hwaccm.s.Event.fPending = false;
-
-    /* Reset state information for real-mode emulation in VT-x. */
-    pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
-    pVCpu->hwaccm.s.vmx.enmPrevGuestMode     = PGMMODE_REAL;
-    pVCpu->hwaccm.s.vmx.enmCurrGuestMode     = PGMMODE_REAL;
-
-    /* Reset the contents of the read cache. */
-    PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
-    for (unsigned j=0;j<pCache->Read.cValidEntries;j++)
-        pCache->Read.aFieldVal[j] = 0;
-
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-    /* Magic marker for searching in crash dumps. */
-    strcpy((char *)pCache->aMagic, "VMCSCACHE Magic");
-    pCache->uMagic = UINT64_C(0xDEADBEEFDEADBEEF);
-#endif
-}
-
-/**
- * The VM is being reset.
- *
- * For the HWACCM component this means that any GDT/LDT/TSS monitors
- * needs to be removed.
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) HWACCMR3Reset(PVM pVM)
-{
-    LogFlow(("HWACCMR3Reset:\n"));
-
-    if (pVM->fHWACCMEnabled)
-        hwaccmR3DisableRawMode(pVM);
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        HWACCMR3ResetCpu(pVCpu);
-    }
-
-    /* Clear all patch information. */
-    pVM->hwaccm.s.pGuestPatchMem         = 0;
-    pVM->hwaccm.s.pFreeGuestPatchMem     = 0;
-    pVM->hwaccm.s.cbGuestPatchMem        = 0;
-    pVM->hwaccm.s.cPatches           = 0;
-    pVM->hwaccm.s.PatchTree          = 0;
-    pVM->hwaccm.s.fTPRPatchingActive = false;
-    ASMMemZero32(pVM->hwaccm.s.aPatches, sizeof(pVM->hwaccm.s.aPatches));
-}
-
-/**
- * Callback to patch a TPR instruction (vmmcall or mov cr8)
- *
- * @returns VBox strict status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU for the EMT we're being called on.
- * @param   pvUser  Unused
- *
- */
-DECLCALLBACK(VBOXSTRICTRC) hwaccmR3RemovePatches(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    VMCPUID idCpu = (VMCPUID)(uintptr_t)pvUser;
-
-    /* Only execute the handler on the VCPU the original patch request was issued. */
-    if (pVCpu->idCpu != idCpu)
-        return VINF_SUCCESS;
-
-    Log(("hwaccmR3RemovePatches\n"));
-    for (unsigned i = 0; i < pVM->hwaccm.s.cPatches; i++)
-    {
-        uint8_t         szInstr[15];
-        PHWACCMTPRPATCH pPatch = &pVM->hwaccm.s.aPatches[i];
-        RTGCPTR         pInstrGC = (RTGCPTR)pPatch->Core.Key;
-        int             rc;
-
-#ifdef LOG_ENABLED
-        char            szOutput[256];
-
-        rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, CPUMGetGuestCS(pVCpu), pInstrGC, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                                szOutput, sizeof(szOutput), NULL);
-        if (RT_SUCCESS(rc))
-            Log(("Patched instr: %s\n", szOutput));
-#endif
-
-        /* Check if the instruction is still the same. */
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, szInstr, pInstrGC, pPatch->cbNewOp);
-        if (rc != VINF_SUCCESS)
-        {
-            Log(("Patched code removed? (rc=%Rrc0\n", rc));
-            continue;   /* swapped out or otherwise removed; skip it. */
-        }
-
-        if (memcmp(szInstr, pPatch->aNewOpcode, pPatch->cbNewOp))
-        {
-            Log(("Patched instruction was changed! (rc=%Rrc0\n", rc));
-            continue;   /* skip it. */
-        }
-
-        rc = PGMPhysSimpleWriteGCPtr(pVCpu, pInstrGC, pPatch->aOpcode, pPatch->cbOp);
-        AssertRC(rc);
-
-#ifdef LOG_ENABLED
-        rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, CPUMGetGuestCS(pVCpu), pInstrGC, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                                szOutput, sizeof(szOutput), NULL);
-        if (RT_SUCCESS(rc))
-            Log(("Original instr: %s\n", szOutput));
-#endif
-    }
-    pVM->hwaccm.s.cPatches           = 0;
-    pVM->hwaccm.s.PatchTree          = 0;
-    pVM->hwaccm.s.pFreeGuestPatchMem = pVM->hwaccm.s.pGuestPatchMem;
-    pVM->hwaccm.s.fTPRPatchingActive = false;
-    return VINF_SUCCESS;
-}
-
-/**
- * Enable patching in a VT-x/AMD-V guest
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   idCpu       VCPU to execute hwaccmR3RemovePatches on
- * @param   pPatchMem   Patch memory range
- * @param   cbPatchMem  Size of the memory range
- */
-int hwaccmR3EnablePatching(PVM pVM, VMCPUID idCpu, RTRCPTR pPatchMem, unsigned cbPatchMem)
-{
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, hwaccmR3RemovePatches, (void *)idCpu);
-    AssertRC(rc);
-
-    pVM->hwaccm.s.pGuestPatchMem      = pPatchMem;
-    pVM->hwaccm.s.pFreeGuestPatchMem  = pPatchMem;
-    pVM->hwaccm.s.cbGuestPatchMem     = cbPatchMem;
-    return VINF_SUCCESS;
-}
-
-/**
- * Enable patching in a VT-x/AMD-V guest
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pPatchMem   Patch memory range
- * @param   cbPatchMem  Size of the memory range
- */
-VMMR3DECL(int)  HWACMMR3EnablePatching(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
-{
-    Log(("HWACMMR3EnablePatching %RGv size %x\n", pPatchMem, cbPatchMem));
-    if (pVM->cCpus > 1)
-    {
-        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
-        int rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE,
-                                    (PFNRT)hwaccmR3EnablePatching, 4, pVM, VMMGetCpuId(pVM), (RTRCPTR)pPatchMem, cbPatchMem);
-        AssertRC(rc);
-        return rc;
-    }
-    return hwaccmR3EnablePatching(pVM, VMMGetCpuId(pVM), (RTRCPTR)pPatchMem, cbPatchMem);
-}
-
-/**
- * Disable patching in a VT-x/AMD-V guest
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pPatchMem   Patch memory range
- * @param   cbPatchMem  Size of the memory range
- */
-VMMR3DECL(int)  HWACMMR3DisablePatching(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
-{
-    Log(("HWACMMR3DisablePatching %RGv size %x\n", pPatchMem, cbPatchMem));
-
-    Assert(pVM->hwaccm.s.pGuestPatchMem == pPatchMem);
-    Assert(pVM->hwaccm.s.cbGuestPatchMem == cbPatchMem);
-
-    /* @todo Potential deadlock when other VCPUs are waiting on the IOM lock (we own it)!! */
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, hwaccmR3RemovePatches, (void *)VMMGetCpuId(pVM));
-    AssertRC(rc);
-
-    pVM->hwaccm.s.pGuestPatchMem      = 0;
-    pVM->hwaccm.s.pFreeGuestPatchMem  = 0;
-    pVM->hwaccm.s.cbGuestPatchMem     = 0;
-    pVM->hwaccm.s.fTPRPatchingActive = false;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Callback to patch a TPR instruction (vmmcall or mov cr8)
- *
- * @returns VBox strict status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU for the EMT we're being called on.
- * @param   pvUser  User specified CPU context
- *
- */
-DECLCALLBACK(VBOXSTRICTRC) hwaccmR3ReplaceTprInstr(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    VMCPUID      idCpu  = (VMCPUID)(uintptr_t)pvUser;
-    PCPUMCTX     pCtx   = CPUMQueryGuestCtxPtr(pVCpu);
-    PDISCPUSTATE pDis   = &pVCpu->hwaccm.s.DisState;
-    unsigned     cbOp;
-
-    /* Only execute the handler on the VCPU the original patch request was issued. (the other CPU(s) might not yet have switched to protected mode) */
-    if (pVCpu->idCpu != idCpu)
-        return VINF_SUCCESS;
-
-    Log(("hwaccmR3ReplaceTprInstr: %RGv\n", pCtx->rip));
-
-    /* Two or more VCPUs were racing to patch this instruction. */
-    PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
-    if (pPatch)
-        return VINF_SUCCESS;
-
-    Assert(pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches));
-
-    int rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
-    AssertRC(rc);
-    if (    rc == VINF_SUCCESS
-        &&  pDis->pCurInstr->opcode == OP_MOV
-        &&  cbOp >= 3)
-    {
-        uint8_t         aVMMCall[3] = { 0xf, 0x1, 0xd9};
-        uint32_t        idx = pVM->hwaccm.s.cPatches;
-
-        pPatch = &pVM->hwaccm.s.aPatches[idx];
-
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, pPatch->aOpcode, pCtx->rip, cbOp);
-        AssertRC(rc);
-
-        pPatch->cbOp     = cbOp;
-
-        if (pDis->param1.flags == USE_DISPLACEMENT32)
-        {
-            /* write. */
-            if (pDis->param2.flags == USE_REG_GEN32)
-            {
-                pPatch->enmType     = HWACCMTPRINSTR_WRITE_REG;
-                pPatch->uSrcOperand = pDis->param2.base.reg_gen;
-            }
-            else
-            {
-                Assert(pDis->param2.flags == USE_IMMEDIATE32);
-                pPatch->enmType     = HWACCMTPRINSTR_WRITE_IMM;
-                pPatch->uSrcOperand = pDis->param2.parval;
-            }
-            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, aVMMCall, sizeof(aVMMCall));
-            AssertRC(rc);
-
-            memcpy(pPatch->aNewOpcode, aVMMCall, sizeof(aVMMCall));
-            pPatch->cbNewOp = sizeof(aVMMCall);
-        }
-        else
-        {
-            RTGCPTR  oldrip   = pCtx->rip;
-            uint32_t oldcbOp  = cbOp;
-            uint32_t uMmioReg = pDis->param1.base.reg_gen;
-
-            /* read */
-            Assert(pDis->param1.flags == USE_REG_GEN32);
-
-            /* Found:
-                *   mov eax, dword [fffe0080]        (5 bytes)
-                * Check if next instruction is:
-                *   shr eax, 4
-                */
-            pCtx->rip += cbOp;
-            rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
-            pCtx->rip = oldrip;
-            if (    rc == VINF_SUCCESS
-                &&  pDis->pCurInstr->opcode == OP_SHR
-                &&  pDis->param1.flags == USE_REG_GEN32
-                &&  pDis->param1.base.reg_gen == uMmioReg
-                &&  pDis->param2.flags == USE_IMMEDIATE8
-                &&  pDis->param2.parval == 4
-                &&  oldcbOp + cbOp < sizeof(pVM->hwaccm.s.aPatches[idx].aOpcode))
-            {
-                uint8_t szInstr[15];
-
-                /* Replacing two instructions now. */
-                rc = PGMPhysSimpleReadGCPtr(pVCpu, &pPatch->aOpcode, pCtx->rip, oldcbOp + cbOp);
-                AssertRC(rc);
-
-                pPatch->cbOp = oldcbOp + cbOp;
-
-                /* 0xF0, 0x0F, 0x20, 0xC0 = mov eax, cr8 */
-                szInstr[0] = 0xF0;
-                szInstr[1] = 0x0F;
-                szInstr[2] = 0x20;
-                szInstr[3] = 0xC0 | pDis->param1.base.reg_gen;
-                for (unsigned i = 4; i < pPatch->cbOp; i++)
-                    szInstr[i] = 0x90;  /* nop */
-
-                rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, szInstr, pPatch->cbOp);
-                AssertRC(rc);
-
-                memcpy(pPatch->aNewOpcode, szInstr, pPatch->cbOp);
-                pPatch->cbNewOp = pPatch->cbOp;
-
-                Log(("Acceptable read/shr candidate!\n"));
-                pPatch->enmType = HWACCMTPRINSTR_READ_SHR4;
-            }
-            else
-            {
-                pPatch->enmType     = HWACCMTPRINSTR_READ;
-                pPatch->uDstOperand = pDis->param1.base.reg_gen;
-
-                rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, aVMMCall, sizeof(aVMMCall));
-                AssertRC(rc);
-
-                memcpy(pPatch->aNewOpcode, aVMMCall, sizeof(aVMMCall));
-                pPatch->cbNewOp = sizeof(aVMMCall);
-            }
-        }
-
-        pPatch->Core.Key = pCtx->eip;
-        rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
-        AssertRC(rc);
-
-        pVM->hwaccm.s.cPatches++;
-        STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRReplaceSuccess);
-        return VINF_SUCCESS;
-    }
-
-    /* Save invalid patch, so we will not try again. */
-    uint32_t  idx = pVM->hwaccm.s.cPatches;
-
-#ifdef LOG_ENABLED
-    char      szOutput[256];
-    rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                            szOutput, sizeof(szOutput), NULL);
-    if (RT_SUCCESS(rc))
-        Log(("Failed to patch instr: %s\n", szOutput));
-#endif
-
-    pPatch = &pVM->hwaccm.s.aPatches[idx];
-    pPatch->Core.Key = pCtx->eip;
-    pPatch->enmType  = HWACCMTPRINSTR_INVALID;
-    rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
-    AssertRC(rc);
-    pVM->hwaccm.s.cPatches++;
-    STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRReplaceFailure);
-    return VINF_SUCCESS;
-}
-
-/**
- * Callback to patch a TPR instruction (jump to generated code)
- *
- * @returns VBox strict status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU for the EMT we're being called on.
- * @param   pvUser  User specified CPU context
- *
- */
-DECLCALLBACK(VBOXSTRICTRC) hwaccmR3PatchTprInstr(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    VMCPUID      idCpu  = (VMCPUID)(uintptr_t)pvUser;
-    PCPUMCTX     pCtx   = CPUMQueryGuestCtxPtr(pVCpu);
-    PDISCPUSTATE pDis   = &pVCpu->hwaccm.s.DisState;
-    unsigned     cbOp;
-    int          rc;
-#ifdef LOG_ENABLED
-    RTGCPTR      pInstr;
-    char         szOutput[256];
-#endif
-
-    /* Only execute the handler on the VCPU the original patch request was issued. (the other CPU(s) might not yet have switched to protected mode) */
-    if (pVCpu->idCpu != idCpu)
-        return VINF_SUCCESS;
-
-    Assert(pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches));
-
-    /* Two or more VCPUs were racing to patch this instruction. */
-    PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
-    if (pPatch)
-    {
-        Log(("hwaccmR3PatchTprInstr: already patched %RGv\n", pCtx->rip));
-        return VINF_SUCCESS;
-    }
-
-    Log(("hwaccmR3PatchTprInstr %RGv\n", pCtx->rip));
-
-    rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
-    AssertRC(rc);
-    if (    rc == VINF_SUCCESS
-        &&  pDis->pCurInstr->opcode == OP_MOV
-        &&  cbOp >= 5)
-    {
-        uint32_t        idx = pVM->hwaccm.s.cPatches;
-        uint8_t         aPatch[64];
-        uint32_t        off = 0;
-
-        pPatch = &pVM->hwaccm.s.aPatches[idx];
-
-#ifdef LOG_ENABLED
-        rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                                szOutput, sizeof(szOutput), NULL);
-        if (RT_SUCCESS(rc))
-            Log(("Original instr: %s\n", szOutput));
-#endif
-
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, pPatch->aOpcode, pCtx->rip, cbOp);
-        AssertRC(rc);
-
-        pPatch->cbOp    = cbOp;
-        pPatch->enmType = HWACCMTPRINSTR_JUMP_REPLACEMENT;
-
-        if (pDis->param1.flags == USE_DISPLACEMENT32)
-        {
-            /*
-                * TPR write:
-                *
-                * push ECX                      [51]
-                * push EDX                      [52]
-                * push EAX                      [50]
-                * xor EDX,EDX                   [31 D2]
-                * mov EAX,EAX                   [89 C0]
-                *  or
-                * mov EAX,0000000CCh            [B8 CC 00 00 00]
-                * mov ECX,0C0000082h            [B9 82 00 00 C0]
-                * wrmsr                         [0F 30]
-                * pop EAX                       [58]
-                * pop EDX                       [5A]
-                * pop ECX                       [59]
-                * jmp return_address            [E9 return_address]
-                *
-                */
-            bool fUsesEax = (pDis->param2.flags == USE_REG_GEN32 && pDis->param2.base.reg_gen == USE_REG_EAX);
-
-            aPatch[off++] = 0x51;    /* push ecx */
-            aPatch[off++] = 0x52;    /* push edx */
-            if (!fUsesEax)
-                aPatch[off++] = 0x50;    /* push eax */
-            aPatch[off++] = 0x31;    /* xor edx, edx */
-            aPatch[off++] = 0xD2;
-            if (pDis->param2.flags == USE_REG_GEN32)
-            {
-                if (!fUsesEax)
-                {
-                    aPatch[off++] = 0x89;    /* mov eax, src_reg */
-                    aPatch[off++] = MAKE_MODRM(3, pDis->param2.base.reg_gen, USE_REG_EAX);
-                }
-            }
-            else
-            {
-                Assert(pDis->param2.flags == USE_IMMEDIATE32);
-                aPatch[off++] = 0xB8;    /* mov eax, immediate */
-                *(uint32_t *)&aPatch[off] = pDis->param2.parval;
-                off += sizeof(uint32_t);
-            }
-            aPatch[off++] = 0xB9;    /* mov ecx, 0xc0000082 */
-            *(uint32_t *)&aPatch[off] = MSR_K8_LSTAR;
-            off += sizeof(uint32_t);
-
-            aPatch[off++] = 0x0F;    /* wrmsr */
-            aPatch[off++] = 0x30;
-            if (!fUsesEax)
-                aPatch[off++] = 0x58;    /* pop eax */
-            aPatch[off++] = 0x5A;    /* pop edx */
-            aPatch[off++] = 0x59;    /* pop ecx */
-        }
-        else
-        {
-            /*
-                * TPR read:
-                *
-                * push ECX                      [51]
-                * push EDX                      [52]
-                * push EAX                      [50]
-                * mov ECX,0C0000082h            [B9 82 00 00 C0]
-                * rdmsr                         [0F 32]
-                * mov EAX,EAX                   [89 C0]
-                * pop EAX                       [58]
-                * pop EDX                       [5A]
-                * pop ECX                       [59]
-                * jmp return_address            [E9 return_address]
-                *
-                */
-            Assert(pDis->param1.flags == USE_REG_GEN32);
-
-            if (pDis->param1.base.reg_gen != USE_REG_ECX)
-                aPatch[off++] = 0x51;    /* push ecx */
-            if (pDis->param1.base.reg_gen != USE_REG_EDX)
-                aPatch[off++] = 0x52;    /* push edx */
-            if (pDis->param1.base.reg_gen != USE_REG_EAX)
-                aPatch[off++] = 0x50;    /* push eax */
-
-            aPatch[off++] = 0x31;    /* xor edx, edx */
-            aPatch[off++] = 0xD2;
-
-            aPatch[off++] = 0xB9;    /* mov ecx, 0xc0000082 */
-            *(uint32_t *)&aPatch[off] = MSR_K8_LSTAR;
-            off += sizeof(uint32_t);
-
-            aPatch[off++] = 0x0F;    /* rdmsr */
-            aPatch[off++] = 0x32;
-
-            if (pDis->param1.base.reg_gen != USE_REG_EAX)
-            {
-                aPatch[off++] = 0x89;    /* mov dst_reg, eax */
-                aPatch[off++] = MAKE_MODRM(3, USE_REG_EAX, pDis->param1.base.reg_gen);
-            }
-
-            if (pDis->param1.base.reg_gen != USE_REG_EAX)
-                aPatch[off++] = 0x58;    /* pop eax */
-            if (pDis->param1.base.reg_gen != USE_REG_EDX)
-                aPatch[off++] = 0x5A;    /* pop edx */
-            if (pDis->param1.base.reg_gen != USE_REG_ECX)
-                aPatch[off++] = 0x59;    /* pop ecx */
-        }
-        aPatch[off++] = 0xE9;    /* jmp return_address */
-        *(RTRCUINTPTR *)&aPatch[off] = ((RTRCUINTPTR)pCtx->eip + cbOp) - ((RTRCUINTPTR)pVM->hwaccm.s.pFreeGuestPatchMem + off + 4);
-        off += sizeof(RTRCUINTPTR);
-
-        if (pVM->hwaccm.s.pFreeGuestPatchMem + off <= pVM->hwaccm.s.pGuestPatchMem + pVM->hwaccm.s.cbGuestPatchMem)
-        {
-            /* Write new code to the patch buffer. */
-            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pVM->hwaccm.s.pFreeGuestPatchMem, aPatch, off);
-            AssertRC(rc);
-
-#ifdef LOG_ENABLED
-            pInstr = pVM->hwaccm.s.pFreeGuestPatchMem;
-            while (true)
-            {
-                uint32_t cb;
-
-                rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pInstr, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                                        szOutput, sizeof(szOutput), &cb);
-                if (RT_SUCCESS(rc))
-                    Log(("Patch instr %s\n", szOutput));
-
-                pInstr += cb;
-
-                if (pInstr >= pVM->hwaccm.s.pFreeGuestPatchMem + off)
-                    break;
-            }
-#endif
-
-            pPatch->aNewOpcode[0] = 0xE9;
-            *(RTRCUINTPTR *)&pPatch->aNewOpcode[1] = ((RTRCUINTPTR)pVM->hwaccm.s.pFreeGuestPatchMem) - ((RTRCUINTPTR)pCtx->eip + 5);
-
-            /* Overwrite the TPR instruction with a jump. */
-            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->eip, pPatch->aNewOpcode, 5);
-            AssertRC(rc);
-
-#ifdef LOG_ENABLED
-            rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                                    szOutput, sizeof(szOutput), NULL);
-            if (RT_SUCCESS(rc))
-                Log(("Jump: %s\n", szOutput));
-#endif
-            pVM->hwaccm.s.pFreeGuestPatchMem += off;
-            pPatch->cbNewOp = 5;
-
-            pPatch->Core.Key = pCtx->eip;
-            rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
-            AssertRC(rc);
-
-            pVM->hwaccm.s.cPatches++;
-            pVM->hwaccm.s.fTPRPatchingActive = true;
-            STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRPatchSuccess);
-            return VINF_SUCCESS;
-        }
-        else
-            Log(("Ran out of space in our patch buffer!\n"));
-    }
-
-    /* Save invalid patch, so we will not try again. */
-    uint32_t  idx = pVM->hwaccm.s.cPatches;
-
-#ifdef LOG_ENABLED
-    rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                            szOutput, sizeof(szOutput), NULL);
-    if (RT_SUCCESS(rc))
-        Log(("Failed to patch instr: %s\n", szOutput));
-#endif
-
-    pPatch = &pVM->hwaccm.s.aPatches[idx];
-    pPatch->Core.Key = pCtx->eip;
-    pPatch->enmType  = HWACCMTPRINSTR_INVALID;
-    rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
-    AssertRC(rc);
-    pVM->hwaccm.s.cPatches++;
-    STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRPatchFailure);
-    return VINF_SUCCESS;
-}
-
-/**
- * Attempt to patch TPR mmio instructions
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VM CPU to operate on.
- * @param   pCtx        CPU context
- */
-VMMR3DECL(int) HWACCMR3PatchTprInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
-{
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, (pVM->hwaccm.s.pGuestPatchMem) ? hwaccmR3PatchTprInstr : hwaccmR3ReplaceTprInstr, (void *)pVCpu->idCpu);
-    AssertRC(rc);
-    return rc;
-}
-
-/**
- * Force execution of the current IO code in the recompiler
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pCtx        Partial VM execution context
- */
-VMMR3DECL(int) HWACCMR3EmulateIoBlock(PVM pVM, PCPUMCTX pCtx)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    Assert(pVM->fHWACCMEnabled);
-    Log(("HWACCMR3EmulateIoBlock\n"));
-
-    /* This is primarily intended to speed up Grub, so we don't care about paged protected mode. */
-    if (HWACCMCanEmulateIoBlockEx(pCtx))
-    {
-        Log(("HWACCMR3EmulateIoBlock -> enabled\n"));
-        pVCpu->hwaccm.s.EmulateIoBlock.fEnabled         = true;
-        pVCpu->hwaccm.s.EmulateIoBlock.GCPtrFunctionEip = pCtx->rip;
-        pVCpu->hwaccm.s.EmulateIoBlock.cr0              = pCtx->cr0;
-        return VINF_EM_RESCHEDULE_REM;
-    }
-    return VINF_SUCCESS;
-}
-
-/**
- * Checks if we can currently use hardware accelerated raw mode.
- *
- * @returns boolean
- * @param   pVM         The VM to operate on.
- * @param   pCtx        Partial VM execution context
- */
-VMMR3DECL(bool) HWACCMR3CanExecuteGuest(PVM pVM, PCPUMCTX pCtx)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    Assert(pVM->fHWACCMEnabled);
-
-    /* If we're still executing the IO code, then return false. */
-    if (    RT_UNLIKELY(pVCpu->hwaccm.s.EmulateIoBlock.fEnabled)
-        &&  pCtx->rip <  pVCpu->hwaccm.s.EmulateIoBlock.GCPtrFunctionEip + 0x200
-        &&  pCtx->rip >  pVCpu->hwaccm.s.EmulateIoBlock.GCPtrFunctionEip - 0x200
-        &&  pCtx->cr0 == pVCpu->hwaccm.s.EmulateIoBlock.cr0)
-        return false;
-
-    pVCpu->hwaccm.s.EmulateIoBlock.fEnabled = false;
-
-    /* AMD-V supports real & protected mode with or without paging. */
-    if (pVM->hwaccm.s.svm.fEnabled)
-    {
-        pVCpu->hwaccm.s.fActive = true;
-        return true;
-    }
-
-    pVCpu->hwaccm.s.fActive = false;
-
-    /* Note! The context supplied by REM is partial. If we add more checks here, be sure to verify that REM provides this info! */
-    Assert((pVM->hwaccm.s.vmx.fUnrestrictedGuest && !pVM->hwaccm.s.vmx.pRealModeTSS) || (!pVM->hwaccm.s.vmx.fUnrestrictedGuest && pVM->hwaccm.s.vmx.pRealModeTSS));
-
-    bool fSupportsRealMode = pVM->hwaccm.s.vmx.fUnrestrictedGuest || PDMVMMDevHeapIsEnabled(pVM);
-    if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
-    {
-        /** The VMM device heap is a requirement for emulating real mode or protected mode without paging when the unrestricted guest execution feature is missing. */
-        if (fSupportsRealMode)
-        {
-            if (CPUMIsGuestInRealModeEx(pCtx))
-            {
-                /* VT-x will not allow high selector bases in v86 mode; fall back to the recompiler in that case.
-                 * The base must also be equal to (sel << 4).
-                 */
-                if (   (   pCtx->cs != (pCtx->csHid.u64Base >> 4)
-                        && pCtx->csHid.u64Base != 0xffff0000 /* we can deal with the BIOS code as it's also mapped into the lower region. */)
-                    || pCtx->ds != (pCtx->dsHid.u64Base >> 4)
-                    || pCtx->es != (pCtx->esHid.u64Base >> 4)
-                    || pCtx->fs != (pCtx->fsHid.u64Base >> 4)
-                    || pCtx->gs != (pCtx->gsHid.u64Base >> 4)
-                    || pCtx->ss != (pCtx->ssHid.u64Base >> 4))
-                {
-                    return false;
-                }
-            }
-            else
-            {
-                PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
-                /* Verify the requirements for executing code in protected mode. VT-x can't handle the CPU state right after a switch
-                 * from real to protected mode. (all sorts of RPL & DPL assumptions)
-                 */
-                if (    pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
-                    &&  enmGuestMode >= PGMMODE_PROTECTED)
-                {
-                    if (   (pCtx->cs & X86_SEL_RPL)
-                        || (pCtx->ds & X86_SEL_RPL)
-                        || (pCtx->es & X86_SEL_RPL)
-                        || (pCtx->fs & X86_SEL_RPL)
-                        || (pCtx->gs & X86_SEL_RPL)
-                        || (pCtx->ss & X86_SEL_RPL))
-                    {
-                        return false;
-                    }
-                }
-                /* VT-x also chokes on invalid tr or ldtr selectors (minix) */
-                if (    pCtx->gdtr.cbGdt
-                    &&  (   pCtx->tr > pCtx->gdtr.cbGdt
-                         || pCtx->ldtr > pCtx->gdtr.cbGdt))
-                {
-                        return false;
-                }
-            }
-        }
-        else
-        {
-            if (    !CPUMIsGuestInLongModeEx(pCtx)
-                &&  !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
-            {
-                /** @todo   This should (probably) be set on every excursion to the REM,
-                 *          however it's too risky right now. So, only apply it when we go
-                 *          back to REM for real mode execution. (The XP hack below doesn't
-                 *          work reliably without this.)
-                 *  Update: Implemented in EM.cpp, see #ifdef EM_NOTIFY_HWACCM.  */
-                pVM->aCpus[0].hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL_GUEST;
-
-                if (    !pVM->hwaccm.s.fNestedPaging        /* requires a fake PD for real *and* protected mode without paging - stored in the VMM device heap*/
-                    ||  CPUMIsGuestInRealModeEx(pCtx))      /* requires a fake TSS for real mode - stored in the VMM device heap */
-                    return false;
-
-                /* Too early for VT-x; Solaris guests will fail with a guru meditation otherwise; same for XP. */
-                if (pCtx->idtr.pIdt == 0 || pCtx->idtr.cbIdt == 0 || pCtx->tr == 0)
-                    return false;
-
-                /* The guest is about to complete the switch to protected mode. Wait a bit longer. */
-                /* Windows XP; switch to protected mode; all selectors are marked not present in the
-                 * hidden registers (possible recompiler bug; see load_seg_vm) */
-                if (pCtx->csHid.Attr.n.u1Present == 0)
-                    return false;
-                if (pCtx->ssHid.Attr.n.u1Present == 0)
-                    return false;
-
-                /* Windows XP: possible same as above, but new recompiler requires new heuristics?
-                   VT-x doesn't seem to like something about the guest state and this stuff avoids it. */
-                /** @todo This check is actually wrong, it doesn't take the direction of the
-                 *        stack segment into account. But, it does the job for now. */
-                if (pCtx->rsp >= pCtx->ssHid.u32Limit)
-                    return false;
-    #if 0
-                if (    pCtx->cs >= pCtx->gdtr.cbGdt
-                    ||  pCtx->ss >= pCtx->gdtr.cbGdt
-                    ||  pCtx->ds >= pCtx->gdtr.cbGdt
-                    ||  pCtx->es >= pCtx->gdtr.cbGdt
-                    ||  pCtx->fs >= pCtx->gdtr.cbGdt
-                    ||  pCtx->gs >= pCtx->gdtr.cbGdt)
-                    return false;
-    #endif
-            }
-        }
-    }
-
-    if (pVM->hwaccm.s.vmx.fEnabled)
-    {
-        uint32_t mask;
-
-        /* if bit N is set in cr0_fixed0, then it must be set in the guest's cr0. */
-        mask = (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0;
-        /* Note: We ignore the NE bit here on purpose; see vmmr0\hwaccmr0.cpp for details. */
-        mask &= ~X86_CR0_NE;
-
-        if (fSupportsRealMode)
-        {
-            /* Note: We ignore the PE & PG bits here on purpose; we emulate real and protected mode without paging. */
-            mask &= ~(X86_CR0_PG|X86_CR0_PE);
-        }
-        else
-        {
-            /* We support protected mode without paging using identity mapping. */
-            mask &= ~X86_CR0_PG;
-        }
-        if ((pCtx->cr0 & mask) != mask)
-            return false;
-
-        /* if bit N is cleared in cr0_fixed1, then it must be zero in the guest's cr0. */
-        mask = (uint32_t)~pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1;
-        if ((pCtx->cr0 & mask) != 0)
-            return false;
-
-        /* if bit N is set in cr4_fixed0, then it must be set in the guest's cr4. */
-        mask  = (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
-        mask &= ~X86_CR4_VMXE;
-        if ((pCtx->cr4 & mask) != mask)
-            return false;
-
-        /* if bit N is cleared in cr4_fixed1, then it must be zero in the guest's cr4. */
-        mask = (uint32_t)~pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1;
-        if ((pCtx->cr4 & mask) != 0)
-            return false;
-
-        pVCpu->hwaccm.s.fActive = true;
-        return true;
-    }
-
-    return false;
-}
-
-/**
- * Checks if we need to reschedule due to VMM device heap changes
- *
- * @returns boolean
- * @param   pVM         The VM to operate on.
- * @param   pCtx        VM execution context
- */
-VMMR3DECL(bool) HWACCMR3IsRescheduleRequired(PVM pVM, PCPUMCTX pCtx)
-{
-    /** The VMM device heap is a requirement for emulating real mode or protected mode without paging when the unrestricted guest execution feature is missing. (VT-x only) */
-    if (    pVM->hwaccm.s.vmx.fEnabled
-        &&  !pVM->hwaccm.s.vmx.fUnrestrictedGuest
-        &&  !CPUMIsGuestInPagedProtectedModeEx(pCtx)
-        &&  !PDMVMMDevHeapIsEnabled(pVM)
-        &&  (pVM->hwaccm.s.fNestedPaging || CPUMIsGuestInRealModeEx(pCtx)))
-        return true;
-
-    return false;
-}
-
-
-/**
- * Notification from EM about a rescheduling into hardware assisted execution
- * mode.
- *
- * @param   pVCpu       Pointer to the current virtual cpu structure.
- */
-VMMR3DECL(void) HWACCMR3NotifyScheduled(PVMCPU pVCpu)
-{
-    pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL_GUEST;
-}
-
-/**
- * Notification from EM about returning from instruction emulation (REM / EM).
- *
- * @param   pVCpu       Pointer to the current virtual cpu structure.
- */
-VMMR3DECL(void) HWACCMR3NotifyEmulated(PVMCPU pVCpu)
-{
-    pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL_GUEST;
-}
-
-/**
- * Checks if we are currently using hardware accelerated raw mode.
- *
- * @returns boolean
- * @param   pVCpu        The VMCPU to operate on.
- */
-VMMR3DECL(bool) HWACCMR3IsActive(PVMCPU pVCpu)
-{
-    return pVCpu->hwaccm.s.fActive;
-}
-
-/**
- * Checks if we are currently using nested paging.
- *
- * @returns boolean
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(bool) HWACCMR3IsNestedPagingActive(PVM pVM)
-{
-    return pVM->hwaccm.s.fNestedPaging;
-}
-
-/**
- * Checks if we are currently using VPID in VT-x mode.
- *
- * @returns boolean
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(bool) HWACCMR3IsVPIDActive(PVM pVM)
-{
-    return pVM->hwaccm.s.vmx.fVPID;
-}
-
-
-/**
- * Checks if internal events are pending. In that case we are not allowed to dispatch interrupts.
- *
- * @returns boolean
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(bool) HWACCMR3IsEventPending(PVMCPU pVCpu)
-{
-    return HWACCMIsEnabled(pVCpu->pVMR3) && pVCpu->hwaccm.s.Event.fPending;
-}
-
-/**
- * Checks if the VMX-preemption timer is being used.
- *
- * @returns true if it is, false if it isn't.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(bool) HWACCMR3IsVmxPreemptionTimerUsed(PVM pVM)
-{
-    return HWACCMIsEnabled(pVM)
-        && pVM->hwaccm.s.vmx.fEnabled
-        && pVM->hwaccm.s.vmx.fUsePreemptTimer;
-}
-
-/**
- * Restart an I/O instruction that was refused in ring-0
- *
- * @returns Strict VBox status code. Informational status codes other than the one documented
- *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
- * @retval  VINF_SUCCESS                Success.
- * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
- *                                      status code must be passed on to EM.
- * @retval  VERR_NOT_FOUND if no pending I/O instruction.
- *
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   pCtx        VCPU register context
- */
-VMMR3DECL(VBOXSTRICTRC) HWACCMR3RestartPendingIOInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
-{
-    HWACCMPENDINGIO enmType = pVCpu->hwaccm.s.PendingIO.enmType;
-
-    pVCpu->hwaccm.s.PendingIO.enmType = HWACCMPENDINGIO_INVALID;
-
-    if (    pVCpu->hwaccm.s.PendingIO.GCPtrRip != pCtx->rip
-        ||  enmType  == HWACCMPENDINGIO_INVALID)
-        return VERR_NOT_FOUND;
-
-    VBOXSTRICTRC rcStrict;
-    switch (enmType)
-    {
-    case HWACCMPENDINGIO_PORT_READ:
-    {
-        uint32_t uAndVal = pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal;
-        uint32_t u32Val  = 0;
-
-        rcStrict = IOMIOPortRead(pVM, pVCpu->hwaccm.s.PendingIO.s.Port.uPort,
-                                 &u32Val,
-                                 pVCpu->hwaccm.s.PendingIO.s.Port.cbSize);
-        if (IOM_SUCCESS(rcStrict))
-        {
-            /* Write back to the EAX register. */
-            pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
-            pCtx->rip = pVCpu->hwaccm.s.PendingIO.GCPtrRipNext;
-        }
-        break;
-    }
-
-    case HWACCMPENDINGIO_PORT_WRITE:
-        rcStrict = IOMIOPortWrite(pVM, pVCpu->hwaccm.s.PendingIO.s.Port.uPort,
-                                  pCtx->eax & pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal,
-                                  pVCpu->hwaccm.s.PendingIO.s.Port.cbSize);
-        if (IOM_SUCCESS(rcStrict))
-            pCtx->rip = pVCpu->hwaccm.s.PendingIO.GCPtrRipNext;
-        break;
-
-    default:
-        AssertFailed();
-        return VERR_INTERNAL_ERROR;
-    }
-
-    return rcStrict;
-}
-
-/**
- * Inject an NMI into a running VM (only VCPU 0!)
- *
- * @returns boolean
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int)  HWACCMR3InjectNMI(PVM pVM)
-{
-    VMCPU_FF_SET(&pVM->aCpus[0], VMCPU_FF_INTERRUPT_NMI);
-    return VINF_SUCCESS;
-}
-
-/**
- * Check fatal VT-x/AMD-V error and produce some meaningful
- * log release message.
- *
- * @param   pVM         The VM to operate on.
- * @param   iStatusCode VBox status code
- */
-VMMR3DECL(void) HWACCMR3CheckError(PVM pVM, int iStatusCode)
-{
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        switch(iStatusCode)
-        {
-        case VERR_VMX_INVALID_VMCS_FIELD:
-            break;
-
-        case VERR_VMX_INVALID_VMCS_PTR:
-            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Current pointer %RGp vs %RGp\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.u64VMCSPhys, pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys));
-            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Current VMCS version %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulVMCSRevision));
-            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Entered Cpu %d\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.idEnteredCpu));
-            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Current Cpu %d\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.idCurrentCpu));
-            break;
-
-        case VERR_VMX_UNABLE_TO_START_VM:
-            LogRel(("VERR_VMX_UNABLE_TO_START_VM: CPU%d instruction error %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulInstrError));
-            LogRel(("VERR_VMX_UNABLE_TO_START_VM: CPU%d exit reason       %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulExitReason));
-#if 0 /* @todo dump the current control fields to the release log */
-            if (pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulInstrError == VMX_ERROR_VMENTRY_INVALID_CONTROL_FIELDS)
-            {
-
-            }
-#endif
-            break;
-
-        case VERR_VMX_UNABLE_TO_RESUME_VM:
-            LogRel(("VERR_VMX_UNABLE_TO_RESUME_VM: CPU%d instruction error %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulInstrError));
-            LogRel(("VERR_VMX_UNABLE_TO_RESUME_VM: CPU%d exit reason       %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulExitReason));
-            break;
-
-        case VERR_VMX_INVALID_VMXON_PTR:
-            break;
-        }
-    }
-}
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) hwaccmR3Save(PVM pVM, PSSMHANDLE pSSM)
-{
-    int rc;
-
-    Log(("hwaccmR3Save:\n"));
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        /*
-         * Save the basic bits - fortunately all the other things can be resynced on load.
-         */
-        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.Event.fPending);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.Event.errCode);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutU64(pSSM, pVM->aCpus[i].hwaccm.s.Event.intInfo);
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.vmx.enmLastSeenGuestMode);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.vmx.enmCurrGuestMode);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.vmx.enmPrevGuestMode);
-        AssertRCReturn(rc, rc);
-    }
-#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
-    rc = SSMR3PutGCPtr(pSSM, pVM->hwaccm.s.pGuestPatchMem);
-    AssertRCReturn(rc, rc);
-    rc = SSMR3PutGCPtr(pSSM, pVM->hwaccm.s.pFreeGuestPatchMem);
-    AssertRCReturn(rc, rc);
-    rc = SSMR3PutU32(pSSM, pVM->hwaccm.s.cbGuestPatchMem);
-    AssertRCReturn(rc, rc);
-
-    /* Store all the guest patch records too. */
-    rc = SSMR3PutU32(pSSM, pVM->hwaccm.s.cPatches);
-    AssertRCReturn(rc, rc);
-
-    for (unsigned i = 0; i < pVM->hwaccm.s.cPatches; i++)
-    {
-        PHWACCMTPRPATCH pPatch = &pVM->hwaccm.s.aPatches[i];
-
-        rc = SSMR3PutU32(pSSM, pPatch->Core.Key);
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutMem(pSSM, pPatch->aOpcode, sizeof(pPatch->aOpcode));
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutU32(pSSM, pPatch->cbOp);
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutMem(pSSM, pPatch->aNewOpcode, sizeof(pPatch->aNewOpcode));
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutU32(pSSM, pPatch->cbNewOp);
-        AssertRCReturn(rc, rc);
-
-        AssertCompileSize(HWACCMTPRINSTR, 4);
-        rc = SSMR3PutU32(pSSM, (uint32_t)pPatch->enmType);
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutU32(pSSM, pPatch->uSrcOperand);
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutU32(pSSM, pPatch->uDstOperand);
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutU32(pSSM, pPatch->pJumpTarget);
-        AssertRCReturn(rc, rc);
-
-        rc = SSMR3PutU32(pSSM, pPatch->cFaults);
-        AssertRCReturn(rc, rc);
-    }
-#endif
-    return VINF_SUCCESS;
-}
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) hwaccmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    int rc;
-
-    Log(("hwaccmR3Load:\n"));
-    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
-
-    /*
-     * Validate version.
-     */
-    if (   uVersion != HWACCM_SSM_VERSION
-        && uVersion != HWACCM_SSM_VERSION_NO_PATCHING
-        && uVersion != HWACCM_SSM_VERSION_2_0_X)
-    {
-        AssertMsgFailed(("hwaccmR3Load: Invalid version uVersion=%d!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        rc = SSMR3GetU32(pSSM, &pVM->aCpus[i].hwaccm.s.Event.fPending);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3GetU32(pSSM, &pVM->aCpus[i].hwaccm.s.Event.errCode);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3GetU64(pSSM, &pVM->aCpus[i].hwaccm.s.Event.intInfo);
-        AssertRCReturn(rc, rc);
-
-        if (uVersion >= HWACCM_SSM_VERSION_NO_PATCHING)
-        {
-            uint32_t val;
-
-            rc = SSMR3GetU32(pSSM, &val);
-            AssertRCReturn(rc, rc);
-            pVM->aCpus[i].hwaccm.s.vmx.enmLastSeenGuestMode = (PGMMODE)val;
-
-            rc = SSMR3GetU32(pSSM, &val);
-            AssertRCReturn(rc, rc);
-            pVM->aCpus[i].hwaccm.s.vmx.enmCurrGuestMode = (PGMMODE)val;
-
-            rc = SSMR3GetU32(pSSM, &val);
-            AssertRCReturn(rc, rc);
-            pVM->aCpus[i].hwaccm.s.vmx.enmPrevGuestMode = (PGMMODE)val;
-        }
-    }
-#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
-    if (uVersion > HWACCM_SSM_VERSION_NO_PATCHING)
-    {
-        rc = SSMR3GetGCPtr(pSSM, &pVM->hwaccm.s.pGuestPatchMem);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3GetGCPtr(pSSM, &pVM->hwaccm.s.pFreeGuestPatchMem);
-        AssertRCReturn(rc, rc);
-        rc = SSMR3GetU32(pSSM, &pVM->hwaccm.s.cbGuestPatchMem);
-        AssertRCReturn(rc, rc);
-
-        /* Fetch all TPR patch records. */
-        rc = SSMR3GetU32(pSSM, &pVM->hwaccm.s.cPatches);
-        AssertRCReturn(rc, rc);
-
-        for (unsigned i = 0; i < pVM->hwaccm.s.cPatches; i++)
-        {
-            PHWACCMTPRPATCH pPatch = &pVM->hwaccm.s.aPatches[i];
-
-            rc = SSMR3GetU32(pSSM, &pPatch->Core.Key);
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetMem(pSSM, pPatch->aOpcode, sizeof(pPatch->aOpcode));
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetU32(pSSM, &pPatch->cbOp);
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetMem(pSSM, pPatch->aNewOpcode, sizeof(pPatch->aNewOpcode));
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetU32(pSSM, &pPatch->cbNewOp);
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetU32(pSSM, (uint32_t *)&pPatch->enmType);
-            AssertRCReturn(rc, rc);
-
-            if (pPatch->enmType == HWACCMTPRINSTR_JUMP_REPLACEMENT)
-                pVM->hwaccm.s.fTPRPatchingActive = true;
-
-            Assert(pPatch->enmType == HWACCMTPRINSTR_JUMP_REPLACEMENT || pVM->hwaccm.s.fTPRPatchingActive == false);
-
-            rc = SSMR3GetU32(pSSM, &pPatch->uSrcOperand);
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetU32(pSSM, &pPatch->uDstOperand);
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetU32(pSSM, &pPatch->cFaults);
-            AssertRCReturn(rc, rc);
-
-            rc = SSMR3GetU32(pSSM, &pPatch->pJumpTarget);
-            AssertRCReturn(rc, rc);
-
-            Log(("hwaccmR3Load: patch %d\n", i));
-            Log(("Key       = %x\n", pPatch->Core.Key));
-            Log(("cbOp      = %d\n", pPatch->cbOp));
-            Log(("cbNewOp   = %d\n", pPatch->cbNewOp));
-            Log(("type      = %d\n", pPatch->enmType));
-            Log(("srcop     = %d\n", pPatch->uSrcOperand));
-            Log(("dstop     = %d\n", pPatch->uDstOperand));
-            Log(("cFaults   = %d\n", pPatch->cFaults));
-            Log(("target    = %x\n", pPatch->pJumpTarget));
-            rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
-            AssertRC(rc);
-        }
-    }
-#endif
-
-    /* Recheck all VCPUs if we can go straight into hwaccm execution mode. */
-    if (HWACCMIsEnabled(pVM))
-    {
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            PVMCPU pVCpu = &pVM->aCpus[i];
-
-            HWACCMR3CanExecuteGuest(pVM, CPUMQueryGuestCtxPtr(pVCpu));
-        }
-    }
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/HWACCMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/HWACCMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,936 +1,0 @@
-/* $Id$ */
-/** @file
- * HWACCM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___HWACCMInternal_h
-#define ___HWACCMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/em.h>
-#include <VBox/stam.h>
-#include <VBox/dis.h>
-#include <VBox/hwaccm.h>
-#include <VBox/hwacc_vmx.h>
-#include <VBox/pgm.h>
-#include <VBox/cpum.h>
-#include <iprt/memobj.h>
-#include <iprt/cpuset.h>
-#include <iprt/mp.h>
-#include <iprt/avl.h>
-
-#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) || defined (VBOX_WITH_64_BITS_GUESTS)
-/* Enable 64 bits guest support. */
-# define VBOX_ENABLE_64_BITS_GUESTS
-#endif
-
-#define VMX_USE_CACHED_VMCS_ACCESSES
-#define HWACCM_VMX_EMULATE_REALMODE
-#define HWACCM_VTX_WITH_EPT
-#define HWACCM_VTX_WITH_VPID
-
-
-#if 0
-/* Seeing somewhat random behaviour on my Nehalem system with auto-save of guest MSRs;
- * for some strange reason the CPU doesn't save the MSRs during the VM-exit.
- * Clearly visible with a dual VCPU configured OpenSolaris 200906 live cd VM.
- *
- * Note: change the assembly files when enabling this! (remove the manual auto load/save)
- */
-#define VBOX_WITH_AUTO_MSR_LOAD_RESTORE
-#endif
-
-RT_C_DECLS_BEGIN
-
-
-/** @defgroup grp_hwaccm_int       Internal
- * @ingroup grp_hwaccm
- * @internal
- * @{
- */
-
-
-/** Maximum number of exit reason statistics counters. */
-#define MAX_EXITREASON_STAT        0x100
-#define MASK_EXITREASON_STAT       0xff
-#define MASK_INJECT_IRQ_STAT       0xff
-
-/** @name Changed flags
- * These flags are used to keep track of which important registers that
- * have been changed since last they were reset.
- * @{
- */
-#define HWACCM_CHANGED_GUEST_FPU                RT_BIT(0)
-#define HWACCM_CHANGED_GUEST_CR0                RT_BIT(1)
-#define HWACCM_CHANGED_GUEST_CR3                RT_BIT(2)
-#define HWACCM_CHANGED_GUEST_CR4                RT_BIT(3)
-#define HWACCM_CHANGED_GUEST_GDTR               RT_BIT(4)
-#define HWACCM_CHANGED_GUEST_IDTR               RT_BIT(5)
-#define HWACCM_CHANGED_GUEST_LDTR               RT_BIT(6)
-#define HWACCM_CHANGED_GUEST_TR                 RT_BIT(7)
-#define HWACCM_CHANGED_GUEST_MSR                RT_BIT(8)
-#define HWACCM_CHANGED_GUEST_SEGMENT_REGS       RT_BIT(9)
-#define HWACCM_CHANGED_GUEST_DEBUG              RT_BIT(10)
-#define HWACCM_CHANGED_HOST_CONTEXT             RT_BIT(11)
-
-#define HWACCM_CHANGED_ALL                  (   HWACCM_CHANGED_GUEST_SEGMENT_REGS \
-                                            |   HWACCM_CHANGED_GUEST_CR0          \
-                                            |   HWACCM_CHANGED_GUEST_CR3          \
-                                            |   HWACCM_CHANGED_GUEST_CR4          \
-                                            |   HWACCM_CHANGED_GUEST_GDTR         \
-                                            |   HWACCM_CHANGED_GUEST_IDTR         \
-                                            |   HWACCM_CHANGED_GUEST_LDTR         \
-                                            |   HWACCM_CHANGED_GUEST_TR           \
-                                            |   HWACCM_CHANGED_GUEST_MSR          \
-                                            |   HWACCM_CHANGED_GUEST_FPU          \
-                                            |   HWACCM_CHANGED_GUEST_DEBUG        \
-                                            |   HWACCM_CHANGED_HOST_CONTEXT)
-
-#define HWACCM_CHANGED_ALL_GUEST            (   HWACCM_CHANGED_GUEST_SEGMENT_REGS \
-                                            |   HWACCM_CHANGED_GUEST_CR0          \
-                                            |   HWACCM_CHANGED_GUEST_CR3          \
-                                            |   HWACCM_CHANGED_GUEST_CR4          \
-                                            |   HWACCM_CHANGED_GUEST_GDTR         \
-                                            |   HWACCM_CHANGED_GUEST_IDTR         \
-                                            |   HWACCM_CHANGED_GUEST_LDTR         \
-                                            |   HWACCM_CHANGED_GUEST_TR           \
-                                            |   HWACCM_CHANGED_GUEST_MSR          \
-                                            |   HWACCM_CHANGED_GUEST_DEBUG        \
-                                            |   HWACCM_CHANGED_GUEST_FPU)
-
-/** @} */
-
-/** @name Intercepted traps
- *  Traps that need to be intercepted so we can correctly dispatch them to the guest if required.
- *  Currently #NM and #PF only
- */
-#ifdef VBOX_STRICT
-#define HWACCM_VMX_TRAP_MASK                RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_PF) | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_MF)
-#define HWACCM_SVM_TRAP_MASK                HWACCM_VMX_TRAP_MASK
-#else
-#define HWACCM_VMX_TRAP_MASK                RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_PF)
-#define HWACCM_SVM_TRAP_MASK                RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_PF)
-#endif
-/* All exceptions have to be intercept in emulated real-mode (minus NM & PF as they are always intercepted. */
-#define HWACCM_VMX_TRAP_MASK_REALMODE       RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_DF) | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) | RT_BIT(X86_XCPT_XF)
-/** @} */
-
-
-/** Maximum number of page flushes we are willing to remember before considering a full TLB flush. */
-#define HWACCM_MAX_TLB_SHOOTDOWN_PAGES      8
-
-/** Size for the EPT identity page table (1024 4 MB pages to cover the entire address space). */
-#define HWACCM_EPT_IDENTITY_PG_TABLE_SIZE   PAGE_SIZE
-/** Size of the TSS structure + 2 pages for the IO bitmap + end byte. */
-#define HWACCM_VTX_TSS_SIZE                 (sizeof(VBOXTSS) + 2*PAGE_SIZE + 1)
-/** Total guest mapped memory needed. */
-#define HWACCM_VTX_TOTAL_DEVHEAP_MEM        (HWACCM_EPT_IDENTITY_PG_TABLE_SIZE + HWACCM_VTX_TSS_SIZE)
-
-/* Enable for TPR guest patching. */
-#define VBOX_HWACCM_WITH_GUEST_PATCHING
-
-/** HWACCM SSM version
- */
-#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
-#define HWACCM_SSM_VERSION                  5
-#define HWACCM_SSM_VERSION_NO_PATCHING      4
-#else
-#define HWACCM_SSM_VERSION                  4
-#define HWACCM_SSM_VERSION_NO_PATCHING      4
-#endif
-#define HWACCM_SSM_VERSION_2_0_X            3
-
-/* Per-cpu information. (host) */
-typedef struct
-{
-    RTCPUID             idCpu;
-
-    RTR0MEMOBJ          pMemObj;
-    /* Current ASID (AMD-V)/VPID (Intel) */
-    uint32_t            uCurrentASID;
-    /* TLB flush count */
-    uint32_t            cTLBFlushes;
-
-    /* Set the first time a cpu is used to make sure we start with a clean TLB. */
-    bool                fFlushTLB;
-
-    /** Configured for VT-x or AMD-V. */
-    bool                fConfigured;
-
-    /** Set if the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE hack is active. */
-    bool                fIgnoreAMDVInUseError;
-
-    /** In use by our code. (for power suspend) */
-    volatile bool       fInUse;
-} HWACCM_CPUINFO;
-typedef HWACCM_CPUINFO *PHWACCM_CPUINFO;
-
-typedef enum
-{
-    HWACCMPENDINGIO_INVALID = 0,
-    HWACCMPENDINGIO_PORT_READ,
-    HWACCMPENDINGIO_PORT_WRITE,
-    HWACCMPENDINGIO_STRING_READ,
-    HWACCMPENDINGIO_STRING_WRITE,
-    /** The usual 32-bit paranoia. */
-    HWACCMPENDINGIO_32BIT_HACK   = 0x7fffffff
-} HWACCMPENDINGIO;
-
-
-typedef enum
-{
-    HWACCMTPRINSTR_INVALID,
-    HWACCMTPRINSTR_READ,
-    HWACCMTPRINSTR_READ_SHR4,
-    HWACCMTPRINSTR_WRITE_REG,
-    HWACCMTPRINSTR_WRITE_IMM,
-    HWACCMTPRINSTR_JUMP_REPLACEMENT,
-    /** The usual 32-bit paranoia. */
-    HWACCMTPRINSTR_32BIT_HACK   = 0x7fffffff
-} HWACCMTPRINSTR;
-
-typedef struct
-{
-    /** The key is the address of patched instruction. (32 bits GC ptr) */
-    AVLOU32NODECORE         Core;
-    /** Original opcode. */
-    uint8_t                 aOpcode[16];
-    /** Instruction size. */
-    uint32_t                cbOp;
-    /** Replacement opcode. */
-    uint8_t                 aNewOpcode[16];
-    /** Replacement instruction size. */
-    uint32_t                cbNewOp;
-    /** Instruction type. */
-    HWACCMTPRINSTR          enmType;
-    /** Source operand. */
-    uint32_t                uSrcOperand;
-    /** Destination operand. */
-    uint32_t                uDstOperand;
-    /** Number of times the instruction caused a fault. */
-    uint32_t                cFaults;
-    /** Patch address of the jump replacement. */
-    RTGCPTR32               pJumpTarget;
-} HWACCMTPRPATCH;
-/** Pointer to HWACCMTPRPATCH. */
-typedef HWACCMTPRPATCH *PHWACCMTPRPATCH;
-
-/**
- * Switcher function, HC to RC.
- *
- * @param   pVM             The VM handle.
- * @param   uOffsetVMCPU    VMCPU offset from pVM
- * @returns Return code indicating the action to take.
- */
-typedef DECLCALLBACK (int) FNHWACCMSWITCHERHC(PVM pVM, uint32_t uOffsetVMCPU);
-/** Pointer to switcher function. */
-typedef FNHWACCMSWITCHERHC *PFNHWACCMSWITCHERHC;
-
-/**
- * HWACCM VM Instance data.
- * Changes to this must checked against the padding of the cfgm union in VM!
- */
-typedef struct HWACCM
-{
-    /** Set when we've initialized VMX or SVM. */
-    bool                        fInitialized;
-
-    /** Set when hardware acceleration is allowed. */
-    bool                        fAllowed;
-
-    /** Set if nested paging is enabled. */
-    bool                        fNestedPaging;
-
-    /** Set if nested paging is allowed. */
-    bool                        fAllowNestedPaging;
-
-    /** Set if large pages are enabled (requires nested paging). */
-    bool                        fLargePages;
-
-    /** Set if we can support 64-bit guests or not. */
-    bool                        fAllow64BitGuests;
-
-    /** Set if an IO-APIC is configured for this VM. */
-    bool                        fHasIoApic;
-
-    /** Set when TPR patching is allowed. */
-    bool                        fTRPPatchingAllowed;
-
-    /** Set when we initialize VT-x or AMD-V once for all CPUs. */
-    bool                        fGlobalInit;
-
-    /** Set when TPR patching is active. */
-    bool                        fTPRPatchingActive;
-    bool                        u8Alignment[6];
-
-    /** And mask for copying register contents. */
-    uint64_t                    u64RegisterMask;
-
-    /** Maximum ASID allowed. */
-    RTUINT                      uMaxASID;
-
-    /** The maximum number of resumes loops allowed in ring-0 (safety precaution).
-     * This number is set much higher when RTThreadPreemptIsPending is reliable. */
-    uint32_t                    cMaxResumeLoops;
-
-    /** Guest allocated memory for patching purposes. */
-    RTGCPTR                     pGuestPatchMem;
-    /** Current free pointer inside the patch block. */
-    RTGCPTR                     pFreeGuestPatchMem;
-    /** Size of the guest patch memory block. */
-    uint32_t                    cbGuestPatchMem;
-    uint32_t                    uPadding1;
-
-#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    /** 32 to 64 bits switcher entrypoint. */
-    R0PTRTYPE(PFNHWACCMSWITCHERHC) pfnHost32ToGuest64R0;
-
-    /* AMD-V 64 bits vmrun handler */
-    RTRCPTR                     pfnSVMGCVMRun64;
-
-    /* VT-x 64 bits vmlaunch handler */
-    RTRCPTR                     pfnVMXGCStartVM64;
-
-    /* RC handler to setup the 64 bits FPU state. */
-    RTRCPTR                     pfnSaveGuestFPU64;
-
-    /* RC handler to setup the 64 bits debug state. */
-    RTRCPTR                     pfnSaveGuestDebug64;
-
-    /* Test handler */
-    RTRCPTR                     pfnTest64;
-
-    RTRCPTR                     uAlignment[2];
-/*#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    uint32_t                    u32Alignment[1]; */
-#endif
-
-    struct
-    {
-        /** Set by the ring-0 side of HWACCM to indicate VMX is supported by the
-         *  CPU. */
-        bool                        fSupported;
-
-        /** Set when we've enabled VMX. */
-        bool                        fEnabled;
-
-        /** Set if VPID is supported. */
-        bool                        fVPID;
-
-        /** Set if VT-x VPID is allowed. */
-        bool                        fAllowVPID;
-
-        /** Set if unrestricted guest execution is allowed (real and protected mode without paging). */
-        bool                        fUnrestrictedGuest;
-
-        /** Whether we're using the preemption timer or not. */
-        bool                        fUsePreemptTimer;
-        /** The shift mask employed by the VMX-Preemption timer. */
-        uint8_t                     cPreemptTimerShift;
-
-        bool                        uAlignment[1];
-
-        /** Virtual address of the TSS page used for real mode emulation. */
-        R3PTRTYPE(PVBOXTSS)         pRealModeTSS;
-
-        /** Virtual address of the identity page table used for real mode and protected mode without paging emulation in EPT mode. */
-        R3PTRTYPE(PX86PD)           pNonPagingModeEPTPageTable;
-
-        /** R0 memory object for the APIC physical page (serves for filtering accesses). */
-        RTR0MEMOBJ                  pMemObjAPIC;
-        /** Physical address of the APIC physical page (serves for filtering accesses). */
-        RTHCPHYS                    pAPICPhys;
-        /** Virtual address of the APIC physical page (serves for filtering accesses). */
-        R0PTRTYPE(uint8_t *)        pAPIC;
-
-        /** R0 memory object for the MSR entry load page (guest MSRs). */
-        RTR0MEMOBJ                  pMemObjMSREntryLoad;
-        /** Physical address of the MSR entry load page (guest MSRs). */
-        RTHCPHYS                    pMSREntryLoadPhys;
-        /** Virtual address of the MSR entry load page (guest MSRs). */
-        R0PTRTYPE(uint8_t *)        pMSREntryLoad;
-
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-        RTR0MEMOBJ                  pMemObjScratch;
-        RTHCPHYS                    pScratchPhys;
-        R0PTRTYPE(uint8_t *)        pScratch;
-#endif
-        /** R0 memory object for the MSR exit store page (guest MSRs). */
-        RTR0MEMOBJ                  pMemObjMSRExitStore;
-        /** Physical address of the MSR exit store page (guest MSRs). */
-        RTHCPHYS                    pMSRExitStorePhys;
-        /** Virtual address of the MSR exit store page (guest MSRs). */
-        R0PTRTYPE(uint8_t *)        pMSRExitStore;
-
-        /** R0 memory object for the MSR exit load page (host MSRs). */
-        RTR0MEMOBJ                  pMemObjMSRExitLoad;
-        /** Physical address of the MSR exit load page (host MSRs). */
-        RTHCPHYS                    pMSRExitLoadPhys;
-        /** Virtual address of the MSR exit load page (host MSRs). */
-        R0PTRTYPE(uint8_t *)        pMSRExitLoad;
-
-        /** Ring 0 handlers for VT-x. */
-        DECLR0CALLBACKMEMBER(void, pfnSetupTaggedTLB, (PVM pVM, PVMCPU pVCpu));
-
-#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
-        uint32_t                    u32Alignment;
-#endif
-        /** Host CR4 value (set by ring-0 VMX init) */
-        uint64_t                    hostCR4;
-
-        /** Host EFER value (set by ring-0 VMX init) */
-        uint64_t                    hostEFER;
-
-        /** VMX MSR values */
-        struct
-        {
-            uint64_t                feature_ctrl;
-            uint64_t                vmx_basic_info;
-            VMX_CAPABILITY          vmx_pin_ctls;
-            VMX_CAPABILITY          vmx_proc_ctls;
-            VMX_CAPABILITY          vmx_proc_ctls2;
-            VMX_CAPABILITY          vmx_exit;
-            VMX_CAPABILITY          vmx_entry;
-            uint64_t                vmx_misc;
-            uint64_t                vmx_cr0_fixed0;
-            uint64_t                vmx_cr0_fixed1;
-            uint64_t                vmx_cr4_fixed0;
-            uint64_t                vmx_cr4_fixed1;
-            uint64_t                vmx_vmcs_enum;
-            uint64_t                vmx_eptcaps;
-        } msr;
-
-        /** Flush types for invept & invvpid; they depend on capabilities. */
-        VMX_FLUSH                   enmFlushPage;
-        VMX_FLUSH                   enmFlushContext;
-    } vmx;
-
-    struct
-    {
-        /** Set by the ring-0 side of HWACCM to indicate SVM is supported by the
-         *  CPU. */
-        bool                        fSupported;
-        /** Set when we've enabled SVM. */
-        bool                        fEnabled;
-        /** Set if erratum 170 affects the AMD cpu. */
-        bool                        fAlwaysFlushTLB;
-        /** Set when the hack to ignore VERR_SVM_IN_USE is active. */
-        bool                        fIgnoreInUseError;
-
-        /** R0 memory object for the IO bitmap (12kb). */
-        RTR0MEMOBJ                  pMemObjIOBitmap;
-        /** Physical address of the IO bitmap (12kb). */
-        RTHCPHYS                    pIOBitmapPhys;
-        /** Virtual address of the IO bitmap. */
-        R0PTRTYPE(void *)           pIOBitmap;
-
-        /* HWCR msr (for diagnostics) */
-        uint64_t                    msrHWCR;
-
-        /** SVM revision. */
-        uint32_t                    u32Rev;
-
-        /** SVM feature bits from cpuid 0x8000000a */
-        uint32_t                    u32Features;
-    } svm;
-
-    /**
-     * AVL tree with all patches (active or disabled) sorted by guest instruction address
-     */
-    AVLOU32TREE                     PatchTree;
-    uint32_t                        cPatches;
-    HWACCMTPRPATCH                  aPatches[64];
-
-    struct
-    {
-        uint32_t                    u32AMDFeatureECX;
-        uint32_t                    u32AMDFeatureEDX;
-    } cpuid;
-
-    /** Saved error from detection */
-    int32_t                 lLastError;
-
-    /** HWACCMR0Init was run */
-    bool                    fHWACCMR0Init;
-    bool                    u8Alignment1[7];
-
-    STAMCOUNTER             StatTPRPatchSuccess;
-    STAMCOUNTER             StatTPRPatchFailure;
-    STAMCOUNTER             StatTPRReplaceSuccess;
-    STAMCOUNTER             StatTPRReplaceFailure;
-} HWACCM;
-/** Pointer to HWACCM VM instance data. */
-typedef HWACCM *PHWACCM;
-
-/* Maximum number of cached entries. */
-#define VMCSCACHE_MAX_ENTRY                             128
-
-/* Structure for storing read and write VMCS actions. */
-typedef struct VMCSCACHE
-{
-#ifdef VBOX_WITH_CRASHDUMP_MAGIC
-    /* Magic marker for searching in crash dumps. */
-    uint8_t         aMagic[16];
-    uint64_t        uMagic;
-    uint64_t        u64TimeEntry;
-    uint64_t        u64TimeSwitch;
-    uint64_t        cResume;
-    uint64_t        interPD;
-    uint64_t        pSwitcher;
-    uint32_t        uPos;
-    uint32_t        idCpu;
-#endif
-    /* CR2 is saved here for EPT syncing. */
-    uint64_t        cr2;
-    struct
-    {
-        uint32_t    cValidEntries;
-        uint32_t    uAlignment;
-        uint32_t    aField[VMCSCACHE_MAX_ENTRY];
-        uint64_t    aFieldVal[VMCSCACHE_MAX_ENTRY];
-    } Write;
-    struct
-    {
-        uint32_t    cValidEntries;
-        uint32_t    uAlignment;
-        uint32_t    aField[VMCSCACHE_MAX_ENTRY];
-        uint64_t    aFieldVal[VMCSCACHE_MAX_ENTRY];
-    } Read;
-#ifdef DEBUG
-    struct
-    {
-        RTHCPHYS    pPageCpuPhys;
-        RTHCPHYS    pVMCSPhys;
-        RTGCPTR     pCache;
-        RTGCPTR     pCtx;
-    } TestIn;
-    struct
-    {
-        RTHCPHYS    pVMCSPhys;
-        RTGCPTR     pCache;
-        RTGCPTR     pCtx;
-        uint64_t    eflags;
-        uint64_t    cr8;
-    } TestOut;
-    struct
-    {
-        uint64_t    param1;
-        uint64_t    param2;
-        uint64_t    param3;
-        uint64_t    param4;
-    } ScratchPad;
-#endif
-} VMCSCACHE;
-/** Pointer to VMCSCACHE. */
-typedef VMCSCACHE *PVMCSCACHE;
-
-/** VMX StartVM function. */
-typedef DECLCALLBACK(int) FNHWACCMVMXSTARTVM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu);
-/** Pointer to a VMX StartVM function. */
-typedef R0PTRTYPE(FNHWACCMVMXSTARTVM *) PFNHWACCMVMXSTARTVM;
-
-/** SVM VMRun function. */
-typedef DECLCALLBACK(int) FNHWACCMSVMVMRUN(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu);
-/** Pointer to a SVM VMRun function. */
-typedef R0PTRTYPE(FNHWACCMSVMVMRUN *) PFNHWACCMSVMVMRUN;
-
-/**
- * HWACCM VMCPU Instance data.
- */
-typedef struct HWACCMCPU
-{
-    /** Old style FPU reporting trap mask override performed (optimization) */
-    bool                        fFPUOldStyleOverride;
-
-    /** Set if we don't have to flush the TLB on VM entry. */
-    bool                        fResumeVM;
-
-    /** Set if we need to flush the TLB during the world switch. */
-    bool                        fForceTLBFlush;
-
-    /** Set when we're using VT-x or AMD-V at that moment. */
-    bool                        fActive;
-
-    /** Set when the TLB has been checked until we return from the world switch. */
-    volatile uint8_t            fCheckedTLBFlush;
-    uint8_t                     bAlignment[3];
-
-    /** HWACCM_CHANGED_* flags. */
-    RTUINT                      fContextUseFlags;
-
-    /** Id of the last cpu we were executing code on (NIL_RTCPUID for the first time) */
-    RTCPUID                     idLastCpu;
-
-    /** TLB flush count */
-    RTUINT                      cTLBFlushes;
-
-    /** Current ASID in use by the VM */
-    RTUINT                      uCurrentASID;
-
-    /** World switch exit counter. */
-    volatile uint32_t           cWorldSwitchExit;
-    uint32_t                    u32Alignment;
-
-    struct
-    {
-        /** Physical address of the VM control structure (VMCS). */
-        RTHCPHYS                    pVMCSPhys;
-        /** R0 memory object for the VM control structure (VMCS). */
-        RTR0MEMOBJ                  pMemObjVMCS;
-        /** Virtual address of the VM control structure (VMCS). */
-        R0PTRTYPE(void *)           pVMCS;
-
-        /** Ring 0 handlers for VT-x. */
-        PFNHWACCMVMXSTARTVM         pfnStartVM;
-
-#if HC_ARCH_BITS == 32
-        uint32_t                    u32Alignment;
-#endif
-
-        /** Current VMX_VMCS_CTRL_PROC_EXEC_CONTROLS. */
-        uint64_t                    proc_ctls;
-
-        /** Current VMX_VMCS_CTRL_PROC_EXEC2_CONTROLS. */
-        uint64_t                    proc_ctls2;
-
-        /** Physical address of the virtual APIC page for TPR caching. */
-        RTHCPHYS                    pVAPICPhys;
-        /** R0 memory object for the virtual APIC page for TPR caching. */
-        RTR0MEMOBJ                  pMemObjVAPIC;
-        /** Virtual address of the virtual APIC page for TPR caching. */
-        R0PTRTYPE(uint8_t *)        pVAPIC;
-
-        /** Current CR0 mask. */
-        uint64_t                    cr0_mask;
-        /** Current CR4 mask. */
-        uint64_t                    cr4_mask;
-
-        /** Current EPTP. */
-        RTHCPHYS                    GCPhysEPTP;
-
-        /** Physical address of the MSR bitmap (1 page). */
-        RTHCPHYS                    pMSRBitmapPhys;
-        /** R0 memory object for the MSR bitmap (1 page). */
-        RTR0MEMOBJ                  pMemObjMSRBitmap;
-        /** Virtual address of the MSR bitmap (1 page). */
-        R0PTRTYPE(uint8_t *)        pMSRBitmap;
-
-#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
-        /** Physical address of the guest MSR load area (1 page). */
-        RTHCPHYS                    pGuestMSRPhys;
-        /** R0 memory object for the guest MSR load area (1 page). */
-        RTR0MEMOBJ                  pMemObjGuestMSR;
-        /** Virtual address of the guest MSR load area (1 page). */
-        R0PTRTYPE(uint8_t *)        pGuestMSR;
-
-        /** Physical address of the MSR load area (1 page). */
-        RTHCPHYS                    pHostMSRPhys;
-        /** R0 memory object for the MSR load area (1 page). */
-        RTR0MEMOBJ                  pMemObjHostMSR;
-        /** Virtual address of the MSR load area (1 page). */
-        R0PTRTYPE(uint8_t *)        pHostMSR;
-#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
-
-        /* Number of automatically loaded/restored MSRs. */
-        uint32_t                    cCachedMSRs;
-        uint32_t                    uAlignement;
-
-        /* Last use TSC offset value. (cached) */
-        uint64_t                    u64TSCOffset;
-
-        /** VMCS cache. */
-        VMCSCACHE                   VMCSCache;
-
-        /** Real-mode emulation state. */
-        struct
-        {
-            X86EFLAGS                   eflags;
-            uint32_t                    fValid;
-        } RealMode;
-
-        struct
-        {
-            uint64_t                u64VMCSPhys;
-            uint32_t                ulVMCSRevision;
-            uint32_t                ulInstrError;
-            uint32_t                ulExitReason;
-            RTCPUID                 idEnteredCpu;
-            RTCPUID                 idCurrentCpu;
-            uint32_t                padding;
-        } lasterror;
-
-        /** The last seen guest paging mode (by VT-x). */
-        PGMMODE                     enmLastSeenGuestMode;
-        /** Current guest paging mode (as seen by HWACCMR3PagingModeChanged). */
-        PGMMODE                     enmCurrGuestMode;
-        /** Previous guest paging mode (as seen by HWACCMR3PagingModeChanged). */
-        PGMMODE                     enmPrevGuestMode;
-    } vmx;
-
-    struct
-    {
-        /** R0 memory object for the host VM control block (VMCB). */
-        RTR0MEMOBJ                  pMemObjVMCBHost;
-        /** Physical address of the host VM control block (VMCB). */
-        RTHCPHYS                    pVMCBHostPhys;
-        /** Virtual address of the host VM control block (VMCB). */
-        R0PTRTYPE(void *)           pVMCBHost;
-
-        /** R0 memory object for the VM control block (VMCB). */
-        RTR0MEMOBJ                  pMemObjVMCB;
-        /** Physical address of the VM control block (VMCB). */
-        RTHCPHYS                    pVMCBPhys;
-        /** Virtual address of the VM control block (VMCB). */
-        R0PTRTYPE(void *)           pVMCB;
-
-        /** Ring 0 handlers for VT-x. */
-        PFNHWACCMSVMVMRUN           pfnVMRun;
-
-        /** R0 memory object for the MSR bitmap (8kb). */
-        RTR0MEMOBJ                  pMemObjMSRBitmap;
-        /** Physical address of the MSR bitmap (8kb). */
-        RTHCPHYS                    pMSRBitmapPhys;
-        /** Virtual address of the MSR bitmap. */
-        R0PTRTYPE(void *)           pMSRBitmap;
-    } svm;
-
-    /** Event injection state. */
-    struct
-    {
-        uint32_t                    fPending;
-        uint32_t                    errCode;
-        uint64_t                    intInfo;
-    } Event;
-
-    /** IO Block emulation state. */
-    struct
-    {
-        bool                    fEnabled;
-        uint8_t                 u8Align[7];
-
-        /** RIP at the start of the io code we wish to emulate in the recompiler. */
-        RTGCPTR                 GCPtrFunctionEip;
-
-        uint64_t                cr0;
-    } EmulateIoBlock;
-
-    struct
-    {
-        /* Pending IO operation type. */
-        HWACCMPENDINGIO         enmType;
-        uint32_t                uPadding;
-        RTGCPTR                 GCPtrRip;
-        RTGCPTR                 GCPtrRipNext;
-        union
-        {
-            struct
-            {
-                unsigned        uPort;
-                unsigned        uAndVal;
-                unsigned        cbSize;
-            } Port;
-            uint64_t            aRaw[2];
-        } s;
-    } PendingIO;
-
-    /** Currently shadow paging mode. */
-    PGMMODE                 enmShadowMode;
-
-    /** The CPU ID of the CPU currently owning the VMCS. Set in
-     * HWACCMR0Enter and cleared in HWACCMR0Leave. */
-    RTCPUID                 idEnteredCpu;
-
-    /** To keep track of pending TLB shootdown pages. (SMP guest only) */
-    struct
-    {
-        RTGCPTR             aPages[HWACCM_MAX_TLB_SHOOTDOWN_PAGES];
-        unsigned            cPages;
-    } TlbShootdown;
-
-    /** For saving stack space, the disassembler state is allocated here instead of
-     * on the stack.
-     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
-    union
-    {
-        /** The disassembler scratch space. */
-        DISCPUSTATE         DisState;
-        /** Padding. */
-        uint8_t             abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
-    };
-
-    RTUINT                  padding2[1];
-
-    STAMPROFILEADV          StatEntry;
-    STAMPROFILEADV          StatExit1;
-    STAMPROFILEADV          StatExit2;
-#if 1 /* temporary for tracking down darwin issues. */
-    STAMPROFILEADV          StatExit2Sub1;
-    STAMPROFILEADV          StatExit2Sub2;
-    STAMPROFILEADV          StatExit2Sub3;
-#endif
-    STAMPROFILEADV          StatInGC;
-
-#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    STAMPROFILEADV          StatWorldSwitch3264;
-#endif
-    STAMPROFILEADV          StatPoke;
-    STAMPROFILEADV          StatSpinPoke;
-    STAMPROFILEADV          StatSpinPokeFailed;
-
-    STAMCOUNTER             StatIntInject;
-
-    STAMCOUNTER             StatExitShadowNM;
-    STAMCOUNTER             StatExitGuestNM;
-    STAMCOUNTER             StatExitShadowPF;
-    STAMCOUNTER             StatExitGuestPF;
-    STAMCOUNTER             StatExitGuestUD;
-    STAMCOUNTER             StatExitGuestSS;
-    STAMCOUNTER             StatExitGuestNP;
-    STAMCOUNTER             StatExitGuestGP;
-    STAMCOUNTER             StatExitGuestDE;
-    STAMCOUNTER             StatExitGuestDB;
-    STAMCOUNTER             StatExitGuestMF;
-    STAMCOUNTER             StatExitInvpg;
-    STAMCOUNTER             StatExitInvd;
-    STAMCOUNTER             StatExitCpuid;
-    STAMCOUNTER             StatExitRdtsc;
-    STAMCOUNTER             StatExitRdpmc;
-    STAMCOUNTER             StatExitCli;
-    STAMCOUNTER             StatExitSti;
-    STAMCOUNTER             StatExitPushf;
-    STAMCOUNTER             StatExitPopf;
-    STAMCOUNTER             StatExitIret;
-    STAMCOUNTER             StatExitInt;
-    STAMCOUNTER             StatExitCRxWrite[16];
-    STAMCOUNTER             StatExitCRxRead[16];
-    STAMCOUNTER             StatExitDRxWrite;
-    STAMCOUNTER             StatExitDRxRead;
-    STAMCOUNTER             StatExitRdmsr;
-    STAMCOUNTER             StatExitWrmsr;
-    STAMCOUNTER             StatExitCLTS;
-    STAMCOUNTER             StatExitHlt;
-    STAMCOUNTER             StatExitMwait;
-    STAMCOUNTER             StatExitMonitor;
-    STAMCOUNTER             StatExitLMSW;
-    STAMCOUNTER             StatExitIOWrite;
-    STAMCOUNTER             StatExitIORead;
-    STAMCOUNTER             StatExitIOStringWrite;
-    STAMCOUNTER             StatExitIOStringRead;
-    STAMCOUNTER             StatExitIrqWindow;
-    STAMCOUNTER             StatExitMaxResume;
-    STAMCOUNTER             StatExitPreemptPending;
-    STAMCOUNTER             StatIntReinject;
-    STAMCOUNTER             StatPendingHostIrq;
-
-    STAMCOUNTER             StatFlushPage;
-    STAMCOUNTER             StatFlushPageManual;
-    STAMCOUNTER             StatFlushPhysPageManual;
-    STAMCOUNTER             StatFlushTLB;
-    STAMCOUNTER             StatFlushTLBManual;
-    STAMCOUNTER             StatFlushPageInvlpg;
-    STAMCOUNTER             StatFlushTLBWorldSwitch;
-    STAMCOUNTER             StatNoFlushTLBWorldSwitch;
-    STAMCOUNTER             StatFlushTLBCRxChange;
-    STAMCOUNTER             StatFlushASID;
-    STAMCOUNTER             StatFlushTLBInvlpga;
-    STAMCOUNTER             StatTlbShootdown;
-    STAMCOUNTER             StatTlbShootdownFlush;
-
-    STAMCOUNTER             StatSwitchGuestIrq;
-    STAMCOUNTER             StatSwitchToR3;
-
-    STAMCOUNTER             StatTSCOffset;
-    STAMCOUNTER             StatTSCIntercept;
-    STAMCOUNTER             StatTSCInterceptOverFlow;
-
-    STAMCOUNTER             StatExitReasonNPF;
-    STAMCOUNTER             StatDRxArmed;
-    STAMCOUNTER             StatDRxContextSwitch;
-    STAMCOUNTER             StatDRxIOCheck;
-
-    STAMCOUNTER             StatLoadMinimal;
-    STAMCOUNTER             StatLoadFull;
-
-#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-    STAMCOUNTER             StatFpu64SwitchBack;
-    STAMCOUNTER             StatDebug64SwitchBack;
-#endif
-
-#ifdef VBOX_WITH_STATISTICS
-    R3PTRTYPE(PSTAMCOUNTER) paStatExitReason;
-    R0PTRTYPE(PSTAMCOUNTER) paStatExitReasonR0;
-    R3PTRTYPE(PSTAMCOUNTER) paStatInjectedIrqs;
-    R0PTRTYPE(PSTAMCOUNTER) paStatInjectedIrqsR0;
-#endif
-} HWACCMCPU;
-/** Pointer to HWACCM VM instance data. */
-typedef HWACCMCPU *PHWACCMCPU;
-
-
-#ifdef IN_RING0
-
-VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpu();
-VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpuEx(RTCPUID idCpu);
-
-
-#ifdef VBOX_STRICT
-VMMR0DECL(void) HWACCMDumpRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
-VMMR0DECL(void) HWACCMR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszMsg);
-#else
-# define HWACCMDumpRegs(a, b ,c)            do { } while (0)
-# define HWACCMR0DumpDescriptor(a, b, c)    do { } while (0)
-#endif
-
-/* Dummy callback handlers. */
-VMMR0DECL(int) HWACCMR0DummyEnter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu);
-VMMR0DECL(int) HWACCMR0DummyLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
-VMMR0DECL(int) HWACCMR0DummyEnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys);
-VMMR0DECL(int) HWACCMR0DummyDisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys);
-VMMR0DECL(int) HWACCMR0DummyInitVM(PVM pVM);
-VMMR0DECL(int) HWACCMR0DummyTermVM(PVM pVM);
-VMMR0DECL(int) HWACCMR0DummySetupVM(PVM pVM);
-VMMR0DECL(int) HWACCMR0DummyRunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
-VMMR0DECL(int) HWACCMR0DummySaveHostState(PVM pVM, PVMCPU pVCpu);
-VMMR0DECL(int) HWACCMR0DummyLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
-
-# ifdef VBOX_WITH_KERNEL_USING_XMM
-DECLASM(int)   hwaccmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu, PFNHWACCMVMXSTARTVM pfnStartVM);
-DECLASM(int)   hwaccmR0SVMRunWrapXMM(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu, PFNHWACCMSVMVMRUN pfnVMRun);
-# endif
-
-# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
-/**
- * Gets 64-bit GDTR and IDTR on darwin.
- * @param  pGdtr        Where to store the 64-bit GDTR.
- * @param  pIdtr        Where to store the 64-bit IDTR.
- */
-DECLASM(void) hwaccmR0Get64bitGDTRandIDTR(PX86XDTR64 pGdtr, PX86XDTR64 pIdtr);
-
-/**
- * Gets 64-bit CR3 on darwin.
- * @returns CR3
- */
-DECLASM(uint64_t) hwaccmR0Get64bitCR3(void);
-# endif
-
-#endif /* IN_RING0 */
-
-/** @} */
-
-RT_C_DECLS_END
-
-#endif
-
Index: unk/src/VBox/VMM/HWACCMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/HWACCMInternal.mac	(revision 35332)
+++ 	(revision )
@@ -1,60 +1,0 @@
-;$Id$
-;; @file
-; HWACCM - Internal header file.
-;
-;
-; Copyright (C) 2006-2007 Oracle Corporation
-;
-; This file is part of VirtualBox Open Source Edition (OSE), as
-; available from http://www.virtualbox.org. This file is free software;
-; you can redistribute it and/or modify it under the terms of the GNU
-; General Public License (GPL) as published by the Free Software
-; Foundation, in version 2 as it comes in the "COPYING" file of the
-; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
-; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
-;
-
-%define VMX_USE_CACHED_VMCS_ACCESSES
-
-;Maximum number of cached entries. 
-%define VMCSCACHE_MAX_ENTRY                             128
-
-; Structure for storing read and write VMCS actions. 
-struc VMCSCACHE
-%ifdef VBOX_WITH_CRASHDUMP_MAGIC
-    .aMagic                   resb    16
-    .uMagic                   resq    1
-    .u64TimeEntry             resq    1
-    .u64TimeSwitch            resq    1
-    .cResume                  resq    1
-    .interPD                  resq    1
-    .pSwitcher                resq    1
-    .uPos                     resd    1
-    .idCpu                    resd    1
-%endif
-    .cr2                      resq    1
-    .Write.cValidEntries      resd    1
-    .Write.uAlignment         resd    1
-    .Write.aField             resd    VMCSCACHE_MAX_ENTRY
-    .Write.aFieldVal          resq    VMCSCACHE_MAX_ENTRY
-    .Read.cValidEntries       resd    1
-    .Read.uAlignment          resd    1
-    .Read.aField              resd    VMCSCACHE_MAX_ENTRY
-    .Read.aFieldVal           resq    VMCSCACHE_MAX_ENTRY
-%ifdef DEBUG
-    .TestIn.pPageCpuPhys      resq    1
-    .TestIn.pVMCSPhys         resq    1
-    .TestIn.pCache            resq    1
-    .TestIn.pCtx              resq    1
-    .TestOut.pVMCSPhys        resq    1
-    .TestOut.pCache           resq    1
-    .TestOut.pCtx             resq    1
-    .TestOut.eflags           resq    1
-    .TestOut.cr8              resq    1
-    .ScratchPad.param1        resq    1
-    .ScratchPad.param2        resq    1
-    .ScratchPad.param3        resq    1
-    .ScratchPad.param4        resq    1
-%endif
-endstruc
-
Index: unk/src/VBox/VMM/IOM.cpp
===================================================================
--- /trunk/src/VBox/VMM/IOM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2082 +1,0 @@
-/* $Id$ */
-/** @file
- * IOM - Input / Output Monitor.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page pg_iom        IOM - The Input / Output Monitor
- *
- * The input/output monitor will handle I/O exceptions routing them to the
- * appropriate device. It implements an API to register and deregister virtual
- * I/0 port handlers and memory mapped I/O handlers. A handler is PDM devices
- * and a set of callback functions.
- *
- * @see grp_iom
- *
- *
- * @section sec_iom_rawmode     Raw-Mode
- *
- * In raw-mode I/O port access is trapped (\#GP(0)) by ensuring that the actual
- * IOPL is 0 regardless of what the guest IOPL is. The \#GP handler use the
- * disassembler (DIS) to figure which instruction caused it (there are a number
- * of instructions in addition to the I/O ones) and if it's an I/O port access
- * it will hand it to IOMGCIOPortHandler (via EMInterpretPortIO).
- * IOMGCIOPortHandler will lookup the port in the AVL tree of registered
- * handlers. If found, the handler will be called otherwise default action is
- * taken. (Default action is to write into the void and read all set bits.)
- *
- * Memory Mapped I/O (MMIO) is implemented as a slightly special case of PGM
- * access handlers. An MMIO range is registered with IOM which then registers it
- * with the PGM access handler sub-system. The access handler catches all
- * access and will be called in the context of a \#PF handler. In RC and R0 this
- * handler is IOMMMIOHandler while in ring-3 it's IOMR3MMIOHandler (although in
- * ring-3 there can be alternative ways). IOMMMIOHandler will attempt to emulate
- * the instruction that is doing the access and pass the corresponding reads /
- * writes to the device.
- *
- * Emulating I/O port access is less complex and should be slightly faster than
- * emulating MMIO, so in most cases we should encourage the OS to use port I/O.
- * Devices which are frequently accessed should register GC handlers to speed up
- * execution.
- *
- *
- * @section sec_iom_hwaccm     Hardware Assisted Virtualization Mode
- *
- * When running in hardware assisted virtualization mode we'll be doing much the
- * same things as in raw-mode. The main difference is that we're running in the
- * host ring-0 context and that we don't get faults (\#GP(0) and \#PG) but
- * exits.
- *
- *
- * @section sec_iom_rem         Recompiled Execution Mode
- *
- * When running in the recompiler things are different. I/O port access is
- * handled by calling IOMIOPortRead and IOMIOPortWrite directly. While MMIO can
- * be handled in one of two ways. The normal way is that we have a registered a
- * special RAM range with the recompiler and in the three callbacks (for byte,
- * word and dword access) we call IOMMMIORead and IOMMMIOWrite directly. The
- * alternative ways that the physical memory access which goes via PGM will take
- * care of it by calling IOMR3MMIOHandler via the PGM access handler machinery
- * - this shouldn't happen but it is an alternative...
- *
- *
- * @section sec_iom_other       Other Accesses
- *
- * I/O ports aren't really exposed in any other way, unless you count the
- * instruction interpreter in EM, but that's just what we're doing in the
- * raw-mode \#GP(0) case really. Now, it's possible to call IOMIOPortRead and
- * IOMIOPortWrite directly to talk to a device, but this is really bad behavior
- * and should only be done as temporary hacks (the PC BIOS device used to setup
- * the CMOS this way back in the dark ages).
- *
- * MMIO has similar direct routes as the I/O ports and these shouldn't be used
- * for the same reasons and with the same restrictions. OTOH since MMIO is
- * mapped into the physical memory address space, it can be accessed in a number
- * of ways thru PGM.
- *
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_IOM
-#include <VBox/iom.h>
-#include <VBox/cpum.h>
-#include <VBox/pgm.h>
-#include <VBox/sup.h>
-#include <VBox/mm.h>
-#include <VBox/stam.h>
-#include <VBox/dbgf.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pdmdev.h>
-#include "IOMInternal.h"
-#include <VBox/vm.h>
-
-#include <VBox/param.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-#include <iprt/string.h>
-#include <VBox/log.h>
-#include <VBox/err.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static void iomR3FlushCache(PVM pVM);
-static DECLCALLBACK(int) iomR3RelocateIOPortCallback(PAVLROIOPORTNODECORE pNode, void *pvUser);
-static DECLCALLBACK(int) iomR3RelocateMMIOCallback(PAVLROGCPHYSNODECORE pNode, void *pvUser);
-static DECLCALLBACK(void) iomR3IOPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(int) iomR3IOPortDummyIn(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t *pu32, unsigned cb);
-static DECLCALLBACK(int) iomR3IOPortDummyOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t u32, unsigned cb);
-static DECLCALLBACK(int) iomR3IOPortDummyInStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrDst, PRTGCUINTREG pcTransfer, unsigned cb);
-static DECLCALLBACK(int) iomR3IOPortDummyOutStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrSrc, PRTGCUINTREG pcTransfer, unsigned cb);
-
-#ifdef VBOX_WITH_STATISTICS
-static const char *iomR3IOPortGetStandardName(RTIOPORT Port);
-#endif
-
-
-/**
- * Initializes the IOM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) IOMR3Init(PVM pVM)
-{
-    LogFlow(("IOMR3Init:\n"));
-
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompileMemberAlignment(VM, iom.s, 32);
-    AssertCompile(sizeof(pVM->iom.s) <= sizeof(pVM->iom.padding));
-    AssertCompileMemberAlignment(IOM, EmtLock, sizeof(uintptr_t));
-
-    /*
-     * Setup any fixed pointers and offsets.
-     */
-    pVM->iom.s.offVM = RT_OFFSETOF(VM, iom);
-
-    /*
-     * Initialize the REM critical section.
-     */
-    int rc = PDMR3CritSectInit(pVM, &pVM->iom.s.EmtLock, RT_SRC_POS, "IOM EMT Lock");
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Allocate the trees structure.
-     */
-    rc = MMHyperAlloc(pVM, sizeof(*pVM->iom.s.pTreesR3), 0, MM_TAG_IOM, (void **)&pVM->iom.s.pTreesR3);
-    if (RT_SUCCESS(rc))
-    {
-        pVM->iom.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->iom.s.pTreesR3);
-        pVM->iom.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->iom.s.pTreesR3);
-        pVM->iom.s.pfnMMIOHandlerRC = NIL_RTGCPTR;
-        pVM->iom.s.pfnMMIOHandlerR0 = NIL_RTR0PTR;
-
-        /*
-         * Info.
-         */
-        DBGFR3InfoRegisterInternal(pVM, "ioport", "Dumps all IOPort ranges. No arguments.", &iomR3IOPortInfo);
-        DBGFR3InfoRegisterInternal(pVM, "mmio", "Dumps all MMIO ranges. No arguments.", &iomR3MMIOInfo);
-
-        /*
-         * Statistics.
-         */
-        STAM_REG(pVM, &pVM->iom.s.StatRZMMIOHandler,      STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler",                      STAMUNIT_TICKS_PER_CALL, "Profiling of the IOMMMIOHandler() body, only success calls.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO1Byte,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access1",              STAMUNIT_OCCURENCES,     "MMIO access by 1 byte counter.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO2Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access2",              STAMUNIT_OCCURENCES,     "MMIO access by 2 bytes counter.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO4Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access4",              STAMUNIT_OCCURENCES,     "MMIO access by 4 bytes counter.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO8Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access8",              STAMUNIT_OCCURENCES,     "MMIO access by 8 bytes counter.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZMMIOFailures,     STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/MMIOFailures",         STAMUNIT_OCCURENCES,     "Number of times IOMMMIOHandler() didn't service the request.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstMov,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOV",             STAMUNIT_TICKS_PER_CALL, "Profiling of the MOV instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstCmp,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/CMP",             STAMUNIT_TICKS_PER_CALL, "Profiling of the CMP instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstAnd,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/AND",             STAMUNIT_TICKS_PER_CALL, "Profiling of the AND instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstOr,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/OR",              STAMUNIT_TICKS_PER_CALL, "Profiling of the OR instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstXor,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XOR",             STAMUNIT_TICKS_PER_CALL, "Profiling of the XOR instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstBt,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/BT",              STAMUNIT_TICKS_PER_CALL, "Profiling of the BT instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstTest,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/TEST",            STAMUNIT_TICKS_PER_CALL, "Profiling of the TEST instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstXchg,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XCHG",            STAMUNIT_TICKS_PER_CALL, "Profiling of the XCHG instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstStos,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/STOS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the STOS instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstLods,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/LODS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the LODS instruction emulation.");
-#ifdef IOM_WITH_MOVS_SUPPORT
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovs,     STAMTYPE_PROFILE_ADV, "/IOM/RZ-MMIOHandler/Inst/MOVS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsToMMIO,   STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/ToMMIO",     STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - Mem2MMIO.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsFromMMIO, STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/FromMMIO",   STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2Mem.");
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsMMIO,     STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/MMIO2MMIO",  STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2MMIO.");
-#endif
-        STAM_REG(pVM, &pVM->iom.s.StatRZInstOther,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Inst/Other",           STAMUNIT_OCCURENCES,     "Other instructions counter.");
-        STAM_REG(pVM, &pVM->iom.s.StatR3MMIOHandler,      STAMTYPE_COUNTER, "/IOM/R3-MMIOHandler",                      STAMUNIT_OCCURENCES,     "Number of calls to IOMR3MMIOHandler.");
-        STAM_REG(pVM, &pVM->iom.s.StatInstIn,             STAMTYPE_COUNTER, "/IOM/IOWork/In",                           STAMUNIT_OCCURENCES,     "Counter of any IN instructions.");
-        STAM_REG(pVM, &pVM->iom.s.StatInstOut,            STAMTYPE_COUNTER, "/IOM/IOWork/Out",                          STAMUNIT_OCCURENCES,     "Counter of any OUT instructions.");
-        STAM_REG(pVM, &pVM->iom.s.StatInstIns,            STAMTYPE_COUNTER, "/IOM/IOWork/Ins",                          STAMUNIT_OCCURENCES,     "Counter of any INS instructions.");
-        STAM_REG(pVM, &pVM->iom.s.StatInstOuts,           STAMTYPE_COUNTER, "/IOM/IOWork/Outs",                         STAMUNIT_OCCURENCES,     "Counter of any OUTS instructions.");
-    }
-
-    /* Redundant, but just in case we change something in the future */
-    iomR3FlushCache(pVM);
-
-    LogFlow(("IOMR3Init: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Flushes the IOM port & statistics lookup cache
- *
- * @param   pVM     The VM.
- */
-static void iomR3FlushCache(PVM pVM)
-{
-    iomLock(pVM);
-    /*
-     * Caching of port and statistics (saves some time in rep outs/ins instruction emulation)
-     */
-    pVM->iom.s.pRangeLastReadR0  = NIL_RTR0PTR;
-    pVM->iom.s.pRangeLastWriteR0 = NIL_RTR0PTR;
-    pVM->iom.s.pStatsLastReadR0  = NIL_RTR0PTR;
-    pVM->iom.s.pStatsLastWriteR0 = NIL_RTR0PTR;
-    pVM->iom.s.pMMIORangeLastR0  = NIL_RTR0PTR;
-    pVM->iom.s.pMMIOStatsLastR0  = NIL_RTR0PTR;
-
-    pVM->iom.s.pRangeLastReadR3  = NULL;
-    pVM->iom.s.pRangeLastWriteR3 = NULL;
-    pVM->iom.s.pStatsLastReadR3  = NULL;
-    pVM->iom.s.pStatsLastWriteR3 = NULL;
-    pVM->iom.s.pMMIORangeLastR3  = NULL;
-    pVM->iom.s.pMMIOStatsLastR3  = NULL;
-
-    pVM->iom.s.pRangeLastReadRC  = NIL_RTRCPTR;
-    pVM->iom.s.pRangeLastWriteRC = NIL_RTRCPTR;
-    pVM->iom.s.pStatsLastReadRC  = NIL_RTRCPTR;
-    pVM->iom.s.pStatsLastWriteRC = NIL_RTRCPTR;
-    pVM->iom.s.pMMIORangeLastRC  = NIL_RTRCPTR;
-    pVM->iom.s.pMMIOStatsLastRC  = NIL_RTRCPTR;
-
-    iomUnlock(pVM);
-}
-
-
-/**
- * The VM is being reset.
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) IOMR3Reset(PVM pVM)
-{
-    iomR3FlushCache(pVM);
-}
-
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * The IOM will update the addresses used by the switcher.
- *
- * @param   pVM     The VM.
- * @param   offDelta    Relocation delta relative to old location.
- */
-VMMR3DECL(void) IOMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    LogFlow(("IOMR3Relocate: offDelta=%d\n", offDelta));
-
-    /*
-     * Apply relocations to the GC callbacks.
-     */
-    pVM->iom.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->iom.s.pTreesR3);
-    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeRC, true, iomR3RelocateIOPortCallback, &offDelta);
-    RTAvlroGCPhysDoWithAll(&pVM->iom.s.pTreesR3->MMIOTree,     true, iomR3RelocateMMIOCallback,   &offDelta);
-
-    if (pVM->iom.s.pfnMMIOHandlerRC)
-        pVM->iom.s.pfnMMIOHandlerRC += offDelta;
-
-    /*
-     * Apply relocations to the cached GC handlers
-     */
-    if (pVM->iom.s.pRangeLastReadRC)
-        pVM->iom.s.pRangeLastReadRC  += offDelta;
-    if (pVM->iom.s.pRangeLastWriteRC)
-        pVM->iom.s.pRangeLastWriteRC += offDelta;
-    if (pVM->iom.s.pStatsLastReadRC)
-        pVM->iom.s.pStatsLastReadRC  += offDelta;
-    if (pVM->iom.s.pStatsLastWriteRC)
-        pVM->iom.s.pStatsLastWriteRC += offDelta;
-    if (pVM->iom.s.pMMIORangeLastRC)
-        pVM->iom.s.pMMIORangeLastRC  += offDelta;
-    if (pVM->iom.s.pMMIOStatsLastRC)
-        pVM->iom.s.pMMIOStatsLastRC  += offDelta;
-}
-
-
-/**
- * Callback function for relocating a I/O port range.
- *
- * @returns 0 (continue enum)
- * @param   pNode       Pointer to a IOMIOPORTRANGERC node.
- * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
- *                      not certain the delta will fit in a void pointer for all possible configs.
- */
-static DECLCALLBACK(int) iomR3RelocateIOPortCallback(PAVLROIOPORTNODECORE pNode, void *pvUser)
-{
-    PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)pNode;
-    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
-
-    Assert(pRange->pDevIns);
-    pRange->pDevIns                 += offDelta;
-    if (pRange->pfnOutCallback)
-        pRange->pfnOutCallback      += offDelta;
-    if (pRange->pfnInCallback)
-        pRange->pfnInCallback       += offDelta;
-    if (pRange->pfnOutStrCallback)
-        pRange->pfnOutStrCallback   += offDelta;
-    if (pRange->pfnInStrCallback)
-        pRange->pfnInStrCallback    += offDelta;
-    if (pRange->pvUser > _64K)
-        pRange->pvUser              += offDelta;
-    return 0;
-}
-
-
-/**
- * Callback function for relocating a MMIO range.
- *
- * @returns 0 (continue enum)
- * @param   pNode       Pointer to a IOMMMIORANGE node.
- * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
- *                      not certain the delta will fit in a void pointer for all possible configs.
- */
-static DECLCALLBACK(int) iomR3RelocateMMIOCallback(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pNode;
-    RTGCINTPTR    offDelta = *(PRTGCINTPTR)pvUser;
-
-    if (pRange->pDevInsRC)
-        pRange->pDevInsRC           += offDelta;
-    if (pRange->pfnWriteCallbackRC)
-        pRange->pfnWriteCallbackRC  += offDelta;
-    if (pRange->pfnReadCallbackRC)
-        pRange->pfnReadCallbackRC   += offDelta;
-    if (pRange->pfnFillCallbackRC)
-        pRange->pfnFillCallbackRC   += offDelta;
-    if (pRange->pvUserRC > _64K)
-        pRange->pvUserRC            += offDelta;
-
-    return 0;
-}
-
-
-/**
- * Terminates the IOM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) IOMR3Term(PVM pVM)
-{
-    /*
-     * IOM is not owning anything but automatically freed resources,
-     * so there's nothing to do here.
-     */
-    return VINF_SUCCESS;
-}
-
-#ifdef VBOX_WITH_STATISTICS
-
-/**
- * Create the statistics node for an I/O port.
- *
- * @returns Pointer to new stats node.
- *
- * @param   pVM         VM handle.
- * @param   Port        Port.
- * @param   pszDesc     Description.
- */
-PIOMIOPORTSTATS iomR3IOPortStatsCreate(PVM pVM, RTIOPORT Port, const char *pszDesc)
-{
-    Assert(IOMIsLockOwner(pVM));
-    /* check if it already exists. */
-    PIOMIOPORTSTATS pPort = (PIOMIOPORTSTATS)RTAvloIOPortGet(&pVM->iom.s.pTreesR3->IOPortStatTree, Port);
-    if (pPort)
-        return pPort;
-
-    /* allocate stats node. */
-    int rc = MMHyperAlloc(pVM, sizeof(*pPort), 0, MM_TAG_IOM_STATS, (void **)&pPort);
-    AssertRC(rc);
-    if (RT_SUCCESS(rc))
-    {
-        /* insert into the tree. */
-        pPort->Core.Key = Port;
-        if (RTAvloIOPortInsert(&pVM->iom.s.pTreesR3->IOPortStatTree, &pPort->Core))
-        {
-            /* put a name on common ports. */
-            if (!pszDesc)
-                pszDesc = iomR3IOPortGetStandardName(Port);
-
-            /* register the statistics counters. */
-            rc = STAMR3RegisterF(pVM, &pPort->InR3,     STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-In-R3", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->OutR3,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-Out-R3", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->InRZ,     STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-In-RZ", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->OutRZ,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-Out-RZ", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->InRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-In-RZtoR3", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->OutRZToR3,STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-Out-RZtoR3", Port); AssertRC(rc);
-
-            /* Profiling */
-            rc = STAMR3RegisterF(pVM, &pPort->ProfInR3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-In-R3/Prof", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->ProfOutR3,STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-Out-R3/Prof", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->ProfInRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-In-RZ/Prof", Port); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pPort->ProfOutRZ,STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-Out-RZ/Prof", Port); AssertRC(rc);
-
-            return pPort;
-        }
-        AssertMsgFailed(("what! Port=%d\n", Port));
-        MMHyperFree(pVM, pPort);
-    }
-    return NULL;
-}
-
-
-/**
- * Create the statistics node for an MMIO address.
- *
- * @returns Pointer to new stats node.
- *
- * @param   pVM         VM handle.
- * @param   GCPhys      The address.
- * @param   pszDesc     Description.
- */
-PIOMMMIOSTATS iomR3MMIOStatsCreate(PVM pVM, RTGCPHYS GCPhys, const char *pszDesc)
-{
-    Assert(IOMIsLockOwner(pVM));
-#ifdef DEBUG_sandervl
-    AssertGCPhys32(GCPhys);
-#endif
-    /* check if it already exists. */
-    PIOMMMIOSTATS pStats = (PIOMMMIOSTATS)RTAvloGCPhysGet(&pVM->iom.s.pTreesR3->MMIOStatTree, GCPhys);
-    if (pStats)
-        return pStats;
-
-    /* allocate stats node. */
-    int rc = MMHyperAlloc(pVM, sizeof(*pStats), 0, MM_TAG_IOM_STATS, (void **)&pStats);
-    AssertRC(rc);
-    if (RT_SUCCESS(rc))
-    {
-        /* insert into the tree. */
-        pStats->Core.Key = GCPhys;
-        if (RTAvloGCPhysInsert(&pVM->iom.s.pTreesR3->MMIOStatTree, &pStats->Core))
-        {
-            rc = STAMR3RegisterF(pVM, &pStats->Accesses,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,     pszDesc, "/IOM/MMIO/%RGp",              GCPhys); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pStats->ProfReadR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Read-R3",      GCPhys); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pStats->ProfWriteR3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Write-R3",     GCPhys); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pStats->ProfReadRZ,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Read-RZ",      GCPhys); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pStats->ProfWriteRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Write-RZ",     GCPhys); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pStats->ReadRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,     pszDesc, "/IOM/MMIO/%RGp/Read-RZtoR3",  GCPhys); AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pStats->WriteRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,     pszDesc, "/IOM/MMIO/%RGp/Write-RZtoR3", GCPhys); AssertRC(rc);
-
-            return pStats;
-        }
-        AssertMsgFailed(("what! GCPhys=%RGp\n", GCPhys));
-        MMHyperFree(pVM, pStats);
-    }
-    return NULL;
-}
-
-#endif /* VBOX_WITH_STATISTICS */
-
-/**
- * Registers a I/O port ring-3 handler.
- *
- * This API is called by PDM on behalf of a device. Devices must first register
- * ring-3 ranges before any GC and R0 ranges can be registered using IOMR3IOPortRegisterRC()
- * and IOMR3IOPortRegisterR0().
- *
- *
- * @returns VBox status code.
- *
- * @param   pVM                 VM handle.
- * @param   pDevIns             PDM device instance owning the port range.
- * @param   PortStart           First port number in the range.
- * @param   cPorts              Number of ports to register.
- * @param   pvUser              User argument for the callbacks.
- * @param   pfnOutCallback      Pointer to function which is gonna handle OUT operations in R3.
- * @param   pfnInCallback       Pointer to function which is gonna handle IN operations in R3.
- * @param   pfnOutStrCallback   Pointer to function which is gonna handle string OUT operations in R3.
- * @param   pfnInStrCallback    Pointer to function which is gonna handle string IN operations in R3.
- * @param   pszDesc             Pointer to description string. This must not be freed.
- */
-VMMR3DECL(int) IOMR3IOPortRegisterR3(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts, RTHCPTR pvUser,
-                                     R3PTRTYPE(PFNIOMIOPORTOUT) pfnOutCallback, R3PTRTYPE(PFNIOMIOPORTIN) pfnInCallback,
-                                     R3PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback, R3PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback, const char *pszDesc)
-{
-    LogFlow(("IOMR3IOPortRegisterR3: pDevIns=%p PortStart=%#x cPorts=%#x pvUser=%RHv pfnOutCallback=%#x pfnInCallback=%#x pfnOutStrCallback=%#x pfnInStrCallback=%#x pszDesc=%s\n",
-             pDevIns, PortStart, cPorts, pvUser, pfnOutCallback, pfnInCallback, pfnOutStrCallback, pfnInStrCallback, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    if (    (RTUINT)PortStart + cPorts <= (RTUINT)PortStart
-        ||  (RTUINT)PortStart + cPorts > 0x10000)
-    {
-        AssertMsgFailed(("Invalid port range %#x-%#x (inclusive)! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
-        return VERR_IOM_INVALID_IOPORT_RANGE;
-    }
-    if (!pfnOutCallback && !pfnInCallback)
-    {
-        AssertMsgFailed(("no handlers specfied for %#x-%#x (inclusive)! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!pfnOutCallback)
-        pfnOutCallback = iomR3IOPortDummyOut;
-    if (!pfnInCallback)
-        pfnInCallback = iomR3IOPortDummyIn;
-    if (!pfnOutStrCallback)
-        pfnOutStrCallback = iomR3IOPortDummyOutStr;
-    if (!pfnInStrCallback)
-        pfnInStrCallback = iomR3IOPortDummyInStr;
-
-    /* Flush the IO port lookup cache */
-    iomR3FlushCache(pVM);
-
-    /*
-     * Allocate new range record and initialize it.
-     */
-    PIOMIOPORTRANGER3 pRange;
-    int rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
-    if (RT_SUCCESS(rc))
-    {
-        pRange->Core.Key        = PortStart;
-        pRange->Core.KeyLast    = PortStart + (cPorts - 1);
-        pRange->Port            = PortStart;
-        pRange->cPorts          = cPorts;
-        pRange->pvUser          = pvUser;
-        pRange->pDevIns         = pDevIns;
-        pRange->pfnOutCallback  = pfnOutCallback;
-        pRange->pfnInCallback   = pfnInCallback;
-        pRange->pfnOutStrCallback = pfnOutStrCallback;
-        pRange->pfnInStrCallback = pfnInStrCallback;
-        pRange->pszDesc         = pszDesc;
-
-        /*
-         * Try Insert it.
-         */
-        iomLock(pVM);
-        if (RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeR3, &pRange->Core))
-        {
-            #ifdef VBOX_WITH_STATISTICS
-            for (unsigned iPort = 0; iPort < cPorts; iPort++)
-                iomR3IOPortStatsCreate(pVM, PortStart + iPort, pszDesc);
-            #endif
-            iomUnlock(pVM);
-            return VINF_SUCCESS;
-        }
-        iomUnlock(pVM);
-
-        /* conflict. */
-        DBGFR3Info(pVM, "ioport", NULL, NULL);
-        AssertMsgFailed(("Port range %#x-%#x (%s) conflicts with existing range(s)!\n", PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
-        MMHyperFree(pVM, pRange);
-        rc = VERR_IOM_IOPORT_RANGE_CONFLICT;
-    }
-
-    return rc;
-}
-
-
-/**
- * Registers a I/O port RC handler.
- *
- * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
- * using IOMIOPortRegisterR3() before calling this function.
- *
- *
- * @returns VBox status code.
- *
- * @param   pVM                 VM handle.
- * @param   pDevIns             PDM device instance owning the port range.
- * @param   PortStart           First port number in the range.
- * @param   cPorts              Number of ports to register.
- * @param   pvUser              User argument for the callbacks.
- * @param   pfnOutCallback      Pointer to function which is gonna handle OUT operations in GC.
- * @param   pfnInCallback       Pointer to function which is gonna handle IN operations in GC.
- * @param   pfnOutStrCallback   Pointer to function which is gonna handle string OUT operations in GC.
- * @param   pfnInStrCallback    Pointer to function which is gonna handle string IN operations in GC.
- * @param   pszDesc             Pointer to description string. This must not be freed.
- */
-VMMR3DECL(int)  IOMR3IOPortRegisterRC(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts, RTRCPTR pvUser,
-                                      RCPTRTYPE(PFNIOMIOPORTOUT) pfnOutCallback, RCPTRTYPE(PFNIOMIOPORTIN) pfnInCallback,
-                                      RCPTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback, RCPTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback, const char *pszDesc)
-{
-    LogFlow(("IOMR3IOPortRegisterRC: pDevIns=%p PortStart=%#x cPorts=%#x pvUser=%RRv pfnOutCallback=%RRv pfnInCallback=%RRv pfnOutStrCallback=%RRv  pfnInStrCallback=%RRv pszDesc=%s\n",
-             pDevIns, PortStart, cPorts, pvUser, pfnOutCallback, pfnInCallback, pfnOutStrCallback, pfnInStrCallback, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    if (    (RTUINT)PortStart + cPorts <= (RTUINT)PortStart
-        ||  (RTUINT)PortStart + cPorts > 0x10000)
-    {
-        AssertMsgFailed(("Invalid port range %#x-%#x! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
-        return VERR_IOM_INVALID_IOPORT_RANGE;
-    }
-    RTIOPORT PortLast = PortStart + (cPorts - 1);
-    if (!pfnOutCallback && !pfnInCallback)
-    {
-        AssertMsgFailed(("Invalid port range %#x-%#x! No callbacks! (%s)\n", PortStart, PortLast, pszDesc));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    iomLock(pVM);
-
-    /*
-     * Validate that there are ring-3 ranges for the ports.
-     */
-    RTIOPORT Port = PortStart;
-    while (Port <= PortLast && Port >= PortStart)
-    {
-        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeR3, Port);
-        if (!pRange)
-        {
-            AssertMsgFailed(("No R3! Port=#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
-            iomUnlock(pVM);
-            return VERR_IOM_NO_HC_IOPORT_RANGE;
-        }
-#ifndef IOM_NO_PDMINS_CHECKS
-# ifndef IN_RC
-        if (pRange->pDevIns != pDevIns)
-# else
-        if (pRange->pDevIns != MMHyperRCToCC(pVM, pDevIns))
-# endif
-        {
-            AssertMsgFailed(("Not owner! Port=%#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
-            iomUnlock(pVM);
-            return VERR_IOM_NOT_IOPORT_RANGE_OWNER;
-        }
-#endif
-        Port = pRange->Core.KeyLast + 1;
-    }
-
-    /* Flush the IO port lookup cache */
-    iomR3FlushCache(pVM);
-
-    /*
-     * Allocate new range record and initialize it.
-     */
-    PIOMIOPORTRANGERC pRange;
-    int rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
-    if (RT_SUCCESS(rc))
-    {
-        pRange->Core.Key        = PortStart;
-        pRange->Core.KeyLast    = PortLast;
-        pRange->Port            = PortStart;
-        pRange->cPorts          = cPorts;
-        pRange->pvUser          = pvUser;
-        pRange->pfnOutCallback  = pfnOutCallback;
-        pRange->pfnInCallback   = pfnInCallback;
-        pRange->pfnOutStrCallback = pfnOutStrCallback;
-        pRange->pfnInStrCallback = pfnInStrCallback;
-        pRange->pDevIns         = MMHyperCCToRC(pVM, pDevIns);
-        pRange->pszDesc         = pszDesc;
-
-        /*
-         * Insert it.
-         */
-        if (RTAvlroIOPortInsert(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeRC, &pRange->Core))
-        {
-            iomUnlock(pVM);
-            return VINF_SUCCESS;
-        }
-
-        /* conflict. */
-        AssertMsgFailed(("Port range %#x-%#x (%s) conflicts with existing range(s)!\n", PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
-        MMHyperFree(pVM, pRange);
-        rc = VERR_IOM_IOPORT_RANGE_CONFLICT;
-    }
-    iomUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Registers a Port IO R0 handler.
- *
- * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
- * using IOMR3IOPortRegisterR3() before calling this function.
- *
- *
- * @returns VBox status code.
- *
- * @param   pVM                 VM handle.
- * @param   pDevIns             PDM device instance owning the port range.
- * @param   PortStart           First port number in the range.
- * @param   cPorts              Number of ports to register.
- * @param   pvUser              User argument for the callbacks.
- * @param   pfnOutCallback      Pointer to function which is gonna handle OUT operations in GC.
- * @param   pfnInCallback       Pointer to function which is gonna handle IN operations in GC.
- * @param   pfnOutStrCallback   Pointer to function which is gonna handle OUT operations in GC.
- * @param   pfnInStrCallback    Pointer to function which is gonna handle IN operations in GC.
- * @param   pszDesc             Pointer to description string. This must not be freed.
- */
-VMMR3DECL(int)  IOMR3IOPortRegisterR0(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts, RTR0PTR pvUser,
-                                      R0PTRTYPE(PFNIOMIOPORTOUT) pfnOutCallback, R0PTRTYPE(PFNIOMIOPORTIN) pfnInCallback,
-                                      R0PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback, R0PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback,
-                                      const char *pszDesc)
-{
-    LogFlow(("IOMR3IOPortRegisterR0: pDevIns=%p PortStart=%#x cPorts=%#x pvUser=%RHv pfnOutCallback=%RHv pfnInCallback=%RHv pfnOutStrCallback=%RHv  pfnInStrCallback=%RHv pszDesc=%s\n",
-             pDevIns, PortStart, cPorts, pvUser, pfnOutCallback, pfnInCallback, pfnOutStrCallback, pfnInStrCallback, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    if (    (RTUINT)PortStart + cPorts <= (RTUINT)PortStart
-        ||  (RTUINT)PortStart + cPorts > 0x10000)
-    {
-        AssertMsgFailed(("Invalid port range %#x-%#x! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
-        return VERR_IOM_INVALID_IOPORT_RANGE;
-    }
-    RTIOPORT PortLast = PortStart + (cPorts - 1);
-    if (!pfnOutCallback && !pfnInCallback)
-    {
-        AssertMsgFailed(("Invalid port range %#x-%#x! No callbacks! (%s)\n", PortStart, PortLast, pszDesc));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    iomLock(pVM);
-    /*
-     * Validate that there are ring-3 ranges for the ports.
-     */
-    RTIOPORT Port = PortStart;
-    while (Port <= PortLast && Port >= PortStart)
-    {
-        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeR3, Port);
-        if (!pRange)
-        {
-            AssertMsgFailed(("No R3! Port=#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
-            iomUnlock(pVM);
-            return VERR_IOM_NO_HC_IOPORT_RANGE;
-        }
-#ifndef IOM_NO_PDMINS_CHECKS
-# ifndef IN_RC
-        if (pRange->pDevIns != pDevIns)
-# else
-        if (pRange->pDevIns != MMHyperRCToCC(pVM, pDevIns))
-# endif
-        {
-            AssertMsgFailed(("Not owner! Port=%#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
-            iomUnlock(pVM);
-            return VERR_IOM_NOT_IOPORT_RANGE_OWNER;
-        }
-#endif
-        Port = pRange->Core.KeyLast + 1;
-    }
-
-    /* Flush the IO port lookup cache */
-    iomR3FlushCache(pVM);
-
-    /*
-     * Allocate new range record and initialize it.
-     */
-    PIOMIOPORTRANGER0 pRange;
-    int rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
-    if (RT_SUCCESS(rc))
-    {
-        pRange->Core.Key        = PortStart;
-        pRange->Core.KeyLast    = PortLast;
-        pRange->Port            = PortStart;
-        pRange->cPorts          = cPorts;
-        pRange->pvUser          = pvUser;
-        pRange->pfnOutCallback  = pfnOutCallback;
-        pRange->pfnInCallback   = pfnInCallback;
-        pRange->pfnOutStrCallback = pfnOutStrCallback;
-        pRange->pfnInStrCallback = pfnInStrCallback;
-        pRange->pDevIns         = MMHyperR3ToR0(pVM, pDevIns);
-        pRange->pszDesc         = pszDesc;
-
-        /*
-         * Insert it.
-         */
-        if (RTAvlroIOPortInsert(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeR0, &pRange->Core))
-        {
-            iomUnlock(pVM);
-            return VINF_SUCCESS;
-        }
-
-        /* conflict. */
-        AssertMsgFailed(("Port range %#x-%#x (%s) conflicts with existing range(s)!\n", PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
-        MMHyperFree(pVM, pRange);
-        rc = VERR_IOM_IOPORT_RANGE_CONFLICT;
-    }
-    iomUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Deregisters a I/O Port range.
- *
- * The specified range must be registered using IOMR3IOPortRegister previous to
- * this call. The range does can be a smaller part of the range specified to
- * IOMR3IOPortRegister, but it can never be larger.
- *
- * This function will remove GC, R0 and R3 context port handlers for this range.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The virtual machine.
- * @param   pDevIns             The device instance associated with the range.
- * @param   PortStart           First port number in the range.
- * @param   cPorts              Number of ports to remove starting at PortStart.
- *
- * @remark  This function mainly for PCI PnP Config and will not do
- *          all the checks you might expect it to do.
- */
-VMMR3DECL(int)  IOMR3IOPortDeregister(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts)
-{
-    LogFlow(("IOMR3IOPortDeregister: pDevIns=%p PortStart=%#x cPorts=%#x\n", pDevIns, PortStart, cPorts));
-
-    /*
-     * Validate input.
-     */
-    if (    (RTUINT)PortStart + cPorts < (RTUINT)PortStart
-        ||  (RTUINT)PortStart + cPorts > 0x10000)
-    {
-        AssertMsgFailed(("Invalid port range %#x-%#x!\n", PortStart, (unsigned)PortStart + cPorts - 1));
-        return VERR_IOM_INVALID_IOPORT_RANGE;
-    }
-
-    iomLock(pVM);
-
-    /* Flush the IO port lookup cache */
-    iomR3FlushCache(pVM);
-
-    /*
-     * Check ownership.
-     */
-    RTIOPORT PortLast = PortStart + (cPorts - 1);
-    RTIOPORT Port = PortStart;
-    while (Port <= PortLast && Port >= PortStart)
-    {
-        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeR3, Port);
-        if (pRange)
-        {
-            Assert(Port <= pRange->Core.KeyLast);
-#ifndef IOM_NO_PDMINS_CHECKS
-            if (pRange->pDevIns != pDevIns)
-            {
-                AssertMsgFailed(("Removal of ports in range %#x-%#x rejected because not owner of %#x-%#x (%s)\n",
-                                 PortStart, PortLast, pRange->Core.Key, pRange->Core.KeyLast, pRange->pszDesc));
-                iomUnlock(pVM);
-                return VERR_IOM_NOT_IOPORT_RANGE_OWNER;
-            }
-#endif /* !IOM_NO_PDMINS_CHECKS */
-            Port = pRange->Core.KeyLast;
-        }
-        Port++;
-    }
-
-    /*
-     * Remove any RC ranges first.
-     */
-    int     rc = VINF_SUCCESS;
-    Port = PortStart;
-    while (Port <= PortLast && Port >= PortStart)
-    {
-        /*
-         * Try find range.
-         */
-        PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeRC, Port);
-        if (pRange)
-        {
-            if (   pRange->Core.Key     == Port
-                && pRange->Core.KeyLast <= PortLast)
-            {
-                /*
-                 * Kick out the entire range.
-                 */
-                void *pv = RTAvlroIOPortRemove(&pVM->iom.s.pTreesR3->IOPortTreeRC, Port);
-                Assert(pv == (void *)pRange); NOREF(pv);
-                Port += pRange->cPorts;
-                MMHyperFree(pVM, pRange);
-            }
-            else if (pRange->Core.Key == Port)
-            {
-                /*
-                 * Cut of the head of the range, done.
-                 */
-                pRange->cPorts  -= Port - pRange->Port;
-                pRange->Core.Key = Port;
-                pRange->Port     = Port;
-                break;
-            }
-            else if (pRange->Core.KeyLast <= PortLast)
-            {
-                /*
-                 * Just cut of the tail.
-                 */
-                unsigned c = pRange->Core.KeyLast - Port + 1;
-                pRange->Core.KeyLast -= c;
-                pRange->cPorts -= c;
-                Port += c;
-            }
-            else
-            {
-                /*
-                 * Split the range, done.
-                 */
-                Assert(pRange->Core.KeyLast > PortLast && pRange->Core.Key < Port);
-                /* create tail. */
-                PIOMIOPORTRANGERC pRangeNew;
-                int rc2 = MMHyperAlloc(pVM, sizeof(*pRangeNew), 0, MM_TAG_IOM, (void **)&pRangeNew);
-                if (RT_FAILURE(rc2))
-                {
-                    iomUnlock(pVM);
-                    return rc2;
-                }
-                *pRangeNew = *pRange;
-                pRangeNew->Core.Key     = PortLast;
-                pRangeNew->Port         = PortLast;
-                pRangeNew->cPorts       = pRangeNew->Core.KeyLast - PortLast + 1;
-
-                LogFlow(("IOMR3IOPortDeregister (rc): split the range; new %x\n", pRangeNew->Core.Key));
-
-                /* adjust head */
-                pRange->Core.KeyLast  = Port - 1;
-                pRange->cPorts        = Port - pRange->Port;
-
-                /* insert */
-                if (!RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeRC, &pRangeNew->Core))
-                {
-                    AssertMsgFailed(("This cannot happen!\n"));
-                    MMHyperFree(pVM, pRangeNew);
-                    rc = VERR_INTERNAL_ERROR;
-                }
-                break;
-            }
-        }
-        else /* next port */
-            Port++;
-    } /* for all ports - RC. */
-
-
-    /*
-     * Remove any R0 ranges.
-     */
-    Port = PortStart;
-    while (Port <= PortLast && Port >= PortStart)
-    {
-        /*
-         * Try find range.
-         */
-        PIOMIOPORTRANGER0 pRange = (PIOMIOPORTRANGER0)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeR0, Port);
-        if (pRange)
-        {
-            if (   pRange->Core.Key     == Port
-                && pRange->Core.KeyLast <= PortLast)
-            {
-                /*
-                 * Kick out the entire range.
-                 */
-                void *pv = RTAvlroIOPortRemove(&pVM->iom.s.pTreesR3->IOPortTreeR0, Port);
-                Assert(pv == (void *)pRange); NOREF(pv);
-                Port += pRange->cPorts;
-                MMHyperFree(pVM, pRange);
-            }
-            else if (pRange->Core.Key == Port)
-            {
-                /*
-                 * Cut of the head of the range, done.
-                 */
-                pRange->cPorts  -= Port - pRange->Port;
-                pRange->Core.Key = Port;
-                pRange->Port     = Port;
-                break;
-            }
-            else if (pRange->Core.KeyLast <= PortLast)
-            {
-                /*
-                 * Just cut of the tail.
-                 */
-                unsigned c = pRange->Core.KeyLast - Port + 1;
-                pRange->Core.KeyLast -= c;
-                pRange->cPorts -= c;
-                Port += c;
-            }
-            else
-            {
-                /*
-                 * Split the range, done.
-                 */
-                Assert(pRange->Core.KeyLast > PortLast && pRange->Core.Key < Port);
-                /* create tail. */
-                PIOMIOPORTRANGER0 pRangeNew;
-                int rc2 = MMHyperAlloc(pVM, sizeof(*pRangeNew), 0, MM_TAG_IOM, (void **)&pRangeNew);
-                if (RT_FAILURE(rc2))
-                {
-                    iomUnlock(pVM);
-                    return rc2;
-                }
-                *pRangeNew = *pRange;
-                pRangeNew->Core.Key     = PortLast;
-                pRangeNew->Port         = PortLast;
-                pRangeNew->cPorts       = pRangeNew->Core.KeyLast - PortLast + 1;
-
-                LogFlow(("IOMR3IOPortDeregister (r0): split the range; new %x\n", pRangeNew->Core.Key));
-
-                /* adjust head */
-                pRange->Core.KeyLast  = Port - 1;
-                pRange->cPorts        = Port - pRange->Port;
-
-                /* insert */
-                if (!RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeR0, &pRangeNew->Core))
-                {
-                    AssertMsgFailed(("This cannot happen!\n"));
-                    MMHyperFree(pVM, pRangeNew);
-                    rc = VERR_INTERNAL_ERROR;
-                }
-                break;
-            }
-        }
-        else /* next port */
-            Port++;
-    } /* for all ports - R0. */
-
-    /*
-     * And the same procedure for ring-3 ranges.
-     */
-    Port = PortStart;
-    while (Port <= PortLast && Port >= PortStart)
-    {
-        /*
-         * Try find range.
-         */
-        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeR3, Port);
-        if (pRange)
-        {
-            if (   pRange->Core.Key     == Port
-                && pRange->Core.KeyLast <= PortLast)
-            {
-                /*
-                 * Kick out the entire range.
-                 */
-                void *pv = RTAvlroIOPortRemove(&pVM->iom.s.pTreesR3->IOPortTreeR3, Port);
-                Assert(pv == (void *)pRange); NOREF(pv);
-                Port += pRange->cPorts;
-                MMHyperFree(pVM, pRange);
-            }
-            else if (pRange->Core.Key == Port)
-            {
-                /*
-                 * Cut of the head of the range, done.
-                 */
-                pRange->cPorts  -= Port - pRange->Port;
-                pRange->Core.Key = Port;
-                pRange->Port     = Port;
-                break;
-            }
-            else if (pRange->Core.KeyLast <= PortLast)
-            {
-                /*
-                 * Just cut of the tail.
-                 */
-                unsigned c = pRange->Core.KeyLast - Port + 1;
-                pRange->Core.KeyLast -= c;
-                pRange->cPorts -= c;
-                Port += c;
-            }
-            else
-            {
-                /*
-                 * Split the range, done.
-                 */
-                Assert(pRange->Core.KeyLast > PortLast && pRange->Core.Key < Port);
-                /* create tail. */
-                PIOMIOPORTRANGER3 pRangeNew;
-                int rc2 = MMHyperAlloc(pVM, sizeof(*pRangeNew), 0, MM_TAG_IOM, (void **)&pRangeNew);
-                if (RT_FAILURE(rc2))
-                {
-                    iomUnlock(pVM);
-                    return rc2;
-                }
-                *pRangeNew = *pRange;
-                pRangeNew->Core.Key     = PortLast;
-                pRangeNew->Port         = PortLast;
-                pRangeNew->cPorts       = pRangeNew->Core.KeyLast - PortLast + 1;
-
-                LogFlow(("IOMR3IOPortDeregister (r3): split the range; new %x\n", pRangeNew->Core.Key));
-
-                /* adjust head */
-                pRange->Core.KeyLast  = Port - 1;
-                pRange->cPorts        = Port - pRange->Port;
-
-                /* insert */
-                if (!RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeR3, &pRangeNew->Core))
-                {
-                    AssertMsgFailed(("This cannot happen!\n"));
-                    MMHyperFree(pVM, pRangeNew);
-                    rc = VERR_INTERNAL_ERROR;
-                }
-                break;
-            }
-        }
-        else /* next port */
-            Port++;
-    } /* for all ports - ring-3. */
-
-    /* done */
-    iomUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Dummy Port I/O Handler for IN operations.
- *
- * @returns VBox status code.
- *
- * @param   pDevIns     The device instance.
- * @param   pvUser      User argument.
- * @param   Port        Port number used for the IN operation.
- * @param   pu32        Where to store the result.
- * @param   cb          Number of bytes read.
- */
-static DECLCALLBACK(int) iomR3IOPortDummyIn(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t *pu32, unsigned cb)
-{
-    switch (cb)
-    {
-        case 1: *pu32 = 0xff; break;
-        case 2: *pu32 = 0xffff; break;
-        case 4: *pu32 = UINT32_C(0xffffffff); break;
-        default:
-            AssertReleaseMsgFailed(("cb=%d\n", cb));
-            return VERR_INTERNAL_ERROR;
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dummy Port I/O Handler for string IN operations.
- *
- * @returns VBox status code.
- *
- * @param   pDevIns     The device instance.
- * @param   pvUser      User argument.
- * @param   Port        Port number used for the string IN operation.
- * @param   pGCPtrDst   Pointer to the destination buffer (GC, incremented appropriately).
- * @param   pcTransfer  Pointer to the number of transfer units to read, on return remaining transfer units.
- * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
- */
-static DECLCALLBACK(int) iomR3IOPortDummyInStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrDst, PRTGCUINTREG pcTransfer, unsigned cb)
-{
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dummy Port I/O Handler for OUT operations.
- *
- * @returns VBox status code.
- *
- * @param   pDevIns     The device instance.
- * @param   pvUser      User argument.
- * @param   Port        Port number used for the OUT operation.
- * @param   u32         The value to output.
- * @param   cb          The value size in bytes.
- */
-static DECLCALLBACK(int) iomR3IOPortDummyOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t u32, unsigned cb)
-{
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dummy Port I/O Handler for string OUT operations.
- *
- * @returns VBox status code.
- *
- * @param   pDevIns     The device instance.
- * @param   pvUser      User argument.
- * @param   Port        Port number used for the string OUT operation.
- * @param   pGCPtrSrc   Pointer to the source buffer (GC, incremented appropriately).
- * @param   pcTransfer  Pointer to the number of transfer units to write, on return remaining transfer units.
- * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
- */
-static DECLCALLBACK(int) iomR3IOPortDummyOutStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrSrc, PRTGCUINTREG pcTransfer, unsigned cb)
-{
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Display a single I/O port ring-3 range.
- *
- * @returns 0
- * @param   pNode   Pointer to I/O port HC range.
- * @param   pvUser  Pointer to info output callback structure.
- */
-static DECLCALLBACK(int) iomR3IOPortInfoOneR3(PAVLROIOPORTNODECORE pNode, void *pvUser)
-{
-    PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)pNode;
-    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
-    pHlp->pfnPrintf(pHlp,
-                    "%04x-%04x %p %p %p %p %s\n",
-                    pRange->Core.Key,
-                    pRange->Core.KeyLast,
-                    pRange->pDevIns,
-                    pRange->pfnInCallback,
-                    pRange->pfnOutCallback,
-                    pRange->pvUser,
-                    pRange->pszDesc);
-    return 0;
-}
-
-
-/**
- * Display a single I/O port GC range.
- *
- * @returns 0
- * @param   pNode   Pointer to IOPORT GC range.
- * @param   pvUser  Pointer to info output callback structure.
- */
-static DECLCALLBACK(int) iomR3IOPortInfoOneRC(PAVLROIOPORTNODECORE pNode, void *pvUser)
-{
-    PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)pNode;
-    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
-    pHlp->pfnPrintf(pHlp,
-                    "%04x-%04x %RRv %RRv %RRv %RRv %s\n",
-                    pRange->Core.Key,
-                    pRange->Core.KeyLast,
-                    pRange->pDevIns,
-                    pRange->pfnInCallback,
-                    pRange->pfnOutCallback,
-                    pRange->pvUser,
-                    pRange->pszDesc);
-    return 0;
-}
-
-
-/**
- * Display all registered I/O port ranges.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) iomR3IOPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-    pHlp->pfnPrintf(pHlp,
-                    "I/O Port R3 ranges (pVM=%p)\n"
-                    "Range     %.*s %.*s %.*s %.*s Description\n",
-                    pVM,
-                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
-                    sizeof(RTHCPTR) * 2,      "In              ",
-                    sizeof(RTHCPTR) * 2,      "Out             ",
-                    sizeof(RTHCPTR) * 2,      "pvUser          ");
-    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeR3, true, iomR3IOPortInfoOneR3, (void *)pHlp);
-
-    pHlp->pfnPrintf(pHlp,
-                    "I/O Port R0 ranges (pVM=%p)\n"
-                    "Range     %.*s %.*s %.*s %.*s Description\n",
-                    pVM,
-                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
-                    sizeof(RTHCPTR) * 2,      "In              ",
-                    sizeof(RTHCPTR) * 2,      "Out             ",
-                    sizeof(RTHCPTR) * 2,      "pvUser          ");
-    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeR0, true, iomR3IOPortInfoOneR3, (void *)pHlp);
-
-    pHlp->pfnPrintf(pHlp,
-                    "I/O Port GC ranges (pVM=%p)\n"
-                    "Range     %.*s %.*s %.*s %.*s Description\n",
-                    pVM,
-                    sizeof(RTRCPTR) * 2,      "pDevIns         ",
-                    sizeof(RTRCPTR) * 2,      "In              ",
-                    sizeof(RTRCPTR) * 2,      "Out             ",
-                    sizeof(RTRCPTR) * 2,      "pvUser          ");
-    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeRC, true, iomR3IOPortInfoOneRC, (void *)pHlp);
-
-    if (pVM->iom.s.pRangeLastReadRC)
-    {
-        PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)MMHyperRCToCC(pVM, pVM->iom.s.pRangeLastReadRC);
-        pHlp->pfnPrintf(pHlp, "RC Read  Ports: %#04x-%#04x %RRv %s\n",
-                        pRange->Port, pRange->Port + pRange->cPorts, pVM->iom.s.pRangeLastReadRC, pRange->pszDesc);
-    }
-    if (pVM->iom.s.pStatsLastReadRC)
-    {
-        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperRCToCC(pVM, pVM->iom.s.pStatsLastReadRC);
-        pHlp->pfnPrintf(pHlp, "RC Read  Stats: %#04x %RRv\n",
-                        pRange->Core.Key, pVM->iom.s.pStatsLastReadRC);
-    }
-
-    if (pVM->iom.s.pRangeLastWriteRC)
-    {
-        PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)MMHyperRCToCC(pVM, pVM->iom.s.pRangeLastWriteRC);
-        pHlp->pfnPrintf(pHlp, "RC Write Ports: %#04x-%#04x %RRv %s\n",
-                        pRange->Port, pRange->Port + pRange->cPorts, pVM->iom.s.pRangeLastWriteRC, pRange->pszDesc);
-    }
-    if (pVM->iom.s.pStatsLastWriteRC)
-    {
-        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperRCToCC(pVM, pVM->iom.s.pStatsLastWriteRC);
-        pHlp->pfnPrintf(pHlp, "RC Write Stats: %#04x %RRv\n",
-                        pRange->Core.Key, pVM->iom.s.pStatsLastWriteRC);
-    }
-
-    if (pVM->iom.s.pRangeLastReadR3)
-    {
-        PIOMIOPORTRANGER3 pRange = pVM->iom.s.pRangeLastReadR3;
-        pHlp->pfnPrintf(pHlp, "R3 Read  Ports: %#04x-%#04x %p %s\n",
-                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
-    }
-    if (pVM->iom.s.pStatsLastReadR3)
-    {
-        PIOMIOPORTSTATS pRange = pVM->iom.s.pStatsLastReadR3;
-        pHlp->pfnPrintf(pHlp, "R3 Read  Stats: %#04x %p\n",
-                        pRange->Core.Key, pRange);
-    }
-
-    if (pVM->iom.s.pRangeLastWriteR3)
-    {
-        PIOMIOPORTRANGER3 pRange = pVM->iom.s.pRangeLastWriteR3;
-        pHlp->pfnPrintf(pHlp, "R3 Write Ports: %#04x-%#04x %p %s\n",
-                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
-    }
-    if (pVM->iom.s.pStatsLastWriteR3)
-    {
-        PIOMIOPORTSTATS pRange = pVM->iom.s.pStatsLastWriteR3;
-        pHlp->pfnPrintf(pHlp, "R3 Write Stats: %#04x %p\n",
-                        pRange->Core.Key, pRange);
-    }
-
-    if (pVM->iom.s.pRangeLastReadR0)
-    {
-        PIOMIOPORTRANGER0 pRange = (PIOMIOPORTRANGER0)MMHyperR0ToCC(pVM, pVM->iom.s.pRangeLastReadR0);
-        pHlp->pfnPrintf(pHlp, "R0 Read  Ports: %#04x-%#04x %p %s\n",
-                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
-    }
-    if (pVM->iom.s.pStatsLastReadR0)
-    {
-        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperR0ToCC(pVM, pVM->iom.s.pStatsLastReadR0);
-        pHlp->pfnPrintf(pHlp, "R0 Read  Stats: %#04x %p\n",
-                        pRange->Core.Key, pRange);
-    }
-
-    if (pVM->iom.s.pRangeLastWriteR0)
-    {
-        PIOMIOPORTRANGER0 pRange = (PIOMIOPORTRANGER0)MMHyperR0ToCC(pVM, pVM->iom.s.pRangeLastWriteR0);
-        pHlp->pfnPrintf(pHlp, "R0 Write Ports: %#04x-%#04x %p %s\n",
-                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
-    }
-    if (pVM->iom.s.pStatsLastWriteR0)
-    {
-        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperR0ToCC(pVM, pVM->iom.s.pStatsLastWriteR0);
-        pHlp->pfnPrintf(pHlp, "R0 Write Stats: %#04x %p\n",
-                        pRange->Core.Key, pRange);
-    }
-}
-
-
-/**
- * Registers a Memory Mapped I/O R3 handler.
- *
- * This API is called by PDM on behalf of a device. Devices must register ring-3 ranges
- * before any GC and R0 ranges can be registered using IOMR3MMIORegisterRC() and IOMR3MMIORegisterR0().
- *
- * @returns VBox status code.
- *
- * @param   pVM                 VM handle.
- * @param   pDevIns             PDM device instance owning the MMIO range.
- * @param   GCPhysStart         First physical address in the range.
- * @param   cbRange             The size of the range (in bytes).
- * @param   pvUser              User argument for the callbacks.
- * @param   pfnWriteCallback    Pointer to function which is gonna handle Write operations.
- * @param   pfnReadCallback     Pointer to function which is gonna handle Read operations.
- * @param   pfnFillCallback     Pointer to function which is gonna handle Fill/memset operations.
- * @param   pszDesc             Pointer to description string. This must not be freed.
- */
-VMMR3DECL(int)  IOMR3MMIORegisterR3(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTHCPTR pvUser,
-                                    R3PTRTYPE(PFNIOMMMIOWRITE) pfnWriteCallback, R3PTRTYPE(PFNIOMMMIOREAD) pfnReadCallback,
-                                    R3PTRTYPE(PFNIOMMMIOFILL) pfnFillCallback, const char *pszDesc)
-{
-    LogFlow(("IOMR3MMIORegisterR3: pDevIns=%p GCPhysStart=%RGp cbRange=%#x pvUser=%RHv pfnWriteCallback=%#x pfnReadCallback=%#x pfnFillCallback=%#x pszDesc=%s\n",
-             pDevIns, GCPhysStart, cbRange, pvUser, pfnWriteCallback, pfnReadCallback, pfnFillCallback, pszDesc));
-    int rc;
-
-    /*
-     * Validate input.
-     */
-    if (GCPhysStart + (cbRange - 1) < GCPhysStart)
-    {
-        AssertMsgFailed(("Wrapped! %RGp %#x bytes\n", GCPhysStart, cbRange));
-        return VERR_IOM_INVALID_MMIO_RANGE;
-    }
-    /** @todo implement per-device locks for MMIO access. */
-    AssertReturn(!pDevIns->pCritSectR3, VERR_INTERNAL_ERROR_2);
-
-    /*
-     * Resolve the GC/R0 handler addresses lazily because of init order.
-     */
-    if (pVM->iom.s.pfnMMIOHandlerR0 == NIL_RTR0PTR)
-    {
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "IOMMMIOHandler", &pVM->iom.s.pfnMMIOHandlerRC);
-        AssertLogRelRCReturn(rc, rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "IOMMMIOHandler", &pVM->iom.s.pfnMMIOHandlerR0);
-        AssertLogRelRCReturn(rc, rc);
-    }
-
-    /*
-     * For the 2nd+ instance, mangle the description string so it's unique.
-     * (PGM requires this.)
-     */
-    if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a string cache. */
-    {
-        pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_IOM, "%s [%u]", pszDesc, pDevIns->iInstance);
-        if (!pszDesc)
-            return VERR_NO_MEMORY;
-    }
-
-
-    /*
-     * Allocate new range record and initialize it.
-     */
-    PIOMMMIORANGE pRange;
-    rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
-    if (RT_SUCCESS(rc))
-    {
-        pRange->Core.Key            = GCPhysStart;
-        pRange->Core.KeyLast        = GCPhysStart + (cbRange - 1);
-        pRange->GCPhys              = GCPhysStart;
-        pRange->cb                  = cbRange;
-        pRange->pszDesc             = pszDesc;
-
-        pRange->pvUserR3            = pvUser;
-        pRange->pDevInsR3           = pDevIns;
-        pRange->pfnReadCallbackR3   = pfnReadCallback;
-        pRange->pfnWriteCallbackR3  = pfnWriteCallback;
-        pRange->pfnFillCallbackR3   = pfnFillCallback;
-
-        //pRange->pvUserR0            = NIL_RTR0PTR;
-        //pRange->pDevInsR0           = NIL_RTR0PTR;
-        //pRange->pfnReadCallbackR0   = NIL_RTR0PTR;
-        //pRange->pfnWriteCallbackR0  = NIL_RTR0PTR;
-        //pRange->pfnFillCallbackR0   = NIL_RTR0PTR;
-
-        //pRange->pvUserRC            = NIL_RTRCPTR;
-        //pRange->pDevInsRC           = NIL_RTRCPTR;
-        //pRange->pfnReadCallbackRC   = NIL_RTRCPTR;
-        //pRange->pfnWriteCallbackRC  = NIL_RTRCPTR;
-        //pRange->pfnFillCallbackRC   = NIL_RTRCPTR;
-
-        /*
-         * Try register it with PGM and then insert it into the tree.
-         */
-        iomLock(pVM);
-        iomR3FlushCache(pVM);
-        rc = PGMR3PhysMMIORegister(pVM, GCPhysStart, cbRange,
-                                   IOMR3MMIOHandler, pRange,
-                                   pVM->iom.s.pfnMMIOHandlerR0, MMHyperR3ToR0(pVM, pRange),
-                                   pVM->iom.s.pfnMMIOHandlerRC, MMHyperR3ToRC(pVM, pRange), pszDesc);
-        if (RT_SUCCESS(rc))
-        {
-            if (RTAvlroGCPhysInsert(&pVM->iom.s.pTreesR3->MMIOTree, &pRange->Core))
-            {
-                iomUnlock(pVM);
-                return VINF_SUCCESS;
-            }
-
-            /* bail out */
-            iomUnlock(pVM);
-            DBGFR3Info(pVM, "mmio", NULL, NULL);
-            AssertMsgFailed(("This cannot happen!\n"));
-            rc = VERR_INTERNAL_ERROR;
-        }
-        else
-            iomUnlock(pVM);
-
-        MMHyperFree(pVM, pRange);
-    }
-    if (pDevIns->iInstance > 0)
-        MMR3HeapFree((void *)pszDesc);
-    return rc;
-}
-
-
-/**
- * Registers a Memory Mapped I/O RC handler range.
- *
- * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
- * using IOMMMIORegisterR3() before calling this function.
- *
- *
- * @returns VBox status code.
- *
- * @param   pVM                 VM handle.
- * @param   pDevIns             PDM device instance owning the MMIO range.
- * @param   GCPhysStart         First physical address in the range.
- * @param   cbRange             The size of the range (in bytes).
- * @param   pvUser              User argument for the callbacks.
- * @param   pfnWriteCallback    Pointer to function which is gonna handle Write operations.
- * @param   pfnReadCallback     Pointer to function which is gonna handle Read operations.
- * @param   pfnFillCallback     Pointer to function which is gonna handle Fill/memset operations.
- */
-VMMR3DECL(int)  IOMR3MMIORegisterRC(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTGCPTR pvUser,
-                                    RCPTRTYPE(PFNIOMMMIOWRITE) pfnWriteCallback, RCPTRTYPE(PFNIOMMMIOREAD) pfnReadCallback,
-                                    RCPTRTYPE(PFNIOMMMIOFILL) pfnFillCallback)
-{
-    LogFlow(("IOMR3MMIORegisterRC: pDevIns=%p GCPhysStart=%RGp cbRange=%#x pvUser=%RGv pfnWriteCallback=%#x pfnReadCallback=%#x pfnFillCallback=%#x\n",
-             pDevIns, GCPhysStart, cbRange, pvUser, pfnWriteCallback, pfnReadCallback, pfnFillCallback));
-
-    /*
-     * Validate input.
-     */
-    if (!pfnWriteCallback && !pfnReadCallback)
-    {
-        AssertMsgFailed(("No callbacks! %RGp LB%#x %s\n", GCPhysStart, cbRange));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Find the MMIO range and check that the input matches.
-     */
-    iomLock(pVM);
-    PIOMMMIORANGE pRange = iomMMIOGetRange(&pVM->iom.s, GCPhysStart);
-    AssertReturnStmt(pRange, iomUnlock(pVM), VERR_IOM_MMIO_RANGE_NOT_FOUND);
-    AssertReturnStmt(pRange->pDevInsR3 == pDevIns, iomUnlock(pVM), VERR_IOM_NOT_MMIO_RANGE_OWNER);
-    AssertReturnStmt(pRange->GCPhys == GCPhysStart, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
-    AssertReturnStmt(pRange->cb == cbRange, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
-
-    pRange->pvUserRC          = pvUser;
-    pRange->pfnReadCallbackRC = pfnReadCallback;
-    pRange->pfnWriteCallbackRC= pfnWriteCallback;
-    pRange->pfnFillCallbackRC = pfnFillCallback;
-    pRange->pDevInsRC         = MMHyperCCToRC(pVM, pDevIns);
-    iomUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Registers a Memory Mapped I/O R0 handler range.
- *
- * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
- * using IOMMR3MIORegisterHC() before calling this function.
- *
- *
- * @returns VBox status code.
- *
- * @param   pVM                 VM handle.
- * @param   pDevIns             PDM device instance owning the MMIO range.
- * @param   GCPhysStart         First physical address in the range.
- * @param   cbRange             The size of the range (in bytes).
- * @param   pvUser              User argument for the callbacks.
- * @param   pfnWriteCallback    Pointer to function which is gonna handle Write operations.
- * @param   pfnReadCallback     Pointer to function which is gonna handle Read operations.
- * @param   pfnFillCallback     Pointer to function which is gonna handle Fill/memset operations.
- */
-VMMR3DECL(int)  IOMR3MMIORegisterR0(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTR0PTR pvUser,
-                                    R0PTRTYPE(PFNIOMMMIOWRITE) pfnWriteCallback,
-                                    R0PTRTYPE(PFNIOMMMIOREAD) pfnReadCallback,
-                                    R0PTRTYPE(PFNIOMMMIOFILL) pfnFillCallback)
-{
-    LogFlow(("IOMR3MMIORegisterR0: pDevIns=%p GCPhysStart=%RGp cbRange=%#x pvUser=%RHv pfnWriteCallback=%#x pfnReadCallback=%#x pfnFillCallback=%#x\n",
-             pDevIns, GCPhysStart, cbRange, pvUser, pfnWriteCallback, pfnReadCallback, pfnFillCallback));
-
-    /*
-     * Validate input.
-     */
-    if (!pfnWriteCallback && !pfnReadCallback)
-    {
-        AssertMsgFailed(("No callbacks! %RGp LB%#x %s\n", GCPhysStart, cbRange));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Find the MMIO range and check that the input matches.
-     */
-    iomLock(pVM);
-    PIOMMMIORANGE pRange = iomMMIOGetRange(&pVM->iom.s, GCPhysStart);
-    AssertReturnStmt(pRange, iomUnlock(pVM), VERR_IOM_MMIO_RANGE_NOT_FOUND);
-    AssertReturnStmt(pRange->pDevInsR3 == pDevIns, iomUnlock(pVM), VERR_IOM_NOT_MMIO_RANGE_OWNER);
-    AssertReturnStmt(pRange->GCPhys == GCPhysStart, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
-    AssertReturnStmt(pRange->cb == cbRange, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
-
-    pRange->pvUserR0          = pvUser;
-    pRange->pfnReadCallbackR0 = pfnReadCallback;
-    pRange->pfnWriteCallbackR0= pfnWriteCallback;
-    pRange->pfnFillCallbackR0 = pfnFillCallback;
-    pRange->pDevInsR0         = MMHyperCCToR0(pVM, pDevIns);
-    iomUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Deregisters a Memory Mapped I/O handler range.
- *
- * Registered GC, R0, and R3 ranges are affected.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The virtual machine.
- * @param   pDevIns             Device instance which the MMIO region is registered.
- * @param   GCPhysStart         First physical address (GC) in the range.
- * @param   cbRange             Number of bytes to deregister.
- *
- * @remark  This function mainly for PCI PnP Config and will not do
- *          all the checks you might expect it to do.
- */
-VMMR3DECL(int)  IOMR3MMIODeregister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange)
-{
-    LogFlow(("IOMR3MMIODeregister: pDevIns=%p GCPhysStart=%RGp cbRange=%#x\n", pDevIns, GCPhysStart, cbRange));
-
-    /*
-     * Validate input.
-     */
-    RTGCPHYS GCPhysLast = GCPhysStart + (cbRange - 1);
-    if (GCPhysLast < GCPhysStart)
-    {
-        AssertMsgFailed(("Wrapped! %#x LB%#x\n", GCPhysStart, cbRange));
-        return VERR_IOM_INVALID_MMIO_RANGE;
-    }
-
-    iomLock(pVM);
-
-    /*
-     * Check ownership and such for the entire area.
-     */
-    RTGCPHYS GCPhys = GCPhysStart;
-    while (GCPhys <= GCPhysLast && GCPhys >= GCPhysStart)
-    {
-        PIOMMMIORANGE pRange = iomMMIOGetRange(&pVM->iom.s, GCPhys);
-        if (!pRange)
-        {
-            iomUnlock(pVM);
-            return VERR_IOM_MMIO_RANGE_NOT_FOUND;
-        }
-        AssertMsgReturnStmt(pRange->pDevInsR3 == pDevIns,
-                            ("Not owner! GCPhys=%RGp %RGp LB%#x %s\n", GCPhys, GCPhysStart, cbRange, pRange->pszDesc),
-                            iomUnlock(pVM),
-                            VERR_IOM_NOT_MMIO_RANGE_OWNER);
-        AssertMsgReturnStmt(pRange->Core.KeyLast <= GCPhysLast,
-                            ("Incomplete R3 range! GCPhys=%RGp %RGp LB%#x %s\n", GCPhys, GCPhysStart, cbRange, pRange->pszDesc),
-                            iomUnlock(pVM),
-                            VERR_IOM_INCOMPLETE_MMIO_RANGE);
-
-        /* next */
-        Assert(GCPhys <= pRange->Core.KeyLast);
-        GCPhys = pRange->Core.KeyLast + 1;
-    }
-
-    /*
-     * Do the actual removing of the MMIO ranges.
-     */
-    GCPhys = GCPhysStart;
-    while (GCPhys <= GCPhysLast && GCPhys >= GCPhysStart)
-    {
-        iomR3FlushCache(pVM);
-
-        PIOMMMIORANGE pRange = (PIOMMMIORANGE)RTAvlroGCPhysRemove(&pVM->iom.s.pTreesR3->MMIOTree, GCPhys);
-        Assert(pRange);
-        Assert(pRange->Core.Key == GCPhys && pRange->Core.KeyLast <= GCPhysLast);
-        iomUnlock(pVM);
-
-        /* remove it from PGM */
-        int rc = PGMR3PhysMMIODeregister(pVM, GCPhys, pRange->cb);
-        AssertRC(rc);
-
-        iomLock(pVM);
-
-        /* advance and free. */
-        GCPhys = pRange->Core.KeyLast + 1;
-        if (pDevIns->iInstance > 0)
-            MMR3HeapFree((void *)pRange->pszDesc);
-        MMHyperFree(pVM, pRange);
-    }
-
-    iomUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * For TM only!
- *
- * @returns Pointer to the critical section.
- * @param   pVM                 The VM handle.
- */
-VMMR3DECL(PPDMCRITSECT) IOMR3GetCritSect(PVM pVM)
-{
-    return &pVM->iom.s.EmtLock;
-}
-
-
-/**
- * Display a single MMIO range.
- *
- * @returns 0
- * @param   pNode   Pointer to MMIO R3 range.
- * @param   pvUser  Pointer to info output callback structure.
- */
-static DECLCALLBACK(int) iomR3MMIOInfoOne(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pNode;
-    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
-    pHlp->pfnPrintf(pHlp,
-                    "%RGp-%RGp %RHv %RHv %RHv %RHv %RHv %s\n",
-                    pRange->Core.Key,
-                    pRange->Core.KeyLast,
-                    pRange->pDevInsR3,
-                    pRange->pfnReadCallbackR3,
-                    pRange->pfnWriteCallbackR3,
-                    pRange->pfnFillCallbackR3,
-                    pRange->pvUserR3,
-                    pRange->pszDesc);
-    pHlp->pfnPrintf(pHlp,
-                    "%*s %RHv %RHv %RHv %RHv %RHv\n",
-                    sizeof(RTGCPHYS) * 2 * 2 + 1, "R0",
-                    pRange->pDevInsR0,
-                    pRange->pfnReadCallbackR0,
-                    pRange->pfnWriteCallbackR0,
-                    pRange->pfnFillCallbackR0,
-                    pRange->pvUserR0);
-    pHlp->pfnPrintf(pHlp,
-                    "%*s %RRv %RRv %RRv %RRv %RRv\n",
-                    sizeof(RTGCPHYS) * 2 * 2 + 1, "RC",
-                    pRange->pDevInsRC,
-                    pRange->pfnReadCallbackRC,
-                    pRange->pfnWriteCallbackRC,
-                    pRange->pfnFillCallbackRC,
-                    pRange->pvUserRC);
-    return 0;
-}
-
-
-/**
- * Display registered MMIO ranges to the log.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-    pHlp->pfnPrintf(pHlp,
-                    "MMIO ranges (pVM=%p)\n"
-                    "%.*s %.*s %.*s %.*s %.*s %.*s %s\n",
-                    pVM,
-                    sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range                    ",
-                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
-                    sizeof(RTHCPTR) * 2,      "Read            ",
-                    sizeof(RTHCPTR) * 2,      "Write           ",
-                    sizeof(RTHCPTR) * 2,      "Fill            ",
-                    sizeof(RTHCPTR) * 2,      "pvUser          ",
-                                              "Description");
-    RTAvlroGCPhysDoWithAll(&pVM->iom.s.pTreesR3->MMIOTree, true, iomR3MMIOInfoOne, (void *)pHlp);
-}
-
-
-#ifdef VBOX_WITH_STATISTICS
-/**
- * Tries to come up with the standard name for a port.
- *
- * @returns Pointer to readonly string if known.
- * @returns NULL if unknown port number.
- *
- * @param   Port    The port to name.
- */
-static const char *iomR3IOPortGetStandardName(RTIOPORT Port)
-{
-    switch (Port)
-    {
-        case 0x00: case 0x10: case 0x20: case 0x30: case 0x40: case 0x50:            case 0x70:
-        case 0x01: case 0x11: case 0x21: case 0x31: case 0x41: case 0x51: case 0x61: case 0x71:
-        case 0x02: case 0x12: case 0x22: case 0x32: case 0x42: case 0x52: case 0x62: case 0x72:
-        case 0x03: case 0x13: case 0x23: case 0x33: case 0x43: case 0x53: case 0x63: case 0x73:
-        case 0x04: case 0x14: case 0x24: case 0x34: case 0x44: case 0x54:            case 0x74:
-        case 0x05: case 0x15: case 0x25: case 0x35: case 0x45: case 0x55: case 0x65: case 0x75:
-        case 0x06: case 0x16: case 0x26: case 0x36: case 0x46: case 0x56: case 0x66: case 0x76:
-        case 0x07: case 0x17: case 0x27: case 0x37: case 0x47: case 0x57: case 0x67: case 0x77:
-        case 0x08: case 0x18: case 0x28: case 0x38: case 0x48: case 0x58: case 0x68: case 0x78:
-        case 0x09: case 0x19: case 0x29: case 0x39: case 0x49: case 0x59: case 0x69: case 0x79:
-        case 0x0a: case 0x1a: case 0x2a: case 0x3a: case 0x4a: case 0x5a: case 0x6a: case 0x7a:
-        case 0x0b: case 0x1b: case 0x2b: case 0x3b: case 0x4b: case 0x5b: case 0x6b: case 0x7b:
-        case 0x0c: case 0x1c: case 0x2c: case 0x3c: case 0x4c: case 0x5c: case 0x6c: case 0x7c:
-        case 0x0d: case 0x1d: case 0x2d: case 0x3d: case 0x4d: case 0x5d: case 0x6d: case 0x7d:
-        case 0x0e: case 0x1e: case 0x2e: case 0x3e: case 0x4e: case 0x5e: case 0x6e: case 0x7e:
-        case 0x0f: case 0x1f: case 0x2f: case 0x3f: case 0x4f: case 0x5f: case 0x6f: case 0x7f:
-
-        case 0x80: case 0x90: case 0xa0: case 0xb0: case 0xc0: case 0xd0: case 0xe0: case 0xf0:
-        case 0x81: case 0x91: case 0xa1: case 0xb1: case 0xc1: case 0xd1: case 0xe1: case 0xf1:
-        case 0x82: case 0x92: case 0xa2: case 0xb2: case 0xc2: case 0xd2: case 0xe2: case 0xf2:
-        case 0x83: case 0x93: case 0xa3: case 0xb3: case 0xc3: case 0xd3: case 0xe3: case 0xf3:
-        case 0x84: case 0x94: case 0xa4: case 0xb4: case 0xc4: case 0xd4: case 0xe4: case 0xf4:
-        case 0x85: case 0x95: case 0xa5: case 0xb5: case 0xc5: case 0xd5: case 0xe5: case 0xf5:
-        case 0x86: case 0x96: case 0xa6: case 0xb6: case 0xc6: case 0xd6: case 0xe6: case 0xf6:
-        case 0x87: case 0x97: case 0xa7: case 0xb7: case 0xc7: case 0xd7: case 0xe7: case 0xf7:
-        case 0x88: case 0x98: case 0xa8: case 0xb8: case 0xc8: case 0xd8: case 0xe8: case 0xf8:
-        case 0x89: case 0x99: case 0xa9: case 0xb9: case 0xc9: case 0xd9: case 0xe9: case 0xf9:
-        case 0x8a: case 0x9a: case 0xaa: case 0xba: case 0xca: case 0xda: case 0xea: case 0xfa:
-        case 0x8b: case 0x9b: case 0xab: case 0xbb: case 0xcb: case 0xdb: case 0xeb: case 0xfb:
-        case 0x8c: case 0x9c: case 0xac: case 0xbc: case 0xcc: case 0xdc: case 0xec: case 0xfc:
-        case 0x8d: case 0x9d: case 0xad: case 0xbd: case 0xcd: case 0xdd: case 0xed: case 0xfd:
-        case 0x8e: case 0x9e: case 0xae: case 0xbe: case 0xce: case 0xde: case 0xee: case 0xfe:
-        case 0x8f: case 0x9f: case 0xaf: case 0xbf: case 0xcf: case 0xdf: case 0xef: case 0xff:
-            return "System Reserved";
-
-        case 0x60:
-        case 0x64:
-            return "Keyboard & Mouse";
-
-        case 0x378:
-        case 0x379:
-        case 0x37a:
-        case 0x37b:
-        case 0x37c:
-        case 0x37d:
-        case 0x37e:
-        case 0x37f:
-        case 0x3bc:
-        case 0x3bd:
-        case 0x3be:
-        case 0x3bf:
-        case 0x278:
-        case 0x279:
-        case 0x27a:
-        case 0x27b:
-        case 0x27c:
-        case 0x27d:
-        case 0x27e:
-        case 0x27f:
-            return "LPT1/2/3";
-
-        case 0x3f8:
-        case 0x3f9:
-        case 0x3fa:
-        case 0x3fb:
-        case 0x3fc:
-        case 0x3fd:
-        case 0x3fe:
-        case 0x3ff:
-            return "COM1";
-
-        case 0x2f8:
-        case 0x2f9:
-        case 0x2fa:
-        case 0x2fb:
-        case 0x2fc:
-        case 0x2fd:
-        case 0x2fe:
-        case 0x2ff:
-            return "COM2";
-
-        case 0x3e8:
-        case 0x3e9:
-        case 0x3ea:
-        case 0x3eb:
-        case 0x3ec:
-        case 0x3ed:
-        case 0x3ee:
-        case 0x3ef:
-            return "COM3";
-
-        case 0x2e8:
-        case 0x2e9:
-        case 0x2ea:
-        case 0x2eb:
-        case 0x2ec:
-        case 0x2ed:
-        case 0x2ee:
-        case 0x2ef:
-            return "COM4";
-
-        case 0x200:
-        case 0x201:
-        case 0x202:
-        case 0x203:
-        case 0x204:
-        case 0x205:
-        case 0x206:
-        case 0x207:
-            return "Joystick";
-
-        case 0x3f0:
-        case 0x3f1:
-        case 0x3f2:
-        case 0x3f3:
-        case 0x3f4:
-        case 0x3f5:
-        case 0x3f6:
-        case 0x3f7:
-            return "Floppy";
-
-        case 0x1f0:
-        case 0x1f1:
-        case 0x1f2:
-        case 0x1f3:
-        case 0x1f4:
-        case 0x1f5:
-        case 0x1f6:
-        case 0x1f7:
-        //case 0x3f6:
-        //case 0x3f7:
-            return "IDE 1st";
-
-        case 0x170:
-        case 0x171:
-        case 0x172:
-        case 0x173:
-        case 0x174:
-        case 0x175:
-        case 0x176:
-        case 0x177:
-        case 0x376:
-        case 0x377:
-            return "IDE 2nd";
-
-        case 0x1e0:
-        case 0x1e1:
-        case 0x1e2:
-        case 0x1e3:
-        case 0x1e4:
-        case 0x1e5:
-        case 0x1e6:
-        case 0x1e7:
-        case 0x3e6:
-        case 0x3e7:
-            return "IDE 3rd";
-
-        case 0x160:
-        case 0x161:
-        case 0x162:
-        case 0x163:
-        case 0x164:
-        case 0x165:
-        case 0x166:
-        case 0x167:
-        case 0x366:
-        case 0x367:
-            return "IDE 4th";
-
-        case 0x130: case 0x140: case 0x150:
-        case 0x131: case 0x141: case 0x151:
-        case 0x132: case 0x142: case 0x152:
-        case 0x133: case 0x143: case 0x153:
-        case 0x134: case 0x144: case 0x154:
-        case 0x135: case 0x145: case 0x155:
-        case 0x136: case 0x146: case 0x156:
-        case 0x137: case 0x147: case 0x157:
-        case 0x138: case 0x148: case 0x158:
-        case 0x139: case 0x149: case 0x159:
-        case 0x13a: case 0x14a: case 0x15a:
-        case 0x13b: case 0x14b: case 0x15b:
-        case 0x13c: case 0x14c: case 0x15c:
-        case 0x13d: case 0x14d: case 0x15d:
-        case 0x13e: case 0x14e: case 0x15e:
-        case 0x13f: case 0x14f: case 0x15f:
-        case 0x220: case 0x230:
-        case 0x221: case 0x231:
-        case 0x222: case 0x232:
-        case 0x223: case 0x233:
-        case 0x224: case 0x234:
-        case 0x225: case 0x235:
-        case 0x226: case 0x236:
-        case 0x227: case 0x237:
-        case 0x228: case 0x238:
-        case 0x229: case 0x239:
-        case 0x22a: case 0x23a:
-        case 0x22b: case 0x23b:
-        case 0x22c: case 0x23c:
-        case 0x22d: case 0x23d:
-        case 0x22e: case 0x23e:
-        case 0x22f: case 0x23f:
-        case 0x330: case 0x340: case 0x350:
-        case 0x331: case 0x341: case 0x351:
-        case 0x332: case 0x342: case 0x352:
-        case 0x333: case 0x343: case 0x353:
-        case 0x334: case 0x344: case 0x354:
-        case 0x335: case 0x345: case 0x355:
-        case 0x336: case 0x346: case 0x356:
-        case 0x337: case 0x347: case 0x357:
-        case 0x338: case 0x348: case 0x358:
-        case 0x339: case 0x349: case 0x359:
-        case 0x33a: case 0x34a: case 0x35a:
-        case 0x33b: case 0x34b: case 0x35b:
-        case 0x33c: case 0x34c: case 0x35c:
-        case 0x33d: case 0x34d: case 0x35d:
-        case 0x33e: case 0x34e: case 0x35e:
-        case 0x33f: case 0x34f: case 0x35f:
-            return "SCSI (typically)";
-
-        case 0x320:
-        case 0x321:
-        case 0x322:
-        case 0x323:
-        case 0x324:
-        case 0x325:
-        case 0x326:
-        case 0x327:
-            return "XT HD";
-
-        case 0x3b0:
-        case 0x3b1:
-        case 0x3b2:
-        case 0x3b3:
-        case 0x3b4:
-        case 0x3b5:
-        case 0x3b6:
-        case 0x3b7:
-        case 0x3b8:
-        case 0x3b9:
-        case 0x3ba:
-        case 0x3bb:
-            return "VGA";
-
-        case 0x3c0: case 0x3d0:
-        case 0x3c1: case 0x3d1:
-        case 0x3c2: case 0x3d2:
-        case 0x3c3: case 0x3d3:
-        case 0x3c4: case 0x3d4:
-        case 0x3c5: case 0x3d5:
-        case 0x3c6: case 0x3d6:
-        case 0x3c7: case 0x3d7:
-        case 0x3c8: case 0x3d8:
-        case 0x3c9: case 0x3d9:
-        case 0x3ca: case 0x3da:
-        case 0x3cb: case 0x3db:
-        case 0x3cc: case 0x3dc:
-        case 0x3cd: case 0x3dd:
-        case 0x3ce: case 0x3de:
-        case 0x3cf: case 0x3df:
-            return "VGA/EGA";
-
-        case 0x240: case 0x260: case 0x280:
-        case 0x241: case 0x261: case 0x281:
-        case 0x242: case 0x262: case 0x282:
-        case 0x243: case 0x263: case 0x283:
-        case 0x244: case 0x264: case 0x284:
-        case 0x245: case 0x265: case 0x285:
-        case 0x246: case 0x266: case 0x286:
-        case 0x247: case 0x267: case 0x287:
-        case 0x248: case 0x268: case 0x288:
-        case 0x249: case 0x269: case 0x289:
-        case 0x24a: case 0x26a: case 0x28a:
-        case 0x24b: case 0x26b: case 0x28b:
-        case 0x24c: case 0x26c: case 0x28c:
-        case 0x24d: case 0x26d: case 0x28d:
-        case 0x24e: case 0x26e: case 0x28e:
-        case 0x24f: case 0x26f: case 0x28f:
-        case 0x300:
-        case 0x301:
-        case 0x388:
-        case 0x389:
-        case 0x38a:
-        case 0x38b:
-            return "Sound Card (typically)";
-
-        default:
-            return NULL;
-    }
-}
-#endif /* VBOX_WITH_STATISTICS */
-
Index: unk/src/VBox/VMM/IOMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/IOMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,567 +1,0 @@
-/* $Id$ */
-/** @file
- * IOM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___IOMInternal_h
-#define ___IOMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/iom.h>
-#include <VBox/stam.h>
-#include <VBox/pgm.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/param.h>
-#include <iprt/assert.h>
-#include <iprt/avl.h>
-
-
-
-/** @defgroup grp_iom_int   Internals
- * @ingroup grp_iom
- * @internal
- * @{
- */
-
-/**
- * MMIO range descriptor.
- */
-typedef struct IOMMMIORANGE
-{
-    /** Avl node core with GCPhys as Key and GCPhys + cbSize - 1 as KeyLast. */
-    AVLROGCPHYSNODECORE         Core;
-    /** Start physical address. */
-    RTGCPHYS                    GCPhys;
-    /** Size of the range. */
-    uint32_t                    cb;
-    uint32_t                    u32Alignment; /**< Alignment padding. */
-
-    /** Pointer to user argument - R3. */
-    RTR3PTR                     pvUserR3;
-    /** Pointer to device instance - R3. */
-    PPDMDEVINSR3                pDevInsR3;
-    /** Pointer to write callback function - R3. */
-    R3PTRTYPE(PFNIOMMMIOWRITE)  pfnWriteCallbackR3;
-    /** Pointer to read callback function - R3. */
-    R3PTRTYPE(PFNIOMMMIOREAD)   pfnReadCallbackR3;
-    /** Pointer to fill (memset) callback function - R3. */
-    R3PTRTYPE(PFNIOMMMIOFILL)   pfnFillCallbackR3;
-
-    /** Pointer to user argument - R0. */
-    RTR0PTR                     pvUserR0;
-    /** Pointer to device instance - R0. */
-    PPDMDEVINSR0                pDevInsR0;
-    /** Pointer to write callback function - R0. */
-    R0PTRTYPE(PFNIOMMMIOWRITE)  pfnWriteCallbackR0;
-    /** Pointer to read callback function - R0. */
-    R0PTRTYPE(PFNIOMMMIOREAD)   pfnReadCallbackR0;
-    /** Pointer to fill (memset) callback function - R0. */
-    R0PTRTYPE(PFNIOMMMIOFILL)   pfnFillCallbackR0;
-
-    /** Pointer to user argument - RC. */
-    RTRCPTR                     pvUserRC;
-    /** Pointer to device instance - RC. */
-    PPDMDEVINSRC                pDevInsRC;
-    /** Pointer to write callback function - RC. */
-    RCPTRTYPE(PFNIOMMMIOWRITE)  pfnWriteCallbackRC;
-    /** Pointer to read callback function - RC. */
-    RCPTRTYPE(PFNIOMMMIOREAD)   pfnReadCallbackRC;
-    /** Pointer to fill (memset) callback function - RC. */
-    RCPTRTYPE(PFNIOMMMIOFILL)   pfnFillCallbackRC;
-    /** Alignment padding. */
-    RTRCPTR                     RCPtrAlignment;
-
-    /** Description / Name. For easing debugging. */
-    R3PTRTYPE(const char *)     pszDesc;
-} IOMMMIORANGE;
-/** Pointer to a MMIO range descriptor, R3 version. */
-typedef struct IOMMMIORANGE *PIOMMMIORANGE;
-
-
-/**
- * MMIO address statistics. (one address)
- *
- * This is a simple way of making on demand statistics, however it's a
- * bit free with the hypervisor heap memory.
- */
-typedef struct IOMMMIOSTATS
-{
-    /** Avl node core with the address as Key. */
-    AVLOGCPHYSNODECORE          Core;
-
-    /** Number of accesses (subtract ReadRZToR3 and WriteRZToR3 to get the right
-     *  number). */
-    STAMCOUNTER                 Accesses;
-
-    /** Profiling read handler overhead in R3. */
-    STAMPROFILE                 ProfReadR3;
-    /** Profiling write handler overhead in R3. */
-    STAMPROFILE                 ProfWriteR3;
-    /** Counting and profiling reads in R0/RC. */
-    STAMPROFILE                 ProfReadRZ;
-    /** Counting and profiling writes in R0/RC. */
-    STAMPROFILE                 ProfWriteRZ;
-
-    /** Number of reads to this address from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 ReadRZToR3;
-    /** Number of writes to this address from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 WriteRZToR3;
-} IOMMMIOSTATS;
-AssertCompileMemberAlignment(IOMMMIOSTATS, Accesses, 8);
-/** Pointer to I/O port statistics. */
-typedef IOMMMIOSTATS *PIOMMMIOSTATS;
-
-
-/**
- * I/O port range descriptor, R3 version.
- */
-typedef struct IOMIOPORTRANGER3
-{
-    /** Avl node core with Port as Key and Port + cPorts - 1 as KeyLast. */
-    AVLROIOPORTNODECORE         Core;
-#if HC_ARCH_BITS == 64 && !defined(RT_OS_WINDOWS)
-    uint32_t                    u32Alignment; /**< The sizeof(Core) differs. */
-#endif
-    /** Start I/O port address. */
-    RTIOPORT                    Port;
-    /** Size of the range. */
-    uint16_t                    cPorts;
-    /** Pointer to user argument. */
-    RTR3PTR                     pvUser;
-    /** Pointer to the associated device instance. */
-    R3PTRTYPE(PPDMDEVINS)       pDevIns;
-    /** Pointer to OUT callback function. */
-    R3PTRTYPE(PFNIOMIOPORTOUT)  pfnOutCallback;
-    /** Pointer to IN callback function. */
-    R3PTRTYPE(PFNIOMIOPORTIN)   pfnInCallback;
-    /** Pointer to string OUT callback function. */
-    R3PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback;
-    /** Pointer to string IN callback function. */
-    R3PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback;
-    /** Description / Name. For easing debugging. */
-    R3PTRTYPE(const char *)     pszDesc;
-} IOMIOPORTRANGER3;
-/** Pointer to I/O port range descriptor, R3 version. */
-typedef IOMIOPORTRANGER3 *PIOMIOPORTRANGER3;
-
-/**
- * I/O port range descriptor, R0 version.
- */
-typedef struct IOMIOPORTRANGER0
-{
-    /** Avl node core with Port as Key and Port + cPorts - 1 as KeyLast. */
-    AVLROIOPORTNODECORE         Core;
-#if HC_ARCH_BITS == 64 && !defined(RT_OS_WINDOWS)
-    uint32_t                    u32Alignment; /**< The sizeof(Core) differs. */
-#endif
-    /** Start I/O port address. */
-    RTIOPORT                    Port;
-    /** Size of the range. */
-    uint16_t                    cPorts;
-    /** Pointer to user argument. */
-    RTR0PTR                     pvUser;
-    /** Pointer to the associated device instance. */
-    R0PTRTYPE(PPDMDEVINS)       pDevIns;
-    /** Pointer to OUT callback function. */
-    R0PTRTYPE(PFNIOMIOPORTOUT)  pfnOutCallback;
-    /** Pointer to IN callback function. */
-    R0PTRTYPE(PFNIOMIOPORTIN)   pfnInCallback;
-    /** Pointer to string OUT callback function. */
-    R0PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback;
-    /** Pointer to string IN callback function. */
-    R0PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback;
-    /** Description / Name. For easing debugging. */
-    R3PTRTYPE(const char *)     pszDesc;
-} IOMIOPORTRANGER0;
-/** Pointer to I/O port range descriptor, R0 version. */
-typedef IOMIOPORTRANGER0 *PIOMIOPORTRANGER0;
-
-/**
- * I/O port range descriptor, RC version.
- */
-typedef struct IOMIOPORTRANGERC
-{
-    /** Avl node core with Port as Key and Port + cPorts - 1 as KeyLast. */
-    AVLROIOPORTNODECORE         Core;
-    /** Start I/O port address. */
-    RTIOPORT                    Port;
-    /** Size of the range. */
-    uint16_t                    cPorts;
-    /** Pointer to user argument. */
-    RTRCPTR                     pvUser;
-    /** Pointer to the associated device instance. */
-    RCPTRTYPE(PPDMDEVINS)       pDevIns;
-    /** Pointer to OUT callback function. */
-    RCPTRTYPE(PFNIOMIOPORTOUT)  pfnOutCallback;
-    /** Pointer to IN callback function. */
-    RCPTRTYPE(PFNIOMIOPORTIN)   pfnInCallback;
-    /** Pointer to string OUT callback function. */
-    RCPTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback;
-    /** Pointer to string IN callback function. */
-    RCPTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback;
-#if HC_ARCH_BITS == 64
-    RTRCPTR                     RCPtrAlignment; /**< pszDesc is 8 byte aligned. */
-#endif
-    /** Description / Name. For easing debugging. */
-    R3PTRTYPE(const char *)     pszDesc;
-} IOMIOPORTRANGERC;
-/** Pointer to I/O port range descriptor, RC version. */
-typedef IOMIOPORTRANGERC *PIOMIOPORTRANGERC;
-
-
-/**
- * I/O port statistics. (one I/O port)
- *
- * This is a simple way of making on demand statistics, however it's a
- * bit free with the hypervisor heap memory.
- */
-typedef struct IOMIOPORTSTATS
-{
-    /** Avl node core with the port as Key. */
-    AVLOIOPORTNODECORE          Core;
-#if HC_ARCH_BITS != 64 || !defined(RT_OS_WINDOWS)
-    uint32_t                    u32Alignment; /**< The sizeof(Core) differs. */
-#endif
-    /** Number of INs to this port from R3. */
-    STAMCOUNTER                 InR3;
-    /** Profiling IN handler overhead in R3. */
-    STAMPROFILE                 ProfInR3;
-    /** Number of OUTs to this port from R3. */
-    STAMCOUNTER                 OutR3;
-    /** Profiling OUT handler overhead in R3. */
-    STAMPROFILE                 ProfOutR3;
-
-    /** Number of INs to this port from R0/RC. */
-    STAMCOUNTER                 InRZ;
-    /** Profiling IN handler overhead in R0/RC. */
-    STAMPROFILE                 ProfInRZ;
-    /** Number of INs to this port from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 InRZToR3;
-
-    /** Number of OUTs to this port from R0/RC. */
-    STAMCOUNTER                 OutRZ;
-    /** Profiling OUT handler overhead in R0/RC. */
-    STAMPROFILE                 ProfOutRZ;
-    /** Number of OUTs to this port from R0/RC which was serviced in R3. */
-    STAMCOUNTER                 OutRZToR3;
-} IOMIOPORTSTATS;
-AssertCompileMemberAlignment(IOMIOPORTSTATS, InR3, 8);
-/** Pointer to I/O port statistics. */
-typedef IOMIOPORTSTATS *PIOMIOPORTSTATS;
-
-
-/**
- * The IOM trees.
- * These are offset based the nodes and root must be in the same
- * memory block in HC. The locations of IOM structure and the hypervisor heap
- * are quite different in R3, R0 and RC.
- */
-typedef struct IOMTREES
-{
-    /** Tree containing I/O port range descriptors registered for HC (IOMIOPORTRANGEHC). */
-    AVLROIOPORTTREE         IOPortTreeR3;
-    /** Tree containing I/O port range descriptors registered for R0 (IOMIOPORTRANGER0). */
-    AVLROIOPORTTREE         IOPortTreeR0;
-    /** Tree containing I/O port range descriptors registered for RC (IOMIOPORTRANGERC). */
-    AVLROIOPORTTREE         IOPortTreeRC;
-
-    /** Tree containing the MMIO range descriptors (IOMMMIORANGE). */
-    AVLROGCPHYSTREE         MMIOTree;
-
-    /** Tree containing I/O port statistics (IOMIOPORTSTATS). */
-    AVLOIOPORTTREE          IOPortStatTree;
-    /** Tree containing MMIO statistics (IOMMMIOSTATS). */
-    AVLOGCPHYSTREE          MMIOStatTree;
-} IOMTREES;
-/** Pointer to the IOM trees. */
-typedef IOMTREES *PIOMTREES;
-
-
-/**
- * Converts an IOM pointer into a VM pointer.
- * @returns Pointer to the VM structure the PGM is part of.
- * @param   pIOM   Pointer to IOM instance data.
- */
-#define IOM2VM(pIOM)  ( (PVM)((char*)pIOM - pIOM->offVM) )
-
-/**
- * IOM Data (part of VM)
- */
-typedef struct IOM
-{
-    /** Offset to the VM structure. */
-    RTINT                           offVM;
-
-    /** Pointer to the trees - RC ptr. */
-    RCPTRTYPE(PIOMTREES)            pTreesRC;
-    /** Pointer to the trees - R3 ptr. */
-    R3PTRTYPE(PIOMTREES)            pTreesR3;
-    /** Pointer to the trees - R0 ptr. */
-    R0PTRTYPE(PIOMTREES)            pTreesR0;
-
-    /** The ring-0 address of IOMMMIOHandler. */
-    R0PTRTYPE(PFNPGMR0PHYSHANDLER)  pfnMMIOHandlerR0;
-    /** The RC address of IOMMMIOHandler. */
-    RCPTRTYPE(PFNPGMRCPHYSHANDLER)  pfnMMIOHandlerRC;
-#if HC_ARCH_BITS == 64
-    RTRCPTR                         padding;
-#endif
-
-    /** Lock serializing EMT access to IOM. */
-    PDMCRITSECT                     EmtLock;
-
-    /** @name Caching of I/O Port and MMIO ranges and statistics.
-     * (Saves quite some time in rep outs/ins instruction emulation.)
-     * @{ */
-    R3PTRTYPE(PIOMIOPORTRANGER3)    pRangeLastReadR3;
-    R3PTRTYPE(PIOMIOPORTRANGER3)    pRangeLastWriteR3;
-    R3PTRTYPE(PIOMIOPORTSTATS)      pStatsLastReadR3;
-    R3PTRTYPE(PIOMIOPORTSTATS)      pStatsLastWriteR3;
-    R3PTRTYPE(PIOMMMIORANGE)        pMMIORangeLastR3;
-    R3PTRTYPE(PIOMMMIOSTATS)        pMMIOStatsLastR3;
-
-    R0PTRTYPE(PIOMIOPORTRANGER0)    pRangeLastReadR0;
-    R0PTRTYPE(PIOMIOPORTRANGER0)    pRangeLastWriteR0;
-    R0PTRTYPE(PIOMIOPORTSTATS)      pStatsLastReadR0;
-    R0PTRTYPE(PIOMIOPORTSTATS)      pStatsLastWriteR0;
-    R0PTRTYPE(PIOMMMIORANGE)        pMMIORangeLastR0;
-    R0PTRTYPE(PIOMMMIOSTATS)        pMMIOStatsLastR0;
-
-    RCPTRTYPE(PIOMIOPORTRANGERC)    pRangeLastReadRC;
-    RCPTRTYPE(PIOMIOPORTRANGERC)    pRangeLastWriteRC;
-    RCPTRTYPE(PIOMIOPORTSTATS)      pStatsLastReadRC;
-    RCPTRTYPE(PIOMIOPORTSTATS)      pStatsLastWriteRC;
-    RCPTRTYPE(PIOMMMIORANGE)        pMMIORangeLastRC;
-    RCPTRTYPE(PIOMMMIOSTATS)        pMMIOStatsLastRC;
-    /** @} */
-
-    /** @name I/O Port statistics.
-     * @{ */
-    STAMCOUNTER                     StatInstIn;
-    STAMCOUNTER                     StatInstOut;
-    STAMCOUNTER                     StatInstIns;
-    STAMCOUNTER                     StatInstOuts;
-    /** @} */
-
-    /** @name MMIO statistics.
-     * @{ */
-    STAMPROFILE                     StatRZMMIOHandler;
-    STAMCOUNTER                     StatRZMMIOFailures;
-
-    STAMPROFILE                     StatRZInstMov;
-    STAMPROFILE                     StatRZInstCmp;
-    STAMPROFILE                     StatRZInstAnd;
-    STAMPROFILE                     StatRZInstOr;
-    STAMPROFILE                     StatRZInstXor;
-    STAMPROFILE                     StatRZInstBt;
-    STAMPROFILE                     StatRZInstTest;
-    STAMPROFILE                     StatRZInstXchg;
-    STAMPROFILE                     StatRZInstStos;
-    STAMPROFILE                     StatRZInstLods;
-#ifdef IOM_WITH_MOVS_SUPPORT
-    STAMPROFILEADV                  StatRZInstMovs;
-    STAMPROFILE                     StatRZInstMovsToMMIO;
-    STAMPROFILE                     StatRZInstMovsFromMMIO;
-    STAMPROFILE                     StatRZInstMovsMMIO;
-#endif
-    STAMCOUNTER                     StatRZInstOther;
-
-    STAMCOUNTER                     StatRZMMIO1Byte;
-    STAMCOUNTER                     StatRZMMIO2Bytes;
-    STAMCOUNTER                     StatRZMMIO4Bytes;
-    STAMCOUNTER                     StatRZMMIO8Bytes;
-
-    STAMCOUNTER                     StatR3MMIOHandler;
-
-    RTUINT                          cMovsMaxBytes;
-    RTUINT                          cStosMaxBytes;
-    /** @} */
-} IOM;
-/** Pointer to IOM instance data. */
-typedef IOM *PIOM;
-
-
-/**
- * IOM per virtual CPU instance data.
- */
-typedef struct IOMCPU
-{
-    /** For saving stack space, the disassembler state is allocated here instead of
-     * on the stack.
-     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
-    union
-    {
-        /** The disassembler scratch space. */
-        DISCPUSTATE                 DisState;
-        /** Padding. */
-        uint8_t                     abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
-    };
-    uint8_t                         Dummy[16];
-} IOMCPU;
-/** Pointer to IOM per virtual CPU instance data. */
-typedef IOMCPU *PIOMCPU;
-
-
-RT_C_DECLS_BEGIN
-
-#ifdef IN_RING3
-PIOMIOPORTSTATS iomR3IOPortStatsCreate(PVM pVM, RTIOPORT Port, const char *pszDesc);
-PIOMMMIOSTATS   iomR3MMIOStatsCreate(PVM pVM, RTGCPHYS GCPhys, const char *pszDesc);
-#endif /* IN_RING3 */
-
-VMMDECL(int)        IOMMMIOHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
-#ifdef IN_RING3
-DECLCALLBACK(int)   IOMR3MMIOHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
-#endif
-
-
-/**
- * Gets the I/O port range for the specified I/O port in the current context.
- *
- * @returns Pointer to I/O port range.
- * @returns NULL if no port registered.
- *
- * @param   pIOM    IOM instance data.
- * @param   Port    Port to lookup.
- */
-DECLINLINE(CTX_SUFF(PIOMIOPORTRANGE)) iomIOPortGetRange(PIOM pIOM, RTIOPORT Port)
-{
-#ifdef IN_RING3
-    if (PDMCritSectIsInitialized(&pIOM->EmtLock))
-#endif
-        Assert(IOMIsLockOwner(IOM2VM(pIOM)));
-    CTX_SUFF(PIOMIOPORTRANGE) pRange = (CTX_SUFF(PIOMIOPORTRANGE))RTAvlroIOPortRangeGet(&pIOM->CTX_SUFF(pTrees)->CTX_SUFF(IOPortTree), Port);
-    return pRange;
-}
-
-
-/**
- * Gets the I/O port range for the specified I/O port in the HC.
- *
- * @returns Pointer to I/O port range.
- * @returns NULL if no port registered.
- *
- * @param   pIOM    IOM instance data.
- * @param   Port    Port to lookup.
- */
-DECLINLINE(PIOMIOPORTRANGER3) iomIOPortGetRangeR3(PIOM pIOM, RTIOPORT Port)
-{
-#ifdef IN_RING3
-    if (PDMCritSectIsInitialized(&pIOM->EmtLock))
-#endif
-        Assert(IOMIsLockOwner(IOM2VM(pIOM)));
-    PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pIOM->CTX_SUFF(pTrees)->IOPortTreeR3, Port);
-    return pRange;
-}
-
-
-/**
- * Gets the MMIO range for the specified physical address in the current context.
- *
- * @returns Pointer to MMIO range.
- * @returns NULL if address not in a MMIO range.
- *
- * @param   pIOM    IOM instance data.
- * @param   GCPhys  Physical address to lookup.
- */
-DECLINLINE(PIOMMMIORANGE) iomMMIOGetRange(PIOM pIOM, RTGCPHYS GCPhys)
-{
-#ifdef IN_RING3
-    if (PDMCritSectIsInitialized(&pIOM->EmtLock))
-#endif
-        Assert(IOMIsLockOwner(IOM2VM(pIOM)));
-    PIOMMMIORANGE pRange = pIOM->CTX_SUFF(pMMIORangeLast);
-    if (    !pRange
-        ||  GCPhys - pRange->GCPhys >= pRange->cb)
-        pIOM->CTX_SUFF(pMMIORangeLast) = pRange = (PIOMMMIORANGE)RTAvlroGCPhysRangeGet(&pIOM->CTX_SUFF(pTrees)->MMIOTree, GCPhys);
-    return pRange;
-}
-
-#ifdef VBOX_STRICT
-/**
- * Gets the MMIO range for the specified physical address in the current context.
- *
- * @returns Pointer to MMIO range.
- * @returns NULL if address not in a MMIO range.
- *
- * @param   pIOM    IOM instance data.
- * @param   GCPhys  Physical address to lookup.
- */
-DECLINLINE(PIOMMMIORANGE) iomMMIOGetRangeUnsafe(PIOM pIOM, RTGCPHYS GCPhys)
-{
-    PIOMMMIORANGE pRange = pIOM->CTX_SUFF(pMMIORangeLast);
-    if (    !pRange
-        ||  GCPhys - pRange->GCPhys >= pRange->cb)
-        pIOM->CTX_SUFF(pMMIORangeLast) = pRange = (PIOMMMIORANGE)RTAvlroGCPhysRangeGet(&pIOM->CTX_SUFF(pTrees)->MMIOTree, GCPhys);
-    return pRange;
-}
-#endif
-
-
-#ifdef VBOX_WITH_STATISTICS
-/**
- * Gets the MMIO statistics record.
- *
- * In ring-3 this will lazily create missing records, while in GC/R0 the caller has to
- * return the appropriate status to defer the operation to ring-3.
- *
- * @returns Pointer to MMIO stats.
- * @returns NULL if not found (R0/GC), or out of memory (R3).
- *
- * @param   pIOM        IOM instance data.
- * @param   GCPhys      Physical address to lookup.
- * @param   pRange      The MMIO range.
- */
-DECLINLINE(PIOMMMIOSTATS) iomMMIOGetStats(PIOM pIOM, RTGCPHYS GCPhys, PIOMMMIORANGE pRange)
-{
-    Assert(IOMIsLockOwner(IOM2VM(pIOM)));
-    /* For large ranges, we'll put everything on the first byte. */
-    if (pRange->cb > PAGE_SIZE)
-        GCPhys = pRange->GCPhys;
-
-    PIOMMMIOSTATS pStats = pIOM->CTX_SUFF(pMMIOStatsLast);
-    if (    !pStats
-        ||  pStats->Core.Key != GCPhys)
-    {
-        pStats = (PIOMMMIOSTATS)RTAvloGCPhysGet(&pIOM->CTX_SUFF(pTrees)->MMIOStatTree, GCPhys);
-# ifdef IN_RING3
-        if (!pStats)
-            pStats = iomR3MMIOStatsCreate(IOM2VM(pIOM), GCPhys, pRange->pszDesc);
-# endif
-    }
-    return pStats;
-}
-#endif
-
-/* IOM locking helpers. */
-int     iomLock(PVM pVM);
-int     iomTryLock(PVM pVM);
-void    iomUnlock(PVM pVM);
-
-/* Disassembly helpers used in IOMAll.cpp & IOMAllMMIO.cpp */
-bool    iomGetRegImmData(PDISCPUSTATE pCpu, PCOP_PARAMETER pParam, PCPUMCTXCORE pRegFrame, uint64_t *pu64Data, unsigned *pcbSize);
-bool    iomSaveDataToReg(PDISCPUSTATE pCpu, PCOP_PARAMETER pParam, PCPUMCTXCORE pRegFrame, uint64_t u32Data);
-
-RT_C_DECLS_END
-
-
-#ifdef IN_RING3
-
-#endif
-
-/** @} */
-
-#endif /* ___IOMInternal_h */
Index: unk/src/VBox/VMM/MM.cpp
===================================================================
--- /trunk/src/VBox/VMM/MM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,804 +1,0 @@
-/* $Id$ */
-/** @file
- * MM - Memory Manager.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page pg_mm     MM - The Memory Manager
- *
- * The memory manager is in charge of the following memory:
- *      - Hypervisor Memory Area (HMA) - Address space management.
- *      - Hypervisor Heap - A memory heap that lives in all contexts.
- *      - Tagged ring-3 heap.
- *      - Page pools - Primarily used by PGM for shadow page tables.
- *      - Locked process memory - Guest RAM and other. (reduce/obsolete this)
- *      - Physical guest memory (RAM & ROM) - Moving to PGM. (obsolete this)
- *
- * The global memory manager (GMM) is the global counter part / partner of MM.
- * MM will provide therefore ring-3 callable interfaces for some of the GMM APIs
- * related to resource tracking (PGM is the user).
- *
- * @see grp_mm
- *
- *
- * @section sec_mm_hma  Hypervisor Memory Area
- *
- * The HMA is used when executing in raw-mode. We borrow, with the help of
- * PGMMap, some unused space (one or more page directory entries to be precise)
- * in the guest's virtual memory context. PGM will monitor the guest's virtual
- * address space for changes and relocate the HMA when required.
- *
- * To give some idea what's in the HMA, study the 'info hma' output:
- * @verbatim
-VBoxDbg> info hma
-Hypervisor Memory Area (HMA) Layout: Base 00000000a0000000, 0x00800000 bytes
-00000000a05cc000-00000000a05cd000                  DYNAMIC                  fence
-00000000a05c4000-00000000a05cc000                  DYNAMIC                  Dynamic mapping
-00000000a05c3000-00000000a05c4000                  DYNAMIC                  fence
-00000000a05b8000-00000000a05c3000                  DYNAMIC                  Paging
-00000000a05b6000-00000000a05b8000                  MMIO2   0000000000000000 PCNetShMem
-00000000a0536000-00000000a05b6000                  MMIO2   0000000000000000 VGA VRam
-00000000a0523000-00000000a0536000 00002aaab3d0c000 LOCKED  autofree         alloc once (PDM_DEVICE)
-00000000a0522000-00000000a0523000                  DYNAMIC                  fence
-00000000a051e000-00000000a0522000 00002aaab36f5000 LOCKED  autofree         VBoxDD2GC.gc
-00000000a051d000-00000000a051e000                  DYNAMIC                  fence
-00000000a04eb000-00000000a051d000 00002aaab36c3000 LOCKED  autofree         VBoxDDGC.gc
-00000000a04ea000-00000000a04eb000                  DYNAMIC                  fence
-00000000a04e9000-00000000a04ea000 00002aaab36c2000 LOCKED  autofree         ram range (High ROM Region)
-00000000a04e8000-00000000a04e9000                  DYNAMIC                  fence
-00000000a040e000-00000000a04e8000 00002aaab2e6d000 LOCKED  autofree         VMMGC.gc
-00000000a0208000-00000000a040e000 00002aaab2c67000 LOCKED  autofree         alloc once (PATM)
-00000000a01f7000-00000000a0208000 00002aaaab92d000 LOCKED  autofree         alloc once (SELM)
-00000000a01e7000-00000000a01f7000 00002aaaab5e8000 LOCKED  autofree         alloc once (SELM)
-00000000a01e6000-00000000a01e7000                  DYNAMIC                  fence
-00000000a01e5000-00000000a01e6000 00002aaaab5e7000 HCPHYS  00000000c363c000 Core Code
-00000000a01e4000-00000000a01e5000                  DYNAMIC                  fence
-00000000a01e3000-00000000a01e4000 00002aaaaab26000 HCPHYS  00000000619cf000 GIP
-00000000a01a2000-00000000a01e3000 00002aaaabf32000 LOCKED  autofree         alloc once (PGM_PHYS)
-00000000a016b000-00000000a01a2000 00002aaab233f000 LOCKED  autofree         alloc once (PGM_POOL)
-00000000a016a000-00000000a016b000                  DYNAMIC                  fence
-00000000a0165000-00000000a016a000                  DYNAMIC                  CR3 mapping
-00000000a0164000-00000000a0165000                  DYNAMIC                  fence
-00000000a0024000-00000000a0164000 00002aaab215f000 LOCKED  autofree         Heap
-00000000a0023000-00000000a0024000                  DYNAMIC                  fence
-00000000a0001000-00000000a0023000 00002aaab1d24000 LOCKED  pages            VM
-00000000a0000000-00000000a0001000                  DYNAMIC                  fence
- @endverbatim
- *
- *
- * @section sec_mm_hyperheap    Hypervisor Heap
- *
- * The heap is accessible from ring-3, ring-0 and the raw-mode context. That
- * said, it's not necessarily mapped into ring-0 on if that's possible since we
- * don't wish to waste kernel address space without a good reason.
- *
- * Allocations within the heap are always in the same relative position in all
- * contexts, so, it's possible to use offset based linking. In fact, the heap is
- * internally using offset based linked lists tracking heap blocks. We use
- * offset linked AVL trees and lists in a lot of places where share structures
- * between RC, R3 and R0, so this is a strict requirement of the heap. However
- * this means that we cannot easily extend the heap since the extension won't
- * necessarily be in the continuation of the current heap memory in all (or any)
- * context.
- *
- * All allocations are tagged. Per tag allocation statistics will be maintaining
- * and exposed thru STAM when VBOX_WITH_STATISTICS is defined.
- *
- *
- * @section sec_mm_r3heap   Tagged Ring-3 Heap
- *
- * The ring-3 heap is a wrapper around the RTMem API adding allocation
- * statistics and automatic cleanup on VM destruction.
- *
- * Per tag allocation statistics will be maintaining and exposed thru STAM when
- * VBOX_WITH_STATISTICS is defined.
- *
- *
- * @section sec_mm_page     Page Pool
- *
- * The MM manages a page pool from which other components can allocate locked,
- * page aligned and page sized memory objects. The pool provides facilities to
- * convert back and forth between (host) physical and virtual addresses (within
- * the pool of course). Several specialized interfaces are provided for the most
- * common allocations and conversions to save the caller from bothersome casting
- * and extra parameter passing.
- *
- *
- * @section sec_mm_locked   Locked Process Memory
- *
- * MM manages the locked process memory. This is used for a bunch of things
- * (count the LOCKED entries in the 'info hma' output found in @ref sec_mm_hma),
- * but the main consumer of memory is currently for guest RAM. There is an
- * ongoing rewrite that will move all the guest RAM allocation to PGM and
- * GMM.
- *
- * The locking of memory is something doing in cooperation with the VirtualBox
- * support driver, SUPDrv (aka. VBoxDrv), thru the support library API,
- * SUPR3 (aka. SUPLib).
- *
- *
- * @section sec_mm_phys     Physical Guest Memory
- *
- * MM is currently managing the physical memory for the guest. It relies heavily
- * on PGM for this. There is an ongoing rewrite that will move this to PGM. (The
- * rewrite is driven by the need for more flexible guest ram allocation, but
- * also motivated by the fact that MMPhys is just adding stupid bureaucracy and
- * that MMR3PhysReserve is a totally weird artifact that must go away.)
- *
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_MM
-#include <VBox/mm.h>
-#include <VBox/pgm.h>
-#include <VBox/cfgm.h>
-#include <VBox/ssm.h>
-#include <VBox/gmm.h>
-#include "MMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-
-#include <VBox/log.h>
-#include <iprt/alloc.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The current saved state version of MM. */
-#define MM_SAVED_STATE_VERSION      2
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) mmR3Save(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int) mmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-
-
-
-
-/**
- * Initializes the MM members of the UVM.
- *
- * This is currently only the ring-3 heap.
- *
- * @returns VBox status code.
- * @param   pUVM    Pointer to the user mode VM structure.
- */
-VMMR3DECL(int) MMR3InitUVM(PUVM pUVM)
-{
-    /*
-     * Assert sizes and order.
-     */
-    AssertCompile(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
-    AssertRelease(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
-    Assert(!pUVM->mm.s.pHeap);
-
-    /*
-     * Init the heap.
-     */
-    int rc = mmR3HeapCreateU(pUVM, &pUVM->mm.s.pHeap);
-    if (RT_SUCCESS(rc))
-    {
-        rc = mmR3UkHeapCreateU(pUVM, &pUVM->mm.s.pUkHeap);
-        if (RT_SUCCESS(rc))
-            return VINF_SUCCESS;
-        mmR3HeapDestroy(pUVM->mm.s.pHeap);
-        pUVM->mm.s.pHeap = NULL;
-    }
-    return rc;
-}
-
-
-/**
- * Initializes the MM.
- *
- * MM is managing the virtual address space (among other things) and
- * setup the hypervisor memory area mapping in the VM structure and
- * the hypervisor alloc-only-heap. Assuming the current init order
- * and components the hypervisor memory area looks like this:
- *      -# VM Structure.
- *      -# Hypervisor alloc only heap (also call Hypervisor memory region).
- *      -# Core code.
- *
- * MM determines the virtual address of the hypervisor memory area by
- * checking for location at previous run. If that property isn't available
- * it will choose a default starting location, currently 0xa0000000.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) MMR3Init(PVM pVM)
-{
-    LogFlow(("MMR3Init\n"));
-
-    /*
-     * Assert alignment, sizes and order.
-     */
-    AssertRelease(!(RT_OFFSETOF(VM, mm.s) & 31));
-    AssertRelease(sizeof(pVM->mm.s) <= sizeof(pVM->mm.padding));
-    AssertMsg(pVM->mm.s.offVM == 0, ("Already initialized!\n"));
-
-    /*
-     * Init the structure.
-     */
-    pVM->mm.s.offVM = RT_OFFSETOF(VM, mm);
-    pVM->mm.s.offLookupHyper = NIL_OFFSET;
-
-    /*
-     * Init the page pool.
-     */
-    int rc = mmR3PagePoolInit(pVM);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Init the hypervisor related stuff.
-         */
-        rc = mmR3HyperInit(pVM);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Register the saved state data unit.
-             */
-            rc = SSMR3RegisterInternal(pVM, "mm", 1, MM_SAVED_STATE_VERSION, sizeof(uint32_t) * 2,
-                                       NULL, NULL, NULL,
-                                       NULL, mmR3Save, NULL,
-                                       NULL, mmR3Load, NULL);
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Statistics.
-                 */
-                STAM_REG(pVM, &pVM->mm.s.cBasePages,   STAMTYPE_U64, "/MM/Reserved/cBasePages",   STAMUNIT_PAGES, "Reserved number of base pages, ROM and Shadow ROM included.");
-                STAM_REG(pVM, &pVM->mm.s.cHandyPages,  STAMTYPE_U32, "/MM/Reserved/cHandyPages",  STAMUNIT_PAGES, "Reserved number of handy pages.");
-                STAM_REG(pVM, &pVM->mm.s.cShadowPages, STAMTYPE_U32, "/MM/Reserved/cShadowPages", STAMUNIT_PAGES, "Reserved number of shadow paging pages.");
-                STAM_REG(pVM, &pVM->mm.s.cFixedPages,  STAMTYPE_U32, "/MM/Reserved/cFixedPages",  STAMUNIT_PAGES, "Reserved number of fixed pages (MMIO2).");
-                STAM_REG(pVM, &pVM->mm.s.cbRamBase,    STAMTYPE_U64, "/MM/cbRamBase",             STAMUNIT_BYTES, "Size of the base RAM.");
-
-                return rc;
-            }
-
-            /* .... failure .... */
-        }
-    }
-    MMR3Term(pVM);
-    return rc;
-}
-
-
-/**
- * Initializes the MM parts which depends on PGM being initialized.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @remark  No cleanup necessary since MMR3Term() will be called on failure.
- */
-VMMR3DECL(int) MMR3InitPaging(PVM pVM)
-{
-    LogFlow(("MMR3InitPaging:\n"));
-
-    /*
-     * Query the CFGM values.
-     */
-    int rc;
-    PCFGMNODE pMMCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "MM");
-    if (!pMMCfg)
-    {
-        rc = CFGMR3InsertNode(CFGMR3GetRoot(pVM), "MM", &pMMCfg);
-        AssertRCReturn(rc, rc);
-    }
-
-    /** @cfgm{RamSize, uint64_t, 0, 16TB, 0}
-     * Specifies the size of the base RAM that is to be set up during
-     * VM initialization.
-     */
-    uint64_t cbRam;
-    rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        cbRam = 0;
-    else
-        AssertMsgRCReturn(rc, ("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc), rc);
-    AssertLogRelMsg(!(cbRam & ~X86_PTE_PAE_PG_MASK), ("%RGp X86_PTE_PAE_PG_MASK=%RX64\n", cbRam, X86_PTE_PAE_PG_MASK));
-    AssertLogRelMsgReturn(cbRam <= GMM_GCPHYS_LAST, ("cbRam=%RGp GMM_GCPHYS_LAST=%RX64\n", cbRam, GMM_GCPHYS_LAST), VERR_OUT_OF_RANGE);
-    cbRam &= X86_PTE_PAE_PG_MASK;
-    pVM->mm.s.cbRamBase = cbRam;
-
-    /** @cfgm{RamHoleSize, uint32_t, 0, 4032MB, 512MB}
-     * Specifies the size of the memory hole. The memory hole is used
-     * to avoid mapping RAM to the range normally used for PCI memory regions.
-     * Must be aligned on a 4MB boundary. */
-    uint32_t cbRamHole;
-    rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
-    AssertLogRelMsgRCReturn(rc, ("Configuration error: Failed to query integer \"RamHoleSize\", rc=%Rrc.\n", rc), rc);
-    AssertLogRelMsgReturn(cbRamHole <= 4032U * _1M,
-                          ("Configuration error: \"RamHoleSize\"=%#RX32 is too large.\n", cbRamHole), VERR_OUT_OF_RANGE);
-    AssertLogRelMsgReturn(cbRamHole > 16 * _1M,
-                          ("Configuration error: \"RamHoleSize\"=%#RX32 is too large.\n", cbRamHole), VERR_OUT_OF_RANGE);
-    AssertLogRelMsgReturn(!(cbRamHole & (_4M - 1)),
-                          ("Configuration error: \"RamHoleSize\"=%#RX32 is misaligned.\n", cbRamHole), VERR_OUT_OF_RANGE);
-    uint64_t const offRamHole = _4G - cbRamHole;
-    if (cbRam < offRamHole)
-        Log(("MM: %RU64 bytes of RAM\n", cbRam));
-    else
-        Log(("MM: %RU64 bytes of RAM with a hole at %RU64 up to 4GB.\n", cbRam, offRamHole));
-
-    /** @cfgm{MM/Policy, string, no overcommitment}
-     * Specifies the policy to use when reserving memory for this VM. The recognized
-     * value is 'no overcommitment' (default). See GMMPOLICY.
-     */
-    GMMOCPOLICY enmOcPolicy;
-    char sz[64];
-    rc = CFGMR3QueryString(CFGMR3GetRoot(pVM), "Policy", sz, sizeof(sz));
-    if (RT_SUCCESS(rc))
-    {
-        if (    !RTStrICmp(sz, "no_oc")
-            ||  !RTStrICmp(sz, "no overcommitment"))
-            enmOcPolicy = GMMOCPOLICY_NO_OC;
-        else
-            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, "Unknown \"MM/Policy\" value \"%s\"", sz);
-    }
-    else if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        enmOcPolicy = GMMOCPOLICY_NO_OC;
-    else
-        AssertMsgFailedReturn(("Configuration error: Failed to query string \"MM/Policy\", rc=%Rrc.\n", rc), rc);
-
-    /** @cfgm{MM/Priority, string, normal}
-     * Specifies the memory priority of this VM. The priority comes into play when the
-     * system is overcommitted and the VMs needs to be milked for memory. The recognized
-     * values are 'low', 'normal' (default) and 'high'. See GMMPRIORITY.
-     */
-    GMMPRIORITY enmPriority;
-    rc = CFGMR3QueryString(CFGMR3GetRoot(pVM), "Priority", sz, sizeof(sz));
-    if (RT_SUCCESS(rc))
-    {
-        if (!RTStrICmp(sz, "low"))
-            enmPriority = GMMPRIORITY_LOW;
-        else if (!RTStrICmp(sz, "normal"))
-            enmPriority = GMMPRIORITY_NORMAL;
-        else if (!RTStrICmp(sz, "high"))
-            enmPriority = GMMPRIORITY_HIGH;
-        else
-            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, "Unknown \"MM/Priority\" value \"%s\"", sz);
-    }
-    else if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        enmPriority = GMMPRIORITY_NORMAL;
-    else
-        AssertMsgFailedReturn(("Configuration error: Failed to query string \"MM/Priority\", rc=%Rrc.\n", rc), rc);
-
-    /*
-     * Make the initial memory reservation with GMM.
-     */
-    uint64_t cBasePages = (cbRam >> PAGE_SHIFT) + pVM->mm.s.cBasePages;
-    rc = GMMR3InitialReservation(pVM,
-                                 RT_MAX(cBasePages + pVM->mm.s.cHandyPages, 1),
-                                 RT_MAX(pVM->mm.s.cShadowPages, 1),
-                                 RT_MAX(pVM->mm.s.cFixedPages, 1),
-                                 enmOcPolicy,
-                                 enmPriority);
-    if (RT_FAILURE(rc))
-    {
-        if (rc == VERR_GMM_MEMORY_RESERVATION_DECLINED)
-            return VMSetError(pVM, rc, RT_SRC_POS,
-                              N_("Insufficient free memory to start the VM (cbRam=%#RX64 enmOcPolicy=%d enmPriority=%d)"),
-                              cbRam, enmOcPolicy, enmPriority);
-        return VMSetError(pVM, rc, RT_SRC_POS, "GMMR3InitialReservation(,%#RX64,0,0,%d,%d)",
-                          cbRam >> PAGE_SHIFT, enmOcPolicy, enmPriority);
-    }
-
-    /*
-     * If RamSize is 0 we're done now.
-     */
-    if (cbRam < PAGE_SIZE)
-    {
-        Log(("MM: No RAM configured\n"));
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Setup the base ram (PGM).
-     */
-    if (cbRam > offRamHole)
-    {
-        rc = PGMR3PhysRegisterRam(pVM, 0, offRamHole, "Base RAM");
-        if (RT_SUCCESS(rc))
-            rc = PGMR3PhysRegisterRam(pVM, _4G, cbRam - offRamHole, "Above 4GB Base RAM");
-    }
-    else
-        rc = PGMR3PhysRegisterRam(pVM, 0, RT_MIN(cbRam, offRamHole), "Base RAM");
-
-    /*
-     * Enabled mmR3UpdateReservation here since we don't want the
-     * PGMR3PhysRegisterRam calls above mess things up.
-     */
-    pVM->mm.s.fDoneMMR3InitPaging = true;
-    AssertMsg(pVM->mm.s.cBasePages == cBasePages || RT_FAILURE(rc), ("%RX64 != %RX64\n", pVM->mm.s.cBasePages, cBasePages));
-
-    LogFlow(("MMR3InitPaging: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Terminates the MM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) MMR3Term(PVM pVM)
-{
-    /*
-     * Destroy the page pool. (first as it used the hyper heap)
-     */
-    mmR3PagePoolTerm(pVM);
-
-    /* Clean up the hypervisor heap. */
-    mmR3HyperTerm(pVM);
-
-    /*
-     * Zero stuff to detect after termination use of the MM interface
-     */
-    pVM->mm.s.offLookupHyper = NIL_OFFSET;
-    pVM->mm.s.pHyperHeapR3   = NULL;        /* freed above. */
-    pVM->mm.s.pHyperHeapR0   = NIL_RTR0PTR; /* freed above. */
-    pVM->mm.s.pHyperHeapRC   = NIL_RTRCPTR; /* freed above. */
-    pVM->mm.s.offVM          = 0;           /* init assertion on this */
-
-    /*
-     * Destroy the User-kernel heap here since the support driver session
-     * may have been terminated by the time we get to MMR3TermUVM.
-     */
-    mmR3UkHeapDestroy(pVM->pUVM->mm.s.pUkHeap);
-    pVM->pUVM->mm.s.pUkHeap = NULL;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Terminates the UVM part of MM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pUVM        Pointer to the user mode VM structure.
- */
-VMMR3DECL(void) MMR3TermUVM(PUVM pUVM)
-{
-    /*
-     * Destroy the heaps.
-     */
-    if (pUVM->mm.s.pUkHeap)
-    {
-        mmR3UkHeapDestroy(pUVM->mm.s.pUkHeap);
-        pUVM->mm.s.pUkHeap = NULL;
-    }
-    mmR3HeapDestroy(pUVM->mm.s.pHeap);
-    pUVM->mm.s.pHeap = NULL;
-}
-
-
-/**
- * Reset notification.
- *
- * @param   pVM             The VM handle.
- */
-VMMR3DECL(void) MMR3Reset(PVM pVM)
-{
-    /* nothing to do anylonger. */
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) mmR3Save(PVM pVM, PSSMHANDLE pSSM)
-{
-    LogFlow(("mmR3Save:\n"));
-
-    /* (PGM saves the physical memory.) */
-    SSMR3PutU64(pSSM, pVM->mm.s.cBasePages);
-    return SSMR3PutU64(pSSM, pVM->mm.s.cbRamBase);
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion       Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) mmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    LogFlow(("mmR3Load:\n"));
-    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
-
-    /*
-     * Validate version.
-     */
-    if (    SSM_VERSION_MAJOR_CHANGED(uVersion, MM_SAVED_STATE_VERSION)
-        ||  !uVersion)
-    {
-        AssertMsgFailed(("mmR3Load: Invalid version uVersion=%d!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    /*
-     * Check the cBasePages and cbRamBase values.
-     */
-    int rc;
-    RTUINT cb1;
-
-    /* cBasePages (ignored) */
-    uint64_t cPages;
-    if (uVersion >= 2)
-        rc = SSMR3GetU64(pSSM, &cPages);
-    else
-    {
-        rc = SSMR3GetUInt(pSSM, &cb1);
-        cPages = cb1 >> PAGE_SHIFT;
-    }
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /* cbRamBase */
-    uint64_t cb;
-    if (uVersion != 1)
-        rc = SSMR3GetU64(pSSM, &cb);
-    else
-    {
-        rc = SSMR3GetUInt(pSSM, &cb1);
-        cb = cb1;
-    }
-    if (RT_FAILURE(rc))
-        return rc;
-    AssertLogRelMsgReturn(cb == pVM->mm.s.cbRamBase,
-                          ("Memory configuration has changed. cbRamBase=%#RX64 save=%#RX64\n", pVM->mm.s.cbRamBase, cb),
-                          VERR_SSM_LOAD_MEMORY_SIZE_MISMATCH);
-
-    /* (PGM restores the physical memory.) */
-    return rc;
-}
-
-
-/**
- * Updates GMM with memory reservation changes.
- *
- * Called when MM::cbRamRegistered, MM::cShadowPages or MM::cFixedPages changes.
- *
- * @returns VBox status code - see GMMR0UpdateReservation.
- * @param   pVM             The shared VM structure.
- */
-int mmR3UpdateReservation(PVM pVM)
-{
-    VM_ASSERT_EMT(pVM);
-    if (pVM->mm.s.fDoneMMR3InitPaging)
-        return GMMR3UpdateReservation(pVM,
-                                      RT_MAX(pVM->mm.s.cBasePages + pVM->mm.s.cHandyPages, 1),
-                                      RT_MAX(pVM->mm.s.cShadowPages, 1),
-                                      RT_MAX(pVM->mm.s.cFixedPages, 1));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Interface for PGM to increase the reservation of RAM and ROM pages.
- *
- * This can be called before MMR3InitPaging.
- *
- * @returns VBox status code. Will set VM error on failure.
- * @param   pVM             The shared VM structure.
- * @param   cAddBasePages   The number of pages to add.
- */
-VMMR3DECL(int) MMR3IncreaseBaseReservation(PVM pVM, uint64_t cAddBasePages)
-{
-    uint64_t cOld = pVM->mm.s.cBasePages;
-    pVM->mm.s.cBasePages += cAddBasePages;
-    LogFlow(("MMR3IncreaseBaseReservation: +%RU64 (%RU64 -> %RU64\n", cAddBasePages, cOld, pVM->mm.s.cBasePages));
-    int rc = mmR3UpdateReservation(pVM);
-    if (RT_FAILURE(rc))
-    {
-        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserved physical memory for the RAM (%#RX64 -> %#RX64 + %#RX32)"),
-                   cOld, pVM->mm.s.cBasePages, pVM->mm.s.cHandyPages);
-        pVM->mm.s.cBasePages = cOld;
-    }
-    return rc;
-}
-
-
-/**
- * Interface for PGM to make reservations for handy pages in addition to the
- * base memory.
- *
- * This can be called before MMR3InitPaging.
- *
- * @returns VBox status code. Will set VM error on failure.
- * @param   pVM             The shared VM structure.
- * @param   cHandyPages     The number of handy pages.
- */
-VMMR3DECL(int) MMR3ReserveHandyPages(PVM pVM, uint32_t cHandyPages)
-{
-    AssertReturn(!pVM->mm.s.cHandyPages, VERR_WRONG_ORDER);
-
-    pVM->mm.s.cHandyPages = cHandyPages;
-    LogFlow(("MMR3ReserveHandyPages: %RU32 (base %RU64)\n", pVM->mm.s.cHandyPages, pVM->mm.s.cBasePages));
-    int rc = mmR3UpdateReservation(pVM);
-    if (RT_FAILURE(rc))
-    {
-        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserved physical memory for the RAM (%#RX64 + %#RX32)"),
-                   pVM->mm.s.cBasePages, pVM->mm.s.cHandyPages);
-        pVM->mm.s.cHandyPages = 0;
-    }
-    return rc;
-}
-
-
-/**
- * Interface for PGM to adjust the reservation of fixed pages.
- *
- * This can be called before MMR3InitPaging.
- *
- * @returns VBox status code. Will set VM error on failure.
- * @param   pVM                 The shared VM structure.
- * @param   cDeltaFixedPages    The number of pages to add (positive) or subtract (negative).
- * @param   pszDesc             Some description associated with the reservation.
- */
-VMMR3DECL(int) MMR3AdjustFixedReservation(PVM pVM, int32_t cDeltaFixedPages, const char *pszDesc)
-{
-    const uint32_t cOld = pVM->mm.s.cFixedPages;
-    pVM->mm.s.cFixedPages += cDeltaFixedPages;
-    LogFlow(("MMR3AdjustFixedReservation: %d (%u -> %u)\n", cDeltaFixedPages, cOld, pVM->mm.s.cFixedPages));
-    int rc = mmR3UpdateReservation(pVM);
-    if (RT_FAILURE(rc))
-    {
-        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserve physical memory (%#x -> %#x; %s)"),
-                   cOld, pVM->mm.s.cFixedPages, pszDesc);
-        pVM->mm.s.cFixedPages = cOld;
-    }
-    return rc;
-}
-
-
-/**
- * Interface for PGM to update the reservation of shadow pages.
- *
- * This can be called before MMR3InitPaging.
- *
- * @returns VBox status code. Will set VM error on failure.
- * @param   pVM             The shared VM structure.
- * @param   cShadowPages    The new page count.
- */
-VMMR3DECL(int) MMR3UpdateShadowReservation(PVM pVM, uint32_t cShadowPages)
-{
-    const uint32_t cOld = pVM->mm.s.cShadowPages;
-    pVM->mm.s.cShadowPages = cShadowPages;
-    LogFlow(("MMR3UpdateShadowReservation: %u -> %u\n", cOld, pVM->mm.s.cShadowPages));
-    int rc = mmR3UpdateReservation(pVM);
-    if (RT_FAILURE(rc))
-    {
-        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserve physical memory for shadow page tables (%#x -> %#x)"), cOld, pVM->mm.s.cShadowPages);
-        pVM->mm.s.cShadowPages = cOld;
-    }
-    return rc;
-}
-
-
-/**
- * Convert HC Physical address to HC Virtual address.
- *
- * @returns VBox status.
- * @param   pVM         VM handle.
- * @param   HCPhys      The host context virtual address.
- * @param   ppv         Where to store the resulting address.
- * @thread  The Emulation Thread.
- *
- * @remarks Avoid whenever possible.
- *          Intended for the debugger facility only.
- * @todo    Rename to indicate the special usage.
- */
-VMMR3DECL(int) MMR3HCPhys2HCVirt(PVM pVM, RTHCPHYS HCPhys, void **ppv)
-{
-    /*
-     * Try page tables.
-     */
-    int rc = MMPagePhys2PageTry(pVM, HCPhys, ppv);
-    if (RT_SUCCESS(rc))
-        return rc;
-
-    /*
-     * Iterate thru the lookup records for HMA.
-     */
-    uint32_t off = HCPhys & PAGE_OFFSET_MASK;
-    HCPhys &= X86_PTE_PAE_PG_MASK;
-    PMMLOOKUPHYPER pCur = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap) + pVM->mm.s.offLookupHyper);
-    for (;;)
-    {
-        switch (pCur->enmType)
-        {
-            case MMLOOKUPHYPERTYPE_LOCKED:
-            {
-                PCRTHCPHYS  paHCPhysPages = pCur->u.Locked.paHCPhysPages;
-                size_t      iPage         = pCur->cb >> PAGE_SHIFT;
-                while (iPage-- > 0)
-                    if (paHCPhysPages[iPage] == HCPhys)
-                    {
-                        *ppv = (char *)pCur->u.Locked.pvR3 + (iPage << PAGE_SHIFT) + off;
-                        return VINF_SUCCESS;
-                    }
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_HCPHYS:
-                if (pCur->u.HCPhys.HCPhys - HCPhys < pCur->cb)
-                {
-                    *ppv = (uint8_t *)pCur->u.HCPhys.pvR3 + pCur->u.HCPhys.HCPhys - HCPhys + off;
-                    return VINF_SUCCESS;
-                }
-                break;
-
-            case MMLOOKUPHYPERTYPE_GCPHYS:  /* (for now we'll not allow these kind of conversions) */
-            case MMLOOKUPHYPERTYPE_MMIO2:
-            case MMLOOKUPHYPERTYPE_DYNAMIC:
-                break;
-
-            default:
-                AssertMsgFailed(("enmType=%d\n", pCur->enmType));
-                break;
-        }
-
-        /* next */
-        if (pCur->offNext ==  (int32_t)NIL_OFFSET)
-            break;
-        pCur = (PMMLOOKUPHYPER)((uint8_t *)pCur + pCur->offNext);
-    }
-    /* give up */
-    return VERR_INVALID_POINTER;
-}
-
-
-
-/**
- * Get the size of the base RAM.
- * This usually means the size of the first contiguous block of physical memory.
- *
- * @returns The guest base RAM size.
- * @param   pVM         The VM handle.
- * @thread  Any.
- *
- * @deprecated
- */
-VMMR3DECL(uint64_t) MMR3PhysGetRamSize(PVM pVM)
-{
-    return pVM->mm.s.cbRamBase;
-}
-
Index: unk/src/VBox/VMM/MMHeap.cpp
===================================================================
--- /trunk/src/VBox/VMM/MMHeap.cpp	(revision 35332)
+++ 	(revision )
@@ -1,694 +1,0 @@
-/* $Id$ */
-/** @file
- * MM - Memory Manager - Heap.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_MM_HEAP
-#include <VBox/mm.h>
-#include <VBox/stam.h>
-#include <VBox/pgm.h>
-#include "MMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/log.h>
-
-#include <iprt/alloc.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero);
-
-
-
-/**
- * Allocate and initialize a heap structure and it's associated substructures.
- *
- * @returns VBox status.
- * @param   pVM     The handle to the VM the heap should be associated with.
- * @param   ppHeap  Where to store the heap pointer.
- */
-int mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap)
-{
-    PMMHEAP pHeap = (PMMHEAP)RTMemAllocZ(sizeof(MMHEAP) + sizeof(MMHEAPSTAT));
-    if (pHeap)
-    {
-        int rc = RTCritSectInit(&pHeap->Lock);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Initialize the global stat record.
-             */
-            pHeap->pUVM = pUVM;
-            pHeap->Stat.pHeap = pHeap;
-#ifdef MMR3HEAP_WITH_STATISTICS
-            PMMHEAPSTAT pStat = &pHeap->Stat;
-            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFrees",           STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
-            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
-                                                                        STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
-            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
-            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
-#endif
-            *ppHeap = pHeap;
-            return VINF_SUCCESS;
-        }
-        AssertRC(rc);
-        RTMemFree(pHeap);
-    }
-    AssertMsgFailed(("failed to allocate heap structure\n"));
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Destroy a heap.
- *
- * @param   pHeap   Heap handle.
- */
-void mmR3HeapDestroy(PMMHEAP pHeap)
-{
-    /*
-     * Start by deleting the lock, that'll trap anyone
-     * attempting to use the heap.
-     */
-    RTCritSectDelete(&pHeap->Lock);
-
-    /*
-     * Walk the node list and free all the memory.
-     */
-    PMMHEAPHDR  pHdr = pHeap->pHead;
-    while (pHdr)
-    {
-        void *pv = pHdr;
-        pHdr = pHdr->pNext;
-        RTMemFree(pv);
-    }
-
-    /*
-     * Free the stat nodes.
-     */
-    /** @todo free all nodes in a AVL tree. */
-    RTMemFree(pHeap);
-}
-
-
-/**
- * Allocate memory associating it with the VM for collective cleanup.
- *
- * The memory will be allocated from the default heap but a header
- * is added in which we keep track of which VM it belongs to and chain
- * all the allocations together so they can be freed in one go.
- *
- * This interface is typically used for memory block which will not be
- * freed during the life of the VM.
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAllocU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
-{
-    Assert(pUVM->mm.s.pHeap);
-    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
-}
-
-
-/**
- * Allocate memory associating it with the VM for collective cleanup.
- *
- * The memory will be allocated from the default heap but a header
- * is added in which we keep track of which VM it belongs to and chain
- * all the allocations together so they can be freed in one go.
- *
- * This interface is typically used for memory block which will not be
- * freed during the life of the VM.
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize)
-{
-    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
-}
-
-
-/**
- * Same as MMR3HeapAllocU().
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    Assert(pUVM->mm.s.pHeap);
-    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Same as MMR3HeapAlloc().
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Same as MMR3HeapAlloc() only the memory is zeroed.
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAllocZU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
-{
-    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
-}
-
-
-/**
- * Same as MMR3HeapAlloc() only the memory is zeroed.
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- */
-VMMR3DECL(void *) MMR3HeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize)
-{
-    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
-}
-
-
-/**
- * Same as MMR3HeapAllocZ().
- *
- * @returns Pointer to allocated memory.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocZExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    Assert(pUVM->mm.s.pHeap);
-    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Same as MMR3HeapAllocZ().
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- */
-VMMR3DECL(int) MMR3HeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
-{
-    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Allocate memory from the heap.
- *
- * @returns Pointer to allocated memory.
- * @param   pHeap       Heap handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   cbSize      Size of the block.
- * @param   fZero       Whether or not to zero the memory block.
- */
-void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
-{
-#ifdef MMR3HEAP_WITH_STATISTICS
-    RTCritSectEnter(&pHeap->Lock);
-
-    /*
-     * Find/alloc statistics nodes.
-     */
-    pHeap->Stat.cAllocations++;
-    PMMHEAPSTAT pStat = (PMMHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
-    if (pStat)
-    {
-        pStat->cAllocations++;
-
-        RTCritSectLeave(&pHeap->Lock);
-    }
-    else
-    {
-        pStat = (PMMHEAPSTAT)RTMemAllocZ(sizeof(MMHEAPSTAT));
-        if (!pStat)
-        {
-            pHeap->Stat.cFailures++;
-            AssertMsgFailed(("Failed to allocate heap stat record.\n"));
-            RTCritSectLeave(&pHeap->Lock);
-            return NULL;
-        }
-        pStat->Core.Key = (AVLULKEY)enmTag;
-        pStat->pHeap    = pHeap;
-        RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
-
-        pStat->cAllocations++;
-        RTCritSectLeave(&pHeap->Lock);
-
-        /* register the statistics */
-        PUVM pUVM = pHeap->pUVM;
-        const char *pszTag = mmGetTagName(enmTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/R3Heap/%s", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.",        "/MM/R3Heap/%s/cAllocations", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.",      "/MM/R3Heap/%s/cReallocations", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.",         "/MM/R3Heap/%s/cFrees", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/R3Heap/%s/cFailures", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/R3Heap/%s/cbAllocated", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/R3Heap/%s/cbFreed", pszTag);
-    }
-#endif
-
-    /*
-     * Validate input.
-     */
-    if (cbSize == 0)
-    {
-#ifdef MMR3HEAP_WITH_STATISTICS
-        RTCritSectEnter(&pHeap->Lock);
-        pStat->cFailures++;
-        pHeap->Stat.cFailures++;
-        RTCritSectLeave(&pHeap->Lock);
-#endif
-        return NULL;
-    }
-
-    /*
-     * Allocate heap block.
-     */
-    cbSize = RT_ALIGN_Z(cbSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
-    PMMHEAPHDR  pHdr = (PMMHEAPHDR)(fZero ? RTMemAllocZ(cbSize) : RTMemAlloc(cbSize));
-    if (!pHdr)
-    {
-        AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cbSize, enmTag, &enmTag));
-#ifdef MMR3HEAP_WITH_STATISTICS
-        RTCritSectEnter(&pHeap->Lock);
-        pStat->cFailures++;
-        pHeap->Stat.cFailures++;
-        RTCritSectLeave(&pHeap->Lock);
-#endif
-        return NULL;
-    }
-    Assert(!((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)));
-
-    RTCritSectEnter(&pHeap->Lock);
-
-    /*
-     * Init and link in the header.
-     */
-    pHdr->pNext = NULL;
-    pHdr->pPrev = pHeap->pTail;
-    if (pHdr->pPrev)
-        pHdr->pPrev->pNext = pHdr;
-    else
-        pHeap->pHead = pHdr;
-    pHeap->pTail = pHdr;
-#ifdef MMR3HEAP_WITH_STATISTICS
-    pHdr->pStat  = pStat;
-#else
-    pHdr->pStat  = &pHeap->Stat;
-#endif
-    pHdr->cbSize = cbSize;
-
-    /*
-     * Update statistics
-     */
-#ifdef MMR3HEAP_WITH_STATISTICS
-    pStat->cbAllocated          += cbSize;
-    pStat->cbCurAllocated       += cbSize;
-    pHeap->Stat.cbAllocated     += cbSize;
-    pHeap->Stat.cbCurAllocated  += cbSize;
-#endif
-
-    RTCritSectLeave(&pHeap->Lock);
-
-    return pHdr + 1;
-}
-
-
-/**
- * Reallocate memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
- *
- * @returns Pointer to reallocated memory.
- * @param   pv          Pointer to the memory block to reallocate.
- *                      Must not be NULL!
- * @param   cbNewSize   New block size.
- */
-VMMR3DECL(void *) MMR3HeapRealloc(void *pv, size_t cbNewSize)
-{
-    AssertMsg(pv, ("Invalid pointer pv=%p\n", pv));
-    if (!pv)
-        return NULL;
-
-    /*
-     * If newsize is zero then this is a free.
-     */
-    if (!cbNewSize)
-    {
-        MMR3HeapFree(pv);
-        return NULL;
-    }
-
-    /*
-     * Validate header.
-     */
-    PMMHEAPHDR  pHdr = (PMMHEAPHDR)pv - 1;
-    if (    pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1)
-        ||  (uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))
-    {
-        AssertMsgFailed(("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
-        return NULL;
-    }
-    Assert(pHdr->pStat != NULL);
-    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
-    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
-
-    PMMHEAP pHeap = pHdr->pStat->pHeap;
-
-#ifdef MMR3HEAP_WITH_STATISTICS
-    RTCritSectEnter(&pHeap->Lock);
-    pHdr->pStat->cReallocations++;
-    pHeap->Stat.cReallocations++;
-    RTCritSectLeave(&pHeap->Lock);
-#endif
-
-    /*
-     * Reallocate the block.
-     */
-    cbNewSize = RT_ALIGN_Z(cbNewSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
-    PMMHEAPHDR pHdrNew = (PMMHEAPHDR)RTMemRealloc(pHdr, cbNewSize);
-    if (!pHdrNew)
-    {
-#ifdef MMR3HEAP_WITH_STATISTICS
-        RTCritSectEnter(&pHeap->Lock);
-        pHdr->pStat->cFailures++;
-        pHeap->Stat.cFailures++;
-        RTCritSectLeave(&pHeap->Lock);
-#endif
-        return NULL;
-    }
-
-    /*
-     * Update pointers.
-     */
-    if (pHdrNew != pHdr)
-    {
-        RTCritSectEnter(&pHeap->Lock);
-        if (pHdrNew->pPrev)
-            pHdrNew->pPrev->pNext = pHdrNew;
-        else
-            pHeap->pHead = pHdrNew;
-
-        if (pHdrNew->pNext)
-            pHdrNew->pNext->pPrev = pHdrNew;
-        else
-            pHeap->pTail = pHdrNew;
-        RTCritSectLeave(&pHeap->Lock);
-    }
-
-    /*
-     * Update statistics.
-     */
-#ifdef MMR3HEAP_WITH_STATISTICS
-    RTCritSectEnter(&pHeap->Lock);
-    pHdrNew->pStat->cbAllocated += cbNewSize - pHdrNew->cbSize;
-    pHeap->Stat.cbAllocated += cbNewSize - pHdrNew->cbSize;
-    RTCritSectLeave(&pHeap->Lock);
-#endif
-
-    pHdrNew->cbSize = cbNewSize;
-
-    return pHdrNew + 1;
-}
-
-
-/**
- * Duplicates the specified string.
- *
- * @returns Pointer to the duplicate.
- * @returns NULL on failure or when input NULL.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   psz         The string to duplicate. NULL is allowed.
- */
-VMMR3DECL(char *) MMR3HeapStrDupU(PUVM pUVM, MMTAG enmTag, const char *psz)
-{
-    if (!psz)
-        return NULL;
-    AssertPtr(psz);
-
-    size_t cch = strlen(psz) + 1;
-    char *pszDup = (char *)MMR3HeapAllocU(pUVM, enmTag, cch);
-    if (pszDup)
-        memcpy(pszDup, psz, cch);
-    return pszDup;
-}
-
-
-/**
- * Duplicates the specified string.
- *
- * @returns Pointer to the duplicate.
- * @returns NULL on failure or when input NULL.
- * @param   pVM         The VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM. See MM_TAG_*.
- * @param   psz         The string to duplicate. NULL is allowed.
- */
-VMMR3DECL(char *) MMR3HeapStrDup(PVM pVM, MMTAG enmTag, const char *psz)
-{
-    return MMR3HeapStrDupU(pVM->pUVM, enmTag, psz);
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pVM         The VM
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   ...         Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintf(PVM pVM, MMTAG enmTag, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    char *psz = MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
-    va_end(va);
-    return psz;
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   ...         Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintfU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    char *psz = MMR3HeapAPrintfVU(pUVM, enmTag, pszFormat, va);
-    va_end(va);
-    return psz;
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pVM         The VM
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   va          Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintfV(PVM pVM, MMTAG enmTag, const char *pszFormat, va_list va)
-{
-    return MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
-}
-
-
-/**
- * Allocating string printf.
- *
- * @returns Pointer to the string.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   enmTag      The statistics tag.
- * @param   pszFormat   The format string.
- * @param   va          Format arguments.
- */
-VMMR3DECL(char *)    MMR3HeapAPrintfVU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, va_list va)
-{
-    /*
-     * The lazy bird way.
-     */
-    char *psz;
-    int cch = RTStrAPrintfV(&psz, pszFormat, va);
-    if (cch < 0)
-        return NULL;
-    Assert(psz[cch] == '\0');
-    char *pszRet = (char *)MMR3HeapAllocU(pUVM, enmTag, cch + 1);
-    if (pszRet)
-        memcpy(pszRet, psz, cch + 1);
-    RTStrFree(psz);
-    return pszRet;
-}
-
-
-/**
- * Releases memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
- *
- * @param   pv          Pointer to the memory block to free.
- */
-VMMR3DECL(void) MMR3HeapFree(void *pv)
-{
-    /* Ignore NULL pointers. */
-    if (!pv)
-        return;
-
-    /*
-     * Validate header.
-     */
-    PMMHEAPHDR  pHdr = (PMMHEAPHDR)pv - 1;
-    if (    pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1)
-        ||  (uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))
-    {
-        AssertMsgFailed(("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
-        return;
-    }
-    Assert(pHdr->pStat != NULL);
-    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
-    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
-
-    /*
-     * Update statistics
-     */
-    PMMHEAP pHeap = pHdr->pStat->pHeap;
-    RTCritSectEnter(&pHeap->Lock);
-
-#ifdef MMR3HEAP_WITH_STATISTICS
-    pHdr->pStat->cFrees++;
-    pHeap->Stat.cFrees++;
-    pHdr->pStat->cbFreed            += pHdr->cbSize;
-    pHeap->Stat.cbFreed             += pHdr->cbSize;
-    pHdr->pStat->cbCurAllocated     -= pHdr->cbSize;
-    pHeap->Stat.cbCurAllocated      -= pHdr->cbSize;
-#endif
-
-    /*
-     * Unlink it.
-     */
-    if (pHdr->pPrev)
-        pHdr->pPrev->pNext = pHdr->pNext;
-    else
-        pHeap->pHead = pHdr->pNext;
-
-    if (pHdr->pNext)
-        pHdr->pNext->pPrev = pHdr->pPrev;
-    else
-        pHeap->pTail = pHdr->pPrev;
-
-    RTCritSectLeave(&pHeap->Lock);
-
-    /*
-     * Free the memory.
-     */
-    RTMemFree(pHdr);
-}
-
Index: unk/src/VBox/VMM/MMHyper.cpp
===================================================================
--- /trunk/src/VBox/VMM/MMHyper.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1407 +1,0 @@
-/* $Id$ */
-/** @file
- * MM - Memory Manager - Hypervisor Memory Area.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_MM_HYPER
-#include <VBox/pgm.h>
-#include <VBox/mm.h>
-#include <VBox/dbgf.h>
-#include "MMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/log.h>
-#include <include/internal/pgm.h>
-#include <iprt/alloc.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(bool) mmR3HyperRelocateCallback(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser);
-static int mmR3HyperMap(PVM pVM, const size_t cb, const char *pszDesc, PRTGCPTR pGCPtr, PMMLOOKUPHYPER *ppLookup);
-static int mmR3HyperHeapCreate(PVM pVM, const size_t cb, PMMHYPERHEAP *ppHeap, PRTR0PTR pR0PtrHeap);
-static int mmR3HyperHeapMap(PVM pVM, PMMHYPERHEAP pHeap, PRTGCPTR ppHeapGC);
-static DECLCALLBACK(void) mmR3HyperInfoHma(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-
-
-DECLINLINE(uint32_t) mmR3ComputeHyperHeapSize(PVM pVM, bool fCanUseLargerHeap)
-{
-    bool fHwVirtExtForced = VMMIsHwVirtExtForced(pVM);
-
-    /* Newer chipset allows for much more devices, putting additional pressure on hyperheap */
-    /* @todo: maybe base hyperheap size upon number of devices attached too */
-    if (fCanUseLargerHeap)
-        return _2M + pVM->cCpus * 2 * _64K;
-
-    if (pVM->cCpus > 1)
-        return _1M + pVM->cCpus * 2 * _64K;
-
-    if (fHwVirtExtForced)
-    {
-        uint64_t cbRam = 0;
-        CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
-
-        /* Need a bit more space for large memory guests. (@todo: only for shadow paging!) */
-        if (cbRam >= _4G)
-            return _1M;
-        else
-            return 640 * _1K;
-    }
-    else
-        /* Size must be kept like this for saved state compatibility (only for raw mode though). */
-        return 1280*_1K;
-}
-
-
-/**
- * Initializes the hypervisor related MM stuff without
- * calling down to PGM.
- *
- * PGM is not initialized at this  point, PGM relies on
- * the heap to initialize.
- *
- * @returns VBox status.
- */
-int mmR3HyperInit(PVM pVM)
-{
-    LogFlow(("mmR3HyperInit:\n"));
-
-    /*
-     * Decide Hypervisor mapping in the guest context
-     * And setup various hypervisor area and heap parameters.
-     */
-    pVM->mm.s.pvHyperAreaGC = (RTGCPTR)MM_HYPER_AREA_ADDRESS;
-    pVM->mm.s.cbHyperArea   = MM_HYPER_AREA_MAX_SIZE;
-    AssertRelease(RT_ALIGN_T(pVM->mm.s.pvHyperAreaGC, 1 << X86_PD_SHIFT, RTGCPTR) == pVM->mm.s.pvHyperAreaGC);
-    Assert(pVM->mm.s.pvHyperAreaGC < 0xff000000);
-
-    /** @todo @bugref{1865}, @bugref{3202}: Change the cbHyperHeap default
-     *        depending on whether VT-x/AMD-V is enabled or not! Don't waste
-     *        precious kernel space on heap for the PATM.
-     */
-    PCFGMNODE pMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "MM");
-    bool fCanUseLargerHeap = false;
-    int rc = CFGMR3QueryBoolDef(pMM, "CanUseLargerHeap", &fCanUseLargerHeap, false);
-    uint32_t cbHyperHeap = mmR3ComputeHyperHeapSize(pVM, fCanUseLargerHeap);
-    rc = CFGMR3QueryU32Def(pMM, "cbHyperHeap", &cbHyperHeap, cbHyperHeap);
-    AssertLogRelRCReturn(rc, rc);
-
-    cbHyperHeap = RT_ALIGN_32(cbHyperHeap, PAGE_SIZE);
-    LogRel(("MM: cbHyperHeap=%#x (%u)\n", cbHyperHeap, cbHyperHeap));
-
-    /*
-     * Allocate the hypervisor heap.
-     *
-     * (This must be done before we start adding memory to the
-     * hypervisor static area because lookup records are allocated from it.)
-     */
-    rc = mmR3HyperHeapCreate(pVM, cbHyperHeap, &pVM->mm.s.pHyperHeapR3, &pVM->mm.s.pHyperHeapR0);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Make a small head fence to fend of accidental sequential access.
-         */
-        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-
-        /*
-         * Map the VM structure into the hypervisor space.
-         */
-        AssertRelease(pVM->cbSelf == RT_UOFFSETOF(VM, aCpus[pVM->cCpus]));
-        RTGCPTR GCPtr;
-        rc = MMR3HyperMapPages(pVM, pVM, pVM->pVMR0, RT_ALIGN_Z(pVM->cbSelf, PAGE_SIZE) >> PAGE_SHIFT, pVM->paVMPagesR3, "VM", &GCPtr);
-        if (RT_SUCCESS(rc))
-        {
-            pVM->pVMRC = (RTRCPTR)GCPtr;
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-                pVM->aCpus[i].pVMRC = pVM->pVMRC;
-
-            /* Reserve a page for fencing. */
-            MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-
-            /*
-             * Map the heap into the hypervisor space.
-             */
-            rc = mmR3HyperHeapMap(pVM, pVM->mm.s.pHyperHeapR3, &GCPtr);
-            if (RT_SUCCESS(rc))
-            {
-                pVM->mm.s.pHyperHeapRC = (RTRCPTR)GCPtr;
-                Assert(pVM->mm.s.pHyperHeapRC == GCPtr);
-
-                /*
-                 * Register info handlers.
-                 */
-                DBGFR3InfoRegisterInternal(pVM, "hma", "Show the layout of the Hypervisor Memory Area.", mmR3HyperInfoHma);
-
-                LogFlow(("mmR3HyperInit: returns VINF_SUCCESS\n"));
-                return VINF_SUCCESS;
-            }
-            /* Caller will do proper cleanup. */
-        }
-    }
-
-    LogFlow(("mmR3HyperInit: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Cleans up the hypervisor heap.
- *
- * @returns VBox status.
- */
-int mmR3HyperTerm(PVM pVM)
-{
-    if (pVM->mm.s.pHyperHeapR3)
-        PDMR3CritSectDelete(&pVM->mm.s.pHyperHeapR3->Lock);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Finalizes the HMA mapping.
- *
- * This is called later during init, most (all) HMA allocations should be done
- * by the time this function is called.
- *
- * @returns VBox status.
- */
-VMMR3DECL(int) MMR3HyperInitFinalize(PVM pVM)
-{
-    LogFlow(("MMR3HyperInitFinalize:\n"));
-
-    /*
-     * Initialize the hyper heap critical section.
-     */
-    int rc = PDMR3CritSectInit(pVM, &pVM->mm.s.pHyperHeapR3->Lock, RT_SRC_POS, "MM-HYPER");
-    AssertRC(rc);
-
-    /*
-     * Adjust and create the HMA mapping.
-     */
-    while ((RTINT)pVM->mm.s.offHyperNextStatic + 64*_1K < (RTINT)pVM->mm.s.cbHyperArea - _4M)
-        pVM->mm.s.cbHyperArea -= _4M;
-    rc = PGMR3MapPT(pVM, pVM->mm.s.pvHyperAreaGC, pVM->mm.s.cbHyperArea, 0 /*fFlags*/,
-                    mmR3HyperRelocateCallback, NULL, "Hypervisor Memory Area");
-    if (RT_FAILURE(rc))
-        return rc;
-    pVM->mm.s.fPGMInitialized = true;
-
-    /*
-     * Do all the delayed mappings.
-     */
-    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uintptr_t)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
-    for (;;)
-    {
-        RTGCPTR     GCPtr = pVM->mm.s.pvHyperAreaGC + pLookup->off;
-        uint32_t    cPages = pLookup->cb >> PAGE_SHIFT;
-        switch (pLookup->enmType)
-        {
-            case MMLOOKUPHYPERTYPE_LOCKED:
-            {
-                PCRTHCPHYS paHCPhysPages = pLookup->u.Locked.paHCPhysPages;
-                for (uint32_t i = 0; i < cPages; i++)
-                {
-                    rc = PGMMap(pVM, GCPtr + (i << PAGE_SHIFT), paHCPhysPages[i], PAGE_SIZE, 0);
-                    AssertRCReturn(rc, rc);
-                }
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_HCPHYS:
-                rc = PGMMap(pVM, GCPtr, pLookup->u.HCPhys.HCPhys, pLookup->cb, 0);
-                break;
-
-            case MMLOOKUPHYPERTYPE_GCPHYS:
-            {
-                const RTGCPHYS  GCPhys = pLookup->u.GCPhys.GCPhys;
-                const uint32_t  cb = pLookup->cb;
-                for (uint32_t off = 0; off < cb; off += PAGE_SIZE)
-                {
-                    RTHCPHYS HCPhys;
-                    rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys + off, &HCPhys);
-                    if (RT_FAILURE(rc))
-                        break;
-                    rc = PGMMap(pVM, GCPtr + off, HCPhys, PAGE_SIZE, 0);
-                    if (RT_FAILURE(rc))
-                        break;
-                }
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_MMIO2:
-            {
-                const RTGCPHYS offEnd = pLookup->u.MMIO2.off + pLookup->cb;
-                for (RTGCPHYS offCur = pLookup->u.MMIO2.off; offCur < offEnd; offCur += PAGE_SIZE)
-                {
-                    RTHCPHYS HCPhys;
-                    rc = PGMR3PhysMMIO2GetHCPhys(pVM, pLookup->u.MMIO2.pDevIns, pLookup->u.MMIO2.iRegion, offCur, &HCPhys);
-                    if (RT_FAILURE(rc))
-                        break;
-                    rc = PGMMap(pVM, GCPtr + (offCur - pLookup->u.MMIO2.off), HCPhys, PAGE_SIZE, 0);
-                    if (RT_FAILURE(rc))
-                        break;
-                }
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_DYNAMIC:
-                /* do nothing here since these are either fences or managed by someone else using PGM. */
-                break;
-
-            default:
-                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
-                break;
-        }
-
-        if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("rc=%Rrc cb=%d off=%#RX32 enmType=%d pszDesc=%s\n",
-                             rc, pLookup->cb, pLookup->off, pLookup->enmType, pLookup->pszDesc));
-            return rc;
-        }
-
-        /* next */
-        if (pLookup->offNext == (int32_t)NIL_OFFSET)
-            break;
-        pLookup = (PMMLOOKUPHYPER)((uintptr_t)pLookup + pLookup->offNext);
-    }
-
-    LogFlow(("MMR3HyperInitFinalize: returns VINF_SUCCESS\n"));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Callback function which will be called when PGM is trying to find
- * a new location for the mapping.
- *
- * The callback is called in two modes, 1) the check mode and 2) the relocate mode.
- * In 1) the callback should say if it objects to a suggested new location. If it
- * accepts the new location, it is called again for doing it's relocation.
- *
- *
- * @returns true if the location is ok.
- * @returns false if another location should be found.
- * @param   pVM         The VM handle.
- * @param   GCPtrOld    The old virtual address.
- * @param   GCPtrNew    The new virtual address.
- * @param   enmMode     Used to indicate the callback mode.
- * @param   pvUser      User argument. Ignored.
- * @remark  The return value is no a failure indicator, it's an acceptance
- *          indicator. Relocation can not fail!
- */
-static DECLCALLBACK(bool) mmR3HyperRelocateCallback(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
-{
-    switch (enmMode)
-    {
-        /*
-         * Verify location - all locations are good for us.
-         */
-        case PGMRELOCATECALL_SUGGEST:
-            return true;
-
-        /*
-         * Execute the relocation.
-         */
-        case PGMRELOCATECALL_RELOCATE:
-        {
-            /*
-             * Accepted!
-             */
-            AssertMsg(GCPtrOld == pVM->mm.s.pvHyperAreaGC, ("GCPtrOld=%RGv pVM->mm.s.pvHyperAreaGC=%RGv\n", GCPtrOld, pVM->mm.s.pvHyperAreaGC));
-            Log(("Relocating the hypervisor from %RGv to %RGv\n", GCPtrOld, GCPtrNew));
-
-            /*
-             * Relocate the VM structure and ourselves.
-             */
-            RTGCINTPTR offDelta = GCPtrNew - GCPtrOld;
-            pVM->pVMRC                          += offDelta;
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-                pVM->aCpus[i].pVMRC              = pVM->pVMRC;
-
-            pVM->mm.s.pvHyperAreaGC             += offDelta;
-            Assert(pVM->mm.s.pvHyperAreaGC < _4G);
-            pVM->mm.s.pHyperHeapRC              += offDelta;
-            pVM->mm.s.pHyperHeapR3->pbHeapRC    += offDelta;
-            pVM->mm.s.pHyperHeapR3->pVMRC        = pVM->pVMRC;
-
-            /*
-             * Relocate the rest.
-             */
-            VMR3Relocate(pVM, offDelta);
-            return true;
-        }
-
-        default:
-            AssertMsgFailed(("Invalid relocation mode %d\n", enmMode));
-    }
-
-    return false;
-}
-
-/**
- * Service a VMMCALLRING3_MMHYPER_LOCK call.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-VMMR3DECL(int) MMR3LockCall(PVM pVM)
-{
-    PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
-
-    int rc = PDMR3CritSectEnterEx(&pHeap->Lock, true /* fHostCall */);
-    AssertRC(rc);
-    return rc;
-}
-
-/**
- * Maps contiguous HC physical memory into the hypervisor region in the GC.
- *
- * @return VBox status code.
- *
- * @param   pVM         VM handle.
- * @param   pvR3        Ring-3 address of the memory. Must be page aligned!
- * @param   pvR0        Optional ring-0 address of the memory.
- * @param   HCPhys      Host context physical address of the memory to be
- *                      mapped. Must be page aligned!
- * @param   cb          Size of the memory. Will be rounded up to nearest page.
- * @param   pszDesc     Description.
- * @param   pGCPtr      Where to store the GC address.
- */
-VMMR3DECL(int) MMR3HyperMapHCPhys(PVM pVM, void *pvR3, RTR0PTR pvR0, RTHCPHYS HCPhys, size_t cb, const char *pszDesc, PRTGCPTR pGCPtr)
-{
-    LogFlow(("MMR3HyperMapHCPhys: pvR3=%p pvR0=%p HCPhys=%RHp cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", pvR3, pvR0, HCPhys, (int)cb, pszDesc, pszDesc, pGCPtr));
-
-    /*
-     * Validate input.
-     */
-    AssertReturn(RT_ALIGN_P(pvR3, PAGE_SIZE) == pvR3, VERR_INVALID_PARAMETER);
-    AssertReturn(RT_ALIGN_T(pvR0, PAGE_SIZE, RTR0PTR) == pvR0, VERR_INVALID_PARAMETER);
-    AssertReturn(RT_ALIGN_T(HCPhys, PAGE_SIZE, RTHCPHYS) == HCPhys, VERR_INVALID_PARAMETER);
-    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
-
-    /*
-     * Add the memory to the hypervisor area.
-     */
-    uint32_t cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
-    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
-    RTGCPTR         GCPtr;
-    PMMLOOKUPHYPER  pLookup;
-    int rc = mmR3HyperMap(pVM, cbAligned, pszDesc, &GCPtr, &pLookup);
-    if (RT_SUCCESS(rc))
-    {
-        pLookup->enmType = MMLOOKUPHYPERTYPE_HCPHYS;
-        pLookup->u.HCPhys.pvR3   = pvR3;
-        pLookup->u.HCPhys.pvR0   = pvR0;
-        pLookup->u.HCPhys.HCPhys = HCPhys;
-
-        /*
-         * Update the page table.
-         */
-        if (pVM->mm.s.fPGMInitialized)
-            rc = PGMMap(pVM, GCPtr, HCPhys, cbAligned, 0);
-        if (RT_SUCCESS(rc))
-            *pGCPtr = GCPtr;
-    }
-    return rc;
-}
-
-
-/**
- * Maps contiguous GC physical memory into the hypervisor region in the GC.
- *
- * @return VBox status code.
- *
- * @param   pVM         VM handle.
- * @param   GCPhys      Guest context physical address of the memory to be mapped. Must be page aligned!
- * @param   cb          Size of the memory. Will be rounded up to nearest page.
- * @param   pszDesc     Mapping description.
- * @param   pGCPtr      Where to store the GC address.
- */
-VMMR3DECL(int) MMR3HyperMapGCPhys(PVM pVM, RTGCPHYS GCPhys, size_t cb, const char *pszDesc, PRTGCPTR pGCPtr)
-{
-    LogFlow(("MMR3HyperMapGCPhys: GCPhys=%RGp cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", GCPhys, (int)cb, pszDesc, pszDesc, pGCPtr));
-
-    /*
-     * Validate input.
-     */
-    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
-    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
-
-    /*
-     * Add the memory to the hypervisor area.
-     */
-    cb = RT_ALIGN_Z(cb, PAGE_SIZE);
-    RTGCPTR         GCPtr;
-    PMMLOOKUPHYPER  pLookup;
-    int rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
-    if (RT_SUCCESS(rc))
-    {
-        pLookup->enmType = MMLOOKUPHYPERTYPE_GCPHYS;
-        pLookup->u.GCPhys.GCPhys = GCPhys;
-
-        /*
-         * Update the page table.
-         */
-        for (unsigned off = 0; off < cb; off += PAGE_SIZE)
-        {
-            RTHCPHYS HCPhys;
-            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys + off, &HCPhys);
-            AssertRC(rc);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("rc=%Rrc GCPhys=%RGp off=%#x %s\n", rc, GCPhys, off, pszDesc));
-                break;
-            }
-            if (pVM->mm.s.fPGMInitialized)
-            {
-                rc = PGMMap(pVM, GCPtr + off, HCPhys, PAGE_SIZE, 0);
-                AssertRC(rc);
-                if (RT_FAILURE(rc))
-                {
-                    AssertMsgFailed(("rc=%Rrc GCPhys=%RGp off=%#x %s\n", rc, GCPhys, off, pszDesc));
-                    break;
-                }
-            }
-        }
-
-        if (RT_SUCCESS(rc) && pGCPtr)
-            *pGCPtr = GCPtr;
-    }
-    return rc;
-}
-
-
-/**
- * Maps a portion of an MMIO2 region into the hypervisor region.
- *
- * Callers of this API must never deregister the MMIO2 region before the
- * VM is powered off. If this becomes a requirement MMR3HyperUnmapMMIO2
- * API will be needed to perform cleanups.
- *
- * @return VBox status code.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pDevIns     The device owning the MMIO2 memory.
- * @param   iRegion     The region.
- * @param   off         The offset into the region. Will be rounded down to closest page boundary.
- * @param   cb          The number of bytes to map. Will be rounded up to the closest page boundary.
- * @param   pszDesc     Mapping description.
- * @param   pRCPtr      Where to store the RC address.
- */
-VMMR3DECL(int) MMR3HyperMapMMIO2(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
-                                const char *pszDesc, PRTRCPTR pRCPtr)
-{
-    LogFlow(("MMR3HyperMapMMIO2: pDevIns=%p iRegion=%#x off=%RGp cb=%RGp pszDesc=%p:{%s} pRCPtr=%p\n",
-             pDevIns, iRegion, off, cb, pszDesc, pszDesc, pRCPtr));
-    int rc;
-
-    /*
-     * Validate input.
-     */
-    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
-    AssertReturn(off + cb > off, VERR_INVALID_PARAMETER);
-    uint32_t const offPage = off & PAGE_OFFSET_MASK;
-    off &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
-    cb += offPage;
-    cb = RT_ALIGN_Z(cb, PAGE_SIZE);
-    const RTGCPHYS offEnd = off + cb;
-    AssertReturn(offEnd > off, VERR_INVALID_PARAMETER);
-    for (RTGCPHYS offCur = off; offCur < offEnd; offCur += PAGE_SIZE)
-    {
-        RTHCPHYS HCPhys;
-        rc = PGMR3PhysMMIO2GetHCPhys(pVM, pDevIns, iRegion, offCur, &HCPhys);
-        AssertMsgRCReturn(rc, ("rc=%Rrc - iRegion=%d off=%RGp\n", rc, iRegion, off), rc);
-    }
-
-    /*
-     * Add the memory to the hypervisor area.
-     */
-    RTGCPTR         GCPtr;
-    PMMLOOKUPHYPER  pLookup;
-    rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
-    if (RT_SUCCESS(rc))
-    {
-        pLookup->enmType = MMLOOKUPHYPERTYPE_MMIO2;
-        pLookup->u.MMIO2.pDevIns = pDevIns;
-        pLookup->u.MMIO2.iRegion = iRegion;
-        pLookup->u.MMIO2.off = off;
-
-        /*
-         * Update the page table.
-         */
-        if (pVM->mm.s.fPGMInitialized)
-        {
-            for (RTGCPHYS offCur = off; offCur < offEnd; offCur += PAGE_SIZE)
-            {
-                RTHCPHYS HCPhys;
-                rc = PGMR3PhysMMIO2GetHCPhys(pVM, pDevIns, iRegion, offCur, &HCPhys);
-                AssertRCReturn(rc, VERR_INTERNAL_ERROR);
-                rc = PGMMap(pVM, GCPtr + (offCur - off), HCPhys, PAGE_SIZE, 0);
-                if (RT_FAILURE(rc))
-                {
-                    AssertMsgFailed(("rc=%Rrc offCur=%RGp %s\n", rc, offCur, pszDesc));
-                    break;
-                }
-            }
-        }
-
-        if (RT_SUCCESS(rc))
-        {
-            GCPtr |= offPage;
-            *pRCPtr = GCPtr;
-            AssertLogRelReturn(*pRCPtr == GCPtr, VERR_INTERNAL_ERROR);
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Maps locked R3 virtual memory into the hypervisor region in the GC.
- *
- * @return VBox status code.
- *
- * @param   pVM         VM handle.
- * @param   pvR3        The ring-3 address of the memory, must be page aligned.
- * @param   pvR0        The ring-0 address of the memory, must be page aligned. (optional)
- * @param   cPages      The number of pages.
- * @param   paPages     The page descriptors.
- * @param   pszDesc     Mapping description.
- * @param   pGCPtr      Where to store the GC address corresponding to pvR3.
- */
-VMMR3DECL(int) MMR3HyperMapPages(PVM pVM, void *pvR3, RTR0PTR pvR0, size_t cPages, PCSUPPAGE paPages, const char *pszDesc, PRTGCPTR pGCPtr)
-{
-    LogFlow(("MMR3HyperMapPages: pvR3=%p pvR0=%p cPages=%zu paPages=%p pszDesc=%p:{%s} pGCPtr=%p\n",
-             pvR3, pvR0, cPages, paPages, pszDesc, pszDesc, pGCPtr));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pvR3, VERR_INVALID_POINTER);
-    AssertPtrReturn(paPages, VERR_INVALID_POINTER);
-    AssertReturn(cPages > 0, VERR_PAGE_COUNT_OUT_OF_RANGE);
-    AssertReturn(cPages <= VBOX_MAX_ALLOC_PAGE_COUNT, VERR_PAGE_COUNT_OUT_OF_RANGE);
-    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
-    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pGCPtr, VERR_INVALID_PARAMETER);
-
-    /*
-     * Add the memory to the hypervisor area.
-     */
-    RTGCPTR         GCPtr;
-    PMMLOOKUPHYPER  pLookup;
-    int rc = mmR3HyperMap(pVM, cPages << PAGE_SHIFT, pszDesc, &GCPtr, &pLookup);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Copy the physical page addresses and tell PGM about them.
-         */
-        PRTHCPHYS paHCPhysPages = (PRTHCPHYS)MMR3HeapAlloc(pVM, MM_TAG_MM, sizeof(RTHCPHYS) * cPages);
-        if (paHCPhysPages)
-        {
-            for (size_t i = 0; i < cPages; i++)
-            {
-                AssertReleaseMsgReturn(paPages[i].Phys != 0 && paPages[i].Phys != NIL_RTHCPHYS && !(paPages[i].Phys & PAGE_OFFSET_MASK),
-                                       ("i=%#zx Phys=%RHp %s\n", i, paPages[i].Phys, pszDesc),
-                                       VERR_INTERNAL_ERROR);
-                paHCPhysPages[i] = paPages[i].Phys;
-            }
-
-            if (pVM->mm.s.fPGMInitialized)
-            {
-                for (size_t i = 0; i < cPages; i++)
-                {
-                    rc = PGMMap(pVM, GCPtr + (i << PAGE_SHIFT), paHCPhysPages[i], PAGE_SIZE, 0);
-                    AssertRCBreak(rc);
-                }
-            }
-            if (RT_SUCCESS(rc))
-            {
-                pLookup->enmType = MMLOOKUPHYPERTYPE_LOCKED;
-                pLookup->u.Locked.pvR3          = pvR3;
-                pLookup->u.Locked.pvR0          = pvR0;
-                pLookup->u.Locked.paHCPhysPages = paHCPhysPages;
-
-                /* done. */
-                *pGCPtr   = GCPtr;
-                return rc;
-            }
-            /* Don't care about failure clean, we're screwed if this fails anyway. */
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * Reserves a hypervisor memory area.
- * Most frequent usage is fence pages and dynamically mappings like the guest PD and PDPT.
- *
- * @return VBox status code.
- *
- * @param   pVM         VM handle.
- * @param   cb          Size of the memory. Will be rounded up to nearest page.
- * @param   pszDesc     Mapping description.
- * @param   pGCPtr      Where to store the assigned GC address. Optional.
- */
-VMMR3DECL(int) MMR3HyperReserve(PVM pVM, unsigned cb, const char *pszDesc, PRTGCPTR pGCPtr)
-{
-    LogFlow(("MMR3HyperMapHCRam: cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", (int)cb, pszDesc, pszDesc, pGCPtr));
-
-    /*
-     * Validate input.
-     */
-    if (    cb <= 0
-        ||  !pszDesc
-        ||  !*pszDesc)
-    {
-        AssertMsgFailed(("Invalid parameter\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Add the memory to the hypervisor area.
-     */
-    RTGCPTR         GCPtr;
-    PMMLOOKUPHYPER  pLookup;
-    int rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
-    if (RT_SUCCESS(rc))
-    {
-        pLookup->enmType = MMLOOKUPHYPERTYPE_DYNAMIC;
-        if (pGCPtr)
-            *pGCPtr = GCPtr;
-        return VINF_SUCCESS;
-    }
-    return rc;
-}
-
-
-/**
- * Adds memory to the hypervisor memory arena.
- *
- * @return VBox status code.
- * @param   pVM         The VM handle.
- * @param   cb          Size of the memory. Will be rounded up to nearest page.
- * @param   pszDesc     The description of the memory.
- * @param   pGCPtr      Where to store the GC address.
- * @param   ppLookup    Where to store the pointer to the lookup record.
- * @remark  We assume the threading structure of VBox imposes natural
- *          serialization of most functions, this one included.
- */
-static int mmR3HyperMap(PVM pVM, const size_t cb, const char *pszDesc, PRTGCPTR pGCPtr, PMMLOOKUPHYPER *ppLookup)
-{
-    /*
-     * Validate input.
-     */
-    const uint32_t cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
-    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
-    if (pVM->mm.s.offHyperNextStatic + cbAligned >= pVM->mm.s.cbHyperArea) /* don't use the last page, it's a fence. */
-    {
-        AssertMsgFailed(("Out of static mapping space in the HMA! offHyperAreaGC=%x cbAligned=%x cbHyperArea=%x\n",
-                         pVM->mm.s.offHyperNextStatic, cbAligned, pVM->mm.s.cbHyperArea));
-        return VERR_NO_MEMORY;
-    }
-
-    /*
-     * Allocate lookup record.
-     */
-    PMMLOOKUPHYPER  pLookup;
-    int rc = MMHyperAlloc(pVM, sizeof(*pLookup), 1, MM_TAG_MM, (void **)&pLookup);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Initialize it and insert it.
-         */
-        pLookup->offNext        = pVM->mm.s.offLookupHyper;
-        pLookup->cb             = cbAligned;
-        pLookup->off            = pVM->mm.s.offHyperNextStatic;
-        pVM->mm.s.offLookupHyper = (uint8_t *)pLookup - (uint8_t *)pVM->mm.s.pHyperHeapR3;
-        if (pLookup->offNext != (int32_t)NIL_OFFSET)
-            pLookup->offNext   -= pVM->mm.s.offLookupHyper;
-        pLookup->enmType        = MMLOOKUPHYPERTYPE_INVALID;
-        memset(&pLookup->u, 0xff, sizeof(pLookup->u));
-        pLookup->pszDesc        = pszDesc;
-
-        /* Mapping. */
-        *pGCPtr = pVM->mm.s.pvHyperAreaGC + pVM->mm.s.offHyperNextStatic;
-        pVM->mm.s.offHyperNextStatic += cbAligned;
-
-        /* Return pointer. */
-        *ppLookup = pLookup;
-    }
-
-    AssertRC(rc);
-    LogFlow(("mmR3HyperMap: returns %Rrc *pGCPtr=%RGv\n", rc, *pGCPtr));
-    return rc;
-}
-
-
-/**
- * Allocates a new heap.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   cb          The size of the new heap.
- * @param   ppHeap      Where to store the heap pointer on successful return.
- * @param   pR0PtrHeap  Where to store the ring-0 address of the heap on
- *                      success.
- */
-static int mmR3HyperHeapCreate(PVM pVM, const size_t cb, PMMHYPERHEAP *ppHeap, PRTR0PTR pR0PtrHeap)
-{
-    /*
-     * Allocate the hypervisor heap.
-     */
-    const uint32_t  cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
-    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
-    uint32_t const  cPages = cbAligned >> PAGE_SHIFT;
-    PSUPPAGE        paPages = (PSUPPAGE)MMR3HeapAlloc(pVM, MM_TAG_MM, cPages * sizeof(paPages[0]));
-    if (!paPages)
-        return VERR_NO_MEMORY;
-    void           *pv;
-    RTR0PTR         pvR0 = NIL_RTR0PTR;
-    int rc = SUPR3PageAllocEx(cPages,
-                              0 /*fFlags*/,
-                              &pv,
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-                              VMMIsHwVirtExtForced(pVM) ? &pvR0 : NULL,
-#else
-                              NULL,
-#endif
-                              paPages);
-    if (RT_SUCCESS(rc))
-    {
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-        if (!VMMIsHwVirtExtForced(pVM))
-            pvR0 = NIL_RTR0PTR;
-#else
-        pvR0 = (uintptr_t)pv;
-#endif
-        memset(pv, 0, cbAligned);
-
-        /*
-         * Initialize the heap and first free chunk.
-         */
-        PMMHYPERHEAP pHeap = (PMMHYPERHEAP)pv;
-        pHeap->u32Magic             = MMHYPERHEAP_MAGIC;
-        pHeap->pbHeapR3             = (uint8_t *)pHeap + MMYPERHEAP_HDR_SIZE;
-        pHeap->pbHeapR0             = pvR0 != NIL_RTR0PTR ? pvR0 + MMYPERHEAP_HDR_SIZE : NIL_RTR0PTR;
-        //pHeap->pbHeapRC           = 0; // set by mmR3HyperHeapMap()
-        pHeap->pVMR3                = pVM;
-        pHeap->pVMR0                = pVM->pVMR0;
-        pHeap->pVMRC                = pVM->pVMRC;
-        pHeap->cbHeap               = cbAligned - MMYPERHEAP_HDR_SIZE;
-        pHeap->cbFree               = pHeap->cbHeap - sizeof(MMHYPERCHUNK);
-        //pHeap->offFreeHead        = 0;
-        //pHeap->offFreeTail        = 0;
-        pHeap->offPageAligned       = pHeap->cbHeap;
-        //pHeap->HyperHeapStatTree  = 0;
-        pHeap->paPages              = paPages;
-
-        PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pHeap->pbHeapR3;
-        pFree->cb                   = pHeap->cbFree;
-        //pFree->core.offNext       = 0;
-        MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
-        pFree->core.offHeap         = -(int32_t)MMYPERHEAP_HDR_SIZE;
-        //pFree->offNext            = 0;
-        //pFree->offPrev            = 0;
-
-        STAMR3Register(pVM, &pHeap->cbHeap, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, "/MM/HyperHeap/cbHeap",  STAMUNIT_BYTES, "The heap size.");
-        STAMR3Register(pVM, &pHeap->cbFree, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, "/MM/HyperHeap/cbFree",  STAMUNIT_BYTES, "The free space.");
-
-        *ppHeap = pHeap;
-        *pR0PtrHeap = pvR0;
-        return VINF_SUCCESS;
-    }
-    AssertMsgFailed(("SUPR3PageAllocEx(%d,,,,) -> %Rrc\n", cbAligned >> PAGE_SHIFT, rc));
-
-    *ppHeap = NULL;
-    return rc;
-}
-
-/**
- * Allocates a new heap.
- */
-static int mmR3HyperHeapMap(PVM pVM, PMMHYPERHEAP pHeap, PRTGCPTR ppHeapGC)
-{
-    Assert(RT_ALIGN_Z(pHeap->cbHeap + MMYPERHEAP_HDR_SIZE, PAGE_SIZE) == pHeap->cbHeap + MMYPERHEAP_HDR_SIZE);
-    Assert(pHeap->paPages);
-    int rc = MMR3HyperMapPages(pVM,
-                               pHeap,
-                               pHeap->pbHeapR0 != NIL_RTR0PTR ? pHeap->pbHeapR0 - MMYPERHEAP_HDR_SIZE : NIL_RTR0PTR,
-                               (pHeap->cbHeap + MMYPERHEAP_HDR_SIZE) >> PAGE_SHIFT,
-                               pHeap->paPages,
-                               "Heap", ppHeapGC);
-    if (RT_SUCCESS(rc))
-    {
-        pHeap->pVMRC    = pVM->pVMRC;
-        pHeap->pbHeapRC = *ppHeapGC + MMYPERHEAP_HDR_SIZE;
-        /* Reserve a page for fencing. */
-        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-
-        /* We won't need these any more. */
-        MMR3HeapFree(pHeap->paPages);
-        pHeap->paPages = NULL;
-    }
-    return rc;
-}
-
-
-/**
- * Allocates memory in the Hypervisor (GC VMM) area which never will
- * be freed and doesn't have any offset based relation to other heap blocks.
- *
- * The latter means that two blocks allocated by this API will not have the
- * same relative position to each other in GC and HC. In short, never use
- * this API for allocating nodes for an offset based AVL tree!
- *
- * The returned memory is of course zeroed.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   cb          Number of bytes to allocate.
- * @param   uAlignment  Required memory alignment in bytes.
- *                      Values are 0,8,16,32 and PAGE_SIZE.
- *                      0 -> default alignment, i.e. 8 bytes.
- * @param   enmTag      The statistics tag.
- * @param   ppv         Where to store the address to the allocated
- *                      memory.
- * @remark  This is assumed not to be used at times when serialization is required.
- */
-VMMR3DECL(int) MMR3HyperAllocOnceNoRel(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
-{
-    return MMR3HyperAllocOnceNoRelEx(pVM, cb, uAlignment, enmTag, 0/*fFlags*/, ppv);
-}
-
-
-/**
- * Allocates memory in the Hypervisor (GC VMM) area which never will
- * be freed and doesn't have any offset based relation to other heap blocks.
- *
- * The latter means that two blocks allocated by this API will not have the
- * same relative position to each other in GC and HC. In short, never use
- * this API for allocating nodes for an offset based AVL tree!
- *
- * The returned memory is of course zeroed.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   cb          Number of bytes to allocate.
- * @param   uAlignment  Required memory alignment in bytes.
- *                      Values are 0,8,16,32 and PAGE_SIZE.
- *                      0 -> default alignment, i.e. 8 bytes.
- * @param   enmTag      The statistics tag.
- * @param   fFlags      Flags, see MMHYPER_AONR_FLAGS_KERNEL_MAPPING.
- * @param   ppv         Where to store the address to the allocated memory.
- * @remark  This is assumed not to be used at times when serialization is required.
- */
-VMMR3DECL(int) MMR3HyperAllocOnceNoRelEx(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, uint32_t fFlags, void **ppv)
-{
-    AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
-    Assert(!(fFlags & ~(MMHYPER_AONR_FLAGS_KERNEL_MAPPING)));
-
-    /*
-     * Choose between allocating a new chunk of HMA memory
-     * and the heap. We will only do BIG allocations from HMA and
-     * only at creation time.
-     */
-    if (   (   cb < _64K
-            && (   uAlignment != PAGE_SIZE
-                || cb < 48*_1K)
-            && !(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING)
-           )
-        ||  VMR3GetState(pVM) != VMSTATE_CREATING
-       )
-    {
-        Assert(!(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING));
-        int rc = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
-        if (    rc != VERR_MM_HYPER_NO_MEMORY
-            ||  cb <= 8*_1K)
-        {
-            Log2(("MMR3HyperAllocOnceNoRel: cb=%#zx uAlignment=%#x returns %Rrc and *ppv=%p\n",
-                  cb, uAlignment, rc, *ppv));
-            return rc;
-        }
-    }
-
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    /*
-     * Set MMHYPER_AONR_FLAGS_KERNEL_MAPPING if we're in going to execute in ring-0.
-     */
-    if (VMMIsHwVirtExtForced(pVM))
-        fFlags |= MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
-#endif
-
-    /*
-     * Validate alignment.
-     */
-    switch (uAlignment)
-    {
-        case 0:
-        case 8:
-        case 16:
-        case 32:
-        case PAGE_SIZE:
-            break;
-        default:
-            AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
-            return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Allocate the pages and map them into HMA space.
-     */
-    uint32_t const  cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
-    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
-    uint32_t const  cPages    = cbAligned >> PAGE_SHIFT;
-    PSUPPAGE        paPages   = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(paPages[0]));
-    if (!paPages)
-        return VERR_NO_TMP_MEMORY;
-    void           *pvPages;
-    RTR0PTR         pvR0 = NIL_RTR0PTR;
-    int rc = SUPR3PageAllocEx(cPages,
-                              0 /*fFlags*/,
-                              &pvPages,
-                              fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING ? &pvR0 : NULL,
-                              paPages);
-    if (RT_SUCCESS(rc))
-    {
-        if (!(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING))
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-            pvR0 = NIL_RTR0PTR;
-#else
-            pvR0 = (RTR0PTR)pvPages;
-#endif
-
-        memset(pvPages, 0, cbAligned);
-
-        RTGCPTR GCPtr;
-        rc = MMR3HyperMapPages(pVM,
-                               pvPages,
-                               pvR0,
-                               cPages,
-                               paPages,
-                               MMR3HeapAPrintf(pVM, MM_TAG_MM, "alloc once (%s)", mmGetTagName(enmTag)),
-                               &GCPtr);
-        if (RT_SUCCESS(rc))
-        {
-            *ppv = pvPages;
-            Log2(("MMR3HyperAllocOnceNoRel: cbAligned=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n",
-                  cbAligned, uAlignment, *ppv));
-            MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-            return rc;
-        }
-        AssertMsgFailed(("Failed to allocate %zd bytes! %Rrc\n", cbAligned, rc));
-        SUPR3PageFreeEx(pvPages, cPages);
-
-
-        /*
-         * HACK ALERT! Try allocate it off the heap so that we don't freak
-         * out during vga/vmmdev mmio2 allocation with certain ram sizes.
-         */
-        /** @todo make a proper fix for this so we will never end up in this kind of situation! */
-        Log(("MMR3HyperAllocOnceNoRel: MMR3HyperMapHCRam failed with rc=%Rrc, try MMHyperAlloc(,%#x,,) instead\n",  rc, cb));
-        int rc2 = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
-        if (RT_SUCCESS(rc2))
-        {
-            Log2(("MMR3HyperAllocOnceNoRel: cb=%#x uAlignment=%#x returns %Rrc and *ppv=%p\n",
-                  cb, uAlignment, rc, *ppv));
-            return rc;
-        }
-    }
-    else
-        AssertMsgFailed(("Failed to allocate %zd bytes! %Rrc\n", cbAligned, rc));
-
-    if (rc == VERR_NO_MEMORY)
-        rc = VERR_MM_HYPER_NO_MEMORY;
-    LogRel(("MMR3HyperAllocOnceNoRel: cb=%#zx uAlignment=%#x returns %Rrc\n", cb, uAlignment, rc));
-    return rc;
-}
-
-
-/**
- * Lookus up a ring-3 pointer to HMA.
- *
- * @returns The lookup record on success, NULL on failure.
- * @param   pVM                 The VM handle.
- * @param   pvR3                The ring-3 address to look up.
- */
-DECLINLINE(PMMLOOKUPHYPER) mmR3HyperLookupR3(PVM pVM, void *pvR3)
-{
-    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
-    for (;;)
-    {
-        switch (pLookup->enmType)
-        {
-            case MMLOOKUPHYPERTYPE_LOCKED:
-            {
-                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.Locked.pvR3;
-                if (off < pLookup->cb)
-                    return pLookup;
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_HCPHYS:
-            {
-                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.HCPhys.pvR3;
-                if (off < pLookup->cb)
-                    return pLookup;
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_GCPHYS:
-            case MMLOOKUPHYPERTYPE_MMIO2:
-            case MMLOOKUPHYPERTYPE_DYNAMIC:
-                /** @todo ?    */
-                break;
-
-            default:
-                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
-                return NULL;
-        }
-
-        /* next */
-        if ((unsigned)pLookup->offNext == NIL_OFFSET)
-            return NULL;
-        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
-    }
-}
-
-
-/**
- * Set / unset guard status on one or more hyper heap pages.
- *
- * @returns VBox status code (first failure).
- * @param   pVM                 The VM handle.
- * @param   pvStart             The hyper heap page address. Must be page
- *                              aligned.
- * @param   cb                  The number of bytes. Must be page aligned.
- * @param   fSet                Whether to set or unset guard page status.
- */
-VMMR3DECL(int) MMR3HyperSetGuard(PVM pVM, void *pvStart, size_t cb, bool fSet)
-{
-    /*
-     * Validate input.
-     */
-    AssertReturn(!((uintptr_t)pvStart & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
-    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(cb <= UINT32_MAX, VERR_INVALID_PARAMETER);
-    PMMLOOKUPHYPER pLookup = mmR3HyperLookupR3(pVM, pvStart);
-    AssertReturn(pLookup, VERR_INVALID_PARAMETER);
-    AssertReturn(pLookup->enmType == MMLOOKUPHYPERTYPE_LOCKED, VERR_INVALID_PARAMETER);
-
-    /*
-     * Get down to business.
-     * Note! We quietly ignore errors from the support library since the
-     *       protection stuff isn't possible to implement on all platforms.
-     */
-    uint8_t    *pbR3  = (uint8_t *)pLookup->u.Locked.pvR3;
-    RTR0PTR     R0Ptr = pLookup->u.Locked.pvR0 != (uintptr_t)pLookup->u.Locked.pvR3
-                      ? pLookup->u.Locked.pvR0
-                      : NIL_RTR0PTR;
-    uint32_t    off   = (uint32_t)((uint8_t *)pvStart - pbR3);
-    int         rc;
-    if (fSet)
-    {
-        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, pvStart), cb, 0);
-        SUPR3PageProtect(pbR3, R0Ptr, off, (uint32_t)cb, RTMEM_PROT_NONE);
-    }
-    else
-    {
-        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, pvStart), cb, X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
-        SUPR3PageProtect(pbR3, R0Ptr, off, (uint32_t)cb, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
-    }
-    return rc;
-}
-
-
-/**
- * Convert hypervisor HC virtual address to HC physical address.
- *
- * @returns HC physical address.
- * @param   pVM         VM Handle
- * @param   pvR3        Host context virtual address.
- */
-VMMR3DECL(RTHCPHYS) MMR3HyperHCVirt2HCPhys(PVM pVM, void *pvR3)
-{
-    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
-    for (;;)
-    {
-        switch (pLookup->enmType)
-        {
-            case MMLOOKUPHYPERTYPE_LOCKED:
-            {
-                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.Locked.pvR3;
-                if (off < pLookup->cb)
-                    return pLookup->u.Locked.paHCPhysPages[off >> PAGE_SHIFT] | (off & PAGE_OFFSET_MASK);
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_HCPHYS:
-            {
-                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.HCPhys.pvR3;
-                if (off < pLookup->cb)
-                    return pLookup->u.HCPhys.HCPhys + off;
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_GCPHYS:
-            case MMLOOKUPHYPERTYPE_MMIO2:
-            case MMLOOKUPHYPERTYPE_DYNAMIC:
-                /* can (or don't want to) convert these kind of records. */
-                break;
-
-            default:
-                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
-                break;
-        }
-
-        /* next */
-        if ((unsigned)pLookup->offNext == NIL_OFFSET)
-            break;
-        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
-    }
-
-    AssertMsgFailed(("pvR3=%p is not inside the hypervisor memory area!\n", pvR3));
-    return NIL_RTHCPHYS;
-}
-
-
-/**
- * Implements the return case of MMR3HyperQueryInfoFromHCPhys.
- *
- * @returns VINF_SUCCESS, VINF_BUFFER_OVERFLOW.
- * @param   pVM                 The VM handle.
- * @param   HCPhys              The host physical address to look for.
- * @param   pLookup             The HMA lookup entry corresponding to HCPhys.
- * @param   pszWhat             Where to return the description.
- * @param   cbWhat              Size of the return buffer.
- * @param   pcbAlloc            Where to return the size of whatever it is.
- */
-static int mmR3HyperQueryInfoFromHCPhysFound(PVM pVM, RTHCPHYS HCPhys, PMMLOOKUPHYPER pLookup,
-                                             char *pszWhat, size_t cbWhat, uint32_t *pcbAlloc)
-{
-    *pcbAlloc = pLookup->cb;
-    int rc = RTStrCopy(pszWhat, cbWhat, pLookup->pszDesc);
-    return rc == VERR_BUFFER_OVERFLOW ? VINF_BUFFER_OVERFLOW : rc;
-}
-
-
-/**
- * Scans the HMA for the physical page and reports back a description if found.
- *
- * @returns VINF_SUCCESS, VINF_BUFFER_OVERFLOW, VERR_NOT_FOUND.
- * @param   pVM                 The VM handle.
- * @param   HCPhys              The host physical address to look for.
- * @param   pszWhat             Where to return the description.
- * @param   cbWhat              Size of the return buffer.
- * @param   pcbAlloc            Where to return the size of whatever it is.
- */
-VMMR3_INT_DECL(int) MMR3HyperQueryInfoFromHCPhys(PVM pVM, RTHCPHYS HCPhys, char *pszWhat, size_t cbWhat, uint32_t *pcbAlloc)
-{
-    RTHCPHYS        HCPhysPage = HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK;
-    PMMLOOKUPHYPER  pLookup    = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
-    for (;;)
-    {
-        switch (pLookup->enmType)
-        {
-            case MMLOOKUPHYPERTYPE_LOCKED:
-            {
-                uint32_t i = pLookup->cb >> PAGE_SHIFT;
-                while (i-- > 0)
-                    if (pLookup->u.Locked.paHCPhysPages[i] == HCPhysPage)
-                        return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_HCPHYS:
-            {
-                if (pLookup->u.HCPhys.HCPhys - HCPhysPage < pLookup->cb)
-                    return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
-                break;
-            }
-
-            case MMLOOKUPHYPERTYPE_MMIO2:
-            case MMLOOKUPHYPERTYPE_GCPHYS:
-            case MMLOOKUPHYPERTYPE_DYNAMIC:
-            {
-                /* brute force. */
-                uint32_t i = pLookup->cb >> PAGE_SHIFT;
-                while (i-- > 0)
-                {
-                    RTGCPTR     GCPtr = pLookup->off + pVM->mm.s.pvHyperAreaGC;
-                    RTHCPHYS    HCPhysCur;
-                    int rc = PGMMapGetPage(pVM, GCPtr, NULL, &HCPhysCur);
-                    if (RT_SUCCESS(rc) && HCPhysCur == HCPhysPage)
-                        return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
-                }
-                break;
-            }
-            default:
-                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
-                break;
-        }
-
-        /* next */
-        if ((unsigned)pLookup->offNext == NIL_OFFSET)
-            break;
-        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
-    }
-    return VERR_NOT_FOUND;
-}
-
-
-#if 0 /* unused, not implemented */
-/**
- * Convert hypervisor HC physical address to HC virtual address.
- *
- * @returns HC virtual address.
- * @param   pVM         VM Handle
- * @param   HCPhys      Host context physical address.
- */
-VMMR3DECL(void *) MMR3HyperHCPhys2HCVirt(PVM pVM, RTHCPHYS HCPhys)
-{
-    void *pv;
-    int rc = MMR3HyperHCPhys2HCVirtEx(pVM, HCPhys, &pv);
-    if (RT_SUCCESS(rc))
-        return pv;
-    AssertMsgFailed(("Invalid address HCPhys=%x rc=%d\n", HCPhys, rc));
-    return NULL;
-}
-
-
-/**
- * Convert hypervisor HC physical address to HC virtual address.
- *
- * @returns VBox status.
- * @param   pVM         VM Handle
- * @param   HCPhys      Host context physical address.
- * @param   ppv         Where to store the HC virtual address.
- */
-VMMR3DECL(int)   MMR3HyperHCPhys2HCVirtEx(PVM pVM, RTHCPHYS HCPhys, void **ppv)
-{
-    /*
-     * Linear search.
-     */
-    /** @todo implement when actually used. */
-    return VERR_INVALID_POINTER;
-}
-#endif /* unused, not implemented */
-
-
-/**
- * Read hypervisor memory from GC virtual address.
- *
- * @returns VBox status.
- * @param   pVM         VM handle.
- * @param   pvDst       Destination address (HC of course).
- * @param   GCPtr       GC virtual address.
- * @param   cb          Number of bytes to read.
- *
- * @remarks For DBGF only.
- */
-VMMR3DECL(int) MMR3HyperReadGCVirt(PVM pVM, void *pvDst, RTGCPTR GCPtr, size_t cb)
-{
-    if (GCPtr - pVM->mm.s.pvHyperAreaGC >= pVM->mm.s.cbHyperArea)
-        return VERR_INVALID_POINTER;
-    return PGMR3MapRead(pVM, pvDst, GCPtr, cb);
-}
-
-
-/**
- * Info handler for 'hma', it dumps the list of lookup records for the hypervisor memory area.
- *
- * @param   pVM         The VM handle.
- * @param   pHlp        Callback functions for doing output.
- * @param   pszArgs     Argument string. Optional and specific to the handler.
- */
-static DECLCALLBACK(void) mmR3HyperInfoHma(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    pHlp->pfnPrintf(pHlp, "Hypervisor Memory Area (HMA) Layout: Base %RGv, 0x%08x bytes\n",
-                    pVM->mm.s.pvHyperAreaGC, pVM->mm.s.cbHyperArea);
-
-    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
-    for (;;)
-    {
-        switch (pLookup->enmType)
-        {
-            case MMLOOKUPHYPERTYPE_LOCKED:
-                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %RHv %RHv LOCKED  %-*s %s\n",
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
-                                pLookup->u.Locked.pvR3,
-                                pLookup->u.Locked.pvR0,
-                                sizeof(RTHCPTR) * 2, "",
-                                pLookup->pszDesc);
-                break;
-
-            case MMLOOKUPHYPERTYPE_HCPHYS:
-                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %RHv %RHv HCPHYS  %RHp %s\n",
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
-                                pLookup->u.HCPhys.pvR3,
-                                pLookup->u.HCPhys.pvR0,
-                                pLookup->u.HCPhys.HCPhys,
-                                pLookup->pszDesc);
-                break;
-
-            case MMLOOKUPHYPERTYPE_GCPHYS:
-                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s GCPHYS  %RGp%*s %s\n",
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
-                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
-                                pLookup->u.GCPhys.GCPhys, RT_ABS((int)(sizeof(RTHCPHYS) - sizeof(RTGCPHYS))) * 2, "",
-                                pLookup->pszDesc);
-                break;
-
-            case MMLOOKUPHYPERTYPE_MMIO2:
-                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s MMIO2   %RGp%*s %s\n",
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
-                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
-                                pLookup->u.MMIO2.off, RT_ABS((int)(sizeof(RTHCPHYS) - sizeof(RTGCPHYS))) * 2, "",
-                                pLookup->pszDesc);
-                break;
-
-            case MMLOOKUPHYPERTYPE_DYNAMIC:
-                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s DYNAMIC %*s %s\n",
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
-                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
-                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
-                                sizeof(RTHCPTR) * 2, "",
-                                pLookup->pszDesc);
-                break;
-
-            default:
-                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
-                break;
-        }
-
-        /* next */
-        if ((unsigned)pLookup->offNext == NIL_OFFSET)
-            break;
-        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
-    }
-}
Index: unk/src/VBox/VMM/MMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/MMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,848 +1,0 @@
-/* $Id$ */
-/** @file
- * MM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___MMInternal_h
-#define ___MMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/sup.h>
-#include <VBox/stam.h>
-#include <VBox/pdmcritsect.h>
-#include <iprt/assert.h>
-#include <iprt/avl.h>
-#include <iprt/critsect.h>
-
-
-
-/** @defgroup grp_mm_int   Internals
- * @internal
- * @ingroup grp_mm
- * @{
- */
-
-
-/** @name MMR3Heap - VM Ring-3 Heap Internals
- * @{
- */
-
-/** @def MMR3HEAP_SIZE_ALIGNMENT
- * The allocation size alignment of the MMR3Heap.
- */
-#define MMR3HEAP_SIZE_ALIGNMENT     16
-
-/** @def MMR3HEAP_WITH_STATISTICS
- * Enable MMR3Heap statistics.
- */
-#if !defined(MMR3HEAP_WITH_STATISTICS) && defined(VBOX_WITH_STATISTICS)
-# define MMR3HEAP_WITH_STATISTICS
-#endif
-
-/**
- * Heap statistics record.
- * There is one global and one per allocation tag.
- */
-typedef struct MMHEAPSTAT
-{
-    /** Core avl node, key is the tag. */
-    AVLULNODECORE           Core;
-    /** Pointer to the heap the memory belongs to. */
-    struct MMHEAP          *pHeap;
-#ifdef MMR3HEAP_WITH_STATISTICS
-# if HC_ARCH_BITS == 32
-    /** Aligning the statistics on an 8 byte boundary (for uint64_t and STAM). */
-    void                   *pvAlignment;
-# endif
-    /** Number of allocation. */
-    uint64_t                cAllocations;
-    /** Number of reallocations. */
-    uint64_t                cReallocations;
-    /** Number of frees. */
-    uint64_t                cFrees;
-    /** Failures. */
-    uint64_t                cFailures;
-    /** Number of bytes allocated (sum). */
-    uint64_t                cbAllocated;
-    /** Number of bytes freed. */
-    uint64_t                cbFreed;
-    /** Number of bytes currently allocated. */
-    size_t                  cbCurAllocated;
-#endif
-} MMHEAPSTAT;
-#if defined(MMR3HEAP_WITH_STATISTICS) && defined(IN_RING3)
-AssertCompileMemberAlignment(MMHEAPSTAT, cAllocations, 8);
-#endif
-/** Pointer to heap statistics record. */
-typedef MMHEAPSTAT *PMMHEAPSTAT;
-
-
-
-
-/**
- * Additional heap block header for relating allocations to the VM.
- */
-typedef struct MMHEAPHDR
-{
-    /** Pointer to the next record. */
-    struct MMHEAPHDR       *pNext;
-    /** Pointer to the previous record. */
-    struct MMHEAPHDR       *pPrev;
-    /** Pointer to the heap statistics record.
-     * (Where the a PVM can be found.) */
-    PMMHEAPSTAT             pStat;
-    /** Size of the allocation (including this header). */
-    size_t                  cbSize;
-} MMHEAPHDR;
-/** Pointer to MM heap header. */
-typedef MMHEAPHDR *PMMHEAPHDR;
-
-
-/** MM Heap structure. */
-typedef struct MMHEAP
-{
-    /** Lock protecting the heap. */
-    RTCRITSECT              Lock;
-    /** Heap block list head. */
-    PMMHEAPHDR              pHead;
-    /** Heap block list tail. */
-    PMMHEAPHDR              pTail;
-    /** Heap per tag statistics tree. */
-    PAVLULNODECORE          pStatTree;
-    /** The VM handle. */
-    PUVM                    pUVM;
-    /** Heap global statistics. */
-    MMHEAPSTAT              Stat;
-} MMHEAP;
-/** Pointer to MM Heap structure. */
-typedef MMHEAP *PMMHEAP;
-
-/** @} */
-
-
-/** @name MMUkHeap - VM User-kernel Heap Internals
- * @{
- */
-
-/** @def MMUKHEAP_SIZE_ALIGNMENT
- * The allocation size alignment of the MMR3UkHeap.
- */
-#define MMUKHEAP_SIZE_ALIGNMENT   16
-
-/** @def MMUKHEAP_WITH_STATISTICS
- * Enable MMUkHeap statistics.
- */
-#if !defined(MMUKHEAP_WITH_STATISTICS) && defined(VBOX_WITH_STATISTICS)
-# define MMUKHEAP_WITH_STATISTICS
-#endif
-
-
-/**
- * Heap statistics record.
- * There is one global and one per allocation tag.
- */
-typedef struct MMUKHEAPSTAT
-{
-    /** Core avl node, key is the tag. */
-    AVLULNODECORE           Core;
-    /** Number of allocation. */
-    uint64_t                cAllocations;
-    /** Number of reallocations. */
-    uint64_t                cReallocations;
-    /** Number of frees. */
-    uint64_t                cFrees;
-    /** Failures. */
-    uint64_t                cFailures;
-    /** Number of bytes allocated (sum). */
-    uint64_t                cbAllocated;
-    /** Number of bytes freed. */
-    uint64_t                cbFreed;
-    /** Number of bytes currently allocated. */
-    size_t                  cbCurAllocated;
-} MMUKHEAPSTAT;
-#ifdef IN_RING3
-AssertCompileMemberAlignment(MMUKHEAPSTAT, cAllocations, 8);
-#endif
-/** Pointer to heap statistics record. */
-typedef MMUKHEAPSTAT *PMMUKHEAPSTAT;
-
-/**
- * Sub heap tracking record.
- */
-typedef struct MMUKHEAPSUB
-{
-    /** Pointer to the next sub-heap. */
-    struct MMUKHEAPSUB     *pNext;
-    /** The base address of the sub-heap. */
-    void                   *pv;
-    /** The size of the sub-heap.  */
-    size_t                  cb;
-    /** The handle of the simple block pointer. */
-    RTHEAPSIMPLE            hSimple;
-    /** The ring-0 address corresponding to MMUKHEAPSUB::pv. */
-    RTR0PTR                 pvR0;
-} MMUKHEAPSUB;
-/** Pointer to a sub-heap tracking record. */
-typedef MMUKHEAPSUB *PMMUKHEAPSUB;
-
-
-/** MM User-kernel Heap structure. */
-typedef struct MMUKHEAP
-{
-    /** Lock protecting the heap. */
-    RTCRITSECT              Lock;
-    /** Head of the sub-heap LIFO. */
-    PMMUKHEAPSUB            pSubHeapHead;
-    /** Heap per tag statistics tree. */
-    PAVLULNODECORE          pStatTree;
-    /** The VM handle. */
-    PUVM                    pUVM;
-#if HC_ARCH_BITS == 32
-    /** Aligning the statistics on an 8 byte boundary (for uint64_t and STAM). */
-    void                   *pvAlignment;
-#endif
-    /** Heap global statistics. */
-    MMUKHEAPSTAT            Stat;
-} MMUKHEAP;
-#ifdef IN_RING3
-AssertCompileMemberAlignment(MMUKHEAP, Stat, 8);
-#endif
-/** Pointer to MM Heap structure. */
-typedef MMUKHEAP *PMMUKHEAP;
-
-/** @} */
-
-
-
-/** @name Hypervisor Heap Internals
- * @{
- */
-
-/** @def MMHYPER_HEAP_FREE_DELAY
- * If defined, it indicates the number of frees that should be delayed.
- */
-#if defined(DOXYGEN_RUNNING)
-# define MMHYPER_HEAP_FREE_DELAY            64
-#endif
-
-/** @def MMHYPER_HEAP_FREE_POISON
- * If defined, it indicates that freed memory should be poisoned
- * with the value it has.
- */
-#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
-# define MMHYPER_HEAP_FREE_POISON           0xcb
-#endif
-
-/** @def MMHYPER_HEAP_STRICT
- * Enables a bunch of assertions in the heap code. */
-#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
-# define MMHYPER_HEAP_STRICT 1
-# if 0 || defined(DOXYGEN_RUNNING)
-/** @def MMHYPER_HEAP_STRICT_FENCE
- * Enables tail fence. */
-#  define MMHYPER_HEAP_STRICT_FENCE
-/** @def MMHYPER_HEAP_STRICT_FENCE_SIZE
- * The fence size in bytes. */
-#  define MMHYPER_HEAP_STRICT_FENCE_SIZE    256
-/** @def MMHYPER_HEAP_STRICT_FENCE_U32
- * The fence filler. */
-#  define MMHYPER_HEAP_STRICT_FENCE_U32     UINT32_C(0xdeadbeef)
-# endif
-#endif
-
-/**
- * Hypervisor heap statistics record.
- * There is one global and one per allocation tag.
- */
-typedef struct MMHYPERSTAT
-{
-    /** Core avl node, key is the tag.
-     * @todo The type is wrong! Get your lazy a$$ over and create that offsetted uint32_t version we need here!  */
-    AVLOGCPHYSNODECORE      Core;
-    /** Aligning the 64-bit fields on a 64-bit line. */
-    uint32_t                u32Padding0;
-    /** Indicator for whether these statistics are registered with STAM or not. */
-    bool                    fRegistered;
-    /** Number of allocation. */
-    uint64_t                cAllocations;
-    /** Number of frees. */
-    uint64_t                cFrees;
-    /** Failures. */
-    uint64_t                cFailures;
-    /** Number of bytes allocated (sum). */
-    uint64_t                cbAllocated;
-    /** Number of bytes freed (sum). */
-    uint64_t                cbFreed;
-    /** Number of bytes currently allocated. */
-    uint32_t                cbCurAllocated;
-    /** Max number of bytes allocated. */
-    uint32_t                cbMaxAllocated;
-} MMHYPERSTAT;
-AssertCompileMemberAlignment(MMHYPERSTAT, cAllocations, 8);
-/** Pointer to hypervisor heap statistics record. */
-typedef MMHYPERSTAT *PMMHYPERSTAT;
-
-/**
- * Hypervisor heap chunk.
- */
-typedef struct MMHYPERCHUNK
-{
-    /** Previous block in the list of all blocks.
-     * This is relative to the start of the heap. */
-    uint32_t                offNext;
-    /** Offset to the previous block relative to this one. */
-    int32_t                 offPrev;
-    /** The statistics record this allocation belongs to (self relative). */
-    int32_t                 offStat;
-    /** Offset to the heap block (self relative). */
-    int32_t                 offHeap;
-} MMHYPERCHUNK;
-/** Pointer to a hypervisor heap chunk. */
-typedef MMHYPERCHUNK *PMMHYPERCHUNK;
-
-
-/**
- * Hypervisor heap chunk.
- */
-typedef struct MMHYPERCHUNKFREE
-{
-    /** Main list. */
-    MMHYPERCHUNK            core;
-    /** Offset of the next chunk in the list of free nodes. */
-    uint32_t                offNext;
-    /** Offset of the previous chunk in the list of free nodes. */
-    int32_t                 offPrev;
-    /** Size of the block. */
-    uint32_t                cb;
-} MMHYPERCHUNKFREE;
-/** Pointer to a free hypervisor heap chunk. */
-typedef MMHYPERCHUNKFREE *PMMHYPERCHUNKFREE;
-
-
-/**
- * The hypervisor heap.
- */
-typedef struct MMHYPERHEAP
-{
-    /** The typical magic (MMHYPERHEAP_MAGIC). */
-    uint32_t                u32Magic;
-    /** The heap size. (This structure is not included!) */
-    uint32_t                cbHeap;
-    /** Lock protecting the heap. */
-    PDMCRITSECT             Lock;
-    /** The HC ring-3 address of the heap. */
-    R3PTRTYPE(uint8_t *)    pbHeapR3;
-    /** The HC ring-3 address of the shared VM structure. */
-    PVMR3                   pVMR3;
-    /** The HC ring-0 address of the heap. */
-    R0PTRTYPE(uint8_t *)    pbHeapR0;
-    /** The HC ring-0 address of the shared VM structure. */
-    PVMR0                   pVMR0;
-    /** The RC address of the heap. */
-    RCPTRTYPE(uint8_t *)    pbHeapRC;
-    /** The RC address of the shared VM structure. */
-    PVMRC                   pVMRC;
-    /** The amount of free memory in the heap. */
-    uint32_t                cbFree;
-    /** Offset of the first free chunk in the heap.
-     * The offset is relative to the start of the heap. */
-    uint32_t                offFreeHead;
-    /** Offset of the last free chunk in the heap.
-     * The offset is relative to the start of the heap. */
-    uint32_t                offFreeTail;
-    /** Offset of the first page aligned block in the heap.
-     * The offset is equal to cbHeap initially. */
-    uint32_t                offPageAligned;
-    /** Tree of hypervisor heap statistics. */
-    AVLOGCPHYSTREE          HyperHeapStatTree;
-#ifdef MMHYPER_HEAP_FREE_DELAY
-    /** Where to insert the next free. */
-    uint32_t                iDelayedFree;
-    /** Array of delayed frees. Circular. Offsets relative to this structure. */
-    struct
-    {
-        /** The free caller address. */
-        RTUINTPTR           uCaller;
-        /** The offset of the freed chunk. */
-        uint32_t            offChunk;
-    } aDelayedFrees[MMHYPER_HEAP_FREE_DELAY];
-#else
-    /** Padding the structure to a 64-bit aligned size. */
-    uint32_t                u32Padding0;
-#endif
-    /** The heap physical pages. */
-    R3PTRTYPE(PSUPPAGE)     paPages;
-#if HC_ARCH_BITS == 32
-    /** Padding the structure to a 64-bit aligned size. */
-    uint32_t                u32Padding1;
-#endif
-} MMHYPERHEAP;
-/** Pointer to the hypervisor heap. */
-typedef MMHYPERHEAP *PMMHYPERHEAP;
-
-/** Magic value for MMHYPERHEAP. (C. S. Lewis) */
-#define MMHYPERHEAP_MAGIC               UINT32_C(0x18981129)
-
-
-/**
- * Hypervisor heap minimum alignment (16 bytes).
- */
-#define MMHYPER_HEAP_ALIGN_MIN          16
-
-/**
- * The aligned size of the the MMHYPERHEAP structure.
- */
-#define MMYPERHEAP_HDR_SIZE             RT_ALIGN_Z(sizeof(MMHYPERHEAP), MMHYPER_HEAP_ALIGN_MIN * 4)
-
-/** @name Hypervisor heap chunk flags.
- * The flags are put in the first bits of the MMHYPERCHUNK::offPrev member.
- * These bits aren't used anyway because of the chunk minimal alignment (16 bytes).
- * @{ */
-/** The chunk is free. (The code ASSUMES this is 0!) */
-#define MMHYPERCHUNK_FLAGS_FREE         0x0
-/** The chunk is in use. */
-#define MMHYPERCHUNK_FLAGS_USED         0x1
-/** The type mask. */
-#define MMHYPERCHUNK_FLAGS_TYPE_MASK    0x1
-/** The flag mask */
-#define MMHYPERCHUNK_FLAGS_MASK         0x1
-
-/** Checks if the chunk is free. */
-#define MMHYPERCHUNK_ISFREE(pChunk)                 ( (((pChunk)->offPrev) & MMHYPERCHUNK_FLAGS_TYPE_MASK) == MMHYPERCHUNK_FLAGS_FREE )
-/** Checks if the chunk is used. */
-#define MMHYPERCHUNK_ISUSED(pChunk)                 ( (((pChunk)->offPrev) & MMHYPERCHUNK_FLAGS_TYPE_MASK) == MMHYPERCHUNK_FLAGS_USED )
-/** Toggles FREE/USED flag of a chunk. */
-#define MMHYPERCHUNK_SET_TYPE(pChunk, type)         do { (pChunk)->offPrev = ((pChunk)->offPrev & ~MMHYPERCHUNK_FLAGS_TYPE_MASK) | ((type) & MMHYPERCHUNK_FLAGS_TYPE_MASK); } while (0)
-
-/** Gets the prev offset without the flags. */
-#define MMHYPERCHUNK_GET_OFFPREV(pChunk)            ((int32_t)((pChunk)->offPrev & ~MMHYPERCHUNK_FLAGS_MASK))
-/** Sets the prev offset without changing the flags. */
-#define MMHYPERCHUNK_SET_OFFPREV(pChunk, off)       do { (pChunk)->offPrev = (off) | ((pChunk)->offPrev & MMHYPERCHUNK_FLAGS_MASK); } while (0)
-#if 0
-/** Clears one or more flags. */
-#define MMHYPERCHUNK_FLAGS_OP_CLEAR(pChunk, fFlags) do { ((pChunk)->offPrev) &= ~((fFlags) & MMHYPERCHUNK_FLAGS_MASK); } while (0)
-/** Sets one or more flags. */
-#define MMHYPERCHUNK_FLAGS_OP_SET(pChunk, fFlags)   do { ((pChunk)->offPrev) |=  ((fFlags) & MMHYPERCHUNK_FLAGS_MASK); } while (0)
-/** Checks if one is set. */
-#define MMHYPERCHUNK_FLAGS_OP_ISSET(pChunk, fFlag)  (!!(((pChunk)->offPrev) & ((fFlag) & MMHYPERCHUNK_FLAGS_MASK)))
-#endif
-/** @} */
-
-/** @} */
-
-
-/** @name Page Pool Internals
- * @{
- */
-
-/**
- * Page sub pool
- *
- * About the allocation of this structure. To keep the number of heap blocks,
- * the number of heap calls, and fragmentation low we allocate all the data
- * related to a MMPAGESUBPOOL node in one chunk. That means that after the
- * bitmap (which is of variable size) comes the SUPPAGE records and then
- * follows the lookup tree nodes. (The heap in question is the hyper heap.)
- */
-typedef struct MMPAGESUBPOOL
-{
-    /** Pointer to next sub pool. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(struct MMPAGESUBPOOL *)   pNext;
-#else
-    R3R0PTRTYPE(struct MMPAGESUBPOOL *) pNext;
-#endif
-    /** Pointer to next sub pool in the free chain.
-     * This is NULL if we're not in the free chain or at the end of it. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(struct MMPAGESUBPOOL *)   pNextFree;
-#else
-    R3R0PTRTYPE(struct MMPAGESUBPOOL *) pNextFree;
-#endif
-    /** Pointer to array of lock ranges.
-     * This is allocated together with the MMPAGESUBPOOL and thus needs no freeing.
-     * It follows immediately after the bitmap.
-     * The reserved field is a pointer to this structure.
-     */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(PSUPPAGE)                 paPhysPages;
-#else
-    R3R0PTRTYPE(PSUPPAGE)               paPhysPages;
-#endif
-    /** Pointer to the first page. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(void *)                   pvPages;
-#else
-    R3R0PTRTYPE(void *)                 pvPages;
-#endif
-    /** Size of the subpool. */
-    uint32_t                            cPages;
-    /** Number of free pages. */
-    uint32_t                            cPagesFree;
-    /** The allocation bitmap.
-     * This may extend beyond the end of the defined array size.
-     */
-    uint32_t                            auBitmap[1];
-    /* ... SUPPAGE                      aRanges[1]; */
-} MMPAGESUBPOOL;
-/** Pointer to page sub pool. */
-typedef MMPAGESUBPOOL *PMMPAGESUBPOOL;
-
-/**
- * Page pool.
- */
-typedef struct MMPAGEPOOL
-{
-    /** List of subpools. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(PMMPAGESUBPOOL)           pHead;
-#else
-    R3R0PTRTYPE(PMMPAGESUBPOOL)         pHead;
-#endif
-    /** Head of subpools with free pages. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(PMMPAGESUBPOOL)           pHeadFree;
-#else
-    R3R0PTRTYPE(PMMPAGESUBPOOL)         pHeadFree;
-#endif
-    /** AVLPV tree for looking up HC virtual addresses.
-     * The tree contains MMLOOKUPVIRTPP records.
-     */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(PAVLPVNODECORE)           pLookupVirt;
-#else
-    R3R0PTRTYPE(PAVLPVNODECORE)         pLookupVirt;
-#endif
-    /** Tree for looking up HC physical addresses.
-     * The tree contains MMLOOKUPPHYSHC records.
-     */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(AVLHCPHYSTREE)            pLookupPhys;
-#else
-    R3R0PTRTYPE(AVLHCPHYSTREE)          pLookupPhys;
-#endif
-    /** Pointer to the VM this pool belongs. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    PVMR3                               pVM;
-#else
-    R3R0PTRTYPE(PVM)                    pVM;
-#endif
-    /** Flag indicating the allocation method.
-     * Set: SUPR3LowAlloc().
-     * Clear: SUPR3PageAllocEx(). */
-    bool                                fLow;
-    /** Number of subpools. */
-    uint32_t                            cSubPools;
-    /** Number of pages in pool. */
-    uint32_t                            cPages;
-#ifdef VBOX_WITH_STATISTICS
-    /** Number of free pages in pool. */
-    uint32_t                            cFreePages;
-# if HC_ARCH_BITS == 32
-    /** Aligning the statistics on an 8 byte boundary. */
-    uint32_t                            u32Alignment;
-# endif
-    /** Number of alloc calls. */
-    STAMCOUNTER                         cAllocCalls;
-    /** Number of free calls. */
-    STAMCOUNTER                         cFreeCalls;
-    /** Number of to phys conversions. */
-    STAMCOUNTER                         cToPhysCalls;
-    /** Number of to virtual conversions. */
-    STAMCOUNTER                         cToVirtCalls;
-    /** Number of real errors. */
-    STAMCOUNTER                         cErrors;
-#endif
-} MMPAGEPOOL;
-#ifndef IN_RC
-AssertCompileMemberAlignment(MMPAGEPOOL, cSubPools, 4);
-# ifdef VBOX_WITH_STATISTICS
-AssertCompileMemberAlignment(MMPAGEPOOL, cAllocCalls, 8);
-# endif
-#endif
-/** Pointer to page pool. */
-typedef MMPAGEPOOL *PMMPAGEPOOL;
-
-/**
- * Lookup record for HC virtual memory in the page pool.
- */
-typedef struct MMPPLOOKUPHCPTR
-{
-    /** The key is virtual address. */
-    AVLPVNODECORE           Core;
-    /** Pointer to subpool if lookup record for a pool. */
-    struct MMPAGESUBPOOL   *pSubPool;
-} MMPPLOOKUPHCPTR;
-/** Pointer to virtual memory lookup record. */
-typedef MMPPLOOKUPHCPTR *PMMPPLOOKUPHCPTR;
-
-/**
- * Lookup record for HC physical memory.
- */
-typedef struct MMPPLOOKUPHCPHYS
-{
-    /** The key is physical address. */
-    AVLHCPHYSNODECORE       Core;
-    /** Pointer to SUPPAGE record for this physical address. */
-    PSUPPAGE                pPhysPage;
-} MMPPLOOKUPHCPHYS;
-/** Pointer to physical memory lookup record. */
-typedef MMPPLOOKUPHCPHYS *PMMPPLOOKUPHCPHYS;
-
-/** @} */
-
-
-/**
- * Hypervisor memory mapping type.
- */
-typedef enum MMLOOKUPHYPERTYPE
-{
-    /** Invalid record. This is used for record which are incomplete. */
-    MMLOOKUPHYPERTYPE_INVALID = 0,
-    /** Mapping of locked memory. */
-    MMLOOKUPHYPERTYPE_LOCKED,
-    /** Mapping of contiguous HC physical memory. */
-    MMLOOKUPHYPERTYPE_HCPHYS,
-    /** Mapping of contiguous GC physical memory. */
-    MMLOOKUPHYPERTYPE_GCPHYS,
-    /** Mapping of MMIO2 memory. */
-    MMLOOKUPHYPERTYPE_MMIO2,
-    /** Dynamic mapping area (MMR3HyperReserve).
-     * A conversion will require to check what's in the page table for the pages. */
-    MMLOOKUPHYPERTYPE_DYNAMIC
-} MMLOOKUPHYPERTYPE;
-
-/**
- * Lookup record for the hypervisor memory area.
- */
-typedef struct MMLOOKUPHYPER
-{
-    /** Byte offset from the start of this record to the next.
-     * If the value is NIL_OFFSET the chain is terminated. */
-    int32_t                 offNext;
-    /** Offset into the hypervisor memory area. */
-    uint32_t                off;
-    /** Size of this part. */
-    uint32_t                cb;
-    /** Locking type. */
-    MMLOOKUPHYPERTYPE       enmType;
-    /** Type specific data */
-    union
-    {
-        /** Locked memory. */
-        struct
-        {
-            /** Host context ring-3 pointer. */
-            R3PTRTYPE(void *)       pvR3;
-            /** Host context ring-0 pointer. Optional. */
-            RTR0PTR                 pvR0;
-            /** Pointer to an array containing the physical address of each page. */
-            R3PTRTYPE(PRTHCPHYS)    paHCPhysPages;
-        } Locked;
-
-        /** Contiguous physical memory. */
-        struct
-        {
-            /** Host context ring-3 pointer. */
-            R3PTRTYPE(void *)       pvR3;
-            /** Host context ring-0 pointer. Optional. */
-            RTR0PTR                 pvR0;
-            /** HC physical address corresponding to pvR3/pvR0. */
-            RTHCPHYS                HCPhys;
-        } HCPhys;
-
-        /** Contiguous guest physical memory. */
-        struct
-        {
-            /** The memory address (Guest Context). */
-            RTGCPHYS                GCPhys;
-        } GCPhys;
-
-        /** MMIO2 memory. */
-        struct
-        {
-            /** The device instance owning the MMIO2 region. */
-            PPDMDEVINSR3            pDevIns;
-            /** The region number. */
-            uint32_t                iRegion;
-            /** The offset into the MMIO2 region. */
-            RTGCPHYS                off;
-        } MMIO2;
-    } u;
-    /** Description. */
-    R3PTRTYPE(const char *) pszDesc;
-} MMLOOKUPHYPER;
-/** Pointer to a hypervisor memory lookup record. */
-typedef MMLOOKUPHYPER *PMMLOOKUPHYPER;
-
-
-/**
- * Converts a MM pointer into a VM pointer.
- * @returns Pointer to the VM structure the MM is part of.
- * @param   pMM   Pointer to MM instance data.
- */
-#define MM2VM(pMM)  ( (PVM)((uint8_t *)pMM - pMM->offVM) )
-
-
-/**
- * MM Data (part of VM)
- */
-typedef struct MM
-{
-    /** Offset to the VM structure.
-     * See MM2VM(). */
-    RTINT                       offVM;
-
-    /** Set if MMR3InitPaging has been called. */
-    bool                        fDoneMMR3InitPaging;
-    /** Set if PGM has been initialized and we can safely call PGMR3Map(). */
-    bool                        fPGMInitialized;
-#if GC_ARCH_BITS == 64 || HC_ARCH_BITS == 64
-    uint32_t                    u32Padding1; /**< alignment padding. */
-#endif
-
-    /** Lookup list for the Hypervisor Memory Area.
-     * The offset is relative to the start of the heap.
-     * Use pHyperHeapR3, pHyperHeapR0 or pHypeRHeapRC to calculate the address.
-     */
-    RTUINT                      offLookupHyper;
-
-    /** The offset of the next static mapping in the Hypervisor Memory Area. */
-    RTUINT                      offHyperNextStatic;
-    /** The size of the HMA.
-     * Starts at 12MB and will be fixed late in the init process. */
-    RTUINT                      cbHyperArea;
-
-    /** Guest address of the Hypervisor Memory Area.
-     * @remarks It's still a bit open whether this should be change to RTRCPTR or
-     *          remain a RTGCPTR. */
-    RTGCPTR                     pvHyperAreaGC;
-
-    /** The hypervisor heap (GC Ptr). */
-    RCPTRTYPE(PMMHYPERHEAP)     pHyperHeapRC;
-#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 64
-    uint32_t                    u32Padding2;
-#endif
-
-    /** The hypervisor heap (R0 Ptr). */
-    R0PTRTYPE(PMMHYPERHEAP)     pHyperHeapR0;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-    /** Page pool - R0 Ptr. */
-    R0PTRTYPE(PMMPAGEPOOL)      pPagePoolR0;
-    /** Page pool pages in low memory R0 Ptr. */
-    R0PTRTYPE(PMMPAGEPOOL)      pPagePoolLowR0;
-#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE */
-
-    /** The hypervisor heap (R3 Ptr). */
-    R3PTRTYPE(PMMHYPERHEAP)     pHyperHeapR3;
-    /** Page pool - R3 Ptr. */
-    R3PTRTYPE(PMMPAGEPOOL)      pPagePoolR3;
-    /** Page pool pages in low memory R3 Ptr. */
-    R3PTRTYPE(PMMPAGEPOOL)      pPagePoolLowR3;
-
-    /** Pointer to the dummy page.
-     * The dummy page is a paranoia thingy used for instance for pure MMIO RAM ranges
-     * to make sure any bugs will not harm whatever the system stores in the first
-     * physical page. */
-    R3PTRTYPE(void *)           pvDummyPage;
-    /** Physical address of the dummy page. */
-    RTHCPHYS                    HCPhysDummyPage;
-
-    /** Size of the base RAM in bytes. (The CFGM RamSize value.) */
-    uint64_t                    cbRamBase;
-    /** The number of base RAM pages that PGM has reserved (GMM).
-     * @remarks Shadow ROMs will be counted twice (RAM+ROM), so it won't be 1:1 with
-     *          what the guest sees. */
-    uint64_t                    cBasePages;
-    /** The number of handy pages that PGM has reserved (GMM).
-     * These are kept out of cBasePages and thus out of the saved state. */
-    uint32_t                    cHandyPages;
-    /** The number of shadow pages PGM has reserved (GMM). */
-    uint32_t                    cShadowPages;
-    /** The number of fixed pages we've reserved (GMM). */
-    uint32_t                    cFixedPages;
-    /** Padding. */
-    uint32_t                    u32Padding0;
-} MM;
-/** Pointer to MM Data (part of VM). */
-typedef MM *PMM;
-
-
-/**
- * MM data kept in the UVM.
- */
-typedef struct MMUSERPERVM
-{
-    /** Pointer to the MM R3 Heap. */
-    R3PTRTYPE(PMMHEAP)          pHeap;
-    /** Pointer to the MM Uk Heap. */
-    R3PTRTYPE(PMMUKHEAP)        pUkHeap;
-} MMUSERPERVM;
-/** Pointer to the MM data kept in the UVM. */
-typedef MMUSERPERVM *PMMUSERPERVM;
-
-
-RT_C_DECLS_BEGIN
-
-
-int  mmR3UpdateReservation(PVM pVM);
-
-int  mmR3PagePoolInit(PVM pVM);
-void mmR3PagePoolTerm(PVM pVM);
-
-int  mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap);
-void mmR3HeapDestroy(PMMHEAP pHeap);
-
-void mmR3UkHeapDestroy(PMMUKHEAP pHeap);
-int  mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap);
-
-
-int  mmR3HyperInit(PVM pVM);
-int  mmR3HyperTerm(PVM pVM);
-int  mmR3HyperInitPaging(PVM pVM);
-
-const char *mmGetTagName(MMTAG enmTag);
-
-/**
- * Converts a pool address to a physical address.
- * The specified allocation type must match with the address.
- *
- * @returns Physical address.
- * @returns NIL_RTHCPHYS if not found or eType is not matching.
- * @param   pPool   Pointer to the page pool.
- * @param   pv      The address to convert.
- * @thread  The Emulation Thread.
- */
-RTHCPHYS mmPagePoolPtr2Phys(PMMPAGEPOOL pPool, void *pv);
-
-/**
- * Converts a pool physical address to a linear address.
- * The specified allocation type must match with the address.
- *
- * @returns Physical address.
- * @returns NULL if not found or eType is not matching.
- * @param   pPool       Pointer to the page pool.
- * @param   HCPhys      The address to convert.
- * @thread  The Emulation Thread.
- */
-void *mmPagePoolPhys2Ptr(PMMPAGEPOOL pPool, RTHCPHYS HCPhys);
-
-RT_C_DECLS_END
-
-/** @} */
-
-#endif
-
Index: unk/src/VBox/VMM/MMPagePool.cpp
===================================================================
--- /trunk/src/VBox/VMM/MMPagePool.cpp	(revision 35332)
+++ 	(revision )
@@ -1,527 +1,0 @@
-/* $Id$ */
-/** @file
- * MM - Memory Manager - Page Pool.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_MM_POOL
-#include <VBox/mm.h>
-#include <VBox/pgm.h>
-#include <VBox/stam.h>
-#include "MMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <include/internal/pgm.h>
-#include <iprt/alloc.h>
-#include <iprt/assert.h>
-#define USE_INLINE_ASM_BIT_OPS
-#ifdef USE_INLINE_ASM_BIT_OPS
-# include <iprt/asm.h>
-#endif
-#include <iprt/string.h>
-
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-#ifdef IN_RING3
-static void *   mmR3PagePoolAlloc(PMMPAGEPOOL pPool);
-static void     mmR3PagePoolFree(PMMPAGEPOOL pPool, void *pv);
-#endif
-
-
-/**
- * Initializes the page pool
- *
- * @return  VBox status.
- * @param   pVM     VM handle.
- * @thread  The Emulation Thread.
- */
-int mmR3PagePoolInit(PVM pVM)
-{
-    AssertMsg(!pVM->mm.s.pPagePoolR3, ("Already initialized!\n"));
-
-    /*
-     * Allocate the pool structures.
-     */
-    /** @todo @bufref{1865},@bufref{3202}: mapping the page pool page into
-     *        ring-0. Need to change the wasy we allocate it... */
-    AssertReleaseReturn(sizeof(*pVM->mm.s.pPagePoolR3) + sizeof(*pVM->mm.s.pPagePoolLowR3) < PAGE_SIZE, VERR_INTERNAL_ERROR);
-    int rc = SUPR3PageAllocEx(1, 0 /*fFlags*/, (void **)&pVM->mm.s.pPagePoolR3, NULL /*pR0Ptr*/, NULL /*paPages*/);
-    if (RT_FAILURE(rc))
-        return rc;
-    memset(pVM->mm.s.pPagePoolR3, 0, PAGE_SIZE);
-    pVM->mm.s.pPagePoolR3->pVM = pVM;
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cPages,         STAMTYPE_U32,     "/MM/Page/Def/cPages",        STAMUNIT_PAGES, "Number of pages in the default pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cFreePages,     STAMTYPE_U32,     "/MM/Page/Def/cFreePages",    STAMUNIT_PAGES, "Number of free pages in the default pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cSubPools,      STAMTYPE_U32,     "/MM/Page/Def/cSubPools",     STAMUNIT_COUNT, "Number of sub pools in the default pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cAllocCalls,    STAMTYPE_COUNTER, "/MM/Page/Def/cAllocCalls",   STAMUNIT_CALLS, "Number of MMR3PageAlloc() calls for the default pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cFreeCalls,     STAMTYPE_COUNTER, "/MM/Page/Def/cFreeCalls",    STAMUNIT_CALLS, "Number of MMR3PageFree()+MMR3PageFreeByPhys() calls for the default pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cToPhysCalls,   STAMTYPE_COUNTER, "/MM/Page/Def/cToPhysCalls",  STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for this pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cToVirtCalls,   STAMTYPE_COUNTER, "/MM/Page/Def/cToVirtCalls",  STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the default pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cErrors,        STAMTYPE_COUNTER, "/MM/Page/Def/cErrors",       STAMUNIT_ERRORS,"Number of errors for the default pool.");
-
-    pVM->mm.s.pPagePoolLowR3 = pVM->mm.s.pPagePoolR3 + 1;
-    pVM->mm.s.pPagePoolLowR3->pVM = pVM;
-    pVM->mm.s.pPagePoolLowR3->fLow = true;
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cPages,      STAMTYPE_U32,     "/MM/Page/Low/cPages",        STAMUNIT_PAGES, "Number of pages in the <4GB pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cFreePages,  STAMTYPE_U32,     "/MM/Page/Low/cFreePages",    STAMUNIT_PAGES, "Number of free pages in the <4GB pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cSubPools,   STAMTYPE_U32,     "/MM/Page/Low/cSubPools",     STAMUNIT_COUNT, "Number of sub pools in the <4GB pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cAllocCalls, STAMTYPE_COUNTER, "/MM/Page/Low/cAllocCalls",   STAMUNIT_CALLS, "Number of MMR3PageAllocLow() calls for the <4GB pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cFreeCalls,  STAMTYPE_COUNTER, "/MM/Page/Low/cFreeCalls",    STAMUNIT_CALLS, "Number of MMR3PageFreeLow()+MMR3PageFreeByPhys() calls for the <4GB pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cToPhysCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToPhysCalls",  STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for the <4GB pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cToVirtCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToVirtCalls",  STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the <4GB pool.");
-    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cErrors,     STAMTYPE_COUNTER, "/MM/Page/Low/cErrors",       STAMUNIT_ERRORS,"Number of errors for the <4GB pool.");
-
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-    pVM->mm.s.pPagePoolR0 = (uintptr_t)pVM->mm.s.pPagePoolR3;
-    pVM->mm.s.pPagePoolLowR0 = (uintptr_t)pVM->mm.s.pPagePoolLowR3;
-#endif
-
-    /** @todo init a mutex? */
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Release all locks and free the allocated memory.
- *
- * @param   pVM     VM handle.
- * @thread  The Emulation Thread.
- */
-void mmR3PagePoolTerm(PVM pVM)
-{
-    if (pVM->mm.s.pPagePoolR3)
-    {
-        /*
-         * Unlock all memory held by subpools and free the memory.
-         * (The MM Heap will free the memory used for internal stuff.)
-         */
-        Assert(!pVM->mm.s.pPagePoolR3->fLow);
-        PMMPAGESUBPOOL  pSubPool = pVM->mm.s.pPagePoolR3->pHead;
-        while (pSubPool)
-        {
-            int rc = SUPR3PageFreeEx(pSubPool->pvPages, pSubPool->cPages);
-            AssertMsgRC(rc, ("SUPR3PageFreeEx(%p) failed with rc=%Rrc\n", pSubPool->pvPages, rc));
-            pSubPool->pvPages = NULL;
-
-            /* next */
-            pSubPool = pSubPool->pNext;
-        }
-        pVM->mm.s.pPagePoolR3 = NULL;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-        pVM->mm.s.pPagePoolR0 = NIL_RTR0PTR;
-#endif
-    }
-
-    if (pVM->mm.s.pPagePoolLowR3)
-    {
-        /*
-         * Free the memory.
-         */
-        Assert(pVM->mm.s.pPagePoolLowR3->fLow);
-        PMMPAGESUBPOOL  pSubPool = pVM->mm.s.pPagePoolLowR3->pHead;
-        while (pSubPool)
-        {
-            int rc = SUPR3LowFree(pSubPool->pvPages, pSubPool->cPages);
-            AssertMsgRC(rc, ("SUPR3LowFree(%p) failed with rc=%d\n", pSubPool->pvPages, rc));
-            pSubPool->pvPages = NULL;
-
-            /* next */
-            pSubPool = pSubPool->pNext;
-        }
-        pVM->mm.s.pPagePoolLowR3 = NULL;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-        pVM->mm.s.pPagePoolLowR0 = NIL_RTR0PTR;
-#endif
-    }
-}
-
-
-/**
- * Allocates a page from the page pool.
- *
- * @returns Pointer to allocated page(s).
- * @returns NULL on failure.
- * @param   pPool   Pointer to the page pool.
- * @thread  The Emulation Thread.
- */
-DECLINLINE(void *) mmR3PagePoolAlloc(PMMPAGEPOOL pPool)
-{
-    VM_ASSERT_EMT(pPool->pVM);
-    STAM_COUNTER_INC(&pPool->cAllocCalls);
-
-    /*
-     * Walk free list.
-     */
-    if (pPool->pHeadFree)
-    {
-        PMMPAGESUBPOOL  pSub = pPool->pHeadFree;
-        /* decrement free count and unlink if no more free entries. */
-        if (!--pSub->cPagesFree)
-            pPool->pHeadFree = pSub->pNextFree;
-#ifdef VBOX_WITH_STATISTICS
-        pPool->cFreePages--;
-#endif
-
-        /* find free spot in bitmap. */
-#ifdef USE_INLINE_ASM_BIT_OPS
-        const int iPage = ASMBitFirstClear(pSub->auBitmap, pSub->cPages);
-        if (iPage >= 0)
-        {
-            Assert(!ASMBitTest(pSub->auBitmap, iPage));
-            ASMBitSet(pSub->auBitmap, iPage);
-            return (uint8_t *)pSub->pvPages + PAGE_SIZE * iPage;
-        }
-#else
-        unsigned   *pu = &pSub->auBitmap[0];
-        unsigned   *puEnd = &pSub->auBitmap[pSub->cPages / (sizeof(pSub->auBitmap) * 8)];
-        while (pu < puEnd)
-        {
-            unsigned u;
-            if ((u = *pu) != ~0U)
-            {
-                unsigned iBit = 0;
-                unsigned uMask = 1;
-                while (iBit < sizeof(pSub->auBitmap[0]) * 8)
-                {
-                    if (!(u & uMask))
-                    {
-                        *pu |= uMask;
-                        return (uint8_t *)pSub->pvPages
-                            + PAGE_SIZE * (iBit + ((uint8_t *)pu - (uint8_t *)&pSub->auBitmap[0]) * 8);
-                    }
-                    iBit++;
-                    uMask <<= 1;
-                }
-                STAM_COUNTER_INC(&pPool->cErrors);
-                AssertMsgFailed(("how odd, expected to find a free bit in %#x, but didn't\n", u));
-            }
-            /* next */
-            pu++;
-        }
-#endif
-        STAM_COUNTER_INC(&pPool->cErrors);
-#ifdef VBOX_WITH_STATISTICS
-        pPool->cFreePages++;
-#endif
-        AssertMsgFailed(("how strange, expected to find a free bit in %p, but didn't (%d pages supposed to be free!)\n", pSub, pSub->cPagesFree + 1));
-    }
-
-    /*
-     * Allocate new subpool.
-     */
-    unsigned        cPages = !pPool->fLow ? 128 : 32;
-    PMMPAGESUBPOOL  pSub;
-    int rc = MMHyperAlloc(pPool->pVM,
-                          RT_OFFSETOF(MMPAGESUBPOOL, auBitmap[cPages / (sizeof(pSub->auBitmap[0] * 8))])
-                          + (sizeof(SUPPAGE) + sizeof(MMPPLOOKUPHCPHYS)) * cPages
-                          + sizeof(MMPPLOOKUPHCPTR),
-                          0,
-                          MM_TAG_MM_PAGE,
-                          (void **)&pSub);
-    if (RT_FAILURE(rc))
-        return NULL;
-
-    PSUPPAGE paPhysPages = (PSUPPAGE)&pSub->auBitmap[cPages / (sizeof(pSub->auBitmap[0]) * 8)];
-    Assert((uintptr_t)paPhysPages >= (uintptr_t)&pSub->auBitmap[1]);
-    if (!pPool->fLow)
-    {
-        rc = SUPR3PageAllocEx(cPages,
-                              0 /* fFlags */,
-                              &pSub->pvPages,
-                              NULL,
-                              paPhysPages);
-        if (RT_FAILURE(rc))
-            rc = VMSetError(pPool->pVM, rc, RT_SRC_POS,
-                            N_("Failed to lock host %zd bytes of memory (out of memory)"), (size_t)cPages << PAGE_SHIFT);
-    }
-    else
-        rc = SUPR3LowAlloc(cPages, &pSub->pvPages, NULL, paPhysPages);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Setup the sub structure and allocate the requested page.
-         */
-        pSub->cPages    = cPages;
-        pSub->cPagesFree= cPages - 1;
-        pSub->paPhysPages = paPhysPages;
-        memset(pSub->auBitmap, 0, cPages / 8);
-        /* allocate first page. */
-        pSub->auBitmap[0] |= 1;
-        /* link into free chain. */
-        pSub->pNextFree = pPool->pHeadFree;
-        pPool->pHeadFree= pSub;
-        /* link into main chain. */
-        pSub->pNext     = pPool->pHead;
-        pPool->pHead    = pSub;
-        /* update pool statistics. */
-        pPool->cSubPools++;
-        pPool->cPages  += cPages;
-#ifdef VBOX_WITH_STATISTICS
-        pPool->cFreePages += cPages - 1;
-#endif
-
-        /*
-         * Initialize the physical pages with backpointer to subpool.
-         */
-        unsigned i = cPages;
-        while (i-- > 0)
-        {
-            AssertMsg(paPhysPages[i].Phys && !(paPhysPages[i].Phys & PAGE_OFFSET_MASK),
-                      ("i=%d Phys=%d\n", i, paPhysPages[i].Phys));
-            paPhysPages[i].uReserved = (RTHCUINTPTR)pSub;
-        }
-
-        /*
-         * Initialize the physical lookup record with backpointers to the physical pages.
-         */
-        PMMPPLOOKUPHCPHYS paLookupPhys = (PMMPPLOOKUPHCPHYS)&paPhysPages[cPages];
-        i = cPages;
-        while (i-- > 0)
-        {
-            paLookupPhys[i].pPhysPage = &paPhysPages[i];
-            paLookupPhys[i].Core.Key = paPhysPages[i].Phys;
-            RTAvlHCPhysInsert(&pPool->pLookupPhys, &paLookupPhys[i].Core);
-        }
-
-        /*
-         * And the one record for virtual memory lookup.
-         */
-        PMMPPLOOKUPHCPTR   pLookupVirt = (PMMPPLOOKUPHCPTR)&paLookupPhys[cPages];
-        pLookupVirt->pSubPool = pSub;
-        pLookupVirt->Core.Key = pSub->pvPages;
-        RTAvlPVInsert(&pPool->pLookupVirt, &pLookupVirt->Core);
-
-        /* return allocated page (first). */
-        return pSub->pvPages;
-    }
-
-    MMHyperFree(pPool->pVM, pSub);
-    STAM_COUNTER_INC(&pPool->cErrors);
-    if (pPool->fLow)
-        VMSetError(pPool->pVM, rc, RT_SRC_POS,
-                   N_("Failed to expand page pool for memory below 4GB. Current size: %d pages"),
-                   pPool->cPages);
-    AssertMsgFailed(("Failed to expand pool%s. rc=%Rrc poolsize=%d\n",
-                     pPool->fLow ? " (<4GB)" : "", rc, pPool->cPages));
-    return NULL;
-}
-
-
-/**
- * Frees a page from the page pool.
- *
- * @param   pPool   Pointer to the page pool.
- * @param   pv      Pointer to the page to free.
- *                  I.e. pointer returned by mmR3PagePoolAlloc().
- * @thread  The Emulation Thread.
- */
-DECLINLINE(void) mmR3PagePoolFree(PMMPAGEPOOL pPool, void *pv)
-{
-    VM_ASSERT_EMT(pPool->pVM);
-    STAM_COUNTER_INC(&pPool->cFreeCalls);
-
-    /*
-     * Lookup the virtual address.
-     */
-    PMMPPLOOKUPHCPTR pLookup = (PMMPPLOOKUPHCPTR)RTAvlPVGetBestFit(&pPool->pLookupVirt, pv, false);
-    if (    !pLookup
-        ||  (uint8_t *)pv >= (uint8_t *)pLookup->pSubPool->pvPages + (pLookup->pSubPool->cPages << PAGE_SHIFT)
-        )
-    {
-        STAM_COUNTER_INC(&pPool->cErrors);
-        AssertMsgFailed(("invalid pointer %p\n", pv));
-        return;
-    }
-
-    /*
-     * Free the page.
-     */
-    PMMPAGESUBPOOL  pSubPool = pLookup->pSubPool;
-    /* clear bitmap bit */
-    const unsigned  iPage = ((uint8_t *)pv - (uint8_t *)pSubPool->pvPages) >> PAGE_SHIFT;
-#ifdef USE_INLINE_ASM_BIT_OPS
-    Assert(ASMBitTest(pSubPool->auBitmap, iPage));
-    ASMBitClear(pSubPool->auBitmap, iPage);
-#else
-    unsigned    iBit   = iPage % (sizeof(pSubPool->auBitmap[0]) * 8);
-    unsigned    iIndex = iPage / (sizeof(pSubPool->auBitmap[0]) * 8);
-    pSubPool->auBitmap[iIndex] &= ~(1 << iBit);
-#endif
-    /* update stats. */
-    pSubPool->cPagesFree++;
-#ifdef VBOX_WITH_STATISTICS
-    pPool->cFreePages++;
-#endif
-    if (pSubPool->cPagesFree == 1)
-    {
-        pSubPool->pNextFree = pPool->pHeadFree;
-        pPool->pHeadFree = pSubPool;
-    }
-}
-
-
-/**
- * Allocates a page from the page pool.
- *
- * This function may returns pages which has physical addresses any
- * where. If you require a page to be within the first 4GB of physical
- * memory, use MMR3PageAllocLow().
- *
- * @returns Pointer to the allocated page page.
- * @returns NULL on failure.
- * @param   pVM         VM handle.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(void *) MMR3PageAlloc(PVM pVM)
-{
-    /* Note: unprotected by locks; currently fine as it's used during init or under the PGM lock */
-    return mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3);
-}
-
-
-/**
- * Allocates a page from the page pool and return its physical address.
- *
- * This function may returns pages which has physical addresses any
- * where. If you require a page to be within the first 4GB of physical
- * memory, use MMR3PageAllocLow().
- *
- * @returns Pointer to the allocated page page.
- * @returns NIL_RTHCPHYS on failure.
- * @param   pVM         VM handle.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(RTHCPHYS) MMR3PageAllocPhys(PVM pVM)
-{
-    /* Note: unprotected by locks; currently fine as it's used during init or under the PGM lock */
-    /** @todo optimize this, it's the most common case now. */
-    void *pv = mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3);
-    if (pv)
-        return mmPagePoolPtr2Phys(pVM->mm.s.pPagePoolR3, pv);
-    return NIL_RTHCPHYS;
-}
-
-
-/**
- * Frees a page allocated from the page pool by MMR3PageAlloc() or
- * MMR3PageAllocPhys().
- *
- * @param   pVM         VM handle.
- * @param   pvPage      Pointer to the page.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(void) MMR3PageFree(PVM pVM, void *pvPage)
-{
-    mmR3PagePoolFree(pVM->mm.s.pPagePoolR3, pvPage);
-}
-
-
-/**
- * Allocates a page from the low page pool.
- *
- * @returns Pointer to the allocated page.
- * @returns NULL on failure.
- * @param   pVM         VM handle.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(void *) MMR3PageAllocLow(PVM pVM)
-{
-    return mmR3PagePoolAlloc(pVM->mm.s.pPagePoolLowR3);
-}
-
-
-/**
- * Frees a page allocated from the page pool by MMR3PageAllocLow().
- *
- * @param   pVM         VM handle.
- * @param   pvPage      Pointer to the page.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(void) MMR3PageFreeLow(PVM pVM, void *pvPage)
-{
-    mmR3PagePoolFree(pVM->mm.s.pPagePoolLowR3, pvPage);
-}
-
-
-/**
- * Free a page allocated from the page pool by physical address.
- * This works for pages allocated by MMR3PageAlloc(), MMR3PageAllocPhys()
- * and MMR3PageAllocLow().
- *
- * @param   pVM         VM handle.
- * @param   HCPhysPage  The physical address of the page to be freed.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(void) MMR3PageFreeByPhys(PVM pVM, RTHCPHYS HCPhysPage)
-{
-    void *pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePoolR3, HCPhysPage);
-    if (!pvPage)
-        pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePoolLowR3, HCPhysPage);
-    if (pvPage)
-        mmR3PagePoolFree(pVM->mm.s.pPagePoolR3, pvPage);
-    else
-        AssertMsgFailed(("Invalid address HCPhysPT=%#x\n", HCPhysPage));
-}
-
-
-/**
- * Gets the HC pointer to the dummy page.
- *
- * The dummy page is used as a place holder to prevent potential bugs
- * from doing really bad things to the system.
- *
- * @returns Pointer to the dummy page.
- * @param   pVM         VM handle.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(void *) MMR3PageDummyHCPtr(PVM pVM)
-{
-    VM_ASSERT_EMT(pVM);
-    if (!pVM->mm.s.pvDummyPage)
-    {
-        pVM->mm.s.pvDummyPage = mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3);
-        AssertRelease(pVM->mm.s.pvDummyPage);
-        pVM->mm.s.HCPhysDummyPage = mmPagePoolPtr2Phys(pVM->mm.s.pPagePoolR3, pVM->mm.s.pvDummyPage);
-        AssertRelease(!(pVM->mm.s.HCPhysDummyPage & ~X86_PTE_PAE_PG_MASK));
-    }
-    return pVM->mm.s.pvDummyPage;
-}
-
-
-/**
- * Gets the HC Phys to the dummy page.
- *
- * The dummy page is used as a place holder to prevent potential bugs
- * from doing really bad things to the system.
- *
- * @returns Pointer to the dummy page.
- * @param   pVM         VM handle.
- * @thread  The Emulation Thread.
- */
-VMMR3DECL(RTHCPHYS) MMR3PageDummyHCPhys(PVM pVM)
-{
-    VM_ASSERT_EMT(pVM);
-    if (!pVM->mm.s.pvDummyPage)
-        MMR3PageDummyHCPtr(pVM);
-    return pVM->mm.s.HCPhysDummyPage;
-}
-
Index: unk/src/VBox/VMM/MMUkHeap.cpp
===================================================================
--- /trunk/src/VBox/VMM/MMUkHeap.cpp	(revision 35332)
+++ 	(revision )
@@ -1,422 +1,0 @@
-/* $Id$ */
-/** @file
- * MM - Memory Manager - Ring-3 Heap with kernel accessible mapping.
- */
-
-/*
- * Copyright (C) 2006-2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_MM_HEAP
-#include <VBox/mm.h>
-#include <VBox/stam.h>
-#include "MMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/log.h>
-
-#include <iprt/assert.h>
-#include <iprt/string.h>
-#include <iprt/heap.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr);
-
-
-
-/**
- * Create a User-kernel heap.
- *
- * This does not require SUPLib to be initialized as we'll lazily allocate the
- * kernel accessible memory on the first alloc call.
- *
- * @returns VBox status.
- * @param   pVM     The handle to the VM the heap should be associated with.
- * @param   ppHeap  Where to store the heap pointer.
- */
-int mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap)
-{
-    PMMUKHEAP pHeap = (PMMUKHEAP)MMR3HeapAllocZU(pUVM, MM_TAG_MM, sizeof(MMUKHEAP));
-    if (pHeap)
-    {
-        int rc = RTCritSectInit(&pHeap->Lock);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Initialize the global stat record.
-             */
-            pHeap->pUVM = pUVM;
-#ifdef MMUKHEAP_WITH_STATISTICS
-            PMMUKHEAPSTAT pStat = &pHeap->Stat;
-            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFrees",           STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.");
-            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
-            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
-                                                                        STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
-            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
-            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
-#endif
-            *ppHeap = pHeap;
-            return VINF_SUCCESS;
-        }
-        AssertRC(rc);
-        MMR3HeapFree(pHeap);
-    }
-    AssertMsgFailed(("failed to allocate heap structure\n"));
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Destroy a User-kernel heap.
- *
- * @param   pHeap   Heap handle.
- */
-void mmR3UkHeapDestroy(PMMUKHEAP pHeap)
-{
-    /*
-     * Start by deleting the lock, that'll trap anyone
-     * attempting to use the heap.
-     */
-    RTCritSectDelete(&pHeap->Lock);
-
-    /*
-     * Walk the sub-heaps and free them.
-     */
-    while (pHeap->pSubHeapHead)
-    {
-        PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
-        pHeap->pSubHeapHead = pSubHeap->pNext;
-        SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
-        //MMR3HeapFree(pSubHeap); - rely on the automatic cleanup.
-    }
-    //MMR3HeapFree(pHeap->stats);
-    //MMR3HeapFree(pHeap);
-}
-
-
-/**
- * Allocate memory associating it with the VM for collective cleanup.
- *
- * The memory will be allocated from the default heap but a header
- * is added in which we keep track of which VM it belongs to and chain
- * all the allocations together so they can be freed in one go.
- *
- * This interface is typically used for memory block which will not be
- * freed during the life of the VM.
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- * @param   pR0Ptr      Where to return the ring-0 address of the memory.
- */
-VMMR3DECL(void *) MMR3UkHeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
-{
-    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
-}
-
-
-/**
- * Same as MMR3UkHeapAlloc().
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- * @param   pR0Ptr      Where to return the ring-0 address of the memory.
- */
-VMMR3DECL(int) MMR3UkHeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
-{
-    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Same as MMR3UkHeapAlloc() only the memory is zeroed.
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- * @param   pR0Ptr      Where to return the ring-0 address of the memory.
- */
-VMMR3DECL(void *) MMR3UkHeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
-{
-    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
-}
-
-
-/**
- * Same as MMR3UkHeapAllocZ().
- *
- * @returns Pointer to allocated memory.
- * @param   pVM         VM handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cbSize      Size of the block.
- * @param   ppv         Where to store the pointer to the allocated memory on success.
- * @param   pR0Ptr      Where to return the ring-0 address of the memory.
- */
-VMMR3DECL(int) MMR3UkHeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
-{
-    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
-    if (pv)
-    {
-        *ppv = pv;
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/***
- * Worker for mmR3UkHeapAlloc that creates and adds a new sub-heap.
- *
- * @returns Pointer to the new sub-heap.
- * @param   pHeap       The heap
- * @param   cbSubHeap   The size of the sub-heap.
- */
-static PMMUKHEAPSUB mmR3UkHeapAddSubHeap(PMMUKHEAP pHeap, size_t cbSubHeap)
-{
-    PMMUKHEAPSUB pSubHeap = (PMMUKHEAPSUB)MMR3HeapAllocU(pHeap->pUVM, MM_TAG_MM/*_UK_HEAP*/, sizeof(*pSubHeap));
-    if (pSubHeap)
-    {
-        pSubHeap->cb = cbSubHeap;
-        int rc = SUPR3PageAllocEx(pSubHeap->cb >> PAGE_SHIFT, 0, &pSubHeap->pv, &pSubHeap->pvR0, NULL);
-        if (RT_SUCCESS(rc))
-        {
-            rc = RTHeapSimpleInit(&pSubHeap->hSimple, pSubHeap->pv, pSubHeap->cb);
-            if (RT_SUCCESS(rc))
-            {
-                pSubHeap->pNext = pHeap->pSubHeapHead;
-                pHeap->pSubHeapHead = pSubHeap;
-                return pSubHeap;
-            }
-
-            /* bail out */
-            SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
-        }
-        MMR3HeapFree(pSubHeap);
-    }
-    return NULL;
-}
-
-
-/**
- * Allocate memory from the heap.
- *
- * @returns Pointer to allocated memory.
- * @param   pHeap       Heap handle.
- * @param   enmTag      Statistics tag. Statistics are collected on a per tag
- *                      basis in addition to a global one. Thus we can easily
- *                      identify how memory is used by the VM.
- * @param   cb          Size of the block.
- * @param   fZero       Whether or not to zero the memory block.
- * @param   pR0Ptr      Where to return the ring-0 pointer.
- */
-static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr)
-{
-    if (pR0Ptr)
-        *pR0Ptr = NIL_RTR0PTR;
-    RTCritSectEnter(&pHeap->Lock);
-
-#ifdef MMUKHEAP_WITH_STATISTICS
-    /*
-     * Find/alloc statistics nodes.
-     */
-    pHeap->Stat.cAllocations++;
-    PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
-    if (pStat)
-        pStat->cAllocations++;
-    else
-    {
-        pStat = (PMMUKHEAPSTAT)MMR3HeapAllocZU(pHeap->pUVM, MM_TAG_MM, sizeof(MMUKHEAPSTAT));
-        if (!pStat)
-        {
-            pHeap->Stat.cFailures++;
-            AssertMsgFailed(("Failed to allocate heap stat record.\n"));
-            RTCritSectLeave(&pHeap->Lock);
-            return NULL;
-        }
-        pStat->Core.Key = (AVLULKEY)enmTag;
-        RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
-
-        pStat->cAllocations++;
-
-        /* register the statistics */
-        PUVM pUVM = pHeap->pUVM;
-        const char *pszTag = mmGetTagName(enmTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/UkHeap/%s", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.",      "/MM/UkHeap/%s/cAllocations", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.",    "/MM/UkHeap/%s/cReallocations", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.",       "/MM/UkHeap/%s/cFrees", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/UkHeap/%s/cFailures", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/UkHeap/%s/cbAllocated", pszTag);
-        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/UkHeap/%s/cbFreed", pszTag);
-    }
-#endif
-
-    /*
-     * Validate input.
-     */
-    if (cb == 0)
-    {
-#ifdef MMUKHEAP_WITH_STATISTICS
-        pStat->cFailures++;
-        pHeap->Stat.cFailures++;
-#endif
-        RTCritSectLeave(&pHeap->Lock);
-        return NULL;
-    }
-
-    /*
-     * Allocate heap block.
-     */
-    cb = RT_ALIGN_Z(cb, MMUKHEAP_SIZE_ALIGNMENT);
-    void *pv = NULL;
-    PMMUKHEAPSUB pSubHeapPrev = NULL;
-    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
-    while (pSubHeap)
-    {
-        if (fZero)
-            pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
-        else
-            pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
-        if (pv)
-        {
-            /* Move the sub-heap with free memory to the head. */
-            if (pSubHeapPrev)
-            {
-                pSubHeapPrev->pNext = pSubHeap->pNext;
-                pSubHeap->pNext = pHeap->pSubHeapHead;
-                pHeap->pSubHeapHead = pSubHeap;
-            }
-            break;
-        }
-        pSubHeapPrev = pSubHeap;
-        pSubHeap = pSubHeap->pNext;
-    }
-    if (RT_UNLIKELY(!pv))
-    {
-        /*
-         * Add another sub-heap.
-         */
-        pSubHeap = mmR3UkHeapAddSubHeap(pHeap, RT_MAX(RT_ALIGN_Z(cb, PAGE_SIZE) + PAGE_SIZE * 16, _256K));
-        if (pSubHeap)
-        {
-            if (fZero)
-                pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
-            else
-                pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
-        }
-        if (RT_UNLIKELY(!pv))
-        {
-            AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cb, enmTag, &enmTag));
-#ifdef MMUKHEAP_WITH_STATISTICS
-            pStat->cFailures++;
-            pHeap->Stat.cFailures++;
-#endif
-            RTCritSectLeave(&pHeap->Lock);
-            return NULL;
-        }
-    }
-
-    /*
-     * Update statistics
-     */
-#ifdef MMUKHEAP_WITH_STATISTICS
-    size_t cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
-    pStat->cbAllocated          += cbActual;
-    pStat->cbCurAllocated       += cbActual;
-    pHeap->Stat.cbAllocated     += cbActual;
-    pHeap->Stat.cbCurAllocated  += cbActual;
-#endif
-
-    if (pR0Ptr)
-        *pR0Ptr = (uintptr_t)pv - (uintptr_t)pSubHeap->pv + pSubHeap->pvR0;
-    RTCritSectLeave(&pHeap->Lock);
-    return pv;
-}
-
-
-/**
- * Releases memory allocated with MMR3UkHeapAlloc() and MMR3UkHeapAllocZ()
- *
- * @param   pVM         The VM handle.
- * @param   pv          Pointer to the memory block to free.
- */
-VMMR3DECL(void) MMR3UkHeapFree(PVM pVM, void *pv, MMTAG enmTag)
-{
-    /* Ignore NULL pointers. */
-    if (!pv)
-        return;
-
-    PMMUKHEAP pHeap = pVM->pUVM->mm.s.pUkHeap;
-    RTCritSectEnter(&pHeap->Lock);
-
-    /*
-     * Find the sub-heap and block
-     */
-#ifdef MMUKHEAP_WITH_STATISTICS
-    size_t cbActual = 0;
-#endif
-    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
-    while (pSubHeap)
-    {
-        if ((uintptr_t)pv - (uintptr_t)pSubHeap->pv < pSubHeap->cb)
-        {
-#ifdef MMUKHEAP_WITH_STATISTICS
-            cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
-            PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
-            if (pStat)
-            {
-                pStat->cFrees++;
-                pStat->cbCurAllocated     -= cbActual;
-                pStat->cbFreed            += cbActual;
-            }
-            pHeap->Stat.cFrees++;
-            pHeap->Stat.cbFreed           += cbActual;
-            pHeap->Stat.cbCurAllocated    -= cbActual;
-#endif
-            RTHeapSimpleFree(pSubHeap->hSimple, pv);
-
-            RTCritSectLeave(&pHeap->Lock);
-            return;
-        }
-    }
-    AssertMsgFailed(("pv=%p\n", pv));
-}
-
Index: /trunk/src/VBox/VMM/Makefile.kmk
===================================================================
--- /trunk/src/VBox/VMM/Makefile.kmk	(revision 35332)
+++ /trunk/src/VBox/VMM/Makefile.kmk	(revision 35333)
@@ -71,65 +71,67 @@
 
 VMMR3_INCS     = \
-	. \
+	include \
 	PATM
+VMMR3_ASINCS   = .
 
 VMMR3_SOURCES   = \
-	CFGM.cpp \
-	CPUM.cpp \
-	DBGF.cpp \
-	DBGFAddr.cpp \
-	DBGFAddrSpace.cpp \
-	DBGFBp.cpp \
-	DBGFCoreWrite.cpp \
-	DBGFCpu.cpp \
-	DBGFDisas.cpp \
-	DBGFInfo.cpp \
-	DBGFLog.cpp \
-	DBGFMem.cpp \
-	DBGFOS.cpp \
-	DBGFReg.cpp \
-	DBGFStack.cpp \
-	DBGFSym.cpp \
-	EM.cpp \
-	EMRaw.cpp \
-	EMHwaccm.cpp \
-	FTM.cpp \
-	IOM.cpp \
-	GMM.cpp \
-	MM.cpp \
-	MMHeap.cpp \
-	MMHyper.cpp \
-	MMPagePool.cpp \
-	MMUkHeap.cpp \
-	PDM.cpp \
-	PDMDevice.cpp \
-	PDMDevHlp.cpp \
-	PDMDevMiscHlp.cpp \
-	PDMDriver.cpp \
-	PDMLdr.cpp \
-	PDMCritSect.cpp \
-	PDMQueue.cpp \
-	PDMThread.cpp \
-	PGM.cpp \
-	PGMDbg.cpp \
-	PGMHandler.cpp \
-	PGMMap.cpp \
-	PGMPhys.cpp \
-	PGMPool.cpp \
-	PGMSavedState.cpp \
-	PGMSharedPage.cpp \
-	SELM.cpp \
-	SSM.cpp \
-	STAM.cpp \
-	TM.cpp \
-	TRPM.cpp \
-	VM.cpp \
-	VMEmt.cpp \
-	VMReq.cpp \
-	VMM.cpp \
-	VMMGuruMeditation.cpp \
-	VMMSwitcher.cpp \
-	VMMTests.cpp \
-	HWACCM.cpp \
+	VMMR3/CFGM.cpp \
+	VMMR3/CPUM.cpp \
+	VMMR3/DBGF.cpp \
+	VMMR3/DBGFAddr.cpp \
+	VMMR3/DBGFAddrSpace.cpp \
+	VMMR3/DBGFBp.cpp \
+	VMMR3/DBGFCoreWrite.cpp \
+	VMMR3/DBGFCpu.cpp \
+	VMMR3/DBGFDisas.cpp \
+	VMMR3/DBGFInfo.cpp \
+	VMMR3/DBGFLog.cpp \
+	VMMR3/DBGFMem.cpp \
+	VMMR3/DBGFOS.cpp \
+	VMMR3/DBGFReg.cpp \
+	VMMR3/DBGFStack.cpp \
+	VMMR3/DBGFSym.cpp \
+	VMMR3/EM.cpp \
+	VMMR3/EMRaw.cpp \
+	VMMR3/EMHwaccm.cpp \
+	VMMR3/FTM.cpp \
+	VMMR3/IOM.cpp \
+	VMMR3/GMM.cpp \
+	VMMR3/MM.cpp \
+	VMMR3/MMHeap.cpp \
+	VMMR3/MMHyper.cpp \
+	VMMR3/MMPagePool.cpp \
+	VMMR3/MMUkHeap.cpp \
+	VMMR3/PDM.cpp \
+	VMMR3/PDMBlkCache.cpp \
+	VMMR3/PDMDevice.cpp \
+	VMMR3/PDMDevHlp.cpp \
+	VMMR3/PDMDevMiscHlp.cpp \
+	VMMR3/PDMDriver.cpp \
+	VMMR3/PDMLdr.cpp \
+	VMMR3/PDMCritSect.cpp \
+	VMMR3/PDMQueue.cpp \
+	VMMR3/PDMThread.cpp \
+	VMMR3/PGM.cpp \
+	VMMR3/PGMDbg.cpp \
+	VMMR3/PGMHandler.cpp \
+	VMMR3/PGMMap.cpp \
+	VMMR3/PGMPhys.cpp \
+	VMMR3/PGMPool.cpp \
+	VMMR3/PGMSavedState.cpp \
+	VMMR3/PGMSharedPage.cpp \
+	VMMR3/SELM.cpp \
+	VMMR3/SSM.cpp \
+	VMMR3/STAM.cpp \
+	VMMR3/TM.cpp \
+	VMMR3/TRPM.cpp \
+	VMMR3/VM.cpp \
+	VMMR3/VMEmt.cpp \
+	VMMR3/VMReq.cpp \
+	VMMR3/VMM.cpp \
+	VMMR3/VMMGuruMeditation.cpp \
+	VMMR3/VMMSwitcher.cpp \
+	VMMR3/VMMTests.cpp \
+	VMMR3/HWACCM.cpp \
 	VMMAll/CPUMAllRegs.cpp \
 	VMMAll/CPUMStack.cpp \
@@ -169,15 +171,14 @@
 	PATM/PATMA.asm \
 	PATM/PATMSSM.cpp \
-	PATM/VMMAll/PATMAll.cpp \
-	PDMBlkCache.cpp
+	PATM/VMMAll/PATMAll.cpp
 ifdef VBOX_WITH_VUSB
- VMMR3_SOURCES += PDMUsb.cpp
+ VMMR3_SOURCES += VMMR3/PDMUsb.cpp
 endif
 ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
  VMMR3_SOURCES += \
- 	PDMAsyncCompletion.cpp \
- 	PDMAsyncCompletionFile.cpp \
- 	PDMAsyncCompletionFileFailsafe.cpp \
- 	PDMAsyncCompletionFileNormal.cpp
+ 	VMMR3/PDMAsyncCompletion.cpp \
+ 	VMMR3/PDMAsyncCompletionFile.cpp \
+ 	VMMR3/PDMAsyncCompletionFileFailsafe.cpp \
+ 	VMMR3/PDMAsyncCompletionFileNormal.cpp
 endif
 if defined(VBOX_WITH_WARNINGS_AS_ERRORS) && "$(KBUILD_TARGET)" == "win"
@@ -202,5 +203,5 @@
 
 # SSM wish to know the build type, host os and arch.
-SSM.cpp_DEFS +=	\
+VMMR3/SSM.cpp_DEFS +=	\
 	KBUILD_TYPE=\"$(KBUILD_TYPE)\" \
 	KBUILD_TARGET=\"$(KBUILD_TARGET)\" \
@@ -216,5 +217,5 @@
 VBoxVMM_DEFS = $(VMMR3_DEFS)
 VBoxVMM_DEFS.$(KBUILD_TARGET) = $(VMMR3_DEFS.$(KBUILD_TARGET))
-VBoxVMM_SOURCES = VBoxVMMDeps.cpp
+VBoxVMM_SOURCES = VMMR3/VBoxVMMDeps.cpp
 VBoxVMM_SONAME.linux = VBoxVMM.so
 
@@ -339,5 +340,5 @@
 
  VMMGC_INCS     := \
- 	. \
+ 	include \
  	VMMGC \
  	PATM
@@ -443,5 +444,5 @@
  endif
  VMMR0_INCS      = \
- 	. \
+ 	include \
  	PATM
 
@@ -526,5 +527,6 @@
  SSMStandalone_TEMPLATE = VBOXR3EXE
  SSMStandalone_DEFS     = IN_VMM_R3 IN_VMM_STATIC SSM_STANDALONE
- SSMStandalone_SOURCES  = SSM.cpp
+ SSMStandalone_INCS     = include
+ SSMStandalone_SOURCES  = VMMR3/SSM.cpp
 endif # !VBOX_ONLY_EXTPACKS
 
Index: /trunk/src/VBox/VMM/PATM/CSAM.cpp
===================================================================
--- /trunk/src/VBox/VMM/PATM/CSAM.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/PATM/CSAM.cpp	(revision 35333)
@@ -48,5 +48,5 @@
 #include <VBox/dis.h>
 #include <VBox/disopcode.h>
-#include <include/internal/pgm.h>
+#include "internal/pgm.h"
 
 
Index: /trunk/src/VBox/VMM/PATM/PATM.cpp
===================================================================
--- /trunk/src/VBox/VMM/PATM/PATM.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/PATM/PATM.cpp	(revision 35333)
@@ -46,5 +46,5 @@
 #include <VBox/dis.h>
 #include <VBox/disopcode.h>
-#include <include/internal/pgm.h>
+#include "internal/pgm.h"
 
 #include <iprt/string.h>
Index: /trunk/src/VBox/VMM/PATM/PATMSSM.cpp
===================================================================
--- /trunk/src/VBox/VMM/PATM/PATMSSM.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/PATM/PATMSSM.cpp	(revision 35333)
@@ -33,5 +33,5 @@
 #include <VBox/vm.h>
 #include <VBox/csam.h>
-#include <include/internal/pgm.h>
+#include "internal/pgm.h"
 #include <VBox/dbg.h>
 #include <VBox/err.h>
Index: unk/src/VBox/VMM/PDM.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2138 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device Manager.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page   pg_pdm      PDM - The Pluggable Device & Driver Manager
- *
- * VirtualBox is designed to be very configurable, i.e. the ability to select
- * virtual devices and configure them uniquely for a VM.  For this reason
- * virtual devices are not statically linked with the VMM but loaded, linked and
- * instantiated at runtime by PDM using the information found in the
- * Configuration Manager (CFGM).
- *
- * While the chief purpose of PDM is to manager of devices their drivers, it
- * also serves as somewhere to put usful things like cross context queues, cross
- * context synchronization (like critsect), VM centric thread management,
- * asynchronous I/O framework, and so on.
- *
- * @see grp_pdm
- *
- *
- * @section sec_pdm_dev     The Pluggable Devices
- *
- * Devices register themselves when the module containing them is loaded.  PDM
- * will call the entry point 'VBoxDevicesRegister' when loading a device module.
- * The device module will then use the supplied callback table to check the VMM
- * version and to register its devices.  Each device have an unique (for the
- * configured VM) name.  The name is not only used in PDM but also in CFGM (to
- * organize device and device instance settings) and by anyone who wants to talk
- * to a specific device instance.
- *
- * When all device modules have been successfully loaded PDM will instantiate
- * those devices which are configured for the VM.  Note that a device may have
- * more than one instance, see network adaptors for instance.  When
- * instantiating a device PDM provides device instance memory and a callback
- * table (aka Device Helpers / DevHlp) with the VM APIs which the device
- * instance is trusted with.
- *
- * Some devices are trusted devices, most are not.  The trusted devices are an
- * integrated part of the VM and can obtain the VM handle from their device
- * instance handles, thus enabling them to call any VM api.  Untrusted devices
- * can only use the callbacks provided during device instantiation.
- *
- * The main purpose in having DevHlps rather than just giving all the devices
- * the VM handle and let them call the internal VM APIs directly, is both to
- * create a binary interface that can be supported across releases and to
- * create a barrier between devices and the VM.  (The trusted / untrusted bit
- * hasn't turned out to be of much use btw., but it's easy to maintain so there
- * isn't any point in removing it.)
- *
- * A device can provide a ring-0 and/or a raw-mode context extension to improve
- * the VM performance by handling exits and traps (respectively) without
- * requiring context switches (to ring-3).  Callbacks for MMIO and I/O ports can
- * needs to be registered specifically for the additional contexts for this to
- * make sense.  Also, the device has to be trusted to be loaded into R0/RC
- * because of the extra privilege it entails.  Note that raw-mode code and data
- * will be subject to relocation.
- *
- *
- * @section sec_pdm_special_devs    Special Devices
- *
- * Several kinds of devices interacts with the VMM and/or other device and PDM
- * will work like a mediator for these. The typical pattern is that the device
- * calls a special registration device helper with a set of callbacks, PDM
- * responds by copying this and providing a pointer to a set helper callbacks
- * for that particular kind of device. Unlike interfaces where the callback
- * table pointer is used a 'this' pointer, these arrangements will use the
- * device instance pointer (PPDMDEVINS) as a kind of 'this' pointer.
- *
- * For an example of this kind of setup, see the PIC. The PIC registers itself
- * by calling PDMDEVHLPR3::pfnPICRegister.  PDM saves the device instance,
- * copies the callback tables (PDMPICREG), resolving the ring-0 and raw-mode
- * addresses in the process, and hands back the pointer to a set of helper
- * methods (PDMPICHLPR3).  The PCI device then queries the ring-0 and raw-mode
- * helpers using PDMPICHLPR3::pfnGetR0Helpers and PDMPICHLPR3::pfnGetRCHelpers.
- * The PCI device repeats ths pfnGetRCHelpers call in it's relocation method
- * since the address changes when RC is relocated.
- *
- * @see grp_pdm_device
- *
- *
- * @section sec_pdm_usbdev  The Pluggable USB Devices
- *
- * USB devices are handled a little bit differently than other devices.  The
- * general concepts wrt. pluggability are mostly the same, but the details
- * varies.  The registration entry point is 'VBoxUsbRegister', the device
- * instance is PDMUSBINS and the callbacks helpers are different.  Also, USB
- * device are restricted to ring-3 and cannot have any ring-0 or raw-mode
- * extensions (at least not yet).
- *
- * The way USB devices work differs greatly from other devices though since they
- * aren't attaches directly to the PCI/ISA/whatever system buses but via a
- * USB host control (OHCI, UHCI or EHCI).  USB devices handles USB requests
- * (URBs) and does not register I/O ports, MMIO ranges or PCI bus
- * devices/functions.
- *
- * @see grp_pdm_usbdev
- *
- *
- * @section sec_pdm_drv     The Pluggable Drivers
- *
- * The VM devices are often accessing host hardware or OS facilities.  For most
- * devices these facilities can be abstracted in one or more levels.  These
- * abstractions are called drivers.
- *
- * For instance take a DVD/CD drive.  This can be connected to a SCSI
- * controller, an ATA controller or a SATA controller.  The basics of the DVD/CD
- * drive implementation remains the same - eject, insert, read, seek, and such.
- * (For the scsi case, you might wanna speak SCSI directly to, but that can of
- * course be fixed - see SCSI passthru.)  So, it
- * makes much sense to have a generic CD/DVD driver which implements this.
- *
- * Then the media 'inserted' into the DVD/CD drive can be a ISO image, or it can
- * be read from a real CD or DVD drive (there are probably other custom formats
- * someone could desire to read or construct too).  So, it would make sense to
- * have abstracted interfaces for dealing with this in a generic way so the
- * cdrom unit doesn't have to implement it all.  Thus we have created the
- * CDROM/DVD media driver family.
- *
- * So, for this example the IDE controller #1 (i.e. secondary) will have
- * the DVD/CD Driver attached to it's LUN #0 (master).  When a media is mounted
- * the DVD/CD Driver will have a ISO, HostDVD or RAW (media) Driver attached.
- *
- * It is possible to configure many levels of drivers inserting filters, loggers,
- * or whatever you desire into the chain.  We're using this for network sniffing
- * for instance.
- *
- * The drivers are loaded in a similar manner to that of the device, namely by
- * iterating a keyspace in CFGM, load the modules listed there and call
- * 'VBoxDriversRegister' with a callback table.
- *
- * @see grp_pdm_driver
- *
- *
- * @section sec_pdm_ifs     Interfaces
- *
- * The pluggable drivers and devices exposes one standard interface (callback
- * table) which is used to construct, destruct, attach, detach,( ++,) and query
- * other interfaces. A device will query the interfaces required for it's
- * operation during init and hot-plug.  PDM may query some interfaces during
- * runtime mounting too.
- *
- * An interface here means a function table contained within the device or
- * driver instance data. Its method are invoked with the function table pointer
- * as the first argument and they will calculate the address of the device or
- * driver instance data from it. (This is one of the aspects which *might* have
- * been better done in C++.)
- *
- * @see grp_pdm_interfaces
- *
- *
- * @section sec_pdm_utils   Utilities
- *
- * As mentioned earlier, PDM is the location of any usful constructs that doesn't
- * quite fit into IPRT. The next subsections will discuss these.
- *
- * One thing these APIs all have in common is that resources will be associated
- * with a device / driver and automatically freed after it has been destroyed if
- * the destructor didn't do this.
- *
- *
- * @subsection sec_pdm_async_completion     Async I/O
- *
- * The PDM Async I/O API provides a somewhat platform agnostic interface for
- * asynchronous I/O.  For reasons of performance and complexity this does not
- * build upon any IPRT API.
- *
- * @todo more details.
- *
- * @see grp_pdm_async_completion
- *
- *
- * @subsection sec_pdm_async_task   Async Task - not implemented
- *
- * @todo implement and describe
- *
- * @see grp_pdm_async_task
- *
- *
- * @subsection sec_pdm_critsect     Critical Section
- *
- * The PDM Critical Section API is currently building on the IPRT API with the
- * same name.  It adds the possibility to use critical sections in ring-0 and
- * raw-mode as well as in ring-3.  There are certain restrictions on the RC and
- * R0 usage though since we're not able to wait on it, nor wake up anyone that
- * is waiting on it.  These restrictions origins with the use of a ring-3 event
- * semaphore.  In a later incarnation we plan to replace the ring-3 event
- * semaphore with a ring-0 one, thus enabling us to wake up waiters while
- * exectuing in ring-0 and making the hardware assisted execution mode more
- * efficient. (Raw-mode won't benefit much from this, naturally.)
- *
- * @see grp_pdm_critsect
- *
- *
- * @subsection sec_pdm_queue        Queue
- *
- * The PDM Queue API is for queuing one or more tasks for later consumption in
- * ring-3 by EMT, and optionally forcing a delayed or ASAP return to ring-3.  The
- * queues can also be run on a timer basis as an alternative to the ASAP thing.
- * The queue will be flushed at forced action time.
- *
- * A queue can also be used by another thread (a I/O worker for instance) to
- * send work / events over to the EMT.
- *
- * @see grp_pdm_queue
- *
- *
- * @subsection sec_pdm_task        Task - not implemented yet
- *
- * The PDM Task API is for flagging a task for execution at a later point when
- * we're back in ring-3, optionally forcing the ring-3 return to happen ASAP.
- * As you can see the concept is similar to queues only simpler.
- *
- * A task can also be scheduled by another thread (a I/O worker for instance) as
- * a mean of getting something done in EMT.
- *
- * @see grp_pdm_task
- *
- *
- * @subsection sec_pdm_thread       Thread
- *
- * The PDM Thread API is there to help devices and drivers manage their threads
- * correctly wrt. power on, suspend, resume, power off and destruction.
- *
- * The general usage pattern for threads in the employ of devices and drivers is
- * that they shuffle data or requests while the VM is running and stop doing
- * this when the VM is paused or powered down. Rogue threads running while the
- * VM is paused can cause the state to change during saving or have other
- * unwanted side effects. The PDM Threads API ensures that this won't happen.
- *
- * @see grp_pdm_thread
- *
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/pgm.h>
-#include <VBox/ssm.h>
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/vmm.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/sup.h>
-
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-#include <iprt/ldr.h>
-#include <iprt/path.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The PDM saved state version. */
-#define PDM_SAVED_STATE_VERSION             4
-#define PDM_SAVED_STATE_VERSION_PRE_NMI_FF  3
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) pdmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
-static DECLCALLBACK(int) pdmR3SaveExec(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int) pdmR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static DECLCALLBACK(int) pdmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM);
-
-
-
-/**
- * Initializes the PDM part of the UVM.
- *
- * This doesn't really do much right now but has to be here for the sake
- * of completeness.
- *
- * @returns VBox status code.
- * @param   pUVM        Pointer to the user mode VM structure.
- */
-VMMR3DECL(int) PDMR3InitUVM(PUVM pUVM)
-{
-    AssertCompile(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
-    AssertRelease(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
-    pUVM->pdm.s.pModules   = NULL;
-    pUVM->pdm.s.pCritSects = NULL;
-    return RTCritSectInit(&pUVM->pdm.s.ListCritSect);
-}
-
-
-/**
- * Initializes the PDM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) PDMR3Init(PVM pVM)
-{
-    LogFlow(("PDMR3Init\n"));
-
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertRelease(!(RT_OFFSETOF(VM, pdm.s) & 31));
-    AssertRelease(sizeof(pVM->pdm.s) <= sizeof(pVM->pdm.padding));
-    AssertCompileMemberAlignment(PDM, CritSect, sizeof(uintptr_t));
-    /*
-     * Init the structure.
-     */
-    pVM->pdm.s.offVM = RT_OFFSETOF(VM, pdm.s);
-    pVM->pdm.s.GCPhysVMMDevHeap = NIL_RTGCPHYS;
-
-    /*
-     * Initialize sub components.
-     */
-    int rc = pdmR3CritSectInitStats(pVM);
-    if (RT_SUCCESS(rc))
-        rc = PDMR3CritSectInit(pVM, &pVM->pdm.s.CritSect, RT_SRC_POS, "PDM");
-    if (RT_SUCCESS(rc))
-        rc = pdmR3LdrInitU(pVM->pUVM);
-#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
-    if (RT_SUCCESS(rc))
-        rc = pdmR3AsyncCompletionInit(pVM);
-#endif
-    if (RT_SUCCESS(rc))
-        rc = pdmR3BlkCacheInit(pVM);
-    if (RT_SUCCESS(rc))
-        rc = pdmR3DrvInit(pVM);
-    if (RT_SUCCESS(rc))
-        rc = pdmR3DevInit(pVM);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Register the saved state data unit.
-         */
-        rc = SSMR3RegisterInternal(pVM, "pdm", 1, PDM_SAVED_STATE_VERSION, 128,
-                                   NULL, pdmR3LiveExec, NULL,
-                                   NULL, pdmR3SaveExec, NULL,
-                                   pdmR3LoadPrep, pdmR3LoadExec, NULL);
-        if (RT_SUCCESS(rc))
-        {
-            LogFlow(("PDM: Successfully initialized\n"));
-            return rc;
-        }
-    }
-
-    /*
-     * Cleanup and return failure.
-     */
-    PDMR3Term(pVM);
-    LogFlow(("PDMR3Init: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM         VM handle.
- * @param   offDelta    Relocation delta relative to old location.
- * @remark  The loader subcomponent is relocated by PDMR3LdrRelocate() very
- *          early in the relocation phase.
- */
-VMMR3DECL(void) PDMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    LogFlow(("PDMR3Relocate\n"));
-
-    /*
-     * Queues.
-     */
-    pdmR3QueueRelocate(pVM, offDelta);
-    pVM->pdm.s.pDevHlpQueueRC = PDMQueueRCPtr(pVM->pdm.s.pDevHlpQueueR3);
-
-    /*
-     * Critical sections.
-     */
-    pdmR3CritSectRelocate(pVM);
-
-    /*
-     * The registered PIC.
-     */
-    if (pVM->pdm.s.Pic.pDevInsRC)
-    {
-        pVM->pdm.s.Pic.pDevInsRC            += offDelta;
-        pVM->pdm.s.Pic.pfnSetIrqRC          += offDelta;
-        pVM->pdm.s.Pic.pfnGetInterruptRC    += offDelta;
-    }
-
-    /*
-     * The registered APIC.
-     */
-    if (pVM->pdm.s.Apic.pDevInsRC)
-    {
-        pVM->pdm.s.Apic.pDevInsRC           += offDelta;
-        pVM->pdm.s.Apic.pfnGetInterruptRC   += offDelta;
-        pVM->pdm.s.Apic.pfnSetBaseRC        += offDelta;
-        pVM->pdm.s.Apic.pfnGetBaseRC        += offDelta;
-        pVM->pdm.s.Apic.pfnSetTPRRC         += offDelta;
-        pVM->pdm.s.Apic.pfnGetTPRRC         += offDelta;
-        pVM->pdm.s.Apic.pfnBusDeliverRC     += offDelta;
-        if (pVM->pdm.s.Apic.pfnLocalInterruptRC)
-            pVM->pdm.s.Apic.pfnLocalInterruptRC += offDelta;
-        pVM->pdm.s.Apic.pfnWriteMSRRC       += offDelta;
-        pVM->pdm.s.Apic.pfnReadMSRRC        += offDelta;
-    }
-
-    /*
-     * The registered I/O APIC.
-     */
-    if (pVM->pdm.s.IoApic.pDevInsRC)
-    {
-        pVM->pdm.s.IoApic.pDevInsRC         += offDelta;
-        pVM->pdm.s.IoApic.pfnSetIrqRC       += offDelta;
-        if (pVM->pdm.s.IoApic.pfnSendMsiRC)
-            pVM->pdm.s.IoApic.pfnSendMsiRC      += offDelta;
-    }
-
-    /*
-     * The register PCI Buses.
-     */
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pdm.s.aPciBuses); i++)
-    {
-        if (pVM->pdm.s.aPciBuses[i].pDevInsRC)
-        {
-            pVM->pdm.s.aPciBuses[i].pDevInsRC   += offDelta;
-            pVM->pdm.s.aPciBuses[i].pfnSetIrqRC += offDelta;
-        }
-    }
-
-    /*
-     * Devices & Drivers.
-     */
-    PCPDMDEVHLPRC pDevHlpRC;
-    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pDevHlpRC);
-    AssertReleaseMsgRC(rc, ("rc=%Rrc when resolving g_pdmRCDevHlp\n", rc));
-
-    PCPDMDRVHLPRC pDrvHlpRC;
-    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pDrvHlpRC);
-    AssertReleaseMsgRC(rc, ("rc=%Rrc when resolving g_pdmRCDevHlp\n", rc));
-
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
-        {
-            pDevIns->pHlpRC             = pDevHlpRC;
-            pDevIns->pvInstanceDataRC   = MMHyperR3ToRC(pVM, pDevIns->pvInstanceDataR3);
-            if (pDevIns->pCritSectR3)
-                pDevIns->pCritSectRC    = MMHyperR3ToRC(pVM, pDevIns->pCritSectR3);
-            pDevIns->Internal.s.pVMRC   = pVM->pVMRC;
-            if (pDevIns->Internal.s.pPciBusR3)
-                pDevIns->Internal.s.pPciBusRC    = MMHyperR3ToRC(pVM, pDevIns->Internal.s.pPciBusR3);
-            if (pDevIns->Internal.s.pPciDeviceR3)
-                pDevIns->Internal.s.pPciDeviceRC = MMHyperR3ToRC(pVM, pDevIns->Internal.s.pPciDeviceR3);
-            if (pDevIns->pReg->pfnRelocate)
-            {
-                LogFlow(("PDMR3Relocate: Relocating device '%s'/%d\n",
-                         pDevIns->pReg->szName, pDevIns->iInstance));
-                pDevIns->pReg->pfnRelocate(pDevIns, offDelta);
-            }
-        }
-
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-        {
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-            {
-                if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
-                {
-                    pDrvIns->pHlpRC = pDrvHlpRC;
-                    pDrvIns->pvInstanceDataRC = MMHyperR3ToRC(pVM, pDrvIns->pvInstanceDataR3);
-                    pDrvIns->Internal.s.pVMRC = pVM->pVMRC;
-                    if (pDrvIns->pReg->pfnRelocate)
-                    {
-                        LogFlow(("PDMR3Relocate: Relocating driver '%s'/%u attached to '%s'/%d/%u\n",
-                                 pDrvIns->pReg->szName, pDrvIns->iInstance,
-                                 pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun));
-                        pDrvIns->pReg->pfnRelocate(pDrvIns, offDelta);
-                    }
-                }
-            }
-        }
-
-    }
-}
-
-
-/**
- * Worker for pdmR3Term that terminates a LUN chain.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pLun        The head of the chain.
- * @param   pszDevice   The name of the device (for logging).
- * @param   iInstance   The device instance number (for logging).
- */
-static void pdmR3TermLuns(PVM pVM, PPDMLUN pLun, const char *pszDevice, unsigned iInstance)
-{
-    for (; pLun; pLun = pLun->pNext)
-    {
-        /*
-         * Destroy them one at a time from the bottom up.
-         * (The serial device/drivers depends on this - bad.)
-         */
-        PPDMDRVINS pDrvIns = pLun->pBottom;
-        pLun->pBottom = pLun->pTop = NULL;
-        while (pDrvIns)
-        {
-            PPDMDRVINS pDrvNext = pDrvIns->Internal.s.pUp;
-
-            if (pDrvIns->pReg->pfnDestruct)
-            {
-                LogFlow(("pdmR3DevTerm: Destroying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                         pDrvIns->pReg->szName, pDrvIns->iInstance, pLun->iLun, pszDevice, iInstance));
-                pDrvIns->pReg->pfnDestruct(pDrvIns);
-            }
-            pDrvIns->Internal.s.pDrv->cInstances--;
-
-            TMR3TimerDestroyDriver(pVM, pDrvIns);
-            //PDMR3QueueDestroyDriver(pVM, pDrvIns);
-            //pdmR3ThreadDestroyDriver(pVM, pDrvIns);
-            SSMR3DeregisterDriver(pVM, pDrvIns, NULL, 0);
-
-            pDrvIns = pDrvNext;
-        }
-    }
-}
-
-
-/**
- * Terminates the PDM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) PDMR3Term(PVM pVM)
-{
-    LogFlow(("PDMR3Term:\n"));
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-
-    /*
-     * Iterate the device instances and attach drivers, doing
-     * relevant destruction processing.
-     *
-     * N.B. There is no need to mess around freeing memory allocated
-     *      from any MM heap since MM will do that in its Term function.
-     */
-    /* usb ones first. */
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        pdmR3TermLuns(pVM, pUsbIns->Internal.s.pLuns, pUsbIns->pReg->szName, pUsbIns->iInstance);
-
-        if (pUsbIns->pReg->pfnDestruct)
-        {
-            LogFlow(("pdmR3DevTerm: Destroying - device '%s'/%d\n",
-                     pUsbIns->pReg->szName, pUsbIns->iInstance));
-            pUsbIns->pReg->pfnDestruct(pUsbIns);
-        }
-
-        //TMR3TimerDestroyUsb(pVM, pUsbIns);
-        //SSMR3DeregisterUsb(pVM, pUsbIns, NULL, 0);
-        pdmR3ThreadDestroyUsb(pVM, pUsbIns);
-    }
-
-    /* then the 'normal' ones. */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        pdmR3TermLuns(pVM, pDevIns->Internal.s.pLunsR3, pDevIns->pReg->szName, pDevIns->iInstance);
-
-        if (pDevIns->pReg->pfnDestruct)
-        {
-            LogFlow(("pdmR3DevTerm: Destroying - device '%s'/%d\n",
-                     pDevIns->pReg->szName, pDevIns->iInstance));
-            pDevIns->pReg->pfnDestruct(pDevIns);
-        }
-
-        TMR3TimerDestroyDevice(pVM, pDevIns);
-        //SSMR3DeregisterDriver(pVM, pDevIns, NULL, 0);
-        pdmR3CritSectDeleteDevice(pVM, pDevIns);
-        //pdmR3ThreadDestroyDevice(pVM, pDevIns);
-        //PDMR3QueueDestroyDevice(pVM, pDevIns);
-        PGMR3PhysMMIO2Deregister(pVM, pDevIns, UINT32_MAX);
-    }
-
-    /*
-     * Destroy all threads.
-     */
-    pdmR3ThreadDestroyAll(pVM);
-
-    /*
-     * Destroy the block cache.
-     */
-    pdmR3BlkCacheTerm(pVM);
-
-#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
-    /*
-     * Free async completion managers.
-     */
-    pdmR3AsyncCompletionTerm(pVM);
-#endif
-
-    /*
-     * Free modules.
-     */
-    pdmR3LdrTermU(pVM->pUVM);
-
-    /*
-     * Destroy the PDM lock.
-     */
-    PDMR3CritSectDelete(&pVM->pdm.s.CritSect);
-    /* The MiscCritSect is deleted by PDMR3CritSectTerm. */
-
-    LogFlow(("PDMR3Term: returns %Rrc\n", VINF_SUCCESS));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Terminates the PDM part of the UVM.
- *
- * This will unload any modules left behind.
- *
- * @param   pUVM        Pointer to the user mode VM structure.
- */
-VMMR3DECL(void) PDMR3TermUVM(PUVM pUVM)
-{
-    /*
-     * In the normal cause of events we will now call pdmR3LdrTermU for
-     * the second time. In the case of init failure however, this might
-     * the first time, which is why we do it.
-     */
-    pdmR3LdrTermU(pUVM);
-
-    Assert(pUVM->pdm.s.pCritSects == NULL);
-    RTCritSectDelete(&pUVM->pdm.s.ListCritSect);
-}
-
-
-/**
- * Bits that are saved in pass 0 and in the final pass.
- *
- * @param   pVM             The VM handle.
- * @param   pSSM            The saved state handle.
- */
-static void pdmR3SaveBoth(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Save the list of device instances so we can check that they're all still
-     * there when we load the state and that nothing new has been added.
-     */
-    uint32_t i = 0;
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3, i++)
-    {
-        SSMR3PutU32(pSSM, i);
-        SSMR3PutStrZ(pSSM, pDevIns->pReg->szName);
-        SSMR3PutU32(pSSM, pDevIns->iInstance);
-    }
-    SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
-}
-
-
-/**
- * Live save.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pSSM            The saved state handle.
- * @param   uPass           The pass.
- */
-static DECLCALLBACK(int) pdmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    LogFlow(("pdmR3LiveExec:\n"));
-    AssertReturn(uPass == 0, VERR_INTERNAL_ERROR_4);
-    pdmR3SaveBoth(pVM, pSSM);
-    return VINF_SSM_DONT_CALL_AGAIN;
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pSSM            The saved state handle.
- */
-static DECLCALLBACK(int) pdmR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
-{
-    LogFlow(("pdmR3SaveExec:\n"));
-
-    /*
-     * Save interrupt and DMA states.
-     */
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[idCpu];
-        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC));
-        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC));
-        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_NMI));
-        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_SMI));
-    }
-    SSMR3PutU32(pSSM, VM_FF_ISSET(pVM, VM_FF_PDM_DMA));
-
-    pdmR3SaveBoth(pVM, pSSM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Prepare state load operation.
- *
- * This will dispatch pending operations and clear the FFs governed by PDM and its devices.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pSSM        The SSM handle.
- */
-static DECLCALLBACK(int) pdmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
-{
-    LogFlow(("pdmR3LoadPrep: %s%s\n",
-             VM_FF_ISSET(pVM, VM_FF_PDM_QUEUES)     ? " VM_FF_PDM_QUEUES" : "",
-             VM_FF_ISSET(pVM, VM_FF_PDM_DMA)        ? " VM_FF_PDM_DMA" : ""));
-#ifdef LOG_ENABLED
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[idCpu];
-        LogFlow(("pdmR3LoadPrep: VCPU %u %s%s\n", idCpu,
-                VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC) ? " VMCPU_FF_INTERRUPT_APIC" : "",
-                VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC)  ? " VMCPU_FF_INTERRUPT_PIC" : ""));
-    }
-#endif
-
-    /*
-     * In case there is work pending that will raise an interrupt,
-     * start a DMA transfer, or release a lock. (unlikely)
-     */
-    if (VM_FF_ISSET(pVM, VM_FF_PDM_QUEUES))
-        PDMR3QueueFlushAll(pVM);
-
-    /* Clear the FFs. */
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[idCpu];
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_SMI);
-    }
-    VM_FF_CLEAR(pVM, VM_FF_PDM_DMA);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) pdmR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    int rc;
-
-    LogFlow(("pdmR3LoadExec: uPass=%#x\n", uPass));
-
-    /*
-     * Validate version.
-     */
-    if (    uVersion != PDM_SAVED_STATE_VERSION
-        &&  uVersion != PDM_SAVED_STATE_VERSION_PRE_NMI_FF)
-    {
-        AssertMsgFailed(("Invalid version uVersion=%d!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    if (uPass == SSM_PASS_FINAL)
-    {
-        /*
-         * Load the interrupt and DMA states.
-         */
-        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-        {
-            PVMCPU pVCpu = &pVM->aCpus[idCpu];
-
-            /* APIC interrupt */
-            uint32_t fInterruptPending = 0;
-            rc = SSMR3GetU32(pSSM, &fInterruptPending);
-            if (RT_FAILURE(rc))
-                return rc;
-            if (fInterruptPending & ~1)
-            {
-                AssertMsgFailed(("fInterruptPending=%#x (APIC)\n", fInterruptPending));
-                return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-            }
-            AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC));
-            if (fInterruptPending)
-                VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC);
-
-            /* PIC interrupt */
-            fInterruptPending = 0;
-            rc = SSMR3GetU32(pSSM, &fInterruptPending);
-            if (RT_FAILURE(rc))
-                return rc;
-            if (fInterruptPending & ~1)
-            {
-                AssertMsgFailed(("fInterruptPending=%#x (PIC)\n", fInterruptPending));
-                return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-            }
-            AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC));
-            if (fInterruptPending)
-                VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);
-
-            if (uVersion > PDM_SAVED_STATE_VERSION_PRE_NMI_FF)
-            {
-                /* NMI interrupt */
-                fInterruptPending = 0;
-                rc = SSMR3GetU32(pSSM, &fInterruptPending);
-                if (RT_FAILURE(rc))
-                    return rc;
-                if (fInterruptPending & ~1)
-                {
-                    AssertMsgFailed(("fInterruptPending=%#x (NMI)\n", fInterruptPending));
-                    return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-                }
-                AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_NMI));
-                if (fInterruptPending)
-                    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
-
-                /* SMI interrupt */
-                fInterruptPending = 0;
-                rc = SSMR3GetU32(pSSM, &fInterruptPending);
-                if (RT_FAILURE(rc))
-                    return rc;
-                if (fInterruptPending & ~1)
-                {
-                    AssertMsgFailed(("fInterruptPending=%#x (SMI)\n", fInterruptPending));
-                    return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-                }
-                AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_SMI));
-                if (fInterruptPending)
-                    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI);
-            }
-        }
-
-        /* DMA pending */
-        uint32_t fDMAPending = 0;
-        rc = SSMR3GetU32(pSSM, &fDMAPending);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (fDMAPending & ~1)
-        {
-            AssertMsgFailed(("fDMAPending=%#x\n", fDMAPending));
-            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-        }
-        if (fDMAPending)
-            VM_FF_SET(pVM, VM_FF_PDM_DMA);
-        Log(("pdmR3LoadExec: VM_FF_PDM_DMA=%RTbool\n", VM_FF_ISSET(pVM, VM_FF_PDM_DMA)));
-    }
-
-    /*
-     * Load the list of devices and verify that they are all there.
-     */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-        pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_FOUND;
-
-    for (uint32_t i = 0; ; i++)
-    {
-        /* Get the sequence number / terminator. */
-        uint32_t    u32Sep;
-        rc = SSMR3GetU32(pSSM, &u32Sep);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (u32Sep == UINT32_MAX)
-            break;
-        if (u32Sep != i)
-            AssertMsgFailedReturn(("Out of sequence. u32Sep=%#x i=%#x\n", u32Sep, i), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-        /* Get the name and instance number. */
-        char szName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
-        rc = SSMR3GetStrZ(pSSM, szName, sizeof(szName));
-        if (RT_FAILURE(rc))
-            return rc;
-        uint32_t iInstance;
-        rc = SSMR3GetU32(pSSM, &iInstance);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /* Try locate it. */
-        PPDMDEVINS pDevIns;
-        for (pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-            if (   !strcmp(szName, pDevIns->pReg->szName)
-                && pDevIns->iInstance == iInstance)
-            {
-                AssertLogRelMsgReturn(!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_FOUND),
-                                      ("%s/#%u\n", pDevIns->pReg->szName, pDevIns->iInstance),
-                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-                pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_FOUND;
-                break;
-            }
-        if (!pDevIns)
-        {
-            LogRel(("Device '%s'/%d not found in current config\n", szName, iInstance));
-            if (SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT)
-                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Device '%s'/%d not found in current config"), szName, iInstance);
-        }
-    }
-
-    /*
-     * Check that no additional devices were configured.
-     */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-        if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_FOUND))
-        {
-            LogRel(("Device '%s'/%d not found in the saved state\n", pDevIns->pReg->szName, pDevIns->iInstance));
-            if (SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT)
-                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Device '%s'/%d not found in the saved state"),
-                                        pDevIns->pReg->szName, pDevIns->iInstance);
-        }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for PDMR3PowerOn that deals with one driver.
- *
- * @param   pDrvIns             The driver instance.
- * @param   pszDeviceName       The parent device name.
- * @param   iDevInstance        The parent device instance number.
- * @param   iLun                The parent LUN number.
- */
-DECLINLINE(int) pdmR3PowerOnDrv(PPDMDRVINS pDrvIns, const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
-{
-    Assert(pDrvIns->Internal.s.fVMSuspended);
-    if (pDrvIns->pReg->pfnPowerOn)
-    {
-        LogFlow(("PDMR3PowerOn: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                 pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-        int rc = VINF_SUCCESS; pDrvIns->pReg->pfnPowerOn(pDrvIns);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDMR3PowerOn: driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
-                    pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance, rc));
-            return rc;
-        }
-    }
-    pDrvIns->Internal.s.fVMSuspended = false;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for PDMR3PowerOn that deals with one USB device instance.
- *
- * @returns VBox status code.
- * @param   pUsbIns             The USB device instance.
- */
-DECLINLINE(int) pdmR3PowerOnUsb(PPDMUSBINS pUsbIns)
-{
-    Assert(pUsbIns->Internal.s.fVMSuspended);
-    if (pUsbIns->pReg->pfnVMPowerOn)
-    {
-        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-        int rc = VINF_SUCCESS; pUsbIns->pReg->pfnVMPowerOn(pUsbIns);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDMR3PowerOn: device '%s'/%d -> %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-            return rc;
-        }
-    }
-    pUsbIns->Internal.s.fVMSuspended = false;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for PDMR3PowerOn that deals with one device instance.
- *
- * @returns VBox status code.
- * @param   pDevIns             The device instance.
- */
-DECLINLINE(int) pdmR3PowerOnDev(PPDMDEVINS pDevIns)
-{
-    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
-    if (pDevIns->pReg->pfnPowerOn)
-    {
-        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-        int rc = VINF_SUCCESS; pDevIns->pReg->pfnPowerOn(pDevIns);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDMR3PowerOn: device '%s'/%d -> %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-    }
-    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * This function will notify all the devices and their
- * attached drivers about the VM now being powered on.
- *
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(void) PDMR3PowerOn(PVM pVM)
-{
-    LogFlow(("PDMR3PowerOn:\n"));
-
-    /*
-     * Iterate thru the device instances and USB device instances,
-     * processing the drivers associated with those.
-     */
-    int rc = VINF_SUCCESS;
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
-                rc = pdmR3PowerOnDrv(pDrvIns, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun);
-        if (RT_SUCCESS(rc))
-            rc = pdmR3PowerOnDev(pDevIns);
-    }
-
-#ifdef VBOX_WITH_USB
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
-                rc = pdmR3PowerOnDrv(pDrvIns, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun);
-        if (RT_SUCCESS(rc))
-            rc = pdmR3PowerOnUsb(pUsbIns);
-    }
-#endif
-
-#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
-    pdmR3AsyncCompletionResume(pVM);
-#endif
-
-    /*
-     * Resume all threads.
-     */
-    if (RT_SUCCESS(rc))
-        pdmR3ThreadResumeAll(pVM);
-
-    /*
-     * On failure, clean up via PDMR3Suspend.
-     */
-    if (RT_FAILURE(rc))
-        PDMR3Suspend(pVM);
-
-    LogFlow(("PDMR3PowerOn: returns %Rrc\n", rc));
-    return /*rc*/;
-}
-
-
-/**
- * Worker for PDMR3Reset that deals with one driver.
- *
- * @param   pDrvIns             The driver instance.
- * @param   pcAsync             The asynchronous reset notification counter.
- * @param   pszDeviceName       The parent device name.
- * @param   iDevInstance        The parent device instance number.
- * @param   iLun                The parent LUN number.
- */
-DECLINLINE(bool) pdmR3ResetDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
-                               const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
-{
-    if (!pDrvIns->Internal.s.fVMReset)
-    {
-        pDrvIns->Internal.s.fVMReset = true;
-        if (pDrvIns->pReg->pfnReset)
-        {
-            if (!pDrvIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3Reset: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-                pDrvIns->pReg->pfnReset(pDrvIns);
-                if (pDrvIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3Reset: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-            }
-            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
-            {
-                pDrvIns->Internal.s.pfnAsyncNotify = false;
-                LogFlow(("PDMR3Reset: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-            }
-            if (pDrvIns->Internal.s.pfnAsyncNotify)
-            {
-                pDrvIns->Internal.s.fVMReset = false;
-                (*pcAsync)++;
-                return false;
-            }
-        }
-    }
-    return true;
-}
-
-
-/**
- * Worker for PDMR3Reset that deals with one USB device instance.
- *
- * @param   pUsbIns             The USB device instance.
- * @param   pcAsync             The asynchronous reset notification counter.
- */
-DECLINLINE(void) pdmR3ResetUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
-{
-    if (!pUsbIns->Internal.s.fVMReset)
-    {
-        pUsbIns->Internal.s.fVMReset = true;
-        if (pUsbIns->pReg->pfnVMReset)
-        {
-            if (!pUsbIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3Reset: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-                pUsbIns->pReg->pfnVMReset(pUsbIns);
-                if (pUsbIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3Reset: Async notification started - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-            }
-            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
-            {
-                LogFlow(("PDMR3Reset: Async notification completed - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
-            }
-            if (pUsbIns->Internal.s.pfnAsyncNotify)
-            {
-                pUsbIns->Internal.s.fVMReset = false;
-                (*pcAsync)++;
-            }
-        }
-    }
-}
-
-
-/**
- * Worker for PDMR3Reset that deals with one device instance.
- *
- * @param   pDevIns             The device instance.
- * @param   pcAsync             The asynchronous reset notification counter.
- */
-DECLINLINE(void) pdmR3ResetDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
-{
-    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_RESET))
-    {
-        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_RESET;
-        if (pDevIns->pReg->pfnReset)
-        {
-            if (!pDevIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3Reset: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-                pDevIns->pReg->pfnReset(pDevIns);
-                if (pDevIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3Reset: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-            }
-            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
-            {
-                LogFlow(("PDMR3Reset: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-                pDevIns->Internal.s.pfnAsyncNotify = NULL;
-            }
-            if (pDevIns->Internal.s.pfnAsyncNotify)
-            {
-                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_RESET;
-                (*pcAsync)++;
-            }
-        }
-    }
-}
-
-
-/**
- * Resets a virtual CPU.
- *
- * Used by PDMR3Reset and CPU hot plugging.
- *
- * @param   pVCpu               The virtual CPU handle.
- */
-VMMR3DECL(void) PDMR3ResetCpu(PVMCPU pVCpu)
-{
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_SMI);
-}
-
-
-/**
- * This function will notify all the devices and their attached drivers about
- * the VM now being reset.
- *
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(void) PDMR3Reset(PVM pVM)
-{
-    LogFlow(("PDMR3Reset:\n"));
-
-    /*
-     * Clear all the reset flags.
-     */
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_RESET;
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                pDrvIns->Internal.s.fVMReset = false;
-    }
-#ifdef VBOX_WITH_USB
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        pUsbIns->Internal.s.fVMReset = false;
-        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                pDrvIns->Internal.s.fVMReset = false;
-    }
-#endif
-
-    /*
-     * The outer loop repeats until there are no more async requests.
-     */
-    unsigned cAsync;
-    for (unsigned iLoop = 0; ; iLoop++)
-    {
-        /*
-         * Iterate thru the device instances and USB device instances,
-         * processing the drivers associated with those.
-         */
-        cAsync = 0;
-        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-        {
-            unsigned const cAsyncStart = cAsync;
-
-            if (cAsync == cAsyncStart)
-                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                        if (!pdmR3ResetDrv(pDrvIns, &cAsync, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
-                            break;
-
-                        if (cAsync == cAsyncStart)
-                pdmR3ResetDev(pDevIns, &cAsync);
-        }
-
-#ifdef VBOX_WITH_USB
-        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-        {
-            unsigned const cAsyncStart = cAsync;
-
-            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                    if (!pdmR3ResetDrv(pDrvIns, &cAsync, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
-                        break;
-
-            if (cAsync == cAsyncStart)
-                pdmR3ResetUsb(pUsbIns, &cAsync);
-        }
-#endif
-        if (!cAsync)
-            break;
-
-        /*
-         * Process requests.
-         */
-        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
-         *        bit I hope... */
-        int rc = VMR3AsyncPdmNotificationWaitU(&pVM->pUVM->aCpus[0]);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        rc = VMR3ReqProcessU(pVM->pUVM, 0/*idDstCpu*/);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-    }
-
-    /*
-     * Clear all pending interrupts and DMA operations.
-     */
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-        PDMR3ResetCpu(&pVM->aCpus[idCpu]);
-    VM_FF_CLEAR(pVM, VM_FF_PDM_DMA);
-
-    LogFlow(("PDMR3Reset: returns void\n"));
-}
-
-
-/**
- * Worker for PDMR3Suspend that deals with one driver.
- *
- * @param   pDrvIns             The driver instance.
- * @param   pcAsync             The asynchronous suspend notification counter.
- * @param   pszDeviceName       The parent device name.
- * @param   iDevInstance        The parent device instance number.
- * @param   iLun                The parent LUN number.
- */
-DECLINLINE(bool) pdmR3SuspendDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
-                                 const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
-{
-    if (!pDrvIns->Internal.s.fVMSuspended)
-    {
-        pDrvIns->Internal.s.fVMSuspended = true;
-        if (pDrvIns->pReg->pfnSuspend)
-        {
-            if (!pDrvIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3Suspend: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-                pDrvIns->pReg->pfnSuspend(pDrvIns);
-                if (pDrvIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3Suspend: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-            }
-            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
-            {
-                pDrvIns->Internal.s.pfnAsyncNotify = false;
-                LogFlow(("PDMR3Suspend: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-            }
-            if (pDrvIns->Internal.s.pfnAsyncNotify)
-            {
-                pDrvIns->Internal.s.fVMSuspended = false;
-                (*pcAsync)++;
-                return false;
-            }
-        }
-    }
-    return true;
-}
-
-
-/**
- * Worker for PDMR3Suspend that deals with one USB device instance.
- *
- * @param   pUsbIns             The USB device instance.
- * @param   pcAsync             The asynchronous suspend notification counter.
- */
-DECLINLINE(void) pdmR3SuspendUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
-{
-    if (!pUsbIns->Internal.s.fVMSuspended)
-    {
-        pUsbIns->Internal.s.fVMSuspended = true;
-        if (pUsbIns->pReg->pfnVMSuspend)
-        {
-            if (!pUsbIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-                pUsbIns->pReg->pfnVMSuspend(pUsbIns);
-                if (pUsbIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3Suspend: Async notification started - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-            }
-            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
-            {
-                LogFlow(("PDMR3Suspend: Async notification completed - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
-            }
-            if (pUsbIns->Internal.s.pfnAsyncNotify)
-            {
-                pUsbIns->Internal.s.fVMSuspended = false;
-                (*pcAsync)++;
-            }
-        }
-    }
-}
-
-
-/**
- * Worker for PDMR3Suspend that deals with one device instance.
- *
- * @param   pDevIns             The device instance.
- * @param   pcAsync             The asynchronous suspend notification counter.
- */
-DECLINLINE(void) pdmR3SuspendDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
-{
-    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
-    {
-        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
-        if (pDevIns->pReg->pfnSuspend)
-        {
-            if (!pDevIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-                pDevIns->pReg->pfnSuspend(pDevIns);
-                if (pDevIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3Suspend: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-            }
-            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
-            {
-                LogFlow(("PDMR3Suspend: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-                pDevIns->Internal.s.pfnAsyncNotify = NULL;
-            }
-            if (pDevIns->Internal.s.pfnAsyncNotify)
-            {
-                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
-                (*pcAsync)++;
-            }
-        }
-    }
-}
-
-
-/**
- * This function will notify all the devices and their attached drivers about
- * the VM now being suspended.
- *
- * @param   pVM     The VM Handle.
- * @thread  EMT(0)
- */
-VMMR3DECL(void) PDMR3Suspend(PVM pVM)
-{
-    LogFlow(("PDMR3Suspend:\n"));
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * The outer loop repeats until there are no more async requests.
-     *
-     * Note! We depend on the suspended indicators to be in the desired state
-     *       and we do not reset them before starting because this allows
-     *       PDMR3PowerOn and PDMR3Resume to use PDMR3Suspend for cleaning up
-     *       on failure.
-     */
-    unsigned cAsync;
-    for (unsigned iLoop = 0; ; iLoop++)
-    {
-        /*
-         * Iterate thru the device instances and USB device instances,
-         * processing the drivers associated with those.
-         *
-         * The attached drivers are normally processed first.  Some devices
-         * (like DevAHCI) though needs to be notified before the drivers so
-         * that it doesn't kick off any new requests after the drivers stopped
-         * taking any. (DrvVD changes to read-only in this particular case.)
-         */
-        cAsync = 0;
-        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-        {
-            unsigned const cAsyncStart = cAsync;
-
-            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION)
-                pdmR3SuspendDev(pDevIns, &cAsync);
-
-            if (cAsync == cAsyncStart)
-                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                        if (!pdmR3SuspendDrv(pDrvIns, &cAsync, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
-                            break;
-
-            if (    cAsync == cAsyncStart
-                && !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION))
-                pdmR3SuspendDev(pDevIns, &cAsync);
-        }
-
-#ifdef VBOX_WITH_USB
-        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-        {
-            unsigned const cAsyncStart = cAsync;
-
-            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                    if (!pdmR3SuspendDrv(pDrvIns, &cAsync, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
-                        break;
-
-            if (cAsync == cAsyncStart)
-                pdmR3SuspendUsb(pUsbIns, &cAsync);
-        }
-#endif
-        if (!cAsync)
-            break;
-
-        /*
-         * Process requests.
-         */
-        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
-         *        bit I hope... */
-        int rc = VMR3AsyncPdmNotificationWaitU(&pVM->pUVM->aCpus[0]);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        rc = VMR3ReqProcessU(pVM->pUVM, 0/*idDstCpu*/);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-    }
-
-    /*
-     * Suspend all threads.
-     */
-    pdmR3ThreadSuspendAll(pVM);
-
-    LogFlow(("PDMR3Suspend: returns void\n"));
-}
-
-
-/**
- * Worker for PDMR3Resume that deals with one driver.
- *
- * @param   pDrvIns             The driver instance.
- * @param   pszDeviceName       The parent device name.
- * @param   iDevInstance        The parent device instance number.
- * @param   iLun                The parent LUN number.
- */
-DECLINLINE(int) pdmR3ResumeDrv(PPDMDRVINS pDrvIns, const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
-{
-    Assert(pDrvIns->Internal.s.fVMSuspended);
-    if (pDrvIns->pReg->pfnResume)
-    {
-        LogFlow(("PDMR3Resume: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                 pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-        int rc = VINF_SUCCESS; pDrvIns->pReg->pfnResume(pDrvIns);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDMR3Resume: driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
-                    pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance, rc));
-            return rc;
-        }
-    }
-    pDrvIns->Internal.s.fVMSuspended = false;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for PDMR3Resume that deals with one USB device instance.
- *
- * @returns VBox status code.
- * @param   pUsbIns             The USB device instance.
- */
-DECLINLINE(int) pdmR3ResumeUsb(PPDMUSBINS pUsbIns)
-{
-    Assert(pUsbIns->Internal.s.fVMSuspended);
-    if (pUsbIns->pReg->pfnVMResume)
-    {
-        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-        int rc = VINF_SUCCESS; pUsbIns->pReg->pfnVMResume(pUsbIns);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDMR3Resume: device '%s'/%d -> %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-            return rc;
-        }
-    }
-    pUsbIns->Internal.s.fVMSuspended = false;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for PDMR3Resume that deals with one device instance.
- *
- * @returns VBox status code.
- * @param   pDevIns             The device instance.
- */
-DECLINLINE(int) pdmR3ResumeDev(PPDMDEVINS pDevIns)
-{
-    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
-    if (pDevIns->pReg->pfnResume)
-    {
-        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-        int rc = VINF_SUCCESS; pDevIns->pReg->pfnResume(pDevIns);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDMR3Resume: device '%s'/%d -> %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-    }
-    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * This function will notify all the devices and their
- * attached drivers about the VM now being resumed.
- *
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(void) PDMR3Resume(PVM pVM)
-{
-    LogFlow(("PDMR3Resume:\n"));
-
-    /*
-     * Iterate thru the device instances and USB device instances,
-     * processing the drivers associated with those.
-     */
-    int rc = VINF_SUCCESS;
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun    = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
-                rc = pdmR3ResumeDrv(pDrvIns, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun);
-        if (RT_SUCCESS(rc))
-            rc = pdmR3ResumeDev(pDevIns);
-    }
-
-#ifdef VBOX_WITH_USB
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
-            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
-                rc = pdmR3ResumeDrv(pDrvIns, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun);
-        if (RT_SUCCESS(rc))
-            rc = pdmR3ResumeUsb(pUsbIns);
-    }
-#endif
-
-    /*
-     * Resume all threads.
-     */
-    if (RT_SUCCESS(rc))
-        pdmR3ThreadResumeAll(pVM);
-
-    /*
-     * Resume the block cache.
-     */
-    if (RT_SUCCESS(rc))
-        pdmR3BlkCacheResume(pVM);
-
-    /*
-     * On failure, clean up via PDMR3Suspend.
-     */
-    if (RT_FAILURE(rc))
-        PDMR3Suspend(pVM);
-
-    LogFlow(("PDMR3Resume: returns %Rrc\n", rc));
-    return /*rc*/;
-}
-
-
-/**
- * Worker for PDMR3PowerOff that deals with one driver.
- *
- * @param   pDrvIns             The driver instance.
- * @param   pcAsync             The asynchronous power off notification counter.
- * @param   pszDeviceName       The parent device name.
- * @param   iDevInstance        The parent device instance number.
- * @param   iLun                The parent LUN number.
- */
-DECLINLINE(bool) pdmR3PowerOffDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
-                                  const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
-{
-    if (!pDrvIns->Internal.s.fVMSuspended)
-    {
-        pDrvIns->Internal.s.fVMSuspended = true;
-        if (pDrvIns->pReg->pfnPowerOff)
-        {
-            if (!pDrvIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3PowerOff: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-                pDrvIns->pReg->pfnPowerOff(pDrvIns);
-                if (pDrvIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3PowerOff: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-            }
-            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
-            {
-                pDrvIns->Internal.s.pfnAsyncNotify = false;
-                LogFlow(("PDMR3PowerOff: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
-                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
-            }
-            if (pDrvIns->Internal.s.pfnAsyncNotify)
-            {
-                pDrvIns->Internal.s.fVMSuspended = false;
-                (*pcAsync)++;
-                return false;
-            }
-        }
-    }
-    return true;
-}
-
-
-/**
- * Worker for PDMR3PowerOff that deals with one USB device instance.
- *
- * @param   pUsbIns             The USB device instance.
- * @param   pcAsync             The asynchronous power off notification counter.
- */
-DECLINLINE(void) pdmR3PowerOffUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
-{
-    if (!pUsbIns->Internal.s.fVMSuspended)
-    {
-        pUsbIns->Internal.s.fVMSuspended = true;
-        if (pUsbIns->pReg->pfnVMPowerOff)
-        {
-            if (!pUsbIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-                pUsbIns->pReg->pfnVMPowerOff(pUsbIns);
-                if (pUsbIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3PowerOff: Async notification started - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-            }
-            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
-            {
-                LogFlow(("PDMR3PowerOff: Async notification completed - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
-            }
-            if (pUsbIns->Internal.s.pfnAsyncNotify)
-            {
-                pUsbIns->Internal.s.fVMSuspended = false;
-                (*pcAsync)++;
-            }
-        }
-    }
-}
-
-
-/**
- * Worker for PDMR3PowerOff that deals with one device instance.
- *
- * @param   pDevIns             The device instance.
- * @param   pcAsync             The asynchronous power off notification counter.
- */
-DECLINLINE(void) pdmR3PowerOffDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
-{
-    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
-    {
-        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
-        if (pDevIns->pReg->pfnPowerOff)
-        {
-            if (!pDevIns->Internal.s.pfnAsyncNotify)
-            {
-                LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-                pDevIns->pReg->pfnPowerOff(pDevIns);
-                if (pDevIns->Internal.s.pfnAsyncNotify)
-                    LogFlow(("PDMR3PowerOff: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-            }
-            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
-            {
-                LogFlow(("PDMR3PowerOff: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-                pDevIns->Internal.s.pfnAsyncNotify = NULL;
-            }
-            if (pDevIns->Internal.s.pfnAsyncNotify)
-            {
-                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
-                (*pcAsync)++;
-            }
-        }
-    }
-}
-
-
-/**
- * This function will notify all the devices and their
- * attached drivers about the VM being powered off.
- *
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(void) PDMR3PowerOff(PVM pVM)
-{
-    LogFlow(("PDMR3PowerOff:\n"));
-
-    /*
-     * The outer loop repeats until there are no more async requests.
-     */
-    unsigned cAsync;
-    for (unsigned iLoop = 0; ; iLoop++)
-    {
-        /*
-         * Iterate thru the device instances and USB device instances,
-         * processing the drivers associated with those.
-         *
-         * The attached drivers are normally processed first.  Some devices
-         * (like DevAHCI) though needs to be notified before the drivers so
-         * that it doesn't kick off any new requests after the drivers stopped
-         * taking any. (DrvVD changes to read-only in this particular case.)
-         */
-        cAsync = 0;
-        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-        {
-            unsigned const cAsyncStart = cAsync;
-
-            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION)
-                pdmR3PowerOffDev(pDevIns, &cAsync);
-
-            if (cAsync == cAsyncStart)
-                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                        if (!pdmR3PowerOffDrv(pDrvIns, &cAsync, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
-                            break;
-
-            if (    cAsync == cAsyncStart
-                && !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION))
-                pdmR3PowerOffDev(pDevIns, &cAsync);
-        }
-
-#ifdef VBOX_WITH_USB
-        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-        {
-            unsigned const cAsyncStart = cAsync;
-
-            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
-                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                    if (!pdmR3PowerOffDrv(pDrvIns, &cAsync, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
-                        break;
-
-            if (cAsync == cAsyncStart)
-                pdmR3PowerOffUsb(pUsbIns, &cAsync);
-        }
-#endif
-        if (!cAsync)
-            break;
-
-        /*
-         * Process requests.
-         */
-        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
-         *        bit I hope... */
-        int rc = VMR3AsyncPdmNotificationWaitU(&pVM->pUVM->aCpus[0]);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        rc = VMR3ReqProcessU(pVM->pUVM, 0/*idDstCpu*/);
-        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-    }
-
-    /*
-     * Suspend all threads.
-     */
-    pdmR3ThreadSuspendAll(pVM);
-
-    LogFlow(("PDMR3PowerOff: returns void\n"));
-}
-
-
-/**
- * Queries the base interface of a device instance.
- *
- * The caller can use this to query other interfaces the device implements
- * and use them to talk to the device.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   ppBase          Where to store the pointer to the base device interface on success.
- * @remark  We're not doing any locking ATM, so don't try call this at times when the
- *          device chain is known to be updated.
- */
-VMMR3DECL(int) PDMR3QueryDevice(PVM pVM, const char *pszDevice, unsigned iInstance, PPDMIBASE *ppBase)
-{
-    LogFlow(("PDMR3DeviceQuery: pszDevice=%p:{%s} iInstance=%u ppBase=%p\n", pszDevice, pszDevice, iInstance, ppBase));
-
-    /*
-     * Iterate registered devices looking for the device.
-     */
-    size_t cchDevice = strlen(pszDevice);
-    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
-    {
-        if (    pDev->cchName == cchDevice
-            &&  !memcmp(pDev->pReg->szName, pszDevice, cchDevice))
-        {
-            /*
-             * Iterate device instances.
-             */
-            for (PPDMDEVINS pDevIns = pDev->pInstances; pDevIns; pDevIns = pDevIns->Internal.s.pPerDeviceNextR3)
-            {
-                if (pDevIns->iInstance == iInstance)
-                {
-                    if (pDevIns->IBase.pfnQueryInterface)
-                    {
-                        *ppBase = &pDevIns->IBase;
-                        LogFlow(("PDMR3DeviceQuery: return VINF_SUCCESS and *ppBase=%p\n", *ppBase));
-                        return VINF_SUCCESS;
-                    }
-
-                    LogFlow(("PDMR3DeviceQuery: returns VERR_PDM_DEVICE_INSTANCE_NO_IBASE\n"));
-                    return VERR_PDM_DEVICE_INSTANCE_NO_IBASE;
-                }
-            }
-
-            LogFlow(("PDMR3DeviceQuery: returns VERR_PDM_DEVICE_INSTANCE_NOT_FOUND\n"));
-            return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND;
-        }
-    }
-
-    LogFlow(("PDMR3QueryDevice: returns VERR_PDM_DEVICE_NOT_FOUND\n"));
-    return VERR_PDM_DEVICE_NOT_FOUND;
-}
-
-
-/**
- * Queries the base interface of a device LUN.
- *
- * This differs from PDMR3QueryLun by that it returns the interface on the
- * device and not the top level driver.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   iLun            The Logical Unit to obtain the interface of.
- * @param   ppBase          Where to store the base interface pointer.
- * @remark  We're not doing any locking ATM, so don't try call this at times when the
- *          device chain is known to be updated.
- */
-VMMR3DECL(int) PDMR3QueryDeviceLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMIBASE *ppBase)
-{
-    LogFlow(("PDMR3QueryLun: pszDevice=%p:{%s} iInstance=%u iLun=%u ppBase=%p\n",
-             pszDevice, pszDevice, iInstance, iLun, ppBase));
-
-    /*
-     * Find the LUN.
-     */
-    PPDMLUN pLun;
-    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
-    if (RT_SUCCESS(rc))
-    {
-        *ppBase = pLun->pBase;
-        LogFlow(("PDMR3QueryDeviceLun: return VINF_SUCCESS and *ppBase=%p\n", *ppBase));
-        return VINF_SUCCESS;
-    }
-    LogFlow(("PDMR3QueryDeviceLun: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Query the interface of the top level driver on a LUN.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   iLun            The Logical Unit to obtain the interface of.
- * @param   ppBase          Where to store the base interface pointer.
- * @remark  We're not doing any locking ATM, so don't try call this at times when the
- *          device chain is known to be updated.
- */
-VMMR3DECL(int) PDMR3QueryLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMIBASE *ppBase)
-{
-    LogFlow(("PDMR3QueryLun: pszDevice=%p:{%s} iInstance=%u iLun=%u ppBase=%p\n",
-             pszDevice, pszDevice, iInstance, iLun, ppBase));
-
-    /*
-     * Find the LUN.
-     */
-    PPDMLUN pLun;
-    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLun->pTop)
-        {
-            *ppBase = &pLun->pTop->IBase;
-            LogFlow(("PDMR3QueryLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
-            return VINF_SUCCESS;
-        }
-        rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
-    }
-    LogFlow(("PDMR3QueryLun: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Query the interface of a named driver on a LUN.
- *
- * If the driver appears more than once in the driver chain, the first instance
- * is returned.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   iLun            The Logical Unit to obtain the interface of.
- * @param   pszDriver       The driver name.
- * @param   ppBase          Where to store the base interface pointer.
- *
- * @remark  We're not doing any locking ATM, so don't try call this at times when the
- *          device chain is known to be updated.
- */
-VMMR3DECL(int) PDMR3QueryDriverOnLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, const char *pszDriver, PPPDMIBASE ppBase)
-{
-    LogFlow(("PDMR3QueryDriverOnLun: pszDevice=%p:{%s} iInstance=%u iLun=%u pszDriver=%p:{%s} ppBase=%p\n",
-             pszDevice, pszDevice, iInstance, iLun, pszDriver, pszDriver, ppBase));
-
-    /*
-     * Find the LUN.
-     */
-    PPDMLUN pLun;
-    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
-    if (RT_SUCCESS(rc))
-    {
-        if (pLun->pTop)
-        {
-            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
-                if (!strcmp(pDrvIns->pReg->szName, pszDriver))
-                {
-                    *ppBase = &pDrvIns->IBase;
-                    LogFlow(("PDMR3QueryDriverOnLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
-                    return VINF_SUCCESS;
-
-                }
-            rc = VERR_PDM_DRIVER_NOT_FOUND;
-        }
-        else
-            rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
-    }
-    LogFlow(("PDMR3QueryDriverOnLun: returns %Rrc\n", rc));
-    return rc;
-}
-
-/**
- * Executes pending DMA transfers.
- * Forced Action handler.
- *
- * @param   pVM             VM handle.
- */
-VMMR3DECL(void) PDMR3DmaRun(PVM pVM)
-{
-    /* Note! Not really SMP safe; restrict it to VCPU 0. */
-    if (VMMGetCpuId(pVM) != 0)
-        return;
-
-    if (VM_FF_TESTANDCLEAR(pVM, VM_FF_PDM_DMA))
-    {
-        if (pVM->pdm.s.pDmac)
-        {
-            bool fMore = pVM->pdm.s.pDmac->Reg.pfnRun(pVM->pdm.s.pDmac->pDevIns);
-            if (fMore)
-                VM_FF_SET(pVM, VM_FF_PDM_DMA);
-        }
-    }
-}
-
-
-/**
- * Service a VMMCALLRING3_PDM_LOCK call.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-VMMR3DECL(int) PDMR3LockCall(PVM pVM)
-{
-    return PDMR3CritSectEnterEx(&pVM->pdm.s.CritSect, true /* fHostCall */);
-}
-
-
-/**
- * Registers the VMM device heap
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   GCPhys          The physical address.
- * @param   pvHeap          Ring-3 pointer.
- * @param   cbSize          Size of the heap.
- */
-VMMR3DECL(int) PDMR3RegisterVMMDevHeap(PVM pVM, RTGCPHYS GCPhys, RTR3PTR pvHeap, unsigned cbSize)
-{
-    Assert(pVM->pdm.s.pvVMMDevHeap == NULL);
-
-    Log(("PDMR3RegisterVMMDevHeap %RGp %RHv %x\n", GCPhys, pvHeap, cbSize));
-    pVM->pdm.s.pvVMMDevHeap     = pvHeap;
-    pVM->pdm.s.GCPhysVMMDevHeap = GCPhys;
-    pVM->pdm.s.cbVMMDevHeap     = cbSize;
-    pVM->pdm.s.cbVMMDevHeapLeft = cbSize;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Unregisters the VMM device heap
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   GCPhys          The physical address.
- */
-VMMR3DECL(int) PDMR3UnregisterVMMDevHeap(PVM pVM, RTGCPHYS GCPhys)
-{
-    Assert(pVM->pdm.s.GCPhysVMMDevHeap == GCPhys);
-
-    Log(("PDMR3UnregisterVMMDevHeap %RGp\n", GCPhys));
-    pVM->pdm.s.pvVMMDevHeap     = NULL;
-    pVM->pdm.s.GCPhysVMMDevHeap = NIL_RTGCPHYS;
-    pVM->pdm.s.cbVMMDevHeap     = 0;
-    pVM->pdm.s.cbVMMDevHeapLeft = 0;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Allocates memory from the VMM device heap
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   cbSize          Allocation size.
- * @param   pv              Ring-3 pointer. (out)
- */
-VMMR3DECL(int) PDMR3VMMDevHeapAlloc(PVM pVM, unsigned cbSize, RTR3PTR *ppv)
-{
-#ifdef DEBUG_bird
-    if (!cbSize || cbSize > pVM->pdm.s.cbVMMDevHeapLeft)
-        return VERR_NO_MEMORY;
-#else
-    AssertReturn(cbSize && cbSize <= pVM->pdm.s.cbVMMDevHeapLeft, VERR_NO_MEMORY);
-#endif
-
-    Log(("PDMR3VMMDevHeapAlloc %x\n", cbSize));
-
-    /** @todo not a real heap as there's currently only one user. */
-    *ppv = pVM->pdm.s.pvVMMDevHeap;
-    pVM->pdm.s.cbVMMDevHeapLeft = 0;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Frees memory from the VMM device heap
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pv              Ring-3 pointer.
- */
-VMMR3DECL(int) PDMR3VMMDevHeapFree(PVM pVM, RTR3PTR pv)
-{
-    Log(("PDMR3VMMDevHeapFree %RHv\n", pv));
-
-    /** @todo not a real heap as there's currently only one user. */
-    pVM->pdm.s.cbVMMDevHeapLeft = pVM->pdm.s.cbVMMDevHeap;
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/PDMAsyncCompletion.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMAsyncCompletion.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1609 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Async I/O - Transport data asynchronous in R3 using EMT.
- */
-
-/*
- * Copyright (C) 2006-2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/rem.h>
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-#include <iprt/mem.h>
-#include <iprt/critsect.h>
-#include <iprt/tcp.h>
-#include <iprt/path.h>
-#include <iprt/string.h>
-
-#include <VBox/pdmasynccompletion.h>
-#include "PDMAsyncCompletionInternal.h"
-
-/**
- * Async I/O type.
- */
-typedef enum PDMASYNCCOMPLETIONTEMPLATETYPE
-{
-    /** Device . */
-    PDMASYNCCOMPLETIONTEMPLATETYPE_DEV = 1,
-    /** Driver consumer. */
-    PDMASYNCCOMPLETIONTEMPLATETYPE_DRV,
-    /** Internal consumer. */
-    PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL,
-    /** Usb consumer. */
-    PDMASYNCCOMPLETIONTEMPLATETYPE_USB
-} PDMASYNCTEMPLATETYPE;
-
-/**
- * PDM Async I/O template.
- */
-typedef struct PDMASYNCCOMPLETIONTEMPLATE
-{
-    /** Pointer to the next template in the list. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE)    pNext;
-    /** Pointer to the previous template in the list. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE)    pPrev;
-    /** Type specific data. */
-    union
-    {
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DEV */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMASYNCCOMPLETEDEV)   pfnCompleted;
-            /** Pointer to the device instance owning the template. */
-            R3PTRTYPE(PPDMDEVINS)               pDevIns;
-        } Dev;
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DRV */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMASYNCCOMPLETEDRV)   pfnCompleted;
-            /** Pointer to the driver instance owning the template. */
-            R3PTRTYPE(PPDMDRVINS)               pDrvIns;
-            /** User argument given during template creation.
-             *  This is only here to make things much easier
-             *  for DrVVD. */
-            void                               *pvTemplateUser;
-        } Drv;
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMASYNCCOMPLETEINT)   pfnCompleted;
-            /** Pointer to user data. */
-            R3PTRTYPE(void *)                   pvUser;
-        } Int;
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_USB */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMASYNCCOMPLETEUSB)   pfnCompleted;
-            /** Pointer to the usb instance owning the template. */
-            R3PTRTYPE(PPDMUSBINS)               pUsbIns;
-        } Usb;
-    } u;
-    /** Template type. */
-    PDMASYNCCOMPLETIONTEMPLATETYPE          enmType;
-    /** Pointer to the VM. */
-    R3PTRTYPE(PVM)                          pVM;
-    /** Use count of the template. */
-    volatile uint32_t                       cUsed;
-} PDMASYNCCOMPLETIONTEMPLATE;
-
-/**
- * Bandwidth control manager instance data
- */
-typedef struct PDMACBWMGR
-{
-    /** Pointer to the next manager in the list. */
-    struct PDMACBWMGR                          *pNext;
-    /** Pointer to the shared UVM structure. */
-    PPDMASYNCCOMPLETIONEPCLASS                  pEpClass;
-    /** Identifier of the manager. */
-    char                                       *pszId;
-    /** Maximum number of bytes the endpoints are allowed to transfer (Max is 4GB/s currently) */
-    volatile uint32_t                           cbTransferPerSecMax;
-    /** Number of bytes we start with */
-    volatile uint32_t                           cbTransferPerSecStart;
-    /** Step after each update */
-    volatile uint32_t                           cbTransferPerSecStep;
-    /** Number of bytes we are allowed to transfer till the next update.
-     * Reset by the refresh timer. */
-    volatile uint32_t                           cbTransferAllowed;
-    /** Timestamp of the last update */
-    volatile uint64_t                           tsUpdatedLast;
-    /** Reference counter - How many endpoints are associated with this manager. */
-    volatile uint32_t                           cRefs;
-} PDMACBWMGR;
-/** Pointer to a bandwidth control manager pointer. */
-typedef PPDMACBWMGR *PPPDMACBWMGR;
-
-static void pdmR3AsyncCompletionPutTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, PPDMASYNCCOMPLETIONTASK pTask);
-
-/**
- * Internal worker for the creation apis
- *
- * @returns VBox status.
- * @param   pVM           VM handle.
- * @param   ppTemplate    Where to store the template handle.
- */
-static int pdmR3AsyncCompletionTemplateCreate(PVM pVM, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE enmType)
-{
-    PUVM pUVM = pVM->pUVM;
-
-    if (ppTemplate == NULL)
-    {
-        AssertMsgFailed(("ppTemplate is NULL\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
-    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION, sizeof(PDMASYNCCOMPLETIONTEMPLATE), (void **)&pTemplate);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Initialize fields.
-     */
-    pTemplate->pVM = pVM;
-    pTemplate->cUsed = 0;
-    pTemplate->enmType = enmType;
-
-    /*
-     * Add template to the global VM template list.
-     */
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    pTemplate->pNext = pUVM->pdm.s.pAsyncCompletionTemplates;
-    if (pUVM->pdm.s.pAsyncCompletionTemplates)
-        pUVM->pdm.s.pAsyncCompletionTemplates->pPrev = pTemplate;
-    pUVM->pdm.s.pAsyncCompletionTemplates = pTemplate;
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-
-    *ppTemplate = pTemplate;
-    return VINF_SUCCESS;
-}
-
-/**
- * Creates a async completion template for a device instance.
- *
- * The template is used when creating new completion tasks.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The device instance.
- * @param   ppTemplate      Where to store the template pointer on success.
- * @param   pfnCompleted    The completion callback routine.
- * @param   pszDesc         Description.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEDEV pfnCompleted, const char *pszDesc)
-{
-    LogFlow(("%s: pDevIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
-              __FUNCTION__, pDevIns, ppTemplate, pfnCompleted, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT(pVM);
-    if (!pfnCompleted)
-    {
-        AssertMsgFailed(("No completion callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (!ppTemplate)
-    {
-        AssertMsgFailed(("Template pointer is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the template.
-     */
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
-    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_DEV);
-    if (RT_SUCCESS(rc))
-    {
-        pTemplate->u.Dev.pDevIns = pDevIns;
-        pTemplate->u.Dev.pfnCompleted = pfnCompleted;
-
-        *ppTemplate = pTemplate;
-        Log(("PDM: Created device template %p: pfnCompleted=%p pDevIns=%p\n",
-             pTemplate, pfnCompleted, pDevIns));
-    }
-
-    return rc;
-}
-
-/**
- * Creates a async completion template for a driver instance.
- *
- * The template is used when creating new completion tasks.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDrvIns         The driver instance.
- * @param   ppTemplate      Where to store the template pointer on success.
- * @param   pfnCompleted    The completion callback routine.
- * @param   pvTemplateUser  Template user argument
- * @param   pszDesc         Description.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEDRV pfnCompleted, void *pvTemplateUser, const char *pszDesc)
-{
-    LogFlow(("%s: pDrvIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
-              __FUNCTION__, pDrvIns, ppTemplate, pfnCompleted, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    if (!pfnCompleted)
-    {
-        AssertMsgFailed(("No completion callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (!ppTemplate)
-    {
-        AssertMsgFailed(("Template pointer is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the template.
-     */
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
-    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_DRV);
-    if (RT_SUCCESS(rc))
-    {
-        pTemplate->u.Drv.pDrvIns        = pDrvIns;
-        pTemplate->u.Drv.pfnCompleted   = pfnCompleted;
-        pTemplate->u.Drv.pvTemplateUser = pvTemplateUser;
-
-        *ppTemplate = pTemplate;
-        Log(("PDM: Created driver template %p: pfnCompleted=%p pDrvIns=%p\n",
-             pTemplate, pfnCompleted, pDrvIns));
-    }
-
-    return rc;
-}
-
-/**
- * Creates a async completion template for a USB device instance.
- *
- * The template is used when creating new completion tasks.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pUsbIns         The USB device instance.
- * @param   ppTemplate      Where to store the template pointer on success.
- * @param   pfnCompleted    The completion callback routine.
- * @param   pszDesc         Description.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEUSB pfnCompleted, const char *pszDesc)
-{
-    LogFlow(("%s: pUsbIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
-              __FUNCTION__, pUsbIns, ppTemplate, pfnCompleted, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT(pVM);
-    if (!pfnCompleted)
-    {
-        AssertMsgFailed(("No completion callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (!ppTemplate)
-    {
-        AssertMsgFailed(("Template pointer is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the template.
-     */
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
-    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_USB);
-    if (RT_SUCCESS(rc))
-    {
-        pTemplate->u.Usb.pUsbIns = pUsbIns;
-        pTemplate->u.Usb.pfnCompleted = pfnCompleted;
-
-        *ppTemplate = pTemplate;
-        Log(("PDM: Created usb template %p: pfnCompleted=%p pDevIns=%p\n",
-             pTemplate, pfnCompleted, pUsbIns));
-    }
-
-    return rc;
-}
-
-/**
- * Creates a async completion template for internally by the VMM.
- *
- * The template is used when creating new completion tasks.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   ppTemplate      Where to store the template pointer on success.
- * @param   pfnCompleted    The completion callback routine.
- * @param   pvUser2         The 2nd user argument for the callback.
- * @param   pszDesc         Description.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateInternal(PVM pVM, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEINT pfnCompleted, void *pvUser2, const char *pszDesc)
-{
-    LogFlow(("%s: ppTemplate=%p pfnCompleted=%p pvUser2=%p pszDesc=%s\n",
-              __FUNCTION__, ppTemplate, pfnCompleted, pvUser2, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT(pVM);
-    if (!pfnCompleted)
-    {
-        AssertMsgFailed(("No completion callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (!ppTemplate)
-    {
-        AssertMsgFailed(("Template pointer is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the template.
-     */
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
-    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL);
-    if (RT_SUCCESS(rc))
-    {
-        pTemplate->u.Int.pvUser = pvUser2;
-        pTemplate->u.Int.pfnCompleted = pfnCompleted;
-
-        *ppTemplate = pTemplate;
-        Log(("PDM: Created internal template %p: pfnCompleted=%p pvUser2=%p\n",
-             pTemplate, pfnCompleted, pvUser2));
-    }
-
-    return rc;
-}
-
-/**
- * Destroys the specified async completion template.
- *
- * @returns VBox status codes:
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if the template is still in use.
- *
- * @param   pTemplate       The template in question.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroy(PPDMASYNCCOMPLETIONTEMPLATE pTemplate)
-{
-    LogFlow(("%s: pTemplate=%p\n", __FUNCTION__, pTemplate));
-
-    if (!pTemplate)
-    {
-        AssertMsgFailed(("pTemplate is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Check if the template is still used.
-     */
-    if (pTemplate->cUsed > 0)
-    {
-        AssertMsgFailed(("Template is still in use\n"));
-        return VERR_PDM_ASYNC_TEMPLATE_BUSY;
-    }
-
-    /*
-     * Unlink the template from the list.
-     */
-    PUVM pUVM = pTemplate->pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-
-    PPDMASYNCCOMPLETIONTEMPLATE pPrev = pTemplate->pPrev;
-    PPDMASYNCCOMPLETIONTEMPLATE pNext = pTemplate->pNext;
-
-    if (pPrev)
-        pPrev->pNext = pNext;
-    else
-        pUVM->pdm.s.pAsyncCompletionTemplates = pNext;
-
-    if (pNext)
-        pNext->pPrev = pPrev;
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-
-    /*
-     * Free the template.
-     */
-    MMR3HeapFree(pTemplate);
-
-    return VINF_SUCCESS;
-}
-
-/**
- * Destroys all the specified async completion templates for the given device instance.
- *
- * @returns VBox status codes:
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The device instance.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
-{
-    LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pDevIns)
-        return VERR_INVALID_PARAMETER;
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Unlink it.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
-    while (pTemplate)
-    {
-        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_DEV
-            &&  pTemplate->u.Dev.pDevIns == pDevIns)
-        {
-            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
-            pTemplate = pTemplate->pNext;
-            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
-            if (RT_FAILURE(rc))
-            {
-                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-                return rc;
-            }
-        }
-        else
-            pTemplate = pTemplate->pNext;
-    }
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return VINF_SUCCESS;
-}
-
-/**
- * Destroys all the specified async completion templates for the given driver instance.
- *
- * @returns VBox status codes:
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDrvIns         The driver instance.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
-{
-    LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDrvIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pDrvIns)
-        return VERR_INVALID_PARAMETER;
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Unlink it.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
-    while (pTemplate)
-    {
-        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_DRV
-            &&  pTemplate->u.Drv.pDrvIns == pDrvIns)
-        {
-            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
-            pTemplate = pTemplate->pNext;
-            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
-            if (RT_FAILURE(rc))
-            {
-                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-                return rc;
-            }
-        }
-        else
-            pTemplate = pTemplate->pNext;
-    }
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return VINF_SUCCESS;
-}
-
-/**
- * Destroys all the specified async completion templates for the given USB device instance.
- *
- * @returns VBox status codes:
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pUsbIns         The USB device instance.
- */
-VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns)
-{
-    LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pUsbIns)
-        return VERR_INVALID_PARAMETER;
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Unlink it.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
-    while (pTemplate)
-    {
-        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_USB
-            &&  pTemplate->u.Usb.pUsbIns == pUsbIns)
-        {
-            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
-            pTemplate = pTemplate->pNext;
-            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
-            if (RT_FAILURE(rc))
-            {
-                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-                return rc;
-            }
-        }
-        else
-            pTemplate = pTemplate->pNext;
-    }
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return VINF_SUCCESS;
-}
-
-
-static PPDMACBWMGR pdmacBwMgrFindById(PPDMASYNCCOMPLETIONEPCLASS pEpClass, const char *pcszId)
-{
-    PPDMACBWMGR pBwMgr = NULL;
-
-    if (VALID_PTR(pcszId))
-    {
-        int rc;
-        rc = RTCritSectEnter(&pEpClass->CritSect);
-        AssertRC(rc);
-
-        pBwMgr = pEpClass->pBwMgrsHead;
-        while (   pBwMgr
-               && RTStrCmp(pBwMgr->pszId, pcszId))
-            pBwMgr = pBwMgr->pNext;
-
-        rc = RTCritSectLeave(&pEpClass->CritSect);
-        AssertRC(rc);
-    }
-
-    return pBwMgr;
-}
-
-static void pdmacBwMgrLink(PPDMACBWMGR pBwMgr)
-{
-    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pBwMgr->pEpClass;
-    int rc;
-
-    rc = RTCritSectEnter(&pEpClass->CritSect);
-    AssertRC(rc);
-
-    pBwMgr->pNext = pEpClass->pBwMgrsHead;
-    pEpClass->pBwMgrsHead = pBwMgr;
-
-    rc = RTCritSectLeave(&pEpClass->CritSect);
-    AssertRC(rc);
-}
-
-static void pdmacBwMgrUnlink(PPDMACBWMGR pBwMgr)
-{
-    int rc;
-    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pBwMgr->pEpClass;
-
-    rc = RTCritSectEnter(&pEpClass->CritSect);
-    AssertRC(rc);
-
-    if (pBwMgr == pEpClass->pBwMgrsHead)
-        pEpClass->pBwMgrsHead = pBwMgr->pNext;
-    else
-    {
-        PPDMACBWMGR pPrev = pEpClass->pBwMgrsHead;
-        while (   pPrev
-               && pPrev->pNext != pBwMgr)
-            pPrev = pPrev->pNext;
-
-        AssertPtr(pPrev);
-        pPrev->pNext = pBwMgr->pNext;
-    }
-
-    rc = RTCritSectLeave(&pEpClass->CritSect);
-    AssertRC(rc);
-}
-
-static int pdmacAsyncCompletionBwMgrCreate(PPDMASYNCCOMPLETIONEPCLASS pEpClass, const char *pcszBwMgr, uint32_t cbTransferPerSecMax,
-                                           uint32_t cbTransferPerSecStart, uint32_t cbTransferPerSecStep)
-{
-    int rc = VINF_SUCCESS;
-    PPDMACBWMGR pBwMgr;
-
-    LogFlowFunc(("pEpClass=%#p pcszBwMgr=%#p{%s} cbTransferPerSecMax=%u cbTransferPerSecStart=%u cbTransferPerSecStep=%u\n",
-                 pEpClass, pcszBwMgr, cbTransferPerSecMax, cbTransferPerSecStart, cbTransferPerSecStep));
-
-    AssertPtrReturn(pEpClass, VERR_INVALID_POINTER);
-    AssertPtrReturn(pcszBwMgr, VERR_INVALID_POINTER);
-    AssertReturn(*pcszBwMgr != '\0', VERR_INVALID_PARAMETER);
-
-    pBwMgr = pdmacBwMgrFindById(pEpClass, pcszBwMgr);
-    if (!pBwMgr)
-    {
-        rc = MMR3HeapAllocZEx(pEpClass->pVM, MM_TAG_PDM_ASYNC_COMPLETION,
-                              sizeof(PDMACBWMGR),
-                              (void **)&pBwMgr);
-        if (RT_SUCCESS(rc))
-        {
-            pBwMgr->pszId = RTStrDup(pcszBwMgr);
-            if (pBwMgr->pszId)
-            {
-                pBwMgr->pEpClass              = pEpClass;
-                pBwMgr->cRefs                 = 0;
-
-                /* Init I/O flow control. */
-                pBwMgr->cbTransferPerSecMax   = cbTransferPerSecMax;
-                pBwMgr->cbTransferPerSecStart = cbTransferPerSecStart;
-                pBwMgr->cbTransferPerSecStep  = cbTransferPerSecStep;
-
-                pBwMgr->cbTransferAllowed     = pBwMgr->cbTransferPerSecStart;
-                pBwMgr->tsUpdatedLast         = RTTimeSystemNanoTS();
-
-                pdmacBwMgrLink(pBwMgr);
-                rc = VINF_SUCCESS;
-            }
-            else
-            {
-                rc = VERR_NO_MEMORY;
-                MMR3HeapFree(pBwMgr);
-            }
-        }
-    }
-    else
-        rc = VERR_ALREADY_EXISTS;
-
-    LogFlowFunc(("returns rc=%Rrc\n", rc));
-    return rc;
-}
-
-DECLINLINE(void) pdmacBwMgrRef(PPDMACBWMGR pBwMgr)
-{
-    ASMAtomicIncU32(&pBwMgr->cRefs);
-}
-
-DECLINLINE(void) pdmacBwMgrUnref(PPDMACBWMGR pBwMgr)
-{
-    Assert(pBwMgr->cRefs > 0);
-    ASMAtomicDecU32(&pBwMgr->cRefs);
-}
-
-bool pdmacEpIsTransferAllowed(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint32_t cbTransfer, RTMSINTERVAL *pmsWhenNext)
-{
-    bool fAllowed = true;
-    PPDMACBWMGR pBwMgr = ASMAtomicReadPtrT(&pEndpoint->pBwMgr, PPDMACBWMGR);
-
-    LogFlowFunc(("pEndpoint=%p pBwMgr=%p cbTransfer=%u\n", pEndpoint, pBwMgr, cbTransfer));
-
-    if (pBwMgr)
-    {
-        uint32_t cbOld = ASMAtomicSubU32(&pBwMgr->cbTransferAllowed, cbTransfer);
-        if (RT_LIKELY(cbOld >= cbTransfer))
-            fAllowed = true;
-        else
-        {
-            fAllowed = false;
-
-            /* We are out of resources  Check if we can update again. */
-            uint64_t tsNow          = RTTimeSystemNanoTS();
-            uint64_t tsUpdatedLast  = ASMAtomicUoReadU64(&pBwMgr->tsUpdatedLast);
-
-            if (tsNow - tsUpdatedLast >= (1000*1000*1000))
-            {
-                if (ASMAtomicCmpXchgU64(&pBwMgr->tsUpdatedLast, tsNow, tsUpdatedLast))
-                {
-                    if (pBwMgr->cbTransferPerSecStart < pBwMgr->cbTransferPerSecMax)
-                    {
-                       pBwMgr->cbTransferPerSecStart = RT_MIN(pBwMgr->cbTransferPerSecMax, pBwMgr->cbTransferPerSecStart + pBwMgr->cbTransferPerSecStep);
-                       LogFlow(("AIOMgr: Increasing maximum bandwidth to %u bytes/sec\n", pBwMgr->cbTransferPerSecStart));
-                    }
-
-                    /* Update */
-                    ASMAtomicWriteU32(&pBwMgr->cbTransferAllowed, pBwMgr->cbTransferPerSecStart - cbTransfer);
-                    fAllowed = true;
-                    LogFlow(("AIOMgr: Refreshed bandwidth\n"));
-                }
-            }
-            else
-            {
-                ASMAtomicAddU32(&pBwMgr->cbTransferAllowed, cbTransfer);
-                *pmsWhenNext = ((1000*1000*1000) - (tsNow - tsUpdatedLast)) / (1000*1000);
-            }
-        }
-    }
-
-    LogFlowFunc(("fAllowed=%RTbool\n", fAllowed));
-
-    return fAllowed;
-}
-
-void pdmR3AsyncCompletionCompleteTask(PPDMASYNCCOMPLETIONTASK pTask, int rc, bool fCallCompletionHandler)
-{
-    LogFlow(("%s: pTask=%#p fCallCompletionHandler=%RTbool\n", __FUNCTION__, pTask, fCallCompletionHandler));
-
-    if (fCallCompletionHandler)
-    {
-        PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pTask->pEndpoint->pTemplate;
-
-        switch (pTemplate->enmType)
-        {
-            case PDMASYNCCOMPLETIONTEMPLATETYPE_DEV:
-            {
-                pTemplate->u.Dev.pfnCompleted(pTemplate->u.Dev.pDevIns, pTask->pvUser, rc);
-                break;
-            }
-            case PDMASYNCCOMPLETIONTEMPLATETYPE_DRV:
-            {
-                pTemplate->u.Drv.pfnCompleted(pTemplate->u.Drv.pDrvIns, pTemplate->u.Drv.pvTemplateUser, pTask->pvUser, rc);
-                break;
-            }
-            case PDMASYNCCOMPLETIONTEMPLATETYPE_USB:
-            {
-                pTemplate->u.Usb.pfnCompleted(pTemplate->u.Usb.pUsbIns, pTask->pvUser, rc);
-                break;
-            }
-            case PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL:
-            {
-                pTemplate->u.Int.pfnCompleted(pTemplate->pVM, pTask->pvUser, pTemplate->u.Int.pvUser, rc);
-                break;
-            }
-            default:
-                AssertMsgFailed(("Unknown template type!\n"));
-        }
-    }
-
-    pdmR3AsyncCompletionPutTask(pTask->pEndpoint, pTask);
-}
-
-/**
- * Worker initializing a endpoint class.
- *
- * @returns VBox status code.
- * @param   pVM        Pointer to the shared VM instance data.
- * @param   pEpClass   Pointer to the endpoint class structure.
- * @param   pCfgHandle Pointer to the the CFGM tree.
- */
-int pdmR3AsyncCompletionEpClassInit(PVM pVM, PCPDMASYNCCOMPLETIONEPCLASSOPS pEpClassOps, PCFGMNODE pCfgHandle)
-{
-    int rc = VINF_SUCCESS;
-
-    /* Validate input. */
-    if (   !pEpClassOps
-        || (pEpClassOps->u32Version != PDMAC_EPCLASS_OPS_VERSION)
-        || (pEpClassOps->u32VersionEnd != PDMAC_EPCLASS_OPS_VERSION))
-        AssertMsgFailedReturn(("Invalid endpoint class data\n"), VERR_VERSION_MISMATCH);
-
-    LogFlowFunc((": pVM=%p pEpClassOps=%p{%s}\n", pVM, pEpClassOps, pEpClassOps->pcszName));
-
-    /* Allocate global class data. */
-    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = NULL;
-
-    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION,
-                          pEpClassOps->cbEndpointClassGlobal,
-                          (void **)&pEndpointClass);
-    if (RT_SUCCESS(rc))
-    {
-        /* Initialize common data. */
-        pEndpointClass->pVM          = pVM;
-        pEndpointClass->pEndpointOps = pEpClassOps;
-
-        rc = RTCritSectInit(&pEndpointClass->CritSect);
-        if (RT_SUCCESS(rc))
-        {
-            PCFGMNODE pCfgNodeClass = CFGMR3GetChild(pCfgHandle, pEpClassOps->pcszName);
-
-            /* Create task cache */
-            rc = RTMemCacheCreate(&pEndpointClass->hMemCacheTasks, pEpClassOps->cbTask,
-                                  0, UINT32_MAX, NULL, NULL, NULL, 0);
-            if (RT_SUCCESS(rc))
-            {
-                /* Call the specific endpoint class initializer. */
-                rc = pEpClassOps->pfnInitialize(pEndpointClass, pCfgNodeClass);
-                if (RT_SUCCESS(rc))
-                {
-                    /* Create all bandwidth groups for resource control. */
-                    PCFGMNODE pCfgBwGrp = CFGMR3GetChild(pCfgNodeClass, "BwGroups");
-
-                    if (pCfgBwGrp)
-                    {
-                        for (PCFGMNODE pCur = CFGMR3GetFirstChild(pCfgBwGrp); pCur; pCur = CFGMR3GetNextChild(pCur))
-                        {
-                            uint32_t cbMax, cbStart, cbStep;
-                            size_t cchName = CFGMR3GetNameLen(pCur) + 1;
-                            char *pszBwGrpId = (char *)RTMemAllocZ(cchName);
-
-                            if (!pszBwGrpId)
-                            {
-                                rc = VERR_NO_MEMORY;
-                                break;
-                            }
-
-                            rc = CFGMR3GetName(pCur, pszBwGrpId, cchName);
-                            AssertRC(rc);
-
-                            if (RT_SUCCESS(rc))
-                                rc = CFGMR3QueryU32(pCur, "Max", &cbMax);
-                            if (RT_SUCCESS(rc))
-                                rc = CFGMR3QueryU32Def(pCur, "Start", &cbStart, cbMax);
-                            if (RT_SUCCESS(rc))
-                                rc = CFGMR3QueryU32Def(pCur, "Step", &cbStep, 0);
-                            if (RT_SUCCESS(rc))
-                                rc = pdmacAsyncCompletionBwMgrCreate(pEndpointClass, pszBwGrpId, cbMax, cbStart, cbStep);
-
-                            RTMemFree(pszBwGrpId);
-
-                            if (RT_FAILURE(rc))
-                                break;
-                        }
-                    }
-
-                    if (RT_SUCCESS(rc))
-                    {
-                        PUVM pUVM = pVM->pUVM;
-                        AssertMsg(!pUVM->pdm.s.apAsyncCompletionEndpointClass[pEpClassOps->enmClassType],
-                                  ("Endpoint class was already initialized\n"));
-
-                        pUVM->pdm.s.apAsyncCompletionEndpointClass[pEpClassOps->enmClassType] = pEndpointClass;
-                        LogFlowFunc((": Initialized endpoint class \"%s\" rc=%Rrc\n", pEpClassOps->pcszName, rc));
-                        return VINF_SUCCESS;
-                    }
-                }
-                RTMemCacheDestroy(pEndpointClass->hMemCacheTasks);
-            }
-            RTCritSectDelete(&pEndpointClass->CritSect);
-        }
-        MMR3HeapFree(pEndpointClass);
-    }
-
-    LogFlowFunc((": Failed to initialize endpoint class rc=%Rrc\n", rc));
-
-    return rc;
-}
-
-/**
- * Worker terminating all endpoint classes.
- *
- * @returns nothing
- * @param   pEndpointClass    Pointer to the endpoint class to terminate.
- *
- * @remarks This method ensures that any still open endpoint is closed.
- */
-static void pdmR3AsyncCompletionEpClassTerminate(PPDMASYNCCOMPLETIONEPCLASS pEndpointClass)
-{
-    int rc = VINF_SUCCESS;
-    PVM pVM = pEndpointClass->pVM;
-
-    /* Close all still open endpoints. */
-    while (pEndpointClass->pEndpointsHead)
-        PDMR3AsyncCompletionEpClose(pEndpointClass->pEndpointsHead);
-
-    /* Destroy the bandwidth managers. */
-    PPDMACBWMGR pBwMgr = pEndpointClass->pBwMgrsHead;
-    while (pBwMgr)
-    {
-        PPDMACBWMGR pFree = pBwMgr;
-        pBwMgr = pBwMgr->pNext;
-        MMR3HeapFree(pFree);
-    }
-
-    /* Call the termination callback of the class. */
-    pEndpointClass->pEndpointOps->pfnTerminate(pEndpointClass);
-
-    RTMemCacheDestroy(pEndpointClass->hMemCacheTasks);
-    RTCritSectDelete(&pEndpointClass->CritSect);
-
-    /* Free the memory of the class finally and clear the entry in the class array. */
-    pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[pEndpointClass->pEndpointOps->enmClassType] = NULL;
-    MMR3HeapFree(pEndpointClass);
-}
-
-/**
- * Initialize the async completion manager.
- *
- * @returns VBox status code
- * @param   pVM Pointer to the shared VM structure.
- */
-int pdmR3AsyncCompletionInit(PVM pVM)
-{
-    int  rc   = VINF_SUCCESS;
-    PUVM pUVM = pVM->pUVM;
-
-    LogFlowFunc((": pVM=%p\n", pVM));
-
-    VM_ASSERT_EMT(pVM);
-
-    PCFGMNODE pCfgRoot            = CFGMR3GetRoot(pVM);
-    PCFGMNODE pCfgAsyncCompletion = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "AsyncCompletion");
-
-    rc = pdmR3AsyncCompletionEpClassInit(pVM, &g_PDMAsyncCompletionEndpointClassFile, pCfgAsyncCompletion);
-
-    LogFlowFunc((": pVM=%p rc=%Rrc\n", pVM, rc));
-
-    return rc;
-}
-
-/**
- * Terminates the async completion manager.
- *
- * @returns VBox status code
- * @param   pVM Pointer to the shared VM structure.
- */
-int pdmR3AsyncCompletionTerm(PVM pVM)
-{
-    LogFlowFunc((": pVM=%p\n", pVM));
-    PUVM pUVM = pVM->pUVM;
-
-    for (size_t i = 0; i < RT_ELEMENTS(pUVM->pdm.s.apAsyncCompletionEndpointClass); i++)
-        if (pUVM->pdm.s.apAsyncCompletionEndpointClass[i])
-            pdmR3AsyncCompletionEpClassTerminate(pUVM->pdm.s.apAsyncCompletionEndpointClass[i]);
-
-    return VINF_SUCCESS;
-}
-
-/**
- * Resume worker for the async completion manager.
- *
- * @returns nothing.
- * @param   pVM Pointer to the shared VM structure.
- */
-void pdmR3AsyncCompletionResume(PVM pVM)
-{
-    LogFlowFunc((": pVM=%p\n", pVM));
-    PUVM pUVM = pVM->pUVM;
-
-    /* Log the bandwidth groups and all assigned endpoints. */
-    for (size_t i = 0; i < RT_ELEMENTS(pUVM->pdm.s.apAsyncCompletionEndpointClass); i++)
-        if (pUVM->pdm.s.apAsyncCompletionEndpointClass[i])
-        {
-            PPDMASYNCCOMPLETIONEPCLASS pEpClass = pUVM->pdm.s.apAsyncCompletionEndpointClass[i];
-            PPDMASYNCCOMPLETIONENDPOINT pEp;
-            PPDMACBWMGR pBwMgr = pEpClass->pBwMgrsHead;
-
-            if (pBwMgr)
-                LogRel(("AIOMgr: Bandwidth groups for class '%s'\n", i == PDMASYNCCOMPLETIONEPCLASSTYPE_FILE
-                                                                     ? "File" : "<Unknown>"));
-
-            while (pBwMgr)
-            {
-                LogRel(("AIOMgr:     Id:    %s\n", pBwMgr->pszId));
-                LogRel(("AIOMgr:     Max:   %u B/s\n", pBwMgr->cbTransferPerSecMax));
-                LogRel(("AIOMgr:     Start: %u B/s\n", pBwMgr->cbTransferPerSecStart));
-                LogRel(("AIOMgr:     Step:  %u B/s\n", pBwMgr->cbTransferPerSecStep));
-                LogRel(("AIOMgr:     Endpoints:\n"));
-
-                pEp = pEpClass->pEndpointsHead;
-                while (pEp)
-                {
-                    if (pEp->pBwMgr == pBwMgr)
-                        LogRel(("AIOMgr:         %s\n", pEp->pszUri));
-
-                    pEp = pEp->pNext;
-                }
-
-                pBwMgr = pBwMgr->pNext;
-            }
-
-            /* Print all endpoints without assigned bandwidth groups. */
-            pEp = pEpClass->pEndpointsHead;
-            if (pEp)
-                LogRel(("AIOMgr: Endpoints without assigned bandwidth groups:\n"));
-
-            while (pEp)
-            {
-                if (!pEp->pBwMgr)
-                    LogRel(("AIOMgr:     %s\n", pEp->pszUri));
-
-                pEp = pEp->pNext;
-            }
-        }
-}
-
-/**
- * Tries to get a free task from the endpoint or class cache
- * allocating the task if it fails.
- *
- * @returns Pointer to a new and initialized task or NULL
- * @param   pEndpoint    The endpoint the task is for.
- * @param   pvUser       Opaque user data for the task.
- */
-static PPDMASYNCCOMPLETIONTASK pdmR3AsyncCompletionGetTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, void *pvUser)
-{
-    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
-    PPDMASYNCCOMPLETIONTASK pTask = NULL;
-
-    pTask = (PPDMASYNCCOMPLETIONTASK)RTMemCacheAlloc(pEndpointClass->hMemCacheTasks);
-
-    if (RT_LIKELY(pTask))
-    {
-        /* Initialize common parts. */
-        pTask->pvUser    = pvUser;
-        pTask->pEndpoint = pEndpoint;
-        /* Clear list pointers for safety. */
-        pTask->pPrev     = NULL;
-        pTask->pNext     = NULL;
-        pTask->tsNsStart = RTTimeNanoTS();
-#ifdef VBOX_WITH_STATISTICS
-        STAM_COUNTER_INC(&pEndpoint->StatIoOpsStarted);
-#endif
-    }
-
-    return pTask;
-}
-
-/**
- * Puts a task in one of the caches.
- *
- * @returns nothing.
- * @param   pEndpoint    The endpoint the task belongs to.
- * @param   pTask        The task to cache.
- */
-static void pdmR3AsyncCompletionPutTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, PPDMASYNCCOMPLETIONTASK pTask)
-{
-    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
-    uint64_t tsRun  = RTTimeNanoTS() - pTask->tsNsStart;
-
-    if (RT_UNLIKELY(tsRun >= (uint64_t)10*1000*1000*1000))
-    {
-        LogRel(("AsyncCompletion: Task %#p completed after %llu seconds\n", pTask, tsRun / ((uint64_t)1000*1000*1000)));
-    }
-
-#ifdef VBOX_WITH_STATISTICS
-    uint64_t iStatIdx;
-
-    if (tsRun < 1000)
-    {
-        /* Update nanoseconds statistics */
-        iStatIdx = tsRun / 100;
-        STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesNs[iStatIdx]);
-    }
-    else
-    {
-        tsRun /= 1000;
-
-        if (tsRun < 1000)
-        {
-            /* Update microsecnds statistics */
-            iStatIdx = tsRun / 100;
-            STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesMicroSec[iStatIdx]);
-        }
-        else
-        {
-            tsRun /= 1000;
-
-            if (tsRun < 1000)
-            {
-                /* Update milliseconds statistics */
-                iStatIdx = tsRun / 100;
-                STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesMs[iStatIdx]);
-            }
-            else
-            {
-                tsRun /= 1000;
-
-                if (tsRun < 1000)
-                {
-                    /* Update seconds statistics */
-                    iStatIdx = tsRun / 10;
-                    STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesSec[iStatIdx]);
-                }
-                else
-                    STAM_COUNTER_INC(&pEndpoint->StatTaskRunOver100Sec);
-            }
-        }
-    }
-
-    STAM_COUNTER_INC(&pEndpoint->StatIoOpsCompleted);
-    pEndpoint->cIoOpsCompleted++;
-    uint64_t tsMsCur = RTTimeMilliTS();
-    uint64_t tsInterval = tsMsCur - pEndpoint->tsIntervalStartMs;
-
-    if (tsInterval >= 1000)
-    {
-        pEndpoint->StatIoOpsPerSec.c = pEndpoint->cIoOpsCompleted / (tsInterval / 1000);
-        pEndpoint->tsIntervalStartMs = tsMsCur;
-        pEndpoint->cIoOpsCompleted = 0;
-    }
-#endif
-
-    RTMemCacheFree(pEndpointClass->hMemCacheTasks, pTask);
-}
-
-static PPDMASYNCCOMPLETIONENDPOINT pdmR3AsyncCompletionFindEndpointWithUri(PPDMASYNCCOMPLETIONEPCLASS pEndpointClass,
-                                                                           const char *pszUri)
-{
-    PPDMASYNCCOMPLETIONENDPOINT pEndpoint = pEndpointClass->pEndpointsHead;
-
-    while (pEndpoint)
-    {
-        if (!RTStrCmp(pEndpoint->pszUri, pszUri))
-            return pEndpoint;
-
-        pEndpoint = pEndpoint->pNext;
-    }
-
-    return NULL;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionEpCreateForFile(PPPDMASYNCCOMPLETIONENDPOINT ppEndpoint,
-                                                   const char *pszFilename, uint32_t fFlags,
-                                                   PPDMASYNCCOMPLETIONTEMPLATE pTemplate)
-{
-    int rc = VINF_SUCCESS;
-
-    LogFlowFunc((": ppEndpoint=%p pszFilename=%p{%s} fFlags=%u pTemplate=%p\n",
-                 ppEndpoint, pszFilename, pszFilename, fFlags, pTemplate));
-
-    /* Sanity checks. */
-    AssertReturn(VALID_PTR(ppEndpoint),  VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(pszFilename), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(pTemplate),   VERR_INVALID_POINTER);
-
-    /* Check that the flags are valid. */
-    AssertReturn(((~(PDMACEP_FILE_FLAGS_READ_ONLY | PDMACEP_FILE_FLAGS_DONT_LOCK) & fFlags) == 0),
-                 VERR_INVALID_PARAMETER);
-
-    PVM  pVM  = pTemplate->pVM;
-    PUVM pUVM = pVM->pUVM;
-    PPDMASYNCCOMPLETIONEPCLASS  pEndpointClass = pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
-    PPDMASYNCCOMPLETIONENDPOINT pEndpoint = NULL;
-
-    AssertMsg(pEndpointClass, ("File endpoint class was not initialized\n"));
-
-    /* Search for a already opened endpoint for this file. */
-    pEndpoint = pdmR3AsyncCompletionFindEndpointWithUri(pEndpointClass, pszFilename);
-
-    if(!pEndpoint)
-    {
-        rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION,
-                              pEndpointClass->pEndpointOps->cbEndpoint,
-                              (void **)&pEndpoint);
-        if (RT_SUCCESS(rc))
-        {
-
-            /* Initialize common parts. */
-            pEndpoint->pNext             = NULL;
-            pEndpoint->pPrev             = NULL;
-            pEndpoint->pEpClass          = pEndpointClass;
-            pEndpoint->pTemplate         = pTemplate;
-            pEndpoint->pszUri            = RTStrDup(pszFilename);
-            pEndpoint->cUsers            = 1;
-            pEndpoint->pBwMgr            = NULL;
-
-            if (   pEndpoint->pszUri
-                && RT_SUCCESS(rc))
-            {
-                /* Call the initializer for the endpoint. */
-                rc = pEndpointClass->pEndpointOps->pfnEpInitialize(pEndpoint, pszFilename, fFlags);
-                if (RT_SUCCESS(rc))
-                {
-                    /* Link it into the list of endpoints. */
-                    rc = RTCritSectEnter(&pEndpointClass->CritSect);
-                    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
-
-                    pEndpoint->pNext = pEndpointClass->pEndpointsHead;
-                    if (pEndpointClass->pEndpointsHead)
-                        pEndpointClass->pEndpointsHead->pPrev = pEndpoint;
-
-                    pEndpointClass->pEndpointsHead = pEndpoint;
-                    pEndpointClass->cEndpoints++;
-
-                    rc = RTCritSectLeave(&pEndpointClass->CritSect);
-                    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
-
-                    /* Reference the template. */
-                    ASMAtomicIncU32(&pTemplate->cUsed);
-
-#ifdef VBOX_WITH_STATISTICS
-                    /* Init the statistics part */
-                    for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesNs); i++)
-                    {
-                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesNs[i], STAMTYPE_COUNTER,
-                                             STAMVISIBILITY_USED,
-                                             STAMUNIT_OCCURENCES,
-                                             "Nanosecond resolution runtime statistics",
-                                             "/PDM/AsyncCompletion/File/%s/TaskRun1Ns-%u-%u",
-                                             RTPathFilename(pEndpoint->pszUri),
-                                             i*100, i*100+100-1);
-                        if (RT_FAILURE(rc))
-                            break;
-                    }
-
-                    if (RT_SUCCESS(rc))
-                    {
-                        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMicroSec); i++)
-                        {
-                            rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesMicroSec[i], STAMTYPE_COUNTER,
-                                                 STAMVISIBILITY_USED,
-                                                 STAMUNIT_OCCURENCES,
-                                                 "Microsecond resolution runtime statistics",
-                                                 "/PDM/AsyncCompletion/File/%s/TaskRun2MicroSec-%u-%u",
-                                                 RTPathFilename(pEndpoint->pszUri),
-                                                 i*100, i*100+100-1);
-                            if (RT_FAILURE(rc))
-                                break;
-                        }
-                    }
-
-                   if (RT_SUCCESS(rc))
-                   {
-                       for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
-                       {
-                           rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesMs[i], STAMTYPE_COUNTER,
-                                                STAMVISIBILITY_USED,
-                                                STAMUNIT_OCCURENCES,
-                                                "Milliseconds resolution runtime statistics",
-                                                "/PDM/AsyncCompletion/File/%s/TaskRun3Ms-%u-%u",
-                                                RTPathFilename(pEndpoint->pszUri),
-                                                i*100, i*100+100-1);
-                           if (RT_FAILURE(rc))
-                               break;
-                       }
-                   }
-
-                   if (RT_SUCCESS(rc))
-                   {
-                       for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
-                       {
-                            rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesSec[i], STAMTYPE_COUNTER,
-                                                 STAMVISIBILITY_USED,
-                                                 STAMUNIT_OCCURENCES,
-                                                 "Second resolution runtime statistics",
-                                                 "/PDM/AsyncCompletion/File/%s/TaskRun4Sec-%u-%u",
-                                                 RTPathFilename(pEndpoint->pszUri),
-                                                 i*10, i*10+10-1);
-                            if (RT_FAILURE(rc))
-                                break;
-                        }
-                    }
-
-                    if (RT_SUCCESS(rc))
-                    {
-                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunOver100Sec, STAMTYPE_COUNTER,
-                                             STAMVISIBILITY_USED,
-                                             STAMUNIT_OCCURENCES,
-                                             "Tasks which ran more than 100sec",
-                                             "/PDM/AsyncCompletion/File/%s/TaskRunSecGreater100Sec",
-                                             RTPathFilename(pEndpoint->pszUri));
-                    }
-
-                    if (RT_SUCCESS(rc))
-                    {
-                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsPerSec, STAMTYPE_COUNTER,
-                                             STAMVISIBILITY_ALWAYS,
-                                             STAMUNIT_OCCURENCES,
-                                             "Processed I/O operations per second",
-                                             "/PDM/AsyncCompletion/File/%s/IoOpsPerSec",
-                                             RTPathFilename(pEndpoint->pszUri));
-                    }
-
-                    if (RT_SUCCESS(rc))
-                    {
-                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsStarted, STAMTYPE_COUNTER,
-                                             STAMVISIBILITY_ALWAYS,
-                                             STAMUNIT_OCCURENCES,
-                                             "Started I/O operations for this endpoint",
-                                             "/PDM/AsyncCompletion/File/%s/IoOpsStarted",
-                                             RTPathFilename(pEndpoint->pszUri));
-                    }
-
-                    if (RT_SUCCESS(rc))
-                    {
-                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsCompleted, STAMTYPE_COUNTER,
-                                             STAMVISIBILITY_ALWAYS,
-                                             STAMUNIT_OCCURENCES,
-                                             "Completed I/O operations for this endpoint",
-                                             "/PDM/AsyncCompletion/File/%s/IoOpsCompleted",
-                                             RTPathFilename(pEndpoint->pszUri));
-                    }
-
-                    pEndpoint->tsIntervalStartMs = RTTimeMilliTS();
-#endif
-
-                    *ppEndpoint = pEndpoint;
-
-                    LogFlowFunc((": Created endpoint for %s: rc=%Rrc\n", pszFilename, rc));
-                    return VINF_SUCCESS;
-                }
-                RTStrFree(pEndpoint->pszUri);
-            }
-            MMR3HeapFree(pEndpoint);
-        }
-    }
-    else
-    {
-        /* Endpoint found. */
-        pEndpoint->cUsers++;
-
-        *ppEndpoint = pEndpoint;
-        return VINF_SUCCESS;
-    }
-
-    LogFlowFunc((": Creation of endpoint for %s failed: rc=%Rrc\n", pszFilename, rc));
-
-    return rc;
-}
-
-VMMR3DECL(void) PDMR3AsyncCompletionEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
-{
-    LogFlowFunc((": pEndpoint=%p\n", pEndpoint));
-
-    /* Sanity checks. */
-    AssertReturnVoid(VALID_PTR(pEndpoint));
-
-    pEndpoint->cUsers--;
-
-    /* If the last user closed the endpoint we will free it. */
-    if (!pEndpoint->cUsers)
-    {
-        PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
-        pEndpointClass->pEndpointOps->pfnEpClose(pEndpoint);
-
-        /* Drop reference from the template. */
-        ASMAtomicDecU32(&pEndpoint->pTemplate->cUsed);
-
-        /* Unlink the endpoint from the list. */
-        int rc = RTCritSectEnter(&pEndpointClass->CritSect);
-        AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
-
-        PPDMASYNCCOMPLETIONENDPOINT pEndpointNext = pEndpoint->pNext;
-        PPDMASYNCCOMPLETIONENDPOINT pEndpointPrev = pEndpoint->pPrev;
-
-        if (pEndpointPrev)
-            pEndpointPrev->pNext = pEndpointNext;
-        else
-            pEndpointClass->pEndpointsHead = pEndpointNext;
-        if (pEndpointNext)
-            pEndpointNext->pPrev = pEndpointPrev;
-
-        pEndpointClass->cEndpoints--;
-
-        rc = RTCritSectLeave(&pEndpointClass->CritSect);
-        AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
-
-#ifdef VBOX_WITH_STATISTICS
-        /* Deregister the statistics part */
-        PVM pVM = pEndpointClass->pVM;
-
-        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesNs); i++)
-            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesNs[i]);
-        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMicroSec); i++)
-            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesMicroSec[i]);
-        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
-            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesMs[i]);
-        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
-            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesSec[i]);
-
-        STAMR3Deregister(pVM, &pEndpoint->StatTaskRunOver100Sec);
-        STAMR3Deregister(pVM, &pEndpoint->StatIoOpsPerSec);
-        STAMR3Deregister(pVM, &pEndpoint->StatIoOpsStarted);
-        STAMR3Deregister(pVM, &pEndpoint->StatIoOpsCompleted);
-#endif
-
-        RTStrFree(pEndpoint->pszUri);
-        MMR3HeapFree(pEndpoint);
-    }
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionEpRead(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
-                                          PCRTSGSEG paSegments, unsigned cSegments,
-                                          size_t cbRead, void *pvUser,
-                                          PPPDMASYNCCOMPLETIONTASK ppTask)
-{
-    int rc = VINF_SUCCESS;
-
-    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(paSegments), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(ppTask), VERR_INVALID_POINTER);
-    AssertReturn(cSegments > 0, VERR_INVALID_PARAMETER);
-    AssertReturn(cbRead > 0, VERR_INVALID_PARAMETER);
-    AssertReturn(off >= 0, VERR_INVALID_PARAMETER);
-
-    PPDMASYNCCOMPLETIONTASK pTask;
-
-    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
-    if (!pTask)
-        return VERR_NO_MEMORY;
-
-    rc = pEndpoint->pEpClass->pEndpointOps->pfnEpRead(pTask, pEndpoint, off,
-                                                      paSegments, cSegments, cbRead);
-    if (RT_SUCCESS(rc))
-    {
-        *ppTask = pTask;
-    }
-    else
-        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
-
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionEpWrite(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
-                                           PCRTSGSEG paSegments, unsigned cSegments,
-                                           size_t cbWrite, void *pvUser,
-                                           PPPDMASYNCCOMPLETIONTASK ppTask)
-{
-    int rc = VINF_SUCCESS;
-
-    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(paSegments), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(ppTask), VERR_INVALID_POINTER);
-    AssertReturn(cSegments > 0, VERR_INVALID_PARAMETER);
-    AssertReturn(cbWrite > 0, VERR_INVALID_PARAMETER);
-    AssertReturn(off >= 0, VERR_INVALID_PARAMETER);
-
-    PPDMASYNCCOMPLETIONTASK pTask;
-
-    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
-    if (!pTask)
-        return VERR_NO_MEMORY;
-
-    rc = pEndpoint->pEpClass->pEndpointOps->pfnEpWrite(pTask, pEndpoint, off,
-                                                       paSegments, cSegments, cbWrite);
-    if (RT_SUCCESS(rc))
-    {
-        *ppTask = pTask;
-    }
-    else
-        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
-
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionEpFlush(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                           void *pvUser,
-                                           PPPDMASYNCCOMPLETIONTASK ppTask)
-{
-    int rc = VINF_SUCCESS;
-
-    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(ppTask), VERR_INVALID_POINTER);
-
-    PPDMASYNCCOMPLETIONTASK pTask;
-
-    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
-    if (!pTask)
-        return VERR_NO_MEMORY;
-
-    rc = pEndpoint->pEpClass->pEndpointOps->pfnEpFlush(pTask, pEndpoint);
-    if (RT_SUCCESS(rc))
-    {
-        *ppTask = pTask;
-    }
-    else
-        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
-
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionEpGetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                             uint64_t *pcbSize)
-{
-    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
-    AssertReturn(VALID_PTR(pcbSize), VERR_INVALID_POINTER);
-
-    if (pEndpoint->pEpClass->pEndpointOps->pfnEpGetSize)
-        return pEndpoint->pEpClass->pEndpointOps->pfnEpGetSize(pEndpoint, pcbSize);
-    else
-        return VERR_NOT_SUPPORTED;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionEpSetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                             uint64_t cbSize)
-{
-    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
-
-    if (pEndpoint->pEpClass->pEndpointOps->pfnEpSetSize)
-        return pEndpoint->pEpClass->pEndpointOps->pfnEpSetSize(pEndpoint, cbSize);
-    else
-        return VERR_NOT_SUPPORTED;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionEpSetBwMgr(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                              const char *pcszBwMgr)
-{
-    int rc = VINF_SUCCESS;
-    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
-    PPDMACBWMGR pBwMgrOld = NULL;
-    PPDMACBWMGR pBwMgrNew = NULL;
-
-    if (pcszBwMgr)
-    {
-        pBwMgrNew = pdmacBwMgrFindById(pEndpoint->pEpClass, pcszBwMgr);
-
-        if (pBwMgrNew)
-            pdmacBwMgrRef(pBwMgrNew);
-        else
-            rc = VERR_NOT_FOUND;
-    }
-
-    if (RT_SUCCESS(rc))
-    {
-        pBwMgrOld = ASMAtomicXchgPtrT(&pEndpoint->pBwMgr, pBwMgrNew, PPDMACBWMGR);
-
-        if (pBwMgrOld)
-            pdmacBwMgrUnref(pBwMgrOld);
-    }
-
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionTaskCancel(PPDMASYNCCOMPLETIONTASK pTask)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
-VMMR3DECL(int) PDMR3AsyncCompletionBwMgrSetMaxForFile(PVM pVM, const char *pcszBwMgr, uint32_t cbMaxNew)
-{
-    int rc = VINF_SUCCESS;
-    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
-    AssertPtrReturn(pcszBwMgr, VERR_INVALID_POINTER);
-    PPDMACBWMGR pBwMgr;
-
-    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
-
-    pBwMgr = pdmacBwMgrFindById(pEpClass, pcszBwMgr);
-    if (pBwMgr)
-    {
-        /*
-         * Set the new value for the start and max value to let the manager pick up
-         * the new limit immediately.
-         */
-        ASMAtomicXchgU32(&pBwMgr->cbTransferPerSecMax, cbMaxNew);
-        ASMAtomicXchgU32(&pBwMgr->cbTransferPerSecStart, cbMaxNew);
-    }
-    else
-        rc = VERR_NOT_FOUND;
-
-    return rc;
-}
-
Index: unk/src/VBox/VMM/PDMAsyncCompletionFile.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMAsyncCompletionFile.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1225 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Async I/O - Transport data asynchronous in R3 using EMT.
- */
-
-/*
- * Copyright (C) 2006-2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <VBox/dbg.h>
-#include <VBox/uvm.h>
-
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/critsect.h>
-#include <iprt/env.h>
-#include <iprt/file.h>
-#include <iprt/mem.h>
-#include <iprt/semaphore.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-#include <iprt/path.h>
-
-#if defined(RT_OS_DARWIN) || defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD)
-# include <errno.h>
-# include <sys/ioctl.h>
-# include <sys/types.h>
-# include <sys/stat.h>
-# include <fcntl.h>
-# include <unistd.h>
-#endif
-
-#ifdef RT_OS_WINDOWS
-# define _WIN32_WINNT 0x0500
-# include <windows.h>
-# include <winioctl.h>
-#endif
-#ifdef RT_OS_DARWIN
-# include <sys/disk.h>
-#endif /* RT_OS_DARWIN */
-#ifdef RT_OS_SOLARIS
-# include <stropts.h>
-# include <sys/dkio.h>
-# include <sys/vtoc.h>
-#endif /* RT_OS_SOLARIS */
-#ifdef RT_OS_FREEBSD
-# include <sys/disk.h>
-#endif /* RT_OS_FREEBSD */
-
-#include "PDMAsyncCompletionFileInternal.h"
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-#ifdef VBOX_WITH_DEBUGGER
-static DECLCALLBACK(int) pdmacEpFileErrorInject(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR pArgs, unsigned cArgs, PDBGCVAR pResult);
-#endif
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-#ifdef VBOX_WITH_DEBUGGER
-static const DBGCVARDESC g_aInjectErrorArgs[] =
-{
-    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
-    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "direction",    "write/read." },
-    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "filename",     "Filename." },
-    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "errcode",      "IPRT error code." },
-};
-
-/** Command descriptors. */
-static const DBGCCMD g_aCmds[] =
-{
-    /* pszCmd,       cArgsMin, cArgsMax, paArgDesc,                cArgDescs,    pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
-    { "injecterror",        3, 3,        &g_aInjectErrorArgs[0],           3,           NULL,             0,     pdmacEpFileErrorInject,        "",   "Inject error into I/O subsystem." },
-};
-#endif
-
-/**
- * Frees a task.
- *
- * @returns nothing.
- * @param   pEndpoint    Pointer to the endpoint the segment was for.
- * @param   pTask        The task to free.
- */
-void pdmacFileTaskFree(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                       PPDMACTASKFILE pTask)
-{
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
-
-    LogFlowFunc((": pEndpoint=%p pTask=%p\n", pEndpoint, pTask));
-
-    /* Try the per endpoint cache first. */
-    if (pEndpoint->cTasksCached < pEpClass->cTasksCacheMax)
-    {
-        /* Add it to the list. */
-        pEndpoint->pTasksFreeTail->pNext = pTask;
-        pEndpoint->pTasksFreeTail        = pTask;
-        ASMAtomicIncU32(&pEndpoint->cTasksCached);
-    }
-    else
-    {
-        Log(("Freeing task %p because all caches are full\n", pTask));
-        MMR3HeapFree(pTask);
-    }
-}
-
-/**
- * Allocates a task segment
- *
- * @returns Pointer to the new task segment or NULL
- * @param   pEndpoint    Pointer to the endpoint
- */
-PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    PPDMACTASKFILE pTask = NULL;
-
-    /* Try the small per endpoint cache first. */
-    if (pEndpoint->pTasksFreeHead == pEndpoint->pTasksFreeTail)
-    {
-        /* Try the bigger endpoint class cache. */
-        PPDMASYNCCOMPLETIONEPCLASSFILE pEndpointClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
-
-        /*
-         * Allocate completely new.
-         * If this fails we return NULL.
-         */
-        int rc = MMR3HeapAllocZEx(pEndpointClass->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION,
-                                  sizeof(PDMACTASKFILE),
-                                  (void **)&pTask);
-        if (RT_FAILURE(rc))
-            pTask = NULL;
-
-        LogFlow(("Allocated task %p\n", pTask));
-    }
-    else
-    {
-        /* Grab a free task from the head. */
-        AssertMsg(pEndpoint->cTasksCached > 0, ("No tasks cached but list contains more than one element\n"));
-
-        pTask = pEndpoint->pTasksFreeHead;
-        pEndpoint->pTasksFreeHead = pTask->pNext;
-        ASMAtomicDecU32(&pEndpoint->cTasksCached);
-    }
-
-    pTask->pNext = NULL;
-
-    return pTask;
-}
-
-PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    PPDMACTASKFILE pTasks = NULL;
-
-    /*
-     * Get pending tasks.
-     */
-    pTasks = ASMAtomicXchgPtrT(&pEndpoint->pTasksNewHead, NULL, PPDMACTASKFILE);
-
-    /* Reverse the list to process in FIFO order. */
-    if (pTasks)
-    {
-        PPDMACTASKFILE pTask = pTasks;
-
-        pTasks = NULL;
-
-        while (pTask)
-        {
-            PPDMACTASKFILE pCur = pTask;
-            pTask = pTask->pNext;
-            pCur->pNext = pTasks;
-            pTasks = pCur;
-        }
-    }
-
-    return pTasks;
-}
-
-static void pdmacFileAioMgrWakeup(PPDMACEPFILEMGR pAioMgr)
-{
-    bool fWokenUp = ASMAtomicXchgBool(&pAioMgr->fWokenUp, true);
-
-    if (!fWokenUp)
-    {
-        int rc = VINF_SUCCESS;
-        bool fWaitingEventSem = ASMAtomicReadBool(&pAioMgr->fWaitingEventSem);
-
-        if (fWaitingEventSem)
-            rc = RTSemEventSignal(pAioMgr->EventSem);
-
-        AssertRC(rc);
-    }
-}
-
-static int pdmacFileAioMgrWaitForBlockingEvent(PPDMACEPFILEMGR pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT enmEvent)
-{
-    int rc = VINF_SUCCESS;
-
-    ASMAtomicWriteU32((volatile uint32_t *)&pAioMgr->enmBlockingEvent, enmEvent);
-    Assert(!pAioMgr->fBlockingEventPending);
-    ASMAtomicXchgBool(&pAioMgr->fBlockingEventPending, true);
-
-    /* Wakeup the async I/O manager */
-    pdmacFileAioMgrWakeup(pAioMgr);
-
-    /* Wait for completion. */
-    rc = RTSemEventWait(pAioMgr->EventSemBlock, RT_INDEFINITE_WAIT);
-    AssertRC(rc);
-
-    ASMAtomicXchgBool(&pAioMgr->fBlockingEventPending, false);
-    ASMAtomicWriteU32((volatile uint32_t *)&pAioMgr->enmBlockingEvent, PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID);
-
-    return rc;
-}
-
-int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    int rc;
-
-    LogFlowFunc(("pAioMgr=%#p pEndpoint=%#p{%s}\n", pAioMgr, pEndpoint, pEndpoint->Core.pszUri));
-
-    /* Update the assigned I/O manager. */
-    ASMAtomicWritePtr(&pEndpoint->pAioMgr, pAioMgr);
-
-    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
-    AssertRCReturn(rc, rc);
-
-    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint, pEndpoint);
-    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT);
-    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint);
-
-    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
-
-    return rc;
-}
-
-static int pdmacFileAioMgrRemoveEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    int rc;
-
-    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
-    AssertRCReturn(rc, rc);
-
-    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint, pEndpoint);
-    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT);
-    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint);
-
-    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
-
-    return rc;
-}
-
-static int pdmacFileAioMgrCloseEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    int rc;
-
-    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
-    AssertRCReturn(rc, rc);
-
-    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint, pEndpoint);
-    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT);
-    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint);
-
-    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
-
-    return rc;
-}
-
-static int pdmacFileAioMgrShutdown(PPDMACEPFILEMGR pAioMgr)
-{
-    int rc;
-
-    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
-    AssertRCReturn(rc, rc);
-
-    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN);
-
-    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
-
-    return rc;
-}
-
-int pdmacFileEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
-{
-    PPDMACTASKFILE pNext;
-    do
-    {
-        pNext = pEndpoint->pTasksNewHead;
-        pTask->pNext = pNext;
-    } while (!ASMAtomicCmpXchgPtr(&pEndpoint->pTasksNewHead, pTask, pNext));
-
-    pdmacFileAioMgrWakeup(ASMAtomicReadPtrT(&pEndpoint->pAioMgr, PPDMACEPFILEMGR));
-
-    return VINF_SUCCESS;
-}
-
-void pdmacFileEpTaskCompleted(PPDMACTASKFILE pTask, void *pvUser, int rc)
-{
-    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pvUser;
-
-    LogFlowFunc(("pTask=%#p pvUser=%#p rc=%Rrc\n", pTask, pvUser, rc));
-
-    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
-    {
-        pdmR3AsyncCompletionCompleteTask(&pTaskFile->Core, rc, true);
-    }
-    else
-    {
-        Assert((uint32_t)pTask->DataSeg.cbSeg == pTask->DataSeg.cbSeg && (int32_t)pTask->DataSeg.cbSeg >= 0);
-        uint32_t uOld = ASMAtomicSubS32(&pTaskFile->cbTransferLeft, (int32_t)pTask->DataSeg.cbSeg);
-
-        /* The first error will be returned. */
-        if (RT_FAILURE(rc))
-            ASMAtomicCmpXchgS32(&pTaskFile->rc, rc, VINF_SUCCESS);
-#ifdef VBOX_WITH_DEBUGGER
-        else
-        {
-            PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pTaskFile->Core.pEndpoint;
-
-            /* Overwrite with injected error code. */
-            if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
-                rc = ASMAtomicXchgS32(&pEpFile->rcReqRead, VINF_SUCCESS);
-            else
-                rc = ASMAtomicXchgS32(&pEpFile->rcReqWrite, VINF_SUCCESS);
-
-            if (RT_FAILURE(rc))
-                ASMAtomicCmpXchgS32(&pTaskFile->rc, rc, VINF_SUCCESS);
-        }
-#endif
-
-        if (!(uOld - pTask->DataSeg.cbSeg)
-            && !ASMAtomicXchgBool(&pTaskFile->fCompleted, true))
-            pdmR3AsyncCompletionCompleteTask(&pTaskFile->Core, pTaskFile->rc, true);
-    }
-}
-
-DECLINLINE(void) pdmacFileEpTaskInit(PPDMASYNCCOMPLETIONTASK pTask, size_t cbTransfer)
-{
-    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
-
-    Assert((uint32_t)cbTransfer == cbTransfer && (int32_t)cbTransfer >= 0);
-    ASMAtomicWriteS32(&pTaskFile->cbTransferLeft, (int32_t)cbTransfer);
-    ASMAtomicWriteBool(&pTaskFile->fCompleted, false);
-    ASMAtomicWriteS32(&pTaskFile->rc, VINF_SUCCESS);
-}
-
-int pdmacFileEpTaskInitiate(PPDMASYNCCOMPLETIONTASK pTask,
-                            PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
-                            PCRTSGSEG paSegments, size_t cSegments,
-                            size_t cbTransfer, PDMACTASKFILETRANSFER enmTransfer)
-{
-    int rc = VINF_SUCCESS;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
-    PPDMACEPFILEMGR pAioMgr = pEpFile->pAioMgr;
-
-    Assert(   (enmTransfer == PDMACTASKFILETRANSFER_READ)
-           || (enmTransfer == PDMACTASKFILETRANSFER_WRITE));
-
-    for (unsigned i = 0; i < cSegments; i++)
-    {
-        PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEpFile);
-        AssertPtr(pIoTask);
-
-        pIoTask->pEndpoint       = pEpFile;
-        pIoTask->enmTransferType = enmTransfer;
-        pIoTask->Off             = off;
-        pIoTask->DataSeg.cbSeg   = paSegments[i].cbSeg;
-        pIoTask->DataSeg.pvSeg   = paSegments[i].pvSeg;
-        pIoTask->pvUser          = pTaskFile;
-        pIoTask->pfnCompleted    = pdmacFileEpTaskCompleted;
-
-        /* Send it off to the I/O manager. */
-        pdmacFileEpAddTask(pEpFile, pIoTask);
-        off        += paSegments[i].cbSeg;
-        cbTransfer -= paSegments[i].cbSeg;
-    }
-
-    AssertMsg(!cbTransfer, ("Incomplete transfer %u bytes left\n", cbTransfer));
-
-    if (ASMAtomicReadS32(&pTaskFile->cbTransferLeft) == 0
-        && !ASMAtomicXchgBool(&pTaskFile->fCompleted, true))
-        pdmR3AsyncCompletionCompleteTask(pTask, pTaskFile->rc, false);
-    else
-        rc = VINF_AIO_TASK_PENDING;
-
-    return rc;
-}
-
-/**
- * Creates a new async I/O manager.
- *
- * @returns VBox status code.
- * @param   pEpClass    Pointer to the endpoint class data.
- * @param   ppAioMgr    Where to store the pointer to the new async I/O manager on success.
- * @param   enmMgrType  Wanted manager type - can be overwritten by the global override.
- */
-int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr,
-                          PDMACEPFILEMGRTYPE enmMgrType)
-{
-    int rc = VINF_SUCCESS;
-    PPDMACEPFILEMGR pAioMgrNew;
-
-    LogFlowFunc((": Entered\n"));
-
-    rc = MMR3HeapAllocZEx(pEpClass->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION, sizeof(PDMACEPFILEMGR), (void **)&pAioMgrNew);
-    if (RT_SUCCESS(rc))
-    {
-        if (enmMgrType < pEpClass->enmMgrTypeOverride)
-            pAioMgrNew->enmMgrType = enmMgrType;
-        else
-            pAioMgrNew->enmMgrType = pEpClass->enmMgrTypeOverride;
-
-        rc = RTSemEventCreate(&pAioMgrNew->EventSem);
-        if (RT_SUCCESS(rc))
-        {
-            rc = RTSemEventCreate(&pAioMgrNew->EventSemBlock);
-            if (RT_SUCCESS(rc))
-            {
-                rc = RTCritSectInit(&pAioMgrNew->CritSectBlockingEvent);
-                if (RT_SUCCESS(rc))
-                {
-                    /* Init the rest of the manager. */
-                    if (pAioMgrNew->enmMgrType != PDMACEPFILEMGRTYPE_SIMPLE)
-                        rc = pdmacFileAioMgrNormalInit(pAioMgrNew);
-
-                    if (RT_SUCCESS(rc))
-                    {
-                        pAioMgrNew->enmState = PDMACEPFILEMGRSTATE_RUNNING;
-
-                        rc = RTThreadCreateF(&pAioMgrNew->Thread,
-                                             pAioMgrNew->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE
-                                             ? pdmacFileAioMgrFailsafe
-                                             : pdmacFileAioMgrNormal,
-                                             pAioMgrNew,
-                                             0,
-                                             RTTHREADTYPE_IO,
-                                             0,
-                                             "AioMgr%d-%s", pEpClass->cAioMgrs,
-                                             pAioMgrNew->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE
-                                             ? "F"
-                                             : "N");
-                        if (RT_SUCCESS(rc))
-                        {
-                            /* Link it into the list. */
-                            RTCritSectEnter(&pEpClass->CritSect);
-                            pAioMgrNew->pNext = pEpClass->pAioMgrHead;
-                            if (pEpClass->pAioMgrHead)
-                                pEpClass->pAioMgrHead->pPrev = pAioMgrNew;
-                            pEpClass->pAioMgrHead = pAioMgrNew;
-                            pEpClass->cAioMgrs++;
-                            RTCritSectLeave(&pEpClass->CritSect);
-
-                            *ppAioMgr = pAioMgrNew;
-
-                            Log(("PDMAC: Successfully created new file AIO Mgr {%s}\n", RTThreadGetName(pAioMgrNew->Thread)));
-                            return VINF_SUCCESS;
-                        }
-                        pdmacFileAioMgrNormalDestroy(pAioMgrNew);
-                    }
-                    RTCritSectDelete(&pAioMgrNew->CritSectBlockingEvent);
-                }
-                RTSemEventDestroy(pAioMgrNew->EventSem);
-            }
-            RTSemEventDestroy(pAioMgrNew->EventSemBlock);
-        }
-        MMR3HeapFree(pAioMgrNew);
-    }
-
-    LogFlowFunc((": Leave rc=%Rrc\n", rc));
-
-    return rc;
-}
-
-/**
- * Destroys a async I/O manager.
- *
- * @returns nothing.
- * @param   pAioMgr    The async I/O manager to destroy.
- */
-static void pdmacFileAioMgrDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile, PPDMACEPFILEMGR pAioMgr)
-{
-    int rc = pdmacFileAioMgrShutdown(pAioMgr);
-    AssertRC(rc);
-
-    /* Unlink from the list. */
-    rc = RTCritSectEnter(&pEpClassFile->CritSect);
-    AssertRC(rc);
-
-    PPDMACEPFILEMGR pPrev = pAioMgr->pPrev;
-    PPDMACEPFILEMGR pNext = pAioMgr->pNext;
-
-    if (pPrev)
-        pPrev->pNext = pNext;
-    else
-        pEpClassFile->pAioMgrHead = pNext;
-
-    if (pNext)
-        pNext->pPrev = pPrev;
-
-    pEpClassFile->cAioMgrs--;
-    rc = RTCritSectLeave(&pEpClassFile->CritSect);
-    AssertRC(rc);
-
-    /* Free the resources. */
-    RTCritSectDelete(&pAioMgr->CritSectBlockingEvent);
-    RTSemEventDestroy(pAioMgr->EventSem);
-    if (pAioMgr->enmMgrType != PDMACEPFILEMGRTYPE_SIMPLE)
-        pdmacFileAioMgrNormalDestroy(pAioMgr);
-
-    MMR3HeapFree(pAioMgr);
-}
-
-static int pdmacFileMgrTypeFromName(const char *pszVal, PPDMACEPFILEMGRTYPE penmMgrType)
-{
-    int rc = VINF_SUCCESS;
-
-    if (!RTStrCmp(pszVal, "Simple"))
-        *penmMgrType = PDMACEPFILEMGRTYPE_SIMPLE;
-    else if (!RTStrCmp(pszVal, "Async"))
-        *penmMgrType = PDMACEPFILEMGRTYPE_ASYNC;
-    else
-        rc = VERR_CFGM_CONFIG_UNKNOWN_VALUE;
-
-    return rc;
-}
-
-static const char *pdmacFileMgrTypeToName(PDMACEPFILEMGRTYPE enmMgrType)
-{
-    if (enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
-        return "Simple";
-    if (enmMgrType == PDMACEPFILEMGRTYPE_ASYNC)
-        return "Async";
-
-    return NULL;
-}
-
-static int pdmacFileBackendTypeFromName(const char *pszVal, PPDMACFILEEPBACKEND penmBackendType)
-{
-    int rc = VINF_SUCCESS;
-
-    if (!RTStrCmp(pszVal, "Buffered"))
-        *penmBackendType = PDMACFILEEPBACKEND_BUFFERED;
-    else if (!RTStrCmp(pszVal, "NonBuffered"))
-        *penmBackendType = PDMACFILEEPBACKEND_NON_BUFFERED;
-    else
-        rc = VERR_CFGM_CONFIG_UNKNOWN_VALUE;
-
-    return rc;
-}
-
-static const char *pdmacFileBackendTypeToName(PDMACFILEEPBACKEND enmBackendType)
-{
-    if (enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
-        return "Buffered";
-    if (enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
-        return "NonBuffered";
-
-    return NULL;
-}
-
-/**
- * Get the size of the given file.
- * Works for block devices too.
- *
- * @returns VBox status code.
- * @param   hFile    The file handle.
- * @param   pcbSize  Where to store the size of the file on success.
- */
-static int pdmacFileEpNativeGetSize(RTFILE hFile, uint64_t *pcbSize)
-{
-    int rc = VINF_SUCCESS;
-    uint64_t cbSize = 0;
-
-    rc = RTFileGetSize(hFile, &cbSize);
-    if (RT_SUCCESS(rc) && (cbSize != 0))
-        *pcbSize = cbSize;
-    else
-    {
-#ifdef RT_OS_WINDOWS
-        DISK_GEOMETRY DriveGeo;
-        DWORD cbDriveGeo;
-        if (DeviceIoControl((HANDLE)hFile,
-                            IOCTL_DISK_GET_DRIVE_GEOMETRY, NULL, 0,
-                            &DriveGeo, sizeof(DriveGeo), &cbDriveGeo, NULL))
-        {
-            if (   DriveGeo.MediaType == FixedMedia
-                || DriveGeo.MediaType == RemovableMedia)
-            {
-                cbSize =     DriveGeo.Cylinders.QuadPart
-                         *   DriveGeo.TracksPerCylinder
-                         *   DriveGeo.SectorsPerTrack
-                         *   DriveGeo.BytesPerSector;
-
-                GET_LENGTH_INFORMATION DiskLenInfo;
-                DWORD junk;
-                if (DeviceIoControl((HANDLE)hFile,
-                                    IOCTL_DISK_GET_LENGTH_INFO, NULL, 0,
-                                    &DiskLenInfo, sizeof(DiskLenInfo), &junk, (LPOVERLAPPED)NULL))
-                {
-                    /* IOCTL_DISK_GET_LENGTH_INFO is supported -- override cbSize. */
-                    cbSize = DiskLenInfo.Length.QuadPart;
-                }
-
-                rc = VINF_SUCCESS;
-            }
-            else
-            {
-                rc = VERR_INVALID_PARAMETER;
-            }
-        }
-        else
-        {
-            rc = RTErrConvertFromWin32(GetLastError());
-        }
-#elif defined(RT_OS_DARWIN)
-        struct stat DevStat;
-        if (!fstat(hFile, &DevStat) && S_ISBLK(DevStat.st_mode))
-        {
-            uint64_t cBlocks;
-            uint32_t cbBlock;
-            if (!ioctl(hFile, DKIOCGETBLOCKCOUNT, &cBlocks))
-            {
-                if (!ioctl(hFile, DKIOCGETBLOCKSIZE, &cbBlock))
-                    cbSize = cBlocks * cbBlock;
-                else
-                    rc = RTErrConvertFromErrno(errno);
-            }
-            else
-                rc = RTErrConvertFromErrno(errno);
-        }
-        else
-            rc = VERR_INVALID_PARAMETER;
-#elif defined(RT_OS_SOLARIS)
-        struct stat DevStat;
-        if (!fstat(hFile, &DevStat) && (   S_ISBLK(DevStat.st_mode)
-                                        || S_ISCHR(DevStat.st_mode)))
-        {
-            struct dk_minfo mediainfo;
-            if (!ioctl(hFile, DKIOCGMEDIAINFO, &mediainfo))
-                cbSize = mediainfo.dki_capacity * mediainfo.dki_lbsize;
-            else
-                rc = RTErrConvertFromErrno(errno);
-        }
-        else
-            rc = VERR_INVALID_PARAMETER;
-#elif defined(RT_OS_FREEBSD)
-        struct stat DevStat;
-        if (!fstat(hFile, &DevStat) && S_ISCHR(DevStat.st_mode))
-        {
-            off_t cbMedia = 0;
-            if (!ioctl(hFile, DIOCGMEDIASIZE, &cbMedia))
-            {
-                cbSize = cbMedia;
-            }
-            else
-                rc = RTErrConvertFromErrno(errno);
-        }
-        else
-            rc = VERR_INVALID_PARAMETER;
-#else
-        /* Could be a block device */
-        rc = RTFileSeek(hFile, 0, RTFILE_SEEK_END, &cbSize);
-#endif
-
-        if (RT_SUCCESS(rc) && (cbSize != 0))
-            *pcbSize = cbSize;
-        else if (RT_SUCCESS(rc))
-            rc = VERR_NOT_SUPPORTED;
-    }
-
-    return rc;
-}
-
-#ifdef VBOX_WITH_DEBUGGER
-/**
- * Error inject callback.
- */
-static DECLCALLBACK(int) pdmacEpFileErrorInject(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR pArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    bool fWrite;
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile;
-
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return DBGCCmdHlpPrintf(pCmdHlp, "error: The command requires a VM to be selected.\n");
-    if (    cArgs != 3
-        ||  pArgs[0].enmType != DBGCVAR_TYPE_STRING
-        ||  pArgs[1].enmType != DBGCVAR_TYPE_STRING
-        ||  pArgs[2].enmType != DBGCVAR_TYPE_STRING)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: parser error, invalid arguments.\n");
-
-    pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
-
-    /* Syntax is "read|write <filename> <status code>" */
-    if (!RTStrCmp(pArgs[0].u.pszString, "read"))
-        fWrite = false;
-    else if (!RTStrCmp(pArgs[0].u.pszString, "write"))
-        fWrite = true;
-    else
-    {
-        DBGCCmdHlpPrintf(pCmdHlp, "error: invalid transefr direction '%s'.\n", pArgs[0].u.pszString);
-        return VINF_SUCCESS;
-    }
-
-    /* Search for the matching endpoint. */
-    RTCritSectEnter(&pEpClassFile->Core.CritSect);
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpClassFile->Core.pEndpointsHead;
-
-    while (pEpFile)
-    {
-        if (!RTStrCmp(pArgs[1].u.pszString, RTPathFilename(pEpFile->Core.pszUri)))
-            break;
-        pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpFile->Core.pNext;
-    }
-
-    if (pEpFile)
-    {
-        int rcToInject = RTStrToInt32(pArgs[2].u.pszString);
-
-        if (fWrite)
-            ASMAtomicXchgS32(&pEpFile->rcReqWrite, rcToInject);
-        else
-            ASMAtomicXchgS32(&pEpFile->rcReqRead, rcToInject);
-
-            DBGCCmdHlpPrintf(pCmdHlp, "Injected %Rrc into '%s' for %s\n",
-                             rcToInject, pArgs[1].u.pszString, pArgs[0].u.pszString);
-    }
-    else
-        DBGCCmdHlpPrintf(pCmdHlp, "No file with name '%s' found\n", NULL, pArgs[1].u.pszString);
-
-    RTCritSectLeave(&pEpClassFile->Core.CritSect);
-    return VINF_SUCCESS;
-}
-#endif
-
-static int pdmacFileInitialize(PPDMASYNCCOMPLETIONEPCLASS pClassGlobals, PCFGMNODE pCfgNode)
-{
-    int rc = VINF_SUCCESS;
-    RTFILEAIOLIMITS AioLimits; /** < Async I/O limitations. */
-
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pClassGlobals;
-
-    rc = RTFileAioGetLimits(&AioLimits);
-#ifdef DEBUG
-    if (RT_SUCCESS(rc) && RTEnvExist("VBOX_ASYNC_IO_FAILBACK"))
-        rc = VERR_ENV_VAR_NOT_FOUND;
-#endif
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("AIO: Async I/O manager not supported (rc=%Rrc). Falling back to simple manager\n",
-                rc));
-        pEpClassFile->enmMgrTypeOverride = PDMACEPFILEMGRTYPE_SIMPLE;
-        pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_BUFFERED;
-    }
-    else
-    {
-        pEpClassFile->uBitmaskAlignment   = AioLimits.cbBufferAlignment ? ~((RTR3UINTPTR)AioLimits.cbBufferAlignment - 1) : RTR3UINTPTR_MAX;
-        pEpClassFile->cReqsOutstandingMax = AioLimits.cReqsOutstandingMax;
-
-        if (pCfgNode)
-        {
-            /* Query the default manager type */
-            char *pszVal = NULL;
-            rc = CFGMR3QueryStringAllocDef(pCfgNode, "IoMgr", &pszVal, "Async");
-            AssertLogRelRCReturn(rc, rc);
-
-            rc = pdmacFileMgrTypeFromName(pszVal, &pEpClassFile->enmMgrTypeOverride);
-            MMR3HeapFree(pszVal);
-            if (RT_FAILURE(rc))
-                return rc;
-
-            LogRel(("AIOMgr: Default manager type is \"%s\"\n", pdmacFileMgrTypeToName(pEpClassFile->enmMgrTypeOverride)));
-
-            /* Query default backend type */
-            rc = CFGMR3QueryStringAllocDef(pCfgNode, "FileBackend", &pszVal, "NonBuffered");
-            AssertLogRelRCReturn(rc, rc);
-
-            rc = pdmacFileBackendTypeFromName(pszVal, &pEpClassFile->enmEpBackendDefault);
-            MMR3HeapFree(pszVal);
-            if (RT_FAILURE(rc))
-                return rc;
-
-            LogRel(("AIOMgr: Default file backend is \"%s\"\n", pdmacFileBackendTypeToName(pEpClassFile->enmEpBackendDefault)));
-
-#ifdef RT_OS_LINUX
-            if (   pEpClassFile->enmMgrTypeOverride == PDMACEPFILEMGRTYPE_ASYNC
-                && pEpClassFile->enmEpBackendDefault == PDMACFILEEPBACKEND_BUFFERED)
-            {
-                LogRel(("AIOMgr: Linux does not support buffered async I/O, changing to non buffered\n"));
-                pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_NON_BUFFERED;
-            }
-#endif
-        }
-        else
-        {
-            /* No configuration supplied, set defaults */
-            pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_NON_BUFFERED;
-            pEpClassFile->enmMgrTypeOverride  = PDMACEPFILEMGRTYPE_ASYNC;
-        }
-    }
-
-    /* Init critical section. */
-    rc = RTCritSectInit(&pEpClassFile->CritSect);
-
-#ifdef VBOX_WITH_DEBUGGER
-    /* Install the error injection handler. */
-    if (RT_SUCCESS(rc))
-    {
-        rc = DBGCRegisterCommands(&g_aCmds[0], 1);
-        AssertRC(rc);
-    }
-#endif
-
-    return rc;
-}
-
-static void pdmacFileTerminate(PPDMASYNCCOMPLETIONEPCLASS pClassGlobals)
-{
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pClassGlobals;
-
-    /* All endpoints should be closed at this point. */
-    AssertMsg(!pEpClassFile->Core.pEndpointsHead, ("There are still endpoints left\n"));
-
-    /* Destroy all left async I/O managers. */
-    while (pEpClassFile->pAioMgrHead)
-        pdmacFileAioMgrDestroy(pEpClassFile, pEpClassFile->pAioMgrHead);
-
-    RTCritSectDelete(&pEpClassFile->CritSect);
-}
-
-static int pdmacFileEpInitialize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                 const char *pszUri, uint32_t fFlags)
-{
-    int rc = VINF_SUCCESS;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;
-    PDMACEPFILEMGRTYPE enmMgrType = pEpClassFile->enmMgrTypeOverride;
-    PDMACFILEEPBACKEND enmEpBackend = pEpClassFile->enmEpBackendDefault;
-
-    AssertMsgReturn((fFlags & ~(PDMACEP_FILE_FLAGS_READ_ONLY | PDMACEP_FILE_FLAGS_DONT_LOCK)) == 0,
-                    ("PDMAsyncCompletion: Invalid flag specified\n"), VERR_INVALID_PARAMETER);
-
-    unsigned fFileFlags = RTFILE_O_OPEN;
-
-    if (fFlags & PDMACEP_FILE_FLAGS_READ_ONLY)
-        fFileFlags |= RTFILE_O_READ | RTFILE_O_DENY_NONE;
-    else
-    {
-        fFileFlags |= RTFILE_O_READWRITE;
-
-        /*
-         * Opened in read/write mode. Check whether the caller wants to
-         * avoid the lock. Return an error in case caching is enabled
-         * because this can lead to data corruption.
-         */
-        if (fFlags & PDMACEP_FILE_FLAGS_DONT_LOCK)
-            fFileFlags |= RTFILE_O_DENY_NONE;
-        else
-            fFileFlags |= RTFILE_O_DENY_WRITE;
-    }
-
-    if (enmMgrType == PDMACEPFILEMGRTYPE_ASYNC)
-        fFileFlags |= RTFILE_O_ASYNC_IO;
-
-    if (enmEpBackend == PDMACFILEEPBACKEND_NON_BUFFERED)
-    {
-        /*
-         * We only disable the cache if the size of the file is a multiple of 512.
-         * Certain hosts like Windows, Linux and Solaris require that transfer sizes
-         * are aligned to the volume sector size.
-         * If not we just make sure that the data is written to disk with RTFILE_O_WRITE_THROUGH
-         * which will trash the host cache but ensures that the host cache will not
-         * contain dirty buffers.
-         */
-        RTFILE File = NIL_RTFILE;
-
-        rc = RTFileOpen(&File, pszUri, RTFILE_O_READ | RTFILE_O_OPEN | RTFILE_O_DENY_NONE);
-        if (RT_SUCCESS(rc))
-        {
-            uint64_t cbSize;
-
-            rc = pdmacFileEpNativeGetSize(File, &cbSize);
-            Assert(RT_FAILURE(rc) || cbSize != 0);
-
-            if (RT_SUCCESS(rc) && ((cbSize % 512) == 0))
-                fFileFlags |= RTFILE_O_NO_CACHE;
-            else
-            {
-                /* Downgrade to the buffered backend */
-                enmEpBackend = PDMACFILEEPBACKEND_BUFFERED;
-
-#ifdef RT_OS_LINUX
-                fFileFlags &= ~RTFILE_O_ASYNC_IO;
-                enmMgrType   = PDMACEPFILEMGRTYPE_SIMPLE;
-#endif
-            }
-            RTFileClose(File);
-        }
-    }
-
-    /* Open with final flags. */
-    rc = RTFileOpen(&pEpFile->File, pszUri, fFileFlags);
-    if ((rc == VERR_INVALID_FUNCTION) || (rc == VERR_INVALID_PARAMETER))
-    {
-        LogRel(("pdmacFileEpInitialize: RTFileOpen %s / %08x failed with %Rrc\n",
-               pszUri, fFileFlags, rc));
-        /*
-         * Solaris doesn't support directio on ZFS so far. :-\
-         * Trying to enable it returns VERR_INVALID_FUNCTION
-         * (ENOTTY). Remove it and hope for the best.
-         * ZFS supports write throttling in case applications
-         * write more data than can be synced to the disk
-         * without blocking the whole application.
-         *
-         * On Linux we have the same problem with cifs.
-         * Have to disable async I/O here too because it requires O_DIRECT.
-         */
-        fFileFlags &= ~RTFILE_O_NO_CACHE;
-        enmEpBackend = PDMACFILEEPBACKEND_BUFFERED;
-
-#ifdef RT_OS_LINUX
-        fFileFlags &= ~RTFILE_O_ASYNC_IO;
-        enmMgrType   = PDMACEPFILEMGRTYPE_SIMPLE;
-#endif
-
-        /* Open again. */
-        rc = RTFileOpen(&pEpFile->File, pszUri, fFileFlags);
-
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("pdmacFileEpInitialize: RTFileOpen %s / %08x failed AGAIN(!) with %Rrc\n",
-                        pszUri, fFileFlags, rc));
-        }
-    }
-
-    if (RT_SUCCESS(rc))
-    {
-        pEpFile->fFlags = fFileFlags;
-
-        rc = pdmacFileEpNativeGetSize(pEpFile->File, (uint64_t *)&pEpFile->cbFile);
-        Assert(RT_FAILURE(rc) || pEpFile->cbFile != 0);
-
-        if (RT_SUCCESS(rc))
-        {
-            /* Initialize the segment cache */
-            rc = MMR3HeapAllocZEx(pEpClassFile->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION,
-                                  sizeof(PDMACTASKFILE),
-                                  (void **)&pEpFile->pTasksFreeHead);
-            if (RT_SUCCESS(rc))
-            {
-                PPDMACEPFILEMGR pAioMgr = NULL;
-
-                pEpFile->pTasksFreeTail = pEpFile->pTasksFreeHead;
-                pEpFile->cTasksCached   = 0;
-                pEpFile->enmBackendType = enmEpBackend;
-                /*
-                 * Disable async flushes on Solaris for now.
-                 * They cause weird hangs which needs more investigations.
-                 */
-#ifndef RT_OS_SOLARIS
-                pEpFile->fAsyncFlushSupported = true;
-#else
-                pEpFile->fAsyncFlushSupported = false;
-#endif
-
-                if (enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
-                {
-                    /* Simple mode. Every file has its own async I/O manager. */
-                    rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, PDMACEPFILEMGRTYPE_SIMPLE);
-                    AssertRC(rc);
-                }
-                else
-                {
-                    pAioMgr = pEpClassFile->pAioMgrHead;
-
-                    /* Check for an idling manager of the same type */
-                    while (pAioMgr)
-                    {
-                        if (pAioMgr->enmMgrType == enmMgrType)
-                            break;
-                        pAioMgr = pAioMgr->pNext;
-                    }
-
-                    if (!pAioMgr)
-                    {
-                        rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, enmMgrType);
-                        AssertRC(rc);
-                    }
-                }
-
-                pEpFile->AioMgr.pTreeRangesLocked = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
-                if (!pEpFile->AioMgr.pTreeRangesLocked)
-                    rc = VERR_NO_MEMORY;
-                else
-                {
-                    pEpFile->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
-
-                    /* Assign the endpoint to the thread. */
-                    rc = pdmacFileAioMgrAddEndpoint(pAioMgr, pEpFile);
-                    if (RT_FAILURE(rc))
-                    {
-                        RTMemFree(pEpFile->AioMgr.pTreeRangesLocked);
-                        MMR3HeapFree(pEpFile->pTasksFreeHead);
-                    }
-                }
-            }
-        }
-
-        if (RT_FAILURE(rc))
-            RTFileClose(pEpFile->File);
-    }
-
-#ifdef VBOX_WITH_STATISTICS
-    if (RT_SUCCESS(rc))
-    {
-        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatRead,
-                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
-                       STAMUNIT_TICKS_PER_CALL, "Time taken to read from the endpoint",
-                       "/PDM/AsyncCompletion/File/%s/Read", RTPathFilename(pEpFile->Core.pszUri));
-
-        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatWrite,
-                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
-                       STAMUNIT_TICKS_PER_CALL, "Time taken to write to the endpoint",
-                       "/PDM/AsyncCompletion/File/%s/Write", RTPathFilename(pEpFile->Core.pszUri));
-    }
-#endif
-
-    if (RT_SUCCESS(rc))
-        LogRel(("AIOMgr: Endpoint for file '%s' (flags %08x) created successfully\n", pszUri, pEpFile->fFlags));
-
-    return rc;
-}
-
-static int pdmacFileEpRangesLockedDestroy(PAVLRFOFFNODECORE pNode, void *pvUser)
-{
-    AssertMsgFailed(("The locked ranges tree should be empty at that point\n"));
-    return VINF_SUCCESS;
-}
-
-static int pdmacFileEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
-{
-    int rc = VINF_SUCCESS;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;
-
-    /* Make sure that all tasks finished for this endpoint. */
-    rc = pdmacFileAioMgrCloseEndpoint(pEpFile->pAioMgr, pEpFile);
-    AssertRC(rc);
-
-    /*
-     * If the async I/O manager is in failsafe mode this is the only endpoint
-     * he processes and thus can be destroyed now.
-     */
-    if (pEpFile->pAioMgr->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
-        pdmacFileAioMgrDestroy(pEpClassFile, pEpFile->pAioMgr);
-
-    /* Free cached tasks. */
-    PPDMACTASKFILE pTask = pEpFile->pTasksFreeHead;
-
-    while (pTask)
-    {
-        PPDMACTASKFILE pTaskFree = pTask;
-        pTask = pTask->pNext;
-        MMR3HeapFree(pTaskFree);
-    }
-
-    /* Destroy the locked ranges tree now. */
-    RTAvlrFileOffsetDestroy(pEpFile->AioMgr.pTreeRangesLocked, pdmacFileEpRangesLockedDestroy, NULL);
-
-    RTFileClose(pEpFile->File);
-
-#ifdef VBOX_WITH_STATISTICS
-    STAMR3Deregister(pEpClassFile->Core.pVM, &pEpFile->StatRead);
-    STAMR3Deregister(pEpClassFile->Core.pVM, &pEpFile->StatWrite);
-#endif
-
-    return VINF_SUCCESS;
-}
-
-static int pdmacFileEpRead(PPDMASYNCCOMPLETIONTASK pTask,
-                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
-                           PCRTSGSEG paSegments, size_t cSegments,
-                           size_t cbRead)
-{
-    int rc = VINF_SUCCESS;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-
-    LogFlowFunc(("pTask=%#p pEndpoint=%#p off=%RTfoff paSegments=%#p cSegments=%zu cbRead=%zu\n",
-                 pTask, pEndpoint, off, paSegments, cSegments, cbRead));
-
-    STAM_PROFILE_ADV_START(&pEpFile->StatRead, Read);
-
-    pdmacFileEpTaskInit(pTask, cbRead);
-
-    rc = pdmacFileEpTaskInitiate(pTask, pEndpoint, off, paSegments, cSegments, cbRead,
-                                 PDMACTASKFILETRANSFER_READ);
-
-    STAM_PROFILE_ADV_STOP(&pEpFile->StatRead, Read);
-
-    return rc;
-}
-
-static int pdmacFileEpWrite(PPDMASYNCCOMPLETIONTASK pTask,
-                            PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
-                            PCRTSGSEG paSegments, size_t cSegments,
-                            size_t cbWrite)
-{
-    int rc = VINF_SUCCESS;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-
-    if (RT_UNLIKELY(pEpFile->fReadonly))
-        return VERR_NOT_SUPPORTED;
-
-    STAM_PROFILE_ADV_START(&pEpFile->StatWrite, Write);
-
-    pdmacFileEpTaskInit(pTask, cbWrite);
-
-    rc = pdmacFileEpTaskInitiate(pTask, pEndpoint, off, paSegments, cSegments, cbWrite,
-                                 PDMACTASKFILETRANSFER_WRITE);
-
-    STAM_PROFILE_ADV_STOP(&pEpFile->StatWrite, Write);
-
-    return rc;
-}
-
-static int pdmacFileEpFlush(PPDMASYNCCOMPLETIONTASK pTask,
-                            PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
-{
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
-
-    if (RT_UNLIKELY(pEpFile->fReadonly))
-        return VERR_NOT_SUPPORTED;
-
-    pdmacFileEpTaskInit(pTask, 0);
-
-    PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEpFile);
-    if (RT_UNLIKELY(!pIoTask))
-        return VERR_NO_MEMORY;
-
-    pIoTask->pEndpoint       = pEpFile;
-    pIoTask->enmTransferType = PDMACTASKFILETRANSFER_FLUSH;
-    pIoTask->pvUser          = pTaskFile;
-    pIoTask->pfnCompleted    = pdmacFileEpTaskCompleted;
-    pdmacFileEpAddTask(pEpFile, pIoTask);
-
-    return VINF_AIO_TASK_PENDING;
-}
-
-static int pdmacFileEpGetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t *pcbSize)
-{
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-
-    *pcbSize = ASMAtomicReadU64(&pEpFile->cbFile);
-
-    return VINF_SUCCESS;
-}
-
-static int pdmacFileEpSetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t cbSize)
-{
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
-
-    ASMAtomicWriteU64(&pEpFile->cbFile, cbSize);
-    return RTFileSetSize(pEpFile->File, cbSize);
-}
-
-const PDMASYNCCOMPLETIONEPCLASSOPS g_PDMAsyncCompletionEndpointClassFile =
-{
-    /* u32Version */
-    PDMAC_EPCLASS_OPS_VERSION,
-    /* pcszName */
-    "File",
-    /* enmClassType */
-    PDMASYNCCOMPLETIONEPCLASSTYPE_FILE,
-    /* cbEndpointClassGlobal */
-    sizeof(PDMASYNCCOMPLETIONEPCLASSFILE),
-    /* cbEndpoint */
-    sizeof(PDMASYNCCOMPLETIONENDPOINTFILE),
-    /* cbTask */
-    sizeof(PDMASYNCCOMPLETIONTASKFILE),
-    /* pfnInitialize */
-    pdmacFileInitialize,
-    /* pfnTerminate */
-    pdmacFileTerminate,
-    /* pfnEpInitialize. */
-    pdmacFileEpInitialize,
-    /* pfnEpClose */
-    pdmacFileEpClose,
-    /* pfnEpRead */
-    pdmacFileEpRead,
-    /* pfnEpWrite */
-    pdmacFileEpWrite,
-    /* pfnEpFlush */
-    pdmacFileEpFlush,
-    /* pfnEpGetSize */
-    pdmacFileEpGetSize,
-    /* pfnEpSetSize */
-    pdmacFileEpSetSize,
-    /* u32VersionEnd */
-    PDMAC_EPCLASS_OPS_VERSION
-};
-
Index: unk/src/VBox/VMM/PDMAsyncCompletionFileFailsafe.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMAsyncCompletionFileFailsafe.cpp	(revision 35332)
+++ 	(revision )
@@ -1,223 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Async I/O - Transport data asynchronous in R3 using EMT.
- * Simple File I/O manager.
- */
-
-/*
- * Copyright (C) 2006-2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <VBox/log.h>
-
-#include "PDMAsyncCompletionFileInternal.h"
-
-
-static int pdmacFileAioMgrFailsafeProcessEndpointTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                                          PPDMACTASKFILE pTasks)
-{
-    int rc = VINF_SUCCESS;
-
-    while (pTasks)
-    {
-        PPDMACTASKFILE pCurr = pTasks;
-
-        pTasks = pTasks->pNext;
-
-        switch (pCurr->enmTransferType)
-        {
-            case PDMACTASKFILETRANSFER_FLUSH:
-            {
-                rc = RTFileFlush(pEndpoint->File);
-                break;
-            }
-            case PDMACTASKFILETRANSFER_READ:
-            case PDMACTASKFILETRANSFER_WRITE:
-            {
-                if (pCurr->enmTransferType == PDMACTASKFILETRANSFER_READ)
-                {
-                    rc = RTFileReadAt(pEndpoint->File, pCurr->Off,
-                                      pCurr->DataSeg.pvSeg,
-                                      pCurr->DataSeg.cbSeg,
-                                      NULL);
-                }
-                else
-                {
-                    if (RT_UNLIKELY((uint64_t)pCurr->Off + pCurr->DataSeg.cbSeg > pEndpoint->cbFile))
-                    {
-                        ASMAtomicWriteU64(&pEndpoint->cbFile, pCurr->Off + pCurr->DataSeg.cbSeg);
-                        RTFileSetSize(pEndpoint->File, pCurr->Off + pCurr->DataSeg.cbSeg);
-                    }
-
-                    rc = RTFileWriteAt(pEndpoint->File, pCurr->Off,
-                                       pCurr->DataSeg.pvSeg,
-                                       pCurr->DataSeg.cbSeg,
-                                       NULL);
-                }
-
-                break;
-            }
-            default:
-                AssertMsgFailed(("Invalid transfer type %d\n", pTasks->enmTransferType));
-        }
-
-        pCurr->pfnCompleted(pCurr, pCurr->pvUser, rc);
-        pdmacFileTaskFree(pEndpoint, pCurr);
-    }
-
-    return VINF_SUCCESS;
-}
-
-static int pdmacFileAioMgrFailsafeProcessEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    int rc = VINF_SUCCESS;
-    PPDMACTASKFILE pTasks = pEndpoint->AioMgr.pReqsPendingHead;
-
-    pEndpoint->AioMgr.pReqsPendingHead = NULL;
-    pEndpoint->AioMgr.pReqsPendingTail = NULL;
-
-    /* Process the request pending list first in case the endpoint was migrated due to an error. */
-    if (pTasks)
-        rc = pdmacFileAioMgrFailsafeProcessEndpointTaskList(pEndpoint, pTasks);
-
-    if (RT_SUCCESS(rc))
-    {
-        pTasks = pdmacFileEpGetNewTasks(pEndpoint);
-
-        if (pTasks)
-            rc = pdmacFileAioMgrFailsafeProcessEndpointTaskList(pEndpoint, pTasks);
-    }
-
-    return rc;
-}
-
-/**
- * A fallback method in case something goes wrong with the normal
- * I/O manager.
- */
-int pdmacFileAioMgrFailsafe(RTTHREAD ThreadSelf, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-    PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
-
-    while (   (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
-           || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING))
-    {
-        ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
-        if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
-            rc = RTSemEventWait(pAioMgr->EventSem, RT_INDEFINITE_WAIT);
-        ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
-        AssertRC(rc);
-
-        LogFlow(("Got woken up\n"));
-        ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
-
-        /* Process endpoint events first. */
-        PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
-        while (pEndpoint)
-        {
-            rc = pdmacFileAioMgrFailsafeProcessEndpoint(pEndpoint);
-            AssertRC(rc);
-            pEndpoint = pEndpoint->AioMgr.pEndpointNext;
-        }
-
-        /* Now check for an external blocking event. */
-        if (pAioMgr->fBlockingEventPending)
-        {
-            switch (pAioMgr->enmBlockingEvent)
-            {
-                case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
-                {
-                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = pAioMgr->BlockingEventData.AddEndpoint.pEndpoint;
-                    AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
-
-                    pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
-
-                    pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
-                    pEndpointNew->AioMgr.pEndpointPrev = NULL;
-                    if (pAioMgr->pEndpointsHead)
-                        pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
-                    pAioMgr->pEndpointsHead = pEndpointNew;
-
-                    pAioMgr->cEndpoints++;
-
-                    /*
-                     * Process the task list the first time. There might be pending requests
-                     * if the endpoint was migrated from another endpoint.
-                     */
-                    rc = pdmacFileAioMgrFailsafeProcessEndpoint(pEndpointNew);
-                    AssertRC(rc);
-                    break;
-                }
-                case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
-                {
-                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint;
-                    AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
-
-                    pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
-
-                    PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
-                    PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
-
-                    if (pPrev)
-                        pPrev->AioMgr.pEndpointNext = pNext;
-                    else
-                        pAioMgr->pEndpointsHead = pNext;
-
-                    if (pNext)
-                        pNext->AioMgr.pEndpointPrev = pPrev;
-
-                    pAioMgr->cEndpoints--;
-                    break;
-                }
-                case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
-                {
-                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint;
-                    AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to Close\n"));
-
-                    pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
-
-                    /* Make sure all tasks finished. */
-                    rc = pdmacFileAioMgrFailsafeProcessEndpoint(pEndpointClose);
-                    AssertRC(rc);
-                    break;
-                }
-                case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
-                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
-                    break;
-                case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
-                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
-                    break;
-                case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
-                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
-                    break;
-                default:
-                    AssertMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
-            }
-
-            ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
-            pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
-
-            /* Release the waiting thread. */
-            rc = RTSemEventSignal(pAioMgr->EventSemBlock);
-            AssertRC(rc);
-        }
-    }
-
-    return rc;
-}
-
Index: unk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h
===================================================================
--- /trunk/src/VBox/VMM/PDMAsyncCompletionFileInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,527 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Async I/O - Transport data asynchronous in R3 using EMT.
- */
-
-/*
- * Copyright (C) 2006-2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___PDMAsyncCompletionFileInternal_h
-#define ___PDMAsyncCompletionFileInternal_h
-
-#include <VBox/cfgm.h>
-#include <VBox/stam.h>
-#include <VBox/tm.h>
-#include <iprt/types.h>
-#include <iprt/file.h>
-#include <iprt/thread.h>
-#include <iprt/semaphore.h>
-#include <iprt/critsect.h>
-#include <iprt/avl.h>
-#include <iprt/list.h>
-#include <iprt/spinlock.h>
-#include <iprt/memcache.h>
-
-#include "PDMAsyncCompletionInternal.h"
-
-/** @todo: Revise the caching of tasks. We have currently four caches:
- *  Per endpoint task cache
- *  Per class cache
- *  Per endpoint task segment cache
- *  Per class task segment cache
- *
- *  We could use the RT heap for this probably or extend MMR3Heap (uses RTMemAlloc
- *  instead of managing larger blocks) to have this global for the whole VM.
- */
-
-RT_C_DECLS_BEGIN
-
-/**
- * A few forward declarations.
- */
-typedef struct PDMASYNCCOMPLETIONENDPOINTFILE *PPDMASYNCCOMPLETIONENDPOINTFILE;
-/** Pointer to a request segment. */
-typedef struct PDMACTASKFILE *PPDMACTASKFILE;
-/** Pointer to the endpoint class data. */
-typedef struct PDMASYNCCOMPLETIONTASKFILE *PPDMASYNCCOMPLETIONTASKFILE;
-/** Pointer to a cache LRU list. */
-typedef struct PDMACFILELRULIST *PPDMACFILELRULIST;
-/** Pointer to the global cache structure. */
-typedef struct PDMACFILECACHEGLOBAL *PPDMACFILECACHEGLOBAL;
-/** Pointer to a task segment. */
-typedef struct PDMACFILETASKSEG *PPDMACFILETASKSEG;
-
-/**
- * Blocking event types.
- */
-typedef enum PDMACEPFILEAIOMGRBLOCKINGEVENT
-{
-    /** Invalid tye */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID = 0,
-    /** An endpoint is added to the manager. */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT,
-    /** An endpoint is removed from the manager. */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT,
-    /** An endpoint is about to be closed. */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT,
-    /** The manager is requested to terminate */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN,
-    /** The manager is requested to suspend */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND,
-    /** The manager is requested to resume */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME,
-    /** 32bit hack */
-    PDMACEPFILEAIOMGRBLOCKINGEVENT_32BIT_HACK = 0x7fffffff
-} PDMACEPFILEAIOMGRBLOCKINGEVENT;
-
-/**
- * I/O manager type.
- */
-typedef enum PDMACEPFILEMGRTYPE
-{
-    /** Simple aka failsafe */
-    PDMACEPFILEMGRTYPE_SIMPLE = 0,
-    /** Async I/O with host cache enabled. */
-    PDMACEPFILEMGRTYPE_ASYNC,
-    /** 32bit hack */
-    PDMACEPFILEMGRTYPE_32BIT_HACK = 0x7fffffff
-} PDMACEPFILEMGRTYPE;
-/** Pointer to a I/O manager type */
-typedef PDMACEPFILEMGRTYPE *PPDMACEPFILEMGRTYPE;
-
-/**
- * States of the I/O manager.
- */
-typedef enum PDMACEPFILEMGRSTATE
-{
-    /** Invalid state. */
-    PDMACEPFILEMGRSTATE_INVALID = 0,
-    /** Normal running state accepting new requests
-     * and processing them.
-     */
-    PDMACEPFILEMGRSTATE_RUNNING,
-    /** Fault state - not accepting new tasks for endpoints but waiting for
-     * remaining ones to finish.
-     */
-    PDMACEPFILEMGRSTATE_FAULT,
-    /** Suspending state - not accepting new tasks for endpoints but waiting
-     * for remaining ones to finish.
-     */
-    PDMACEPFILEMGRSTATE_SUSPENDING,
-    /** Shutdown state - not accepting new tasks for endpoints but waiting
-     * for remaining ones to finish.
-     */
-    PDMACEPFILEMGRSTATE_SHUTDOWN,
-    /** The I/O manager waits for all active requests to complete and doesn't queue
-     * new ones because it needs to grow to handle more requests.
-     */
-    PDMACEPFILEMGRSTATE_GROWING,
-    /** 32bit hack */
-    PDMACEPFILEMGRSTATE_32BIT_HACK = 0x7fffffff
-} PDMACEPFILEMGRSTATE;
-
-/**
- * State of a async I/O manager.
- */
-typedef struct PDMACEPFILEMGR
-{
-    /** Next Aio manager in the list. */
-    R3PTRTYPE(struct PDMACEPFILEMGR *)     pNext;
-    /** Previous Aio manager in the list. */
-    R3PTRTYPE(struct PDMACEPFILEMGR *)     pPrev;
-    /** Manager type */
-    PDMACEPFILEMGRTYPE                     enmMgrType;
-    /** Current state of the manager. */
-    PDMACEPFILEMGRSTATE                    enmState;
-    /** Event semaphore the manager sleeps on when waiting for new requests. */
-    RTSEMEVENT                             EventSem;
-    /** Flag whether the thread waits in the event semaphore. */
-    volatile bool                          fWaitingEventSem;
-    /** Thread data */
-    RTTHREAD                               Thread;
-    /** The async I/O context for this manager. */
-    RTFILEAIOCTX                           hAioCtx;
-    /** Flag whether the I/O manager was woken up. */
-    volatile bool                          fWokenUp;
-    /** List of endpoints assigned to this manager. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointsHead;
-    /** Number of endpoints assigned to the manager. */
-    unsigned                               cEndpoints;
-    /** Number of requests active currently. */
-    unsigned                               cRequestsActive;
-    /** Number of maximum requests active. */
-    uint32_t                               cRequestsActiveMax;
-    /** Pointer to an array of free async I/O request handles. */
-    RTFILEAIOREQ                          *pahReqsFree;
-    /** Index of the next free entry in the cache. */
-    uint32_t                               iFreeEntry;
-    /** Size of the array. */
-    unsigned                               cReqEntries;
-    /** Memory cache for file range locks. */
-    RTMEMCACHE                             hMemCacheRangeLocks;
-    /** Number of milliseconds to wait until the bandwidth is refreshed for at least
-     * one endpoint and it is possible to process more requests. */
-    RTMSINTERVAL                           msBwLimitExpired;
-    /** Critical section protecting the blocking event handling. */
-    RTCRITSECT                             CritSectBlockingEvent;
-    /** Event semaphore for blocking external events.
-     * The caller waits on it until the async I/O manager
-     * finished processing the event. */
-    RTSEMEVENT                             EventSemBlock;
-    /** Flag whether a blocking event is pending and needs
-     * processing by the I/O manager. */
-    volatile bool                          fBlockingEventPending;
-    /** Blocking event type */
-    volatile PDMACEPFILEAIOMGRBLOCKINGEVENT enmBlockingEvent;
-    /** Event type data */
-    union
-    {
-        /** Add endpoint event. */
-        struct
-        {
-            /** The endpoint to be added */
-            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
-        } AddEndpoint;
-        /** Remove endpoint event. */
-        struct
-        {
-            /** The endpoint to be removed */
-            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
-        } RemoveEndpoint;
-        /** Close endpoint event. */
-        struct
-        {
-            /** The endpoint to be closed */
-            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
-        } CloseEndpoint;
-    } BlockingEventData;
-} PDMACEPFILEMGR;
-/** Pointer to a async I/O manager state. */
-typedef PDMACEPFILEMGR *PPDMACEPFILEMGR;
-/** Pointer to a async I/O manager state pointer. */
-typedef PPDMACEPFILEMGR *PPPDMACEPFILEMGR;
-
-/**
- * A file access range lock.
- */
-typedef struct PDMACFILERANGELOCK
-{
-    /** AVL node in the locked range tree of the endpoint. */
-    AVLRFOFFNODECORE            Core;
-    /** How many tasks have locked this range. */
-    uint32_t                    cRefs;
-    /** Flag whether this is a read or write lock. */
-    bool                        fReadLock;
-    /** List of tasks which are waiting that the range gets unlocked. */
-    PPDMACTASKFILE              pWaitingTasksHead;
-    /** List of tasks which are waiting that the range gets unlocked. */
-    PPDMACTASKFILE              pWaitingTasksTail;
-} PDMACFILERANGELOCK, *PPDMACFILERANGELOCK;
-
-/**
- * Backend type for the endpoint.
- */
-typedef enum PDMACFILEEPBACKEND
-{
-    /** Non buffered. */
-    PDMACFILEEPBACKEND_NON_BUFFERED = 0,
-    /** Buffered (i.e host cache enabled) */
-    PDMACFILEEPBACKEND_BUFFERED,
-    /** 32bit hack */
-    PDMACFILEEPBACKEND_32BIT_HACK = 0x7fffffff
-} PDMACFILEEPBACKEND;
-/** Pointer to a backend type. */
-typedef PDMACFILEEPBACKEND *PPDMACFILEEPBACKEND;
-
-/**
- * Global data for the file endpoint class.
- */
-typedef struct PDMASYNCCOMPLETIONEPCLASSFILE
-{
-    /** Common data. */
-    PDMASYNCCOMPLETIONEPCLASS           Core;
-    /** Override I/O manager type - set to SIMPLE after failure. */
-    PDMACEPFILEMGRTYPE                  enmMgrTypeOverride;
-    /** Default backend type for the endpoint. */
-    PDMACFILEEPBACKEND                  enmEpBackendDefault;
-    RTCRITSECT                          CritSect;
-    /** Pointer to the head of the async I/O managers. */
-    R3PTRTYPE(PPDMACEPFILEMGR)          pAioMgrHead;
-    /** Number of async I/O managers currently running. */
-    unsigned                            cAioMgrs;
-    /** Maximum number of segments to cache per endpoint */
-    unsigned                            cTasksCacheMax;
-    /** Maximum number of simultaneous outstandingrequests. */
-    uint32_t                            cReqsOutstandingMax;
-    /** Bitmask for checking the alignment of a buffer. */
-    RTR3UINTPTR                         uBitmaskAlignment;
-    /** Flag whether the out of resources warning was printed already. */
-    bool                                fOutOfResourcesWarningPrinted;
-} PDMASYNCCOMPLETIONEPCLASSFILE;
-/** Pointer to the endpoint class data. */
-typedef PDMASYNCCOMPLETIONEPCLASSFILE *PPDMASYNCCOMPLETIONEPCLASSFILE;
-
-typedef enum PDMACEPFILEBLOCKINGEVENT
-{
-    /** The invalid event type */
-    PDMACEPFILEBLOCKINGEVENT_INVALID = 0,
-    /** A task is about to be canceled */
-    PDMACEPFILEBLOCKINGEVENT_CANCEL,
-    /** Usual 32bit hack */
-    PDMACEPFILEBLOCKINGEVENT_32BIT_HACK = 0x7fffffff
-} PDMACEPFILEBLOCKINGEVENT;
-
-/**
- * States of the endpoint.
- */
-typedef enum PDMASYNCCOMPLETIONENDPOINTFILESTATE
-{
-    /** Invalid state. */
-    PDMASYNCCOMPLETIONENDPOINTFILESTATE_INVALID = 0,
-    /** Normal running state accepting new requests
-     * and processing them.
-     */
-    PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
-    /** The endpoint is about to be closed - not accepting new tasks for endpoints but waiting for
-     *  remaining ones to finish.
-     */
-    PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING,
-    /** Removing from current I/O manager state - not processing new tasks for endpoints but waiting
-     * for remaining ones to finish.
-     */
-    PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING,
-    /** The current endpoint will be migrated to another I/O manager. */
-    PDMASYNCCOMPLETIONENDPOINTFILESTATE_MIGRATING,
-    /** 32bit hack */
-    PDMASYNCCOMPLETIONENDPOINTFILESTATE_32BIT_HACK = 0x7fffffff
-} PDMASYNCCOMPLETIONENDPOINTFILESTATE;
-
-/**
- * Data for the file endpoint.
- */
-typedef struct PDMASYNCCOMPLETIONENDPOINTFILE
-{
-    /** Common data. */
-    PDMASYNCCOMPLETIONENDPOINT             Core;
-    /** Current state of the endpoint. */
-    PDMASYNCCOMPLETIONENDPOINTFILESTATE    enmState;
-    /** The backend to use for this endpoint. */
-    PDMACFILEEPBACKEND                     enmBackendType;
-    /** async I/O manager this endpoint is assigned to. */
-    R3PTRTYPE(volatile PPDMACEPFILEMGR)    pAioMgr;
-    /** Flags for opening the file. */
-    unsigned                               fFlags;
-    /** File handle. */
-    RTFILE                                 File;
-    /**
-     * Real size of the file. Only updated if
-     * data is appended.
-     */
-    volatile uint64_t                      cbFile;
-    /** List of new tasks. */
-    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksNewHead;
-
-    /** Head of the small cache for allocated task segments for exclusive
-     * use by this endpoint. */
-    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksFreeHead;
-    /** Tail of the small cache for allocated task segments for exclusive
-     * use by this endpoint. */
-    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksFreeTail;
-    /** Number of elements in the cache. */
-    volatile uint32_t                      cTasksCached;
-
-    /** Flag whether a flush request is currently active */
-    PPDMACTASKFILE                         pFlushReq;
-
-#ifdef VBOX_WITH_STATISTICS
-    /** Time spend in a read. */
-    STAMPROFILEADV                         StatRead;
-    /** Time spend in a write. */
-    STAMPROFILEADV                         StatWrite;
-#endif
-
-    /** Event semaphore for blocking external events.
-     * The caller waits on it until the async I/O manager
-     * finished processing the event. */
-    RTSEMEVENT                             EventSemBlock;
-    /** Flag whether caching is enabled for this file. */
-    bool                                   fCaching;
-    /** Flag whether the file was opened readonly. */
-    bool                                   fReadonly;
-    /** Flag whether the host supports the async flush API. */
-    bool                                   fAsyncFlushSupported;
-#ifdef VBOX_WITH_DEBUGGER
-    /** Status code to inject for the next complete read. */
-    volatile int                           rcReqRead;
-    /** Status code to inject for the next complete write. */
-    volatile int                           rcReqWrite;
-#endif
-    /** Flag whether a blocking event is pending and needs
-     * processing by the I/O manager. */
-    bool                                   fBlockingEventPending;
-    /** Blocking event type */
-    PDMACEPFILEBLOCKINGEVENT               enmBlockingEvent;
-
-    /** Additional data needed for the event types. */
-    union
-    {
-        /** Cancelation event. */
-        struct
-        {
-            /** The task to cancel. */
-            PPDMACTASKFILE                 pTask;
-        } Cancel;
-    } BlockingEventData;
-    /** Data for exclusive use by the assigned async I/O manager. */
-    struct
-    {
-        /** Pointer to the next endpoint assigned to the manager. */
-        R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointNext;
-        /** Pointer to the previous endpoint assigned to the manager. */
-        R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointPrev;
-        /** List of pending requests (not submitted due to usage restrictions
-         *  or a pending flush request) */
-        R3PTRTYPE(PPDMACTASKFILE)                  pReqsPendingHead;
-        /** Tail of pending requests. */
-        R3PTRTYPE(PPDMACTASKFILE)                  pReqsPendingTail;
-        /** Tree of currently locked ranges.
-         * If a write task is enqueued the range gets locked and any other
-         * task writing to that range has to wait until the task completes.
-         */
-        PAVLRFOFFTREE                              pTreeRangesLocked;
-        /** Number of requests currently being processed for this endpoint
-         * (excluded flush requests). */
-        unsigned                                   cRequestsActive;
-        /** Number of requests processed during the last second. */
-        unsigned                                   cReqsPerSec;
-        /** Current number of processed requests for the current update period. */
-        unsigned                                   cReqsProcessed;
-        /** Flag whether the endpoint is about to be moved to another manager. */
-        bool                                       fMoving;
-        /** Destination I/O manager. */
-        PPDMACEPFILEMGR                            pAioMgrDst;
-    } AioMgr;
-} PDMASYNCCOMPLETIONENDPOINTFILE;
-/** Pointer to the endpoint class data. */
-typedef PDMASYNCCOMPLETIONENDPOINTFILE *PPDMASYNCCOMPLETIONENDPOINTFILE;
-#ifdef VBOX_WITH_STATISTICS
-AssertCompileMemberAlignment(PDMASYNCCOMPLETIONENDPOINTFILE, StatRead, sizeof(uint64_t));
-#endif
-
-/** Request completion function */
-typedef DECLCALLBACK(void)   FNPDMACTASKCOMPLETED(PPDMACTASKFILE pTask, void *pvUser, int rc);
-/** Pointer to a request completion function. */
-typedef FNPDMACTASKCOMPLETED *PFNPDMACTASKCOMPLETED;
-
-/**
- * Transfer type.
- */
-typedef enum PDMACTASKFILETRANSFER
-{
-    /** Invalid. */
-    PDMACTASKFILETRANSFER_INVALID = 0,
-    /** Read transfer. */
-    PDMACTASKFILETRANSFER_READ,
-    /** Write transfer. */
-    PDMACTASKFILETRANSFER_WRITE,
-    /** Flush transfer. */
-    PDMACTASKFILETRANSFER_FLUSH
-} PDMACTASKFILETRANSFER;
-
-/**
- * Data of a request.
- */
-typedef struct PDMACTASKFILE
-{
-    /** Pointer to the range lock we are waiting for */
-    PPDMACFILERANGELOCK                  pRangeLock;
-    /** Next task in the list. (Depending on the state) */
-    struct PDMACTASKFILE                *pNext;
-    /** Endpoint */
-    PPDMASYNCCOMPLETIONENDPOINTFILE      pEndpoint;
-    /** Transfer type. */
-    PDMACTASKFILETRANSFER                enmTransferType;
-    /** Start offset */
-    RTFOFF                               Off;
-    /** Data segment. */
-    RTSGSEG                              DataSeg;
-    /** When non-zero the segment uses a bounce buffer because the provided buffer
-     * doesn't meet host requirements. */
-    size_t                               cbBounceBuffer;
-    /** Pointer to the used bounce buffer if any. */
-    void                                *pvBounceBuffer;
-    /** Start offset in the bounce buffer to copy from. */
-    uint32_t                             offBounceBuffer;
-    /** Flag whether this is a prefetch request. */
-    bool                                 fPrefetch;
-    /** Already prepared native I/O request.
-     * Used if the request is prepared already but
-     * was not queued because the host has not enough
-     * resources. */
-    RTFILEAIOREQ                         hReq;
-    /** Completion function to call on completion. */
-    PFNPDMACTASKCOMPLETED                pfnCompleted;
-    /** User data */
-    void                                *pvUser;
-} PDMACTASKFILE;
-
-/**
- * Per task data.
- */
-typedef struct PDMASYNCCOMPLETIONTASKFILE
-{
-    /** Common data. */
-    PDMASYNCCOMPLETIONTASK Core;
-    /** Number of bytes to transfer until this task completes. */
-    volatile int32_t      cbTransferLeft;
-    /** Flag whether the task completed. */
-    volatile bool         fCompleted;
-    /** Return code. */
-    volatile int          rc;
-} PDMASYNCCOMPLETIONTASKFILE;
-
-int pdmacFileAioMgrFailsafe(RTTHREAD ThreadSelf, void *pvUser);
-int pdmacFileAioMgrNormal(RTTHREAD ThreadSelf, void *pvUser);
-
-int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr);
-void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr);
-
-int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr, PDMACEPFILEMGRTYPE enmMgrType);
-
-int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
-
-PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
-PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
-void pdmacFileTaskFree(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                       PPDMACTASKFILE pTask);
-
-int pdmacFileEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask);
-
-void pdmacFileEpTaskCompleted(PPDMACTASKFILE pTask, void *pvUser, int rc);
-
-int pdmacFileCacheInit(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile, PCFGMNODE pCfgNode);
-void pdmacFileCacheDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile);
-int pdmacFileEpCacheInit(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile);
-void pdmacFileEpCacheDestroy(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
-
-int pdmacFileEpCacheRead(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
-                         RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
-                         size_t cbRead);
-int pdmacFileEpCacheWrite(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
-                          RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
-                          size_t cbWrite);
-int pdmacFileEpCacheFlush(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
-
-RT_C_DECLS_END
-
-#endif
-
Index: unk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMAsyncCompletionFileNormal.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1705 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Async I/O - Transport data asynchronous in R3 using EMT.
- * Async File I/O manager.
- */
-
-/*
- * Copyright (C) 2006-2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
-#include <iprt/types.h>
-#include <iprt/asm.h>
-#include <iprt/file.h>
-#include <iprt/mem.h>
-#include <iprt/string.h>
-#include <iprt/assert.h>
-#include <VBox/log.h>
-
-#include "PDMAsyncCompletionFileInternal.h"
-
-/** The update period for the I/O load statistics in ms. */
-#define PDMACEPFILEMGR_LOAD_UPDATE_PERIOD 1000
-/** Maximum number of requests a manager will handle. */
-#define PDMACEPFILEMGR_REQS_STEP 512
-
-/*******************************************************************************
-*   Internal functions                                                         *
-*******************************************************************************/
-static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
-                                                PPDMACEPFILEMGR pAioMgr,
-                                                PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
-
-static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
-                                                         PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                                         PPDMACFILERANGELOCK pRangeLock);
-
-static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
-                                               int rc, size_t cbTransfered);
-
-int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr)
-{
-    int rc = VINF_SUCCESS;
-
-    pAioMgr->cRequestsActiveMax = PDMACEPFILEMGR_REQS_STEP;
-
-    rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, RTFILEAIO_UNLIMITED_REQS);
-    if (rc == VERR_OUT_OF_RANGE)
-        rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, pAioMgr->cRequestsActiveMax);
-
-    if (RT_SUCCESS(rc))
-    {
-        /* Initialize request handle array. */
-        pAioMgr->iFreeEntry       = 0;
-        pAioMgr->cReqEntries      = pAioMgr->cRequestsActiveMax;
-        pAioMgr->pahReqsFree      = (RTFILEAIOREQ *)RTMemAllocZ(pAioMgr->cReqEntries * sizeof(RTFILEAIOREQ));
-        pAioMgr->msBwLimitExpired = RT_INDEFINITE_WAIT;
-
-        if (pAioMgr->pahReqsFree)
-        {
-            /* Create the range lock memcache. */
-            rc = RTMemCacheCreate(&pAioMgr->hMemCacheRangeLocks, sizeof(PDMACFILERANGELOCK),
-                                  0, UINT32_MAX, NULL, NULL, NULL, 0);
-            if (RT_SUCCESS(rc))
-                return VINF_SUCCESS;
-
-            RTMemFree(pAioMgr->pahReqsFree);
-        }
-        else
-        {
-            RTFileAioCtxDestroy(pAioMgr->hAioCtx);
-            rc = VERR_NO_MEMORY;
-        }
-    }
-
-    return rc;
-}
-
-void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr)
-{
-    RTFileAioCtxDestroy(pAioMgr->hAioCtx);
-
-    while (pAioMgr->iFreeEntry > 0)
-    {
-        pAioMgr->iFreeEntry--;
-        Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] != NIL_RTFILEAIOREQ);
-        RTFileAioReqDestroy(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry]);
-    }
-
-    RTMemFree(pAioMgr->pahReqsFree);
-    RTMemCacheDestroy(pAioMgr->hMemCacheRangeLocks);
-}
-
-/**
- * Sorts the endpoint list with insertion sort.
- */
-static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
-{
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
-
-    pEpPrev = pAioMgr->pEndpointsHead;
-    pEpCurr = pEpPrev->AioMgr.pEndpointNext;
-
-    while (pEpCurr)
-    {
-        /* Remember the next element to sort because the list might change. */
-        pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
-
-        /* Unlink the current element from the list. */
-        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
-        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
-
-        if (pPrev)
-            pPrev->AioMgr.pEndpointNext = pNext;
-        else
-            pAioMgr->pEndpointsHead = pNext;
-
-        if (pNext)
-            pNext->AioMgr.pEndpointPrev = pPrev;
-
-        /* Go back until we reached the place to insert the current endpoint into. */
-        while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
-            pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
-
-        /* Link the endpoint into the list. */
-        if (pEpPrev)
-            pNext = pEpPrev->AioMgr.pEndpointNext;
-        else
-            pNext = pAioMgr->pEndpointsHead;
-
-        pEpCurr->AioMgr.pEndpointNext = pNext;
-        pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
-
-        if (pNext)
-            pNext->AioMgr.pEndpointPrev = pEpCurr;
-
-        if (pEpPrev)
-            pEpPrev->AioMgr.pEndpointNext = pEpCurr;
-        else
-            pAioMgr->pEndpointsHead = pEpCurr;
-
-        pEpCurr = pEpNextToSort;
-    }
-
-#ifdef DEBUG
-    /* Validate sorting algorithm */
-    unsigned cEndpoints = 0;
-    pEpCurr = pAioMgr->pEndpointsHead;
-
-    AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
-    AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
-
-    while (pEpCurr)
-    {
-        cEndpoints++;
-
-        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
-        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
-
-        Assert(!pNext || pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
-        Assert(!pPrev || pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
-
-        pEpCurr = pNext;
-    }
-
-    AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
-
-#endif
-}
-
-/**
- * Removes an endpoint from the currently assigned manager.
- *
- * @returns TRUE if there are still requests pending on the current manager for this endpoint.
- *          FALSE otherwise.
- * @param   pEndpointRemove    The endpoint to remove.
- */
-static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
-{
-    PPDMASYNCCOMPLETIONENDPOINTFILE pPrev   = pEndpointRemove->AioMgr.pEndpointPrev;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pNext   = pEndpointRemove->AioMgr.pEndpointNext;
-    PPDMACEPFILEMGR                 pAioMgr = pEndpointRemove->pAioMgr;
-
-    pAioMgr->cEndpoints--;
-
-    if (pPrev)
-        pPrev->AioMgr.pEndpointNext = pNext;
-    else
-        pAioMgr->pEndpointsHead = pNext;
-
-    if (pNext)
-        pNext->AioMgr.pEndpointPrev = pPrev;
-
-    /* Make sure that there is no request pending on this manager for the endpoint. */
-    if (!pEndpointRemove->AioMgr.cRequestsActive)
-    {
-        Assert(!pEndpointRemove->pFlushReq);
-
-        /* Reopen the file so that the new endpoint can re-associate with the file */
-        RTFileClose(pEndpointRemove->File);
-        int rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
-        AssertRC(rc);
-        return false;
-    }
-
-    return true;
-}
-
-static bool pdmacFileAioMgrNormalIsBalancePossible(PPDMACEPFILEMGR pAioMgr)
-{
-    /* Balancing doesn't make sense with only one endpoint. */
-    if (pAioMgr->cEndpoints == 1)
-        return false;
-
-    /* Doesn't make sens to move endpoints if only one produces the whole load */
-    unsigned cEndpointsWithLoad = 0;
-
-    PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
-
-    while (pCurr)
-    {
-        if (pCurr->AioMgr.cReqsPerSec)
-            cEndpointsWithLoad++;
-
-        pCurr = pCurr->AioMgr.pEndpointNext;
-    }
-
-    return (cEndpointsWithLoad > 1);
-}
-
-/**
- * Creates a new I/O manager and spreads the I/O load of the endpoints
- * between the given I/O manager and the new one.
- *
- * @returns nothing.
- * @param   pAioMgr    The I/O manager with high I/O load.
- */
-static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
-{
-    PPDMACEPFILEMGR pAioMgrNew = NULL;
-    int rc = VINF_SUCCESS;
-
-    /*
-     * Check if balancing would improve the situation.
-     */
-    if (pdmacFileAioMgrNormalIsBalancePossible(pAioMgr))
-    {
-        PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
-
-        rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgrNew, PDMACEPFILEMGRTYPE_ASYNC);
-        if (RT_SUCCESS(rc))
-        {
-            /* We will sort the list by request count per second. */
-            pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
-
-            /* Now move some endpoints to the new manager. */
-            unsigned cReqsHere  = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
-            unsigned cReqsOther = 0;
-            PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
-
-            while (pCurr)
-            {
-                if (cReqsHere <= cReqsOther)
-                {
-                    /*
-                     * The other manager has more requests to handle now.
-                     * We will keep the current endpoint.
-                     */
-                    Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
-                    cReqsHere += pCurr->AioMgr.cReqsPerSec;
-                    pCurr = pCurr->AioMgr.pEndpointNext;
-                }
-                else
-                {
-                    /* Move to other endpoint. */
-                    Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr, pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
-                    cReqsOther += pCurr->AioMgr.cReqsPerSec;
-
-                    PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
-
-                    pCurr = pCurr->AioMgr.pEndpointNext;
-
-                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
-
-                    if (fReqsPending)
-                    {
-                        pMove->enmState          = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
-                        pMove->AioMgr.fMoving    = true;
-                        pMove->AioMgr.pAioMgrDst = pAioMgrNew;
-                    }
-                    else
-                    {
-                        pMove->AioMgr.fMoving    = false;
-                        pMove->AioMgr.pAioMgrDst = NULL;
-                        pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
-                    }
-                }
-            }
-        }
-        else
-        {
-            /* Don't process further but leave a log entry about reduced performance. */
-            LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
-        }
-    }
-    else
-        Log(("AIOMgr: Load balancing would not improve anything\n"));
-}
-
-/**
- * Increase the maximum number of active requests for the given I/O manager.
- *
- * @returns VBox status code.
- * @param   pAioMgr    The I/O manager to grow.
- */
-static int pdmacFileAioMgrNormalGrow(PPDMACEPFILEMGR pAioMgr)
-{
-    int rc = VINF_SUCCESS;
-    RTFILEAIOCTX hAioCtxNew = NIL_RTFILEAIOCTX;
-
-    LogFlowFunc(("pAioMgr=%#p\n", pAioMgr));
-
-    AssertMsg(    pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING
-              && !pAioMgr->cRequestsActive,
-              ("Invalid state of the I/O manager\n"));
-
-#ifdef RT_OS_WINDOWS
-    /*
-     * Reopen the files of all assigned endpoints first so we can assign them to the new
-     * I/O context.
-     */
-    PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
-
-    while (pCurr)
-    {
-        RTFileClose(pCurr->File);
-        rc = RTFileOpen(&pCurr->File, pCurr->Core.pszUri, pCurr->fFlags);
-        AssertRC(rc);
-
-        pCurr = pCurr->AioMgr.pEndpointNext;
-    }
-#endif
-
-    /* Create the new bigger context. */
-    pAioMgr->cRequestsActiveMax += PDMACEPFILEMGR_REQS_STEP;
-
-    rc = RTFileAioCtxCreate(&hAioCtxNew, RTFILEAIO_UNLIMITED_REQS);
-    if (rc == VERR_OUT_OF_RANGE)
-        rc = RTFileAioCtxCreate(&hAioCtxNew, pAioMgr->cRequestsActiveMax);
-
-    if (RT_SUCCESS(rc))
-    {
-        /* Close the old context. */
-        rc = RTFileAioCtxDestroy(pAioMgr->hAioCtx);
-        AssertRC(rc);
-
-        pAioMgr->hAioCtx = hAioCtxNew;
-
-        /* Create a new I/O task handle array */
-        uint32_t cReqEntriesNew = pAioMgr->cRequestsActiveMax + 1;
-        RTFILEAIOREQ *pahReqNew = (RTFILEAIOREQ *)RTMemAllocZ(cReqEntriesNew * sizeof(RTFILEAIOREQ));
-
-        if (pahReqNew)
-        {
-            /* Copy the cached request handles. */
-            for (uint32_t iReq = 0; iReq < pAioMgr->cReqEntries; iReq++)
-                pahReqNew[iReq] = pAioMgr->pahReqsFree[iReq];
-
-            RTMemFree(pAioMgr->pahReqsFree);
-            pAioMgr->pahReqsFree = pahReqNew;
-            pAioMgr->cReqEntries = cReqEntriesNew;
-            LogFlowFunc(("I/O manager increased to handle a maximum of %u requests\n",
-                         pAioMgr->cRequestsActiveMax));
-        }
-        else
-            rc = VERR_NO_MEMORY;
-    }
-
-#ifdef RT_OS_WINDOWS
-    /* Assign the file to the new context. */
-    pCurr = pAioMgr->pEndpointsHead;
-
-    while (pCurr)
-    {
-        rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pCurr->File);
-        AssertRC(rc);
-
-        pCurr = pCurr->AioMgr.pEndpointNext;
-    }
-#endif
-
-    if (RT_FAILURE(rc))
-    {
-        LogFlow(("Increasing size of the I/O manager failed with rc=%Rrc\n", rc));
-        pAioMgr->cRequestsActiveMax -= PDMACEPFILEMGR_REQS_STEP;
-    }
-
-    pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
-    LogFlowFunc(("returns rc=%Rrc\n", rc));
-
-    return rc;
-}
-
-/**
- * Checks if a given status code is fatal.
- * Non fatal errors can be fixed by migrating the endpoint to a
- * failsafe manager.
- *
- * @returns true If the error is fatal and migrating to a failsafe manager doesn't help
- *          false If the error can be fixed by a migration. (image on NFS disk for example)
- * @param   rcReq    The status code to check.
- */
-DECLINLINE(bool) pdmacFileAioMgrNormalRcIsFatal(int rcReq)
-{
-    return    rcReq == VERR_DEV_IO_ERROR
-           || rcReq == VERR_FILE_IO_ERROR
-           || rcReq == VERR_DISK_IO_ERROR
-           || rcReq == VERR_DISK_FULL
-           || rcReq == VERR_FILE_TOO_BIG;
-}
-
-/**
- * Error handler which will create the failsafe managers and destroy the failed I/O manager.
- *
- * @returns VBox status code
- * @param   pAioMgr    The I/O manager the error occurred on.
- * @param   rc         The error code.
- */
-static int pdmacFileAioMgrNormalErrorHandler(PPDMACEPFILEMGR pAioMgr, int rc, RT_SRC_POS_DECL)
-{
-    LogRel(("AIOMgr: I/O manager %#p encountered a critical error (rc=%Rrc) during operation. Falling back to failsafe mode. Expect reduced performance\n",
-            pAioMgr, rc));
-    LogRel(("AIOMgr: Error happened in %s:(%u){%s}\n", RT_SRC_POS_ARGS));
-    LogRel(("AIOMgr: Please contact the product vendor\n"));
-
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
-
-    pAioMgr->enmState = PDMACEPFILEMGRSTATE_FAULT;
-    ASMAtomicWriteU32((volatile uint32_t *)&pEpClassFile->enmMgrTypeOverride, PDMACEPFILEMGRTYPE_SIMPLE);
-
-    AssertMsgFailed(("Implement\n"));
-    return VINF_SUCCESS;
-}
-
-/**
- * Put a list of tasks in the pending request list of an endpoint.
- */
-DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
-{
-    /* Add the rest of the tasks to the pending list */
-    if (!pEndpoint->AioMgr.pReqsPendingHead)
-    {
-        Assert(!pEndpoint->AioMgr.pReqsPendingTail);
-        pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
-    }
-    else
-    {
-        Assert(pEndpoint->AioMgr.pReqsPendingTail);
-        pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
-    }
-
-    /* Update the tail. */
-    while (pTaskHead->pNext)
-        pTaskHead = pTaskHead->pNext;
-
-    pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
-    pTaskHead->pNext = NULL;
-}
-
-/**
- * Put one task in the pending request list of an endpoint.
- */
-DECLINLINE(void) pdmacFileAioMgrEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
-{
-    /* Add the rest of the tasks to the pending list */
-    if (!pEndpoint->AioMgr.pReqsPendingHead)
-    {
-        Assert(!pEndpoint->AioMgr.pReqsPendingTail);
-        pEndpoint->AioMgr.pReqsPendingHead = pTask;
-    }
-    else
-    {
-        Assert(pEndpoint->AioMgr.pReqsPendingTail);
-        pEndpoint->AioMgr.pReqsPendingTail->pNext = pTask;
-    }
-
-    pEndpoint->AioMgr.pReqsPendingTail = pTask;
-    pTask->pNext = NULL;
-}
-
-/**
- * Allocates a async I/O request.
- *
- * @returns Handle to the request.
- * @param   pAioMgr    The I/O manager.
- */
-static RTFILEAIOREQ pdmacFileAioMgrNormalRequestAlloc(PPDMACEPFILEMGR pAioMgr)
-{
-    RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
-
-    /* Get a request handle. */
-    if (pAioMgr->iFreeEntry > 0)
-    {
-        pAioMgr->iFreeEntry--;
-        hReq = pAioMgr->pahReqsFree[pAioMgr->iFreeEntry];
-        pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = NIL_RTFILEAIOREQ;
-        Assert(hReq != NIL_RTFILEAIOREQ);
-    }
-    else
-    {
-        int rc = RTFileAioReqCreate(&hReq);
-        AssertRC(rc);
-    }
-
-    return hReq;
-}
-
-/**
- * Frees a async I/O request handle.
- *
- * @returns nothing.
- * @param   pAioMgr    The I/O manager.
- * @param   hReq       The I/O request handle to free.
- */
-static void pdmacFileAioMgrNormalRequestFree(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
-{
-    Assert(pAioMgr->iFreeEntry < pAioMgr->cReqEntries);
-    Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] == NIL_RTFILEAIOREQ);
-
-    pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = hReq;
-    pAioMgr->iFreeEntry++;
-}
-
-/**
- * Wrapper around RTFIleAioCtxSubmit() which is also doing error handling.
- */
-static int pdmacFileAioMgrNormalReqsEnqueue(PPDMACEPFILEMGR pAioMgr,
-                                            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                            PRTFILEAIOREQ pahReqs, unsigned cReqs)
-{
-    int rc;
-
-    pAioMgr->cRequestsActive += cReqs;
-    pEndpoint->AioMgr.cRequestsActive += cReqs;
-
-    LogFlow(("Enqueuing %d requests. I/O manager has a total of %d active requests now\n", cReqs, pAioMgr->cRequestsActive));
-    LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
-
-    rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, pahReqs, cReqs);
-    if (RT_FAILURE(rc))
-    {
-        if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
-        {
-            PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
-
-            /* Append any not submitted task to the waiting list. */
-            for (size_t i = 0; i < cReqs; i++)
-            {
-                int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
-
-                if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
-                {
-                    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
-
-                    Assert(pTask->hReq == pahReqs[i]);
-                    pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
-                    pAioMgr->cRequestsActive--;
-                    pEndpoint->AioMgr.cRequestsActive--;
-
-                    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
-                    {
-                        /* Clear the pending flush */
-                        Assert(pEndpoint->pFlushReq == pTask);
-                        pEndpoint->pFlushReq = NULL;
-                    }
-                }
-            }
-
-            pAioMgr->cRequestsActiveMax = pAioMgr->cRequestsActive;
-
-            /* Print an entry in the release log */
-            if (RT_UNLIKELY(!pEpClass->fOutOfResourcesWarningPrinted))
-            {
-                pEpClass->fOutOfResourcesWarningPrinted = true;
-                LogRel(("AIOMgr: Host limits number of active IO requests to %u. Expect a performance impact.\n",
-                        pAioMgr->cRequestsActive));
-            }
-
-            LogFlow(("Removed requests. I/O manager has a total of %u active requests now\n", pAioMgr->cRequestsActive));
-            LogFlow(("Endpoint has a total of %u active requests now\n", pEndpoint->AioMgr.cRequestsActive));
-            rc = VINF_SUCCESS;
-        }
-        else /* Another kind of error happened (full disk, ...) */
-        {
-            /* An error happened. Find out which one caused the error and resubmit all other tasks. */
-            for (size_t i = 0; i < cReqs; i++)
-            {
-                int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
-
-                if (rcReq == VERR_FILE_AIO_NOT_SUBMITTED)
-                {
-                    /* We call ourself again to do any error handling which might come up now. */
-                    rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &pahReqs[i], 1);
-                    AssertRC(rc);
-                }
-                else if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
-                {
-                    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
-
-                    pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, pahReqs[i], rcReq, 0);
-                }
-            }
-
-
-            if (    pEndpoint->pFlushReq
-                && !pAioMgr->cRequestsActive
-                && !pEndpoint->fAsyncFlushSupported)
-            {
-                /*
-                 * Complete a pending flush if we don't have requests enqueued and the host doesn't support
-                 * the async flush API.
-                 * Happens only if this we just noticed that this is not supported
-                 * and the only active request was a flush.
-                 */
-                PPDMACTASKFILE pFlush = pEndpoint->pFlushReq;
-                pEndpoint->pFlushReq = NULL;
-                pFlush->pfnCompleted(pFlush, pFlush->pvUser, VINF_SUCCESS);
-                pdmacFileTaskFree(pEndpoint, pFlush);
-            }
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-static bool pdmacFileAioMgrNormalIsRangeLocked(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                               RTFOFF offStart, size_t cbRange,
-                                               PPDMACTASKFILE pTask)
-{
-    PPDMACFILERANGELOCK pRangeLock = NULL; /** < Range lock */
-
-    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
-              || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ,
-                 ("Invalid task type %d\n", pTask->enmTransferType));
-
-    pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetRangeGet(pEndpoint->AioMgr.pTreeRangesLocked, offStart);
-    if (!pRangeLock)
-    {
-        pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGetBestFit(pEndpoint->AioMgr.pTreeRangesLocked, offStart, true);
-        /* Check if we intersect with the range. */
-        if (   !pRangeLock
-            || !(   (pRangeLock->Core.Key) <= (offStart + (RTFOFF)cbRange - 1)
-                && (pRangeLock->Core.KeyLast) >= offStart))
-        {
-            pRangeLock = NULL; /* False alarm */
-        }
-    }
-
-    /* Check whether we have one of the situations explained below */
-    if (   pRangeLock
-#if 0 /** @todo: later. For now we will just block all requests if they interfere */
-        && (   (pRangeLock->fReadLock && pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
-            || (!pRangeLock->fReadLock)
-#endif
-        )
-    {
-        /* Add to the list. */
-        pTask->pNext = NULL;
-
-        if (!pRangeLock->pWaitingTasksHead)
-        {
-            Assert(!pRangeLock->pWaitingTasksTail);
-            pRangeLock->pWaitingTasksHead = pTask;
-            pRangeLock->pWaitingTasksTail = pTask;
-        }
-        else
-        {
-            AssertPtr(pRangeLock->pWaitingTasksTail);
-            pRangeLock->pWaitingTasksTail->pNext = pTask;
-            pRangeLock->pWaitingTasksTail = pTask;
-        }
-        return true;
-    }
-
-    return false;
-}
-
-static int pdmacFileAioMgrNormalRangeLock(PPDMACEPFILEMGR pAioMgr,
-                                          PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                          RTFOFF offStart, size_t cbRange,
-                                          PPDMACTASKFILE pTask)
-{
-    AssertMsg(!pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbRange, pTask),
-              ("Range is already locked offStart=%RTfoff cbRange=%u\n",
-               offStart, cbRange));
-
-    PPDMACFILERANGELOCK pRangeLock = (PPDMACFILERANGELOCK)RTMemCacheAlloc(pAioMgr->hMemCacheRangeLocks);
-    if (!pRangeLock)
-        return VERR_NO_MEMORY;
-
-    /* Init the lock. */
-    pRangeLock->Core.Key          = offStart;
-    pRangeLock->Core.KeyLast      = offStart + cbRange - 1;
-    pRangeLock->cRefs             = 1;
-    pRangeLock->fReadLock         = pTask->enmTransferType == PDMACTASKFILETRANSFER_READ;
-    pRangeLock->pWaitingTasksHead = NULL;
-    pRangeLock->pWaitingTasksTail = NULL;
-
-    bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->AioMgr.pTreeRangesLocked, &pRangeLock->Core);
-    AssertMsg(fInserted, ("Range lock was not inserted!\n"));
-
-    /* Let the task point to its lock. */
-    pTask->pRangeLock = pRangeLock;
-
-    return VINF_SUCCESS;
-}
-
-static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
-                                                         PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                                         PPDMACFILERANGELOCK pRangeLock)
-{
-    PPDMACTASKFILE pTasksWaitingHead;
-
-    AssertPtr(pRangeLock);
-    Assert(pRangeLock->cRefs == 1);
-
-    RTAvlrFileOffsetRemove(pEndpoint->AioMgr.pTreeRangesLocked, pRangeLock->Core.Key);
-    pTasksWaitingHead = pRangeLock->pWaitingTasksHead;
-    pRangeLock->pWaitingTasksHead = NULL;
-    pRangeLock->pWaitingTasksTail = NULL;
-    RTMemCacheFree(pAioMgr->hMemCacheRangeLocks, pRangeLock);
-
-    return pTasksWaitingHead;
-}
-
-static int pdmacFileAioMgrNormalTaskPrepareBuffered(PPDMACEPFILEMGR pAioMgr,
-                                                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                                    PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
-{
-    int rc = VINF_SUCCESS;
-    RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
-    void *pvBuf = pTask->DataSeg.pvSeg;
-
-    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
-                || (uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) <= pEndpoint->cbFile,
-                ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
-                pTask->Off, pTask->DataSeg.cbSeg, pEndpoint->cbFile));
-
-    pTask->fPrefetch = false;
-    pTask->cbBounceBuffer = 0;
-
-    /*
-     * Before we start to setup the request we have to check whether there is a task
-     * already active which range intersects with ours. We have to defer execution
-     * of this task in two cases:
-     *     - The pending task is a write and the current is either read or write
-     *     - The pending task is a read and the current task is a write task.
-     *
-     * To check whether a range is currently "locked" we use the AVL tree where every pending task
-     * is stored by its file offset range. The current task will be added to the active task
-     * and will be executed when the active one completes. (The method below
-     * which checks whether a range is already used will add the task)
-     *
-     * This is necessary because of the requirement to align all requests to a 512 boundary
-     * which is enforced by the host OS (Linux and Windows atm). It is possible that
-     * we have to process unaligned tasks and need to align them using bounce buffers.
-     * While the data is fetched from the file another request might arrive writing to
-     * the same range. This will result in data corruption if both are executed concurrently.
-     */
-    bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, pTask->Off, pTask->DataSeg.cbSeg, pTask);
-
-    if (!fLocked)
-    {
-        /* Get a request handle. */
-        hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
-        AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
-
-        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
-        {
-            /* Grow the file if needed. */
-            if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
-            {
-                ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
-                RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
-            }
-
-            rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
-                                          pTask->Off, pTask->DataSeg.pvSeg,
-                                          pTask->DataSeg.cbSeg, pTask);
-        }
-        else
-            rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
-                                         pTask->Off, pTask->DataSeg.pvSeg,
-                                         pTask->DataSeg.cbSeg, pTask);
-        AssertRC(rc);
-
-        rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, pTask->Off,
-                                            pTask->DataSeg.cbSeg,
-                                            pTask);
-
-        if (RT_SUCCESS(rc))
-        {
-            pTask->hReq = hReq;
-            *phReq = hReq;
-        }
-    }
-    else
-        LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
-
-    return rc;
-}
-
-static int pdmacFileAioMgrNormalTaskPrepareNonBuffered(PPDMACEPFILEMGR pAioMgr,
-                                                       PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
-                                                       PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
-{
-    int rc = VINF_SUCCESS;
-    RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
-    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
-    void *pvBuf = pTask->DataSeg.pvSeg;
-
-    /*
-     * Check if the alignment requirements are met.
-     * Offset, transfer size and buffer address
-     * need to be on a 512 boundary.
-     */
-    RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
-    size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
-    PDMACTASKFILETRANSFER enmTransferType = pTask->enmTransferType;
-
-    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
-                || (uint64_t)(offStart + cbToTransfer) <= pEndpoint->cbFile,
-                ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
-                offStart, cbToTransfer, pEndpoint->cbFile));
-
-    pTask->fPrefetch = false;
-
-    /*
-     * Before we start to setup the request we have to check whether there is a task
-     * already active which range intersects with ours. We have to defer execution
-     * of this task in two cases:
-     *     - The pending task is a write and the current is either read or write
-     *     - The pending task is a read and the current task is a write task.
-     *
-     * To check whether a range is currently "locked" we use the AVL tree where every pending task
-     * is stored by its file offset range. The current task will be added to the active task
-     * and will be executed when the active one completes. (The method below
-     * which checks whether a range is already used will add the task)
-     *
-     * This is necessary because of the requirement to align all requests to a 512 boundary
-     * which is enforced by the host OS (Linux and Windows atm). It is possible that
-     * we have to process unaligned tasks and need to align them using bounce buffers.
-     * While the data is fetched from the file another request might arrive writing to
-     * the same range. This will result in data corruption if both are executed concurrently.
-     */
-    bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbToTransfer, pTask);
-
-    if (!fLocked)
-    {
-        /* Get a request handle. */
-        hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
-        AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
-
-        if (   RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
-            || RT_UNLIKELY(offStart != pTask->Off)
-            || ((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) != (RTR3UINTPTR)pvBuf))
-        {
-            LogFlow(("Using bounce buffer for task %#p cbToTransfer=%zd cbSeg=%zd offStart=%RTfoff off=%RTfoff\n",
-                     pTask, cbToTransfer, pTask->DataSeg.cbSeg, offStart, pTask->Off));
-
-            /* Create bounce buffer. */
-            pTask->cbBounceBuffer = cbToTransfer;
-
-            AssertMsg(pTask->Off >= offStart, ("Overflow in calculation Off=%llu offStart=%llu\n",
-                      pTask->Off, offStart));
-            pTask->offBounceBuffer = pTask->Off - offStart;
-
-            /** @todo: I think we need something like a RTMemAllocAligned method here.
-             * Current assumption is that the maximum alignment is 4096byte
-             * (GPT disk on Windows)
-             * so we can use RTMemPageAlloc here.
-             */
-            pTask->pvBounceBuffer = RTMemPageAlloc(cbToTransfer);
-            if (RT_LIKELY(pTask->pvBounceBuffer))
-            {
-                pvBuf = pTask->pvBounceBuffer;
-
-                if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
-                {
-                    if (   RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
-                        || RT_UNLIKELY(offStart != pTask->Off))
-                    {
-                        /* We have to fill the buffer first before we can update the data. */
-                        LogFlow(("Prefetching data for task %#p\n", pTask));
-                        pTask->fPrefetch = true;
-                        enmTransferType = PDMACTASKFILETRANSFER_READ;
-                    }
-                    else
-                        memcpy(pvBuf, pTask->DataSeg.pvSeg, pTask->DataSeg.cbSeg);
-                }
-            }
-            else
-                rc = VERR_NO_MEMORY;
-        }
-        else
-            pTask->cbBounceBuffer = 0;
-
-        if (RT_SUCCESS(rc))
-        {
-            AssertMsg((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) == (RTR3UINTPTR)pvBuf,
-                      ("AIO: Alignment restrictions not met! pvBuf=%p uBitmaskAlignment=%p\n", pvBuf, pEpClassFile->uBitmaskAlignment));
-
-            if (enmTransferType == PDMACTASKFILETRANSFER_WRITE)
-            {
-                /* Grow the file if needed. */
-                if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
-                {
-                    ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
-                    RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
-                }
-
-                rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
-                                              offStart, pvBuf, cbToTransfer, pTask);
-            }
-            else
-                rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
-                                             offStart, pvBuf, cbToTransfer, pTask);
-            AssertRC(rc);
-
-            rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, offStart, cbToTransfer, pTask);
-
-            if (RT_SUCCESS(rc))
-            {
-                pTask->hReq = hReq;
-                *phReq = hReq;
-            }
-            else
-            {
-                /* Cleanup */
-                if (pTask->cbBounceBuffer)
-                    RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
-            }
-        }
-    }
-    else
-        LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
-
-    return rc;
-}
-
-static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
-                                                PPDMACEPFILEMGR pAioMgr,
-                                                PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    RTFILEAIOREQ  apReqs[20];
-    unsigned      cRequests = 0;
-    unsigned      cMaxRequests = pAioMgr->cRequestsActiveMax - pAioMgr->cRequestsActive;
-    int           rc = VINF_SUCCESS;
-
-    AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
-              ("Trying to process request lists of a non active endpoint!\n"));
-
-    /* Go through the list and queue the requests until we get a flush request */
-    while (   pTaskHead
-           && !pEndpoint->pFlushReq
-           && (pAioMgr->cRequestsActive + cRequests < pAioMgr->cRequestsActiveMax)
-           && RT_SUCCESS(rc))
-    {
-        RTMSINTERVAL msWhenNext;
-        PPDMACTASKFILE pCurr = pTaskHead;
-
-        if (!pdmacEpIsTransferAllowed(&pEndpoint->Core, (uint32_t)pCurr->DataSeg.cbSeg, &msWhenNext))
-        {
-            pAioMgr->msBwLimitExpired = RT_MIN(pAioMgr->msBwLimitExpired, msWhenNext);
-            break;
-        }
-
-        pTaskHead = pTaskHead->pNext;
-
-        pCurr->pNext = NULL;
-
-        AssertMsg(VALID_PTR(pCurr->pEndpoint) && (pCurr->pEndpoint == pEndpoint),
-                  ("Endpoints do not match\n"));
-
-        switch (pCurr->enmTransferType)
-        {
-            case PDMACTASKFILETRANSFER_FLUSH:
-            {
-                /* If there is no data transfer request this flush request finished immediately. */
-                if (pEndpoint->fAsyncFlushSupported)
-                {
-                    /* Issue a flush to the host. */
-                    RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
-                    AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
-
-                    LogFlow(("Flush request %#p\n", hReq));
-
-                    rc = RTFileAioReqPrepareFlush(hReq, pEndpoint->File, pCurr);
-                    if (RT_FAILURE(rc))
-                    {
-                        pEndpoint->fAsyncFlushSupported = false;
-                        pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
-                        rc = VINF_SUCCESS; /* Fake success */
-                    }
-                    else
-                    {
-                        pCurr->hReq = hReq;
-                        apReqs[cRequests] = hReq;
-                        pEndpoint->AioMgr.cReqsProcessed++;
-                        cRequests++;
-                    }
-                }
-
-                if (   !pEndpoint->AioMgr.cRequestsActive
-                    && !pEndpoint->fAsyncFlushSupported)
-                {
-                    pCurr->pfnCompleted(pCurr, pCurr->pvUser, VINF_SUCCESS);
-                    pdmacFileTaskFree(pEndpoint, pCurr);
-                }
-                else
-                {
-                    Assert(!pEndpoint->pFlushReq);
-                    pEndpoint->pFlushReq = pCurr;
-                }
-                break;
-            }
-            case PDMACTASKFILETRANSFER_READ:
-            case PDMACTASKFILETRANSFER_WRITE:
-            {
-                RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
-
-                if (pCurr->hReq == NIL_RTFILEAIOREQ)
-                {
-                    if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
-                        rc = pdmacFileAioMgrNormalTaskPrepareBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
-                    else if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
-                        rc = pdmacFileAioMgrNormalTaskPrepareNonBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
-                    else
-                        AssertMsgFailed(("Invalid backend type %d\n", pEndpoint->enmBackendType));
-
-                    AssertRC(rc);
-                }
-                else
-                {
-                    LogFlow(("Task %#p has I/O request %#p already\n", pCurr, pCurr->hReq));
-                    hReq = pCurr->hReq;
-                }
-
-                LogFlow(("Read/Write request %#p\n", hReq));
-
-                if (hReq != NIL_RTFILEAIOREQ)
-                {
-                    apReqs[cRequests] = hReq;
-                    cRequests++;
-                }
-                break;
-            }
-            default:
-                AssertMsgFailed(("Invalid transfer type %d\n", pCurr->enmTransferType));
-        } /* switch transfer type */
-
-        /* Queue the requests if the array is full. */
-        if (cRequests == RT_ELEMENTS(apReqs))
-        {
-            rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
-            cRequests = 0;
-            AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
-                      ("Unexpected return code\n"));
-        }
-    }
-
-    if (cRequests)
-    {
-        rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
-        AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
-                  ("Unexpected return code rc=%Rrc\n", rc));
-    }
-
-    if (pTaskHead)
-    {
-        /* Add the rest of the tasks to the pending list */
-        pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead);
-
-        if (RT_UNLIKELY(   pAioMgr->cRequestsActiveMax == pAioMgr->cRequestsActive
-                        && !pEndpoint->pFlushReq))
-        {
-#if 0
-            /*
-             * The I/O manager has no room left for more requests
-             * but there are still requests to process.
-             * Create a new I/O manager and let it handle some endpoints.
-             */
-            pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
-#else
-            /* Grow the I/O manager */
-            pAioMgr->enmState = PDMACEPFILEMGRSTATE_GROWING;
-#endif
-        }
-    }
-
-    /* Insufficient resources are not fatal. */
-    if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
-        rc = VINF_SUCCESS;
-
-    return rc;
-}
-
-/**
- * Adds all pending requests for the given endpoint
- * until a flush request is encountered or there is no
- * request anymore.
- *
- * @returns VBox status code.
- * @param   pAioMgr    The async I/O manager for the endpoint
- * @param   pEndpoint  The endpoint to get the requests from.
- */
-static int pdmacFileAioMgrNormalQueueReqs(PPDMACEPFILEMGR pAioMgr,
-                                          PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
-{
-    int rc = VINF_SUCCESS;
-    PPDMACTASKFILE pTasksHead = NULL;
-
-    AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
-              ("Trying to process request lists of a non active endpoint!\n"));
-
-    Assert(!pEndpoint->pFlushReq);
-
-    /* Check the pending list first */
-    if (pEndpoint->AioMgr.pReqsPendingHead)
-    {
-        LogFlow(("Queuing pending requests first\n"));
-
-        pTasksHead = pEndpoint->AioMgr.pReqsPendingHead;
-        /*
-         * Clear the list as the processing routine will insert them into the list
-         * again if it gets a flush request.
-         */
-        pEndpoint->AioMgr.pReqsPendingHead = NULL;
-        pEndpoint->AioMgr.pReqsPendingTail = NULL;
-        rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
-        AssertRC(rc);
-    }
-
-    if (!pEndpoint->pFlushReq && !pEndpoint->AioMgr.pReqsPendingHead)
-    {
-        /* Now the request queue. */
-        pTasksHead = pdmacFileEpGetNewTasks(pEndpoint);
-        if (pTasksHead)
-        {
-            rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
-            AssertRC(rc);
-        }
-    }
-
-    return rc;
-}
-
-static int pdmacFileAioMgrNormalProcessBlockingEvent(PPDMACEPFILEMGR pAioMgr)
-{
-    int rc = VINF_SUCCESS;
-    bool fNotifyWaiter = false;
-
-    LogFlowFunc((": Enter\n"));
-
-    Assert(pAioMgr->fBlockingEventPending);
-
-    switch (pAioMgr->enmBlockingEvent)
-    {
-        case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
-        {
-            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
-            AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
-
-            pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
-
-            pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
-            pEndpointNew->AioMgr.pEndpointPrev = NULL;
-            if (pAioMgr->pEndpointsHead)
-                pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
-            pAioMgr->pEndpointsHead = pEndpointNew;
-
-            /* Assign the completion point to this file. */
-            rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->File);
-            fNotifyWaiter = true;
-            pAioMgr->cEndpoints++;
-            break;
-        }
-        case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
-        {
-            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
-            AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
-
-            pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
-            fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
-            break;
-        }
-        case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
-        {
-            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
-            AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to close\n"));
-
-            if (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
-            {
-                LogFlowFunc((": Closing endpoint %#p{%s}\n", pEndpointClose, pEndpointClose->Core.pszUri));
-
-                /* Make sure all tasks finished. Process the queues a last time first. */
-                rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpointClose);
-                AssertRC(rc);
-
-                pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
-                fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
-            }
-            else if (   (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING)
-                     && (!pEndpointClose->AioMgr.cRequestsActive))
-                fNotifyWaiter = true;
-            break;
-        }
-        case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
-        {
-            pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
-            if (!pAioMgr->cRequestsActive)
-                fNotifyWaiter = true;
-            break;
-        }
-        case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
-        {
-            pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
-            break;
-        }
-        case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
-        {
-            pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
-            fNotifyWaiter = true;
-            break;
-        }
-        default:
-            AssertReleaseMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
-    }
-
-    if (fNotifyWaiter)
-    {
-        ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
-        pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
-
-        /* Release the waiting thread. */
-        LogFlow(("Signalling waiter\n"));
-        rc = RTSemEventSignal(pAioMgr->EventSemBlock);
-        AssertRC(rc);
-    }
-
-    LogFlowFunc((": Leave\n"));
-    return rc;
-}
-
-/**
- * Checks all endpoints for pending events or new requests.
- *
- * @returns VBox status code.
- * @param   pAioMgr    The I/O manager handle.
- */
-static int pdmacFileAioMgrNormalCheckEndpoints(PPDMACEPFILEMGR pAioMgr)
-{
-    /* Check the assigned endpoints for new tasks if there isn't a flush request active at the moment. */
-    int rc = VINF_SUCCESS;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
-
-    pAioMgr->msBwLimitExpired = RT_INDEFINITE_WAIT;
-
-    while (pEndpoint)
-    {
-        if (!pEndpoint->pFlushReq
-            && (pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
-            && !pEndpoint->AioMgr.fMoving)
-        {
-            rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpoint);
-            if (RT_FAILURE(rc))
-                return rc;
-        }
-        else if (   !pEndpoint->AioMgr.cRequestsActive
-                 && pEndpoint->enmState != PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
-        {
-            /* Reopen the file so that the new endpoint can re-associate with the file */
-            RTFileClose(pEndpoint->File);
-            rc = RTFileOpen(&pEndpoint->File, pEndpoint->Core.pszUri, pEndpoint->fFlags);
-            AssertRC(rc);
-
-            if (pEndpoint->AioMgr.fMoving)
-            {
-                pEndpoint->AioMgr.fMoving = false;
-                pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
-            }
-            else
-            {
-                Assert(pAioMgr->fBlockingEventPending);
-                ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
-
-                /* Release the waiting thread. */
-                LogFlow(("Signalling waiter\n"));
-                rc = RTSemEventSignal(pAioMgr->EventSemBlock);
-                AssertRC(rc);
-            }
-        }
-
-        pEndpoint = pEndpoint->AioMgr.pEndpointNext;
-    }
-
-    return rc;
-}
-
-/**
- * Wrapper around pdmacFileAioMgrNormalReqCompleteRc().
- */
-static void pdmacFileAioMgrNormalReqComplete(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
-{
-    size_t cbTransfered = 0;
-    int rcReq = RTFileAioReqGetRC(hReq, &cbTransfered);
-
-    pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, hReq, rcReq, cbTransfered);
-}
-
-static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
-                                               int rcReq, size_t cbTransfered)
-{
-    int rc = VINF_SUCCESS;
-    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
-    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(hReq);
-    PPDMACTASKFILE pTasksWaiting;
-
-    LogFlowFunc(("pAioMgr=%#p hReq=%#p\n", pAioMgr, hReq));
-
-    pEndpoint = pTask->pEndpoint;
-
-    pTask->hReq = NIL_RTFILEAIOREQ;
-
-    pAioMgr->cRequestsActive--;
-    pEndpoint->AioMgr.cRequestsActive--;
-    pEndpoint->AioMgr.cReqsProcessed++;
-
-    /*
-     * It is possible that the request failed on Linux with kernels < 2.6.23
-     * if the passed buffer was allocated with remap_pfn_range or if the file
-     * is on an NFS endpoint which does not support async and direct I/O at the same time.
-     * The endpoint will be migrated to a failsafe manager in case a request fails.
-     */
-    if (RT_FAILURE(rcReq))
-    {
-        /* Free bounce buffers and the IPRT request. */
-        pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
-
-        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
-        {
-            LogFlow(("Async flushes are not supported for this endpoint, disabling\n"));
-            pEndpoint->fAsyncFlushSupported = false;
-            AssertMsg(pEndpoint->pFlushReq == pTask, ("Failed flush request doesn't match active one\n"));
-            /* The other method will take over now. */
-        }
-        else
-        {
-            /* Free the lock and process pending tasks if necessary */
-            pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
-            rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
-            AssertRC(rc);
-
-            if (pTask->cbBounceBuffer)
-                RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
-
-            /*
-             * Fatal errors are reported to the guest and non-fatal errors
-             * will cause a migration to the failsafe manager in the hope
-             * that the error disappears.
-             */
-            if (!pdmacFileAioMgrNormalRcIsFatal(rcReq))
-            {
-                /* Queue the request on the pending list. */
-                pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead;
-                pEndpoint->AioMgr.pReqsPendingHead = pTask;
-
-                /* Create a new failsafe manager if necessary. */
-                if (!pEndpoint->AioMgr.fMoving)
-                {
-                    PPDMACEPFILEMGR pAioMgrFailsafe;
-
-                    LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n",
-                            RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri));
-
-                    pEndpoint->AioMgr.fMoving = true;
-
-                    rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass,
-                                                &pAioMgrFailsafe, PDMACEPFILEMGRTYPE_SIMPLE);
-                    AssertRC(rc);
-
-                    pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe;
-
-                    /* Update the flags to open the file with. Disable async I/O and enable the host cache. */
-                    pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO | RTFILE_O_NO_CACHE);
-                }
-
-                /* If this was the last request for the endpoint migrate it to the new manager. */
-                if (!pEndpoint->AioMgr.cRequestsActive)
-                {
-                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
-                    Assert(!fReqsPending);
-
-                    rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
-                    AssertRC(rc);
-                }
-            }
-            else
-            {
-                pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
-                pdmacFileTaskFree(pEndpoint, pTask);
-            }
-        }
-    }
-    else
-    {
-        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
-        {
-            /* Clear pending flush */
-            AssertMsg(pEndpoint->pFlushReq == pTask, ("Completed flush request doesn't match active one\n"));
-            pEndpoint->pFlushReq = NULL;
-            pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
-
-            /* Call completion callback */
-            LogFlow(("Flush task=%#p completed with %Rrc\n", pTask, rcReq));
-            pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
-            pdmacFileTaskFree(pEndpoint, pTask);
-        }
-        else
-        {
-            /*
-             * Restart an incomplete transfer.
-             * This usually means that the request will return an error now
-             * but to get the cause of the error (disk full, file too big, I/O error, ...)
-             * the transfer needs to be continued.
-             */
-            if (RT_UNLIKELY(   cbTransfered < pTask->DataSeg.cbSeg
-                            || (   pTask->cbBounceBuffer
-                                && cbTransfered < pTask->cbBounceBuffer)))
-            {
-                RTFOFF offStart;
-                size_t cbToTransfer;
-                uint8_t *pbBuf = NULL;
-
-                LogFlow(("Restarting incomplete transfer %#p (%zu bytes transferred)\n",
-                         pTask, cbTransfered));
-                Assert(cbTransfered % 512 == 0);
-
-                if (pTask->cbBounceBuffer)
-                {
-                    AssertPtr(pTask->pvBounceBuffer);
-                    offStart     = (pTask->Off & ~((RTFOFF)512-1)) + cbTransfered;
-                    cbToTransfer = pTask->cbBounceBuffer - cbTransfered;
-                    pbBuf        = (uint8_t *)pTask->pvBounceBuffer + cbTransfered;
-                }
-                else
-                {
-                    Assert(!pTask->pvBounceBuffer);
-                    offStart     = pTask->Off + cbTransfered;
-                    cbToTransfer = pTask->DataSeg.cbSeg - cbTransfered;
-                    pbBuf        = (uint8_t *)pTask->DataSeg.pvSeg + cbTransfered;
-                }
-
-                if (pTask->fPrefetch || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
-                {
-                    rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File, offStart,
-                                                 pbBuf, cbToTransfer, pTask);
-                }
-                else
-                {
-                    AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE,
-                                  ("Invalid transfer type\n"));
-                    rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File, offStart,
-                                                  pbBuf, cbToTransfer, pTask);
-                }
-                AssertRC(rc);
-
-                pTask->hReq = hReq;
-                rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
-                AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
-                          ("Unexpected return code rc=%Rrc\n", rc));
-            }
-            else if (pTask->fPrefetch)
-            {
-                Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE);
-                Assert(pTask->cbBounceBuffer);
-
-                memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
-                        pTask->DataSeg.pvSeg,
-                        pTask->DataSeg.cbSeg);
-
-                /* Write it now. */
-                pTask->fPrefetch = false;
-                size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg, 512);
-                RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
-
-                /* Grow the file if needed. */
-                if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
-                {
-                    ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
-                    RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
-                }
-
-                rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
-                                              offStart, pTask->pvBounceBuffer, cbToTransfer, pTask);
-                AssertRC(rc);
-                pTask->hReq = hReq;
-                rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
-                AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
-                          ("Unexpected return code rc=%Rrc\n", rc));
-            }
-            else
-            {
-                if (RT_SUCCESS(rc) && pTask->cbBounceBuffer)
-                {
-                    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
-                        memcpy(pTask->DataSeg.pvSeg,
-                               ((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
-                               pTask->DataSeg.cbSeg);
-
-                    RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
-                }
-
-                pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
-
-                /* Free the lock and process pending tasks if necessary */
-                pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
-                if (pTasksWaiting)
-                {
-                    rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
-                    AssertRC(rc);
-                }
-
-                /* Call completion callback */
-                LogFlow(("Task=%#p completed with %Rrc\n", pTask, rcReq));
-                pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
-                pdmacFileTaskFree(pEndpoint, pTask);
-
-                /*
-                 * If there is no request left on the endpoint but a flush request is set
-                 * it completed now and we notify the owner.
-                 * Furthermore we look for new requests and continue.
-                 */
-                if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq)
-                {
-                    /* Call completion callback */
-                    pTask = pEndpoint->pFlushReq;
-                    pEndpoint->pFlushReq = NULL;
-
-                    AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n"));
-
-                    pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
-                    pdmacFileTaskFree(pEndpoint, pTask);
-                }
-                else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving))
-                {
-                    /* If the endpoint is about to be migrated do it now. */
-                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
-                    Assert(!fReqsPending);
-
-                    rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
-                    AssertRC(rc);
-                }
-            }
-        } /* Not a flush request */
-    } /* request completed successfully */
-}
-
-/** Helper macro for checking for error codes. */
-#define CHECK_RC(pAioMgr, rc) \
-    if (RT_FAILURE(rc)) \
-    {\
-        int rc2 = pdmacFileAioMgrNormalErrorHandler(pAioMgr, rc, RT_SRC_POS);\
-        return rc2;\
-    }
-
-/**
- * The normal I/O manager using the RTFileAio* API
- *
- * @returns VBox status code.
- * @param ThreadSelf    Handle of the thread.
- * @param pvUser        Opaque user data.
- */
-int pdmacFileAioMgrNormal(RTTHREAD ThreadSelf, void *pvUser)
-{
-    int rc                  = VINF_SUCCESS;
-    PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
-    uint64_t uMillisEnd     = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
-
-    while (   (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
-           || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING)
-           || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
-    {
-        if (!pAioMgr->cRequestsActive)
-        {
-            ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
-            if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
-                rc = RTSemEventWait(pAioMgr->EventSem, pAioMgr->msBwLimitExpired);
-            ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
-            Assert(RT_SUCCESS(rc) || rc == VERR_TIMEOUT);
-
-            LogFlow(("Got woken up\n"));
-            ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
-        }
-
-        /* Check for an external blocking event first. */
-        if (pAioMgr->fBlockingEventPending)
-        {
-            rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
-            CHECK_RC(pAioMgr, rc);
-        }
-
-        if (RT_LIKELY(    pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
-                      ||  pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
-        {
-            /* We got woken up because an endpoint issued new requests. Queue them. */
-            rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
-            CHECK_RC(pAioMgr, rc);
-
-            while (pAioMgr->cRequestsActive)
-            {
-                RTFILEAIOREQ apReqs[20];
-                uint32_t     cReqsCompleted = 0;
-                size_t       cReqsWait;
-
-                if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
-                    cReqsWait = RT_ELEMENTS(apReqs);
-                else
-                    cReqsWait = pAioMgr->cRequestsActive;
-
-                LogFlow(("Waiting for %d of %d tasks to complete\n", 1, cReqsWait));
-
-                rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
-                                      1,
-                                      RT_INDEFINITE_WAIT, apReqs,
-                                      cReqsWait, &cReqsCompleted);
-                if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
-                    CHECK_RC(pAioMgr, rc);
-
-                LogFlow(("%d tasks completed\n", cReqsCompleted));
-
-                for (uint32_t i = 0; i < cReqsCompleted; i++)
-                    pdmacFileAioMgrNormalReqComplete(pAioMgr, apReqs[i]);
-
-                /* Check for an external blocking event before we go to sleep again. */
-                if (pAioMgr->fBlockingEventPending)
-                {
-                    rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
-                    CHECK_RC(pAioMgr, rc);
-                }
-
-                /* Update load statistics. */
-                uint64_t uMillisCurr = RTTimeMilliTS();
-                if (uMillisCurr > uMillisEnd)
-                {
-                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead;
-
-                    /* Calculate timespan. */
-                    uMillisCurr -= uMillisEnd;
-
-                    while (pEndpointCurr)
-                    {
-                        pEndpointCurr->AioMgr.cReqsPerSec    = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD);
-                        pEndpointCurr->AioMgr.cReqsProcessed = 0;
-                        pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext;
-                    }
-
-                    /* Set new update interval */
-                    uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
-                }
-
-                /* Check endpoints for new requests. */
-                if (pAioMgr->enmState != PDMACEPFILEMGRSTATE_GROWING)
-                {
-                    rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
-                    CHECK_RC(pAioMgr, rc);
-                }
-            } /* while requests are active. */
-
-            if (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
-            {
-                rc = pdmacFileAioMgrNormalGrow(pAioMgr);
-                AssertRC(rc);
-                Assert(pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING);
-
-                rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
-                CHECK_RC(pAioMgr, rc);
-            }
-        } /* if still running */
-    } /* while running */
-
-    LogFlowFunc(("rc=%Rrc\n", rc));
-    return rc;
-}
-
-#undef CHECK_RC
-
Index: unk/src/VBox/VMM/PDMAsyncCompletionInternal.h
===================================================================
--- /trunk/src/VBox/VMM/PDMAsyncCompletionInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,277 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device Manager, Async I/O Completion internal header.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___PDMAsyncCompletionInternal_h
-#define ___PDMAsyncCompletionInternal_h
-
-#include <iprt/critsect.h>
-#include <iprt/memcache.h>
-#include <iprt/sg.h>
-#include <VBox/types.h>
-#include <VBox/cfgm.h>
-#include <VBox/stam.h>
-#include <VBox/pdmasynccompletion.h>
-#include "PDMInternal.h"
-
-RT_C_DECLS_BEGIN
-
-
-/**
- * PDM Async completion endpoint operations.
- */
-typedef struct PDMASYNCCOMPLETIONEPCLASSOPS
-{
-    /** Version identifier. */
-    uint32_t                      u32Version;
-    /** Name of the endpoint class. */
-    const char                   *pcszName;
-    /** Class type. */
-    PDMASYNCCOMPLETIONEPCLASSTYPE enmClassType;
-    /** Size of the global endpoint class data in bytes. */
-    size_t                        cbEndpointClassGlobal;
-    /** Size of an endpoint in bytes. */
-    size_t                        cbEndpoint;
-    /** size of a task in bytes. */
-    size_t                        cbTask;
-
-    /**
-     * Initializes the global data for a endpoint class.
-     *
-     * @returns VBox status code.
-     * @param   pClassGlobals    Pointer to the uninitialized globals data.
-     * @param   pCfgNode         Node for querying configuration data.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnInitialize, (PPDMASYNCCOMPLETIONEPCLASS pClassGlobals, PCFGMNODE pCfgNode));
-
-    /**
-     * Frees all allocated resources which were allocated during init.
-     *
-     * @returns VBox status code.
-     * @param   pClassGlobals    Pointer to the globals data.
-     */
-    DECLR3CALLBACKMEMBER(void, pfnTerminate, (PPDMASYNCCOMPLETIONEPCLASS pClassGlobals));
-
-    /**
-     * Initializes a given endpoint.
-     *
-     * @returns VBox status code.
-     * @param   pEndpoint     Pointer to the uninitialized endpoint.
-     * @param   pszUri        Pointer to the string containing the endpoint
-     *                        destination (filename, IP address, ...)
-     * @param   fFlags        Creation flags.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnEpInitialize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                                const char *pszUri, uint32_t fFlags));
-
-    /**
-     * Closes a endpoint finishing all tasks.
-     *
-     * @returns VBox status code.
-     * @param   pEndpoint     Pointer to the endpoint to be closed.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnEpClose, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint));
-
-    /**
-     * Initiates a read request from the given endpoint.
-     *
-     * @returns VBox status code.
-     * @param   pTask         Pointer to the task object associated with the request.
-     * @param   pEndpoint     Endpoint the request is for.
-     * @param   off           Where to start reading from.
-     * @param   paSegments    Scatter gather list to store the data in.
-     * @param   cSegments     Number of segments in the list.
-     * @param   cbRead        The overall number of bytes to read.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnEpRead, (PPDMASYNCCOMPLETIONTASK pTask,
-                                          PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
-                                          PCRTSGSEG paSegments, size_t cSegments,
-                                         size_t cbRead));
-
-    /**
-     * Initiates a write request to the given endpoint.
-     *
-     * @returns VBox status code.
-     * @param   pTask         Pointer to the task object associated with the request.
-     * @param   pEndpoint     Endpoint the request is for.
-     * @param   off           Where to start writing to.
-     * @param   paSegments    Scatter gather list to store the data in.
-     * @param   cSegments     Number of segments in the list.
-     * @param   cbRead        The overall number of bytes to write.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnEpWrite, (PPDMASYNCCOMPLETIONTASK pTask,
-                                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
-                                           PCRTSGSEG paSegments, size_t cSegments,
-                                           size_t cbWrite));
-
-    /**
-     * Initiates a flush request on the given endpoint.
-     *
-     * @returns VBox status code.
-     * @param   pTask         Pointer to the task object associated with the request.
-     * @param   pEndpoint     Endpoint the request is for.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnEpFlush, (PPDMASYNCCOMPLETIONTASK pTask,
-                                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint));
-
-    /**
-     * Queries the size of the endpoint. Optional.
-     *
-     * @returns VBox status code.
-     * @param   pEndpoint     Endpoint the request is for.
-     * @param   pcbSize       Where to store the size of the endpoint.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnEpGetSize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                             uint64_t *pcbSize));
-
-    /**
-     * Sets the size of the endpoint. Optional.
-     * This is a synchronous operation.
-     *
-     *
-     * @returns VBox status code.
-     * @param   pEndpoint     Endpoint the request is for.
-     * @param   cbSize        New size for the endpoint.
-     */
-    DECLR3CALLBACKMEMBER(int, pfnEpSetSize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
-                                             uint64_t cbSize));
-
-    /** Initialization safety marker. */
-    uint32_t    u32VersionEnd;
-} PDMASYNCCOMPLETIONEPCLASSOPS;
-/** Pointer to a async completion endpoint class operation table. */
-typedef PDMASYNCCOMPLETIONEPCLASSOPS *PPDMASYNCCOMPLETIONEPCLASSOPS;
-/** Const pointer to a async completion endpoint class operation table. */
-typedef const PDMASYNCCOMPLETIONEPCLASSOPS *PCPDMASYNCCOMPLETIONEPCLASSOPS;
-
-/** Version for the endpoint class operations structure. */
-#define PDMAC_EPCLASS_OPS_VERSION 0x00000001
-
-/** Pointer to a bandwidth control manager. */
-typedef struct PDMACBWMGR *PPDMACBWMGR;
-
-/**
- * PDM Async completion endpoint class.
- * Common data.
- */
-typedef struct PDMASYNCCOMPLETIONEPCLASS
-{
-    /** Pointer to the shared VM structure. */
-    PVM                                         pVM;
-    /** Critical section protecting the lists below. */
-    RTCRITSECT                                  CritSect;
-    /** Number of endpoints in the list. */
-    volatile unsigned                           cEndpoints;
-    /** Head of endpoints with this class. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pEndpointsHead;
-    /** Head of the bandwidth managers for this class. */
-    R3PTRTYPE(PPDMACBWMGR)                      pBwMgrsHead;
-    /** Pointer to the callback table. */
-    R3PTRTYPE(PCPDMASYNCCOMPLETIONEPCLASSOPS)   pEndpointOps;
-    /** Task cache. */
-    RTMEMCACHE                                  hMemCacheTasks;
-} PDMASYNCCOMPLETIONEPCLASS;
-/** Pointer to the PDM async completion endpoint class data. */
-typedef PDMASYNCCOMPLETIONEPCLASS *PPDMASYNCCOMPLETIONEPCLASS;
-
-/**
- * A PDM Async completion endpoint.
- * Common data.
- */
-typedef struct PDMASYNCCOMPLETIONENDPOINT
-{
-    /** Next endpoint in the list. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pNext;
-    /** Previous endpoint in the list. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pPrev;
-    /** Pointer to the class this endpoint belongs to. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONEPCLASS)       pEpClass;
-    /** Template associated with this endpoint. */
-    PPDMASYNCCOMPLETIONTEMPLATE                 pTemplate;
-    /** Reference count. */
-    unsigned                                    cUsers;
-    /** URI describing the endpoint */
-    char                                       *pszUri;
-    /** Pointer to the assigned bandwidth manager. */
-    volatile PPDMACBWMGR                        pBwMgr;
-#ifdef VBOX_WITH_STATISTICS
-    uint32_t                                    u32Alignment;
-    STAMCOUNTER                                 StatTaskRunTimesNs[10];
-    STAMCOUNTER                                 StatTaskRunTimesMicroSec[10];
-    STAMCOUNTER                                 StatTaskRunTimesMs[10];
-    STAMCOUNTER                                 StatTaskRunTimesSec[10];
-    STAMCOUNTER                                 StatTaskRunOver100Sec;
-    STAMCOUNTER                                 StatIoOpsPerSec;
-    STAMCOUNTER                                 StatIoOpsStarted;
-    STAMCOUNTER                                 StatIoOpsCompleted;
-    uint64_t                                    tsIntervalStartMs;
-    uint64_t                                    cIoOpsCompleted;
-#endif
-} PDMASYNCCOMPLETIONENDPOINT;
-#ifdef VBOX_WITH_STATISTICS
-AssertCompileMemberAlignment(PDMASYNCCOMPLETIONENDPOINT, StatTaskRunTimesNs, sizeof(uint64_t));
-#endif
-
-/**
- * A PDM async completion task handle.
- * Common data.
- */
-typedef struct PDMASYNCCOMPLETIONTASK
-{
-    /** Next task in the list
-     * (for free and assigned tasks). */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONTASK)      pNext;
-    /** Previous task in the list
-     * (for free and assigned tasks). */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONTASK)      pPrev;
-    /** Endpoint this task is assigned to. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)  pEndpoint;
-    /** Opaque user data for this task. */
-    void                                   *pvUser;
-    /** Start timestamp. */
-    uint64_t                                tsNsStart;
-} PDMASYNCCOMPLETIONTASK;
-
-/**
- * Called by the endpoint if a task has finished.
- *
- * @returns nothing
- * @param   pTask                     Pointer to the finished task.
- * @param   rc                        Status code of the completed request.
- * @param   fCallCompletionHandler    Flag whether the completion handler should be called to
- *                                    inform the owner of the task that it has completed.
- */
-void pdmR3AsyncCompletionCompleteTask(PPDMASYNCCOMPLETIONTASK pTask, int rc, bool fCallCompletionHandler);
-
-/**
- * Checks if the endpoint is allowed to transfer the given amount of bytes.
- *
- * @returns true if the endpoint is allowed to transfer the data.
- *          false otherwise
- * @param   pEndpoint                 The endpoint.
- * @param   cbTransfer                The number of bytes to transfer.
- * @param   pmsWhenNext               Where to store the number of milliseconds
- *                                    until the bandwidth is refreshed.
- *                                    Only set if false is returned.
- */
-bool pdmacEpIsTransferAllowed(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint32_t cbTransfer, RTMSINTERVAL *pmsWhenNext);
-
-RT_C_DECLS_END
-
-extern const PDMASYNCCOMPLETIONEPCLASSOPS g_PDMAsyncCompletionEndpointClassFile;
-
-#endif /* !___PDMAsyncCompletionInternal_h */
-
Index: unk/src/VBox/VMM/PDMBlkCache.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMBlkCache.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2612 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Block Cache.
- */
-
-/*
- * Copyright (C) 2006-2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_pdm_block_cache     PDM Block Cache - The I/O cache
- * This component implements an I/O cache based on the 2Q cache algorithm.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_BLK_CACHE
-#include "PDMInternal.h"
-#include <iprt/asm.h>
-#include <iprt/mem.h>
-#include <iprt/path.h>
-#include <iprt/string.h>
-#include <VBox/log.h>
-#include <VBox/stam.h>
-#include <VBox/uvm.h>
-#include <VBox/vm.h>
-
-#include "PDMBlkCacheInternal.h"
-
-#ifdef VBOX_STRICT
-# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
-    do \
-    { \
-     AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
-               ("Thread does not own critical section\n"));\
-    } while(0)
-
-# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
-    do \
-    { \
-        AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
-                  ("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
-    } while(0)
-
-# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
-    do \
-    { \
-        AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
-                  ("Thread is not read owner of the per endpoint RW semaphore\n")); \
-    } while(0)
-
-#else
-# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0)
-# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while(0)
-# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while(0)
-#endif
-
-#define PDM_BLK_CACHE_SAVED_STATE_VERSION 1
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-
-static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
-                                               uint64_t off, size_t cbData, uint8_t *pbBuffer);
-static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry);
-
-/**
- * Decrement the reference counter of the given cache entry.
- *
- * @returns nothing.
- * @param   pEntry    The entry to release.
- */
-DECLINLINE(void) pdmBlkCacheEntryRelease(PPDMBLKCACHEENTRY pEntry)
-{
-    AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
-    ASMAtomicDecU32(&pEntry->cRefs);
-}
-
-/**
- * Increment the reference counter of the given cache entry.
- *
- * @returns nothing.
- * @param   pEntry    The entry to reference.
- */
-DECLINLINE(void) pdmBlkCacheEntryRef(PPDMBLKCACHEENTRY pEntry)
-{
-    ASMAtomicIncU32(&pEntry->cRefs);
-}
-
-#ifdef DEBUG
-static void pdmBlkCacheValidate(PPDMBLKCACHEGLOBAL pCache)
-{
-    /* Amount of cached data should never exceed the maximum amount. */
-    AssertMsg(pCache->cbCached <= pCache->cbMax,
-              ("Current amount of cached data exceeds maximum\n"));
-
-    /* The amount of cached data in the LRU and FRU list should match cbCached */
-    AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
-              ("Amount of cached data doesn't match\n"));
-
-    AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
-              ("Paged out list exceeds maximum\n"));
-}
-#endif
-
-DECLINLINE(void) pdmBlkCacheLockEnter(PPDMBLKCACHEGLOBAL pCache)
-{
-    RTCritSectEnter(&pCache->CritSect);
-#ifdef DEBUG
-    pdmBlkCacheValidate(pCache);
-#endif
-}
-
-DECLINLINE(void) pdmBlkCacheLockLeave(PPDMBLKCACHEGLOBAL pCache)
-{
-#ifdef DEBUG
-    pdmBlkCacheValidate(pCache);
-#endif
-    RTCritSectLeave(&pCache->CritSect);
-}
-
-DECLINLINE(void) pdmBlkCacheSub(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
-{
-    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
-    pCache->cbCached -= cbAmount;
-}
-
-DECLINLINE(void) pdmBlkCacheAdd(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
-{
-    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
-    pCache->cbCached += cbAmount;
-}
-
-DECLINLINE(void) pdmBlkCacheListAdd(PPDMBLKLRULIST pList, uint32_t cbAmount)
-{
-    pList->cbCached += cbAmount;
-}
-
-DECLINLINE(void) pdmBlkCacheListSub(PPDMBLKLRULIST pList, uint32_t cbAmount)
-{
-    pList->cbCached -= cbAmount;
-}
-
-#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
-/**
- * Checks consistency of a LRU list.
- *
- * @returns nothing
- * @param    pList         The LRU list to check.
- * @param    pNotInList    Element which is not allowed to occur in the list.
- */
-static void pdmBlkCacheCheckList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pNotInList)
-{
-    PPDMBLKCACHEENTRY pCurr = pList->pHead;
-
-    /* Check that there are no double entries and no cycles in the list. */
-    while (pCurr)
-    {
-        PPDMBLKCACHEENTRY pNext = pCurr->pNext;
-
-        while (pNext)
-        {
-            AssertMsg(pCurr != pNext,
-                      ("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
-                      pCurr, pList));
-            pNext = pNext->pNext;
-        }
-
-        AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
-
-        if (!pCurr->pNext)
-            AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
-
-        pCurr = pCurr->pNext;
-    }
-}
-#endif
-
-/**
- * Unlinks a cache entry from the LRU list it is assigned to.
- *
- * @returns nothing.
- * @param   pEntry    The entry to unlink.
- */
-static void pdmBlkCacheEntryRemoveFromList(PPDMBLKCACHEENTRY pEntry)
-{
-    PPDMBLKLRULIST pList = pEntry->pList;
-    PPDMBLKCACHEENTRY pPrev, pNext;
-
-    LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
-
-    AssertPtr(pList);
-
-#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
-    pdmBlkCacheCheckList(pList, NULL);
-#endif
-
-    pPrev = pEntry->pPrev;
-    pNext = pEntry->pNext;
-
-    AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
-    AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
-
-    if (pPrev)
-        pPrev->pNext = pNext;
-    else
-    {
-        pList->pHead = pNext;
-
-        if (pNext)
-            pNext->pPrev = NULL;
-    }
-
-    if (pNext)
-        pNext->pPrev = pPrev;
-    else
-    {
-        pList->pTail = pPrev;
-
-        if (pPrev)
-            pPrev->pNext = NULL;
-    }
-
-    pEntry->pList    = NULL;
-    pEntry->pPrev    = NULL;
-    pEntry->pNext    = NULL;
-    pdmBlkCacheListSub(pList, pEntry->cbData);
-#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
-    pdmBlkCacheCheckList(pList, pEntry);
-#endif
-}
-
-/**
- * Adds a cache entry to the given LRU list unlinking it from the currently
- * assigned list if needed.
- *
- * @returns nothing.
- * @param    pList    List to the add entry to.
- * @param    pEntry   Entry to add.
- */
-static void pdmBlkCacheEntryAddToList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pEntry)
-{
-    LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
-#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
-    pdmBlkCacheCheckList(pList, NULL);
-#endif
-
-    /* Remove from old list if needed */
-    if (pEntry->pList)
-        pdmBlkCacheEntryRemoveFromList(pEntry);
-
-    pEntry->pNext = pList->pHead;
-    if (pList->pHead)
-        pList->pHead->pPrev = pEntry;
-    else
-    {
-        Assert(!pList->pTail);
-        pList->pTail = pEntry;
-    }
-
-    pEntry->pPrev    = NULL;
-    pList->pHead     = pEntry;
-    pdmBlkCacheListAdd(pList, pEntry->cbData);
-    pEntry->pList    = pList;
-#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
-    pdmBlkCacheCheckList(pList, NULL);
-#endif
-}
-
-/**
- * Destroys a LRU list freeing all entries.
- *
- * @returns nothing
- * @param   pList    Pointer to the LRU list to destroy.
- *
- * @note The caller must own the critical section of the cache.
- */
-static void pdmBlkCacheDestroyList(PPDMBLKLRULIST pList)
-{
-    while (pList->pHead)
-    {
-        PPDMBLKCACHEENTRY pEntry = pList->pHead;
-
-        pList->pHead = pEntry->pNext;
-
-        AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)),
-                  ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
-
-        RTMemPageFree(pEntry->pbData, pEntry->cbData);
-        RTMemFree(pEntry);
-    }
-}
-
-/**
- * Tries to remove the given amount of bytes from a given list in the cache
- * moving the entries to one of the given ghosts lists
- *
- * @returns Amount of data which could be freed.
- * @param    pCache           Pointer to the global cache data.
- * @param    cbData           The amount of the data to free.
- * @param    pListSrc         The source list to evict data from.
- * @param    pGhostListSrc    The ghost list removed entries should be moved to
- *                            NULL if the entry should be freed.
- * @param    fReuseBuffer     Flag whether a buffer should be reused if it has the same size
- * @param    ppbBuf           Where to store the address of the buffer if an entry with the
- *                            same size was found and fReuseBuffer is true.
- *
- * @note    This function may return fewer bytes than requested because entries
- *          may be marked as non evictable if they are used for I/O at the
- *          moment.
- */
-static size_t pdmBlkCacheEvictPagesFrom(PPDMBLKCACHEGLOBAL pCache, size_t cbData,
-                                        PPDMBLKLRULIST pListSrc, PPDMBLKLRULIST pGhostListDst,
-                                        bool fReuseBuffer, uint8_t **ppbBuffer)
-{
-    size_t cbEvicted = 0;
-
-    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
-
-    AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
-    AssertMsg(   !pGhostListDst
-              || (pGhostListDst == &pCache->LruRecentlyUsedOut),
-              ("Destination list must be NULL or the recently used but paged out list\n"));
-
-    if (fReuseBuffer)
-    {
-        AssertPtr(ppbBuffer);
-        *ppbBuffer = NULL;
-    }
-
-    /* Start deleting from the tail. */
-    PPDMBLKCACHEENTRY pEntry = pListSrc->pTail;
-
-    while ((cbEvicted < cbData) && pEntry)
-    {
-        PPDMBLKCACHEENTRY pCurr = pEntry;
-
-        pEntry = pEntry->pPrev;
-
-        /* We can't evict pages which are currently in progress or dirty but not in progress */
-        if (   !(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
-            && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
-        {
-            /* Ok eviction candidate. Grab the endpoint semaphore and check again
-             * because somebody else might have raced us. */
-            PPDMBLKCACHE pBlkCache = pCurr->pBlkCache;
-            RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-
-            if (!(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
-                && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
-            {
-                LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
-
-                if (fReuseBuffer && (pCurr->cbData == cbData))
-                {
-                    STAM_COUNTER_INC(&pCache->StatBuffersReused);
-                    *ppbBuffer = pCurr->pbData;
-                }
-                else if (pCurr->pbData)
-                    RTMemPageFree(pCurr->pbData, pCurr->cbData);
-
-                pCurr->pbData = NULL;
-                cbEvicted += pCurr->cbData;
-
-                pdmBlkCacheEntryRemoveFromList(pCurr);
-                pdmBlkCacheSub(pCache, pCurr->cbData);
-
-                if (pGhostListDst)
-                {
-                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-
-                    PPDMBLKCACHEENTRY pGhostEntFree = pGhostListDst->pTail;
-
-                    /* We have to remove the last entries from the paged out list. */
-                    while (   ((pGhostListDst->cbCached + pCurr->cbData) > pCache->cbRecentlyUsedOutMax)
-                           && pGhostEntFree)
-                    {
-                        PPDMBLKCACHEENTRY pFree = pGhostEntFree;
-                        PPDMBLKCACHE pBlkCacheFree = pFree->pBlkCache;
-
-                        pGhostEntFree = pGhostEntFree->pPrev;
-
-                        RTSemRWRequestWrite(pBlkCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
-
-                        if (ASMAtomicReadU32(&pFree->cRefs) == 0)
-                        {
-                            pdmBlkCacheEntryRemoveFromList(pFree);
-
-                            STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
-                            RTAvlrU64Remove(pBlkCacheFree->pTree, pFree->Core.Key);
-                            STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
-
-                            RTMemFree(pFree);
-                        }
-
-                        RTSemRWReleaseWrite(pBlkCacheFree->SemRWEntries);
-                    }
-
-                    if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
-                    {
-                        /* Couldn't remove enough entries. Delete */
-                        STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
-                        RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
-                        STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
-
-                        RTMemFree(pCurr);
-                    }
-                    else
-                        pdmBlkCacheEntryAddToList(pGhostListDst, pCurr);
-                }
-                else
-                {
-                    /* Delete the entry from the AVL tree it is assigned to. */
-                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
-                    RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
-                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
-
-                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-                    RTMemFree(pCurr);
-                }
-            }
-
-        }
-        else
-            LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
-    }
-
-    return cbEvicted;
-}
-
-static bool pdmBlkCacheReclaim(PPDMBLKCACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
-{
-    size_t cbRemoved = 0;
-
-    if ((pCache->cbCached + cbData) < pCache->cbMax)
-        return true;
-    else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
-    {
-        /* Try to evict as many bytes as possible from A1in */
-        cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
-                                                 &pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
-
-        /*
-         * If it was not possible to remove enough entries
-         * try the frequently accessed cache.
-         */
-        if (cbRemoved < cbData)
-        {
-            Assert(!fReuseBuffer || !*ppbBuffer); /* It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
-
-            /*
-             * If we removed something we can't pass the reuse buffer flag anymore because
-             * we don't need to evict that much data
-             */
-            if (!cbRemoved)
-                cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
-                                                          NULL, fReuseBuffer, ppbBuffer);
-            else
-                cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
-                                                          NULL, false, NULL);
-        }
-    }
-    else
-    {
-        /* We have to remove entries from frequently access list. */
-        cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
-                                                 NULL, fReuseBuffer, ppbBuffer);
-    }
-
-    LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
-    return (cbRemoved >= cbData);
-}
-
-DECLINLINE(int) pdmBlkCacheEnqueue(PPDMBLKCACHE pBlkCache, uint64_t off, PPDMBLKCACHEIOXFER pIoXfer)
-{
-    int rc = VINF_SUCCESS;
-
-    switch (pBlkCache->enmType)
-    {
-        case PDMBLKCACHETYPE_DEV:
-        {
-            rc = pBlkCache->u.Dev.pfnXferEnqueue(pBlkCache->u.Dev.pDevIns,
-                                                 pIoXfer->enmXferDir,
-                                                 off, pIoXfer->cbXfer,
-                                                 &pIoXfer->SgBuf, pIoXfer);
-            break;
-        }
-        case PDMBLKCACHETYPE_DRV:
-        {
-            rc = pBlkCache->u.Drv.pfnXferEnqueue(pBlkCache->u.Drv.pDrvIns,
-                                                 pIoXfer->enmXferDir,
-                                                 off, pIoXfer->cbXfer,
-                                                 &pIoXfer->SgBuf, pIoXfer);
-            break;
-        }
-        case PDMBLKCACHETYPE_USB:
-        {
-            rc = pBlkCache->u.Usb.pfnXferEnqueue(pBlkCache->u.Usb.pUsbIns,
-                                                 pIoXfer->enmXferDir,
-                                                 off, pIoXfer->cbXfer,
-                                                 &pIoXfer->SgBuf, pIoXfer);
-            break;
-        }
-        case PDMBLKCACHETYPE_INTERNAL:
-        {
-            rc = pBlkCache->u.Int.pfnXferEnqueue(pBlkCache->u.Int.pvUser,
-                                                 pIoXfer->enmXferDir,
-                                                 off, pIoXfer->cbXfer,
-                                                 &pIoXfer->SgBuf, pIoXfer);
-            break;
-        }
-        default:
-            AssertMsgFailed(("Unknown block cache type!\n"));
-    }
-
-    return rc;
-}
-
-/**
- * Initiates a read I/O task for the given entry.
- *
- * @returns VBox status code.
- * @param   pEntry    The entry to fetch the data to.
- */
-static int pdmBlkCacheEntryReadFromMedium(PPDMBLKCACHEENTRY pEntry)
-{
-    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
-    LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
-
-    /* Make sure no one evicts the entry while it is accessed. */
-    pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
-
-    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
-    if (RT_UNLIKELY(!pIoXfer))
-        return VERR_NO_MEMORY;
-
-    AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
-
-    pIoXfer->fIoCache = true;
-    pIoXfer->pEntry = pEntry;
-    pIoXfer->SgSeg.pvSeg = pEntry->pbData;
-    pIoXfer->SgSeg.cbSeg = pEntry->cbData;
-    pIoXfer->cbXfer      = pEntry->cbData;
-    pIoXfer->enmXferDir  = PDMBLKCACHEXFERDIR_READ;
-    RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
-
-    return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
-}
-
-/**
- * Initiates a write I/O task for the given entry.
- *
- * @returns nothing.
- * @param    pEntry The entry to read the data from.
- */
-static int pdmBlkCacheEntryWriteToMedium(PPDMBLKCACHEENTRY pEntry)
-{
-    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
-    LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
-
-    /* Make sure no one evicts the entry while it is accessed. */
-    pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
-
-    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
-    if (RT_UNLIKELY(!pIoXfer))
-        return VERR_NO_MEMORY;
-
-    AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
-
-    pIoXfer->fIoCache = true;
-    pIoXfer->pEntry = pEntry;
-    pIoXfer->SgSeg.pvSeg = pEntry->pbData;
-    pIoXfer->SgSeg.cbSeg = pEntry->cbData;
-    pIoXfer->cbXfer      = pEntry->cbData;
-    pIoXfer->enmXferDir  = PDMBLKCACHEXFERDIR_WRITE;
-    RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
-
-    return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
-}
-
-/**
- * Passthrough a part of a request directly to the I/O manager
- * handling the endpoint.
- *
- * @returns VBox status code.
- * @param   pEndpoint          The endpoint.
- * @param   pTask              The task.
- * @param   pIoMemCtx          The I/O memory context to use.
- * @param   offStart           Offset to start transfer from.
- * @param   cbData             Amount of data to transfer.
- * @param   enmTransferType    The transfer type (read/write)
- */
-static int pdmBlkCacheRequestPassthrough(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
-                                         PRTSGBUF pSgBuf, uint64_t offStart, size_t cbData,
-                                         PDMBLKCACHEXFERDIR enmXferDir)
-{
-
-    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
-    if (RT_UNLIKELY(!pIoXfer))
-        return VERR_NO_MEMORY;
-
-    ASMAtomicIncU32(&pReq->cXfersPending);
-    pIoXfer->fIoCache    = false;
-    pIoXfer->pReq        = pReq;
-    pIoXfer->cbXfer      = cbData;
-    pIoXfer->enmXferDir  = enmXferDir;
-    if (pSgBuf)
-    {
-        RTSgBufClone(&pIoXfer->SgBuf, pSgBuf);
-        RTSgBufAdvance(pSgBuf, cbData);
-    }
-
-    return pdmBlkCacheEnqueue(pBlkCache, offStart, pIoXfer);
-}
-
-/**
- * Commit a single dirty entry to the endpoint
- *
- * @returns nothing
- * @param   pEntry    The entry to commit.
- */
-static void pdmBlkCacheEntryCommit(PPDMBLKCACHEENTRY pEntry)
-{
-    AssertMsg(   (pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY)
-              && !(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
-              ("Invalid flags set for entry %#p\n", pEntry));
-
-    pdmBlkCacheEntryWriteToMedium(pEntry);
-}
-
-/**
- * Commit all dirty entries for a single endpoint.
- *
- * @returns nothing.
- * @param   pBlkCache    The endpoint cache to commit.
- */
-static void pdmBlkCacheCommit(PPDMBLKCACHE pBlkCache)
-{
-    uint32_t cbCommitted = 0;
-
-    /* Return if the cache was suspended. */
-    if (pBlkCache->fSuspended)
-        return;
-
-    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-
-    /* The list is moved to a new header to reduce locking overhead. */
-    RTLISTNODE ListDirtyNotCommitted;
-    RTSPINLOCKTMP Tmp;
-
-    RTListInit(&ListDirtyNotCommitted);
-    RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
-    RTListMove(&ListDirtyNotCommitted, &pBlkCache->ListDirtyNotCommitted);
-    RTSpinlockRelease(pBlkCache->LockList, &Tmp);
-
-    if (!RTListIsEmpty(&ListDirtyNotCommitted))
-    {
-        PPDMBLKCACHEENTRY pEntry = RTListGetFirst(&ListDirtyNotCommitted, PDMBLKCACHEENTRY, NodeNotCommitted);
-
-        while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
-        {
-            PPDMBLKCACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMBLKCACHEENTRY,
-                                                        NodeNotCommitted);
-            pdmBlkCacheEntryCommit(pEntry);
-            cbCommitted += pEntry->cbData;
-            RTListNodeRemove(&pEntry->NodeNotCommitted);
-            pEntry = pNext;
-        }
-
-        /* Commit the last endpoint */
-        Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
-        pdmBlkCacheEntryCommit(pEntry);
-        RTListNodeRemove(&pEntry->NodeNotCommitted);
-        AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
-                  ("Committed all entries but list is not empty\n"));
-    }
-
-    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-    AssertMsg(pBlkCache->pCache->cbDirty >= cbCommitted,
-              ("Number of committed bytes exceeds number of dirty bytes\n"));
-    uint32_t cbDirtyOld = ASMAtomicSubU32(&pBlkCache->pCache->cbDirty, cbCommitted);
-
-    /* Reset the commit timer if we don't have any dirty bits. */
-    if (   !(cbDirtyOld - cbCommitted)
-        && pBlkCache->pCache->u32CommitTimeoutMs != 0)
-        TMTimerStop(pBlkCache->pCache->pTimerCommit);
-}
-
-/**
- * Commit all dirty entries in the cache.
- *
- * @returns nothing.
- * @param   pCache    The global cache instance.
- */
-static void pdmBlkCacheCommitDirtyEntries(PPDMBLKCACHEGLOBAL pCache)
-{
-    bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
-
-    if (!fCommitInProgress)
-    {
-        pdmBlkCacheLockEnter(pCache);
-        Assert(!RTListIsEmpty(&pCache->ListUsers));
-
-        PPDMBLKCACHE pBlkCache = RTListGetFirst(&pCache->ListUsers, PDMBLKCACHE, NodeCacheUser);
-        AssertPtr(pBlkCache);
-
-        while (!RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser))
-        {
-            pdmBlkCacheCommit(pBlkCache);
-
-            pBlkCache = RTListNodeGetNext(&pBlkCache->NodeCacheUser, PDMBLKCACHE,
-                                          NodeCacheUser);
-        }
-
-        /* Commit the last endpoint */
-        Assert(RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser));
-        pdmBlkCacheCommit(pBlkCache);
-
-        pdmBlkCacheLockLeave(pCache);
-        ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
-    }
-}
-
-/**
- * Adds the given entry as a dirty to the cache.
- *
- * @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
- * @param   pBlkCache    The endpoint cache the entry belongs to.
- * @param   pEntry       The entry to add.
- */
-static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
-{
-    bool fDirtyBytesExceeded = false;
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-
-    /* If the commit timer is disabled we commit right away. */
-    if (pCache->u32CommitTimeoutMs == 0)
-    {
-        pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
-        pdmBlkCacheEntryCommit(pEntry);
-    }
-    else if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY))
-    {
-        pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
-
-        RTSPINLOCKTMP Tmp;
-        RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
-        RTListAppend(&pBlkCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
-        RTSpinlockRelease(pBlkCache->LockList, &Tmp);
-
-        uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
-
-        /* Prevent committing if the VM was suspended. */
-        if (RT_LIKELY(!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended)))
-            fDirtyBytesExceeded = (cbDirty + pEntry->cbData >= pCache->cbCommitDirtyThreshold);
-        else if (!cbDirty && pCache->u32CommitTimeoutMs > 0)
-        {
-            /* Arm the commit timer. */
-            TMTimerSetMillies(pCache->pTimerCommit, pCache->u32CommitTimeoutMs);
-        }
-    }
-
-    return fDirtyBytesExceeded;
-}
-
-static PPDMBLKCACHE pdmR3BlkCacheFindById(PPDMBLKCACHEGLOBAL pBlkCacheGlobal, const char *pcszId)
-{
-    bool fFound = false;
-    PPDMBLKCACHE pBlkCache = NULL;
-
-    RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
-    {
-        if (!RTStrCmp(pBlkCache->pszId, pcszId))
-        {
-            fFound = true;
-            break;
-        }
-    }
-
-    return fFound ?  pBlkCache : NULL;
-}
-
-/**
- * Commit timer callback.
- */
-static void pdmBlkCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
-{
-    PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
-
-    LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
-
-    if (   ASMAtomicReadU32(&pCache->cbDirty) > 0
-        && !ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
-        pdmBlkCacheCommitDirtyEntries(pCache);
-
-    LogFlowFunc(("Entries committed, going to sleep\n"));
-}
-
-static DECLCALLBACK(int) pdmR3BlkCacheSaveExec(PVM pVM, PSSMHANDLE pSSM)
-{
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-
-    AssertPtr(pBlkCacheGlobal);
-
-    pdmBlkCacheLockEnter(pBlkCacheGlobal);
-
-    SSMR3PutU32(pSSM, pBlkCacheGlobal->cRefs);
-
-    /* Go through the list and save all dirty entries. */
-    PPDMBLKCACHE pBlkCache;
-    RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
-    {
-        uint32_t cEntries = 0;
-        PPDMBLKCACHEENTRY pEntry;
-
-        RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-        SSMR3PutU32(pSSM, strlen(pBlkCache->pszId));
-        SSMR3PutStrZ(pSSM, pBlkCache->pszId);
-
-        /* Count the number of entries to safe. */
-        RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
-        {
-            cEntries++;
-        }
-
-        SSMR3PutU32(pSSM, cEntries);
-
-        /* Walk the list of all dirty entries and save them. */
-        RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
-        {
-            /* A few sanity checks. */
-            AssertMsg(!pEntry->cRefs, ("The entry is still referenced\n"));
-            AssertMsg(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY, ("Entry is not dirty\n"));
-            AssertMsg(!(pEntry->fFlags & ~PDMBLKCACHE_ENTRY_IS_DIRTY), ("Invalid flags set\n"));
-            AssertMsg(!pEntry->pWaitingHead && !pEntry->pWaitingTail, ("There are waiting requests\n"));
-            AssertMsg(   pEntry->pList == &pBlkCacheGlobal->LruRecentlyUsedIn
-                      || pEntry->pList == &pBlkCacheGlobal->LruFrequentlyUsed,
-                      ("Invalid list\n"));
-            AssertMsg(pEntry->cbData == pEntry->Core.KeyLast - pEntry->Core.Key + 1,
-                      ("Size and range do not match\n"));
-
-            /* Save */
-            SSMR3PutU64(pSSM, pEntry->Core.Key);
-            SSMR3PutU32(pSSM, pEntry->cbData);
-            SSMR3PutMem(pSSM, pEntry->pbData, pEntry->cbData);
-        }
-
-        RTSemRWReleaseRead(pBlkCache->SemRWEntries);
-    }
-
-    pdmBlkCacheLockLeave(pBlkCacheGlobal);
-
-    /* Terminator */
-    return SSMR3PutU32(pSSM, UINT32_MAX);
-}
-
-static DECLCALLBACK(int) pdmR3BlkCacheLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    int rc = VINF_SUCCESS;
-    uint32_t cRefs;
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-
-    AssertPtr(pBlkCacheGlobal);
-
-    pdmBlkCacheLockEnter(pBlkCacheGlobal);
-
-    if (uVersion != PDM_BLK_CACHE_SAVED_STATE_VERSION)
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-
-    SSMR3GetU32(pSSM, &cRefs);
-
-    if (cRefs == pBlkCacheGlobal->cRefs)
-    {
-        char *pszId = NULL;
-
-        while (   cRefs > 0
-               && RT_SUCCESS(rc))
-        {
-            PPDMBLKCACHE pBlkCache = NULL;
-            uint32_t cbId = 0;
-
-            SSMR3GetU32(pSSM, &cbId);
-            Assert(cbId > 0);
-
-            cbId++; /* Include terminator */
-            pszId = (char *)RTMemAllocZ(cbId * sizeof(char));
-            if (!pszId)
-            {
-                rc = VERR_NO_MEMORY;
-                break;
-            }
-
-            rc = SSMR3GetStrZ(pSSM, pszId, cbId);
-            AssertRC(rc);
-
-            /* Search for the block cache with the provided id. */
-            pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pszId);
-            if (!pBlkCache)
-            {
-                rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
-                                      N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
-                break;
-            }
-
-            RTStrFree(pszId);
-            pszId = NULL;
-
-            /* Get the entries */
-            uint32_t cEntries;
-            SSMR3GetU32(pSSM, &cEntries);
-
-            while (cEntries > 0)
-            {
-                PPDMBLKCACHEENTRY pEntry;
-                uint64_t off;
-                uint32_t cbEntry;
-
-                SSMR3GetU64(pSSM, &off);
-                SSMR3GetU32(pSSM, &cbEntry);
-
-                pEntry = pdmBlkCacheEntryAlloc(pBlkCache, off, cbEntry, NULL);
-                if (!pEntry)
-                {
-                    rc = VERR_NO_MEMORY;
-                    break;
-                }
-
-                rc = SSMR3GetMem(pSSM, pEntry->pbData, cbEntry);
-                if (RT_FAILURE(rc))
-                {
-                    RTMemFree(pEntry->pbData);
-                    RTMemFree(pEntry);
-                    break;
-                }
-
-                /* Insert into the tree. */
-                bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
-                Assert(fInserted);
-
-                /* Add to the dirty list. */
-                pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
-                pdmBlkCacheEntryAddToList(&pBlkCacheGlobal->LruRecentlyUsedIn, pEntry);
-                pdmBlkCacheAdd(pBlkCacheGlobal, cbEntry);
-                pdmBlkCacheEntryRelease(pEntry);
-                cEntries--;
-            }
-
-            cRefs--;
-        }
-
-        if (pszId)
-            RTStrFree(pszId);
-    }
-    else
-        rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
-                              N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
-
-    pdmBlkCacheLockLeave(pBlkCacheGlobal);
-
-    if (RT_SUCCESS(rc))
-    {
-        uint32_t u32 = 0;
-        rc = SSMR3GetU32(pSSM, &u32);
-        if (RT_SUCCESS(rc))
-            AssertMsgReturn(u32 == UINT32_MAX, ("%#x\n", u32), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-    }
-
-    return rc;
-}
-
-int pdmR3BlkCacheInit(PVM pVM)
-{
-    int  rc   = VINF_SUCCESS;
-    PUVM pUVM = pVM->pUVM;
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal;
-
-    LogFlowFunc((": pVM=%p\n", pVM));
-
-    VM_ASSERT_EMT(pVM);
-
-    PCFGMNODE pCfgRoot     = CFGMR3GetRoot(pVM);
-    PCFGMNODE pCfgBlkCache = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "BlkCache");
-
-    pBlkCacheGlobal = (PPDMBLKCACHEGLOBAL)RTMemAllocZ(sizeof(PDMBLKCACHEGLOBAL));
-    if (!pBlkCacheGlobal)
-        return VERR_NO_MEMORY;
-
-    RTListInit(&pBlkCacheGlobal->ListUsers);
-    pBlkCacheGlobal->pVM = pVM;
-    pBlkCacheGlobal->cRefs = 0;
-    pBlkCacheGlobal->cbCached  = 0;
-    pBlkCacheGlobal->fCommitInProgress = false;
-
-    /* Initialize members */
-    pBlkCacheGlobal->LruRecentlyUsedIn.pHead    = NULL;
-    pBlkCacheGlobal->LruRecentlyUsedIn.pTail    = NULL;
-    pBlkCacheGlobal->LruRecentlyUsedIn.cbCached = 0;
-
-    pBlkCacheGlobal->LruRecentlyUsedOut.pHead    = NULL;
-    pBlkCacheGlobal->LruRecentlyUsedOut.pTail    = NULL;
-    pBlkCacheGlobal->LruRecentlyUsedOut.cbCached = 0;
-
-    pBlkCacheGlobal->LruFrequentlyUsed.pHead    = NULL;
-    pBlkCacheGlobal->LruFrequentlyUsed.pTail    = NULL;
-    pBlkCacheGlobal->LruFrequentlyUsed.cbCached = 0;
-
-    do
-    {
-        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheSize", &pBlkCacheGlobal->cbMax, 5 * _1M);
-        AssertLogRelRCBreak(rc);
-        LogFlowFunc(("Maximum number of bytes cached %u\n", pBlkCacheGlobal->cbMax));
-
-        pBlkCacheGlobal->cbRecentlyUsedInMax  = (pBlkCacheGlobal->cbMax / 100) * 25; /* 25% of the buffer size */
-        pBlkCacheGlobal->cbRecentlyUsedOutMax = (pBlkCacheGlobal->cbMax / 100) * 50; /* 50% of the buffer size */
-        LogFlowFunc(("cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n",
-                     pBlkCacheGlobal->cbRecentlyUsedInMax, pBlkCacheGlobal->cbRecentlyUsedOutMax));
-
-        /** @todo r=aeichner: Experiment to find optimal default values */
-        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitIntervalMs", &pBlkCacheGlobal->u32CommitTimeoutMs, 10000 /* 10sec */);
-        AssertLogRelRCBreak(rc);
-        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitThreshold", &pBlkCacheGlobal->cbCommitDirtyThreshold, pBlkCacheGlobal->cbMax / 2);
-        AssertLogRelRCBreak(rc);
-    } while (0);
-
-    if (RT_SUCCESS(rc))
-    {
-        STAMR3Register(pVM, &pBlkCacheGlobal->cbMax,
-                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/cbMax",
-                       STAMUNIT_BYTES,
-                       "Maximum cache size");
-        STAMR3Register(pVM, &pBlkCacheGlobal->cbCached,
-                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/cbCached",
-                       STAMUNIT_BYTES,
-                       "Currently used cache");
-        STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedIn.cbCached,
-                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/cbCachedMruIn",
-                       STAMUNIT_BYTES,
-                       "Number of bytes cached in MRU list");
-        STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedOut.cbCached,
-                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/cbCachedMruOut",
-                       STAMUNIT_BYTES,
-                       "Number of bytes cached in FRU list");
-        STAMR3Register(pVM, &pBlkCacheGlobal->LruFrequentlyUsed.cbCached,
-                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/cbCachedFru",
-                       STAMUNIT_BYTES,
-                       "Number of bytes cached in FRU ghost list");
-
-#ifdef VBOX_WITH_STATISTICS
-        STAMR3Register(pVM, &pBlkCacheGlobal->cHits,
-                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheHits",
-                       STAMUNIT_COUNT, "Number of hits in the cache");
-        STAMR3Register(pVM, &pBlkCacheGlobal->cPartialHits,
-                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CachePartialHits",
-                       STAMUNIT_COUNT, "Number of partial hits in the cache");
-        STAMR3Register(pVM, &pBlkCacheGlobal->cMisses,
-                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheMisses",
-                       STAMUNIT_COUNT, "Number of misses when accessing the cache");
-        STAMR3Register(pVM, &pBlkCacheGlobal->StatRead,
-                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheRead",
-                       STAMUNIT_BYTES, "Number of bytes read from the cache");
-        STAMR3Register(pVM, &pBlkCacheGlobal->StatWritten,
-                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheWritten",
-                       STAMUNIT_BYTES, "Number of bytes written to the cache");
-        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeGet,
-                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheTreeGet",
-                       STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
-        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeInsert,
-                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheTreeInsert",
-                       STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
-        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeRemove,
-                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheTreeRemove",
-                       STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
-        STAMR3Register(pVM, &pBlkCacheGlobal->StatBuffersReused,
-                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
-                       "/PDM/BlkCache/CacheBuffersReused",
-                       STAMUNIT_COUNT, "Number of times a buffer could be reused");
-#endif
-
-        /* Initialize the critical section */
-        rc = RTCritSectInit(&pBlkCacheGlobal->CritSect);
-    }
-
-    if (RT_SUCCESS(rc))
-    {
-        /* Create the commit timer */
-        if (pBlkCacheGlobal->u32CommitTimeoutMs > 0)
-            rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL,
-                                         pdmBlkCacheCommitTimerCallback,
-                                         pBlkCacheGlobal,
-                                         "BlkCache-Commit",
-                                         &pBlkCacheGlobal->pTimerCommit);
-
-        if (RT_SUCCESS(rc))
-        {
-            /* Register saved state handler. */
-            rc = SSMR3RegisterInternal(pVM, "pdmblkcache", 0, PDM_BLK_CACHE_SAVED_STATE_VERSION, pBlkCacheGlobal->cbMax,
-                                       NULL, NULL, NULL,
-                                       NULL, pdmR3BlkCacheSaveExec, NULL,
-                                       NULL, pdmR3BlkCacheLoadExec, NULL);
-            if (RT_SUCCESS(rc))
-            {
-                LogRel(("BlkCache: Cache successfully initialised. Cache size is %u bytes\n", pBlkCacheGlobal->cbMax));
-                LogRel(("BlkCache: Cache commit interval is %u ms\n", pBlkCacheGlobal->u32CommitTimeoutMs));
-                LogRel(("BlkCache: Cache commit threshold is %u bytes\n", pBlkCacheGlobal->cbCommitDirtyThreshold));
-                pUVM->pdm.s.pBlkCacheGlobal = pBlkCacheGlobal;
-                return VINF_SUCCESS;
-            }
-        }
-
-        RTCritSectDelete(&pBlkCacheGlobal->CritSect);
-    }
-
-    if (pBlkCacheGlobal)
-        RTMemFree(pBlkCacheGlobal);
-
-    LogFlowFunc((": returns rc=%Rrc\n", pVM, rc));
-    return rc;
-}
-
-void pdmR3BlkCacheTerm(PVM pVM)
-{
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-
-    if (pBlkCacheGlobal)
-    {
-        /* Make sure no one else uses the cache now */
-        pdmBlkCacheLockEnter(pBlkCacheGlobal);
-
-        /* Cleanup deleting all cache entries waiting for in progress entries to finish. */
-        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedIn);
-        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedOut);
-        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruFrequentlyUsed);
-
-        pdmBlkCacheLockLeave(pBlkCacheGlobal);
-
-        RTCritSectDelete(&pBlkCacheGlobal->CritSect);
-        RTMemFree(pBlkCacheGlobal);
-        pVM->pUVM->pdm.s.pBlkCacheGlobal = NULL;
-    }
-}
-
-int pdmR3BlkCacheResume(PVM pVM)
-{
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-
-    LogFlowFunc(("pVM=%#p\n", pVM));
-
-    if (   pBlkCacheGlobal
-        && ASMAtomicXchgBool(&pBlkCacheGlobal->fIoErrorVmSuspended, false))
-    {
-        /* The VM was suspended because of an I/O error, commit all dirty entries. */
-        pdmBlkCacheCommitDirtyEntries(pBlkCacheGlobal);
-    }
-
-    return VINF_SUCCESS;
-}
-
-static int pdmR3BlkCacheRetain(PVM pVM, PPPDMBLKCACHE ppBlkCache, const char *pcszId)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHE pBlkCache = NULL;
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-
-    if (!pBlkCacheGlobal)
-        return VERR_NOT_SUPPORTED;
-
-    /*
-     * Check that no other user cache has the same id first,
-     * Unique id's are necessary in case the state is saved.
-     */
-    pdmBlkCacheLockEnter(pBlkCacheGlobal);
-
-    pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pcszId);
-
-    if (!pBlkCache)
-    {
-        pBlkCache = (PPDMBLKCACHE)RTMemAllocZ(sizeof(PDMBLKCACHE));
-
-        if (pBlkCache)
-            pBlkCache->pszId = RTStrDup(pcszId);
-
-        if (   pBlkCache
-            && pBlkCache->pszId)
-        {
-            pBlkCache->fSuspended = false;
-            pBlkCache->pCache = pBlkCacheGlobal;
-            RTListInit(&pBlkCache->ListDirtyNotCommitted);
-
-            rc = RTSpinlockCreate(&pBlkCache->LockList);
-            if (RT_SUCCESS(rc))
-            {
-                rc = RTSemRWCreate(&pBlkCache->SemRWEntries);
-                if (RT_SUCCESS(rc))
-                {
-                    pBlkCache->pTree  = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
-                    if (pBlkCache->pTree)
-                    {
-#ifdef VBOX_WITH_STATISTICS
-                        STAMR3RegisterF(pBlkCacheGlobal->pVM, &pBlkCache->StatWriteDeferred,
-                                        STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
-                                        STAMUNIT_COUNT, "Number of deferred writes",
-                                        "/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
-#endif
-
-                        /* Add to the list of users. */
-                        pBlkCacheGlobal->cRefs++;
-                        RTListAppend(&pBlkCacheGlobal->ListUsers, &pBlkCache->NodeCacheUser);
-                        pdmBlkCacheLockLeave(pBlkCacheGlobal);
-
-                        *ppBlkCache = pBlkCache;
-                        LogFlowFunc(("returns success\n"));
-                        return VINF_SUCCESS;
-                    }
-                    else
-                        rc = VERR_NO_MEMORY;
-
-                    RTSemRWDestroy(pBlkCache->SemRWEntries);
-                }
-
-                RTSpinlockDestroy(pBlkCache->LockList);
-            }
-
-            RTStrFree(pBlkCache->pszId);
-        }
-        else
-            rc = VERR_NO_MEMORY;
-
-        if (pBlkCache)
-            RTMemFree(pBlkCache);
-    }
-    else
-        rc = VERR_ALREADY_EXISTS;
-
-    pdmBlkCacheLockLeave(pBlkCacheGlobal);
-
-    LogFlowFunc(("Leave rc=%Rrc\n", rc));
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3BlkCacheRetainDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
-                                         PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
-                                         PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
-                                         const char *pcszId)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHE pBlkCache;
-
-    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
-    if (RT_SUCCESS(rc))
-    {
-        pBlkCache->enmType = PDMBLKCACHETYPE_DRV;
-        pBlkCache->u.Drv.pfnXferComplete = pfnXferComplete;
-        pBlkCache->u.Drv.pfnXferEnqueue  = pfnXferEnqueue;
-        pBlkCache->u.Drv.pDrvIns         = pDrvIns;
-        *ppBlkCache = pBlkCache;
-    }
-
-    LogFlowFunc(("Leave rc=%Rrc\n", rc));
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3BlkCacheRetainDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMBLKCACHE ppBlkCache,
-                                         PFNPDMBLKCACHEXFERCOMPLETEDEV pfnXferComplete,
-                                         PFNPDMBLKCACHEXFERENQUEUEDEV pfnXferEnqueue,
-                                         const char *pcszId)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHE pBlkCache;
-
-    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
-    if (RT_SUCCESS(rc))
-    {
-        pBlkCache->enmType = PDMBLKCACHETYPE_DEV;
-        pBlkCache->u.Dev.pfnXferComplete = pfnXferComplete;
-        pBlkCache->u.Dev.pfnXferEnqueue  = pfnXferEnqueue;
-        pBlkCache->u.Dev.pDevIns         = pDevIns;
-        *ppBlkCache = pBlkCache;
-    }
-
-    LogFlowFunc(("Leave rc=%Rrc\n", rc));
-    return rc;
-
-}
-
-VMMR3DECL(int) PDMR3BlkCacheRetainUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMBLKCACHE ppBlkCache,
-                                      PFNPDMBLKCACHEXFERCOMPLETEUSB pfnXferComplete,
-                                      PFNPDMBLKCACHEXFERENQUEUEUSB pfnXferEnqueue,
-                                      const char *pcszId)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHE pBlkCache;
-
-    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
-    if (RT_SUCCESS(rc))
-    {
-        pBlkCache->enmType = PDMBLKCACHETYPE_USB;
-        pBlkCache->u.Usb.pfnXferComplete = pfnXferComplete;
-        pBlkCache->u.Usb.pfnXferEnqueue  = pfnXferEnqueue;
-        pBlkCache->u.Usb.pUsbIns         = pUsbIns;
-        *ppBlkCache = pBlkCache;
-    }
-
-    LogFlowFunc(("Leave rc=%Rrc\n", rc));
-    return rc;
-
-}
-
-VMMR3DECL(int) PDMR3BlkCacheRetainInt(PVM pVM, void *pvUser, PPPDMBLKCACHE ppBlkCache,
-                                      PFNPDMBLKCACHEXFERCOMPLETEINT pfnXferComplete,
-                                      PFNPDMBLKCACHEXFERENQUEUEINT pfnXferEnqueue,
-                                      const char *pcszId)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHE pBlkCache;
-
-    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
-    if (RT_SUCCESS(rc))
-    {
-        pBlkCache->enmType = PDMBLKCACHETYPE_INTERNAL;
-        pBlkCache->u.Int.pfnXferComplete = pfnXferComplete;
-        pBlkCache->u.Int.pfnXferEnqueue  = pfnXferEnqueue;
-        pBlkCache->u.Int.pvUser          = pvUser;
-        *ppBlkCache = pBlkCache;
-    }
-
-    LogFlowFunc(("Leave rc=%Rrc\n", rc));
-    return rc;
-
-}
-
-/**
- * Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
- *
- * @returns IPRT status code.
- * @param    pNode     The node to destroy.
- * @param    pvUser    Opaque user data.
- */
-static int pdmBlkCacheEntryDestroy(PAVLRU64NODECORE pNode, void *pvUser)
-{
-    PPDMBLKCACHEENTRY  pEntry = (PPDMBLKCACHEENTRY)pNode;
-    PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
-    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
-
-    while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
-    {
-        /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
-        pdmBlkCacheEntryRef(pEntry);
-        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-        pdmBlkCacheLockLeave(pCache);
-
-        RTThreadSleep(250);
-
-        /* Re-enter all locks */
-        pdmBlkCacheLockEnter(pCache);
-        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-        pdmBlkCacheEntryRelease(pEntry);
-    }
-
-    AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
-                ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
-
-    bool fUpdateCache =    pEntry->pList == &pCache->LruFrequentlyUsed
-                        || pEntry->pList == &pCache->LruRecentlyUsedIn;
-
-    pdmBlkCacheEntryRemoveFromList(pEntry);
-
-    if (fUpdateCache)
-        pdmBlkCacheSub(pCache, pEntry->cbData);
-
-    RTMemPageFree(pEntry->pbData, pEntry->cbData);
-    RTMemFree(pEntry);
-
-    return VINF_SUCCESS;
-}
-
-/**
- * Destroys all cache resources used by the given endpoint.
- *
- * @returns nothing.
- * @param    pEndpoint    The endpoint to the destroy.
- */
-VMMR3DECL(void) PDMR3BlkCacheRelease(PPDMBLKCACHE pBlkCache)
-{
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-
-    /*
-     * Commit all dirty entries now (they are waited on for completion during the
-     * destruction of the AVL tree below).
-     * The exception is if the VM was paused because of an I/O error before.
-     */
-    if (!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
-        pdmBlkCacheCommit(pBlkCache);
-
-    /* Make sure nobody is accessing the cache while we delete the tree. */
-    pdmBlkCacheLockEnter(pCache);
-    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-    RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
-    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-
-    RTSpinlockDestroy(pBlkCache->LockList);
-
-    pCache->cRefs--;
-    RTListNodeRemove(&pBlkCache->NodeCacheUser);
-
-    pdmBlkCacheLockLeave(pCache);
-
-    RTSemRWDestroy(pBlkCache->SemRWEntries);
-
-#ifdef VBOX_WITH_STATISTICS
-    STAMR3Deregister(pCache->pVM, &pBlkCache->StatWriteDeferred);
-#endif
-
-    RTStrFree(pBlkCache->pszId);
-    RTMemFree(pBlkCache);
-}
-
-VMMR3DECL(void) PDMR3BlkCacheReleaseDevice(PVM pVM, PPDMDEVINS pDevIns)
-{
-    LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pDevIns)
-        return;
-    VM_ASSERT_EMT(pVM);
-
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
-
-    /* Return silently if not supported. */
-    if (!pBlkCacheGlobal)
-        return;
-
-    pdmBlkCacheLockEnter(pBlkCacheGlobal);
-
-    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
-    {
-        if (    pBlkCache->enmType == PDMBLKCACHETYPE_DEV
-            &&  pBlkCache->u.Dev.pDevIns == pDevIns)
-            PDMR3BlkCacheRelease(pBlkCache);
-    }
-
-    pdmBlkCacheLockLeave(pBlkCacheGlobal);
-}
-
-VMMR3DECL(void) PDMR3BlkCacheReleaseDriver(PVM pVM, PPDMDRVINS pDrvIns)
-{
-    LogFlow(("%s: pDrvIns=%p\n", __FUNCTION__, pDrvIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pDrvIns)
-        return;
-    VM_ASSERT_EMT(pVM);
-
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
-
-    /* Return silently if not supported. */
-    if (!pBlkCacheGlobal)
-        return;
-
-    pdmBlkCacheLockEnter(pBlkCacheGlobal);
-
-    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
-    {
-        if (    pBlkCache->enmType == PDMBLKCACHETYPE_DRV
-            &&  pBlkCache->u.Drv.pDrvIns == pDrvIns)
-            PDMR3BlkCacheRelease(pBlkCache);
-    }
-
-    pdmBlkCacheLockLeave(pBlkCacheGlobal);
-}
-
-VMMR3DECL(void) PDMR3BlkCacheReleaseUsb(PVM pVM, PPDMUSBINS pUsbIns)
-{
-    LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pUsbIns)
-        return;
-    VM_ASSERT_EMT(pVM);
-
-    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
-    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
-
-    /* Return silently if not supported. */
-    if (!pBlkCacheGlobal)
-        return;
-
-    pdmBlkCacheLockEnter(pBlkCacheGlobal);
-
-    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
-    {
-        if (    pBlkCache->enmType == PDMBLKCACHETYPE_USB
-            &&  pBlkCache->u.Usb.pUsbIns == pUsbIns)
-            PDMR3BlkCacheRelease(pBlkCache);
-    }
-
-    pdmBlkCacheLockLeave(pBlkCacheGlobal);
-}
-
-static PPDMBLKCACHEENTRY pdmBlkCacheGetCacheEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off)
-{
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-    PPDMBLKCACHEENTRY pEntry = NULL;
-
-    STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
-
-    RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-    pEntry = (PPDMBLKCACHEENTRY)RTAvlrU64RangeGet(pBlkCache->pTree, off);
-    if (pEntry)
-        pdmBlkCacheEntryRef(pEntry);
-    RTSemRWReleaseRead(pBlkCache->SemRWEntries);
-
-    STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
-
-    return pEntry;
-}
-
-/**
- * Return the best fit cache entries for the given offset.
- *
- * @returns nothing.
- * @param   pBlkCache    The endpoint cache.
- * @param   off               The offset.
- * @param   pEntryAbove       Where to store the pointer to the best fit entry above the
- *                            the given offset. NULL if not required.
- */
-static void pdmBlkCacheGetCacheBestFitEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off,
-                                                    PPDMBLKCACHEENTRY *ppEntryAbove)
-{
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-
-    STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
-
-    RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-    if (ppEntryAbove)
-    {
-        *ppEntryAbove = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, true /*fAbove*/);
-        if (*ppEntryAbove)
-            pdmBlkCacheEntryRef(*ppEntryAbove);
-    }
-
-    RTSemRWReleaseRead(pBlkCache->SemRWEntries);
-
-    STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
-}
-
-static void pdmBlkCacheInsertEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
-{
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-
-    STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
-    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-    bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
-    AssertMsg(fInserted, ("Node was not inserted into tree\n"));
-    STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
-    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-}
-
-/**
- * Allocates and initializes a new entry for the cache.
- * The entry has a reference count of 1.
- *
- * @returns Pointer to the new cache entry or NULL if out of memory.
- * @param   pBlkCache The cache the entry belongs to.
- * @param   off       Start offset.
- * @param   cbData    Size of the cache entry.
- * @param   pbBuffer  Pointer to the buffer to use.
- *                    NULL if a new buffer should be allocated.
- *                    The buffer needs to have the same size of the entry.
- */
-static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
-                                               uint64_t off, size_t cbData, uint8_t *pbBuffer)
-{
-    PPDMBLKCACHEENTRY pEntryNew = (PPDMBLKCACHEENTRY)RTMemAllocZ(sizeof(PDMBLKCACHEENTRY));
-
-    if (RT_UNLIKELY(!pEntryNew))
-        return NULL;
-
-    pEntryNew->Core.Key      = off;
-    pEntryNew->Core.KeyLast  = off + cbData - 1;
-    pEntryNew->pBlkCache     = pBlkCache;
-    pEntryNew->fFlags        = 0;
-    pEntryNew->cRefs         = 1; /* We are using it now. */
-    pEntryNew->pList         = NULL;
-    pEntryNew->cbData        = cbData;
-    pEntryNew->pWaitingHead  = NULL;
-    pEntryNew->pWaitingTail  = NULL;
-    if (pbBuffer)
-        pEntryNew->pbData    = pbBuffer;
-    else
-        pEntryNew->pbData    = (uint8_t *)RTMemPageAlloc(cbData);
-
-    if (RT_UNLIKELY(!pEntryNew->pbData))
-    {
-        RTMemFree(pEntryNew);
-        return NULL;
-    }
-
-    return pEntryNew;
-}
-
-/**
- * Checks that a set of flags is set/clear acquiring the R/W semaphore
- * in exclusive mode.
- *
- * @returns true if the flag in fSet is set and the one in fClear is clear.
- *          false otherwise.
- *          The R/W semaphore is only held if true is returned.
- *
- * @param   pBlkCache   The endpoint cache instance data.
- * @param   pEntry           The entry to check the flags for.
- * @param   fSet             The flag which is tested to be set.
- * @param   fClear           The flag which is tested to be clear.
- */
-DECLINLINE(bool) pdmBlkCacheEntryFlagIsSetClearAcquireLock(PPDMBLKCACHE pBlkCache,
-                                                           PPDMBLKCACHEENTRY pEntry,
-                                                           uint32_t fSet, uint32_t fClear)
-{
-    uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
-    bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
-
-    if (fPassed)
-    {
-        /* Acquire the lock and check again because the completion callback might have raced us. */
-        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-
-        fFlags = ASMAtomicReadU32(&pEntry->fFlags);
-        fPassed = ((fFlags & fSet) && !(fFlags & fClear));
-
-        /* Drop the lock if we didn't passed the test. */
-        if (!fPassed)
-            RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-    }
-
-    return fPassed;
-}
-
-/**
- * Adds a segment to the waiting list for a cache entry
- * which is currently in progress.
- *
- * @returns nothing.
- * @param   pEntry    The cache entry to add the segment to.
- * @param   pSeg      The segment to add.
- */
-DECLINLINE(void) pdmBlkCacheEntryAddWaiter(PPDMBLKCACHEENTRY pEntry,
-                                           PPDMBLKCACHEWAITER pWaiter)
-{
-    pWaiter->pNext = NULL;
-
-    if (pEntry->pWaitingHead)
-    {
-        AssertPtr(pEntry->pWaitingTail);
-
-        pEntry->pWaitingTail->pNext = pWaiter;
-        pEntry->pWaitingTail = pWaiter;
-    }
-    else
-    {
-        Assert(!pEntry->pWaitingTail);
-
-        pEntry->pWaitingHead = pWaiter;
-        pEntry->pWaitingTail = pWaiter;
-    }
-}
-
-/**
- * Add a buffer described by the I/O memory context
- * to the entry waiting for completion.
- *
- * @returns VBox status code.
- * @param   pEntry    The entry to add the buffer to.
- * @param   pTask     Task associated with the buffer.
- * @param   pIoMemCtx The memory context to use.
- * @param   offDiff   Offset from the start of the buffer
- *                    in the entry.
- * @param   cbData    Amount of data to wait for onthis entry.
- * @param   fWrite    Flag whether the task waits because it wants to write
- *                    to the cache entry.
- */
-static int pdmBlkCacheEntryWaitersAdd(PPDMBLKCACHEENTRY pEntry,
-                                      PPDMBLKCACHEREQ pReq,
-                                      PRTSGBUF pSgBuf, uint64_t offDiff,
-                                      size_t cbData, bool fWrite)
-{
-    PPDMBLKCACHEWAITER pWaiter  = (PPDMBLKCACHEWAITER)RTMemAllocZ(sizeof(PDMBLKCACHEWAITER));
-    if (!pWaiter)
-        return VERR_NO_MEMORY;
-
-    ASMAtomicIncU32(&pReq->cXfersPending);
-    pWaiter->pReq          = pReq;
-    pWaiter->offCacheEntry = offDiff;
-    pWaiter->cbTransfer    = cbData;
-    pWaiter->fWrite        = fWrite;
-    RTSgBufClone(&pWaiter->SgBuf, pSgBuf);
-    RTSgBufAdvance(pSgBuf, cbData);
-
-    pdmBlkCacheEntryAddWaiter(pEntry, pWaiter);
-
-    return VINF_SUCCESS;
-}
-
-/**
- * Calculate aligned offset and size for a new cache entry
- * which do not intersect with an already existing entry and the
- * file end.
- *
- * @returns The number of bytes the entry can hold of the requested amount
- *          of byte.
- * @param   pEndpoint        The endpoint.
- * @param   pBlkCache   The endpoint cache.
- * @param   off              The start offset.
- * @param   cb               The number of bytes the entry needs to hold at least.
- * @param   uAlignment       Alignment of the boundary sizes.
- * @param   poffAligned      Where to store the aligned offset.
- * @param   pcbAligned       Where to store the aligned size of the entry.
- */
-static size_t pdmBlkCacheEntryBoundariesCalc(PPDMBLKCACHE pBlkCache,
-                                             uint64_t off, size_t cb,
-                                             unsigned uAlignment,
-                                             uint64_t *poffAligned, size_t *pcbAligned)
-{
-    size_t cbAligned;
-    size_t cbInEntry = 0;
-    uint64_t offAligned;
-    PPDMBLKCACHEENTRY pEntryAbove = NULL;
-
-    /* Get the best fit entries around the offset */
-    pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
-
-    /* Log the info */
-    LogFlow(("%sest fit entry above off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
-             pEntryAbove ? "B" : "No b",
-             off,
-             pEntryAbove ? pEntryAbove->Core.Key : 0,
-             pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
-             pEntryAbove ? pEntryAbove->cbData : 0));
-
-    offAligned = off;
-
-    if (    pEntryAbove
-        &&  off + cb > pEntryAbove->Core.Key)
-    {
-        cbInEntry = pEntryAbove->Core.Key - off;
-        cbAligned = pEntryAbove->Core.Key - offAligned;
-    }
-    else
-    {
-        cbAligned = cb;
-        cbInEntry = cb;
-    }
-
-    /* A few sanity checks */
-    AssertMsg(!pEntryAbove || (offAligned + cbAligned) <= pEntryAbove->Core.Key,
-              ("Aligned size intersects with another cache entry\n"));
-    Assert(cbInEntry <= cbAligned);
-
-    if (pEntryAbove)
-        pdmBlkCacheEntryRelease(pEntryAbove);
-
-    LogFlow(("offAligned=%llu cbAligned=%u\n", offAligned, cbAligned));
-
-    *poffAligned = offAligned;
-    *pcbAligned  = cbAligned;
-
-    return cbInEntry;
-}
-
-/**
- * Create a new cache entry evicting data from the cache if required.
- *
- * @returns Pointer to the new cache entry or NULL
- *          if not enough bytes could be evicted from the cache.
- * @param   pEndpoint         The endpoint.
- * @param   pBlkCache    The endpoint cache.
- * @param   off               The offset.
- * @param   cb                Number of bytes the cache entry should have.
- * @param   uAlignment        Alignment the size of the entry should have.
- * @param   pcbData           Where to store the number of bytes the new
- *                            entry can hold. May be lower than actually requested
- *                            due to another entry intersecting the access range.
- */
-static PPDMBLKCACHEENTRY pdmBlkCacheEntryCreate(PPDMBLKCACHE pBlkCache,
-                                                uint64_t off, size_t cb,
-                                                unsigned uAlignment,
-                                                size_t *pcbData)
-{
-    uint64_t offStart = 0;
-    size_t cbEntry = 0;
-    PPDMBLKCACHEENTRY pEntryNew = NULL;
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-    uint8_t *pbBuffer = NULL;
-
-    *pcbData = pdmBlkCacheEntryBoundariesCalc(pBlkCache, off, cb, uAlignment,
-                                              &offStart, &cbEntry);
-
-    pdmBlkCacheLockEnter(pCache);
-    bool fEnough = pdmBlkCacheReclaim(pCache, cbEntry, true, &pbBuffer);
-
-    if (fEnough)
-    {
-        LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
-
-        pEntryNew = pdmBlkCacheEntryAlloc(pBlkCache, offStart, cbEntry, pbBuffer);
-        if (RT_LIKELY(pEntryNew))
-        {
-            pdmBlkCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
-            pdmBlkCacheAdd(pCache, cbEntry);
-            pdmBlkCacheLockLeave(pCache);
-
-            pdmBlkCacheInsertEntry(pBlkCache, pEntryNew);
-
-            AssertMsg(   (off >= pEntryNew->Core.Key)
-                      && (off + *pcbData <= pEntryNew->Core.KeyLast + 1),
-                      ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
-                       off, pEntryNew->Core.Key));
-        }
-        else
-            pdmBlkCacheLockLeave(pCache);
-    }
-    else
-        pdmBlkCacheLockLeave(pCache);
-
-    return pEntryNew;
-}
-
-static PPDMBLKCACHEREQ pdmBlkCacheReqAlloc(size_t cbXfer, void *pvUser)
-{
-    PPDMBLKCACHEREQ pReq = (PPDMBLKCACHEREQ)RTMemAlloc(sizeof(PDMBLKCACHEREQ));
-
-    if (RT_LIKELY(pReq))
-    {
-        pReq->pvUser = pvUser;
-        pReq->cbXfer = cbXfer;
-        pReq->rcReq  = VINF_SUCCESS;
-        pReq->cXfersPending = 0;
-    }
-
-    return pReq;
-}
-
-static void pdmBlkCacheReqComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq)
-{
-    switch (pBlkCache->enmType)
-    {
-        case PDMBLKCACHETYPE_DEV:
-        {
-            pBlkCache->u.Dev.pfnXferComplete(pBlkCache->u.Dev.pDevIns,
-                                             pReq->pvUser, pReq->rcReq);
-            break;
-        }
-        case PDMBLKCACHETYPE_DRV:
-        {
-            pBlkCache->u.Drv.pfnXferComplete(pBlkCache->u.Drv.pDrvIns,
-                                             pReq->pvUser, pReq->rcReq);
-            break;
-        }
-        case PDMBLKCACHETYPE_USB:
-        {
-            pBlkCache->u.Usb.pfnXferComplete(pBlkCache->u.Usb.pUsbIns,
-                                             pReq->pvUser, pReq->rcReq);
-            break;
-        }
-        case PDMBLKCACHETYPE_INTERNAL:
-        {
-            pBlkCache->u.Int.pfnXferComplete(pBlkCache->u.Int.pvUser,
-                                             pReq->pvUser, pReq->rcReq);
-            break;
-        }
-        default:
-            AssertMsgFailed(("Unknown block cache type!\n"));
-    }
-
-    RTMemFree(pReq);
-}
-
-static bool pdmBlkCacheReqUpdate(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
-                                 size_t cbComplete, int rcReq, bool fCallHandler)
-{
-    if (RT_FAILURE(rcReq))
-        ASMAtomicCmpXchgS32(&pReq->rcReq, rcReq, VINF_SUCCESS);
-
-    AssertMsg(pReq->cbXfer >= cbComplete, ("Completed more than left\n"));
-    AssertMsg(pReq->cXfersPending > 0, ("No transfers are pending for this request\n"));
-    uint32_t cbOld = ASMAtomicSubU32(&pReq->cbXfer, cbComplete);
-    uint32_t cXfersPending = ASMAtomicDecU32(&pReq->cXfersPending);
-
-    if (   !(cbOld - cbComplete)
-        && !cXfersPending)
-    {
-        if (fCallHandler)
-            pdmBlkCacheReqComplete(pBlkCache, pReq);
-        return true;
-    }
-
-    LogFlowFunc(("pReq=%#p cXfersPending=%u cbXfer=%u\n", pReq, cXfersPending, (cbOld - cbComplete)));
-    return false;
-}
-
-VMMR3DECL(int) PDMR3BlkCacheRead(PPDMBLKCACHE pBlkCache, uint64_t off,
-                                 PCRTSGBUF pcSgBuf, size_t cbRead, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-    PPDMBLKCACHEENTRY  pEntry;
-    PPDMBLKCACHEREQ    pReq;
-
-    LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbRead=%u pvUser=%#p\n",
-                 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbRead, pvUser));
-
-    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
-    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
-
-    RTSGBUF SgBuf;
-    RTSgBufClone(&SgBuf, pcSgBuf);
-
-    /* Allocate new request structure. */
-    pReq = pdmBlkCacheReqAlloc(cbRead, pvUser);
-    if (RT_UNLIKELY(!pReq))
-        return VERR_NO_MEMORY;
-
-    /* Increment data transfer counter to keep the request valid while we access it. */
-    ASMAtomicIncU32(&pReq->cXfersPending);
-
-    while (cbRead)
-    {
-        size_t cbToRead;
-
-        pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
-
-        /*
-         * If there is no entry we try to create a new one eviciting unused pages
-         * if the cache is full. If this is not possible we will pass the request through
-         * and skip the caching (all entries may be still in progress so they can't
-         * be evicted)
-         * If we have an entry it can be in one of the LRU lists where the entry
-         * contains data (recently used or frequently used LRU) so we can just read
-         * the data we need and put the entry at the head of the frequently used LRU list.
-         * In case the entry is in one of the ghost lists it doesn't contain any data.
-         * We have to fetch it again evicting pages from either T1 or T2 to make room.
-         */
-        if (pEntry)
-        {
-            uint64_t offDiff = off - pEntry->Core.Key;
-
-            AssertMsg(off >= pEntry->Core.Key,
-                      ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
-                      off, pEntry->Core.Key));
-
-            AssertPtr(pEntry->pList);
-
-            cbToRead = RT_MIN(pEntry->cbData - offDiff, cbRead);
-
-            AssertMsg(off + cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
-                      ("Buffer of cache entry exceeded off=%llu cbToRead=%d\n",
-                       off, cbToRead));
-
-            cbRead  -= cbToRead;
-
-            if (!cbRead)
-                STAM_COUNTER_INC(&pCache->cHits);
-            else
-                STAM_COUNTER_INC(&pCache->cPartialHits);
-
-            STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
-
-            /* Ghost lists contain no data. */
-            if (   (pEntry->pList == &pCache->LruRecentlyUsedIn)
-                || (pEntry->pList == &pCache->LruFrequentlyUsed))
-            {
-                if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
-                                                              PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
-                                                              PDMBLKCACHE_ENTRY_IS_DIRTY))
-                {
-                    /* Entry didn't completed yet. Append to the list */
-                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
-                                               &SgBuf, offDiff, cbToRead,
-                                               false /* fWrite */);
-                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-                }
-                else
-                {
-                    /* Read as much as we can from the entry. */
-                    RTSgBufCopyFromBuf(&SgBuf, pEntry->pbData + offDiff, cbToRead);
-                    ASMAtomicSubU32(&pReq->cbXfer, cbToRead);
-                }
-
-                /* Move this entry to the top position */
-                if (pEntry->pList == &pCache->LruFrequentlyUsed)
-                {
-                    pdmBlkCacheLockEnter(pCache);
-                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
-                    pdmBlkCacheLockLeave(pCache);
-                }
-                /* Release the entry */
-                pdmBlkCacheEntryRelease(pEntry);
-            }
-            else
-            {
-                uint8_t *pbBuffer = NULL;
-
-                LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
-
-                pdmBlkCacheLockEnter(pCache);
-                pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
-                bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
-
-                /* Move the entry to Am and fetch it to the cache. */
-                if (fEnough)
-                {
-                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
-                    pdmBlkCacheAdd(pCache, pEntry->cbData);
-                    pdmBlkCacheLockLeave(pCache);
-
-                    if (pbBuffer)
-                        pEntry->pbData = pbBuffer;
-                    else
-                        pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
-                    AssertPtr(pEntry->pbData);
-
-                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
-                                               &SgBuf, offDiff, cbToRead,
-                                               false /* fWrite */);
-                    pdmBlkCacheEntryReadFromMedium(pEntry);
-                    /* Release the entry */
-                    pdmBlkCacheEntryRelease(pEntry);
-                }
-                else
-                {
-                    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
-                    RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
-                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
-                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-
-                    pdmBlkCacheLockLeave(pCache);
-
-                    RTMemFree(pEntry);
-
-                    pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
-                                                  &SgBuf, off, cbToRead,
-                                                  PDMBLKCACHEXFERDIR_READ);
-                }
-            }
-        }
-        else
-        {
-#ifdef VBOX_WITH_IO_READ_CACHE
-            /* No entry found for this offset. Create a new entry and fetch the data to the cache. */
-            PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
-                                                                 off, cbRead,
-                                                                 PAGE_SIZE,
-                                                                 &cbToRead);
-
-            cbRead -= cbToRead;
-
-            if (pEntryNew)
-            {
-                if (!cbRead)
-                    STAM_COUNTER_INC(&pCache->cMisses);
-                else
-                    STAM_COUNTER_INC(&pCache->cPartialHits);
-
-                pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
-                                           &SgBuf,
-                                           off - pEntryNew->Core.Key,
-                                           cbToRead,
-                                           false /* fWrite */);
-                pdmBlkCacheEntryReadFromMedium(pEntryNew);
-                pdmBlkCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
-            }
-            else
-            {
-                /*
-                 * There is not enough free space in the cache.
-                 * Pass the request directly to the I/O manager.
-                 */
-                LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
-
-                pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
-                                              &SgBuf, off, cbToRead,
-                                              PDMBLKCACHEXFERDIR_READ);
-            }
-#else
-            /* Clip read size if necessary. */
-            PPDMBLKCACHEENTRY pEntryAbove;
-            pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
-
-            if (pEntryAbove)
-            {
-                if (off + cbRead > pEntryAbove->Core.Key)
-                    cbToRead = pEntryAbove->Core.Key - off;
-                else
-                    cbToRead = cbRead;
-
-                pdmBlkCacheEntryRelease(pEntryAbove);
-            }
-            else
-                cbToRead = cbRead;
-
-            cbRead -= cbToRead;
-            pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
-                                          &SgBuf, off, cbToRead,
-                                          PDMBLKCACHEXFERDIR_READ);
-#endif
-        }
-        off += cbToRead;
-    }
-
-    if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
-        rc = VINF_AIO_TASK_PENDING;
-
-    LogFlowFunc((": Leave rc=%Rrc\n", rc));
-
-   return rc;
-}
-
-VMMR3DECL(int) PDMR3BlkCacheWrite(PPDMBLKCACHE pBlkCache, uint64_t off,
-                                  PCRTSGBUF pcSgBuf, size_t cbWrite, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
-    PPDMBLKCACHEENTRY pEntry;
-    PPDMBLKCACHEREQ pReq;
-
-    LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbWrite=%u pvUser=%#p\n",
-                 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbWrite, pvUser));
-
-    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
-    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
-
-    RTSGBUF SgBuf;
-    RTSgBufClone(&SgBuf, pcSgBuf);
-
-    /* Allocate new request structure. */
-    pReq = pdmBlkCacheReqAlloc(cbWrite, pvUser);
-    if (RT_UNLIKELY(!pReq))
-        return VERR_NO_MEMORY;
-
-    /* Increment data transfer counter to keep the request valid while we access it. */
-    ASMAtomicIncU32(&pReq->cXfersPending);
-
-    while (cbWrite)
-    {
-        size_t cbToWrite;
-
-        pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
-
-        if (pEntry)
-        {
-            /* Write the data into the entry and mark it as dirty */
-            AssertPtr(pEntry->pList);
-
-            uint64_t offDiff = off - pEntry->Core.Key;
-
-            AssertMsg(off >= pEntry->Core.Key,
-                      ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
-                      off, pEntry->Core.Key));
-
-            cbToWrite = RT_MIN(pEntry->cbData - offDiff, cbWrite);
-            cbWrite  -= cbToWrite;
-
-            if (!cbWrite)
-                STAM_COUNTER_INC(&pCache->cHits);
-            else
-                STAM_COUNTER_INC(&pCache->cPartialHits);
-
-            STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
-
-            /* Ghost lists contain no data. */
-            if (   (pEntry->pList == &pCache->LruRecentlyUsedIn)
-                || (pEntry->pList == &pCache->LruFrequentlyUsed))
-            {
-                /* Check if the entry is dirty. */
-                if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
-                                                              PDMBLKCACHE_ENTRY_IS_DIRTY,
-                                                              0))
-                {
-                    /* If it is already dirty but not in progress just update the data. */
-                    if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
-                    {
-                        RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff,
-                                         cbToWrite);
-                        ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
-                    }
-                    else
-                    {
-                        /* The data isn't written to the file yet */
-                        pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
-                                                   &SgBuf, offDiff, cbToWrite,
-                                                   true /* fWrite */);
-                        STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
-                    }
-
-                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-                }
-                else /* Dirty bit not set */
-                {
-                    /*
-                     * Check if a read is in progress for this entry.
-                     * We have to defer processing in that case.
-                     */
-                    if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
-                                                                 PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
-                                                                 0))
-                    {
-                        pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
-                                                   &SgBuf, offDiff, cbToWrite,
-                                                   true /* fWrite */);
-                        STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
-                        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-                    }
-                    else /* I/O in progress flag not set */
-                    {
-                        /* Write as much as we can into the entry and update the file. */
-                        RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
-                        ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
-
-                        bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
-                        if (fCommit)
-                            pdmBlkCacheCommitDirtyEntries(pCache);
-                    }
-                } /* Dirty bit not set */
-
-                /* Move this entry to the top position */
-                if (pEntry->pList == &pCache->LruFrequentlyUsed)
-                {
-                    pdmBlkCacheLockEnter(pCache);
-                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
-                    pdmBlkCacheLockLeave(pCache);
-                }
-
-                pdmBlkCacheEntryRelease(pEntry);
-            }
-            else /* Entry is on the ghost list */
-            {
-                uint8_t *pbBuffer = NULL;
-
-                pdmBlkCacheLockEnter(pCache);
-                pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
-                bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
-
-                if (fEnough)
-                {
-                    /* Move the entry to Am and fetch it to the cache. */
-                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
-                    pdmBlkCacheAdd(pCache, pEntry->cbData);
-                    pdmBlkCacheLockLeave(pCache);
-
-                    if (pbBuffer)
-                        pEntry->pbData = pbBuffer;
-                    else
-                        pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
-                    AssertPtr(pEntry->pbData);
-
-                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
-                                               &SgBuf, offDiff, cbToWrite,
-                                               true /* fWrite */);
-                    STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
-                    pdmBlkCacheEntryReadFromMedium(pEntry);
-
-                    /* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
-                    pdmBlkCacheEntryRelease(pEntry);
-                }
-                else
-                {
-                    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
-                    RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
-                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
-                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-
-                    pdmBlkCacheLockLeave(pCache);
-
-                    RTMemFree(pEntry);
-                    pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
-                                                  &SgBuf, off, cbToWrite,
-                                                  PDMBLKCACHEXFERDIR_WRITE);
-                }
-            }
-        }
-        else /* No entry found */
-        {
-            /*
-             * No entry found. Try to create a new cache entry to store the data in and if that fails
-             * write directly to the file.
-             */
-            PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
-                                                                 off, cbWrite,
-                                                                 512, &cbToWrite);
-
-            cbWrite -= cbToWrite;
-
-            if (pEntryNew)
-            {
-                uint64_t offDiff = off - pEntryNew->Core.Key;
-
-                STAM_COUNTER_INC(&pCache->cHits);
-
-                /*
-                 * Check if it is possible to just write the data without waiting
-                 * for it to get fetched first.
-                 */
-                if (!offDiff && pEntryNew->cbData == cbToWrite)
-                {
-                    RTSgBufCopyToBuf(&SgBuf, pEntryNew->pbData, cbToWrite);
-                    ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
-
-                    bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntryNew);
-                    if (fCommit)
-                        pdmBlkCacheCommitDirtyEntries(pCache);
-                    STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
-                }
-                else
-                {
-                    /* Defer the write and fetch the data from the endpoint. */
-                    pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
-                                               &SgBuf, offDiff, cbToWrite,
-                                               true /* fWrite */);
-                    STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
-                    pdmBlkCacheEntryReadFromMedium(pEntryNew);
-                }
-
-                pdmBlkCacheEntryRelease(pEntryNew);
-            }
-            else
-            {
-                /*
-                 * There is not enough free space in the cache.
-                 * Pass the request directly to the I/O manager.
-                 */
-                LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
-
-                STAM_COUNTER_INC(&pCache->cMisses);
-
-                pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
-                                              &SgBuf, off, cbToWrite,
-                                              PDMBLKCACHEXFERDIR_WRITE);
-            }
-        }
-
-        off += cbToWrite;
-    }
-
-    if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
-        rc = VINF_AIO_TASK_PENDING;
-
-    LogFlowFunc((": Leave rc=%Rrc\n", rc));
-
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3BlkCacheFlush(PPDMBLKCACHE pBlkCache, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-    PPDMBLKCACHEREQ pReq;
-
-    LogFlowFunc((": pBlkCache=%#p{%s}\n", pBlkCache, pBlkCache->pszId));
-
-    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
-    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
-
-    /* Commit dirty entries in the cache. */
-    pdmBlkCacheCommit(pBlkCache);
-
-    /* Allocate new request structure. */
-    pReq = pdmBlkCacheReqAlloc(0, pvUser);
-    if (RT_UNLIKELY(!pReq))
-        return VERR_NO_MEMORY;
-
-    rc = pdmBlkCacheRequestPassthrough(pBlkCache, pReq, NULL, 0, 0,
-                                       PDMBLKCACHEXFERDIR_FLUSH);
-    AssertRC(rc);
-
-    LogFlowFunc((": Leave rc=%Rrc\n", rc));
-    return VINF_AIO_TASK_PENDING;
-}
-
-/**
- * Completes a task segment freeing all resources and completes the task handle
- * if everything was transferred.
- *
- * @returns Next task segment handle.
- * @param   pTaskSeg          Task segment to complete.
- * @param   rc                Status code to set.
- */
-static PPDMBLKCACHEWAITER pdmBlkCacheWaiterComplete(PPDMBLKCACHE pBlkCache,
-                                                    PPDMBLKCACHEWAITER pWaiter,
-                                                    int rc)
-{
-    PPDMBLKCACHEWAITER pNext = pWaiter->pNext;
-    PPDMBLKCACHEREQ pReq = pWaiter->pReq;
-
-    pdmBlkCacheReqUpdate(pBlkCache, pWaiter->pReq, pWaiter->cbTransfer, rc, true);
-
-    RTMemFree(pWaiter);
-
-    return pNext;
-}
-
-static void pdmBlkCacheIoXferCompleteEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
-{
-    PPDMBLKCACHEENTRY  pEntry    = hIoXfer->pEntry;
-    PPDMBLKCACHEGLOBAL pCache    = pBlkCache->pCache;
-
-    /* Reference the entry now as we are clearing the I/O in progress flag
-     * which protected the entry till now. */
-    pdmBlkCacheEntryRef(pEntry);
-
-    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-    pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
-
-    /* Process waiting segment list. The data in entry might have changed in-between. */
-    bool fDirty = false;
-    PPDMBLKCACHEWAITER pComplete = pEntry->pWaitingHead;
-    PPDMBLKCACHEWAITER pCurr     = pComplete;
-
-    AssertMsg((pCurr && pEntry->pWaitingTail) || (!pCurr && !pEntry->pWaitingTail),
-                ("The list tail was not updated correctly\n"));
-    pEntry->pWaitingTail = NULL;
-    pEntry->pWaitingHead = NULL;
-
-    if (hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_WRITE)
-    {
-        /*
-         * An error here is difficult to handle as the original request completed already.
-         * The error is logged for now and the VM is paused.
-         * If the user continues the entry is written again in the hope
-         * the user fixed the problem and the next write succeeds.
-         */
-        if (RT_FAILURE(rcIoXfer))
-        {
-            LogRel(("I/O cache: Error while writing entry at offset %llu (%u bytes) to medium \"%s\" (rc=%Rrc)\n",
-                    pEntry->Core.Key, pEntry->cbData, pBlkCache->pszId, rcIoXfer));
-
-            if (!ASMAtomicXchgBool(&pCache->fIoErrorVmSuspended, true))
-            {
-                int rc = VMSetRuntimeError(pCache->pVM, VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_NO_WAIT, "BLKCACHE_IOERR",
-                                           N_("The I/O cache encountered an error while updating data in medium \"%s\" (rc=%Rrc). "
-                                              "Make sure there is enough free space on the disk and that the disk is working properly. "
-                                              "Operation can be resumed afterwards"),
-                                           pBlkCache->pszId, rcIoXfer);
-                AssertRC(rc);
-            }
-
-            /*
-             * The entry is still marked as dirty which prevents eviction.
-             * Add the waiters to the list again.
-             */
-            pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY; /* Clear so it gets added to the list again. */
-            fDirty = true;
-
-            if (pComplete)
-            {
-                pEntry->pWaitingHead = pComplete;
-                while (pComplete->pNext)
-                    pComplete = pComplete->pNext;
-                pEntry->pWaitingTail = pComplete;
-                pComplete = NULL;
-            }
-        }
-        else
-        {
-            pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY;
-
-            while (pCurr)
-            {
-                AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
-
-                RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
-                fDirty = true;
-
-                pCurr = pCurr->pNext;
-            }
-        }
-    }
-    else
-    {
-        AssertMsg(hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_READ, ("Invalid transfer type\n"));
-        AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY),
-                  ("Invalid flags set\n"));
-
-        while (pCurr)
-        {
-            if (pCurr->fWrite)
-            {
-                RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
-                fDirty = true;
-            }
-            else
-                RTSgBufCopyFromBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
-
-            pCurr = pCurr->pNext;
-        }
-    }
-
-    bool fCommit = false;
-    if (fDirty)
-        fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
-
-    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-
-    /* Dereference so that it isn't protected anymore except we issued anyother write for it. */
-    pdmBlkCacheEntryRelease(pEntry);
-
-    if (fCommit)
-        pdmBlkCacheCommitDirtyEntries(pCache);
-
-    /* Complete waiters now. */
-    while (pComplete)
-        pComplete = pdmBlkCacheWaiterComplete(pBlkCache, pComplete, rcIoXfer);
-}
-
-VMMR3DECL(void) PDMR3BlkCacheIoXferComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
-{
-    LogFlowFunc(("pBlkCache=%#p hIoXfer=%#p rcIoXfer=%Rrc\n", pBlkCache, hIoXfer, rcIoXfer));
-
-    if (hIoXfer->fIoCache)
-        pdmBlkCacheIoXferCompleteEntry(pBlkCache, hIoXfer, rcIoXfer);
-    else
-        pdmBlkCacheReqUpdate(pBlkCache, hIoXfer->pReq, hIoXfer->cbXfer, rcIoXfer, true);
-}
-
-/**
- * Callback for the AVL do with all routine. Waits for a cachen entry to finish any pending I/O.
- *
- * @returns IPRT status code.
- * @param    pNode     The node to destroy.
- * @param    pvUser    Opaque user data.
- */
-static int pdmBlkCacheEntryQuiesce(PAVLRU64NODECORE pNode, void *pvUser)
-{
-    PPDMBLKCACHEENTRY  pEntry = (PPDMBLKCACHEENTRY)pNode;
-    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
-
-    while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
-    {
-        /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
-        pdmBlkCacheEntryRef(pEntry);
-        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-
-        RTThreadSleep(1);
-
-        /* Re-enter all locks and drop the reference. */
-        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-        pdmBlkCacheEntryRelease(pEntry);
-    }
-
-    AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
-                ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
-
-    return VINF_SUCCESS;
-}
-
-VMMR3DECL(int) PDMR3BlkCacheSuspend(PPDMBLKCACHE pBlkCache)
-{
-    int rc = VINF_SUCCESS;
-    LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
-
-    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
-
-    ASMAtomicXchgBool(&pBlkCache->fSuspended, true);
-
-    /* Wait for all I/O to complete. */
-    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
-    rc = RTAvlrU64DoWithAll(pBlkCache->pTree, true, pdmBlkCacheEntryQuiesce, NULL);
-    AssertRC(rc);
-    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
-
-    return rc;
-}
-
-VMMR3DECL(int) PDMR3BlkCacheResume(PPDMBLKCACHE pBlkCache)
-{
-    LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
-
-    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
-
-    ASMAtomicXchgBool(&pBlkCache->fSuspended, false);
-
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/PDMBlkCacheInternal.h
===================================================================
--- /trunk/src/VBox/VMM/PDMBlkCacheInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,320 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Block Cache.
- */
-
-/*
- * Copyright (C) 2006-2008 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___PDMBlkCacheInternal_h
-#define ___PDMBlkCacheInternal_h
-
-#include <VBox/cfgm.h>
-#include <VBox/stam.h>
-#include <VBox/tm.h>
-#include <VBox/pdmblkcache.h>
-#include <iprt/types.h>
-#include <iprt/file.h>
-#include <iprt/thread.h>
-#include <iprt/semaphore.h>
-#include <iprt/critsect.h>
-#include <iprt/avl.h>
-#include <iprt/list.h>
-#include <iprt/spinlock.h>
-#include <iprt/memcache.h>
-
-RT_C_DECLS_BEGIN
-
-/**
- * A few forward declarations.
- */
-/** Pointer to a cache LRU list. */
-typedef struct PDMBLKLRULIST *PPDMBLKLRULIST;
-/** Pointer to the global cache structure. */
-typedef struct PDMBLKCACHEGLOBAL *PPDMBLKCACHEGLOBAL;
-/** Pointer to a cache entry waiter structure. */
-typedef struct PDMBLKCACHEWAITER *PPDMBLKCACHEWAITER;
-
-/**
- * A cache entry
- */
-typedef struct PDMBLKCACHEENTRY
-{
-    /** The AVL entry data. */
-    AVLRU64NODECORE                 Core;
-    /** Pointer to the previous element. Used in one of the LRU lists.*/
-    struct PDMBLKCACHEENTRY        *pPrev;
-    /** Pointer to the next element. Used in one of the LRU lists.*/
-    struct PDMBLKCACHEENTRY        *pNext;
-    /** Pointer to the list the entry is in. */
-    PPDMBLKLRULIST                  pList;
-    /** Cache the entry belongs to. */
-    PPDMBLKCACHE                    pBlkCache;
-    /** Flags for this entry. Combinations of PDMACFILECACHE_* #defines */
-    volatile uint32_t               fFlags;
-    /** Reference counter. Prevents eviction of the entry if > 0. */
-    volatile uint32_t               cRefs;
-    /** Size of the entry. */
-    size_t                          cbData;
-    /** Pointer to the memory containing the data. */
-    uint8_t                        *pbData;
-    /** Head of list of tasks waiting for this one to finish. */
-    PPDMBLKCACHEWAITER              pWaitingHead;
-    /** Tail of list of tasks waiting for this one to finish. */
-    PPDMBLKCACHEWAITER              pWaitingTail;
-    /** Node for dirty but not yet committed entries list per endpoint. */
-    RTLISTNODE                      NodeNotCommitted;
-} PDMBLKCACHEENTRY, *PPDMBLKCACHEENTRY;
-/** I/O is still in progress for this entry. This entry is not evictable. */
-#define PDMBLKCACHE_ENTRY_IO_IN_PROGRESS RT_BIT(0)
-/** Entry is locked and thus not evictable. */
-#define PDMBLKCACHE_ENTRY_LOCKED         RT_BIT(1)
-/** Entry is dirty */
-#define PDMBLKCACHE_ENTRY_IS_DIRTY       RT_BIT(2)
-/** Entry is not evictable. */
-#define PDMBLKCACHE_NOT_EVICTABLE  (PDMBLKCACHE_ENTRY_LOCKED | PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)
-
-/**
- * LRU list data
- */
-typedef struct PDMBLKLRULIST
-{
-    /** Head of the list. */
-    PPDMBLKCACHEENTRY pHead;
-    /** Tail of the list. */
-    PPDMBLKCACHEENTRY pTail;
-    /** Number of bytes cached in the list. */
-    uint32_t          cbCached;
-} PDMBLKLRULIST;
-
-/**
- * Global cache data.
- */
-typedef struct PDMBLKCACHEGLOBAL
-{
-    /** Pointer to the owning VM instance. */
-    PVM               pVM;
-    /** Maximum size of the cache in bytes. */
-    uint32_t          cbMax;
-    /** Current size of the cache in bytes. */
-    uint32_t          cbCached;
-    /** Critical section protecting the cache. */
-    RTCRITSECT        CritSect;
-    /** Maximum number of bytes cached. */
-    uint32_t          cbRecentlyUsedInMax;
-    /** Maximum number of bytes in the paged out list .*/
-    uint32_t          cbRecentlyUsedOutMax;
-    /** Recently used cache entries list */
-    PDMBLKLRULIST     LruRecentlyUsedIn;
-    /** Scorecard cache entry list. */
-    PDMBLKLRULIST     LruRecentlyUsedOut;
-    /** List of frequently used cache entries */
-    PDMBLKLRULIST     LruFrequentlyUsed;
-    /** Commit timeout in milli seconds */
-    uint32_t          u32CommitTimeoutMs;
-    /** Number of dirty bytes needed to start a commit of the data to the disk. */
-    uint32_t          cbCommitDirtyThreshold;
-    /** Current number of dirty bytes in the cache. */
-    volatile uint32_t cbDirty;
-    /** Flag whether a commit is currently in progress. */
-    volatile bool     fCommitInProgress;
-    /** Commit interval timer */
-    PTMTIMERR3        pTimerCommit;
-    /** Number of endpoints using the cache. */
-    uint32_t          cRefs;
-    /** List of all users of this cache. */
-    RTLISTNODE        ListUsers;
-#ifdef VBOX_WITH_STATISTICS
-    /** Hit counter. */
-    STAMCOUNTER       cHits;
-    /** Partial hit counter. */
-    STAMCOUNTER       cPartialHits;
-    /** Miss counter. */
-    STAMCOUNTER       cMisses;
-    /** Bytes read from cache. */
-    STAMCOUNTER       StatRead;
-    /** Bytes written to the cache. */
-    STAMCOUNTER       StatWritten;
-    /** Time spend to get an entry in the AVL tree. */
-    STAMPROFILEADV    StatTreeGet;
-    /** Time spend to insert an entry in the AVL tree. */
-    STAMPROFILEADV    StatTreeInsert;
-    /** Time spend to remove an entry in the AVL tree. */
-    STAMPROFILEADV    StatTreeRemove;
-    /** Number of times a buffer could be reused. */
-    STAMCOUNTER       StatBuffersReused;
-#endif
-    /** Flag whether the VM was suspended becaus of an I/O error. */
-    volatile bool     fIoErrorVmSuspended;
-} PDMBLKCACHEGLOBAL;
-#ifdef VBOX_WITH_STATISTICS
-AssertCompileMemberAlignment(PDMBLKCACHEGLOBAL, cHits, sizeof(uint64_t));
-#endif
-
-/**
- * Block cache type.
- */
-typedef enum PDMBLKCACHETYPE
-{
-    /** Device . */
-    PDMBLKCACHETYPE_DEV = 1,
-    /** Driver consumer. */
-    PDMBLKCACHETYPE_DRV,
-    /** Internal consumer. */
-    PDMBLKCACHETYPE_INTERNAL,
-    /** Usb consumer. */
-    PDMBLKCACHETYPE_USB
-} PDMBLKCACHETYPE;
-
-/**
- * Per user cache data.
- */
-typedef struct PDMBLKCACHE
-{
-    /** Pointer to the id for the cache. */
-    char                         *pszId;
-    /** AVL tree managing cache entries. */
-    PAVLRU64TREE                  pTree;
-    /** R/W semaphore protecting cached entries for this endpoint. */
-    RTSEMRW                       SemRWEntries;
-    /** Pointer to the gobal cache data */
-    PPDMBLKCACHEGLOBAL            pCache;
-    /** Lock protecting the dirty entries list. */
-    RTSPINLOCK                    LockList;
-    /** List of dirty but not committed entries for this endpoint. */
-    RTLISTNODE                    ListDirtyNotCommitted;
-    /** Node of the cache user list. */
-    RTLISTNODE                    NodeCacheUser;
-    /** Block cache type. */
-    PDMBLKCACHETYPE               enmType;
-    /** Type specific data. */
-    union
-    {
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DEV */
-        struct
-        {
-            /** Pointer to the device instance owning the block cache. */
-            R3PTRTYPE(PPDMDEVINS)                     pDevIns;
-            /** Complete callback to the user. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEDEV)  pfnXferComplete;
-            /** I/O enqueue callback. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDEV)   pfnXferEnqueue;
-        } Dev;
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DRV */
-        struct
-        {
-            /** Pointer to the driver instance owning the block cache. */
-            R3PTRTYPE(PPDMDRVINS)                     pDrvIns;
-            /** Complete callback to the user. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEDRV)  pfnXferComplete;
-            /** I/O enqueue callback. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDRV)   pfnXferEnqueue;
-        } Drv;
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL */
-        struct
-        {
-            /** Pointer to user data. */
-            R3PTRTYPE(void *)                         pvUser;
-            /** Complete callback to the user. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEINT)  pfnXferComplete;
-            /** I/O enqueue callback. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEINT)   pfnXferEnqueue;
-        } Int;
-        /** PDMASYNCCOMPLETIONTEMPLATETYPE_USB */
-        struct
-        {
-            /** Pointer to the usb instance owning the template. */
-            R3PTRTYPE(PPDMUSBINS)                     pUsbIns;
-            /** Complete callback to the user. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEUSB)  pfnXferComplete;
-            /** I/O enqueue callback. */
-            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEUSB)   pfnXferEnqueue;
-        } Usb;
-    } u;
-
-#ifdef VBOX_WITH_STATISTICS
-    uint32_t    u32Alignment;
-    /** Number of times a write was deferred because the cache entry was still in progress */
-    STAMCOUNTER StatWriteDeferred;
-#endif
-
-    /** Flag whether the cache was suspended. */
-    volatile bool                 fSuspended;
-
-} PDMBLKCACHE, *PPDMBLKCACHE;
-#ifdef VBOX_WITH_STATISTICS
-AssertCompileMemberAlignment(PDMBLKCACHE, StatWriteDeferred, sizeof(uint64_t));
-#endif
-
-/**
- * I/O task.
- */
-typedef struct PDMBLKCACHEREQ
-{
-    /** Opaque user data returned on completion. */
-    void             *pvUser;
-    /** Number of bytes to transfer. */
-    volatile uint32_t cbXfer;
-    /** Number of pending transfers (waiting for a cache entry and passed through). */
-    volatile uint32_t cXfersPending;
-    /** Status code. */
-    volatile int      rcReq;
-} PDMBLKCACHEREQ, *PPDMBLKCACHEREQ;
-
-/**
- * I/O transfer from the cache to the underlying medium.
- */
-typedef struct PDMBLKCACHEIOXFER
-{
-    /** Flag whether the I/O xfer updates a cache entry or updates the request directl. */
-    bool fIoCache;
-    /** Type dependent data. */
-    union
-    {
-        /** Pointer to the entry the transfer updates. */
-        PPDMBLKCACHEENTRY pEntry;
-        /** Pointer to the request the ztransfer updates. */
-        PPDMBLKCACHEREQ   pReq;
-    };
-    /** Segment used if a cache entry is updated. */
-    RTSGSEG SgSeg;
-    /** S/G buffer. */
-    RTSGBUF SgBuf;
-    /** Transfer direction. */
-    PDMBLKCACHEXFERDIR enmXferDir;
-    /** Size of the transfer. */
-    size_t             cbXfer;
-} PDMBLKCACHEIOXFER;
-
-/**
- * Cache waiter
- */
-typedef struct PDMBLKCACHEWAITER
-{
-    /* Next waiter in the list. */
-    struct PDMBLKCACHEWAITER *pNext;
-    /** S/G buffer holding or receiving data. */
-    RTSGBUF                   SgBuf;
-    /** Offset into the cache entry to start the transfer. */
-    uint32_t                  offCacheEntry;
-    /** How many bytes to transfer. */
-    size_t                    cbTransfer;
-    /** Flag whether the task wants to read or write into the entry. */
-    bool                      fWrite;
-    /** Task the waiter is for. */
-    PPDMBLKCACHEREQ           pReq;
-} PDMBLKCACHEWAITER;
-
-RT_C_DECLS_END
-
-#endif
-
Index: unk/src/VBox/VMM/PDMCritSect.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMCritSect.cpp	(revision 35332)
+++ 	(revision )
@@ -1,638 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Critical Sections, Ring-3.
- */
-
-/*
- * Copyright (C) 2006-2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM//_CRITSECT
-#include "PDMInternal.h"
-#include <VBox/pdmcritsect.h>
-#include <VBox/mm.h>
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <VBox/sup.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/lockvalidator.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int pdmR3CritSectDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTINT pCritSect, PPDMCRITSECTINT pPrev, bool fFinal);
-
-
-
-/**
- * Register statistics related to the critical sections.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- */
-int pdmR3CritSectInitStats(PVM pVM)
-{
-    STAM_REG(pVM, &pVM->pdm.s.StatQueuedCritSectLeaves, STAMTYPE_COUNTER, "/PDM/QueuedCritSectLeaves", STAMUNIT_OCCURENCES,
-             "Number of times a critical section leave request needed to be queued for ring-3 execution.");
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Relocates all the critical sections.
- *
- * @param   pVM         The VM handle.
- */
-void pdmR3CritSectRelocate(PVM pVM)
-{
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-
-    for (PPDMCRITSECTINT pCur = pUVM->pdm.s.pCritSects;
-         pCur;
-         pCur = pCur->pNext)
-        pCur->pVMRC = pVM->pVMRC;
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-}
-
-
-/**
- * Deletes all remaining critical sections.
- *
- * This is called at the very end of the termination process.  It is also called
- * at the end of vmR3CreateU failure cleanup, which may cause it to be called
- * twice depending on where vmR3CreateU actually failed.  We have to do the
- * latter call because other components expect the critical sections to be
- * automatically deleted.
- *
- * @returns VBox status.
- *          First error code, rest is lost.
- * @param   pVMU        The user mode VM handle.
- * @remark  Don't confuse this with PDMR3CritSectDelete.
- */
-VMMDECL(int) PDMR3CritSectTerm(PVM pVM)
-{
-    PUVM    pUVM = pVM->pUVM;
-    int     rc   = VINF_SUCCESS;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-
-    while (pUVM->pdm.s.pCritSects)
-    {
-        int rc2 = pdmR3CritSectDeleteOne(pVM, pUVM, pUVM->pdm.s.pCritSects, NULL, true /* final */);
-        AssertRC(rc2);
-        if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-            rc = rc2;
-    }
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-/**
- * Initializes a critical section and inserts it into the list.
- *
- * @returns VBox status code.
- * @param   pVM             The Vm handle.
- * @param   pCritSect       The critical section.
- * @param   pvKey           The owner key.
- * @param   RT_SRC_POS_DECL The source position.
- * @param   pszName         The name of the critical section (for statistics).
- * @param   pszNameFmt      Format string for naming the critical section.  For
- *                          statistics and lock validation.
- * @param   va              Arguments for the format string.
- */
-static int pdmR3CritSectInitOne(PVM pVM, PPDMCRITSECTINT pCritSect, void *pvKey, RT_SRC_POS_DECL, const char *pszNameFmt, va_list va)
-{
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Allocate the semaphore.
-     */
-    AssertCompile(sizeof(SUPSEMEVENT) == sizeof(pCritSect->Core.EventSem));
-    int rc = SUPSemEventCreate(pVM->pSession, (PSUPSEMEVENT)&pCritSect->Core.EventSem);
-    if (RT_SUCCESS(rc))
-    {
-        /* Only format the name once. */
-        char *pszName = RTStrAPrintf2V(pszNameFmt, va); /** @todo plug the "leak"... */
-        if (pszName)
-        {
-#ifndef PDMCRITSECT_STRICT
-            pCritSect->Core.pValidatorRec = NULL;
-#else
-            rc = RTLockValidatorRecExclCreate(&pCritSect->Core.pValidatorRec,
-# ifdef RT_LOCK_STRICT_ORDER
-                                              RTLockValidatorClassForSrcPos(RT_SRC_POS_ARGS, "%s", pszName),
-# else
-                                              NIL_RTLOCKVALCLASS,
-# endif
-                                              RTLOCKVAL_SUB_CLASS_NONE,
-                                              pCritSect, true, "%s", pszName);
-#endif
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Initialize the structure (first bit is c&p from RTCritSectInitEx).
-                 */
-                pCritSect->Core.u32Magic             = RTCRITSECT_MAGIC;
-                pCritSect->Core.fFlags               = 0;
-                pCritSect->Core.cNestings            = 0;
-                pCritSect->Core.cLockers             = -1;
-                pCritSect->Core.NativeThreadOwner    = NIL_RTNATIVETHREAD;
-                pCritSect->pVMR3                     = pVM;
-                pCritSect->pVMR0                     = pVM->pVMR0;
-                pCritSect->pVMRC                     = pVM->pVMRC;
-                pCritSect->pvKey                     = pvKey;
-                pCritSect->EventToSignal             = NIL_RTSEMEVENT;
-                pCritSect->pNext                     = pVM->pUVM->pdm.s.pCritSects;
-                pCritSect->pszName                   = pszName;
-                pVM->pUVM->pdm.s.pCritSects = pCritSect;
-                STAMR3RegisterF(pVM, &pCritSect->StatContentionRZLock,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionRZLock", pCritSect->pszName);
-                STAMR3RegisterF(pVM, &pCritSect->StatContentionRZUnlock,STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionRZUnlock", pCritSect->pszName);
-                STAMR3RegisterF(pVM, &pCritSect->StatContentionR3,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionR3", pCritSect->pszName);
-#ifdef VBOX_WITH_STATISTICS
-                STAMR3RegisterF(pVM, &pCritSect->StatLocked,        STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_OCCURENCE, NULL, "/PDM/CritSects/%s/Locked", pCritSect->pszName);
-#endif
-                return VINF_SUCCESS;
-            }
-
-            RTStrFree(pszName);
-        }
-        else
-            rc = VERR_NO_STR_MEMORY;
-        SUPSemEventClose(pVM->pSession, (SUPSEMEVENT)pCritSect->Core.EventSem);
-    }
-    return rc;
-}
-
-
-/**
- * Initializes a PDM critical section for internal use.
- *
- * The PDM critical sections are derived from the IPRT critical sections, but
- * works in GC as well.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pDevIns         Device instance.
- * @param   pCritSect       Pointer to the critical section.
- * @param   RT_SRC_POS_DECL Use RT_SRC_POS.
- * @param   pszNameFmt      Format string for naming the critical section.  For
- *                          statistics and lock validation.
- * @param   ...             Arguments for the format string.
- * @thread  EMT(0)
- */
-VMMR3DECL(int) PDMR3CritSectInit(PVM pVM, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, ...)
-{
-#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
-    AssertCompile(sizeof(pCritSect->padding) >= sizeof(pCritSect->s));
-#endif
-    Assert(RT_ALIGN_P(pCritSect, sizeof(uintptr_t)) == pCritSect);
-    va_list va;
-    va_start(va, pszNameFmt);
-    int rc = pdmR3CritSectInitOne(pVM, &pCritSect->s, pCritSect, RT_SRC_POS_ARGS, pszNameFmt, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Initializes a PDM critical section for a device.
- *
- * The PDM critical sections are derived from the IPRT critical sections, but
- * works in GC as well.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pDevIns         Device instance.
- * @param   pCritSect       Pointer to the critical section.
- * @param   pszNameFmt      Format string for naming the critical section.  For
- *                          statistics and lock validation.
- * @param   va              Arguments for the format string.
- */
-int pdmR3CritSectInitDevice(PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
-                            const char *pszNameFmt, va_list va)
-{
-    return pdmR3CritSectInitOne(pVM, &pCritSect->s, pDevIns, RT_SRC_POS_ARGS, pszNameFmt, va);
-}
-
-
-/**
- * Initializes a PDM critical section for a driver.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pDrvIns         Driver instance.
- * @param   pCritSect       Pointer to the critical section.
- * @param   pszNameFmt      Format string for naming the critical section.  For
- *                          statistics and lock validation.
- * @param   ...             Arguments for the format string.
- */
-int pdmR3CritSectInitDriver(PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
-                            const char *pszNameFmt, ...)
-{
-    va_list va;
-    va_start(va, pszNameFmt);
-    int rc = pdmR3CritSectInitOne(pVM, &pCritSect->s, pDrvIns, RT_SRC_POS_ARGS, pszNameFmt, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Deletes one critical section.
- *
- * @returns Return code from RTCritSectDelete.
- *
- * @param   pVM         The VM handle.
- * @param   pCritSect   The critical section.
- * @param   pPrev       The previous critical section in the list.
- * @param   fFinal      Set if this is the final call and statistics shouldn't be deregistered.
- *
- * @remarks Caller must have entered the ListCritSect.
- */
-static int pdmR3CritSectDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTINT pCritSect, PPDMCRITSECTINT pPrev, bool fFinal)
-{
-    /*
-     * Assert free waiters and so on (c&p from RTCritSectDelete).
-     */
-    Assert(pCritSect->Core.u32Magic == RTCRITSECT_MAGIC);
-    Assert(pCritSect->Core.cNestings == 0);
-    Assert(pCritSect->Core.cLockers == -1);
-    Assert(pCritSect->Core.NativeThreadOwner == NIL_RTNATIVETHREAD);
-    Assert(RTCritSectIsOwner(&pUVM->pdm.s.ListCritSect));
-
-    /*
-     * Unlink it.
-     */
-    if (pPrev)
-        pPrev->pNext = pCritSect->pNext;
-    else
-        pUVM->pdm.s.pCritSects = pCritSect->pNext;
-
-    /*
-     * Delete it (parts taken from RTCritSectDelete).
-     * In case someone is waiting we'll signal the semaphore cLockers + 1 times.
-     */
-    ASMAtomicWriteU32(&pCritSect->Core.u32Magic, 0);
-    SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->Core.EventSem;
-    pCritSect->Core.EventSem = NIL_RTSEMEVENT;
-    while (pCritSect->Core.cLockers-- >= 0)
-        SUPSemEventSignal(pVM->pSession, hEvent);
-    ASMAtomicWriteS32(&pCritSect->Core.cLockers, -1);
-    int rc = SUPSemEventClose(pVM->pSession, hEvent);
-    AssertRC(rc);
-    RTLockValidatorRecExclDestroy(&pCritSect->Core.pValidatorRec);
-    pCritSect->pNext   = NULL;
-    pCritSect->pvKey   = NULL;
-    pCritSect->pVMR3   = NULL;
-    pCritSect->pVMR0   = NIL_RTR0PTR;
-    pCritSect->pVMRC   = NIL_RTRCPTR;
-    RTStrFree((char *)pCritSect->pszName);
-    pCritSect->pszName = NULL;
-    if (!fFinal)
-    {
-        STAMR3Deregister(pVM, &pCritSect->StatContentionRZLock);
-        STAMR3Deregister(pVM, &pCritSect->StatContentionRZUnlock);
-        STAMR3Deregister(pVM, &pCritSect->StatContentionR3);
-#ifdef VBOX_WITH_STATISTICS
-        STAMR3Deregister(pVM, &pCritSect->StatLocked);
-#endif
-    }
-    return rc;
-}
-
-
-/**
- * Deletes all critical sections with a give initializer key.
- *
- * @returns VBox status code.
- *          The entire list is processed on failure, so we'll only
- *          return the first error code. This shouldn't be a problem
- *          since errors really shouldn't happen here.
- * @param   pVM     The VM handle.
- * @param   pvKey   The initializer key.
- */
-static int pdmR3CritSectDeleteByKey(PVM pVM, void *pvKey)
-{
-    /*
-     * Iterate the list and match key.
-     */
-    PUVM            pUVM  = pVM->pUVM;
-    int             rc    = VINF_SUCCESS;
-    PPDMCRITSECTINT pPrev = NULL;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMCRITSECTINT pCur  = pUVM->pdm.s.pCritSects;
-    while (pCur)
-    {
-        if (pCur->pvKey == pvKey)
-        {
-            int rc2 = pdmR3CritSectDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
-            AssertRC(rc2);
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-        }
-
-        /* next */
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-/**
- * Deletes all undeleted critical sections initialized by a given device.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pDevIns     The device handle.
- */
-int pdmR3CritSectDeleteDevice(PVM pVM, PPDMDEVINS pDevIns)
-{
-    return pdmR3CritSectDeleteByKey(pVM, pDevIns);
-}
-
-
-/**
- * Deletes all undeleted critical sections initialized by a given driver.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pDrvIns     The driver handle.
- */
-int pdmR3CritSectDeleteDriver(PVM pVM, PPDMDRVINS pDrvIns)
-{
-    return pdmR3CritSectDeleteByKey(pVM, pDrvIns);
-}
-
-
-/**
- * Deletes the critical section.
- *
- * @returns VBox status code.
- * @param   pCritSect           The PDM critical section to destroy.
- */
-VMMR3DECL(int) PDMR3CritSectDelete(PPDMCRITSECT pCritSect)
-{
-    if (!RTCritSectIsInitialized(&pCritSect->s.Core))
-        return VINF_SUCCESS;
-
-    /*
-     * Find and unlink it.
-     */
-    PVM             pVM   = pCritSect->s.pVMR3;
-    PUVM            pUVM  = pVM->pUVM;
-    AssertReleaseReturn(pVM, VERR_INTERNAL_ERROR);
-    PPDMCRITSECTINT pPrev = NULL;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMCRITSECTINT pCur  = pUVM->pdm.s.pCritSects;
-    while (pCur)
-    {
-        if (pCur == &pCritSect->s)
-        {
-            int rc = pdmR3CritSectDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-            return rc;
-        }
-
-        /* next */
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    AssertReleaseMsgFailed(("pCritSect=%p wasn't found!\n", pCritSect));
-    return VERR_INTERNAL_ERROR;
-}
-
-
-/**
- * Gets the name of the critical section.
- *
- *
- * @returns Pointer to the critical section name (read only) on success,
- *          NULL on failure (invalid critical section).
- * @param   pCritSect           The critical section.
- */
-VMMR3DECL(const char *) PDMR3CritSectName(PCPDMCRITSECT pCritSect)
-{
-    AssertPtrReturn(pCritSect, NULL);
-    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, NULL);
-    return pCritSect->s.pszName;
-}
-
-
-/**
- * Yield the critical section if someone is waiting on it.
- *
- * When yielding, we'll leave the critical section and try to make sure the
- * other waiting threads get a chance of entering before we reclaim it.
- *
- * @retval  true if yielded.
- * @retval  false if not yielded.
- * @param   pCritSect           The critical section.
- */
-VMMR3DECL(bool) PDMR3CritSectYield(PPDMCRITSECT pCritSect)
-{
-    AssertPtrReturn(pCritSect, false);
-    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, false);
-    Assert(pCritSect->s.Core.NativeThreadOwner == RTThreadNativeSelf());
-
-    /* No recursion allowed here. */
-    int32_t const cNestings = pCritSect->s.Core.cNestings;
-    AssertReturn(cNestings == 1, false);
-
-    int32_t const cLockers  = ASMAtomicReadS32(&pCritSect->s.Core.cLockers);
-    if (cLockers < cNestings)
-        return false;
-
-#ifdef PDMCRITSECT_STRICT
-    RTLOCKVALSRCPOS const SrcPos = pCritSect->s.Core.pValidatorRec->SrcPos;
-#endif
-    PDMCritSectLeave(pCritSect);
-
-    /*
-     * If we're lucky, then one of the waiters has entered the lock already.
-     * We spin a little bit in hope for this to happen so we can avoid the
-     * yield detour.
-     */
-    if (ASMAtomicUoReadS32(&pCritSect->s.Core.cNestings) == 0)
-    {
-        int cLoops = 20;
-        while (   cLoops > 0
-               && ASMAtomicUoReadS32(&pCritSect->s.Core.cNestings) == 0
-               && ASMAtomicUoReadS32(&pCritSect->s.Core.cLockers)  >= 0)
-        {
-            ASMNopPause();
-            cLoops--;
-        }
-        if (cLoops == 0)
-            RTThreadYield();
-    }
-
-#ifdef PDMCRITSECT_STRICT
-    int rc = PDMCritSectEnterDebug(pCritSect, VERR_INTERNAL_ERROR,
-                                   SrcPos.uId, SrcPos.pszFile, SrcPos.uLine, SrcPos.pszFunction);
-#else
-    int rc = PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
-#endif
-    AssertLogRelRC(rc);
-    return true;
-}
-
-
-/**
- * Schedule a event semaphore for signalling upon critsect exit.
- *
- * @returns VINF_SUCCESS on success.
- * @returns VERR_TOO_MANY_SEMAPHORES if an event was already scheduled.
- * @returns VERR_NOT_OWNER if we're not the critsect owner.
- * @returns VERR_SEM_DESTROYED if RTCritSectDelete was called while waiting.
- *
- * @param   pCritSect       The critical section.
- * @param   EventToSignal   The semaphore that should be signalled.
- */
-VMMR3DECL(int) PDMR3CritSectScheduleExitEvent(PPDMCRITSECT pCritSect, RTSEMEVENT EventToSignal)
-{
-    Assert(EventToSignal != NIL_RTSEMEVENT);
-    if (RT_UNLIKELY(!RTCritSectIsOwner(&pCritSect->s.Core)))
-        return VERR_NOT_OWNER;
-    if (RT_LIKELY(   pCritSect->s.EventToSignal == NIL_RTSEMEVENT
-                  || pCritSect->s.EventToSignal == EventToSignal))
-    {
-        pCritSect->s.EventToSignal = EventToSignal;
-        return VINF_SUCCESS;
-    }
-    return VERR_TOO_MANY_SEMAPHORES;
-}
-
-
-/**
- * Counts the critical sections owned by the calling thread, optionally
- * returning a comma separated list naming them.
- *
- * This is for diagnostic purposes only.
- *
- * @returns Lock count.
- *
- * @param   pVM             The VM handle.
- * @param   pszNames        Where to return the critical section names.
- * @param   cbNames         The size of the buffer.
- */
-VMMR3DECL(uint32_t) PDMR3CritSectCountOwned(PVM pVM, char *pszNames, size_t cbNames)
-{
-    /*
-     * Init the name buffer.
-     */
-    size_t cchLeft = cbNames;
-    if (cchLeft)
-    {
-        cchLeft--;
-        pszNames[0] = pszNames[cchLeft] = '\0';
-    }
-
-    /*
-     * Iterate the critical sections.
-     */
-    /* This is unsafe, but wtf. */
-    RTNATIVETHREAD const    hNativeThread = RTThreadNativeSelf();
-    uint32_t                cCritSects = 0;
-    for (PPDMCRITSECTINT pCur = pVM->pUVM->pdm.s.pCritSects;
-         pCur;
-         pCur = pCur->pNext)
-    {
-        /* Same as RTCritSectIsOwner(). */
-        if (pCur->Core.NativeThreadOwner == hNativeThread)
-        {
-            cCritSects++;
-
-            /*
-             * Copy the name if there is space. Fun stuff.
-             */
-            if (cchLeft)
-            {
-                /* try add comma. */
-                if (cCritSects != 1)
-                {
-                    *pszNames++ = ',';
-                    if (--cchLeft)
-                    {
-                        *pszNames++ = ' ';
-                        cchLeft--;
-                    }
-                }
-
-                /* try copy the name. */
-                if (cchLeft)
-                {
-                    size_t const cchName = strlen(pCur->pszName);
-                    if (cchName < cchLeft)
-                    {
-                        memcpy(pszNames, pCur->pszName, cchName);
-                        pszNames += cchName;
-                        cchLeft -= cchName;
-                    }
-                    else
-                    {
-                        if (cchLeft > 2)
-                        {
-                            memcpy(pszNames, pCur->pszName, cchLeft - 2);
-                            pszNames += cchLeft - 2;
-                            cchLeft = 2;
-                        }
-                        while (cchLeft-- > 0)
-                            *pszNames++ = '+';
-                    }
-                }
-                *pszNames = '\0';
-            }
-        }
-    }
-
-    return cCritSects;
-}
-
-
-/**
- * Leave all critical sections the calling thread owns.
- *
- * @param   pVM         The VM handle.
- */
-void PDMR3CritSectLeaveAll(PVM pVM)
-{
-    RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
-    PUVM                 pUVM        = pVM->pUVM;
-
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMCRITSECTINT pCur = pUVM->pdm.s.pCritSects;
-         pCur;
-         pCur = pCur->pNext)
-    {
-        while (     pCur->Core.NativeThreadOwner == hNativeSelf
-               &&   pCur->Core.cNestings > 0)
-            PDMCritSectLeave((PPDMCRITSECT)pCur);
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-}
-
Index: unk/src/VBox/VMM/PDMDevHlp.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDevHlp.cpp	(revision 35332)
+++ 	(revision )
@@ -1,3471 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device and Driver Manager, Device Helpers.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_DEVICE
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/pgm.h>
-#include <VBox/iom.h>
-#include <VBox/rem.h>
-#include <VBox/dbgf.h>
-#include <VBox/vmapi.h>
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/vmm.h>
-
-#include <VBox/version.h>
-#include <VBox/log.h>
-#include <VBox/err.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/ctype.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** @def PDM_DEVHLP_DEADLOCK_DETECTION
- * Define this to enable the deadlock detection when accessing physical memory.
- */
-#if /*defined(DEBUG_bird) ||*/ defined(DOXYGEN_RUNNING)
-# define PDM_DEVHLP_DEADLOCK_DETECTION /**< @todo enable DevHlp deadlock detection! */
-#endif
-
-
-
-/**
- * Wrapper around PDMR3LdrGetSymbolRCLazy.
- */
-DECLINLINE(int) pdmR3DevGetSymbolRCLazy(PPDMDEVINS pDevIns, const char *pszSymbol, PRTRCPTR ppvValue)
-{
-    return PDMR3LdrGetSymbolRCLazy(pDevIns->Internal.s.pVMR3,
-                                   pDevIns->Internal.s.pDevR3->pReg->szRCMod,
-                                   pDevIns->Internal.s.pDevR3->pszRCSearchPath,
-                                   pszSymbol, ppvValue);
-}
-
-
-/**
- * Wrapper around PDMR3LdrGetSymbolR0Lazy.
- */
-DECLINLINE(int) pdmR3DevGetSymbolR0Lazy(PPDMDEVINS pDevIns, const char *pszSymbol, PRTR0PTR ppvValue)
-{
-    return PDMR3LdrGetSymbolR0Lazy(pDevIns->Internal.s.pVMR3,
-                                   pDevIns->Internal.s.pDevR3->pReg->szR0Mod,
-                                   pDevIns->Internal.s.pDevR3->pszR0SearchPath,
-                                   pszSymbol, ppvValue);
-}
-
-
-/** @name R3 DevHlp
- * @{
- */
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_IOPortRegister(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts, RTHCPTR pvUser, PFNIOMIOPORTOUT pfnOut, PFNIOMIOPORTIN pfnIn,
-                                                    PFNIOMIOPORTOUTSTRING pfnOutStr, PFNIOMIOPORTINSTRING pfnInStr, const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_IOPortRegister: caller='%s'/%d: Port=%#x cPorts=%#x pvUser=%p pfnOut=%p pfnIn=%p pfnOutStr=%p pfnInStr=%p p32_tszDesc=%p:{%s}\n", pDevIns->pReg->szName, pDevIns->iInstance,
-             Port, cPorts, pvUser, pfnOut, pfnIn, pfnOutStr, pfnInStr, pszDesc, pszDesc));
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-
-#if 0 /** @todo needs a real string cache for this */
-    if (pDevIns->iInstance > 0)
-    {
-         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
-         if (pszDesc2)
-             pszDesc = pszDesc2;
-    }
-#endif
-
-    int rc = IOMR3IOPortRegisterR3(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts, pvUser, pfnOut, pfnIn, pfnOutStr, pfnInStr, pszDesc);
-
-    LogFlow(("pdmR3DevHlp_IOPortRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortRegisterRC} */
-static DECLCALLBACK(int) pdmR3DevHlp_IOPortRegisterRC(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts, RTRCPTR pvUser,
-                                                      const char *pszOut, const char *pszIn,
-                                                      const char *pszOutStr, const char *pszInStr, const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_IOPortRegisterRC: caller='%s'/%d: Port=%#x cPorts=%#x pvUser=%p pszOut=%p:{%s} pszIn=%p:{%s} pszOutStr=%p:{%s} pszInStr=%p:{%s} pszDesc=%p:{%s}\n", pDevIns->pReg->szName, pDevIns->iInstance,
-             Port, cPorts, pvUser, pszOut, pszOut, pszIn, pszIn, pszOutStr, pszOutStr, pszInStr, pszInStr, pszDesc, pszDesc));
-
-    /*
-     * Resolve the functions (one of the can be NULL).
-     */
-    int rc = VINF_SUCCESS;
-    if (    pDevIns->pReg->szRCMod[0]
-        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC))
-    {
-        RTRCPTR RCPtrIn = NIL_RTRCPTR;
-        if (pszIn)
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszIn, &RCPtrIn);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszIn)\n", pDevIns->pReg->szRCMod, pszIn));
-        }
-        RTRCPTR RCPtrOut = NIL_RTRCPTR;
-        if (pszOut && RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszOut, &RCPtrOut);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOut)\n", pDevIns->pReg->szRCMod, pszOut));
-        }
-        RTRCPTR RCPtrInStr = NIL_RTRCPTR;
-        if (pszInStr && RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszInStr, &RCPtrInStr);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszInStr)\n", pDevIns->pReg->szRCMod, pszInStr));
-        }
-        RTRCPTR RCPtrOutStr = NIL_RTRCPTR;
-        if (pszOutStr && RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszOutStr, &RCPtrOutStr);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOutStr)\n", pDevIns->pReg->szRCMod, pszOutStr));
-        }
-
-        if (RT_SUCCESS(rc))
-        {
-#if 0 /** @todo needs a real string cache for this */
-            if (pDevIns->iInstance > 0)
-            {
-                 char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
-                 if (pszDesc2)
-                     pszDesc = pszDesc2;
-            }
-#endif
-
-            rc = IOMR3IOPortRegisterRC(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts, pvUser, RCPtrOut, RCPtrIn, RCPtrOutStr, RCPtrInStr, pszDesc);
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("No GC module for this driver!\n"));
-        rc = VERR_INVALID_PARAMETER;
-    }
-
-    LogFlow(("pdmR3DevHlp_IOPortRegisterRC: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortRegisterR0} */
-static DECLCALLBACK(int) pdmR3DevHlp_IOPortRegisterR0(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts, RTR0PTR pvUser,
-                                                      const char *pszOut, const char *pszIn,
-                                                      const char *pszOutStr, const char *pszInStr, const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_IOPortRegisterR0: caller='%s'/%d: Port=%#x cPorts=%#x pvUser=%p pszOut=%p:{%s} pszIn=%p:{%s} pszOutStr=%p:{%s} pszInStr=%p:{%s} pszDesc=%p:{%s}\n", pDevIns->pReg->szName, pDevIns->iInstance,
-             Port, cPorts, pvUser, pszOut, pszOut, pszIn, pszIn, pszOutStr, pszOutStr, pszInStr, pszInStr, pszDesc, pszDesc));
-
-    /*
-     * Resolve the functions (one of the can be NULL).
-     */
-    int rc = VINF_SUCCESS;
-    if (    pDevIns->pReg->szR0Mod[0]
-        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0))
-    {
-        R0PTRTYPE(PFNIOMIOPORTIN) pfnR0PtrIn = 0;
-        if (pszIn)
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszIn, &pfnR0PtrIn);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszIn)\n", pDevIns->pReg->szR0Mod, pszIn));
-        }
-        R0PTRTYPE(PFNIOMIOPORTOUT) pfnR0PtrOut = 0;
-        if (pszOut && RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszOut, &pfnR0PtrOut);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOut)\n", pDevIns->pReg->szR0Mod, pszOut));
-        }
-        R0PTRTYPE(PFNIOMIOPORTINSTRING) pfnR0PtrInStr = 0;
-        if (pszInStr && RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszInStr, &pfnR0PtrInStr);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszInStr)\n", pDevIns->pReg->szR0Mod, pszInStr));
-        }
-        R0PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnR0PtrOutStr = 0;
-        if (pszOutStr && RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszOutStr, &pfnR0PtrOutStr);
-            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOutStr)\n", pDevIns->pReg->szR0Mod, pszOutStr));
-        }
-
-        if (RT_SUCCESS(rc))
-        {
-#if 0 /** @todo needs a real string cache for this */
-            if (pDevIns->iInstance > 0)
-            {
-                 char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
-                 if (pszDesc2)
-                     pszDesc = pszDesc2;
-            }
-#endif
-
-            rc = IOMR3IOPortRegisterR0(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts, pvUser, pfnR0PtrOut, pfnR0PtrIn, pfnR0PtrOutStr, pfnR0PtrInStr, pszDesc);
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("No R0 module for this driver!\n"));
-        rc = VERR_INVALID_PARAMETER;
-    }
-
-    LogFlow(("pdmR3DevHlp_IOPortRegisterR0: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortDeregister} */
-static DECLCALLBACK(int) pdmR3DevHlp_IOPortDeregister(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_IOPortDeregister: caller='%s'/%d: Port=%#x cPorts=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance,
-             Port, cPorts));
-
-    int rc = IOMR3IOPortDeregister(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts);
-
-    LogFlow(("pdmR3DevHlp_IOPortDeregister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnMMIORegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIORegister(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTHCPTR pvUser,
-                                                  PFNIOMMMIOWRITE pfnWrite, PFNIOMMMIOREAD pfnRead, PFNIOMMMIOFILL pfnFill,
-                                                  const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIORegister: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvUser=%p pfnWrite=%p pfnRead=%p pfnFill=%p pszDesc=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvUser, pfnWrite, pfnRead, pfnFill, pszDesc, pszDesc));
-
-/** @todo IOMR3MMIORegisterR3 mangles the description, move it here. */
-    int rc = IOMR3MMIORegisterR3(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvUser, pfnWrite, pfnRead, pfnFill, pszDesc);
-
-    LogFlow(("pdmR3DevHlp_MMIORegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnMMIORegisterRC} */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIORegisterRC(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTGCPTR pvUser,
-                                                    const char *pszWrite, const char *pszRead, const char *pszFill,
-                                                    const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIORegisterRC: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvUser=%p pszWrite=%p:{%s} pszRead=%p:{%s} pszFill=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvUser, pszWrite, pszWrite, pszRead, pszRead, pszFill, pszFill));
-
-/** @todo pszDesc is unused here, drop it.  */
-
-    /*
-     * Resolve the functions.
-     * Not all function have to present, leave it to IOM to enforce this.
-     */
-    int rc = VINF_SUCCESS;
-    if (    pDevIns->pReg->szRCMod[0]
-        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC))
-    {
-        RTRCPTR RCPtrWrite = NIL_RTRCPTR;
-        if (pszWrite)
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszWrite, &RCPtrWrite);
-
-        RTRCPTR RCPtrRead = NIL_RTRCPTR;
-        int rc2 = VINF_SUCCESS;
-        if (pszRead)
-            rc2 = pdmR3DevGetSymbolRCLazy(pDevIns, pszRead, &RCPtrRead);
-
-        RTRCPTR RCPtrFill = NIL_RTRCPTR;
-        int rc3 = VINF_SUCCESS;
-        if (pszFill)
-            rc3 = pdmR3DevGetSymbolRCLazy(pDevIns, pszFill, &RCPtrFill);
-
-        if (RT_SUCCESS(rc) && RT_SUCCESS(rc2) && RT_SUCCESS(rc3))
-            rc = IOMR3MMIORegisterRC(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvUser, RCPtrWrite, RCPtrRead, RCPtrFill);
-        else
-        {
-            AssertMsgRC(rc,  ("Failed to resolve %s.%s (pszWrite)\n", pDevIns->pReg->szRCMod, pszWrite));
-            AssertMsgRC(rc2, ("Failed to resolve %s.%s (pszRead)\n",  pDevIns->pReg->szRCMod, pszRead));
-            AssertMsgRC(rc3, ("Failed to resolve %s.%s (pszFill)\n",  pDevIns->pReg->szRCMod, pszFill));
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-            if (RT_FAILURE(rc3) && RT_SUCCESS(rc))
-                rc = rc3;
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("No GC module for this driver!\n"));
-        rc = VERR_INVALID_PARAMETER;
-    }
-
-    LogFlow(("pdmR3DevHlp_MMIORegisterRC: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnMMIORegisterR0} */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIORegisterR0(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTR0PTR pvUser,
-                                                    const char *pszWrite, const char *pszRead, const char *pszFill,
-                                                    const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIORegisterHC: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvUser=%p pszWrite=%p:{%s} pszRead=%p:{%s} pszFill=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvUser, pszWrite, pszWrite, pszRead, pszRead, pszFill, pszFill));
-
-/** @todo pszDesc is unused here, remove it.  */
-
-    /*
-     * Resolve the functions.
-     * Not all function have to present, leave it to IOM to enforce this.
-     */
-    int rc = VINF_SUCCESS;
-    if (    pDevIns->pReg->szR0Mod[0]
-        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0))
-    {
-        R0PTRTYPE(PFNIOMMMIOWRITE) pfnR0PtrWrite = 0;
-        if (pszWrite)
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszWrite, &pfnR0PtrWrite);
-        R0PTRTYPE(PFNIOMMMIOREAD) pfnR0PtrRead = 0;
-        int rc2 = VINF_SUCCESS;
-        if (pszRead)
-            rc2 = pdmR3DevGetSymbolR0Lazy(pDevIns, pszRead, &pfnR0PtrRead);
-        R0PTRTYPE(PFNIOMMMIOFILL) pfnR0PtrFill = 0;
-        int rc3 = VINF_SUCCESS;
-        if (pszFill)
-            rc3 = pdmR3DevGetSymbolR0Lazy(pDevIns, pszFill, &pfnR0PtrFill);
-        if (RT_SUCCESS(rc) && RT_SUCCESS(rc2) && RT_SUCCESS(rc3))
-            rc = IOMR3MMIORegisterR0(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvUser, pfnR0PtrWrite, pfnR0PtrRead, pfnR0PtrFill);
-        else
-        {
-            AssertMsgRC(rc,  ("Failed to resolve %s.%s (pszWrite)\n", pDevIns->pReg->szR0Mod, pszWrite));
-            AssertMsgRC(rc2, ("Failed to resolve %s.%s (pszRead)\n",  pDevIns->pReg->szR0Mod, pszRead));
-            AssertMsgRC(rc3, ("Failed to resolve %s.%s (pszFill)\n",  pDevIns->pReg->szR0Mod, pszFill));
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-            if (RT_FAILURE(rc3) && RT_SUCCESS(rc))
-                rc = rc3;
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("No R0 module for this driver!\n"));
-        rc = VERR_INVALID_PARAMETER;
-    }
-
-    LogFlow(("pdmR3DevHlp_MMIORegisterR0: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnMMIODeregister} */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIODeregister(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIODeregister: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange));
-
-    int rc = IOMR3MMIODeregister(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange);
-
-    LogFlow(("pdmR3DevHlp_MMIODeregister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnMMIO2Register
- */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Register(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void **ppv, const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIO2Register: caller='%s'/%d: iRegion=%#x cb=%#RGp fFlags=%RX32 ppv=%p pszDescp=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, cb, fFlags, ppv, pszDesc, pszDesc));
-
-/** @todo PGMR3PhysMMIO2Register mangles the description, move it here and
- *        use a real string cache. */
-    int rc = PGMR3PhysMMIO2Register(pDevIns->Internal.s.pVMR3, pDevIns, iRegion, cb, fFlags, ppv, pszDesc);
-
-    LogFlow(("pdmR3DevHlp_MMIO2Register: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnMMIO2Deregister
- */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Deregister(PPDMDEVINS pDevIns, uint32_t iRegion)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIO2Deregister: caller='%s'/%d: iRegion=%#x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iRegion));
-
-    AssertReturn(iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
-
-    int rc = PGMR3PhysMMIO2Deregister(pDevIns->Internal.s.pVMR3, pDevIns, iRegion);
-
-    LogFlow(("pdmR3DevHlp_MMIO2Deregister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnMMIO2Map
- */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Map(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIO2Map: caller='%s'/%d: iRegion=%#x GCPhys=%#RGp\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, GCPhys));
-
-    int rc = PGMR3PhysMMIO2Map(pDevIns->Internal.s.pVMR3, pDevIns, iRegion, GCPhys);
-
-    LogFlow(("pdmR3DevHlp_MMIO2Map: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnMMIO2Unmap
- */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Unmap(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_MMIO2Unmap: caller='%s'/%d: iRegion=%#x GCPhys=%#RGp\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, GCPhys));
-
-    int rc = PGMR3PhysMMIO2Unmap(pDevIns->Internal.s.pVMR3, pDevIns, iRegion, GCPhys);
-
-    LogFlow(("pdmR3DevHlp_MMIO2Unmap: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnMMHyperMapMMIO2
- */
-static DECLCALLBACK(int) pdmR3DevHlp_MMHyperMapMMIO2(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
-                                                     const char *pszDesc, PRTRCPTR pRCPtr)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_MMHyperMapMMIO2: caller='%s'/%d: iRegion=%#x off=%RGp cb=%RGp pszDesc=%p:{%s} pRCPtr=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, off, cb, pszDesc, pszDesc, pRCPtr));
-
-    if (pDevIns->iInstance > 0)
-    {
-         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
-         if (pszDesc2)
-             pszDesc = pszDesc2;
-    }
-
-    int rc = MMR3HyperMapMMIO2(pVM, pDevIns, iRegion, off, cb, pszDesc, pRCPtr);
-
-    LogFlow(("pdmR3DevHlp_MMHyperMapMMIO2: caller='%s'/%d: returns %Rrc *pRCPtr=%RRv\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *pRCPtr));
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnMMIO2MapKernel
- */
-static DECLCALLBACK(int) pdmR3DevHlp_MMIO2MapKernel(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
-                                                    const char *pszDesc, PRTR0PTR pR0Ptr)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_MMIO2MapKernel: caller='%s'/%d: iRegion=%#x off=%RGp cb=%RGp pszDesc=%p:{%s} pR0Ptr=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, off, cb, pszDesc, pszDesc, pR0Ptr));
-
-    if (pDevIns->iInstance > 0)
-    {
-         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
-         if (pszDesc2)
-             pszDesc = pszDesc2;
-    }
-
-    int rc = PGMR3PhysMMIO2MapKernel(pVM, pDevIns, iRegion, off, cb, pszDesc, pR0Ptr);
-
-    LogFlow(("pdmR3DevHlp_MMIO2MapKernel: caller='%s'/%d: returns %Rrc *pR0Ptr=%RHv\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *pR0Ptr));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnROMRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_ROMRegister(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange,
-                                                 const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_ROMRegister: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvBinary=%p cbBinary=%#x fFlags=%#RX32 pszDesc=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvBinary, cbBinary, fFlags, pszDesc, pszDesc));
-
-/** @todo can we mangle pszDesc? */
-    int rc = PGMR3PhysRomRegister(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvBinary, cbBinary, fFlags, pszDesc);
-
-    LogFlow(("pdmR3DevHlp_ROMRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnROMProtectShadow} */
-static DECLCALLBACK(int) pdmR3DevHlp_ROMProtectShadow(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, PGMROMPROT enmProt)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_ROMProtectShadow: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x enmProt=%d\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, enmProt));
-
-    int rc = PGMR3PhysRomProtect(pDevIns->Internal.s.pVMR3, GCPhysStart, cbRange, enmProt);
-
-    LogFlow(("pdmR3DevHlp_ROMProtectShadow: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnSSMRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_SSMRegister(PPDMDEVINS pDevIns, uint32_t uVersion, size_t cbGuess, const char *pszBefore,
-                                                 PFNSSMDEVLIVEPREP pfnLivePrep, PFNSSMDEVLIVEEXEC pfnLiveExec, PFNSSMDEVLIVEVOTE pfnLiveVote,
-                                                 PFNSSMDEVSAVEPREP pfnSavePrep, PFNSSMDEVSAVEEXEC pfnSaveExec, PFNSSMDEVSAVEDONE pfnSaveDone,
-                                                 PFNSSMDEVLOADPREP pfnLoadPrep, PFNSSMDEVLOADEXEC pfnLoadExec, PFNSSMDEVLOADDONE pfnLoadDone)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_SSMRegister: caller='%s'/%d: uVersion=#x cbGuess=%#x pszBefore=%p:{%s}\n"
-             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoadDone=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, uVersion, cbGuess, pszBefore, pszBefore,
-             pfnLivePrep, pfnLiveExec, pfnLiveVote,
-             pfnSavePrep, pfnSaveExec, pfnSaveDone,
-             pfnLoadPrep, pfnLoadExec, pfnLoadDone));
-
-    int rc = SSMR3RegisterDevice(pDevIns->Internal.s.pVMR3, pDevIns, pDevIns->pReg->szName, pDevIns->iInstance,
-                                 uVersion, cbGuess, pszBefore,
-                                 pfnLivePrep, pfnLiveExec, pfnLiveVote,
-                                 pfnSavePrep, pfnSaveExec, pfnSaveDone,
-                                 pfnLoadPrep, pfnLoadExec, pfnLoadDone);
-
-    LogFlow(("pdmR3DevHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimerCreate} */
-static DECLCALLBACK(int) pdmR3DevHlp_TMTimerCreate(PPDMDEVINS pDevIns, TMCLOCK enmClock, PFNTMTIMERDEV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
-
-    if (pDevIns->iInstance > 0) /** @todo use a string cache here later. */
-    {
-         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
-         if (pszDesc2)
-             pszDesc = pszDesc2;
-    }
-
-    int rc = TMR3TimerCreateDevice(pVM, pDevIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer);
-
-    LogFlow(("pdmR3DevHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnTMUtcNow} */
-static DECLCALLBACK(PRTTIMESPEC) pdmR3DevHlp_TMUtcNow(PPDMDEVINS pDevIns, PRTTIMESPEC pTime)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_TMUtcNow: caller='%s'/%d: pTime=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pTime));
-
-    pTime = TMR3UtcNow(pDevIns->Internal.s.pVMR3, pTime);
-
-    LogFlow(("pdmR3DevHlp_TMUtcNow: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, RTTimeSpecGetNano(pTime)));
-    return pTime;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGet} */
-static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGet(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_TMTimeVirtGet: caller='%s'\n",
-             pDevIns->pReg->szName, pDevIns->iInstance));
-
-    uint64_t u64Time = TMVirtualSyncGet(pDevIns->Internal.s.pVMR3);
-
-    LogFlow(("pdmR3DevHlp_TMTimeVirtGet: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Time));
-    return u64Time;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGetFreq} */
-static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGetFreq(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_TMTimeVirtGetFreq: caller='%s'\n",
-             pDevIns->pReg->szName, pDevIns->iInstance));
-
-    uint64_t u64Freq = TMVirtualGetFreq(pDevIns->Internal.s.pVMR3);
-
-    LogFlow(("pdmR3DevHlp_TMTimeVirtGetFreq: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Freq));
-    return u64Freq;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGetNano} */
-static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGetNano(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_TMTimeVirtGetNano: caller='%s'\n",
-             pDevIns->pReg->szName, pDevIns->iInstance));
-
-    uint64_t u64Time = TMVirtualSyncGet(pDevIns->Internal.s.pVMR3);
-    uint64_t u64Nano = TMVirtualToNano(pDevIns->Internal.s.pVMR3, u64Time);
-
-    LogFlow(("pdmR3DevHlp_TMTimeVirtGetNano: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Nano));
-    return u64Nano;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysRead} */
-static DECLCALLBACK(int) pdmR3DevHlp_PhysRead(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    LogFlow(("pdmR3DevHlp_PhysRead: caller='%s'/%d: GCPhys=%RGp pvBuf=%p cbRead=%#x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, pvBuf, cbRead));
-
-#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
-    if (!VM_IS_EMT(pVM))
-    {
-        char szNames[128];
-        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
-        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
-    }
-#endif
-
-    int rc;
-    if (VM_IS_EMT(pVM))
-        rc = PGMPhysRead(pVM, GCPhys, pvBuf, cbRead);
-    else
-        rc = PGMR3PhysReadExternal(pVM, GCPhys, pvBuf, cbRead);
-
-    Log(("pdmR3DevHlp_PhysRead: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysWrite} */
-static DECLCALLBACK(int) pdmR3DevHlp_PhysWrite(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    LogFlow(("pdmR3DevHlp_PhysWrite: caller='%s'/%d: GCPhys=%RGp pvBuf=%p cbWrite=%#x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, pvBuf, cbWrite));
-
-#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
-    if (!VM_IS_EMT(pVM))
-    {
-        char szNames[128];
-        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
-        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
-    }
-#endif
-
-    int rc;
-    if (VM_IS_EMT(pVM))
-        rc = PGMPhysWrite(pVM, GCPhys, pvBuf, cbWrite);
-    else
-        rc = PGMR3PhysWriteExternal(pVM, GCPhys, pvBuf, cbWrite, pDevIns->pReg->szName);
-
-    Log(("pdmR3DevHlp_PhysWrite: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPhys2CCPtr} */
-static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPhys2CCPtr(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t fFlags, void **ppv, PPGMPAGEMAPLOCK pLock)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    LogFlow(("pdmR3DevHlp_PhysGCPhys2CCPtr: caller='%s'/%d: GCPhys=%RGp fFlags=%#x ppv=%p pLock=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, fFlags, ppv, pLock));
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
-
-#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
-    if (!VM_IS_EMT(pVM))
-    {
-        char szNames[128];
-        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
-        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
-    }
-#endif
-
-    int rc = PGMR3PhysGCPhys2CCPtrExternal(pVM, GCPhys, ppv, pLock);
-
-    Log(("pdmR3DevHlp_PhysGCPhys2CCPtr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPhys2CCPtrReadOnly} */
-static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t fFlags, const void **ppv, PPGMPAGEMAPLOCK pLock)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    LogFlow(("pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly: caller='%s'/%d: GCPhys=%RGp fFlags=%#x ppv=%p pLock=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, fFlags, ppv, pLock));
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
-
-#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
-    if (!VM_IS_EMT(pVM))
-    {
-        char szNames[128];
-        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
-        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
-    }
-#endif
-
-    int rc = PGMR3PhysGCPhys2CCPtrReadOnlyExternal(pVM, GCPhys, ppv, pLock);
-
-    Log(("pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysReleasePageMappingLock} */
-static DECLCALLBACK(void) pdmR3DevHlp_PhysReleasePageMappingLock(PPDMDEVINS pDevIns, PPGMPAGEMAPLOCK pLock)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    LogFlow(("pdmR3DevHlp_PhysReleasePageMappingLock: caller='%s'/%d: pLock=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pLock));
-
-    PGMPhysReleasePageMappingLock(pVM, pLock);
-
-    Log(("pdmR3DevHlp_PhysReleasePageMappingLock: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysReadGCVirt} */
-static DECLCALLBACK(int) pdmR3DevHlp_PhysReadGCVirt(PPDMDEVINS pDevIns, void *pvDst, RTGCPTR GCVirtSrc, size_t cb)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_PhysReadGCVirt: caller='%s'/%d: pvDst=%p GCVirt=%RGv cb=%#x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pvDst, GCVirtSrc, cb));
-
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    if (!pVCpu)
-        return VERR_ACCESS_DENIED;
-#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
-    /** @todo SMP. */
-#endif
-
-    int rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCVirtSrc, cb);
-
-    LogFlow(("pdmR3DevHlp_PhysReadGCVirt: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysWriteGCVirt} */
-static DECLCALLBACK(int) pdmR3DevHlp_PhysWriteGCVirt(PPDMDEVINS pDevIns, RTGCPTR GCVirtDst, const void *pvSrc, size_t cb)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_PhysWriteGCVirt: caller='%s'/%d: GCVirtDst=%RGv pvSrc=%p cb=%#x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCVirtDst, pvSrc, cb));
-
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    if (!pVCpu)
-        return VERR_ACCESS_DENIED;
-#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
-    /** @todo SMP. */
-#endif
-
-    int rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCVirtDst, pvSrc, cb);
-
-    LogFlow(("pdmR3DevHlp_PhysWriteGCVirt: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPtr2GCPhys} */
-static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPtr2GCPhys(PPDMDEVINS pDevIns, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_PhysGCPtr2GCPhys: caller='%s'/%d: GCPtr=%RGv pGCPhys=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, GCPtr, pGCPhys));
-
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    if (!pVCpu)
-        return VERR_ACCESS_DENIED;
-#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
-    /** @todo SMP. */
-#endif
-
-    int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, pGCPhys);
-
-    LogFlow(("pdmR3DevHlp_PhysGCPtr2GCPhys: caller='%s'/%d: returns %Rrc *pGCPhys=%RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *pGCPhys));
-
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapAlloc} */
-static DECLCALLBACK(void *) pdmR3DevHlp_MMHeapAlloc(PPDMDEVINS pDevIns, size_t cb)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: cb=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cb));
-
-    void *pv = MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE_USER, cb);
-
-    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
-    return pv;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapAllocZ} */
-static DECLCALLBACK(void *) pdmR3DevHlp_MMHeapAllocZ(PPDMDEVINS pDevIns, size_t cb)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_MMHeapAllocZ: caller='%s'/%d: cb=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cb));
-
-    void *pv = MMR3HeapAllocZ(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE_USER, cb);
-
-    LogFlow(("pdmR3DevHlp_MMHeapAllocZ: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
-    return pv;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapFree} */
-static DECLCALLBACK(void) pdmR3DevHlp_MMHeapFree(PPDMDEVINS pDevIns, void *pv)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_MMHeapFree: caller='%s'/%d: pv=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
-
-    MMR3HeapFree(pv);
-
-    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMState} */
-static DECLCALLBACK(VMSTATE) pdmR3DevHlp_VMState(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-
-    VMSTATE enmVMState = VMR3GetState(pDevIns->Internal.s.pVMR3);
-
-    LogFlow(("pdmR3DevHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pDevIns->pReg->szName, pDevIns->iInstance,
-             enmVMState, VMR3GetStateName(enmVMState)));
-    return enmVMState;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMTeleportedAndNotFullyResumedYet} */
-static DECLCALLBACK(bool) pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-
-    bool fRc = VMR3TeleportedAndNotFullyResumedYet(pDevIns->Internal.s.pVMR3);
-
-    LogFlow(("pdmR3DevHlp_VMState: caller='%s'/%d: returns %RTbool\n", pDevIns->pReg->szName, pDevIns->iInstance,
-             fRc));
-    return fRc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetError} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMSetError(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    va_list args;
-    va_start(args, pszFormat);
-    int rc2 = VMSetErrorV(pDevIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
-    va_end(args);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetErrorV} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMSetErrorV(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    int rc2 = VMSetErrorV(pDevIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetRuntimeError} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMSetRuntimeError(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    va_list args;
-    va_start(args, pszFormat);
-    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, args);
-    va_end(args);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetRuntimeErrorV} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMSetRuntimeErrorV(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, va);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFStopV} */
-static DECLCALLBACK(int) pdmR3DevHlp_DBGFStopV(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction, const char *pszFormat, va_list args)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-#ifdef LOG_ENABLED
-    va_list va2;
-    va_copy(va2, args);
-    LogFlow(("pdmR3DevHlp_DBGFStopV: caller='%s'/%d: pszFile=%p:{%s} iLine=%d pszFunction=%p:{%s} pszFormat=%p:{%s} (%N)\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pszFile, pszFile, iLine, pszFunction, pszFunction, pszFormat, pszFormat, pszFormat, &va2));
-    va_end(va2);
-#endif
-
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    int rc = DBGFR3EventSrcV(pVM, DBGFEVENT_DEV_STOP, pszFile, iLine, pszFunction, pszFormat, args);
-    if (rc == VERR_DBGF_NOT_ATTACHED)
-        rc = VINF_SUCCESS;
-
-    LogFlow(("pdmR3DevHlp_DBGFStopV: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFInfoRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_DBGFInfoRegister(PPDMDEVINS pDevIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDEV pfnHandler)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
-
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    int rc = DBGFR3InfoRegisterDevice(pVM, pszName, pszDesc, pfnHandler, pDevIns);
-
-    LogFlow(("pdmR3DevHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegister} */
-static DECLCALLBACK(void) pdmR3DevHlp_STAMRegister(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    STAM_REG(pVM, pvSample, enmType, pszName, enmUnit, pszDesc);
-    NOREF(pVM);
-}
-
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegisterF} */
-static DECLCALLBACK(void) pdmR3DevHlp_STAMRegisterF(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
-                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, ...)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    va_list args;
-    va_start(args, pszName);
-    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-    va_end(args);
-    AssertRC(rc);
-
-    NOREF(pVM);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegisterV} */
-static DECLCALLBACK(void) pdmR3DevHlp_STAMRegisterV(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
-                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-    AssertRC(rc);
-
-    NOREF(pVM);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPCIRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_PCIRegister(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: pPciDev=%p:{.config={%#.256Rhxs}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pPciDev->config));
-
-    /*
-     * Validate input.
-     */
-    if (!pPciDev)
-    {
-        Assert(pPciDev);
-        LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc (pPciDev)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!pPciDev->config[0] && !pPciDev->config[1])
-    {
-        Assert(pPciDev->config[0] || pPciDev->config[1]);
-        LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc (vendor)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (pDevIns->Internal.s.pPciDeviceR3)
-    {
-        /** @todo the PCI device vs. PDM device designed is a bit flawed if we have to
-         * support a PDM device with multiple PCI devices. This might become a problem
-         * when upgrading the chipset for instance because of multiple functions in some
-         * devices...
-         */
-        AssertMsgFailed(("Only one PCI device per device is currently implemented!\n"));
-        return VERR_INTERNAL_ERROR;
-    }
-
-    /*
-     * Choose the PCI bus for the device.
-     *
-     * This is simple. If the device was configured for a particular bus, the PCIBusNo
-     * configuration value will be set. If not the default bus is 0.
-     */
-    int rc;
-    PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3;
-    if (!pBus)
-    {
-        uint8_t u8Bus;
-        rc = CFGMR3QueryU8Def(pDevIns->Internal.s.pCfgHandle, "PCIBusNo", &u8Bus, 0);
-        AssertLogRelMsgRCReturn(rc, ("Configuration error: PCIBusNo query failed with rc=%Rrc (%s/%d)\n",
-                                     rc, pDevIns->pReg->szName, pDevIns->iInstance), rc);
-        AssertLogRelMsgReturn(u8Bus < RT_ELEMENTS(pVM->pdm.s.aPciBuses),
-                              ("Configuration error: PCIBusNo=%d, max is %d. (%s/%d)\n", u8Bus,
-                               RT_ELEMENTS(pVM->pdm.s.aPciBuses), pDevIns->pReg->szName, pDevIns->iInstance),
-                              VERR_PDM_NO_PCI_BUS);
-        pBus = pDevIns->Internal.s.pPciBusR3 = &pVM->pdm.s.aPciBuses[u8Bus];
-    }
-    if (pBus->pDevInsR3)
-    {
-        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
-            pDevIns->Internal.s.pPciBusR0 = MMHyperR3ToR0(pVM, pDevIns->Internal.s.pPciBusR3);
-        else
-            pDevIns->Internal.s.pPciBusR0 = NIL_RTR0PTR;
-
-        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
-            pDevIns->Internal.s.pPciBusRC = MMHyperR3ToRC(pVM, pDevIns->Internal.s.pPciBusR3);
-        else
-            pDevIns->Internal.s.pPciBusRC = NIL_RTRCPTR;
-
-        /*
-         * Check the configuration for PCI device and function assignment.
-         */
-        int iDev = -1;
-        uint8_t     u8Device;
-        rc = CFGMR3QueryU8(pDevIns->Internal.s.pCfgHandle, "PCIDeviceNo", &u8Device);
-        if (RT_SUCCESS(rc))
-        {
-            if (u8Device > 31)
-            {
-                AssertMsgFailed(("Configuration error: PCIDeviceNo=%d, max is 31. (%s/%d)\n",
-                                 u8Device, pDevIns->pReg->szName, pDevIns->iInstance));
-                return VERR_INTERNAL_ERROR;
-            }
-
-            uint8_t     u8Function;
-            rc = CFGMR3QueryU8(pDevIns->Internal.s.pCfgHandle, "PCIFunctionNo", &u8Function);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("Configuration error: PCIDeviceNo, but PCIFunctionNo query failed with rc=%Rrc (%s/%d)\n",
-                                 rc, pDevIns->pReg->szName, pDevIns->iInstance));
-                return rc;
-            }
-            if (u8Function > 7)
-            {
-                AssertMsgFailed(("Configuration error: PCIFunctionNo=%d, max is 7. (%s/%d)\n",
-                                 u8Function, pDevIns->pReg->szName, pDevIns->iInstance));
-                return VERR_INTERNAL_ERROR;
-            }
-            iDev = (u8Device << 3) | u8Function;
-        }
-        else if (rc != VERR_CFGM_VALUE_NOT_FOUND)
-        {
-            AssertMsgFailed(("Configuration error: PCIDeviceNo query failed with rc=%Rrc (%s/%d)\n",
-                             rc, pDevIns->pReg->szName, pDevIns->iInstance));
-            return rc;
-        }
-
-        /*
-         * Call the pci bus device to do the actual registration.
-         */
-        pdmLock(pVM);
-        rc = pBus->pfnRegisterR3(pBus->pDevInsR3, pPciDev, pDevIns->pReg->szName, iDev);
-        pdmUnlock(pVM);
-        if (RT_SUCCESS(rc))
-        {
-            pPciDev->pDevIns = pDevIns;
-
-            pDevIns->Internal.s.pPciDeviceR3 = pPciDev;
-            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
-                pDevIns->Internal.s.pPciDeviceR0 = MMHyperR3ToR0(pVM, pPciDev);
-            else
-                pDevIns->Internal.s.pPciDeviceR0 = NIL_RTR0PTR;
-
-            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
-                pDevIns->Internal.s.pPciDeviceRC = MMHyperR3ToRC(pVM, pPciDev);
-            else
-                pDevIns->Internal.s.pPciDeviceRC = NIL_RTRCPTR;
-
-            Log(("PDM: Registered device '%s'/%d as PCI device %d on bus %d\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, pPciDev->devfn, pDevIns->Internal.s.pPciBusR3->iBus));
-        }
-    }
-    else
-    {
-        AssertLogRelMsgFailed(("Configuration error: No PCI bus available. This could be related to init order too!\n"));
-        rc = VERR_PDM_NO_PCI_BUS;
-    }
-
-    LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPCIIORegionRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_PCIIORegionRegister(PPDMDEVINS pDevIns, int iRegion, uint32_t cbRegion, PCIADDRESSSPACE enmType, PFNPCIIOREGIONMAP pfnCallback)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: iRegion=%d cbRegion=%#x enmType=%d pfnCallback=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, cbRegion, enmType, pfnCallback));
-
-    /*
-     * Validate input.
-     */
-    if (iRegion < 0 || iRegion >= PCI_NUM_REGIONS)
-    {
-        Assert(iRegion >= 0 && iRegion < PCI_NUM_REGIONS);
-        LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (iRegion)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    switch (enmType)
-    {
-        case PCI_ADDRESS_SPACE_IO:
-            /*
-             * Sanity check: don't allow to register more than 32K of the PCI I/O space.
-             */
-            AssertMsgReturn(cbRegion <= _32K,
-                            ("caller='%s'/%d: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cbRegion),
-                            VERR_INVALID_PARAMETER);
-            break;
-
-        case PCI_ADDRESS_SPACE_MEM:
-        case PCI_ADDRESS_SPACE_MEM_PREFETCH:
-            /*
-             * Sanity check: don't allow to register more than 512MB of the PCI MMIO space for
-             * now. If this limit is increased beyond 2GB, adapt the aligned check below as well!
-             */
-            AssertMsgReturn(cbRegion <= 512 * _1M,
-                            ("caller='%s'/%d: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cbRegion),
-                            VERR_INVALID_PARAMETER);
-            break;
-        default:
-            AssertMsgFailed(("enmType=%#x is unknown\n", enmType));
-            LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (enmType)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-            return VERR_INVALID_PARAMETER;
-    }
-    if (!pfnCallback)
-    {
-        Assert(pfnCallback);
-        LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (callback)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    AssertRelease(VMR3GetState(pVM) != VMSTATE_RUNNING);
-
-    /*
-     * Must have a PCI device registered!
-     */
-    int rc;
-    PPCIDEVICE pPciDev = pDevIns->Internal.s.pPciDeviceR3;
-    if (pPciDev)
-    {
-        /*
-         * We're currently restricted to page aligned MMIO regions.
-         */
-        if (    (enmType == PCI_ADDRESS_SPACE_MEM || enmType == PCI_ADDRESS_SPACE_MEM_PREFETCH)
-            &&  cbRegion != RT_ALIGN_32(cbRegion, PAGE_SIZE))
-        {
-            Log(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: aligning cbRegion %#x -> %#x\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, cbRegion, RT_ALIGN_32(cbRegion, PAGE_SIZE)));
-            cbRegion = RT_ALIGN_32(cbRegion, PAGE_SIZE);
-        }
-
-        /*
-         * For registering PCI MMIO memory or PCI I/O memory, the size of the region must be a power of 2!
-         */
-        int iLastSet = ASMBitLastSetU32(cbRegion);
-        Assert(iLastSet > 0);
-        uint32_t cbRegionAligned = RT_BIT_32(iLastSet - 1);
-        if (cbRegion > cbRegionAligned)
-            cbRegion = cbRegionAligned * 2; /* round up */
-
-        PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3;
-        Assert(pBus);
-        pdmLock(pVM);
-        rc = pBus->pfnIORegionRegisterR3(pBus->pDevInsR3, pPciDev, iRegion, cbRegion, enmType, pfnCallback);
-        pdmUnlock(pVM);
-    }
-    else
-    {
-        AssertMsgFailed(("No PCI device registered!\n"));
-        rc = VERR_PDM_NOT_PCI_DEVICE;
-    }
-
-    LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetConfigCallbacks} */
-static DECLCALLBACK(void) pdmR3DevHlp_PCISetConfigCallbacks(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PFNPCICONFIGREAD pfnRead, PPFNPCICONFIGREAD ppfnReadOld,
-                                                            PFNPCICONFIGWRITE pfnWrite, PPFNPCICONFIGWRITE ppfnWriteOld)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_PCISetConfigCallbacks: caller='%s'/%d: pPciDev=%p pfnRead=%p ppfnReadOld=%p pfnWrite=%p ppfnWriteOld=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pfnRead, ppfnReadOld, pfnWrite, ppfnWriteOld));
-
-    /*
-     * Validate input and resolve defaults.
-     */
-    AssertPtr(pfnRead);
-    AssertPtr(pfnWrite);
-    AssertPtrNull(ppfnReadOld);
-    AssertPtrNull(ppfnWriteOld);
-    AssertPtrNull(pPciDev);
-
-    if (!pPciDev)
-        pPciDev = pDevIns->Internal.s.pPciDeviceR3;
-    AssertReleaseMsg(pPciDev, ("You must register your device first!\n"));
-    PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3;
-    AssertRelease(pBus);
-    AssertRelease(VMR3GetState(pVM) != VMSTATE_RUNNING);
-
-    /*
-     * Do the job.
-     */
-    pdmLock(pVM);
-    pBus->pfnSetConfigCallbacksR3(pBus->pDevInsR3, pPciDev, pfnRead, ppfnReadOld, pfnWrite, ppfnWriteOld);
-    pdmUnlock(pVM);
-
-    LogFlow(("pdmR3DevHlp_PCISetConfigCallbacks: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetIrq} */
-static DECLCALLBACK(void) pdmR3DevHlp_PCISetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
-
-    /*
-     * Validate input.
-     */
-    /** @todo iIrq and iLevel checks. */
-
-    /*
-     * Must have a PCI device registered!
-     */
-    PPCIDEVICE pPciDev = pDevIns->Internal.s.pPciDeviceR3;
-    if (pPciDev)
-    {
-        PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3; /** @todo the bus should be associated with the PCI device not the PDM device. */
-        Assert(pBus);
-        PVM pVM = pDevIns->Internal.s.pVMR3;
-        pdmLock(pVM);
-        pBus->pfnSetIrqR3(pBus->pDevInsR3, pPciDev, iIrq, iLevel);
-        pdmUnlock(pVM);
-    }
-    else
-        AssertReleaseMsgFailed(("No PCI device registered!\n"));
-
-    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetIrqNoWait} */
-static DECLCALLBACK(void) pdmR3DevHlp_PCISetIrqNoWait(PPDMDEVINS pDevIns, int iIrq, int iLevel)
-{
-    pdmR3DevHlp_PCISetIrq(pDevIns, iIrq, iLevel);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPCIRegisterMsi} */
-static DECLCALLBACK(int) pdmR3DevHlp_PCIRegisterMsi(PPDMDEVINS pDevIns, PPDMMSIREG pMsiReg)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_PCIRegisterMsi: caller='%s'/%d: %d MSI vectors %d MSI-X vectors\n", pDevIns->pReg->szName, pDevIns->iInstance, pMsiReg->cMsiVectors,pMsiReg->cMsixVectors ));
-    int rc = VINF_SUCCESS;
-
-    /*
-     * Must have a PCI device registered!
-     */
-    PPCIDEVICE pPciDev = pDevIns->Internal.s.pPciDeviceR3;
-    if (pPciDev)
-    {
-        PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3; /** @todo the bus should be associated with the PCI device not the PDM device. */
-        Assert(pBus);
-
-        PVM pVM = pDevIns->Internal.s.pVMR3;
-        pdmLock(pVM);
-        if (!pBus->pfnRegisterMsiR3)
-            rc = VERR_NOT_IMPLEMENTED;
-        else
-            rc = pBus->pfnRegisterMsiR3(pBus->pDevInsR3, pPciDev, pMsiReg);
-        pdmUnlock(pVM);
-    }
-    else
-        AssertReleaseMsgFailed(("No PCI device registered!\n"));
-
-    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return rc;
-}
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnISASetIrq} */
-static DECLCALLBACK(void) pdmR3DevHlp_ISASetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_ISASetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
-
-    /*
-     * Validate input.
-     */
-    /** @todo iIrq and iLevel checks. */
-
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    PDMIsaSetIrq(pVM, iIrq, iLevel);    /* (The API takes the lock.) */
-
-    LogFlow(("pdmR3DevHlp_ISASetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnISASetIrqNoWait} */
-static DECLCALLBACK(void) pdmR3DevHlp_ISASetIrqNoWait(PPDMDEVINS pDevIns, int iIrq, int iLevel)
-{
-    pdmR3DevHlp_ISASetIrq(pDevIns, iIrq, iLevel);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDriverAttach} */
-static DECLCALLBACK(int) pdmR3DevHlp_DriverAttach(PPDMDEVINS pDevIns, RTUINT iLun, PPDMIBASE pBaseInterface, PPDMIBASE *ppBaseInterface, const char *pszDesc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: iLun=%d pBaseInterface=%p ppBaseInterface=%p pszDesc=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iLun, pBaseInterface, ppBaseInterface, pszDesc, pszDesc));
-
-    /*
-     * Lookup the LUN, it might already be registered.
-     */
-    PPDMLUN pLunPrev = NULL;
-    PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;
-    for (; pLun; pLunPrev = pLun, pLun = pLun->pNext)
-        if (pLun->iLun == iLun)
-            break;
-
-    /*
-     * Create the LUN if if wasn't found, else check if driver is already attached to it.
-     */
-    if (!pLun)
-    {
-        if (    !pBaseInterface
-            ||  !pszDesc
-            ||  !*pszDesc)
-        {
-            Assert(pBaseInterface);
-            Assert(pszDesc || *pszDesc);
-            return VERR_INVALID_PARAMETER;
-        }
-
-        pLun = (PPDMLUN)MMR3HeapAlloc(pVM, MM_TAG_PDM_LUN, sizeof(*pLun));
-        if (!pLun)
-            return VERR_NO_MEMORY;
-
-        pLun->iLun      = iLun;
-        pLun->pNext     = pLunPrev ? pLunPrev->pNext : NULL;
-        pLun->pTop      = NULL;
-        pLun->pBottom   = NULL;
-        pLun->pDevIns   = pDevIns;
-        pLun->pUsbIns   = NULL;
-        pLun->pszDesc   = pszDesc;
-        pLun->pBase     = pBaseInterface;
-        if (!pLunPrev)
-            pDevIns->Internal.s.pLunsR3 = pLun;
-        else
-            pLunPrev->pNext = pLun;
-        Log(("pdmR3DevHlp_DriverAttach: Registered LUN#%d '%s' with device '%s'/%d.\n",
-             iLun, pszDesc, pDevIns->pReg->szName, pDevIns->iInstance));
-    }
-    else if (pLun->pTop)
-    {
-        AssertMsgFailed(("Already attached! The device should keep track of such things!\n"));
-        LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_PDM_DRIVER_ALREADY_ATTACHED));
-        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
-    }
-    Assert(pLun->pBase == pBaseInterface);
-
-
-    /*
-     * Get the attached driver configuration.
-     */
-    int rc;
-    PCFGMNODE pNode = CFGMR3GetChildF(pDevIns->Internal.s.pCfgHandle, "LUN#%u", iLun);
-    if (pNode)
-        rc = pdmR3DrvInstantiate(pVM, pNode, pBaseInterface, NULL /*pDrvAbove*/, pLun, ppBaseInterface);
-    else
-        rc = VERR_PDM_NO_ATTACHED_DRIVER;
-
-
-    LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnQueueCreate} */
-static DECLCALLBACK(int) pdmR3DevHlp_QueueCreate(PPDMDEVINS pDevIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
-                                                 PFNPDMQUEUEDEV pfnCallback, bool fGCEnabled, const char *pszName, PPDMQUEUE *ppQueue)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_QueueCreate: caller='%s'/%d: cbItem=%#x cItems=%#x cMilliesInterval=%u pfnCallback=%p fGCEnabled=%RTbool pszName=%p:{%s} ppQueue=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, fGCEnabled, pszName, pszName, ppQueue));
-
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    if (pDevIns->iInstance > 0)
-    {
-        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s_%u", pszName, pDevIns->iInstance);
-        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
-    }
-
-    int rc = PDMR3QueueCreateDevice(pVM, pDevIns, cbItem, cItems, cMilliesInterval, pfnCallback, fGCEnabled, pszName, ppQueue);
-
-    LogFlow(("pdmR3DevHlp_QueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *ppQueue));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectInit} */
-static DECLCALLBACK(int) pdmR3DevHlp_CritSectInit(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
-                                                  const char *pszNameFmt, va_list va)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_CritSectInit: caller='%s'/%d: pCritSect=%p pszNameFmt=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect, pszNameFmt, pszNameFmt));
-
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    int rc = pdmR3CritSectInitDevice(pVM, pDevIns, pCritSect, RT_SRC_POS_ARGS, pszNameFmt, va);
-
-    LogFlow(("pdmR3DevHlp_CritSectInit: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnThreadCreate} */
-static DECLCALLBACK(int) pdmR3DevHlp_ThreadCreate(PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
-                                                  PFNPDMTHREADWAKEUPDEV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_ThreadCreate: caller='%s'/%d: ppThread=%p pvUser=%p pfnThread=%p pfnWakeup=%p cbStack=%#zx enmType=%d pszName=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName, pszName));
-
-    int rc = pdmR3ThreadCreateDevice(pDevIns->Internal.s.pVMR3, pDevIns, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName);
-
-    LogFlow(("pdmR3DevHlp_ThreadCreate: caller='%s'/%d: returns %Rrc *ppThread=%RTthrd\n", pDevIns->pReg->szName, pDevIns->iInstance,
-            rc, *ppThread));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnSetAsyncNotification} */
-static DECLCALLBACK(int) pdmR3DevHlp_SetAsyncNotification(PPDMDEVINS pDevIns, PFNPDMDEVASYNCNOTIFY pfnAsyncNotify)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT0(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pfnAsyncNotify));
-
-    int rc = VINF_SUCCESS;
-    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
-    AssertStmt(!pDevIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
-    AssertStmt(pDevIns->Internal.s.fIntFlags & (PDMDEVINSINT_FLAGS_SUSPENDED | PDMDEVINSINT_FLAGS_RESET), rc = VERR_WRONG_ORDER);
-    VMSTATE enmVMState = VMR3GetState(pDevIns->Internal.s.pVMR3);
-    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
-               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
-               || enmVMState == VMSTATE_SUSPENDING_LS
-               || enmVMState == VMSTATE_RESETTING
-               || enmVMState == VMSTATE_RESETTING_LS
-               || enmVMState == VMSTATE_POWERING_OFF
-               || enmVMState == VMSTATE_POWERING_OFF_LS,
-               rc = VERR_INVALID_STATE);
-
-    if (RT_SUCCESS(rc))
-        pDevIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
-
-    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnAsyncNotificationCompleted} */
-static DECLCALLBACK(void) pdmR3DevHlp_AsyncNotificationCompleted(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    if (   enmVMState == VMSTATE_SUSPENDING
-        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
-        || enmVMState == VMSTATE_SUSPENDING_LS
-        || enmVMState == VMSTATE_RESETTING
-        || enmVMState == VMSTATE_RESETTING_LS
-        || enmVMState == VMSTATE_POWERING_OFF
-        || enmVMState == VMSTATE_POWERING_OFF_LS)
-    {
-        LogFlow(("pdmR3DevHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
-        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
-    }
-    else
-        LogFlow(("pdmR3DevHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, enmVMState));
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnRTCRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_RTCRegister(PPDMDEVINS pDevIns, PCPDMRTCREG pRtcReg, PCPDMRTCHLP *ppRtcHlp)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: pRtcReg=%p:{.u32Version=%#x, .pfnWrite=%p, .pfnRead=%p} ppRtcHlp=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pRtcReg, pRtcReg->u32Version, pRtcReg->pfnWrite,
-             pRtcReg->pfnWrite, ppRtcHlp));
-
-    /*
-     * Validate input.
-     */
-    if (pRtcReg->u32Version != PDM_RTCREG_VERSION)
-    {
-        AssertMsgFailed(("u32Version=%#x expected %#x\n", pRtcReg->u32Version,
-                         PDM_RTCREG_VERSION));
-        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (version)\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    !pRtcReg->pfnWrite
-        ||  !pRtcReg->pfnRead)
-    {
-        Assert(pRtcReg->pfnWrite);
-        Assert(pRtcReg->pfnRead);
-        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (callbacks)\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (!ppRtcHlp)
-    {
-        Assert(ppRtcHlp);
-        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (ppRtcHlp)\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Only one DMA device.
-     */
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    if (pVM->pdm.s.pRtc)
-    {
-        AssertMsgFailed(("Only one RTC device is supported!\n"));
-        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Allocate and initialize pci bus structure.
-     */
-    int rc = VINF_SUCCESS;
-    PPDMRTC pRtc = (PPDMRTC)MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE, sizeof(*pRtc));
-    if (pRtc)
-    {
-        pRtc->pDevIns   = pDevIns;
-        pRtc->Reg       = *pRtcReg;
-        pVM->pdm.s.pRtc = pRtc;
-
-        /* set the helper pointer. */
-        *ppRtcHlp = &g_pdmR3DevRtcHlp;
-        Log(("PDM: Registered RTC device '%s'/%d pDevIns=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
-    }
-    else
-        rc = VERR_NO_MEMORY;
-
-    LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDMARegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_DMARegister(PPDMDEVINS pDevIns, unsigned uChannel, PFNDMATRANSFERHANDLER pfnTransferHandler, void *pvUser)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_DMARegister: caller='%s'/%d: uChannel=%d pfnTransferHandler=%p pvUser=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pfnTransferHandler, pvUser));
-    int rc = VINF_SUCCESS;
-    if (pVM->pdm.s.pDmac)
-        pVM->pdm.s.pDmac->Reg.pfnRegister(pVM->pdm.s.pDmac->pDevIns, uChannel, pfnTransferHandler, pvUser);
-    else
-    {
-        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
-        rc = VERR_PDM_NO_DMAC_INSTANCE;
-    }
-    LogFlow(("pdmR3DevHlp_DMARegister: caller='%s'/%d: returns %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDMAReadMemory} */
-static DECLCALLBACK(int) pdmR3DevHlp_DMAReadMemory(PPDMDEVINS pDevIns, unsigned uChannel, void *pvBuffer, uint32_t off, uint32_t cbBlock, uint32_t *pcbRead)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_DMAReadMemory: caller='%s'/%d: uChannel=%d pvBuffer=%p off=%#x cbBlock=%#x pcbRead=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pvBuffer, off, cbBlock, pcbRead));
-    int rc = VINF_SUCCESS;
-    if (pVM->pdm.s.pDmac)
-    {
-        uint32_t cb = pVM->pdm.s.pDmac->Reg.pfnReadMemory(pVM->pdm.s.pDmac->pDevIns, uChannel, pvBuffer, off, cbBlock);
-        if (pcbRead)
-            *pcbRead = cb;
-    }
-    else
-    {
-        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
-        rc = VERR_PDM_NO_DMAC_INSTANCE;
-    }
-    LogFlow(("pdmR3DevHlp_DMAReadMemory: caller='%s'/%d: returns %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDMAWriteMemory} */
-static DECLCALLBACK(int) pdmR3DevHlp_DMAWriteMemory(PPDMDEVINS pDevIns, unsigned uChannel, const void *pvBuffer, uint32_t off, uint32_t cbBlock, uint32_t *pcbWritten)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_DMAWriteMemory: caller='%s'/%d: uChannel=%d pvBuffer=%p off=%#x cbBlock=%#x pcbWritten=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pvBuffer, off, cbBlock, pcbWritten));
-    int rc = VINF_SUCCESS;
-    if (pVM->pdm.s.pDmac)
-    {
-        uint32_t cb = pVM->pdm.s.pDmac->Reg.pfnWriteMemory(pVM->pdm.s.pDmac->pDevIns, uChannel, pvBuffer, off, cbBlock);
-        if (pcbWritten)
-            *pcbWritten = cb;
-    }
-    else
-    {
-        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
-        rc = VERR_PDM_NO_DMAC_INSTANCE;
-    }
-    LogFlow(("pdmR3DevHlp_DMAWriteMemory: caller='%s'/%d: returns %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDMASetDREQ} */
-static DECLCALLBACK(int) pdmR3DevHlp_DMASetDREQ(PPDMDEVINS pDevIns, unsigned uChannel, unsigned uLevel)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_DMASetDREQ: caller='%s'/%d: uChannel=%d uLevel=%d\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, uLevel));
-    int rc = VINF_SUCCESS;
-    if (pVM->pdm.s.pDmac)
-        pVM->pdm.s.pDmac->Reg.pfnSetDREQ(pVM->pdm.s.pDmac->pDevIns, uChannel, uLevel);
-    else
-    {
-        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
-        rc = VERR_PDM_NO_DMAC_INSTANCE;
-    }
-    LogFlow(("pdmR3DevHlp_DMASetDREQ: caller='%s'/%d: returns %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDMAGetChannelMode} */
-static DECLCALLBACK(uint8_t) pdmR3DevHlp_DMAGetChannelMode(PPDMDEVINS pDevIns, unsigned uChannel)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_DMAGetChannelMode: caller='%s'/%d: uChannel=%d\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, uChannel));
-    uint8_t u8Mode;
-    if (pVM->pdm.s.pDmac)
-        u8Mode = pVM->pdm.s.pDmac->Reg.pfnGetChannelMode(pVM->pdm.s.pDmac->pDevIns, uChannel);
-    else
-    {
-        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
-        u8Mode = 3 << 2 /* illegal mode type */;
-    }
-    LogFlow(("pdmR3DevHlp_DMAGetChannelMode: caller='%s'/%d: returns %#04x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, u8Mode));
-    return u8Mode;
-}
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDMASchedule} */
-static DECLCALLBACK(void) pdmR3DevHlp_DMASchedule(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_DMASchedule: caller='%s'/%d: VM_FF_PDM_DMA %d -> 1\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, VM_FF_ISSET(pVM, VM_FF_PDM_DMA)));
-
-    AssertMsg(pVM->pdm.s.pDmac, ("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
-    VM_FF_SET(pVM, VM_FF_PDM_DMA);
-    REMR3NotifyDmaPending(pVM);
-    VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnCMOSWrite} */
-static DECLCALLBACK(int) pdmR3DevHlp_CMOSWrite(PPDMDEVINS pDevIns, unsigned iReg, uint8_t u8Value)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: iReg=%#04x u8Value=%#04x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iReg, u8Value));
-    int rc;
-    if (pVM->pdm.s.pRtc)
-        rc = pVM->pdm.s.pRtc->Reg.pfnWrite(pVM->pdm.s.pRtc->pDevIns, iReg, u8Value);
-    else
-        rc = VERR_PDM_NO_RTC_INSTANCE;
-
-    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: return %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnCMOSRead} */
-static DECLCALLBACK(int) pdmR3DevHlp_CMOSRead(PPDMDEVINS pDevIns, unsigned iReg, uint8_t *pu8Value)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: iReg=%#04x pu8Value=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iReg, pu8Value));
-    int rc;
-    if (pVM->pdm.s.pRtc)
-        rc = pVM->pdm.s.pRtc->Reg.pfnRead(pVM->pdm.s.pRtc->pDevIns, iReg, pu8Value);
-    else
-        rc = VERR_PDM_NO_RTC_INSTANCE;
-
-    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: return %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnAssertEMT} */
-static DECLCALLBACK(bool) pdmR3DevHlp_AssertEMT(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    if (VM_IS_EMT(pDevIns->Internal.s.pVMR3))
-        return true;
-
-    char szMsg[100];
-    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance);
-    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
-    AssertBreakpoint();
-    return false;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnAssertOther} */
-static DECLCALLBACK(bool) pdmR3DevHlp_AssertOther(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    if (!VM_IS_EMT(pDevIns->Internal.s.pVMR3))
-        return true;
-
-    char szMsg[100];
-    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance);
-    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
-    AssertBreakpoint();
-    return false;
-}
-
-
-/** @interface_method_impl{PDMDEVHLP,pfnLdrGetRCInterfaceSymbols} */
-static DECLCALLBACK(int) pdmR3DevHlp_LdrGetRCInterfaceSymbols(PPDMDEVINS pDevIns, void *pvInterface, size_t cbInterface,
-                                                              const char *pszSymPrefix, const char *pszSymList)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_PDMLdrGetRCInterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
-
-    int rc;
-    if (   strncmp(pszSymPrefix, "dev", 3) == 0
-        && RTStrIStr(pszSymPrefix + 3, pDevIns->pReg->szName) != NULL)
-    {
-        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
-            rc = PDMR3LdrGetInterfaceSymbols(pDevIns->Internal.s.pVMR3,
-                                             pvInterface, cbInterface,
-                                             pDevIns->pReg->szRCMod, pDevIns->Internal.s.pDevR3->pszRCSearchPath,
-                                             pszSymPrefix, pszSymList,
-                                             false /*fRing0OrRC*/);
-        else
-        {
-            AssertMsgFailed(("Not a raw-mode enabled driver\n"));
-            rc = VERR_PERMISSION_DENIED;
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'dev' and contain the driver name!\n",
-                         pszSymPrefix, pDevIns->pReg->szName));
-        rc = VERR_INVALID_NAME;
-    }
-
-    LogFlow(("pdmR3DevHlp_PDMLdrGetRCInterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
-             pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLP,pfnLdrGetR0InterfaceSymbols} */
-static DECLCALLBACK(int) pdmR3DevHlp_LdrGetR0InterfaceSymbols(PPDMDEVINS pDevIns, void *pvInterface, size_t cbInterface,
-                                                              const char *pszSymPrefix, const char *pszSymList)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_PDMLdrGetR0InterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
-
-    int rc;
-    if (   strncmp(pszSymPrefix, "dev", 3) == 0
-        && RTStrIStr(pszSymPrefix + 3, pDevIns->pReg->szName) != NULL)
-    {
-        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
-            rc = PDMR3LdrGetInterfaceSymbols(pDevIns->Internal.s.pVMR3,
-                                             pvInterface, cbInterface,
-                                             pDevIns->pReg->szR0Mod, pDevIns->Internal.s.pDevR3->pszR0SearchPath,
-                                             pszSymPrefix, pszSymList,
-                                             true /*fRing0OrRC*/);
-        else
-        {
-            AssertMsgFailed(("Not a ring-0 enabled driver\n"));
-            rc = VERR_PERMISSION_DENIED;
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'dev' and contain the driver name!\n",
-                         pszSymPrefix, pDevIns->pReg->szName));
-        rc = VERR_INVALID_NAME;
-    }
-
-    LogFlow(("pdmR3DevHlp_PDMLdrGetR0InterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
-             pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLP,pfnCallR0} */
-static DECLCALLBACK(int) pdmR3DevHlp_CallR0(PPDMDEVINS pDevIns, uint32_t uOperation, uint64_t u64Arg)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_CallR0: caller='%s'/%d: uOperation=%#x u64Arg=%#RX64\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, uOperation, u64Arg));
-
-    /*
-     * Resolve the ring-0 entry point.  There is not need to remember this like
-     * we do for drivers since this is mainly for construction time hacks and
-     * other things that aren't performance critical.
-     */
-    int rc;
-    if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
-    {
-        char szSymbol[          sizeof("devR0") + sizeof(pDevIns->pReg->szName) + sizeof("ReqHandler")];
-        strcat(strcat(strcpy(szSymbol, "devR0"),         pDevIns->pReg->szName),         "ReqHandler");
-        szSymbol[sizeof("devR0") - 1] = RT_C_TO_UPPER(szSymbol[sizeof("devR0") - 1]);
-
-        PFNPDMDRVREQHANDLERR0 pfnReqHandlerR0;
-        rc = pdmR3DevGetSymbolR0Lazy(pDevIns, szSymbol, &pfnReqHandlerR0);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Make the ring-0 call.
-             */
-            PDMDEVICECALLREQHANDLERREQ Req;
-            Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
-            Req.Hdr.cbReq       = sizeof(Req);
-            Req.pDevInsR0       = PDMDEVINS_2_R0PTR(pDevIns);
-            Req.pfnReqHandlerR0 = pfnReqHandlerR0;
-            Req.uOperation      = uOperation;
-            Req.u32Alignment    = 0;
-            Req.u64Arg          = u64Arg;
-            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_PDM_DEVICE_CALL_REQ_HANDLER, 0, &Req.Hdr);
-        }
-        else
-            pfnReqHandlerR0 = NIL_RTR0PTR;
-    }
-    else
-        rc = VERR_ACCESS_DENIED;
-    LogFlow(("pdmR3DevHlp_CallR0: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
-             pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnGetVM} */
-static DECLCALLBACK(PVM) pdmR3DevHlp_GetVM(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3DevHlp_GetVM: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns->Internal.s.pVMR3));
-    return pDevIns->Internal.s.pVMR3;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnGetVMCPU} */
-static DECLCALLBACK(PVMCPU) pdmR3DevHlp_GetVMCPU(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_GetVMCPU: caller='%s'/%d for CPU %u\n", pDevIns->pReg->szName, pDevIns->iInstance, VMMGetCpuId(pDevIns->Internal.s.pVMR3)));
-    return VMMGetCpu(pDevIns->Internal.s.pVMR3);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPCIBusRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_PCIBusRegister(PPDMDEVINS pDevIns, PPDMPCIBUSREG pPciBusReg, PCPDMPCIHLPR3 *ppPciHlpR3)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: pPciBusReg=%p:{.u32Version=%#x, .pfnRegisterR3=%p, .pfnIORegionRegisterR3=%p, .pfnSetIrqR3=%p, "
-             ".pfnSaveExecR3=%p, .pfnLoadExecR3=%p, .pfnFakePCIBIOSR3=%p, .pszSetIrqRC=%p:{%s}, .pszSetIrqR0=%p:{%s}} ppPciHlpR3=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pPciBusReg, pPciBusReg->u32Version, pPciBusReg->pfnRegisterR3,
-             pPciBusReg->pfnIORegionRegisterR3, pPciBusReg->pfnSetIrqR3, pPciBusReg->pfnSaveExecR3, pPciBusReg->pfnLoadExecR3,
-             pPciBusReg->pfnFakePCIBIOSR3, pPciBusReg->pszSetIrqRC, pPciBusReg->pszSetIrqRC, pPciBusReg->pszSetIrqR0, pPciBusReg->pszSetIrqR0, ppPciHlpR3));
-
-    /*
-     * Validate the structure.
-     */
-    if (pPciBusReg->u32Version != PDM_PCIBUSREG_VERSION)
-    {
-        AssertMsgFailed(("u32Version=%#x expected %#x\n", pPciBusReg->u32Version, PDM_PCIBUSREG_VERSION));
-        LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    !pPciBusReg->pfnRegisterR3
-        ||  !pPciBusReg->pfnIORegionRegisterR3
-        ||  !pPciBusReg->pfnSetIrqR3
-        ||  !pPciBusReg->pfnSaveExecR3
-        ||  !pPciBusReg->pfnLoadExecR3
-        ||  (!pPciBusReg->pfnFakePCIBIOSR3 && !pVM->pdm.s.aPciBuses[0].pDevInsR3)) /* Only the first bus needs to do the BIOS work. */
-    {
-        Assert(pPciBusReg->pfnRegisterR3);
-        Assert(pPciBusReg->pfnIORegionRegisterR3);
-        Assert(pPciBusReg->pfnSetIrqR3);
-        Assert(pPciBusReg->pfnSaveExecR3);
-        Assert(pPciBusReg->pfnLoadExecR3);
-        Assert(pPciBusReg->pfnFakePCIBIOSR3);
-        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pPciBusReg->pszSetIrqRC
-        &&  !VALID_PTR(pPciBusReg->pszSetIrqRC))
-    {
-        Assert(VALID_PTR(pPciBusReg->pszSetIrqRC));
-        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pPciBusReg->pszSetIrqR0
-        &&  !VALID_PTR(pPciBusReg->pszSetIrqR0))
-    {
-        Assert(VALID_PTR(pPciBusReg->pszSetIrqR0));
-        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-     if (!ppPciHlpR3)
-    {
-        Assert(ppPciHlpR3);
-        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (ppPciHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Find free PCI bus entry.
-     */
-    unsigned iBus = 0;
-    for (iBus = 0; iBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses); iBus++)
-        if (!pVM->pdm.s.aPciBuses[iBus].pDevInsR3)
-            break;
-    if (iBus >= RT_ELEMENTS(pVM->pdm.s.aPciBuses))
-    {
-        AssertMsgFailed(("Too many PCI buses. Max=%u\n", RT_ELEMENTS(pVM->pdm.s.aPciBuses)));
-        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (pci bus)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    PPDMPCIBUS pPciBus = &pVM->pdm.s.aPciBuses[iBus];
-
-    /*
-     * Resolve and init the RC bits.
-     */
-    if (pPciBusReg->pszSetIrqRC)
-    {
-        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pPciBusReg->pszSetIrqRC, &pPciBus->pfnSetIrqRC);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pPciBusReg->pszSetIrqRC, rc));
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pPciBus->pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
-    }
-    else
-    {
-        pPciBus->pfnSetIrqRC  = 0;
-        pPciBus->pDevInsRC    = 0;
-    }
-
-    /*
-     * Resolve and init the R0 bits.
-     */
-    if (pPciBusReg->pszSetIrqR0)
-    {
-        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pPciBusReg->pszSetIrqR0, &pPciBus->pfnSetIrqR0);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pPciBusReg->pszSetIrqR0, rc));
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pPciBus->pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
-    }
-    else
-    {
-        pPciBus->pfnSetIrqR0 = 0;
-        pPciBus->pDevInsR0   = 0;
-    }
-
-    /*
-     * Init the R3 bits.
-     */
-    pPciBus->iBus                    = iBus;
-    pPciBus->pDevInsR3               = pDevIns;
-    pPciBus->pfnRegisterR3           = pPciBusReg->pfnRegisterR3;
-    pPciBus->pfnRegisterMsiR3        = pPciBusReg->pfnRegisterMsiR3;
-    pPciBus->pfnIORegionRegisterR3   = pPciBusReg->pfnIORegionRegisterR3;
-    pPciBus->pfnSetConfigCallbacksR3 = pPciBusReg->pfnSetConfigCallbacksR3;
-    pPciBus->pfnSetIrqR3             = pPciBusReg->pfnSetIrqR3;
-    pPciBus->pfnSaveExecR3           = pPciBusReg->pfnSaveExecR3;
-    pPciBus->pfnLoadExecR3           = pPciBusReg->pfnLoadExecR3;
-    pPciBus->pfnFakePCIBIOSR3        = pPciBusReg->pfnFakePCIBIOSR3;
-
-    Log(("PDM: Registered PCI bus device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
-
-    /* set the helper pointer and return. */
-    *ppPciHlpR3 = &g_pdmR3DevPciHlp;
-    LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
-    return VINF_SUCCESS;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnPICRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_PICRegister(PPDMDEVINS pDevIns, PPDMPICREG pPicReg, PCPDMPICHLPR3 *ppPicHlpR3)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: pPicReg=%p:{.u32Version=%#x, .pfnSetIrqR3=%p, .pfnGetInterruptR3=%p, .pszGetIrqRC=%p:{%s}, .pszGetInterruptRC=%p:{%s}, .pszGetIrqR0=%p:{%s}, .pszGetInterruptR0=%p:{%s} } ppPicHlpR3=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pPicReg, pPicReg->u32Version, pPicReg->pfnSetIrqR3, pPicReg->pfnGetInterruptR3,
-             pPicReg->pszSetIrqRC, pPicReg->pszSetIrqRC, pPicReg->pszGetInterruptRC, pPicReg->pszGetInterruptRC,
-             pPicReg->pszSetIrqR0, pPicReg->pszSetIrqR0, pPicReg->pszGetInterruptR0, pPicReg->pszGetInterruptR0,
-             ppPicHlpR3));
-
-    /*
-     * Validate input.
-     */
-    if (pPicReg->u32Version != PDM_PICREG_VERSION)
-    {
-        AssertMsgFailed(("u32Version=%#x expected %#x\n", pPicReg->u32Version, PDM_PICREG_VERSION));
-        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    !pPicReg->pfnSetIrqR3
-        ||  !pPicReg->pfnGetInterruptR3)
-    {
-        Assert(pPicReg->pfnSetIrqR3);
-        Assert(pPicReg->pfnGetInterruptR3);
-        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    (   pPicReg->pszSetIrqRC
-             || pPicReg->pszGetInterruptRC)
-        &&  (   !VALID_PTR(pPicReg->pszSetIrqRC)
-             || !VALID_PTR(pPicReg->pszGetInterruptRC))
-       )
-    {
-        Assert(VALID_PTR(pPicReg->pszSetIrqRC));
-        Assert(VALID_PTR(pPicReg->pszGetInterruptRC));
-        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (RC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pPicReg->pszSetIrqRC
-        &&  !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC))
-    {
-        Assert(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC);
-        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (RC flag)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pPicReg->pszSetIrqR0
-        &&  !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0))
-    {
-        Assert(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0);
-        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (R0 flag)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!ppPicHlpR3)
-    {
-        Assert(ppPicHlpR3);
-        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (ppPicHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Only one PIC device.
-     */
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    if (pVM->pdm.s.Pic.pDevInsR3)
-    {
-        AssertMsgFailed(("Only one pic device is supported!\n"));
-        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * RC stuff.
-     */
-    if (pPicReg->pszSetIrqRC)
-    {
-        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pPicReg->pszSetIrqRC, &pVM->pdm.s.Pic.pfnSetIrqRC);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pPicReg->pszSetIrqRC, rc));
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pPicReg->pszGetInterruptRC, &pVM->pdm.s.Pic.pfnGetInterruptRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pPicReg->pszGetInterruptRC, rc));
-        }
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pVM->pdm.s.Pic.pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
-    }
-    else
-    {
-        pVM->pdm.s.Pic.pDevInsRC = 0;
-        pVM->pdm.s.Pic.pfnSetIrqRC = 0;
-        pVM->pdm.s.Pic.pfnGetInterruptRC = 0;
-    }
-
-    /*
-     * R0 stuff.
-     */
-    if (pPicReg->pszSetIrqR0)
-    {
-        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pPicReg->pszSetIrqR0, &pVM->pdm.s.Pic.pfnSetIrqR0);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pPicReg->pszSetIrqR0, rc));
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pPicReg->pszGetInterruptR0, &pVM->pdm.s.Pic.pfnGetInterruptR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pPicReg->pszGetInterruptR0, rc));
-        }
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pVM->pdm.s.Pic.pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
-        Assert(pVM->pdm.s.Pic.pDevInsR0);
-    }
-    else
-    {
-        pVM->pdm.s.Pic.pfnSetIrqR0 = 0;
-        pVM->pdm.s.Pic.pfnGetInterruptR0 = 0;
-        pVM->pdm.s.Pic.pDevInsR0 = 0;
-    }
-
-    /*
-     * R3 stuff.
-     */
-    pVM->pdm.s.Pic.pDevInsR3 = pDevIns;
-    pVM->pdm.s.Pic.pfnSetIrqR3 = pPicReg->pfnSetIrqR3;
-    pVM->pdm.s.Pic.pfnGetInterruptR3 = pPicReg->pfnGetInterruptR3;
-    Log(("PDM: Registered PIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
-
-    /* set the helper pointer and return. */
-    *ppPicHlpR3 = &g_pdmR3DevPicHlp;
-    LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
-    return VINF_SUCCESS;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnAPICRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_APICRegister(PPDMDEVINS pDevIns, PPDMAPICREG pApicReg, PCPDMAPICHLPR3 *ppApicHlpR3)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: pApicReg=%p:{.u32Version=%#x, .pfnGetInterruptR3=%p, .pfnSetBaseR3=%p, .pfnGetBaseR3=%p, "
-             ".pfnSetTPRR3=%p, .pfnGetTPRR3=%p, .pfnWriteMSR3=%p, .pfnReadMSR3=%p, .pfnBusDeliverR3=%p, .pfnLocalInterruptR3=%p, pszGetInterruptRC=%p:{%s}, pszSetBaseRC=%p:{%s}, pszGetBaseRC=%p:{%s}, "
-             ".pszSetTPRRC=%p:{%s}, .pszGetTPRRC=%p:{%s}, .pszWriteMSRRC=%p:{%s}, .pszReadMSRRC=%p:{%s}, .pszBusDeliverRC=%p:{%s}, .pszLocalInterruptRC=%p:{%s}} ppApicHlpR3=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pApicReg, pApicReg->u32Version, pApicReg->pfnGetInterruptR3, pApicReg->pfnSetBaseR3,
-             pApicReg->pfnGetBaseR3, pApicReg->pfnSetTPRR3, pApicReg->pfnGetTPRR3, pApicReg->pfnWriteMSRR3, pApicReg->pfnReadMSRR3, pApicReg->pfnBusDeliverR3, pApicReg->pfnLocalInterruptR3, pApicReg->pszGetInterruptRC,
-             pApicReg->pszGetInterruptRC, pApicReg->pszSetBaseRC, pApicReg->pszSetBaseRC, pApicReg->pszGetBaseRC, pApicReg->pszGetBaseRC,
-             pApicReg->pszSetTPRRC, pApicReg->pszSetTPRRC, pApicReg->pszGetTPRRC, pApicReg->pszGetTPRRC, pApicReg->pszWriteMSRRC, pApicReg->pszWriteMSRRC, pApicReg->pszReadMSRRC, pApicReg->pszReadMSRRC, pApicReg->pszBusDeliverRC,
-             pApicReg->pszBusDeliverRC, pApicReg->pszLocalInterruptRC, pApicReg->pszLocalInterruptRC, ppApicHlpR3));
-
-    /*
-     * Validate input.
-     */
-    if (pApicReg->u32Version != PDM_APICREG_VERSION)
-    {
-        AssertMsgFailed(("u32Version=%#x expected %#x\n", pApicReg->u32Version, PDM_APICREG_VERSION));
-        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    !pApicReg->pfnGetInterruptR3
-        ||  !pApicReg->pfnHasPendingIrqR3
-        ||  !pApicReg->pfnSetBaseR3
-        ||  !pApicReg->pfnGetBaseR3
-        ||  !pApicReg->pfnSetTPRR3
-        ||  !pApicReg->pfnGetTPRR3
-        ||  !pApicReg->pfnWriteMSRR3
-        ||  !pApicReg->pfnReadMSRR3
-        ||  !pApicReg->pfnBusDeliverR3
-        ||  !pApicReg->pfnLocalInterruptR3)
-    {
-        Assert(pApicReg->pfnGetInterruptR3);
-        Assert(pApicReg->pfnHasPendingIrqR3);
-        Assert(pApicReg->pfnSetBaseR3);
-        Assert(pApicReg->pfnGetBaseR3);
-        Assert(pApicReg->pfnSetTPRR3);
-        Assert(pApicReg->pfnGetTPRR3);
-        Assert(pApicReg->pfnWriteMSRR3);
-        Assert(pApicReg->pfnReadMSRR3);
-        Assert(pApicReg->pfnBusDeliverR3);
-        Assert(pApicReg->pfnLocalInterruptR3);
-        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (   (    pApicReg->pszGetInterruptRC
-            ||  pApicReg->pszHasPendingIrqRC
-            ||  pApicReg->pszSetBaseRC
-            ||  pApicReg->pszGetBaseRC
-            ||  pApicReg->pszSetTPRRC
-            ||  pApicReg->pszGetTPRRC
-            ||  pApicReg->pszWriteMSRRC
-            ||  pApicReg->pszReadMSRRC
-            ||  pApicReg->pszBusDeliverRC
-            ||  pApicReg->pszLocalInterruptRC)
-        &&  (   !VALID_PTR(pApicReg->pszGetInterruptRC)
-            ||  !VALID_PTR(pApicReg->pszHasPendingIrqRC)
-            ||  !VALID_PTR(pApicReg->pszSetBaseRC)
-            ||  !VALID_PTR(pApicReg->pszGetBaseRC)
-            ||  !VALID_PTR(pApicReg->pszSetTPRRC)
-            ||  !VALID_PTR(pApicReg->pszGetTPRRC)
-            ||  !VALID_PTR(pApicReg->pszWriteMSRRC)
-            ||  !VALID_PTR(pApicReg->pszReadMSRRC)
-            ||  !VALID_PTR(pApicReg->pszBusDeliverRC)
-            ||  !VALID_PTR(pApicReg->pszLocalInterruptRC))
-       )
-    {
-        Assert(VALID_PTR(pApicReg->pszGetInterruptRC));
-        Assert(VALID_PTR(pApicReg->pszHasPendingIrqRC));
-        Assert(VALID_PTR(pApicReg->pszSetBaseRC));
-        Assert(VALID_PTR(pApicReg->pszGetBaseRC));
-        Assert(VALID_PTR(pApicReg->pszSetTPRRC));
-        Assert(VALID_PTR(pApicReg->pszGetTPRRC));
-        Assert(VALID_PTR(pApicReg->pszReadMSRRC));
-        Assert(VALID_PTR(pApicReg->pszWriteMSRRC));
-        Assert(VALID_PTR(pApicReg->pszBusDeliverRC));
-        Assert(VALID_PTR(pApicReg->pszLocalInterruptRC));
-        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (RC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (   (    pApicReg->pszGetInterruptR0
-            ||  pApicReg->pszHasPendingIrqR0
-            ||  pApicReg->pszSetBaseR0
-            ||  pApicReg->pszGetBaseR0
-            ||  pApicReg->pszSetTPRR0
-            ||  pApicReg->pszGetTPRR0
-            ||  pApicReg->pszWriteMSRR0
-            ||  pApicReg->pszReadMSRR0
-            ||  pApicReg->pszBusDeliverR0
-            ||  pApicReg->pszLocalInterruptR0)
-        &&  (   !VALID_PTR(pApicReg->pszGetInterruptR0)
-            ||  !VALID_PTR(pApicReg->pszHasPendingIrqR0)
-            ||  !VALID_PTR(pApicReg->pszSetBaseR0)
-            ||  !VALID_PTR(pApicReg->pszGetBaseR0)
-            ||  !VALID_PTR(pApicReg->pszSetTPRR0)
-            ||  !VALID_PTR(pApicReg->pszGetTPRR0)
-            ||  !VALID_PTR(pApicReg->pszReadMSRR0)
-            ||  !VALID_PTR(pApicReg->pszWriteMSRR0)
-            ||  !VALID_PTR(pApicReg->pszBusDeliverR0)
-            ||  !VALID_PTR(pApicReg->pszLocalInterruptR0))
-       )
-    {
-        Assert(VALID_PTR(pApicReg->pszGetInterruptR0));
-        Assert(VALID_PTR(pApicReg->pszHasPendingIrqR0));
-        Assert(VALID_PTR(pApicReg->pszSetBaseR0));
-        Assert(VALID_PTR(pApicReg->pszGetBaseR0));
-        Assert(VALID_PTR(pApicReg->pszSetTPRR0));
-        Assert(VALID_PTR(pApicReg->pszGetTPRR0));
-        Assert(VALID_PTR(pApicReg->pszReadMSRR0));
-        Assert(VALID_PTR(pApicReg->pszWriteMSRR0));
-        Assert(VALID_PTR(pApicReg->pszBusDeliverR0));
-        Assert(VALID_PTR(pApicReg->pszLocalInterruptR0));
-        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (R0 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!ppApicHlpR3)
-    {
-        Assert(ppApicHlpR3);
-        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (ppApicHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Only one APIC device. On SMP we have single logical device covering all LAPICs,
-     * as they need to communicate and share state easily.
-     */
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    if (pVM->pdm.s.Apic.pDevInsR3)
-    {
-        AssertMsgFailed(("Only one apic device is supported!\n"));
-        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Resolve & initialize the RC bits.
-     */
-    if (pApicReg->pszGetInterruptRC)
-    {
-        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszGetInterruptRC, &pVM->pdm.s.Apic.pfnGetInterruptRC);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszGetInterruptRC, rc));
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszHasPendingIrqRC, &pVM->pdm.s.Apic.pfnHasPendingIrqRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszHasPendingIrqRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszSetBaseRC, &pVM->pdm.s.Apic.pfnSetBaseRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszSetBaseRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszGetBaseRC, &pVM->pdm.s.Apic.pfnGetBaseRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszGetBaseRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszSetTPRRC, &pVM->pdm.s.Apic.pfnSetTPRRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszSetTPRRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszGetTPRRC, &pVM->pdm.s.Apic.pfnGetTPRRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszGetTPRRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszWriteMSRRC, &pVM->pdm.s.Apic.pfnWriteMSRRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszWriteMSRRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszReadMSRRC, &pVM->pdm.s.Apic.pfnReadMSRRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszReadMSRRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszBusDeliverRC, &pVM->pdm.s.Apic.pfnBusDeliverRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszBusDeliverRC, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszLocalInterruptRC, &pVM->pdm.s.Apic.pfnLocalInterruptRC);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszLocalInterruptRC, rc));
-        }
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pVM->pdm.s.Apic.pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
-    }
-    else
-    {
-        pVM->pdm.s.Apic.pDevInsRC           = 0;
-        pVM->pdm.s.Apic.pfnGetInterruptRC   = 0;
-        pVM->pdm.s.Apic.pfnHasPendingIrqRC  = 0;
-        pVM->pdm.s.Apic.pfnSetBaseRC        = 0;
-        pVM->pdm.s.Apic.pfnGetBaseRC        = 0;
-        pVM->pdm.s.Apic.pfnSetTPRRC         = 0;
-        pVM->pdm.s.Apic.pfnGetTPRRC         = 0;
-        pVM->pdm.s.Apic.pfnWriteMSRRC       = 0;
-        pVM->pdm.s.Apic.pfnReadMSRRC        = 0;
-        pVM->pdm.s.Apic.pfnBusDeliverRC     = 0;
-        pVM->pdm.s.Apic.pfnLocalInterruptRC = 0;
-    }
-
-    /*
-     * Resolve & initialize the R0 bits.
-     */
-    if (pApicReg->pszGetInterruptR0)
-    {
-        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszGetInterruptR0, &pVM->pdm.s.Apic.pfnGetInterruptR0);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszGetInterruptR0, rc));
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszHasPendingIrqR0, &pVM->pdm.s.Apic.pfnHasPendingIrqR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszHasPendingIrqR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszSetBaseR0, &pVM->pdm.s.Apic.pfnSetBaseR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszSetBaseR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszGetBaseR0, &pVM->pdm.s.Apic.pfnGetBaseR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszGetBaseR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszSetTPRR0, &pVM->pdm.s.Apic.pfnSetTPRR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszSetTPRR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszGetTPRR0, &pVM->pdm.s.Apic.pfnGetTPRR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszGetTPRR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszWriteMSRR0, &pVM->pdm.s.Apic.pfnWriteMSRR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszWriteMSRR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszReadMSRR0, &pVM->pdm.s.Apic.pfnReadMSRR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszReadMSRR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszBusDeliverR0, &pVM->pdm.s.Apic.pfnBusDeliverR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszBusDeliverR0, rc));
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszLocalInterruptR0, &pVM->pdm.s.Apic.pfnLocalInterruptR0);
-            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszLocalInterruptR0, rc));
-        }
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pVM->pdm.s.Apic.pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
-        Assert(pVM->pdm.s.Apic.pDevInsR0);
-    }
-    else
-    {
-        pVM->pdm.s.Apic.pfnGetInterruptR0   = 0;
-        pVM->pdm.s.Apic.pfnHasPendingIrqR0  = 0;
-        pVM->pdm.s.Apic.pfnSetBaseR0        = 0;
-        pVM->pdm.s.Apic.pfnGetBaseR0        = 0;
-        pVM->pdm.s.Apic.pfnSetTPRR0         = 0;
-        pVM->pdm.s.Apic.pfnGetTPRR0         = 0;
-        pVM->pdm.s.Apic.pfnWriteMSRR0       = 0;
-        pVM->pdm.s.Apic.pfnReadMSRR0        = 0;
-        pVM->pdm.s.Apic.pfnBusDeliverR0     = 0;
-        pVM->pdm.s.Apic.pfnLocalInterruptR0 = 0;
-        pVM->pdm.s.Apic.pDevInsR0           = 0;
-    }
-
-    /*
-     * Initialize the HC bits.
-     */
-    pVM->pdm.s.Apic.pDevInsR3           = pDevIns;
-    pVM->pdm.s.Apic.pfnGetInterruptR3   = pApicReg->pfnGetInterruptR3;
-    pVM->pdm.s.Apic.pfnHasPendingIrqR3  = pApicReg->pfnHasPendingIrqR3;
-    pVM->pdm.s.Apic.pfnSetBaseR3        = pApicReg->pfnSetBaseR3;
-    pVM->pdm.s.Apic.pfnGetBaseR3        = pApicReg->pfnGetBaseR3;
-    pVM->pdm.s.Apic.pfnSetTPRR3         = pApicReg->pfnSetTPRR3;
-    pVM->pdm.s.Apic.pfnGetTPRR3         = pApicReg->pfnGetTPRR3;
-    pVM->pdm.s.Apic.pfnWriteMSRR3       = pApicReg->pfnWriteMSRR3;
-    pVM->pdm.s.Apic.pfnReadMSRR3        = pApicReg->pfnReadMSRR3;
-    pVM->pdm.s.Apic.pfnBusDeliverR3     = pApicReg->pfnBusDeliverR3;
-    pVM->pdm.s.Apic.pfnLocalInterruptR3 = pApicReg->pfnLocalInterruptR3;
-    Log(("PDM: Registered APIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
-
-    /* set the helper pointer and return. */
-    *ppApicHlpR3 = &g_pdmR3DevApicHlp;
-    LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
-    return VINF_SUCCESS;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnIOAPICRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_IOAPICRegister(PPDMDEVINS pDevIns, PPDMIOAPICREG pIoApicReg, PCPDMIOAPICHLPR3 *ppIoApicHlpR3)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: pIoApicReg=%p:{.u32Version=%#x, .pfnSetIrqR3=%p, .pszSetIrqRC=%p:{%s}, .pszSetIrqR0=%p:{%s}} ppIoApicHlpR3=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg, pIoApicReg->u32Version, pIoApicReg->pfnSetIrqR3,
-             pIoApicReg->pszSetIrqRC, pIoApicReg->pszSetIrqRC, pIoApicReg->pszSetIrqR0, pIoApicReg->pszSetIrqR0, ppIoApicHlpR3));
-
-    /*
-     * Validate input.
-     */
-    if (pIoApicReg->u32Version != PDM_IOAPICREG_VERSION)
-    {
-        AssertMsgFailed(("u32Version=%#x expected %#x\n", pIoApicReg->u32Version, PDM_IOAPICREG_VERSION));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!pIoApicReg->pfnSetIrqR3 || !pIoApicReg->pfnSendMsiR3)
-    {
-        Assert(pIoApicReg->pfnSetIrqR3);
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pIoApicReg->pszSetIrqRC
-        &&  !VALID_PTR(pIoApicReg->pszSetIrqRC))
-    {
-        Assert(VALID_PTR(pIoApicReg->pszSetIrqRC));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pIoApicReg->pszSendMsiRC
-        &&  !VALID_PTR(pIoApicReg->pszSendMsiRC))
-    {
-        Assert(VALID_PTR(pIoApicReg->pszSendMsiRC));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pIoApicReg->pszSetIrqR0
-        &&  !VALID_PTR(pIoApicReg->pszSetIrqR0))
-    {
-        Assert(VALID_PTR(pIoApicReg->pszSetIrqR0));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pIoApicReg->pszSendMsiR0
-        &&  !VALID_PTR(pIoApicReg->pszSendMsiR0))
-    {
-        Assert(VALID_PTR(pIoApicReg->pszSendMsiR0));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!ppIoApicHlpR3)
-    {
-        Assert(ppIoApicHlpR3);
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (ppApicHlp)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * The I/O APIC requires the APIC to be present (hacks++).
-     * If the I/O APIC does GC stuff so must the APIC.
-     */
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    if (!pVM->pdm.s.Apic.pDevInsR3)
-    {
-        AssertMsgFailed(("Configuration error / Init order error! No APIC!\n"));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (no APIC)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    pIoApicReg->pszSetIrqRC
-        &&  !pVM->pdm.s.Apic.pDevInsRC)
-    {
-        AssertMsgFailed(("Configuration error! APIC doesn't do GC, I/O APIC does!\n"));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (no GC APIC)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Only one I/O APIC device.
-     */
-    if (pVM->pdm.s.IoApic.pDevInsR3)
-    {
-        AssertMsgFailed(("Only one ioapic device is supported!\n"));
-        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (only one)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Resolve & initialize the GC bits.
-     */
-    if (pIoApicReg->pszSetIrqRC)
-    {
-        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pIoApicReg->pszSetIrqRC, &pVM->pdm.s.IoApic.pfnSetIrqRC);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pIoApicReg->pszSetIrqRC, rc));
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pVM->pdm.s.IoApic.pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
-    }
-    else
-    {
-        pVM->pdm.s.IoApic.pDevInsRC   = 0;
-        pVM->pdm.s.IoApic.pfnSetIrqRC = 0;
-    }
-
-    if (pIoApicReg->pszSendMsiRC)
-    {
-        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pIoApicReg->pszSetIrqRC, &pVM->pdm.s.IoApic.pfnSendMsiRC);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pIoApicReg->pszSendMsiRC, rc));
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-    }
-    else
-    {
-        pVM->pdm.s.IoApic.pfnSendMsiRC = 0;
-    }
-
-    /*
-     * Resolve & initialize the R0 bits.
-     */
-    if (pIoApicReg->pszSetIrqR0)
-    {
-        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pIoApicReg->pszSetIrqR0, &pVM->pdm.s.IoApic.pfnSetIrqR0);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pIoApicReg->pszSetIrqR0, rc));
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-        pVM->pdm.s.IoApic.pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
-        Assert(pVM->pdm.s.IoApic.pDevInsR0);
-    }
-    else
-    {
-        pVM->pdm.s.IoApic.pfnSetIrqR0 = 0;
-        pVM->pdm.s.IoApic.pDevInsR0   = 0;
-    }
-
-    if (pIoApicReg->pszSendMsiR0)
-    {
-        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pIoApicReg->pszSetIrqR0, &pVM->pdm.s.IoApic.pfnSendMsiR0);
-        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pIoApicReg->pszSendMsiR0, rc));
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            return rc;
-        }
-    }
-    else
-    {
-        pVM->pdm.s.IoApic.pfnSendMsiR0 = 0;
-    }
-
-
-    /*
-     * Initialize the R3 bits.
-     */
-    pVM->pdm.s.IoApic.pDevInsR3   = pDevIns;
-    pVM->pdm.s.IoApic.pfnSetIrqR3 = pIoApicReg->pfnSetIrqR3;
-    pVM->pdm.s.IoApic.pfnSendMsiR3 = pIoApicReg->pfnSendMsiR3;
-    Log(("PDM: Registered I/O APIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
-
-    /* set the helper pointer and return. */
-    *ppIoApicHlpR3 = &g_pdmR3DevIoApicHlp;
-    LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
-    return VINF_SUCCESS;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnHPETRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_HPETRegister(PPDMDEVINS pDevIns, PPDMHPETREG pHpetReg, PCPDMHPETHLPR3 *ppHpetHlpR3)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d:\n"));
-
-    /*
-     * Validate input.
-     */
-    if (pHpetReg->u32Version != PDM_HPETREG_VERSION)
-    {
-        AssertMsgFailed(("u32Version=%#x expected %#x\n", pHpetReg->u32Version, PDM_HPETREG_VERSION));
-        LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (!ppHpetHlpR3)
-    {
-        Assert(ppHpetHlpR3);
-        LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d: returns %Rrc (ppApicHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /* set the helper pointer and return. */
-    *ppHpetHlpR3 = &g_pdmR3DevHpetHlp;
-    LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
-    return VINF_SUCCESS;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnDMACRegister} */
-static DECLCALLBACK(int) pdmR3DevHlp_DMACRegister(PPDMDEVINS pDevIns, PPDMDMACREG pDmacReg, PCPDMDMACHLP *ppDmacHlp)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: pDmacReg=%p:{.u32Version=%#x, .pfnRun=%p, .pfnRegister=%p, .pfnReadMemory=%p, .pfnWriteMemory=%p, .pfnSetDREQ=%p, .pfnGetChannelMode=%p} ppDmacHlp=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pDmacReg, pDmacReg->u32Version, pDmacReg->pfnRun, pDmacReg->pfnRegister,
-             pDmacReg->pfnReadMemory, pDmacReg->pfnWriteMemory, pDmacReg->pfnSetDREQ, pDmacReg->pfnGetChannelMode, ppDmacHlp));
-
-    /*
-     * Validate input.
-     */
-    if (pDmacReg->u32Version != PDM_DMACREG_VERSION)
-    {
-        AssertMsgFailed(("u32Version=%#x expected %#x\n", pDmacReg->u32Version,
-                         PDM_DMACREG_VERSION));
-        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (version)\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (    !pDmacReg->pfnRun
-        ||  !pDmacReg->pfnRegister
-        ||  !pDmacReg->pfnReadMemory
-        ||  !pDmacReg->pfnWriteMemory
-        ||  !pDmacReg->pfnSetDREQ
-        ||  !pDmacReg->pfnGetChannelMode)
-    {
-        Assert(pDmacReg->pfnRun);
-        Assert(pDmacReg->pfnRegister);
-        Assert(pDmacReg->pfnReadMemory);
-        Assert(pDmacReg->pfnWriteMemory);
-        Assert(pDmacReg->pfnSetDREQ);
-        Assert(pDmacReg->pfnGetChannelMode);
-        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (callbacks)\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    if (!ppDmacHlp)
-    {
-        Assert(ppDmacHlp);
-        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (ppDmacHlp)\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Only one DMA device.
-     */
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    if (pVM->pdm.s.pDmac)
-    {
-        AssertMsgFailed(("Only one DMA device is supported!\n"));
-        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Allocate and initialize pci bus structure.
-     */
-    int rc = VINF_SUCCESS;
-    PPDMDMAC  pDmac = (PPDMDMAC)MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE, sizeof(*pDmac));
-    if (pDmac)
-    {
-        pDmac->pDevIns   = pDevIns;
-        pDmac->Reg       = *pDmacReg;
-        pVM->pdm.s.pDmac = pDmac;
-
-        /* set the helper pointer. */
-        *ppDmacHlp = &g_pdmR3DevDmacHlp;
-        Log(("PDM: Registered DMAC device '%s'/%d pDevIns=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
-    }
-    else
-        rc = VERR_NO_MEMORY;
-
-    LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnRegisterVMMDevHeap
- */
-static DECLCALLBACK(int) pdmR3DevHlp_RegisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTR3PTR pvHeap, unsigned cbSize)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-
-    int rc = PDMR3RegisterVMMDevHeap(pDevIns->Internal.s.pVMR3, GCPhys, pvHeap, cbSize);
-    return rc;
-}
-
-
-/**
- * @copydoc PDMDEVHLPR3::pfnUnregisterVMMDevHeap
- */
-static DECLCALLBACK(int) pdmR3DevHlp_UnregisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-
-    int rc = PDMR3UnregisterVMMDevHeap(pDevIns->Internal.s.pVMR3, GCPhys);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMReset} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMReset(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_VMReset: caller='%s'/%d: VM_FF_RESET %d -> 1\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, VM_FF_ISSET(pVM, VM_FF_RESET)));
-
-    /*
-     * We postpone this operation because we're likely to be inside a I/O instruction
-     * and the EIP will be updated when we return.
-     * We still return VINF_EM_RESET to break out of any execution loops and force FF evaluation.
-     */
-    bool fHaltOnReset;
-    int rc = CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "HaltOnReset", &fHaltOnReset);
-    if (RT_SUCCESS(rc) && fHaltOnReset)
-    {
-        Log(("pdmR3DevHlp_VMReset: Halt On Reset!\n"));
-        rc = VINF_EM_HALT;
-    }
-    else
-    {
-        VM_FF_SET(pVM, VM_FF_RESET);
-        rc = VINF_EM_RESET;
-    }
-
-    LogFlow(("pdmR3DevHlp_VMReset: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspend} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMSuspend(PPDMDEVINS pDevIns)
-{
-    int rc;
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_VMSuspend: caller='%s'/%d:\n",
-             pDevIns->pReg->szName, pDevIns->iInstance));
-
-    /** @todo Always take the SMP path - fewer code paths. */
-    if (pVM->cCpus > 1)
-    {
-        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
-        rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)VMR3Suspend, 1, pVM);
-        AssertRC(rc);
-        rc = VINF_EM_SUSPEND;
-    }
-    else
-        rc = VMR3Suspend(pVM);
-
-    LogFlow(("pdmR3DevHlp_VMSuspend: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/**
- * Worker for pdmR3DevHlp_VMSuspendSaveAndPowerOff that is invoked via a queued
- * EMT request to avoid deadlocks.
- *
- * @returns VBox status code fit for scheduling.
- * @param   pVM                 The VM handle.
- * @param   pDevIns             The device that triggered this action.
- */
-static DECLCALLBACK(int) pdmR3DevHlp_VMSuspendSaveAndPowerOffWorker(PVM pVM, PPDMDEVINS pDevIns)
-{
-    /*
-     * Suspend the VM first then do the saving.
-     */
-    int rc = VMR3Suspend(pVM);
-    if (RT_SUCCESS(rc))
-    {
-        rc = pVM->pUVM->pVmm2UserMethods->pfnSaveState(pVM->pUVM->pVmm2UserMethods, pVM);
-
-        /*
-         * On success, power off the VM, on failure we'll leave it suspended.
-         */
-        if (RT_SUCCESS(rc))
-        {
-            rc = VMR3PowerOff(pVM);
-            if (RT_FAILURE(rc))
-                LogRel(("%s/SSP: VMR3PowerOff failed: %Rrc\n", pDevIns->pReg->szName, rc));
-        }
-        else
-            LogRel(("%s/SSP: pfnSaveState failed: %Rrc\n", pDevIns->pReg->szName, rc));
-    }
-    else
-        LogRel(("%s/SSP: Suspend failed: %Rrc\n", pDevIns->pReg->szName, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspendSaveAndPowerOff} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMSuspendSaveAndPowerOff(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_VMSuspendSaveAndPowerOff: caller='%s'/%d:\n",
-             pDevIns->pReg->szName, pDevIns->iInstance));
-
-    int rc;
-    if (   pVM->pUVM->pVmm2UserMethods
-        && pVM->pUVM->pVmm2UserMethods->pfnSaveState)
-    {
-        rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)pdmR3DevHlp_VMSuspendSaveAndPowerOffWorker, 2, pVM, pDevIns);
-        if (RT_SUCCESS(rc))
-        {
-            LogRel(("%s: Suspending, Saving and Powering Off the VM\n", pDevIns->pReg->szName));
-            rc = VINF_EM_SUSPEND;
-        }
-    }
-    else
-        rc = VERR_NOT_SUPPORTED;
-
-    LogFlow(("pdmR3DevHlp_VMSuspendSaveAndPowerOff: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMPowerOff} */
-static DECLCALLBACK(int) pdmR3DevHlp_VMPowerOff(PPDMDEVINS pDevIns)
-{
-    int rc;
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DevHlp_VMPowerOff: caller='%s'/%d:\n",
-             pDevIns->pReg->szName, pDevIns->iInstance));
-
-    /** @todo Always take the SMP path - fewer code paths. */
-    if (pVM->cCpus > 1)
-    {
-        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
-        rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)VMR3PowerOff, 1, pVM);
-        AssertRC(rc);
-        /* Set the VCPU state to stopped here as well to make sure no
-         * inconsistency with the EM state occurs.
-         */
-        VMCPU_SET_STATE(VMMGetCpu(pVM), VMCPUSTATE_STOPPED);
-        rc = VINF_EM_OFF;
-    }
-    else
-        rc = VMR3PowerOff(pVM);
-
-    LogFlow(("pdmR3DevHlp_VMPowerOff: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnA20IsEnabled} */
-static DECLCALLBACK(bool) pdmR3DevHlp_A20IsEnabled(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-
-    bool fRc = PGMPhysIsA20Enabled(VMMGetCpu(pDevIns->Internal.s.pVMR3));
-
-    LogFlow(("pdmR3DevHlp_A20IsEnabled: caller='%s'/%d: returns %d\n", pDevIns->pReg->szName, pDevIns->iInstance, fRc));
-    return fRc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnA20Set} */
-static DECLCALLBACK(void) pdmR3DevHlp_A20Set(PPDMDEVINS pDevIns, bool fEnable)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DevHlp_A20Set: caller='%s'/%d: fEnable=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, fEnable));
-    PGMR3PhysSetA20(VMMGetCpu(pDevIns->Internal.s.pVMR3), fEnable);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnGetCpuId} */
-static DECLCALLBACK(void) pdmR3DevHlp_GetCpuId(PPDMDEVINS pDevIns, uint32_t iLeaf,
-                                               uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-
-    LogFlow(("pdmR3DevHlp_GetCpuId: caller='%s'/%d: iLeaf=%d pEax=%p pEbx=%p pEcx=%p pEdx=%p\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, iLeaf, pEax, pEbx, pEcx, pEdx));
-    AssertPtr(pEax); AssertPtr(pEbx); AssertPtr(pEcx); AssertPtr(pEdx);
-
-    CPUMGetGuestCpuId(VMMGetCpu(pDevIns->Internal.s.pVMR3), iLeaf, pEax, pEbx, pEcx, pEdx);
-
-    LogFlow(("pdmR3DevHlp_GetCpuId: caller='%s'/%d: returns void - *pEax=%#x *pEbx=%#x *pEcx=%#x *pEdx=%#x\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, *pEax, *pEbx, *pEcx, *pEdx));
-}
-
-
-/**
- * The device helper structure for trusted devices.
- */
-const PDMDEVHLPR3 g_pdmR3DevHlpTrusted =
-{
-    PDM_DEVHLPR3_VERSION,
-    pdmR3DevHlp_IOPortRegister,
-    pdmR3DevHlp_IOPortRegisterRC,
-    pdmR3DevHlp_IOPortRegisterR0,
-    pdmR3DevHlp_IOPortDeregister,
-    pdmR3DevHlp_MMIORegister,
-    pdmR3DevHlp_MMIORegisterRC,
-    pdmR3DevHlp_MMIORegisterR0,
-    pdmR3DevHlp_MMIODeregister,
-    pdmR3DevHlp_MMIO2Register,
-    pdmR3DevHlp_MMIO2Deregister,
-    pdmR3DevHlp_MMIO2Map,
-    pdmR3DevHlp_MMIO2Unmap,
-    pdmR3DevHlp_MMHyperMapMMIO2,
-    pdmR3DevHlp_MMIO2MapKernel,
-    pdmR3DevHlp_ROMRegister,
-    pdmR3DevHlp_ROMProtectShadow,
-    pdmR3DevHlp_SSMRegister,
-    pdmR3DevHlp_TMTimerCreate,
-    pdmR3DevHlp_TMUtcNow,
-    pdmR3DevHlp_PhysRead,
-    pdmR3DevHlp_PhysWrite,
-    pdmR3DevHlp_PhysGCPhys2CCPtr,
-    pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly,
-    pdmR3DevHlp_PhysReleasePageMappingLock,
-    pdmR3DevHlp_PhysReadGCVirt,
-    pdmR3DevHlp_PhysWriteGCVirt,
-    pdmR3DevHlp_PhysGCPtr2GCPhys,
-    pdmR3DevHlp_MMHeapAlloc,
-    pdmR3DevHlp_MMHeapAllocZ,
-    pdmR3DevHlp_MMHeapFree,
-    pdmR3DevHlp_VMState,
-    pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet,
-    pdmR3DevHlp_VMSetError,
-    pdmR3DevHlp_VMSetErrorV,
-    pdmR3DevHlp_VMSetRuntimeError,
-    pdmR3DevHlp_VMSetRuntimeErrorV,
-    pdmR3DevHlp_DBGFStopV,
-    pdmR3DevHlp_DBGFInfoRegister,
-    pdmR3DevHlp_STAMRegister,
-    pdmR3DevHlp_STAMRegisterF,
-    pdmR3DevHlp_STAMRegisterV,
-    pdmR3DevHlp_PCIRegister,
-    pdmR3DevHlp_PCIRegisterMsi,
-    pdmR3DevHlp_PCIIORegionRegister,
-    pdmR3DevHlp_PCISetConfigCallbacks,
-    pdmR3DevHlp_PCISetIrq,
-    pdmR3DevHlp_PCISetIrqNoWait,
-    pdmR3DevHlp_ISASetIrq,
-    pdmR3DevHlp_ISASetIrqNoWait,
-    pdmR3DevHlp_DriverAttach,
-    pdmR3DevHlp_QueueCreate,
-    pdmR3DevHlp_CritSectInit,
-    pdmR3DevHlp_ThreadCreate,
-    pdmR3DevHlp_SetAsyncNotification,
-    pdmR3DevHlp_AsyncNotificationCompleted,
-    pdmR3DevHlp_RTCRegister,
-    pdmR3DevHlp_PCIBusRegister,
-    pdmR3DevHlp_PICRegister,
-    pdmR3DevHlp_APICRegister,
-    pdmR3DevHlp_IOAPICRegister,
-    pdmR3DevHlp_HPETRegister,
-    pdmR3DevHlp_DMACRegister,
-    pdmR3DevHlp_DMARegister,
-    pdmR3DevHlp_DMAReadMemory,
-    pdmR3DevHlp_DMAWriteMemory,
-    pdmR3DevHlp_DMASetDREQ,
-    pdmR3DevHlp_DMAGetChannelMode,
-    pdmR3DevHlp_DMASchedule,
-    pdmR3DevHlp_CMOSWrite,
-    pdmR3DevHlp_CMOSRead,
-    pdmR3DevHlp_AssertEMT,
-    pdmR3DevHlp_AssertOther,
-    pdmR3DevHlp_LdrGetRCInterfaceSymbols,
-    pdmR3DevHlp_LdrGetR0InterfaceSymbols,
-    pdmR3DevHlp_CallR0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    pdmR3DevHlp_GetVM,
-    pdmR3DevHlp_GetVMCPU,
-    pdmR3DevHlp_RegisterVMMDevHeap,
-    pdmR3DevHlp_UnregisterVMMDevHeap,
-    pdmR3DevHlp_VMReset,
-    pdmR3DevHlp_VMSuspend,
-    pdmR3DevHlp_VMSuspendSaveAndPowerOff,
-    pdmR3DevHlp_VMPowerOff,
-    pdmR3DevHlp_A20IsEnabled,
-    pdmR3DevHlp_A20Set,
-    pdmR3DevHlp_GetCpuId,
-    pdmR3DevHlp_TMTimeVirtGet,
-    pdmR3DevHlp_TMTimeVirtGetFreq,
-    pdmR3DevHlp_TMTimeVirtGetNano,
-    PDM_DEVHLPR3_VERSION /* the end */
-};
-
-
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnGetVM} */
-static DECLCALLBACK(PVM) pdmR3DevHlp_Untrusted_GetVM(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return NULL;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnGetVMCPU} */
-static DECLCALLBACK(PVMCPU) pdmR3DevHlp_Untrusted_GetVMCPU(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return NULL;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnRegisterVMMDevHeap} */
-static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_RegisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTR3PTR pvHeap, unsigned cbSize)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return VERR_ACCESS_DENIED;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnUnregisterVMMDevHeap} */
-static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_UnregisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return VERR_ACCESS_DENIED;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMReset} */
-static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMReset(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return VERR_ACCESS_DENIED;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspend} */
-static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMSuspend(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return VERR_ACCESS_DENIED;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspendSaveAndPowerOff} */
-static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMSuspendSaveAndPowerOff(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return VERR_ACCESS_DENIED;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMPowerOff} */
-static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMPowerOff(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return VERR_ACCESS_DENIED;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnA20IsEnabled} */
-static DECLCALLBACK(bool) pdmR3DevHlp_Untrusted_A20IsEnabled(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return false;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnA20Set} */
-static DECLCALLBACK(void) pdmR3DevHlp_Untrusted_A20Set(PPDMDEVINS pDevIns, bool fEnable)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    NOREF(fEnable);
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnGetCpuId} */
-static DECLCALLBACK(void) pdmR3DevHlp_Untrusted_GetCpuId(PPDMDEVINS pDevIns, uint32_t iLeaf,
-                                                         uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-}
-
-
-/**
- * The device helper structure for non-trusted devices.
- */
-const PDMDEVHLPR3 g_pdmR3DevHlpUnTrusted =
-{
-    PDM_DEVHLPR3_VERSION,
-    pdmR3DevHlp_IOPortRegister,
-    pdmR3DevHlp_IOPortRegisterRC,
-    pdmR3DevHlp_IOPortRegisterR0,
-    pdmR3DevHlp_IOPortDeregister,
-    pdmR3DevHlp_MMIORegister,
-    pdmR3DevHlp_MMIORegisterRC,
-    pdmR3DevHlp_MMIORegisterR0,
-    pdmR3DevHlp_MMIODeregister,
-    pdmR3DevHlp_MMIO2Register,
-    pdmR3DevHlp_MMIO2Deregister,
-    pdmR3DevHlp_MMIO2Map,
-    pdmR3DevHlp_MMIO2Unmap,
-    pdmR3DevHlp_MMHyperMapMMIO2,
-    pdmR3DevHlp_MMIO2MapKernel,
-    pdmR3DevHlp_ROMRegister,
-    pdmR3DevHlp_ROMProtectShadow,
-    pdmR3DevHlp_SSMRegister,
-    pdmR3DevHlp_TMTimerCreate,
-    pdmR3DevHlp_TMUtcNow,
-    pdmR3DevHlp_PhysRead,
-    pdmR3DevHlp_PhysWrite,
-    pdmR3DevHlp_PhysGCPhys2CCPtr,
-    pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly,
-    pdmR3DevHlp_PhysReleasePageMappingLock,
-    pdmR3DevHlp_PhysReadGCVirt,
-    pdmR3DevHlp_PhysWriteGCVirt,
-    pdmR3DevHlp_PhysGCPtr2GCPhys,
-    pdmR3DevHlp_MMHeapAlloc,
-    pdmR3DevHlp_MMHeapAllocZ,
-    pdmR3DevHlp_MMHeapFree,
-    pdmR3DevHlp_VMState,
-    pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet,
-    pdmR3DevHlp_VMSetError,
-    pdmR3DevHlp_VMSetErrorV,
-    pdmR3DevHlp_VMSetRuntimeError,
-    pdmR3DevHlp_VMSetRuntimeErrorV,
-    pdmR3DevHlp_DBGFStopV,
-    pdmR3DevHlp_DBGFInfoRegister,
-    pdmR3DevHlp_STAMRegister,
-    pdmR3DevHlp_STAMRegisterF,
-    pdmR3DevHlp_STAMRegisterV,
-    pdmR3DevHlp_PCIRegister,
-    pdmR3DevHlp_PCIRegisterMsi,
-    pdmR3DevHlp_PCIIORegionRegister,
-    pdmR3DevHlp_PCISetConfigCallbacks,
-    pdmR3DevHlp_PCISetIrq,
-    pdmR3DevHlp_PCISetIrqNoWait,
-    pdmR3DevHlp_ISASetIrq,
-    pdmR3DevHlp_ISASetIrqNoWait,
-    pdmR3DevHlp_DriverAttach,
-    pdmR3DevHlp_QueueCreate,
-    pdmR3DevHlp_CritSectInit,
-    pdmR3DevHlp_ThreadCreate,
-    pdmR3DevHlp_SetAsyncNotification,
-    pdmR3DevHlp_AsyncNotificationCompleted,
-    pdmR3DevHlp_RTCRegister,
-    pdmR3DevHlp_PCIBusRegister,
-    pdmR3DevHlp_PICRegister,
-    pdmR3DevHlp_APICRegister,
-    pdmR3DevHlp_IOAPICRegister,
-    pdmR3DevHlp_HPETRegister,
-    pdmR3DevHlp_DMACRegister,
-    pdmR3DevHlp_DMARegister,
-    pdmR3DevHlp_DMAReadMemory,
-    pdmR3DevHlp_DMAWriteMemory,
-    pdmR3DevHlp_DMASetDREQ,
-    pdmR3DevHlp_DMAGetChannelMode,
-    pdmR3DevHlp_DMASchedule,
-    pdmR3DevHlp_CMOSWrite,
-    pdmR3DevHlp_CMOSRead,
-    pdmR3DevHlp_AssertEMT,
-    pdmR3DevHlp_AssertOther,
-    pdmR3DevHlp_LdrGetRCInterfaceSymbols,
-    pdmR3DevHlp_LdrGetR0InterfaceSymbols,
-    pdmR3DevHlp_CallR0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    pdmR3DevHlp_Untrusted_GetVM,
-    pdmR3DevHlp_Untrusted_GetVMCPU,
-    pdmR3DevHlp_Untrusted_RegisterVMMDevHeap,
-    pdmR3DevHlp_Untrusted_UnregisterVMMDevHeap,
-    pdmR3DevHlp_Untrusted_VMReset,
-    pdmR3DevHlp_Untrusted_VMSuspend,
-    pdmR3DevHlp_Untrusted_VMSuspendSaveAndPowerOff,
-    pdmR3DevHlp_Untrusted_VMPowerOff,
-    pdmR3DevHlp_Untrusted_A20IsEnabled,
-    pdmR3DevHlp_Untrusted_A20Set,
-    pdmR3DevHlp_Untrusted_GetCpuId,
-    pdmR3DevHlp_TMTimeVirtGet,
-    pdmR3DevHlp_TMTimeVirtGetFreq,
-    pdmR3DevHlp_TMTimeVirtGetNano,
-    PDM_DEVHLPR3_VERSION /* the end */
-};
-
-
-
-/**
- * Queue consumer callback for internal component.
- *
- * @returns Success indicator.
- *          If false the item will not be removed and the flushing will stop.
- * @param   pVM         The VM handle.
- * @param   pItem       The item to consume. Upon return this item will be freed.
- */
-DECLCALLBACK(bool) pdmR3DevHlpQueueConsumer(PVM pVM, PPDMQUEUEITEMCORE pItem)
-{
-    PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)pItem;
-    LogFlow(("pdmR3DevHlpQueueConsumer: enmOp=%d pDevIns=%p\n", pTask->enmOp, pTask->pDevInsR3));
-    switch (pTask->enmOp)
-    {
-        case PDMDEVHLPTASKOP_ISA_SET_IRQ:
-            PDMIsaSetIrq(pVM, pTask->u.SetIRQ.iIrq, pTask->u.SetIRQ.iLevel);
-            break;
-
-        case PDMDEVHLPTASKOP_PCI_SET_IRQ:
-            pdmR3DevHlp_PCISetIrq(pTask->pDevInsR3, pTask->u.SetIRQ.iIrq, pTask->u.SetIRQ.iLevel);
-            break;
-
-        case PDMDEVHLPTASKOP_IOAPIC_SET_IRQ:
-            PDMIoApicSetIrq(pVM, pTask->u.SetIRQ.iIrq, pTask->u.SetIRQ.iLevel);
-            break;
-
-        default:
-            AssertReleaseMsgFailed(("Invalid operation %d\n", pTask->enmOp));
-            break;
-    }
-    return true;
-}
-
-/** @} */
-
Index: unk/src/VBox/VMM/PDMDevMiscHlp.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDevMiscHlp.cpp	(revision 35332)
+++ 	(revision )
@@ -1,676 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device and Driver Manager, Misc. Device Helpers.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_DEVICE
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/rem.h>
-#include <VBox/vm.h>
-#include <VBox/vmm.h>
-
-#include <VBox/log.h>
-#include <VBox/err.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-
-
-
-/** @name Ring-3 PIC Helpers
- * @{
- */
-
-/** @interface_method_impl{PDMPICHLPR3,pfnSetInterruptFF} */
-static DECLCALLBACK(void) pdmR3PicHlp_SetInterruptFF(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-
-    if (pVM->pdm.s.Apic.pfnLocalInterruptR3)
-    {
-        LogFlow(("pdmR3PicHlp_SetInterruptFF: caller='%s'/%d: Setting local interrupt on LAPIC\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance));
-        /* Raise the LAPIC's LINT0 line instead of signaling the CPU directly. */
-        pVM->pdm.s.Apic.pfnLocalInterruptR3(pVM->pdm.s.Apic.pDevInsR3, 0, 1);
-        return;
-    }
-
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* for PIC we always deliver to CPU 0, MP use APIC */
-
-    LogFlow(("pdmR3PicHlp_SetInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT_PIC %d -> 1\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC)));
-
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);
-    REMR3NotifyInterruptSet(pVM, pVCpu);
-    VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE);
-}
-
-
-/** @interface_method_impl{PDMPICHLPR3,pfnClearInterruptFF} */
-static DECLCALLBACK(void) pdmR3PicHlp_ClearInterruptFF(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* for PIC we always deliver to CPU 0, MP use APIC */
-
-    if (pVM->pdm.s.Apic.pfnLocalInterruptR3)
-    {
-        /* Raise the LAPIC's LINT0 line instead of signaling the CPU directly. */
-        LogFlow(("pdmR3PicHlp_ClearInterruptFF: caller='%s'/%d: Clearing local interrupt on LAPIC\n",
-                 pDevIns->pReg->szName, pDevIns->iInstance));
-        /* Lower the LAPIC's LINT0 line instead of signaling the CPU directly. */
-        pVM->pdm.s.Apic.pfnLocalInterruptR3(pVM->pdm.s.Apic.pDevInsR3, 0, 0);
-        return;
-    }
-
-    LogFlow(("pdmR3PicHlp_ClearInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT_PIC %d -> 0\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC)));
-
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
-    REMR3NotifyInterruptClear(pVM, pVCpu);
-}
-
-
-/** @interface_method_impl{PDMPICHLPR3,pfnLock} */
-static DECLCALLBACK(int) pdmR3PicHlp_Lock(PPDMDEVINS pDevIns, int rc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
-}
-
-
-/** @interface_method_impl{PDMPICHLPR3,pfnUnlock} */
-static DECLCALLBACK(void) pdmR3PicHlp_Unlock(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    pdmUnlock(pDevIns->Internal.s.pVMR3);
-}
-
-
-/** @interface_method_impl{PDMPICHLPR3,pfnGetRCHelpers} */
-static DECLCALLBACK(PCPDMPICHLPRC) pdmR3PicHlp_GetRCHelpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    RTRCPTR pRCHelpers = 0;
-    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCPicHlp", &pRCHelpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pRCHelpers);
-    LogFlow(("pdmR3PicHlp_GetRCHelpers: caller='%s'/%d: returns %RRv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
-    return pRCHelpers;
-}
-
-
-/** @interface_method_impl{PDMPICHLPR3,pfnGetR0Helpers} */
-static DECLCALLBACK(PCPDMPICHLPR0) pdmR3PicHlp_GetR0Helpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    PCPDMPICHLPR0 pR0Helpers = 0;
-    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0PicHlp", &pR0Helpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pR0Helpers);
-    LogFlow(("pdmR3PicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
-    return pR0Helpers;
-}
-
-
-/**
- * PIC Device Helpers.
- */
-const PDMPICHLPR3 g_pdmR3DevPicHlp =
-{
-    PDM_PICHLPR3_VERSION,
-    pdmR3PicHlp_SetInterruptFF,
-    pdmR3PicHlp_ClearInterruptFF,
-    pdmR3PicHlp_Lock,
-    pdmR3PicHlp_Unlock,
-    pdmR3PicHlp_GetRCHelpers,
-    pdmR3PicHlp_GetR0Helpers,
-    PDM_PICHLPR3_VERSION /* the end */
-};
-
-/** @} */
-
-
-
-
-/** @name R3 APIC Helpers
- * @{
- */
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnSetInterruptFF} */
-static DECLCALLBACK(void) pdmR3ApicHlp_SetInterruptFF(PPDMDEVINS pDevIns, PDMAPICIRQ enmType, VMCPUID idCpu)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    PVMCPU pVCpu = &pVM->aCpus[idCpu];
-
-    AssertReturnVoid(idCpu < pVM->cCpus);
-
-    LogFlow(("pdmR3ApicHlp_SetInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT(%d) %d -> 1\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, idCpu, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC)));
-
-    switch (enmType)
-    {
-        case PDMAPICIRQ_HARDWARE:
-            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC);
-            break;
-        case PDMAPICIRQ_NMI:
-            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
-            break;
-        case PDMAPICIRQ_SMI:
-            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI);
-            break;
-        case PDMAPICIRQ_EXTINT:
-            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);
-            break;
-        default:
-            AssertMsgFailed(("enmType=%d\n", enmType));
-            break;
-    }
-    REMR3NotifyInterruptSet(pVM, pVCpu);
-    VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE);
-}
-
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnClearInterruptFF} */
-static DECLCALLBACK(void) pdmR3ApicHlp_ClearInterruptFF(PPDMDEVINS pDevIns, PDMAPICIRQ enmType, VMCPUID idCpu)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    PVMCPU pVCpu = &pVM->aCpus[idCpu];
-
-    AssertReturnVoid(idCpu < pVM->cCpus);
-
-    LogFlow(("pdmR3ApicHlp_ClearInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT(%d) %d -> 0\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, idCpu, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC)));
-
-    /* Note: NMI/SMI can't be cleared. */
-    switch (enmType)
-    {
-        case PDMAPICIRQ_HARDWARE:
-            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
-            break;
-        case PDMAPICIRQ_EXTINT:
-            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
-            break;
-        default:
-            AssertMsgFailed(("enmType=%d\n", enmType));
-            break;
-    }
-    REMR3NotifyInterruptClear(pVM, pVCpu);
-}
-
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnChangeFeature} */
-static DECLCALLBACK(void) pdmR3ApicHlp_ChangeFeature(PPDMDEVINS pDevIns, PDMAPICVERSION enmVersion)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3ApicHlp_ChangeFeature: caller='%s'/%d: version=%d\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, (int)enmVersion));
-    switch (enmVersion)
-    {
-        case PDMAPICVERSION_NONE:
-            CPUMClearGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_APIC);
-            CPUMClearGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_X2APIC);
-            break;
-        case PDMAPICVERSION_APIC:
-            CPUMSetGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_APIC);
-            CPUMClearGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_X2APIC);
-            break;
-        case PDMAPICVERSION_X2APIC:
-            CPUMSetGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_X2APIC);
-            CPUMSetGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_APIC);
-            break;
-        default:
-            AssertMsgFailed(("Unknown APIC version: %d\n", (int)enmVersion));
-    }
-}
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnGetCpuId} */
-static DECLCALLBACK(VMCPUID) pdmR3ApicHlp_GetCpuId(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    return VMMGetCpuId(pDevIns->Internal.s.pVMR3);
-}
-
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnSendSipi} */
-static DECLCALLBACK(void) pdmR3ApicHlp_SendSipi(PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t uVector)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    VMMR3SendSipi(pDevIns->Internal.s.pVMR3, idCpu, uVector);
-}
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnSendInitIpi} */
-static DECLCALLBACK(void) pdmR3ApicHlp_SendInitIpi(PPDMDEVINS pDevIns, VMCPUID idCpu)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    VMMR3SendInitIpi(pDevIns->Internal.s.pVMR3, idCpu);
-}
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnGetRCHelpers} */
-static DECLCALLBACK(PCPDMAPICHLPRC) pdmR3ApicHlp_GetRCHelpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    RTRCPTR pRCHelpers = 0;
-    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCApicHlp", &pRCHelpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pRCHelpers);
-    LogFlow(("pdmR3ApicHlp_GetRCHelpers: caller='%s'/%d: returns %RRv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
-    return pRCHelpers;
-}
-
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnGetR0Helpers} */
-static DECLCALLBACK(PCPDMAPICHLPR0) pdmR3ApicHlp_GetR0Helpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    PCPDMAPICHLPR0 pR0Helpers = 0;
-    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0ApicHlp", &pR0Helpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pR0Helpers);
-    LogFlow(("pdmR3ApicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
-    return pR0Helpers;
-}
-
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnGetR3CritSect} */
-static DECLCALLBACK(R3PTRTYPE(PPDMCRITSECT)) pdmR3ApicHlp_GetR3CritSect(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3ApicHlp_Lock: caller='%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    return &pDevIns->Internal.s.pVMR3->pdm.s.CritSect;
-}
-
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnGetRCCritSect} */
-static DECLCALLBACK(RCPTRTYPE(PPDMCRITSECT)) pdmR3ApicHlp_GetRCCritSect(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    RTRCPTR RCPtr = MMHyperCCToRC(pVM, &pVM->pdm.s.CritSect);
-    LogFlow(("pdmR3ApicHlp_GetR0CritSect: caller='%s'/%d: return %RRv\n", pDevIns->pReg->szName, pDevIns->iInstance, RCPtr));
-    return RCPtr;
-}
-
-
-/** @interface_method_impl{PDMAPICHLPR3,pfnGetR3CritSect} */
-static DECLCALLBACK(R0PTRTYPE(PPDMCRITSECT)) pdmR3ApicHlp_GetR0CritSect(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    RTR0PTR R0Ptr = MMHyperCCToR0(pVM, &pVM->pdm.s.CritSect);
-    LogFlow(("pdmR3ApicHlp_GetR0CritSect: caller='%s'/%d: return %RHv\n", pDevIns->pReg->szName, pDevIns->iInstance, R0Ptr));
-    return R0Ptr;
-}
-
-
-
-/**
- * APIC Device Helpers.
- */
-const PDMAPICHLPR3 g_pdmR3DevApicHlp =
-{
-    PDM_APICHLPR3_VERSION,
-    pdmR3ApicHlp_SetInterruptFF,
-    pdmR3ApicHlp_ClearInterruptFF,
-    pdmR3ApicHlp_ChangeFeature,
-    pdmR3ApicHlp_GetCpuId,
-    pdmR3ApicHlp_SendSipi,
-    pdmR3ApicHlp_SendInitIpi,
-    pdmR3ApicHlp_GetRCHelpers,
-    pdmR3ApicHlp_GetR0Helpers,
-    pdmR3ApicHlp_GetR3CritSect,
-    pdmR3ApicHlp_GetRCCritSect,
-    pdmR3ApicHlp_GetR0CritSect,
-    PDM_APICHLPR3_VERSION /* the end */
-};
-
-/** @} */
-
-
-
-
-/** @name Ring-3 I/O APIC Helpers
- * @{
- */
-
-/** @interface_method_impl{PDMIOAPICHLPR3,pfnApicBusDeliver} */
-static DECLCALLBACK(int) pdmR3IoApicHlp_ApicBusDeliver(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
-                                                        uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    PVM pVM = pDevIns->Internal.s.pVMR3;
-    LogFlow(("pdmR3IoApicHlp_ApicBusDeliver: caller='%s'/%d: u8Dest=%RX8 u8DestMode=%RX8 u8DeliveryMode=%RX8 iVector=%RX8 u8Polarity=%RX8 u8TriggerMode=%RX8\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, u8Dest, u8DestMode, u8DeliveryMode, iVector, u8Polarity, u8TriggerMode));
-    if (pVM->pdm.s.Apic.pfnBusDeliverR3)
-        return pVM->pdm.s.Apic.pfnBusDeliverR3(pVM->pdm.s.Apic.pDevInsR3, u8Dest, u8DestMode, u8DeliveryMode, iVector, u8Polarity, u8TriggerMode);
-    return VINF_SUCCESS;
-}
-
-
-/** @interface_method_impl{PDMIOAPICHLPR3,pfnLock} */
-static DECLCALLBACK(int) pdmR3IoApicHlp_Lock(PPDMDEVINS pDevIns, int rc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3IoApicHlp_Lock: caller='%s'/%d: rc=%Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
-}
-
-
-/** @interface_method_impl{PDMIOAPICHLPR3,pfnUnlock} */
-static DECLCALLBACK(void) pdmR3IoApicHlp_Unlock(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3IoApicHlp_Unlock: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    pdmUnlock(pDevIns->Internal.s.pVMR3);
-}
-
-
-/** @interface_method_impl{PDMIOAPICHLPR3,pfnGetRCHelpers} */
-static DECLCALLBACK(PCPDMIOAPICHLPRC) pdmR3IoApicHlp_GetRCHelpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    RTRCPTR pRCHelpers = 0;
-    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCIoApicHlp", &pRCHelpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pRCHelpers);
-    LogFlow(("pdmR3IoApicHlp_GetRCHelpers: caller='%s'/%d: returns %RRv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
-    return pRCHelpers;
-}
-
-
-/** @interface_method_impl{PDMIOAPICHLPR3,pfnGetR0Helpers} */
-static DECLCALLBACK(PCPDMIOAPICHLPR0) pdmR3IoApicHlp_GetR0Helpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    PCPDMIOAPICHLPR0 pR0Helpers = 0;
-    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0IoApicHlp", &pR0Helpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pR0Helpers);
-    LogFlow(("pdmR3IoApicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
-    return pR0Helpers;
-}
-
-
-/**
- * I/O APIC Device Helpers.
- */
-const PDMIOAPICHLPR3 g_pdmR3DevIoApicHlp =
-{
-    PDM_IOAPICHLPR3_VERSION,
-    pdmR3IoApicHlp_ApicBusDeliver,
-    pdmR3IoApicHlp_Lock,
-    pdmR3IoApicHlp_Unlock,
-    pdmR3IoApicHlp_GetRCHelpers,
-    pdmR3IoApicHlp_GetR0Helpers,
-    PDM_IOAPICHLPR3_VERSION /* the end */
-};
-
-/** @} */
-
-
-
-
-/** @name Ring-3 PCI Bus Helpers
- * @{
- */
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnIsaSetIrq} */
-static DECLCALLBACK(void) pdmR3PciHlp_IsaSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    Log4(("pdmR3PciHlp_IsaSetIrq: iIrq=%d iLevel=%d\n", iIrq, iLevel));
-    PDMIsaSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel);
-}
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnIoApicSetIrq} */
-static DECLCALLBACK(void) pdmR3PciHlp_IoApicSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    Log4(("pdmR3PciHlp_IoApicSetIrq: iIrq=%d iLevel=%d\n", iIrq, iLevel));
-    PDMIoApicSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel);
-}
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnIoApicSendMsi} */
-static DECLCALLBACK(void) pdmR3PciHlp_IoApicSendMsi(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    Log4(("pdmR3PciHlp_IoApicSendMsi: address=%p value=%x\n", GCAddr, uValue));
-    PDMIoApicSendMsi(pDevIns->Internal.s.pVMR3, GCAddr, uValue);
-}
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnIsMMIO2Base} */
-static DECLCALLBACK(bool) pdmR3PciHlp_IsMMIO2Base(PPDMDEVINS pDevIns, PPDMDEVINS pOwner, RTGCPHYS GCPhys)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    bool fRc = PGMR3PhysMMIO2IsBase(pDevIns->Internal.s.pVMR3, pOwner, GCPhys);
-    Log4(("pdmR3PciHlp_IsMMIO2Base: pOwner=%p GCPhys=%RGp -> %RTbool\n", pOwner, GCPhys, fRc));
-    return fRc;
-}
-
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnLock} */
-static DECLCALLBACK(int) pdmR3PciHlp_Lock(PPDMDEVINS pDevIns, int rc)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3PciHlp_Lock: caller='%s'/%d: rc=%Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
-}
-
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnUnlock} */
-static DECLCALLBACK(void) pdmR3PciHlp_Unlock(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3PciHlp_Unlock: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
-    pdmUnlock(pDevIns->Internal.s.pVMR3);
-}
-
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnGetRCHelpers} */
-static DECLCALLBACK(PCPDMPCIHLPRC) pdmR3PciHlp_GetRCHelpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    RTRCPTR pRCHelpers = 0;
-    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCPciHlp", &pRCHelpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pRCHelpers);
-    LogFlow(("pdmR3IoApicHlp_GetGCHelpers: caller='%s'/%d: returns %RRv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
-    return pRCHelpers;
-}
-
-
-/** @interface_method_impl{PDMPCIHLPR3,pfnGetR0Helpers} */
-static DECLCALLBACK(PCPDMPCIHLPR0) pdmR3PciHlp_GetR0Helpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    PCPDMPCIHLPR0 pR0Helpers = 0;
-    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0PciHlp", &pR0Helpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pR0Helpers);
-    LogFlow(("pdmR3IoApicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
-    return pR0Helpers;
-}
-
-
-/**
- * PCI Bus Device Helpers.
- */
-const PDMPCIHLPR3 g_pdmR3DevPciHlp =
-{
-    PDM_PCIHLPR3_VERSION,
-    pdmR3PciHlp_IsaSetIrq,
-    pdmR3PciHlp_IoApicSetIrq,
-    pdmR3PciHlp_IoApicSendMsi,
-    pdmR3PciHlp_IsMMIO2Base,
-    pdmR3PciHlp_GetRCHelpers,
-    pdmR3PciHlp_GetR0Helpers,
-    pdmR3PciHlp_Lock,
-    pdmR3PciHlp_Unlock,
-    PDM_PCIHLPR3_VERSION, /* the end */
-};
-
-/** @} */
-
-
-
-
-/** @name Ring-3 HPET Helpers
- * {@
- */
-
-/** @interface_method_impl{PDMHPETHLPR3,pfnSetLegacyMode} */
-static DECLCALLBACK(int) pdmR3HpetHlp_SetLegacyMode(PPDMDEVINS pDevIns, bool fActivated)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3HpetHlp_SetLegacyMode: caller='%s'/%d: fActivated=%RTbool\n", pDevIns->pReg->szName, pDevIns->iInstance, fActivated));
-
-    size_t                      i;
-    int                         rc = VINF_SUCCESS;
-    static const char * const   s_apszDevsToNotify[] =
-    {
-        "i8254",
-        "mc146818"
-    };
-    for (i = 0; i < RT_ELEMENTS(s_apszDevsToNotify); i++)
-    {
-        PPDMIBASE pBase;
-        rc = PDMR3QueryDevice(pDevIns->Internal.s.pVMR3, "i8254", 0, &pBase);
-        if (RT_SUCCESS(rc))
-        {
-            PPDMIHPETLEGACYNOTIFY pPort = PDMIBASE_QUERY_INTERFACE(pBase, PDMIHPETLEGACYNOTIFY);
-            AssertLogRelMsgBreakStmt(pPort, ("%s\n", s_apszDevsToNotify[i]), rc = VERR_INTERNAL_ERROR_3);
-            pPort->pfnModeChanged(pPort, fActivated);
-        }
-        else if (   rc == VERR_PDM_DEVICE_NOT_FOUND
-                 || rc == VERR_PDM_DEVICE_INSTANCE_NOT_FOUND)
-            rc = VINF_SUCCESS; /* the device isn't configured, ignore. */
-        else
-            AssertLogRelMsgFailedBreak(("%s -> %Rrc\n", s_apszDevsToNotify[i], rc));
-    }
-
-    /* Don't bother cleaning up, any failure here will cause a guru meditation. */
-
-    LogFlow(("pdmR3HpetHlp_SetLegacyMode: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMHPETHLPR3,pfnSetIrq} */
-static DECLCALLBACK(int) pdmR3HpetHlp_SetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    LogFlow(("pdmR3HpetHlp_SetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
-    PDMIsaSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel);
-    return 0;
-}
-
-
-/** @interface_method_impl{PDMHPETHLPR3,pfnGetRCHelpers} */
-static DECLCALLBACK(PCPDMHPETHLPRC) pdmR3HpetHlp_GetRCHelpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    RTRCPTR pRCHelpers = 0;
-    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCHpetHlp", &pRCHelpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pRCHelpers);
-    LogFlow(("pdmR3HpetHlp_GetGCHelpers: caller='%s'/%d: returns %RRv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
-    return pRCHelpers;
-}
-
-
-/** @interface_method_impl{PDMHPETHLPR3,pfnGetR0Helpers} */
-static DECLCALLBACK(PCPDMHPETHLPR0) pdmR3HpetHlp_GetR0Helpers(PPDMDEVINS pDevIns)
-{
-    PDMDEV_ASSERT_DEVINS(pDevIns);
-    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
-    PCPDMHPETHLPR0 pR0Helpers = 0;
-    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0HpetHlp", &pR0Helpers);
-    AssertReleaseRC(rc);
-    AssertRelease(pR0Helpers);
-    LogFlow(("pdmR3HpetHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
-             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
-    return pR0Helpers;
-}
-
-
-/**
- * HPET Device Helpers.
- */
-const PDMHPETHLPR3 g_pdmR3DevHpetHlp =
-{
-    PDM_HPETHLPR3_VERSION,
-    pdmR3HpetHlp_GetRCHelpers,
-    pdmR3HpetHlp_GetR0Helpers,
-    pdmR3HpetHlp_SetLegacyMode,
-    pdmR3HpetHlp_SetIrq,
-    PDM_HPETHLPR3_VERSION, /* the end */
-};
-
-/** @} */
-
-
-
-/* none yet */
-
-/**
- * DMAC Device Helpers.
- */
-const PDMDMACHLP g_pdmR3DevDmacHlp =
-{
-    PDM_DMACHLP_VERSION
-};
-
-
-
-
-/* none yet */
-
-/**
- * RTC Device Helpers.
- */
-const PDMRTCHLP g_pdmR3DevRtcHlp =
-{
-    PDM_RTCHLP_VERSION
-};
Index: unk/src/VBox/VMM/PDMDevice.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDevice.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1042 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device and Driver Manager, Device parts.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_DEVICE
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/pgm.h>
-#include <VBox/iom.h>
-#include <VBox/cfgm.h>
-#include <VBox/rem.h>
-#include <VBox/dbgf.h>
-#include <VBox/vm.h>
-#include <VBox/vmm.h>
-
-#include <VBox/version.h>
-#include <VBox/log.h>
-#include <VBox/err.h>
-#include <iprt/alloc.h>
-#include <iprt/alloca.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/path.h>
-#include <iprt/semaphore.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Internal callback structure pointer.
- * The main purpose is to define the extra data we associate
- * with PDMDEVREGCB so we can find the VM instance and so on.
- */
-typedef struct PDMDEVREGCBINT
-{
-    /** The callback structure. */
-    PDMDEVREGCB     Core;
-    /** A bit of padding. */
-    uint32_t        u32[4];
-    /** VM Handle. */
-    PVM             pVM;
-    /** Pointer to the configuration node the registrations should be
-     * associated with.  Can be NULL. */
-    PCFGMNODE       pCfgNode;
-} PDMDEVREGCBINT;
-/** Pointer to a PDMDEVREGCBINT structure. */
-typedef PDMDEVREGCBINT *PPDMDEVREGCBINT;
-/** Pointer to a const PDMDEVREGCBINT structure. */
-typedef const PDMDEVREGCBINT *PCPDMDEVREGCBINT;
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int)    pdmR3DevReg_Register(PPDMDEVREGCB pCallbacks, PCPDMDEVREG pReg);
-static int                  pdmR3DevLoadModules(PVM pVM);
-static int                  pdmR3DevLoad(PVM pVM, PPDMDEVREGCBINT pRegCB, const char *pszFilename, const char *pszName);
-
-
-
-
-/**
- * This function will initialize the devices for this VM instance.
- *
- *
- * First of all this mean loading the builtin device and letting them
- * register themselves. Beyond that any additional device modules are
- * loaded and called for registration.
- *
- * Then the device configuration is enumerated, the instantiation order
- * is determined, and finally they are instantiated.
- *
- * After all devices have been successfully instantiated the primary
- * PCI Bus device is called to emulate the PCI BIOS, i.e. making the
- * resource assignments. If there is no PCI device, this step is of course
- * skipped.
- *
- * Finally the init completion routines of the instantiated devices
- * are called.
- *
- * @returns VBox status code.
- * @param   pVM     VM Handle.
- */
-int pdmR3DevInit(PVM pVM)
-{
-    LogFlow(("pdmR3DevInit:\n"));
-
-    AssertRelease(!(RT_OFFSETOF(PDMDEVINS, achInstanceData) & 15));
-    AssertRelease(sizeof(pVM->pdm.s.pDevInstances->Internal.s) <= sizeof(pVM->pdm.s.pDevInstances->Internal.padding));
-
-    /*
-     * Load device modules.
-     */
-    int rc = pdmR3DevLoadModules(pVM);
-    if (RT_FAILURE(rc))
-        return rc;
-
-#ifdef VBOX_WITH_USB
-    /* ditto for USB Devices. */
-    rc = pdmR3UsbLoadModules(pVM);
-    if (RT_FAILURE(rc))
-        return rc;
-#endif
-
-    /*
-     * Get the RC & R0 devhlps and create the devhlp R3 task queue.
-     */
-    PCPDMDEVHLPRC pHlpRC;
-    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pHlpRC);
-    AssertReleaseRCReturn(rc, rc);
-
-    PCPDMDEVHLPR0 pHlpR0;
-    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "g_pdmR0DevHlp", &pHlpR0);
-    AssertReleaseRCReturn(rc, rc);
-
-    rc = PDMR3QueueCreateInternal(pVM, sizeof(PDMDEVHLPTASK), 8, 0, pdmR3DevHlpQueueConsumer, true, "DevHlp",
-                                  &pVM->pdm.s.pDevHlpQueueR3);
-    AssertRCReturn(rc, rc);
-    pVM->pdm.s.pDevHlpQueueR0 = PDMQueueR0Ptr(pVM->pdm.s.pDevHlpQueueR3);
-    pVM->pdm.s.pDevHlpQueueRC = PDMQueueRCPtr(pVM->pdm.s.pDevHlpQueueR3);
-
-
-    /*
-     *
-     * Enumerate the device instance configurations
-     * and come up with a instantiation order.
-     *
-     */
-    /* Switch to /Devices, which contains the device instantiations. */
-    PCFGMNODE pDevicesNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "Devices");
-
-    /*
-     * Count the device instances.
-     */
-    PCFGMNODE pCur;
-    PCFGMNODE pInstanceNode;
-    unsigned cDevs = 0;
-    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
-        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
-            cDevs++;
-    if (!cDevs)
-    {
-        Log(("PDM: No devices were configured!\n"));
-        return VINF_SUCCESS;
-    }
-    Log2(("PDM: cDevs=%d!\n", cDevs));
-
-    /*
-     * Collect info on each device instance.
-     */
-    struct DEVORDER
-    {
-        /** Configuration node. */
-        PCFGMNODE   pNode;
-        /** Pointer to device. */
-        PPDMDEV     pDev;
-        /** Init order. */
-        uint32_t    u32Order;
-        /** VBox instance number. */
-        uint32_t    iInstance;
-    } *paDevs = (struct DEVORDER *)alloca(sizeof(paDevs[0]) * (cDevs + 1)); /* (One extra for swapping) */
-    Assert(paDevs);
-    unsigned i = 0;
-    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
-    {
-        /* Get the device name. */
-        char szName[sizeof(paDevs[0].pDev->pReg->szName)];
-        rc = CFGMR3GetName(pCur, szName, sizeof(szName));
-        AssertMsgRCReturn(rc, ("Configuration error: device name is too long (or something)! rc=%Rrc\n", rc), rc);
-
-        /* Find the device. */
-        PPDMDEV pDev = pdmR3DevLookup(pVM, szName);
-        AssertMsgReturn(pDev, ("Configuration error: device '%s' not found!\n", szName), VERR_PDM_DEVICE_NOT_FOUND);
-
-        /* Configured priority or use default based on device class? */
-        uint32_t u32Order;
-        rc = CFGMR3QueryU32(pCur, "Priority", &u32Order);
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        {
-            uint32_t u32 = pDev->pReg->fClass;
-            for (u32Order = 1; !(u32 & u32Order); u32Order <<= 1)
-                /* nop */;
-        }
-        else
-            AssertMsgRCReturn(rc, ("Configuration error: reading \"Priority\" for the '%s' device failed rc=%Rrc!\n", szName, rc), rc);
-
-        /* Enumerate the device instances. */
-        uint32_t const iStart = i;
-        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
-        {
-            paDevs[i].pNode = pInstanceNode;
-            paDevs[i].pDev = pDev;
-            paDevs[i].u32Order = u32Order;
-
-            /* Get the instance number. */
-            char szInstance[32];
-            rc = CFGMR3GetName(pInstanceNode, szInstance, sizeof(szInstance));
-            AssertMsgRCReturn(rc, ("Configuration error: instance name is too long (or something)! rc=%Rrc\n", rc), rc);
-            char *pszNext = NULL;
-            rc = RTStrToUInt32Ex(szInstance, &pszNext, 0, &paDevs[i].iInstance);
-            AssertMsgRCReturn(rc, ("Configuration error: RTStrToInt32Ex failed on the instance name '%s'! rc=%Rrc\n", szInstance, rc), rc);
-            AssertMsgReturn(!*pszNext, ("Configuration error: the instance name '%s' isn't all digits. (%s)\n", szInstance, pszNext), VERR_INVALID_PARAMETER);
-
-            /* next instance */
-            i++;
-        }
-
-        /* check the number of instances */
-        if (i - iStart > pDev->pReg->cMaxInstances)
-            AssertLogRelMsgFailedReturn(("Configuration error: Too many instances of %s was configured: %u, max %u\n",
-                                         szName, i - iStart, pDev->pReg->cMaxInstances),
-                                        VERR_PDM_TOO_MANY_DEVICE_INSTANCES);
-    } /* devices */
-    Assert(i == cDevs);
-
-    /*
-     * Sort the device array ascending on u32Order. (bubble)
-     */
-    unsigned c = cDevs - 1;
-    while (c)
-    {
-        unsigned j = 0;
-        for (i = 0; i < c; i++)
-            if (paDevs[i].u32Order > paDevs[i + 1].u32Order)
-            {
-                paDevs[cDevs] = paDevs[i + 1];
-                paDevs[i + 1] = paDevs[i];
-                paDevs[i] = paDevs[cDevs];
-                j = i;
-            }
-        c = j;
-    }
-
-
-    /*
-     *
-     * Instantiate the devices.
-     *
-     */
-    for (i = 0; i < cDevs; i++)
-    {
-        /*
-         * Gather a bit of config.
-         */
-        /* trusted */
-        bool fTrusted;
-        rc = CFGMR3QueryBool(paDevs[i].pNode, "Trusted", &fTrusted);
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-            fTrusted = false;
-        else if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("configuration error: failed to query boolean \"Trusted\", rc=%Rrc\n", rc));
-            return rc;
-        }
-        /* config node */
-        PCFGMNODE pConfigNode = CFGMR3GetChild(paDevs[i].pNode, "Config");
-        if (!pConfigNode)
-        {
-            rc = CFGMR3InsertNode(paDevs[i].pNode, "Config", &pConfigNode);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("Failed to create Config node! rc=%Rrc\n", rc));
-                return rc;
-            }
-        }
-        CFGMR3SetRestrictedRoot(pConfigNode);
-
-        /*
-         * Allocate the device instance.
-         */
-        AssertReturn(paDevs[i].pDev->cInstances < paDevs[i].pDev->pReg->cMaxInstances, VERR_PDM_TOO_MANY_DEVICE_INSTANCES);
-        size_t cb = RT_OFFSETOF(PDMDEVINS, achInstanceData[paDevs[i].pDev->pReg->cbInstance]);
-        cb = RT_ALIGN_Z(cb, 16);
-        PPDMDEVINS pDevIns;
-        if (paDevs[i].pDev->pReg->fFlags & (PDM_DEVREG_FLAGS_RC | PDM_DEVREG_FLAGS_R0))
-            rc = MMR3HyperAllocOnceNoRel(pVM, cb, 0, MM_TAG_PDM_DEVICE, (void **)&pDevIns);
-        else
-            rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_DEVICE, cb, (void **)&pDevIns);
-        if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("Failed to allocate %d bytes of instance data for device '%s'. rc=%Rrc\n",
-                             cb, paDevs[i].pDev->pReg->szName, rc));
-            return rc;
-        }
-
-        /*
-         * Initialize it.
-         */
-        pDevIns->u32Version                     = PDM_DEVINS_VERSION;
-        //pDevIns->Internal.s.pNextR3             = NULL;
-        //pDevIns->Internal.s.pPerDeviceNextR3    = NULL;
-        pDevIns->Internal.s.pDevR3              = paDevs[i].pDev;
-        pDevIns->Internal.s.pVMR3               = pVM;
-        pDevIns->Internal.s.pVMR0               = pVM->pVMR0;
-        pDevIns->Internal.s.pVMRC               = pVM->pVMRC;
-        //pDevIns->Internal.s.pLunsR3             = NULL;
-        pDevIns->Internal.s.pCfgHandle          = paDevs[i].pNode;
-        //pDevIns->Internal.s.pPciDeviceR3        = NULL;
-        //pDevIns->Internal.s.pPciBusR3           = NULL;
-        //pDevIns->Internal.s.pPciDeviceR0        = 0;
-        //pDevIns->Internal.s.pPciBusR0           = 0;
-        //pDevIns->Internal.s.pPciDeviceRC        = 0;
-        //pDevIns->Internal.s.pPciBusRC           = 0;
-        pDevIns->Internal.s.fIntFlags           = PDMDEVINSINT_FLAGS_SUSPENDED;
-        pDevIns->pHlpR3                         = fTrusted ? &g_pdmR3DevHlpTrusted : &g_pdmR3DevHlpUnTrusted;
-        pDevIns->pHlpRC                         = pHlpRC;
-        pDevIns->pHlpR0                         = pHlpR0;
-        pDevIns->pReg                           = paDevs[i].pDev->pReg;
-        pDevIns->pCfg                           = pConfigNode;
-        pDevIns->iInstance                      = paDevs[i].iInstance;
-        pDevIns->pvInstanceDataR3               = &pDevIns->achInstanceData[0];
-        pDevIns->pvInstanceDataRC               = pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC
-                                                ? MMHyperR3ToRC(pVM, pDevIns->pvInstanceDataR3) : NIL_RTRCPTR;
-        pDevIns->pvInstanceDataR0               = pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0
-                                                ? MMHyperR3ToR0(pVM, pDevIns->pvInstanceDataR3) : NIL_RTR0PTR;
-        //pDevIns->pCritSectR3                    = NULL;
-        //pDevIns->pCritSectR0                    = NIL_RTR0PTR;
-        //pDevIns->pCritSectRC                    = NIL_RTRCPTR;
-
-        /*
-         * Link it into all the lists.
-         */
-        /* The global instance FIFO. */
-        PPDMDEVINS pPrev1 = pVM->pdm.s.pDevInstances;
-        if (!pPrev1)
-            pVM->pdm.s.pDevInstances = pDevIns;
-        else
-        {
-            while (pPrev1->Internal.s.pNextR3)
-                pPrev1 = pPrev1->Internal.s.pNextR3;
-            pPrev1->Internal.s.pNextR3 = pDevIns;
-        }
-
-        /* The per device instance FIFO. */
-        PPDMDEVINS pPrev2 = paDevs[i].pDev->pInstances;
-        if (!pPrev2)
-            paDevs[i].pDev->pInstances = pDevIns;
-        else
-        {
-            while (pPrev2->Internal.s.pPerDeviceNextR3)
-                pPrev2 = pPrev2->Internal.s.pPerDeviceNextR3;
-            pPrev2->Internal.s.pPerDeviceNextR3 = pDevIns;
-        }
-
-        /*
-         * Call the constructor.
-         */
-        paDevs[i].pDev->cInstances++;
-        Log(("PDM: Constructing device '%s' instance %d...\n", pDevIns->pReg->szName, pDevIns->iInstance));
-        rc = pDevIns->pReg->pfnConstruct(pDevIns, pDevIns->iInstance, pDevIns->pCfg);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Per-device critsect fun.
-             */
-            if (pDevIns->pCritSectR3)
-            {
-                AssertStmt(PDMCritSectIsInitialized(pDevIns->pCritSectR3), rc = VERR_INTERNAL_ERROR_4);
-                if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
-                {
-                    pDevIns->pCritSectR0 = MMHyperCCToR0(pVM, pDevIns->pCritSectR3);
-                    AssertStmt(pDevIns->pCritSectR0 != NIL_RTR0PTR, rc = VERR_INTERNAL_ERROR_3);
-                }
-                else
-                    AssertStmt(pDevIns->pCritSectR0 == NIL_RTRCPTR, rc = VERR_INTERNAL_ERROR_2);
-
-                if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
-                {
-                    pDevIns->pCritSectRC = MMHyperCCToRC(pVM, pDevIns->pCritSectR3);
-                    AssertStmt(pDevIns->pCritSectRC != NIL_RTRCPTR, rc = VERR_INTERNAL_ERROR_3);
-                }
-                else
-                    AssertStmt(pDevIns->pCritSectRC == NIL_RTRCPTR, rc = VERR_INTERNAL_ERROR_2);
-            }
-            else
-                AssertStmt(   pDevIns->pCritSectRC == NIL_RTRCPTR
-                           && pDevIns->pCritSectR0 == NIL_RTR0PTR,
-                           rc = VERR_INTERNAL_ERROR_5);
-        }
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDM: Failed to construct '%s'/%d! %Rra\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
-            paDevs[i].pDev->cInstances--;
-            /* because we're damn lazy right now, we'll say that the destructor will be called even if the constructor fails. */
-            return rc;
-        }
-    } /* for device instances */
-
-#ifdef VBOX_WITH_USB
-    /* ditto for USB Devices. */
-    rc = pdmR3UsbInstantiateDevices(pVM);
-    if (RT_FAILURE(rc))
-        return rc;
-#endif
-
-
-    /*
-     *
-     * PCI BIOS Fake and Init Complete.
-     *
-     */
-    if (pVM->pdm.s.aPciBuses[0].pDevInsR3)
-    {
-        pdmLock(pVM);
-        rc = pVM->pdm.s.aPciBuses[0].pfnFakePCIBIOSR3(pVM->pdm.s.aPciBuses[0].pDevInsR3);
-        pdmUnlock(pVM);
-        if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("PCI BIOS fake failed rc=%Rrc\n", rc));
-            return rc;
-        }
-    }
-
-    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
-    {
-        if (pDevIns->pReg->pfnInitComplete)
-        {
-            rc = pDevIns->pReg->pfnInitComplete(pDevIns);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("InitComplete on device '%s'/%d failed with rc=%Rrc\n",
-                                 pDevIns->pReg->szName, pDevIns->iInstance, rc));
-                return rc;
-            }
-        }
-    }
-
-#ifdef VBOX_WITH_USB
-    /* ditto for USB Devices. */
-    rc = pdmR3UsbVMInitComplete(pVM);
-    if (RT_FAILURE(rc))
-        return rc;
-#endif
-
-    LogFlow(("pdmR3DevInit: returns %Rrc\n", VINF_SUCCESS));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Lookups a device structure by name.
- * @internal
- */
-PPDMDEV pdmR3DevLookup(PVM pVM, const char *pszName)
-{
-    size_t cchName = strlen(pszName);
-    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
-        if (    pDev->cchName == cchName
-            &&  !strcmp(pDev->pReg->szName, pszName))
-            return pDev;
-    return NULL;
-}
-
-
-/**
- * Loads the device modules.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- */
-static int pdmR3DevLoadModules(PVM pVM)
-{
-    /*
-     * Initialize the callback structure.
-     */
-    PDMDEVREGCBINT RegCB;
-    RegCB.Core.u32Version  = PDM_DEVREG_CB_VERSION;
-    RegCB.Core.pfnRegister = pdmR3DevReg_Register;
-    RegCB.pVM              = pVM;
-    RegCB.pCfgNode         = NULL;
-
-    /*
-     * Load the builtin module
-     */
-    PCFGMNODE pDevicesNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/Devices");
-    bool fLoadBuiltin;
-    int rc = CFGMR3QueryBool(pDevicesNode, "LoadBuiltin", &fLoadBuiltin);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-        fLoadBuiltin = true;
-    else if (RT_FAILURE(rc))
-    {
-        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
-        return rc;
-    }
-    if (fLoadBuiltin)
-    {
-        /* make filename */
-        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
-        if (!pszFilename)
-            return VERR_NO_TMP_MEMORY;
-        rc = pdmR3DevLoad(pVM, &RegCB, pszFilename, "VBoxDD");
-        RTMemTmpFree(pszFilename);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /* make filename */
-        pszFilename = pdmR3FileR3("VBoxDD2", true /*fShared*/);
-        if (!pszFilename)
-            return VERR_NO_TMP_MEMORY;
-        rc = pdmR3DevLoad(pVM, &RegCB, pszFilename, "VBoxDD2");
-        RTMemTmpFree(pszFilename);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Load additional device modules.
-     */
-    PCFGMNODE pCur;
-    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
-    {
-        /*
-         * Get the name and path.
-         */
-        char szName[PDMMOD_NAME_LEN];
-        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
-        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
-        {
-            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
-            return VERR_PDM_MODULE_NAME_TOO_LONG;
-        }
-        else if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
-            return rc;
-        }
-
-        /* the path is optional, if no path the module name + path is used. */
-        char szFilename[RTPATH_MAX];
-        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-            strcpy(szFilename, szName);
-        else if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
-            return rc;
-        }
-
-        /* prepend path? */
-        if (!RTPathHavePath(szFilename))
-        {
-            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
-            if (!psz)
-                return VERR_NO_TMP_MEMORY;
-            size_t cch = strlen(psz) + 1;
-            if (cch > sizeof(szFilename))
-            {
-                RTMemTmpFree(psz);
-                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
-                return VERR_FILENAME_TOO_LONG;
-            }
-            memcpy(szFilename, psz, cch);
-            RTMemTmpFree(psz);
-        }
-
-        /*
-         * Load the module and register it's devices.
-         */
-        RegCB.pCfgNode = pCur;
-        rc = pdmR3DevLoad(pVM, &RegCB, szFilename, szName);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Loads one device module and call the registration entry point.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pRegCB          The registration callback stuff.
- * @param   pszFilename     Module filename.
- * @param   pszName         Module name.
- */
-static int pdmR3DevLoad(PVM pVM, PPDMDEVREGCBINT pRegCB, const char *pszFilename, const char *pszName)
-{
-    /*
-     * Load it.
-     */
-    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Get the registration export and call it.
-         */
-        FNPDMVBOXDEVICESREGISTER *pfnVBoxDevicesRegister;
-        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxDevicesRegister", (void **)&pfnVBoxDevicesRegister);
-        if (RT_SUCCESS(rc))
-        {
-            Log(("PDM: Calling VBoxDevicesRegister (%p) of %s (%s)\n", pfnVBoxDevicesRegister, pszName, pszFilename));
-            rc = pfnVBoxDevicesRegister(&pRegCB->Core, VBOX_VERSION);
-            if (RT_SUCCESS(rc))
-                Log(("PDM: Successfully loaded device module %s (%s).\n", pszName, pszFilename));
-            else
-                AssertMsgFailed(("VBoxDevicesRegister failed with rc=%Rrc for module %s (%s)\n", rc, pszName, pszFilename));
-        }
-        else
-        {
-            AssertMsgFailed(("Failed to locate 'VBoxDevicesRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
-            if (rc == VERR_SYMBOL_NOT_FOUND)
-                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
-        }
-    }
-    else
-        AssertMsgFailed(("Failed to load %s %s!\n", pszFilename, pszName));
-    return rc;
-}
-
-
-/**
- * @interface_method_impl{PDMDEVREGCB,pfnRegister}
- */
-static DECLCALLBACK(int) pdmR3DevReg_Register(PPDMDEVREGCB pCallbacks, PCPDMDEVREG pReg)
-{
-    /*
-     * Validate the registration structure.
-     */
-    Assert(pReg);
-    AssertMsgReturn(pReg->u32Version == PDM_DEVREG_VERSION,
-                    ("Unknown struct version %#x!\n", pReg->u32Version),
-                    VERR_PDM_UNKNOWN_DEVREG_VERSION);
-
-    AssertMsgReturn(    pReg->szName[0]
-                    &&  strlen(pReg->szName) < sizeof(pReg->szName),
-                    ("Invalid name '%s'\n", pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertMsgReturn(   !(pReg->fFlags & PDM_DEVREG_FLAGS_RC)
-                    || (   pReg->szRCMod[0]
-                        && strlen(pReg->szRCMod) < sizeof(pReg->szRCMod)),
-                    ("Invalid GC module name '%s' - (Device %s)\n", pReg->szRCMod, pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertMsgReturn(   !(pReg->fFlags & PDM_DEVREG_FLAGS_R0)
-                    || (   pReg->szR0Mod[0]
-                        && strlen(pReg->szR0Mod) < sizeof(pReg->szR0Mod)),
-                    ("Invalid R0 module name '%s' - (Device %s)\n", pReg->szR0Mod, pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_HOST_BITS_MASK) == PDM_DEVREG_FLAGS_HOST_BITS_DEFAULT,
-                    ("Invalid host bits flags! fFlags=%#x (Device %s)\n", pReg->fFlags, pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_HOST_BITS);
-    AssertMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_GUEST_BITS_MASK),
-                    ("Invalid guest bits flags! fFlags=%#x (Device %s)\n", pReg->fFlags, pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertMsgReturn(pReg->fClass,
-                    ("No class! (Device %s)\n", pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertMsgReturn(pReg->cMaxInstances > 0,
-                    ("Max instances %u! (Device %s)\n", pReg->cMaxInstances, pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertMsgReturn(pReg->cbInstance <= (uint32_t)(pReg->fFlags & (PDM_DEVREG_FLAGS_RC | PDM_DEVREG_FLAGS_R0)  ? 96 * _1K : _1M),
-                    ("Instance size %d bytes! (Device %s)\n", pReg->cbInstance, pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertMsgReturn(pReg->pfnConstruct,
-                    ("No constructor! (Device %s)\n", pReg->szName),
-                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
-    AssertLogRelMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_GUEST_BITS_MASK) == PDM_DEVREG_FLAGS_GUEST_BITS_DEFAULT,
-                          ("PDM: Rejected device '%s' because it didn't match the guest bits.\n", pReg->szName),
-                          VERR_PDM_INVALID_DEVICE_GUEST_BITS);
-    AssertLogRelMsg(pReg->u32VersionEnd == PDM_DEVREG_VERSION,
-                    ("u32VersionEnd=%#x, expected %#x. (szName=%s)\n",
-                     pReg->u32VersionEnd, PDM_DEVREG_VERSION, pReg->szName));
-
-    /*
-     * Check for duplicate and find FIFO entry at the same time.
-     */
-    PCPDMDEVREGCBINT pRegCB = (PCPDMDEVREGCBINT)pCallbacks;
-    PPDMDEV pDevPrev = NULL;
-    PPDMDEV pDev = pRegCB->pVM->pdm.s.pDevs;
-    for (; pDev; pDevPrev = pDev, pDev = pDev->pNext)
-        AssertMsgReturn(strcmp(pDev->pReg->szName, pReg->szName),
-                        ("Device '%s' already exists\n", pReg->szName),
-                        VERR_PDM_DEVICE_NAME_CLASH);
-
-    /*
-     * Allocate new device structure, initialize and insert it into the list.
-     */
-    int rc;
-    pDev = (PPDMDEV)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DEVICE, sizeof(*pDev));
-    if (pDev)
-    {
-        pDev->pNext      = NULL;
-        pDev->cInstances = 0;
-        pDev->pInstances = NULL;
-        pDev->pReg       = pReg;
-        pDev->cchName    = (uint32_t)strlen(pReg->szName);
-        rc = CFGMR3QueryStringAllocDef(    pRegCB->pCfgNode, "RCSearchPath", &pDev->pszRCSearchPath, NULL);
-        if (RT_SUCCESS(rc))
-            rc = CFGMR3QueryStringAllocDef(pRegCB->pCfgNode, "R0SearchPath", &pDev->pszR0SearchPath, NULL);
-        if (RT_SUCCESS(rc))
-        {
-            if (pDevPrev)
-                pDevPrev->pNext = pDev;
-            else
-                pRegCB->pVM->pdm.s.pDevs = pDev;
-            Log(("PDM: Registered device '%s'\n", pReg->szName));
-            return VINF_SUCCESS;
-        }
-
-        MMR3HeapFree(pDev);
-    }
-    else
-        rc = VERR_NO_MEMORY;
-    return rc;
-}
-
-
-/**
- * Locates a LUN.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   iLun            The Logical Unit to obtain the interface of.
- * @param   ppLun           Where to store the pointer to the LUN if found.
- * @thread  Try only do this in EMT...
- */
-int pdmR3DevFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPPDMLUN ppLun)
-{
-    /*
-     * Iterate registered devices looking for the device.
-     */
-    size_t cchDevice = strlen(pszDevice);
-    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
-    {
-        if (    pDev->cchName == cchDevice
-            &&  !memcmp(pDev->pReg->szName, pszDevice, cchDevice))
-        {
-            /*
-             * Iterate device instances.
-             */
-            for (PPDMDEVINS pDevIns = pDev->pInstances; pDevIns; pDevIns = pDevIns->Internal.s.pPerDeviceNextR3)
-            {
-                if (pDevIns->iInstance == iInstance)
-                {
-                    /*
-                     * Iterate luns.
-                     */
-                    for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
-                    {
-                        if (pLun->iLun == iLun)
-                        {
-                            *ppLun = pLun;
-                            return VINF_SUCCESS;
-                        }
-                    }
-                    return VERR_PDM_LUN_NOT_FOUND;
-                }
-            }
-            return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND;
-        }
-    }
-    return VERR_PDM_DEVICE_NOT_FOUND;
-}
-
-
-/**
- * Attaches a preconfigured driver to an existing device instance.
- *
- * This is used to change drivers and suchlike at runtime.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   iLun            The Logical Unit to obtain the interface of.
- * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
- * @param   ppBase          Where to store the base interface pointer. Optional.
- * @thread  EMT
- */
-VMMR3DECL(int) PDMR3DeviceAttach(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags, PPPDMIBASE ppBase)
-{
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("PDMR3DeviceAttach: pszDevice=%p:{%s} iInstance=%d iLun=%d fFlags=%#x ppBase=%p\n",
-             pszDevice, pszDevice, iInstance, iLun, fFlags, ppBase));
-
-    /*
-     * Find the LUN in question.
-     */
-    PPDMLUN pLun;
-    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Can we attach anything at runtime?
-         */
-        PPDMDEVINS pDevIns = pLun->pDevIns;
-        if (pDevIns->pReg->pfnAttach)
-        {
-            if (!pLun->pTop)
-            {
-                rc = pDevIns->pReg->pfnAttach(pDevIns, iLun, fFlags);
-            }
-            else
-                rc = VERR_PDM_DRIVER_ALREADY_ATTACHED;
-        }
-        else
-            rc = VERR_PDM_DEVICE_NO_RT_ATTACH;
-
-        if (ppBase)
-            *ppBase = pLun->pTop ? &pLun->pTop->IBase : NULL;
-    }
-    else if (ppBase)
-        *ppBase = NULL;
-
-    if (ppBase)
-        LogFlow(("PDMR3DeviceAttach: returns %Rrc *ppBase=%p\n", rc, *ppBase));
-    else
-        LogFlow(("PDMR3DeviceAttach: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Detaches a driver chain from an existing device instance.
- *
- * This is used to change drivers and suchlike at runtime.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   iLun            The Logical Unit to obtain the interface of.
- * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
- * @thread  EMT
- */
-VMMR3DECL(int) PDMR3DeviceDetach(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags)
-{
-    return PDMR3DriverDetach(pVM, pszDevice, iInstance, iLun, NULL, 0, fFlags);
-}
-
-
-/**
- * Attaches a preconfigured driver to an existing device or driver instance.
- *
- * This is used to change drivers and suchlike at runtime.  The driver or device
- * at the end of the chain will be told to attach to whatever is configured
- * below it.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iInstance       Device instance.
- * @param   iLun            The Logical Unit to obtain the interface of.
- * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
- * @param   ppBase          Where to store the base interface pointer. Optional.
- *
- * @thread  EMT
- */
-VMMR3DECL(int) PDMR3DriverAttach(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags, PPPDMIBASE ppBase)
-{
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("PDMR3DriverAttach: pszDevice=%p:{%s} iInstance=%d iLun=%d fFlags=%#x ppBase=%p\n",
-             pszDevice, pszDevice, iInstance, iLun, fFlags, ppBase));
-
-    if (ppBase)
-        *ppBase = NULL;
-
-    /*
-     * Find the LUN in question.
-     */
-    PPDMLUN pLun;
-    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Anything attached to the LUN?
-         */
-        PPDMDRVINS pDrvIns = pLun->pTop;
-        if (!pDrvIns)
-        {
-            /* No, ask the device to attach to the new stuff. */
-            PPDMDEVINS pDevIns = pLun->pDevIns;
-            if (pDevIns->pReg->pfnAttach)
-            {
-                rc = pDevIns->pReg->pfnAttach(pDevIns, iLun, fFlags);
-                if (RT_SUCCESS(rc) && ppBase)
-                    *ppBase = pLun->pTop ? &pLun->pTop->IBase : NULL;
-            }
-            else
-                rc = VERR_PDM_DEVICE_NO_RT_ATTACH;
-        }
-        else
-        {
-            /* Yes, find the bottom most driver and ask it to attach to the new stuff. */
-            while (pDrvIns->Internal.s.pDown)
-                pDrvIns = pDrvIns->Internal.s.pDown;
-            if (pDrvIns->pReg->pfnAttach)
-            {
-                rc = pDrvIns->pReg->pfnAttach(pDrvIns, fFlags);
-                if (RT_SUCCESS(rc) && ppBase)
-                    *ppBase = pDrvIns->Internal.s.pDown
-                            ? &pDrvIns->Internal.s.pDown->IBase
-                            : NULL;
-            }
-            else
-                rc = VERR_PDM_DRIVER_NO_RT_ATTACH;
-        }
-    }
-
-    if (ppBase)
-        LogFlow(("PDMR3DriverAttach: returns %Rrc *ppBase=%p\n", rc, *ppBase));
-    else
-        LogFlow(("PDMR3DriverAttach: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Detaches the specified driver instance.
- *
- * This is used to replumb drivers at runtime for simulating hot plugging and
- * media changes.
- *
- * This is a superset of PDMR3DeviceDetach.  It allows detaching drivers from
- * any driver or device by specifying the driver to start detaching at.  The
- * only prerequisite is that the driver or device above implements the
- * pfnDetach callback (PDMDRVREG / PDMDEVREG).
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iDevIns         Device instance.
- * @param   iLun            The Logical Unit in which to look for the driver.
- * @param   pszDriver       The name of the driver which to detach.  If NULL
- *                          then the entire driver chain is detatched.
- * @param   iOccurance      The occurrence of that driver in the chain.  This is
- *                          usually 0.
- * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
- * @thread  EMT
- */
-VMMR3DECL(int) PDMR3DriverDetach(PVM pVM, const char *pszDevice, unsigned iDevIns, unsigned iLun,
-                                 const char *pszDriver, unsigned iOccurance, uint32_t fFlags)
-{
-    LogFlow(("PDMR3DriverDetach: pszDevice=%p:{%s} iDevIns=%u iLun=%u pszDriver=%p:{%s} iOccurance=%u fFlags=%#x\n",
-             pszDevice, pszDevice, iDevIns, iLun, pszDriver, iOccurance, fFlags));
-    VM_ASSERT_EMT(pVM);
-    AssertPtr(pszDevice);
-    AssertPtrNull(pszDriver);
-    Assert(iOccurance == 0 || pszDriver);
-    Assert(!(fFlags & ~(PDM_TACH_FLAGS_NOT_HOT_PLUG)));
-
-    /*
-     * Find the LUN in question.
-     */
-    PPDMLUN pLun;
-    int rc = pdmR3DevFindLun(pVM, pszDevice, iDevIns, iLun, &pLun);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Locate the driver.
-         */
-        PPDMDRVINS pDrvIns = pLun->pTop;
-        if (pDrvIns)
-        {
-            if (pszDriver)
-            {
-                while (pDrvIns)
-                {
-                    if (!strcmp(pDrvIns->pReg->szName, pszDriver))
-                    {
-                        if (iOccurance == 0)
-                            break;
-                        iOccurance--;
-                    }
-                    pDrvIns = pDrvIns->Internal.s.pDown;
-                }
-            }
-            if (pDrvIns)
-                rc = pdmR3DrvDetach(pDrvIns, fFlags);
-            else
-                rc = VERR_PDM_DRIVER_INSTANCE_NOT_FOUND;
-        }
-        else
-            rc = VINF_PDM_NO_DRIVER_ATTACHED_TO_LUN;
-    }
-
-    LogFlow(("PDMR3DriverDetach: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Runtime detach and reattach of a new driver chain or sub chain.
- *
- * This is intended to be called on a non-EMT thread, this will instantiate the
- * new driver (sub-)chain, and then the EMTs will do the actual replumbing.  The
- * destruction of the old driver chain will be taken care of on the calling
- * thread.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pszDevice       Device name.
- * @param   iDevIns         Device instance.
- * @param   iLun            The Logical Unit in which to look for the driver.
- * @param   pszDriver       The name of the driver which to detach and replace.
- *                          If NULL then the entire driver chain is to be
- *                          reattached.
- * @param   iOccurance      The occurrence of that driver in the chain.  This is
- *                          usually 0.
- * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
- * @param   pCfg            The configuration of the new driver chain that is
- *                          going to be attached.  The subtree starts with the
- *                          node containing a Driver key, a Config subtree and
- *                          optionally an AttachedDriver subtree.
- *                          If this parameter is NULL, then this call will work
- *                          like at a non-pause version of PDMR3DriverDetach.
- * @param   ppBase          Where to store the base interface pointer to the new
- *                          driver.  Optional.
- *
- * @thread  Any thread. The EMTs will be involved at some point though.
- */
-VMMR3DECL(int)  PDMR3DriverReattach(PVM pVM, const char *pszDevice, unsigned iDevIns, unsigned iLun,
-                                    const char *pszDriver, unsigned iOccurance, uint32_t fFlags,
-                                    PCFGMNODE pCfg, PPPDMIBASE ppBase)
-{
-    return VERR_NOT_IMPLEMENTED;
-}
-
Index: unk/src/VBox/VMM/PDMDriver.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMDriver.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1476 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device and Driver Manager, Driver parts.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_DRIVER
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/cfgm.h>
-#include <VBox/vmm.h>
-#include <VBox/sup.h>
-#include <VBox/vm.h>
-#include <VBox/version.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <iprt/ctype.h>
-#include <iprt/mem.h>
-#include <iprt/thread.h>
-#include <iprt/path.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Internal callback structure pointer.
- *
- * The main purpose is to define the extra data we associate
- * with PDMDRVREGCB so we can find the VM instance and so on.
- */
-typedef struct PDMDRVREGCBINT
-{
-    /** The callback structure. */
-    PDMDRVREGCB     Core;
-    /** A bit of padding. */
-    uint32_t        u32[4];
-    /** VM Handle. */
-    PVM             pVM;
-    /** Pointer to the configuration node the registrations should be
-     * associated with.  Can be NULL. */
-    PCFGMNODE       pCfgNode;
-} PDMDRVREGCBINT, *PPDMDRVREGCBINT;
-typedef const PDMDRVREGCBINT *PCPDMDRVREGCBINT;
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) pdmR3DrvRegister(PCPDMDRVREGCB pCallbacks, PCPDMDRVREG pReg);
-static int pdmR3DrvLoad(PVM pVM, PPDMDRVREGCBINT pRegCB, const char *pszFilename, const char *pszName);
-
-
-
-/**
- * Register external drivers
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pfnCallback Driver registration callback
- */
-VMMR3DECL(int) PDMR3RegisterDrivers(PVM pVM, FNPDMVBOXDRIVERSREGISTER pfnCallback)
-{
-    /*
-     * The registration callbacks.
-     */
-    PDMDRVREGCBINT  RegCB;
-    RegCB.Core.u32Version   = PDM_DRVREG_CB_VERSION;
-    RegCB.Core.pfnRegister  = pdmR3DrvRegister;
-    RegCB.pVM               = pVM;
-
-    int rc = pfnCallback(&RegCB.Core, VBOX_VERSION);
-    if (RT_FAILURE(rc))
-        AssertMsgFailed(("VBoxDriversRegister failed with rc=%Rrc\n"));
-
-    return rc;
-}
-
-/**
- * This function will initialize the drivers for this VM instance.
- *
- * First of all this mean loading the builtin drivers and letting them
- * register themselves. Beyond that any additional driver modules are
- * loaded and called for registration.
- *
- * @returns VBox status code.
- * @param   pVM     VM Handle.
- */
-int pdmR3DrvInit(PVM pVM)
-{
-    LogFlow(("pdmR3DrvInit:\n"));
-
-    AssertRelease(!(RT_OFFSETOF(PDMDRVINS, achInstanceData) & 15));
-    PPDMDRVINS pDrvInsAssert; NOREF(pDrvInsAssert);
-    AssertCompile(sizeof(pDrvInsAssert->Internal.s) <= sizeof(pDrvInsAssert->Internal.padding));
-    AssertRelease(sizeof(pDrvInsAssert->Internal.s) <= sizeof(pDrvInsAssert->Internal.padding));
-
-    /*
-     * The registration callbacks.
-     */
-    PDMDRVREGCBINT  RegCB;
-    RegCB.Core.u32Version   = PDM_DRVREG_CB_VERSION;
-    RegCB.Core.pfnRegister  = pdmR3DrvRegister;
-    RegCB.pVM               = pVM;
-    RegCB.pCfgNode          = NULL;
-
-    /*
-     * Load the builtin module
-     */
-    PCFGMNODE pDriversNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/Drivers");
-    bool fLoadBuiltin;
-    int rc = CFGMR3QueryBool(pDriversNode, "LoadBuiltin", &fLoadBuiltin);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-        fLoadBuiltin = true;
-    else if (RT_FAILURE(rc))
-    {
-        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
-        return rc;
-    }
-    if (fLoadBuiltin)
-    {
-        /* make filename */
-        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
-        if (!pszFilename)
-            return VERR_NO_TMP_MEMORY;
-        rc = pdmR3DrvLoad(pVM, &RegCB, pszFilename, "VBoxDD");
-        RTMemTmpFree(pszFilename);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Load additional driver modules.
-     */
-    for (PCFGMNODE pCur = CFGMR3GetFirstChild(pDriversNode); pCur; pCur = CFGMR3GetNextChild(pCur))
-    {
-        /*
-         * Get the name and path.
-         */
-        char szName[PDMMOD_NAME_LEN];
-        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
-        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
-        {
-            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
-            return VERR_PDM_MODULE_NAME_TOO_LONG;
-        }
-        else if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
-            return rc;
-        }
-
-        /* the path is optional, if no path the module name + path is used. */
-        char szFilename[RTPATH_MAX];
-        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-            strcpy(szFilename, szName);
-        else if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
-            return rc;
-        }
-
-        /* prepend path? */
-        if (!RTPathHavePath(szFilename))
-        {
-            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
-            if (!psz)
-                return VERR_NO_TMP_MEMORY;
-            size_t cch = strlen(psz) + 1;
-            if (cch > sizeof(szFilename))
-            {
-                RTMemTmpFree(psz);
-                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
-                return VERR_FILENAME_TOO_LONG;
-            }
-            memcpy(szFilename, psz, cch);
-            RTMemTmpFree(psz);
-        }
-
-        /*
-         * Load the module and register it's drivers.
-         */
-        RegCB.pCfgNode = pCur;
-        rc = pdmR3DrvLoad(pVM, &RegCB, szFilename, szName);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    LogFlow(("pdmR3DrvInit: returns VINF_SUCCESS\n"));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Loads one driver module and call the registration entry point.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pRegCB          The registration callback stuff.
- * @param   pszFilename     Module filename.
- * @param   pszName         Module name.
- */
-static int pdmR3DrvLoad(PVM pVM, PPDMDRVREGCBINT pRegCB, const char *pszFilename, const char *pszName)
-{
-    /*
-     * Load it.
-     */
-    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Get the registration export and call it.
-         */
-        FNPDMVBOXDRIVERSREGISTER *pfnVBoxDriversRegister;
-        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxDriversRegister", (void **)&pfnVBoxDriversRegister);
-        if (RT_SUCCESS(rc))
-        {
-            Log(("PDM: Calling VBoxDriversRegister (%p) of %s (%s)\n", pfnVBoxDriversRegister, pszName, pszFilename));
-            rc = pfnVBoxDriversRegister(&pRegCB->Core, VBOX_VERSION);
-            if (RT_SUCCESS(rc))
-                Log(("PDM: Successfully loaded driver module %s (%s).\n", pszName, pszFilename));
-            else
-                AssertMsgFailed(("VBoxDriversRegister failed with rc=%Rrc\n"));
-        }
-        else
-        {
-            AssertMsgFailed(("Failed to locate 'VBoxDriversRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
-            if (rc == VERR_SYMBOL_NOT_FOUND)
-                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
-        }
-    }
-    else
-        AssertMsgFailed(("Failed to load %s (%s) rc=%Rrc!\n", pszName, pszFilename, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVREGCB,pfnRegister} */
-static DECLCALLBACK(int) pdmR3DrvRegister(PCPDMDRVREGCB pCallbacks, PCPDMDRVREG pReg)
-{
-    /*
-     * Validate the registration structure.
-     */
-    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
-    AssertMsgReturn(pReg->u32Version == PDM_DRVREG_VERSION,
-                    ("%#x\n", pReg->u32Version),
-                    VERR_PDM_UNKNOWN_DRVREG_VERSION);
-    AssertReturn(pReg->szName[0], VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(RTStrEnd(pReg->szName, sizeof(pReg->szName)),
-                    (".*s\n", sizeof(pReg->szName), pReg->szName),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(    !(pReg->fFlags & PDM_DRVREG_FLAGS_R0)
-                    ||  (   pReg->szR0Mod[0]
-                         && RTStrEnd(pReg->szR0Mod, sizeof(pReg->szR0Mod))),
-                    ("%s: %.*s\n", pReg->szName, sizeof(pReg->szR0Mod), pReg->szR0Mod),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(    !(pReg->fFlags & PDM_DRVREG_FLAGS_RC)
-                    ||  (   pReg->szRCMod[0]
-                         && RTStrEnd(pReg->szRCMod, sizeof(pReg->szRCMod))),
-                    ("%s: %.*s\n", pReg->szName, sizeof(pReg->szRCMod), pReg->szRCMod),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(VALID_PTR(pReg->pszDescription),
-                    ("%s: %p\n", pReg->szName, pReg->pszDescription),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(!(pReg->fFlags & ~(PDM_DRVREG_FLAGS_HOST_BITS_MASK | PDM_DRVREG_FLAGS_R0 | PDM_DRVREG_FLAGS_RC)),
-                    ("%s: %#x\n", pReg->szName, pReg->fFlags),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn((pReg->fFlags & PDM_DRVREG_FLAGS_HOST_BITS_MASK) == PDM_DRVREG_FLAGS_HOST_BITS_DEFAULT,
-                    ("%s: %#x\n", pReg->szName, pReg->fFlags),
-                    VERR_PDM_INVALID_DRIVER_HOST_BITS);
-    AssertMsgReturn(pReg->cMaxInstances > 0,
-                    ("%s: %#x\n", pReg->szName, pReg->cMaxInstances),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(pReg->cbInstance <= _1M,
-                    ("%s: %#x\n", pReg->szName, pReg->cbInstance),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(VALID_PTR(pReg->pfnConstruct),
-                    ("%s: %p\n", pReg->szName, pReg->pfnConstruct),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(VALID_PTR(pReg->pfnRelocate) || !(pReg->fFlags & PDM_DRVREG_FLAGS_RC),
-                    ("%s: %#x\n", pReg->szName, pReg->cbInstance),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(pReg->pfnSoftReset == NULL,
-                    ("%s: %p\n", pReg->szName, pReg->pfnSoftReset),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-    AssertMsgReturn(pReg->u32VersionEnd == PDM_DRVREG_VERSION,
-                    ("%s: #x\n", pReg->szName, pReg->u32VersionEnd),
-                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
-
-    /*
-     * Check for duplicate and find FIFO entry at the same time.
-     */
-    PCPDMDRVREGCBINT pRegCB = (PCPDMDRVREGCBINT)pCallbacks;
-    PPDMDRV pDrvPrev = NULL;
-    PPDMDRV pDrv = pRegCB->pVM->pdm.s.pDrvs;
-    for (; pDrv; pDrvPrev = pDrv, pDrv = pDrv->pNext)
-    {
-        if (!strcmp(pDrv->pReg->szName, pReg->szName))
-        {
-            AssertMsgFailed(("Driver '%s' already exists\n", pReg->szName));
-            return VERR_PDM_DRIVER_NAME_CLASH;
-        }
-    }
-
-    /*
-     * Allocate new driver structure and insert it into the list.
-     */
-    int rc;
-    pDrv = (PPDMDRV)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DRIVER, sizeof(*pDrv));
-    if (pDrv)
-    {
-        pDrv->pNext         = NULL;
-        pDrv->cInstances    = 0;
-        pDrv->iNextInstance = 0;
-        pDrv->pReg          = pReg;
-        rc = CFGMR3QueryStringAllocDef(    pRegCB->pCfgNode, "RCSearchPath", &pDrv->pszRCSearchPath, NULL);
-        if (RT_SUCCESS(rc))
-            rc = CFGMR3QueryStringAllocDef(pRegCB->pCfgNode, "R0SearchPath", &pDrv->pszR0SearchPath, NULL);
-        if (RT_SUCCESS(rc))
-        {
-            if (pDrvPrev)
-                pDrvPrev->pNext = pDrv;
-            else
-                pRegCB->pVM->pdm.s.pDrvs = pDrv;
-            Log(("PDM: Registered driver '%s'\n", pReg->szName));
-            return VINF_SUCCESS;
-        }
-        MMR3HeapFree(pDrv);
-    }
-    else
-        rc = VERR_NO_MEMORY;
-    return rc;
-}
-
-
-/**
- * Lookups a driver structure by name.
- * @internal
- */
-PPDMDRV pdmR3DrvLookup(PVM pVM, const char *pszName)
-{
-    for (PPDMDRV pDrv = pVM->pdm.s.pDrvs; pDrv; pDrv = pDrv->pNext)
-        if (!strcmp(pDrv->pReg->szName, pszName))
-            return pDrv;
-    return NULL;
-}
-
-
-/**
- * Instantiate a driver.
- *
- * @returns VBox status code, including informational statuses.
- *
- * @param   pVM                 The VM handle.
- * @param   pNode               The CFGM node for the driver.
- * @param   pBaseInterface      The base interface.
- * @param   pDrvAbove           The driver above it.  NULL if it's the top-most
- *                              driver.
- * @param   pLun                The LUN the driver is being attached to.  NULL
- *                              if we're instantiating a driver chain before
- *                              attaching it - untested.
- * @param   ppBaseInterface     Where to return the pointer to the base
- *                              interface of the newly created driver.
- *
- * @remarks Recursive calls to this function is normal as the drivers will
- *          attach to anything below them during the pfnContruct call.
- */
-int pdmR3DrvInstantiate(PVM pVM, PCFGMNODE pNode, PPDMIBASE pBaseInterface, PPDMDRVINS pDrvAbove,
-                        PPDMLUN pLun, PPDMIBASE *ppBaseInterface)
-{
-    Assert(!pDrvAbove || !pDrvAbove->Internal.s.pDown);
-    Assert(!pDrvAbove || !pDrvAbove->pDownBase);
-
-    Assert(pBaseInterface->pfnQueryInterface(pBaseInterface, PDMIBASE_IID) == pBaseInterface);
-
-    /*
-     * Find the driver.
-     */
-    char *pszName;
-    int rc = CFGMR3QueryStringAlloc(pNode, "Driver", &pszName);
-    if (RT_SUCCESS(rc))
-    {
-        PPDMDRV pDrv = pdmR3DrvLookup(pVM, pszName);
-        MMR3HeapFree(pszName);
-        if (    pDrv
-            &&  pDrv->cInstances < pDrv->pReg->cMaxInstances)
-        {
-            /* config node */
-            PCFGMNODE pConfigNode = CFGMR3GetChild(pNode, "Config");
-            if (!pConfigNode)
-                rc = CFGMR3InsertNode(pNode, "Config", &pConfigNode);
-            if (RT_SUCCESS(rc))
-            {
-                CFGMR3SetRestrictedRoot(pConfigNode);
-
-                /*
-                 * Allocate the driver instance.
-                 */
-                size_t cb = RT_OFFSETOF(PDMDRVINS, achInstanceData[pDrv->pReg->cbInstance]);
-                cb = RT_ALIGN_Z(cb, 16);
-                bool const fHyperHeap = !!(pDrv->pReg->fFlags & (PDM_DRVREG_FLAGS_R0 | PDM_DRVREG_FLAGS_RC));
-                PPDMDRVINS pNew;
-                if (fHyperHeap)
-                    rc = MMHyperAlloc(pVM, cb, 64, MM_TAG_PDM_DRIVER, (void **)&pNew);
-                else
-                    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_DRIVER, cb, (void **)&pNew);
-                if (pNew)
-                {
-                    /*
-                     * Initialize the instance structure (declaration order).
-                     */
-                    pNew->u32Version                = PDM_DRVINS_VERSION;
-                    pNew->Internal.s.pUp            = pDrvAbove ? pDrvAbove : NULL;
-                    //pNew->Internal.s.pDown          = NULL;
-                    pNew->Internal.s.pLun           = pLun;
-                    pNew->Internal.s.pDrv           = pDrv;
-                    pNew->Internal.s.pVMR3          = pVM;
-                    pNew->Internal.s.pVMR0          = pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_R0 ? pVM->pVMR0 : NIL_RTR0PTR;
-                    pNew->Internal.s.pVMRC          = pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_RC ? pVM->pVMRC : NIL_RTRCPTR;
-                    //pNew->Internal.s.fDetaching     = false;
-                    pNew->Internal.s.fVMSuspended   = true;
-                    //pNew->Internal.s.fVMReset       = false;
-                    pNew->Internal.s.fHyperHeap     = fHyperHeap;
-                    //pNew->Internal.s.pfnAsyncNotify = NULL;
-                    pNew->Internal.s.pCfgHandle     = pNode;
-                    pNew->pReg                      = pDrv->pReg;
-                    pNew->pCfg                      = pConfigNode;
-                    pNew->iInstance                 = pDrv->iNextInstance;
-                    pNew->pUpBase                   = pBaseInterface;
-                    Assert(!pDrvAbove || pBaseInterface == &pDrvAbove->IBase);
-                    //pNew->pDownBase                 = NULL;
-                    //pNew->IBase.pfnQueryInterface   = NULL;
-                    pNew->pHlpR3                    = &g_pdmR3DrvHlp;
-                    pNew->pvInstanceDataR3          = &pNew->achInstanceData[0];
-                    if (pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
-                    {
-                        pNew->pvInstanceDataR0      = MMHyperR3ToR0(pVM, &pNew->achInstanceData[0]);
-                        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "g_pdmR0DrvHlp", &pNew->pHlpR0);
-                        AssertReleaseRCReturn(rc, rc);
-
-                    }
-                    if (pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
-                    {
-                        pNew->pvInstanceDataR0      = MMHyperR3ToRC(pVM, &pNew->achInstanceData[0]);
-                        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDrvHlp", &pNew->pHlpRC);
-                        AssertReleaseRCReturn(rc, rc);
-                    }
-
-                    pDrv->iNextInstance++;
-                    pDrv->cInstances++;
-
-                    /*
-                     * Link with it with the driver above / LUN.
-                     */
-                    if (pDrvAbove)
-                    {
-                        pDrvAbove->pDownBase        = &pNew->IBase;
-                        pDrvAbove->Internal.s.pDown = pNew;
-                    }
-                    else if (pLun)
-                        pLun->pTop                  = pNew;
-                    if (pLun)
-                        pLun->pBottom               = pNew;
-
-                    /*
-                     * Invoke the constructor.
-                     */
-                    rc = pDrv->pReg->pfnConstruct(pNew, pNew->pCfg, 0 /*fFlags*/);
-                    if (RT_SUCCESS(rc))
-                    {
-                        AssertPtr(pNew->IBase.pfnQueryInterface);
-                        Assert(pNew->IBase.pfnQueryInterface(&pNew->IBase, PDMIBASE_IID) == &pNew->IBase);
-
-                        /* Success! */
-                        *ppBaseInterface = &pNew->IBase;
-                        if (pLun)
-                            Log(("PDM: Attached driver %p:'%s'/%d to LUN#%d on device '%s'/%d, pDrvAbove=%p:'%s'/%d\n",
-                                 pNew, pDrv->pReg->szName, pNew->iInstance,
-                                 pLun->iLun,
-                                 pLun->pDevIns ? pLun->pDevIns->pReg->szName : pLun->pUsbIns->pReg->szName,
-                                 pLun->pDevIns ? pLun->pDevIns->iInstance    : pLun->pUsbIns->iInstance,
-                                 pDrvAbove, pDrvAbove ? pDrvAbove->pReg->szName : "", pDrvAbove ? pDrvAbove->iInstance : UINT32_MAX));
-                        else
-                            Log(("PDM: Attached driver %p:'%s'/%d, pDrvAbove=%p:'%s'/%d\n",
-                                 pNew, pDrv->pReg->szName, pNew->iInstance,
-                                 pDrvAbove, pDrvAbove ? pDrvAbove->pReg->szName : "", pDrvAbove ? pDrvAbove->iInstance : UINT32_MAX));
-                    }
-                    else
-                        pdmR3DrvDestroyChain(pNew, PDM_TACH_FLAGS_NO_CALLBACKS);
-                }
-                else
-                {
-                    AssertMsgFailed(("Failed to allocate %d bytes for instantiating driver '%s'\n", cb, pszName));
-                    rc = VERR_NO_MEMORY;
-                }
-            }
-            else
-                AssertMsgFailed(("Failed to create Config node! rc=%Rrc\n", rc));
-        }
-        else if (pDrv)
-        {
-            AssertMsgFailed(("Too many instances of driver '%s', max is %u\n", pszName, pDrv->pReg->cMaxInstances));
-            rc = VERR_PDM_TOO_MANY_DRIVER_INSTANCES;
-        }
-        else
-        {
-            AssertMsgFailed(("Driver '%s' wasn't found!\n", pszName));
-            rc = VERR_PDM_DRIVER_NOT_FOUND;
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("Query for string value of \"Driver\" -> %Rrc\n", rc));
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-            rc = VERR_PDM_CFG_MISSING_DRIVER_NAME;
-    }
-    return rc;
-}
-
-
-/**
- * Detaches a driver from whatever it's attached to.
- * This will of course lead to the destruction of the driver and all drivers below it in the chain.
- *
- * @returns VINF_SUCCESS
- * @param   pDrvIns     The driver instance to detach.
- * @param   fFlags      Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
- */
-int pdmR3DrvDetach(PPDMDRVINS pDrvIns, uint32_t fFlags)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvDetach: pDrvIns=%p '%s'/%d\n", pDrvIns, pDrvIns->pReg->szName, pDrvIns->iInstance));
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    /*
-     * Check that we're not doing this recursively, that could have unwanted sideeffects!
-     */
-    if (pDrvIns->Internal.s.fDetaching)
-    {
-        AssertMsgFailed(("Recursive detach! '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance));
-        return VINF_SUCCESS;           }
-
-    /*
-     * Check that we actually can detach this instance.
-     * The requirement is that the driver/device above has a detach method.
-     */
-    if (pDrvIns->Internal.s.pUp
-        ? !pDrvIns->Internal.s.pUp->pReg->pfnDetach
-        : !pDrvIns->Internal.s.pLun->pDevIns->pReg->pfnDetach)
-    {
-        AssertMsgFailed(("Cannot detach driver instance because the driver/device above doesn't support it!\n"));
-        return VERR_PDM_DRIVER_DETACH_NOT_POSSIBLE;
-    }
-
-    /*
-     * Join paths with pdmR3DrvDestroyChain.
-     */
-    pdmR3DrvDestroyChain(pDrvIns, fFlags);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Destroys a driver chain starting with the specified driver.
- *
- * This is used when unplugging a device at run time.
- *
- * @param   pDrvIns     Pointer to the driver instance to start with.
- * @param   fFlags      PDM_TACH_FLAGS_NOT_HOT_PLUG, PDM_TACH_FLAGS_NO_CALLBACKS
- *                      or 0.
- */
-void pdmR3DrvDestroyChain(PPDMDRVINS pDrvIns, uint32_t fFlags)
-{
-    PVM pVM = pDrvIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Detach the bottommost driver until we've detached pDrvIns.
-     */
-    pDrvIns->Internal.s.fDetaching = true;
-    PPDMDRVINS pCur;
-    do
-    {
-        /* find the driver to detach. */
-        pCur = pDrvIns;
-        while (pCur->Internal.s.pDown)
-            pCur = pCur->Internal.s.pDown;
-        LogFlow(("pdmR3DrvDestroyChain: pCur=%p '%s'/%d\n", pCur, pCur->pReg->szName, pCur->iInstance));
-
-        /*
-         * Unlink it and notify parent.
-         */
-        pCur->Internal.s.fDetaching = true;
-
-        PPDMLUN pLun = pCur->Internal.s.pLun;
-        Assert(pLun->pBottom == pCur);
-        pLun->pBottom = pCur->Internal.s.pUp;
-
-        if (pCur->Internal.s.pUp)
-        {
-            /* driver parent */
-            PPDMDRVINS pParent = pCur->Internal.s.pUp;
-            pCur->Internal.s.pUp = NULL;
-            pParent->Internal.s.pDown = NULL;
-
-            if (!(fFlags & PDM_TACH_FLAGS_NO_CALLBACKS) && pParent->pReg->pfnDetach)
-                pParent->pReg->pfnDetach(pParent, fFlags);
-
-            pParent->pDownBase = NULL;
-        }
-        else
-        {
-            /* device parent */
-            Assert(pLun->pTop == pCur);
-            pLun->pTop = NULL;
-            if (!(fFlags & PDM_TACH_FLAGS_NO_CALLBACKS) && pLun->pDevIns->pReg->pfnDetach)
-                pLun->pDevIns->pReg->pfnDetach(pLun->pDevIns, pLun->iLun, fFlags);
-        }
-
-        /*
-         * Call destructor.
-         */
-        pCur->pUpBase = NULL;
-        if (pCur->pReg->pfnDestruct)
-            pCur->pReg->pfnDestruct(pCur);
-        pCur->Internal.s.pDrv->cInstances--;
-
-        /*
-         * Free all resources allocated by the driver.
-         */
-        /* Queues. */
-        int rc = PDMR3QueueDestroyDriver(pVM, pCur);
-        AssertRC(rc);
-
-        /* Timers. */
-        rc = TMR3TimerDestroyDriver(pVM, pCur);
-        AssertRC(rc);
-
-        /* SSM data units. */
-        rc = SSMR3DeregisterDriver(pVM, pCur, NULL, 0);
-        AssertRC(rc);
-
-        /* PDM threads. */
-        rc = pdmR3ThreadDestroyDriver(pVM, pCur);
-        AssertRC(rc);
-
-        /* Info handlers. */
-        rc = DBGFR3InfoDeregisterDriver(pVM, pCur, NULL);
-        AssertRC(rc);
-
-        /* PDM critsects. */
-        rc = pdmR3CritSectDeleteDriver(pVM, pCur);
-        AssertRC(rc);
-
-        /* Block caches. */
-        PDMR3BlkCacheReleaseDriver(pVM, pCur);
-
-        /* Finally, the driver it self. */
-        bool fHyperHeap = pCur->Internal.s.fHyperHeap;
-        ASMMemFill32(pCur, RT_OFFSETOF(PDMDRVINS, achInstanceData[pCur->pReg->cbInstance]), 0xdeadd0d0);
-        if (fHyperHeap)
-            MMHyperFree(pVM, pCur);
-        else
-            MMR3HeapFree(pCur);
-
-    } while (pCur != pDrvIns);
-}
-
-
-
-
-/** @name Driver Helpers
- * @{
- */
-
-/** @interface_method_impl{PDMDRVHLP,pfnAttach} */
-static DECLCALLBACK(int) pdmR3DrvHlp_Attach(PPDMDRVINS pDrvIns, uint32_t fFlags, PPDMIBASE *ppBaseInterface)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    PVM pVM = pDrvIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DrvHlp_Attach: caller='%s'/%d: fFlags=%#x\n", pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
-    Assert(!(fFlags & ~(PDM_TACH_FLAGS_NOT_HOT_PLUG)));
-
-    /*
-     * Check that there isn't anything attached already.
-     */
-    int rc;
-    if (!pDrvIns->Internal.s.pDown)
-    {
-        Assert(pDrvIns->Internal.s.pLun->pBottom == pDrvIns);
-
-        /*
-         * Get the attached driver configuration.
-         */
-        PCFGMNODE pNode = CFGMR3GetChild(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver");
-        if (pNode)
-            rc = pdmR3DrvInstantiate(pVM, pNode, &pDrvIns->IBase, pDrvIns, pDrvIns->Internal.s.pLun, ppBaseInterface);
-        else
-            rc = VERR_PDM_NO_ATTACHED_DRIVER;
-    }
-    else
-    {
-        AssertMsgFailed(("Already got a driver attached. The driver should keep track of such things!\n"));
-        rc = VERR_PDM_DRIVER_ALREADY_ATTACHED;
-    }
-
-    LogFlow(("pdmR3DrvHlp_Attach: caller='%s'/%d: return %Rrc\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnDetach} */
-static DECLCALLBACK(int) pdmR3DrvHlp_Detach(PPDMDRVINS pDrvIns, uint32_t fFlags)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_Detach: caller='%s'/%d: fFlags=%#x\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    /*
-     * Anything attached?
-     */
-    int rc;
-    if (pDrvIns->Internal.s.pDown)
-        rc = pdmR3DrvDetach(pDrvIns->Internal.s.pDown, fFlags);
-    else
-    {
-        AssertMsgFailed(("Nothing attached!\n"));
-        rc = VERR_PDM_NO_DRIVER_ATTACHED;
-    }
-
-    LogFlow(("pdmR3DrvHlp_Detach: caller='%s'/%d: returns %Rrc\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnDetachSelf} */
-static DECLCALLBACK(int) pdmR3DrvHlp_DetachSelf(PPDMDRVINS pDrvIns, uint32_t fFlags)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_DetachSelf: caller='%s'/%d: fFlags=%#x\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    int rc = pdmR3DrvDetach(pDrvIns, fFlags);
-
-    LogFlow(("pdmR3DrvHlp_Detach: returns %Rrc\n", rc)); /* pDrvIns is freed by now. */
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnMountPrepare} */
-static DECLCALLBACK(int) pdmR3DrvHlp_MountPrepare(PPDMDRVINS pDrvIns, const char *pszFilename, const char *pszCoreDriver)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: pszFilename=%p:{%s} pszCoreDriver=%p:{%s}\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, pszFilename, pszFilename, pszCoreDriver, pszCoreDriver));
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    /*
-     * Do the caller have anything attached below itself?
-     */
-    if (pDrvIns->Internal.s.pDown)
-    {
-        AssertMsgFailed(("Cannot prepare a mount when something's attached to you!\n"));
-        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
-    }
-
-    /*
-     * We're asked to prepare, so we'll start off by nuking the
-     * attached configuration tree.
-     */
-    PCFGMNODE   pNode = CFGMR3GetChild(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver");
-    if (pNode)
-        CFGMR3RemoveNode(pNode);
-
-    /*
-     * If there is no core driver, we'll have to probe for it.
-     */
-    if (!pszCoreDriver)
-    {
-        /** @todo implement image probing. */
-        AssertReleaseMsgFailed(("Not implemented!\n"));
-        return VERR_NOT_IMPLEMENTED;
-    }
-
-    /*
-     * Construct the basic attached driver configuration.
-     */
-    int rc = CFGMR3InsertNode(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver", &pNode);
-    if (RT_SUCCESS(rc))
-    {
-        rc = CFGMR3InsertString(pNode, "Driver", pszCoreDriver);
-        if (RT_SUCCESS(rc))
-        {
-            PCFGMNODE pCfg;
-            rc = CFGMR3InsertNode(pNode, "Config", &pCfg);
-            if (RT_SUCCESS(rc))
-            {
-                rc = CFGMR3InsertString(pCfg, "Path", pszFilename);
-                if (RT_SUCCESS(rc))
-                {
-                    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: returns %Rrc (Driver=%s)\n",
-                             pDrvIns->pReg->szName, pDrvIns->iInstance, rc, pszCoreDriver));
-                    return rc;
-                }
-                else
-                    AssertMsgFailed(("Path string insert failed, rc=%Rrc\n", rc));
-            }
-            else
-                AssertMsgFailed(("Config node failed, rc=%Rrc\n", rc));
-        }
-        else
-            AssertMsgFailed(("Driver string insert failed, rc=%Rrc\n", rc));
-        CFGMR3RemoveNode(pNode);
-    }
-    else
-        AssertMsgFailed(("AttachedDriver node insert failed, rc=%Rrc\n", rc));
-
-    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: returns %Rrc\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnAssertEMT} */
-static DECLCALLBACK(bool) pdmR3DrvHlp_AssertEMT(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    if (VM_IS_EMT(pDrvIns->Internal.s.pVMR3))
-        return true;
-
-    char szMsg[100];
-    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance);
-    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
-    AssertBreakpoint();
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    return false;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnAssertOther} */
-static DECLCALLBACK(bool) pdmR3DrvHlp_AssertOther(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    if (!VM_IS_EMT(pDrvIns->Internal.s.pVMR3))
-        return true;
-
-    char szMsg[100];
-    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance);
-    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
-    AssertBreakpoint();
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    return false;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnVMSetError} */
-static DECLCALLBACK(int) pdmR3DrvHlp_VMSetError(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    va_list args;
-    va_start(args, pszFormat);
-    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
-    va_end(args);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnVMSetErrorV} */
-static DECLCALLBACK(int) pdmR3DrvHlp_VMSetErrorV(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnVMSetRuntimeError} */
-static DECLCALLBACK(int) pdmR3DrvHlp_VMSetRuntimeError(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    va_list args;
-    va_start(args, pszFormat);
-    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, args);
-    va_end(args);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnVMSetRuntimeErrorV} */
-static DECLCALLBACK(int) pdmR3DrvHlp_VMSetRuntimeErrorV(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, va);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMState} */
-static DECLCALLBACK(VMSTATE) pdmR3DrvHlp_VMState(PPDMDRVINS pDrvIns)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-
-    VMSTATE enmVMState = VMR3GetState(pDrvIns->Internal.s.pVMR3);
-
-    LogFlow(("pdmR3DrvHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
-             enmVMState, VMR3GetStateName(enmVMState)));
-    return enmVMState;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnVMTeleportedAndNotFullyResumedYet} */
-static DECLCALLBACK(bool) pdmR3DrvHlp_VMTeleportedAndNotFullyResumedYet(PPDMDRVINS pDrvIns)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-
-    bool fRc = VMR3TeleportedAndNotFullyResumedYet(pDrvIns->Internal.s.pVMR3);
-
-    LogFlow(("pdmR3DrvHlp_VMState: caller='%s'/%d: returns %RTbool)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
-             fRc));
-    return fRc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLPR3,pfnGetSupDrvSession} */
-static DECLCALLBACK(PSUPDRVSESSION) pdmR3DrvHlp_GetSupDrvSession(PPDMDRVINS pDrvIns)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-
-    PSUPDRVSESSION pSession = pDrvIns->Internal.s.pVMR3->pSession;
-    LogFlow(("pdmR3DrvHlp_GetSupDrvSession: caller='%s'/%d: returns %p)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
-             pSession));
-    return pSession;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnQueueCreate} */
-static DECLCALLBACK(int) pdmR3DrvHlp_QueueCreate(PPDMDRVINS pDrvIns, uint32_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
-                                                 PFNPDMQUEUEDRV pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_PDMQueueCreate: caller='%s'/%d: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%p:{%s} ppQueue=%p\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, pszName, ppQueue, ppQueue));
-    PVM pVM = pDrvIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-
-    if (pDrvIns->iInstance > 0)
-    {
-        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DRIVER_DESC, "%s_%u", pszName, pDrvIns->iInstance);
-        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
-    }
-
-    int rc = PDMR3QueueCreateDriver(pVM, pDrvIns, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, ppQueue);
-
-    LogFlow(("pdmR3DrvHlp_PDMQueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc, *ppQueue));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnTMGetVirtualFreq} */
-static DECLCALLBACK(uint64_t) pdmR3DrvHlp_TMGetVirtualFreq(PPDMDRVINS pDrvIns)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-
-    return TMVirtualGetFreq(pDrvIns->Internal.s.pVMR3);
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnTMGetVirtualTime} */
-static DECLCALLBACK(uint64_t) pdmR3DrvHlp_TMGetVirtualTime(PPDMDRVINS pDrvIns)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-
-    return TMVirtualGet(pDrvIns->Internal.s.pVMR3);
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnTMTimerCreate} */
-static DECLCALLBACK(int) pdmR3DrvHlp_TMTimerCreate(PPDMDRVINS pDrvIns, TMCLOCK enmClock, PFNTMTIMERDRV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
-
-    int rc = TMR3TimerCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer);
-
-    LogFlow(("pdmR3DrvHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc *ppTimer=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc, *ppTimer));
-    return rc;
-}
-
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSSMRegister} */
-static DECLCALLBACK(int) pdmR3DrvHlp_SSMRegister(PPDMDRVINS pDrvIns, uint32_t uVersion, size_t cbGuess,
-                                                 PFNSSMDRVLIVEPREP pfnLivePrep, PFNSSMDRVLIVEEXEC pfnLiveExec, PFNSSMDRVLIVEVOTE pfnLiveVote,
-                                                 PFNSSMDRVSAVEPREP pfnSavePrep, PFNSSMDRVSAVEEXEC pfnSaveExec, PFNSSMDRVSAVEDONE pfnSaveDone,
-                                                 PFNSSMDRVLOADPREP pfnLoadPrep, PFNSSMDRVLOADEXEC pfnLoadExec, PFNSSMDRVLOADDONE pfnLoadDone)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DrvHlp_SSMRegister: caller='%s'/%d: uVersion=#x cbGuess=%#x \n"
-             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p  pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoaddone=%p\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, uVersion, cbGuess,
-             pfnLivePrep, pfnLiveExec, pfnLiveVote,
-             pfnSavePrep, pfnSaveExec, pfnSaveDone, pfnLoadPrep, pfnLoadExec, pfnLoadDone));
-
-    int rc = SSMR3RegisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pDrvIns->pReg->szName, pDrvIns->iInstance,
-                                 uVersion, cbGuess,
-                                 pfnLivePrep, pfnLiveExec, pfnLiveVote,
-                                 pfnSavePrep, pfnSaveExec, pfnSaveDone,
-                                 pfnLoadPrep, pfnLoadExec, pfnLoadDone);
-
-    LogFlow(("pdmR3DrvHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSSMDeregister} */
-static DECLCALLBACK(int) pdmR3DrvHlp_SSMDeregister(PPDMDRVINS pDrvIns, const char *pszName, uint32_t u32Instance)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DrvHlp_SSMDeregister: caller='%s'/%d: pszName=%p:{%s} u32Instance=%#x\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName, u32Instance));
-
-    int rc = SSMR3DeregisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pszName, u32Instance);
-
-    LogFlow(("pdmR3DrvHlp_SSMDeregister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLP,pfnDBGFInfoRegister} */
-static DECLCALLBACK(int) pdmR3DrvHlp_DBGFInfoRegister(PPDMDRVINS pDrvIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDRV pfnHandler)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
-
-    int rc = DBGFR3InfoRegisterDriver(pDrvIns->Internal.s.pVMR3, pszName, pszDesc, pfnHandler, pDrvIns);
-
-    LogFlow(("pdmR3DrvHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDEVHLP,pfnDBGFInfoDeregister} */
-static DECLCALLBACK(int) pdmR3DrvHlp_DBGFInfoDeregister(PPDMDRVINS pDrvIns, const char *pszName)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_DBGFInfoDeregister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName));
-
-    int rc = DBGFR3InfoDeregisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pszName);
-
-    LogFlow(("pdmR3DrvHlp_DBGFInfoDeregister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSTAMRegister} */
-static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegister(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    STAM_REG(pDrvIns->Internal.s.pVMR3, pvSample, enmType, pszName, enmUnit, pszDesc);
-    /** @todo track the samples so they can be dumped & deregistered when the driver instance is destroyed.
-     * For now we just have to be careful not to use this call for drivers which can be unloaded. */
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSTAMRegisterF} */
-static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegisterF(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
-                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, ...)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    va_list args;
-    va_start(args, pszName);
-    int rc = STAMR3RegisterV(pDrvIns->Internal.s.pVMR3, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-    va_end(args);
-    AssertRC(rc);
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSTAMRegisterV} */
-static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegisterV(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
-                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    int rc = STAMR3RegisterV(pDrvIns->Internal.s.pVMR3, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-    AssertRC(rc);
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSTAMDeregister} */
-static DECLCALLBACK(int) pdmR3DrvHlp_STAMDeregister(PPDMDRVINS pDrvIns, void *pvSample)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-
-    int rc = STAMR3DeregisterU(pDrvIns->Internal.s.pVMR3->pUVM, pvSample);
-    AssertRC(rc);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSUPCallVMMR0Ex} */
-static DECLCALLBACK(int) pdmR3DrvHlp_SUPCallVMMR0Ex(PPDMDRVINS pDrvIns, unsigned uOperation, void *pvArg, unsigned cbArg)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_SSMCallVMMR0Ex: caller='%s'/%d: uOperation=%u pvArg=%p cbArg=%d\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, uOperation, pvArg, cbArg));
-    int rc;
-    if (    uOperation >= VMMR0_DO_SRV_START
-        &&  uOperation <  VMMR0_DO_SRV_END)
-        rc = SUPR3CallVMMR0Ex(pDrvIns->Internal.s.pVMR3->pVMR0, NIL_VMCPUID, uOperation, 0, (PSUPVMMR0REQHDR)pvArg);
-    else
-    {
-        AssertMsgFailed(("Invalid uOperation=%u\n", uOperation));
-        rc = VERR_INVALID_PARAMETER;
-    }
-
-    LogFlow(("pdmR3DrvHlp_SUPCallVMMR0Ex: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnUSBRegisterHub} */
-static DECLCALLBACK(int) pdmR3DrvHlp_USBRegisterHub(PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DrvHlp_USBRegisterHub: caller='%s'/%d: fVersions=%#x cPorts=%#x pUsbHubReg=%p ppUsbHubHlp=%p\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, fVersions, cPorts, pUsbHubReg, ppUsbHubHlp));
-
-#ifdef VBOX_WITH_USB
-    int rc = pdmR3UsbRegisterHub(pDrvIns->Internal.s.pVMR3, pDrvIns, fVersions, cPorts, pUsbHubReg, ppUsbHubHlp);
-#else
-    int rc = VERR_NOT_SUPPORTED;
-#endif
-
-    LogFlow(("pdmR3DrvHlp_USBRegisterHub: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnSetAsyncNotification} */
-static DECLCALLBACK(int) pdmR3DrvHlp_SetAsyncNotification(PPDMDRVINS pDrvIns, PFNPDMDRVASYNCNOTIFY pfnAsyncNotify)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT0(pDrvIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, pfnAsyncNotify));
-
-    int rc = VINF_SUCCESS;
-    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
-    AssertStmt(!pDrvIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
-    AssertStmt(pDrvIns->Internal.s.fVMSuspended || pDrvIns->Internal.s.fVMReset, rc = VERR_WRONG_ORDER);
-    VMSTATE enmVMState = VMR3GetState(pDrvIns->Internal.s.pVMR3);
-    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
-               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
-               || enmVMState == VMSTATE_SUSPENDING_LS
-               || enmVMState == VMSTATE_RESETTING
-               || enmVMState == VMSTATE_RESETTING_LS
-               || enmVMState == VMSTATE_POWERING_OFF
-               || enmVMState == VMSTATE_POWERING_OFF_LS,
-               rc = VERR_INVALID_STATE);
-
-    if (RT_SUCCESS(rc))
-        pDrvIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
-
-    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnAsyncNotificationCompleted} */
-static DECLCALLBACK(void) pdmR3DrvHlp_AsyncNotificationCompleted(PPDMDRVINS pDrvIns)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    PVM pVM = pDrvIns->Internal.s.pVMR3;
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    if (   enmVMState == VMSTATE_SUSPENDING
-        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
-        || enmVMState == VMSTATE_SUSPENDING_LS
-        || enmVMState == VMSTATE_RESETTING
-        || enmVMState == VMSTATE_RESETTING_LS
-        || enmVMState == VMSTATE_POWERING_OFF
-        || enmVMState == VMSTATE_POWERING_OFF_LS)
-    {
-        LogFlow(("pdmR3DrvHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pDrvIns->pReg->szName, pDrvIns->iInstance));
-        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
-    }
-    else
-        LogFlow(("pdmR3DrvHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance, enmVMState));
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnThreadCreate} */
-static DECLCALLBACK(int) pdmR3DrvHlp_ThreadCreate(PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
-                                                  PFNPDMTHREADWAKEUPDRV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DrvHlp_ThreadCreate: caller='%s'/%d: ppThread=%p pvUser=%p pfnThread=%p pfnWakeup=%p cbStack=%#zx enmType=%d pszName=%p:{%s}\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName, pszName));
-
-    int rc = pdmR3ThreadCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName);
-
-    LogFlow(("pdmR3DrvHlp_ThreadCreate: caller='%s'/%d: returns %Rrc *ppThread=%RTthrd\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
-            rc, *ppThread));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnAsyncCompletionTemplateCreate} */
-static DECLCALLBACK(int) pdmR3DrvHlp_AsyncCompletionTemplateCreate(PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
-                                                                   PFNPDMASYNCCOMPLETEDRV pfnCompleted, void *pvTemplateUser,
-                                                                   const char *pszDesc)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    LogFlow(("pdmR3DrvHlp_AsyncCompletionTemplateCreate: caller='%s'/%d: ppTemplate=%p pfnCompleted=%p pszDesc=%p:{%s}\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, ppTemplate, pfnCompleted, pszDesc, pszDesc));
-
-    int rc = PDMR3AsyncCompletionTemplateCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppTemplate, pfnCompleted, pvTemplateUser, pszDesc);
-
-    LogFlow(("pdmR3DrvHlp_AsyncCompletionTemplateCreate: caller='%s'/%d: returns %Rrc *ppThread=%p\n", pDrvIns->pReg->szName,
-             pDrvIns->iInstance, rc, *ppTemplate));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnLdrGetRCInterfaceSymbols} */
-static DECLCALLBACK(int) pdmR3DrvHlp_LdrGetRCInterfaceSymbols(PPDMDRVINS pDrvIns, void *pvInterface, size_t cbInterface,
-                                                              const char *pszSymPrefix, const char *pszSymList)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DrvHlp_LdrGetRCInterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
-
-    int rc;
-    if (   strncmp(pszSymPrefix, "drv", 3) == 0
-        && RTStrIStr(pszSymPrefix + 3, pDrvIns->pReg->szName) != NULL)
-    {
-        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
-            rc = PDMR3LdrGetInterfaceSymbols(pDrvIns->Internal.s.pVMR3,
-                                             pvInterface, cbInterface,
-                                             pDrvIns->pReg->szRCMod, pDrvIns->Internal.s.pDrv->pszRCSearchPath,
-                                             pszSymPrefix, pszSymList,
-                                             false /*fRing0OrRC*/);
-        else
-        {
-            AssertMsgFailed(("Not a raw-mode enabled driver\n"));
-            rc = VERR_PERMISSION_DENIED;
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'drv' and contain the driver name!\n",
-                         pszSymPrefix, pDrvIns->pReg->szName));
-        rc = VERR_INVALID_NAME;
-    }
-
-    LogFlow(("pdmR3DrvHlp_LdrGetRCInterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
-             pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnLdrGetR0InterfaceSymbols} */
-static DECLCALLBACK(int) pdmR3DrvHlp_LdrGetR0InterfaceSymbols(PPDMDRVINS pDrvIns, void *pvInterface, size_t cbInterface,
-                                                              const char *pszSymPrefix, const char *pszSymList)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
-    LogFlow(("pdmR3DrvHlp_LdrGetR0InterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
-
-    int rc;
-    if (   strncmp(pszSymPrefix, "drv", 3) == 0
-        && RTStrIStr(pszSymPrefix + 3, pDrvIns->pReg->szName) != NULL)
-    {
-        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
-            rc = PDMR3LdrGetInterfaceSymbols(pDrvIns->Internal.s.pVMR3,
-                                             pvInterface, cbInterface,
-                                             pDrvIns->pReg->szR0Mod, pDrvIns->Internal.s.pDrv->pszR0SearchPath,
-                                             pszSymPrefix, pszSymList,
-                                             true /*fRing0OrRC*/);
-        else
-        {
-            AssertMsgFailed(("Not a ring-0 enabled driver\n"));
-            rc = VERR_PERMISSION_DENIED;
-        }
-    }
-    else
-    {
-        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'drv' and contain the driver name!\n",
-                         pszSymPrefix, pDrvIns->pReg->szName));
-        rc = VERR_INVALID_NAME;
-    }
-
-    LogFlow(("pdmR3DrvHlp_LdrGetR0InterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
-             pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnCritSectInit} */
-static DECLCALLBACK(int) pdmR3DrvHlp_CritSectInit(PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect,
-                                                  RT_SRC_POS_DECL, const char *pszName)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    PVM pVM = pDrvIns->Internal.s.pVMR3;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3DrvHlp_CritSectInit: caller='%s'/%d: pCritSect=%p pszName=%s\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, pCritSect, pszName));
-
-    int rc = pdmR3CritSectInitDriver(pVM, pDrvIns, pCritSect, RT_SRC_POS_ARGS, "%s_%u", pszName, pDrvIns->iInstance);
-
-    LogFlow(("pdmR3DrvHlp_CritSectInit: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
-             pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnCallR0} */
-static DECLCALLBACK(int) pdmR3DrvHlp_CallR0(PPDMDRVINS pDrvIns, uint32_t uOperation, uint64_t u64Arg)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    PVM pVM = pDrvIns->Internal.s.pVMR3;
-    LogFlow(("pdmR3DrvHlp_CallR0: caller='%s'/%d: uOperation=%#x u64Arg=%#RX64\n",
-             pDrvIns->pReg->szName, pDrvIns->iInstance, uOperation, u64Arg));
-
-    /*
-     * Lazy resolve the ring-0 entry point.
-     */
-    int rc = VINF_SUCCESS;
-    PFNPDMDRVREQHANDLERR0 pfnReqHandlerR0 = pDrvIns->Internal.s.pfnReqHandlerR0;
-    if (RT_UNLIKELY(pfnReqHandlerR0 == NIL_RTR0PTR))
-    {
-        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
-        {
-            char szSymbol[          sizeof("drvR0") + sizeof(pDrvIns->pReg->szName) + sizeof("ReqHandler")];
-            strcat(strcat(strcpy(szSymbol, "drvR0"),         pDrvIns->pReg->szName),         "ReqHandler");
-            szSymbol[sizeof("drvR0") - 1] = RT_C_TO_UPPER(szSymbol[sizeof("drvR0") - 1]);
-
-            rc = PDMR3LdrGetSymbolR0Lazy(pVM, pDrvIns->pReg->szR0Mod, pDrvIns->Internal.s.pDrv->pszR0SearchPath, szSymbol,
-                                         &pfnReqHandlerR0);
-            if (RT_SUCCESS(rc))
-                pDrvIns->Internal.s.pfnReqHandlerR0 = pfnReqHandlerR0;
-            else
-                pfnReqHandlerR0 = NIL_RTR0PTR;
-        }
-        else
-            rc = VERR_ACCESS_DENIED;
-    }
-    if (RT_LIKELY(pfnReqHandlerR0 != NIL_RTR0PTR))
-    {
-        /*
-         * Make the ring-0 call.
-         */
-        PDMDRIVERCALLREQHANDLERREQ Req;
-        Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
-        Req.Hdr.cbReq       = sizeof(Req);
-        Req.pDrvInsR0       = PDMDRVINS_2_R0PTR(pDrvIns);
-        Req.uOperation      = uOperation;
-        Req.u32Alignment    = 0;
-        Req.u64Arg          = u64Arg;
-        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER, 0, &Req.Hdr);
-    }
-
-    LogFlow(("pdmR3DrvHlp_CallR0: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
-             pDrvIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnFTSetCheckpoint} */
-static DECLCALLBACK(int) pdmR3DrvHlp_FTSetCheckpoint(PPDMDRVINS pDrvIns, FTMCHECKPOINTTYPE enmType)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    return FTMSetCheckpoint(pDrvIns->Internal.s.pVMR3, enmType);
-}
-
-
-/** @interface_method_impl{PDMDRVHLP,pfnBlkCacheRetain} */
-static DECLCALLBACK(int) pdmR3DrvHlp_BlkCacheRetain(PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
-                                                    PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
-                                                    PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
-                                                    const char *pcszId)
-{
-    PDMDRV_ASSERT_DRVINS(pDrvIns);
-    return PDMR3BlkCacheRetainDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppBlkCache,
-                                     pfnXferComplete, pfnXferEnqueue, pcszId);
-}
-
-
-/**
- * The driver helper structure.
- */
-const PDMDRVHLPR3 g_pdmR3DrvHlp =
-{
-    PDM_DRVHLPR3_VERSION,
-    pdmR3DrvHlp_Attach,
-    pdmR3DrvHlp_Detach,
-    pdmR3DrvHlp_DetachSelf,
-    pdmR3DrvHlp_MountPrepare,
-    pdmR3DrvHlp_AssertEMT,
-    pdmR3DrvHlp_AssertOther,
-    pdmR3DrvHlp_VMSetError,
-    pdmR3DrvHlp_VMSetErrorV,
-    pdmR3DrvHlp_VMSetRuntimeError,
-    pdmR3DrvHlp_VMSetRuntimeErrorV,
-    pdmR3DrvHlp_VMState,
-    pdmR3DrvHlp_VMTeleportedAndNotFullyResumedYet,
-    pdmR3DrvHlp_GetSupDrvSession,
-    pdmR3DrvHlp_QueueCreate,
-    pdmR3DrvHlp_TMGetVirtualFreq,
-    pdmR3DrvHlp_TMGetVirtualTime,
-    pdmR3DrvHlp_TMTimerCreate,
-    pdmR3DrvHlp_SSMRegister,
-    pdmR3DrvHlp_SSMDeregister,
-    pdmR3DrvHlp_DBGFInfoRegister,
-    pdmR3DrvHlp_DBGFInfoDeregister,
-    pdmR3DrvHlp_STAMRegister,
-    pdmR3DrvHlp_STAMRegisterF,
-    pdmR3DrvHlp_STAMRegisterV,
-    pdmR3DrvHlp_STAMDeregister,
-    pdmR3DrvHlp_SUPCallVMMR0Ex,
-    pdmR3DrvHlp_USBRegisterHub,
-    pdmR3DrvHlp_SetAsyncNotification,
-    pdmR3DrvHlp_AsyncNotificationCompleted,
-    pdmR3DrvHlp_ThreadCreate,
-    pdmR3DrvHlp_AsyncCompletionTemplateCreate,
-    pdmR3DrvHlp_LdrGetRCInterfaceSymbols,
-    pdmR3DrvHlp_LdrGetR0InterfaceSymbols,
-    pdmR3DrvHlp_CritSectInit,
-    pdmR3DrvHlp_CallR0,
-    pdmR3DrvHlp_FTSetCheckpoint,
-    pdmR3DrvHlp_BlkCacheRetain,
-    PDM_DRVHLPR3_VERSION /* u32TheEnd */
-};
-
-/** @} */
Index: unk/src/VBox/VMM/PDMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/PDMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,1192 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___PDMInternal_h
-#define ___PDMInternal_h
-
-#include <VBox/types.h>
-#include <VBox/param.h>
-#include <VBox/cfgm.h>
-#include <VBox/stam.h>
-#include <VBox/vusb.h>
-#include <VBox/pdmasynccompletion.h>
-#include <VBox/pdmblkcache.h>
-#include <VBox/pdmcommon.h>
-#include <iprt/assert.h>
-#include <iprt/critsect.h>
-#ifdef IN_RING3
-# include <iprt/thread.h>
-#endif
-
-RT_C_DECLS_BEGIN
-
-
-/** @defgroup grp_pdm_int       Internal
- * @ingroup grp_pdm
- * @internal
- * @{
- */
-
-/** @def PDM_WITH_R3R0_CRIT_SECT
- * Enables or disabled ring-3/ring-0 critical sections. */
-#if defined(DOXYGEN_RUNNING) || 1
-# define PDM_WITH_R3R0_CRIT_SECT
-#endif
-
-/** @def PDMCRITSECT_STRICT
- * Enables/disables PDM critsect strictness like deadlock detection. */
-#if (defined(RT_LOCK_STRICT) && defined(IN_RING3)) || defined(DOXYGEN_RUNNING)
-# define PDMCRITSECT_STRICT
-#endif
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-
-/** Pointer to a PDM Device. */
-typedef struct PDMDEV *PPDMDEV;
-/** Pointer to a pointer to a PDM Device. */
-typedef PPDMDEV *PPPDMDEV;
-
-/** Pointer to a PDM USB Device. */
-typedef struct PDMUSB *PPDMUSB;
-/** Pointer to a pointer to a PDM USB Device. */
-typedef PPDMUSB *PPPDMUSB;
-
-/** Pointer to a PDM Driver. */
-typedef struct PDMDRV *PPDMDRV;
-/** Pointer to a pointer to a PDM Driver. */
-typedef PPDMDRV *PPPDMDRV;
-
-/** Pointer to a PDM Logical Unit. */
-typedef struct PDMLUN *PPDMLUN;
-/** Pointer to a pointer to a PDM Logical Unit. */
-typedef PPDMLUN *PPPDMLUN;
-
-/** Pointer to a PDM PCI Bus instance. */
-typedef struct PDMPCIBUS *PPDMPCIBUS;
-/** Pointer to a DMAC instance. */
-typedef struct PDMDMAC *PPDMDMAC;
-/** Pointer to a RTC instance. */
-typedef struct PDMRTC *PPDMRTC;
-
-/** Pointer to an USB HUB registration record. */
-typedef struct PDMUSBHUB *PPDMUSBHUB;
-
-/**
- * Supported asynchronous completion endpoint classes.
- */
-typedef enum PDMASYNCCOMPLETIONEPCLASSTYPE
-{
-    /** File class. */
-    PDMASYNCCOMPLETIONEPCLASSTYPE_FILE = 0,
-    /** Number of supported classes. */
-    PDMASYNCCOMPLETIONEPCLASSTYPE_MAX,
-    /** 32bit hack. */
-    PDMASYNCCOMPLETIONEPCLASSTYPE_32BIT_HACK = 0x7fffffff
-} PDMASYNCCOMPLETIONEPCLASSTYPE;
-
-/**
- * Private device instance data.
- */
-typedef struct PDMDEVINSINT
-{
-    /** Pointer to the next instance (HC Ptr).
-     * (Head is pointed to by PDM::pDevInstances.) */
-    R3PTRTYPE(PPDMDEVINS)           pNextR3;
-    /** Pointer to the next per device instance (HC Ptr).
-     * (Head is pointed to by PDMDEV::pInstances.) */
-    R3PTRTYPE(PPDMDEVINS)           pPerDeviceNextR3;
-    /** Pointer to device structure - HC Ptr. */
-    R3PTRTYPE(PPDMDEV)              pDevR3;
-    /** Pointer to the list of logical units associated with the device. (FIFO) */
-    R3PTRTYPE(PPDMLUN)              pLunsR3;
-    /** Pointer to the asynchronous notification callback set while in
-     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
-    R3PTRTYPE(PFNPDMDEVASYNCNOTIFY) pfnAsyncNotify;
-    /** Configuration handle to the instance node. */
-    R3PTRTYPE(PCFGMNODE)            pCfgHandle;
-
-    /** R3 pointer to the VM this instance was created for. */
-    PVMR3                           pVMR3;
-    /** R3 pointer to associated PCI device structure. */
-    R3PTRTYPE(struct PCIDevice *)   pPciDeviceR3;
-    /** R3 pointer to associated PCI bus structure. */
-    R3PTRTYPE(PPDMPCIBUS)           pPciBusR3;
-
-    /** R0 pointer to the VM this instance was created for. */
-    PVMR0                           pVMR0;
-    /** R0 pointer to associated PCI device structure. */
-    R0PTRTYPE(struct PCIDevice *)   pPciDeviceR0;
-    /** R0 pointer to associated PCI bus structure. */
-    R0PTRTYPE(PPDMPCIBUS)           pPciBusR0;
-
-    /** RC pointer to the VM this instance was created for. */
-    PVMRC                           pVMRC;
-    /** RC pointer to associated PCI device structure. */
-    RCPTRTYPE(struct PCIDevice *)   pPciDeviceRC;
-    /** RC pointer to associated PCI bus structure. */
-    RCPTRTYPE(PPDMPCIBUS)           pPciBusRC;
-
-    /** Flags, see PDMDEVINSINT_FLAGS_XXX. */
-    uint32_t                        fIntFlags;
-} PDMDEVINSINT;
-
-/** @name PDMDEVINSINT::fIntFlags
- * @{ */
-/** Used by pdmR3Load to mark device instances it found in the saved state. */
-#define PDMDEVINSINT_FLAGS_FOUND         RT_BIT_32(0)
-/** Indicates that the device hasn't been powered on or resumed.
- * This is used by PDMR3PowerOn, PDMR3Resume, PDMR3Suspend and PDMR3PowerOff
- * to make sure each device gets exactly one notification for each of those
- * events.  PDMR3Resume and PDMR3PowerOn also makes use of it to bail out on
- * a failure (already resumed/powered-on devices are suspended). */
-#define PDMDEVINSINT_FLAGS_SUSPENDED     RT_BIT_32(1)
-/** Indicates that the device has been reset already.  Used by PDMR3Reset. */
-#define PDMDEVINSINT_FLAGS_RESET         RT_BIT_32(2)
-/** @} */
-
-
-/**
- * Private USB device instance data.
- */
-typedef struct PDMUSBINSINT
-{
-    /** The UUID of this instance. */
-    RTUUID                          Uuid;
-    /** Pointer to the next instance.
-     * (Head is pointed to by PDM::pUsbInstances.) */
-    R3PTRTYPE(PPDMUSBINS)           pNext;
-    /** Pointer to the next per USB device instance.
-     * (Head is pointed to by PDMUSB::pInstances.) */
-    R3PTRTYPE(PPDMUSBINS)           pPerDeviceNext;
-
-    /** Pointer to device structure. */
-    R3PTRTYPE(PPDMUSB)              pUsbDev;
-
-    /** Pointer to the VM this instance was created for. */
-    PVMR3                           pVM;
-    /** Pointer to the list of logical units associated with the device. (FIFO) */
-    R3PTRTYPE(PPDMLUN)              pLuns;
-    /** The per instance device configuration. */
-    R3PTRTYPE(PCFGMNODE)            pCfg;
-    /** Same as pCfg if the configuration should be deleted when detaching the device. */
-    R3PTRTYPE(PCFGMNODE)            pCfgDelete;
-    /** The global device configuration. */
-    R3PTRTYPE(PCFGMNODE)            pCfgGlobal;
-
-    /** Pointer to the USB hub this device is attached to.
-     * This is NULL if the device isn't connected to any HUB. */
-    R3PTRTYPE(PPDMUSBHUB)           pHub;
-    /** The port number that we're connected to. */
-    uint32_t                        iPort;
-    /** Indicates that the USB device hasn't been powered on or resumed.
-     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
-    bool                            fVMSuspended;
-    /** Indicates that the USB device has been reset. */
-    bool                            fVMReset;
-    /** Pointer to the asynchronous notification callback set while in
-     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
-    R3PTRTYPE(PFNPDMUSBASYNCNOTIFY) pfnAsyncNotify;
-} PDMUSBINSINT;
-
-
-/**
- * Private driver instance data.
- */
-typedef struct PDMDRVINSINT
-{
-    /** Pointer to the driver instance above.
-     * This is NULL for the topmost drive. */
-    R3PTRTYPE(PPDMDRVINS)           pUp;
-    /** Pointer to the driver instance below.
-     * This is NULL for the bottommost driver. */
-    R3PTRTYPE(PPDMDRVINS)           pDown;
-    /** Pointer to the logical unit this driver chained on. */
-    R3PTRTYPE(PPDMLUN)              pLun;
-    /** Pointer to driver structure from which this was instantiated. */
-    R3PTRTYPE(PPDMDRV)              pDrv;
-    /** Pointer to the VM this instance was created for, ring-3 context. */
-    PVMR3                           pVMR3;
-    /** Pointer to the VM this instance was created for, ring-0 context. */
-    PVMR0                           pVMR0;
-    /** Pointer to the VM this instance was created for, raw-mode context. */
-    PVMRC                           pVMRC;
-    /** Flag indicating that the driver is being detached and destroyed.
-     * (Helps detect potential recursive detaching.) */
-    bool                            fDetaching;
-    /** Indicates that the driver hasn't been powered on or resumed.
-     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
-    bool                            fVMSuspended;
-    /** Indicates that the driver has been reset already. */
-    bool                            fVMReset;
-    /** Set if allocated on the hyper heap, false if on the ring-3 heap. */
-    bool                            fHyperHeap;
-    /** Pointer to the asynchronous notification callback set while in
-     * PDMUSBREG::pfnVMSuspend or PDMUSBREG::pfnVMPowerOff. */
-    R3PTRTYPE(PFNPDMDRVASYNCNOTIFY) pfnAsyncNotify;
-    /** Configuration handle to the instance node. */
-    R3PTRTYPE(PCFGMNODE)            pCfgHandle;
-    /** Pointer to the ring-0 request handler function. */
-    PFNPDMDRVREQHANDLERR0           pfnReqHandlerR0;
-} PDMDRVINSINT;
-
-
-/**
- * Private critical section data.
- */
-typedef struct PDMCRITSECTINT
-{
-    /** The critical section core which is shared with IPRT. */
-    RTCRITSECT                      Core;
-    /** Pointer to the next critical section.
-     * This chain is used for relocating pVMRC and device cleanup. */
-    R3PTRTYPE(struct PDMCRITSECTINT *) pNext;
-    /** Owner identifier.
-     * This is pDevIns if the owner is a device. Similarly for a driver or service.
-     * PDMR3CritSectInit() sets this to point to the critsect itself. */
-    RTR3PTR                         pvKey;
-    /** Pointer to the VM - R3Ptr. */
-    PVMR3                           pVMR3;
-    /** Pointer to the VM - R0Ptr. */
-    PVMR0                           pVMR0;
-    /** Pointer to the VM - GCPtr. */
-    PVMRC                           pVMRC;
-    /** Alignment padding. */
-    uint32_t                        padding;
-    /** Event semaphore that is scheduled to be signaled upon leaving the
-     * critical section. This is Ring-3 only of course. */
-    RTSEMEVENT                      EventToSignal;
-    /** The lock name. */
-    R3PTRTYPE(const char *)         pszName;
-    /** R0/RC lock contention. */
-    STAMCOUNTER                     StatContentionRZLock;
-    /** R0/RC unlock contention. */
-    STAMCOUNTER                     StatContentionRZUnlock;
-    /** R3 lock contention. */
-    STAMCOUNTER                     StatContentionR3;
-    /** Profiling the time the section is locked. */
-    STAMPROFILEADV                  StatLocked;
-} PDMCRITSECTINT;
-AssertCompileMemberAlignment(PDMCRITSECTINT, StatContentionRZLock, 8);
-/** Pointer to private critical section data. */
-typedef PDMCRITSECTINT *PPDMCRITSECTINT;
-
-/** Indicates that the critical section is queued for unlock.
- * PDMCritSectIsOwner and PDMCritSectIsOwned optimizations. */
-#define PDMCRITSECT_FLAGS_PENDING_UNLOCK    RT_BIT_32(17)
-
-
-/**
- * The usual device/driver/internal/external stuff.
- */
-typedef enum
-{
-    /** The usual invalid entry. */
-    PDMTHREADTYPE_INVALID = 0,
-    /** Device type. */
-    PDMTHREADTYPE_DEVICE,
-    /** USB Device type. */
-    PDMTHREADTYPE_USB,
-    /** Driver type. */
-    PDMTHREADTYPE_DRIVER,
-    /** Internal type. */
-    PDMTHREADTYPE_INTERNAL,
-    /** External type. */
-    PDMTHREADTYPE_EXTERNAL,
-    /** The usual 32-bit hack. */
-    PDMTHREADTYPE_32BIT_HACK = 0x7fffffff
-} PDMTHREADTYPE;
-
-
-/**
- * The internal structure for the thread.
- */
-typedef struct PDMTHREADINT
-{
-    /** The VM pointer. */
-    PVMR3                           pVM;
-    /** The event semaphore the thread blocks on when not running. */
-    RTSEMEVENTMULTI                 BlockEvent;
-    /** The event semaphore the thread sleeps on while running. */
-    RTSEMEVENTMULTI                 SleepEvent;
-    /** Pointer to the next thread. */
-    R3PTRTYPE(struct PDMTHREAD *)   pNext;
-    /** The thread type. */
-    PDMTHREADTYPE                   enmType;
-} PDMTHREADINT;
-
-
-
-/* Must be included after PDMDEVINSINT is defined. */
-#define PDMDEVINSINT_DECLARED
-#define PDMUSBINSINT_DECLARED
-#define PDMDRVINSINT_DECLARED
-#define PDMCRITSECTINT_DECLARED
-#define PDMTHREADINT_DECLARED
-#ifdef ___VBox_pdm_h
-# error "Invalid header PDM order. Include PDMInternal.h before VBox/pdm.h!"
-#endif
-RT_C_DECLS_END
-#include <VBox/pdm.h>
-RT_C_DECLS_BEGIN
-
-/**
- * PDM Logical Unit.
- *
- * This typically the representation of a physical port on a
- * device, like for instance the PS/2 keyboard port on the
- * keyboard controller device. The LUNs are chained on the
- * device the belong to (PDMDEVINSINT::pLunsR3).
- */
-typedef struct PDMLUN
-{
-    /** The LUN - The Logical Unit Number. */
-    RTUINT                          iLun;
-    /** Pointer to the next LUN. */
-    PPDMLUN                         pNext;
-    /** Pointer to the top driver in the driver chain. */
-    PPDMDRVINS                      pTop;
-    /** Pointer to the bottom driver in the driver chain. */
-    PPDMDRVINS                      pBottom;
-    /** Pointer to the device instance which the LUN belongs to.
-     * Either this is set or pUsbIns is set. Both is never set at the same time. */
-    PPDMDEVINS                      pDevIns;
-    /** Pointer to the USB device instance which the LUN belongs to. */
-    PPDMUSBINS                      pUsbIns;
-    /** Pointer to the device base interface. */
-    PPDMIBASE                       pBase;
-    /** Description of this LUN. */
-    const char                     *pszDesc;
-} PDMLUN;
-
-
-/**
- * PDM Device.
- */
-typedef struct PDMDEV
-{
-    /** Pointer to the next device (R3 Ptr). */
-    R3PTRTYPE(PPDMDEV)              pNext;
-    /** Device name length. (search optimization) */
-    RTUINT                          cchName;
-    /** Registration structure. */
-    R3PTRTYPE(const struct PDMDEVREG *) pReg;
-    /** Number of instances. */
-    uint32_t                        cInstances;
-    /** Pointer to chain of instances (R3 Ptr). */
-    PPDMDEVINSR3                    pInstances;
-    /** The search path for raw-mode context modules (';' as separator). */
-    char                           *pszRCSearchPath;
-    /** The search path for ring-0 context modules (';' as separator). */
-    char                           *pszR0SearchPath;
-} PDMDEV;
-
-
-/**
- * PDM USB Device.
- */
-typedef struct PDMUSB
-{
-    /** Pointer to the next device (R3 Ptr). */
-    R3PTRTYPE(PPDMUSB)              pNext;
-    /** Device name length. (search optimization) */
-    RTUINT                          cchName;
-    /** Registration structure. */
-    R3PTRTYPE(const struct PDMUSBREG *) pReg;
-    /** Next instance number. */
-    uint32_t                        iNextInstance;
-    /** Pointer to chain of instances (R3 Ptr). */
-    R3PTRTYPE(PPDMUSBINS)           pInstances;
-} PDMUSB;
-
-
-/**
- * PDM Driver.
- */
-typedef struct PDMDRV
-{
-    /** Pointer to the next device. */
-    PPDMDRV                         pNext;
-    /** Registration structure. */
-    const struct PDMDRVREG *        pReg;
-    /** Current number of instances. */
-    uint32_t                        cInstances;
-    /** The next instance number. */
-    uint32_t                        iNextInstance;
-    /** The search path for raw-mode context modules (';' as separator). */
-    char                           *pszRCSearchPath;
-    /** The search path for ring-0 context modules (';' as separator). */
-    char                           *pszR0SearchPath;
-} PDMDRV;
-
-
-/**
- * PDM registered PIC device.
- */
-typedef struct PDMPIC
-{
-    /** Pointer to the PIC device instance - R3. */
-    PPDMDEVINSR3                    pDevInsR3;
-    /** @copydoc PDMPICREG::pfnSetIrqR3 */
-    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
-    /** @copydoc PDMPICREG::pfnGetInterruptR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnGetInterruptR3,(PPDMDEVINS pDevIns));
-
-    /** Pointer to the PIC device instance - R0. */
-    PPDMDEVINSR0                    pDevInsR0;
-    /** @copydoc PDMPICREG::pfnSetIrqR3 */
-    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
-    /** @copydoc PDMPICREG::pfnGetInterruptR3 */
-    DECLR0CALLBACKMEMBER(int,       pfnGetInterruptR0,(PPDMDEVINS pDevIns));
-
-    /** Pointer to the PIC device instance - RC. */
-    PPDMDEVINSRC                    pDevInsRC;
-    /** @copydoc PDMPICREG::pfnSetIrqR3 */
-    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
-    /** @copydoc PDMPICREG::pfnGetInterruptR3 */
-    DECLRCCALLBACKMEMBER(int,       pfnGetInterruptRC,(PPDMDEVINS pDevIns));
-    /** Alignment padding. */
-    RTRCPTR                         RCPtrPadding;
-} PDMPIC;
-
-
-/**
- * PDM registered APIC device.
- */
-typedef struct PDMAPIC
-{
-    /** Pointer to the APIC device instance - R3 Ptr. */
-    PPDMDEVINSR3                    pDevInsR3;
-    /** @copydoc PDMAPICREG::pfnGetInterruptR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnGetInterruptR3,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnHasPendingIrqR3 */
-    DECLR3CALLBACKMEMBER(bool,      pfnHasPendingIrqR3,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnSetBaseR3 */
-    DECLR3CALLBACKMEMBER(void,      pfnSetBaseR3,(PPDMDEVINS pDevIns, uint64_t u64Base));
-    /** @copydoc PDMAPICREG::pfnGetBaseR3 */
-    DECLR3CALLBACKMEMBER(uint64_t,  pfnGetBaseR3,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnSetTPRR3 */
-    DECLR3CALLBACKMEMBER(void,      pfnSetTPRR3,(PPDMDEVINS pDevIns, VMCPUID idCpu, uint8_t u8TPR));
-    /** @copydoc PDMAPICREG::pfnGetTPRR3 */
-    DECLR3CALLBACKMEMBER(uint8_t,   pfnGetTPRR3,(PPDMDEVINS pDevIns, VMCPUID idCpu));
-    /** @copydoc PDMAPICREG::pfnWriteMSRR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnWriteMSRR3, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t u64Value));
-    /** @copydoc PDMAPICREG::pfnReadMSRR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnReadMSRR3, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t *pu64Value));
-    /** @copydoc PDMAPICREG::pfnBusDeliverR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnBusDeliverR3,(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
-                                                     uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode));
-    /** @copydoc PDMAPICREG::pfnLocalInterruptR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnLocalInterruptR3,(PPDMDEVINS pDevIns, uint8_t u8Pin, uint8_t u8Level));
-
-    /** Pointer to the APIC device instance - R0 Ptr. */
-    PPDMDEVINSR0                    pDevInsR0;
-    /** @copydoc PDMAPICREG::pfnGetInterruptR3 */
-    DECLR0CALLBACKMEMBER(int,       pfnGetInterruptR0,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnHasPendingIrqR3 */
-    DECLR0CALLBACKMEMBER(bool,      pfnHasPendingIrqR0,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnSetBaseR3 */
-    DECLR0CALLBACKMEMBER(void,      pfnSetBaseR0,(PPDMDEVINS pDevIns, uint64_t u64Base));
-    /** @copydoc PDMAPICREG::pfnGetBaseR3 */
-    DECLR0CALLBACKMEMBER(uint64_t,  pfnGetBaseR0,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnSetTPRR3 */
-    DECLR0CALLBACKMEMBER(void,      pfnSetTPRR0,(PPDMDEVINS pDevIns, VMCPUID idCpu, uint8_t u8TPR));
-    /** @copydoc PDMAPICREG::pfnGetTPRR3 */
-    DECLR0CALLBACKMEMBER(uint8_t,   pfnGetTPRR0,(PPDMDEVINS pDevIns, VMCPUID idCpu));
-     /** @copydoc PDMAPICREG::pfnWriteMSRR3 */
-    DECLR0CALLBACKMEMBER(uint32_t,  pfnWriteMSRR0, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t u64Value));
-    /** @copydoc PDMAPICREG::pfnReadMSRR3 */
-    DECLR0CALLBACKMEMBER(uint32_t,  pfnReadMSRR0, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t *pu64Value));
-    /** @copydoc PDMAPICREG::pfnBusDeliverR3 */
-    DECLR0CALLBACKMEMBER(int,       pfnBusDeliverR0,(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
-                                                     uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode));
-    /** @copydoc PDMAPICREG::pfnLocalInterruptR3 */
-    DECLR0CALLBACKMEMBER(int,       pfnLocalInterruptR0,(PPDMDEVINS pDevIns, uint8_t u8Pin, uint8_t u8Level));
-
-    /** Pointer to the APIC device instance - RC Ptr. */
-    PPDMDEVINSRC                    pDevInsRC;
-    /** @copydoc PDMAPICREG::pfnGetInterruptR3 */
-    DECLRCCALLBACKMEMBER(int,       pfnGetInterruptRC,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnHasPendingIrqR3 */
-    DECLRCCALLBACKMEMBER(bool,      pfnHasPendingIrqRC,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnSetBaseR3 */
-    DECLRCCALLBACKMEMBER(void,      pfnSetBaseRC,(PPDMDEVINS pDevIns, uint64_t u64Base));
-    /** @copydoc PDMAPICREG::pfnGetBaseR3 */
-    DECLRCCALLBACKMEMBER(uint64_t,  pfnGetBaseRC,(PPDMDEVINS pDevIns));
-    /** @copydoc PDMAPICREG::pfnSetTPRR3 */
-    DECLRCCALLBACKMEMBER(void,      pfnSetTPRRC,(PPDMDEVINS pDevIns, VMCPUID idCpu, uint8_t u8TPR));
-    /** @copydoc PDMAPICREG::pfnGetTPRR3 */
-    DECLRCCALLBACKMEMBER(uint8_t,   pfnGetTPRRC,(PPDMDEVINS pDevIns, VMCPUID idCpu));
-    /** @copydoc PDMAPICREG::pfnWriteMSRR3 */
-    DECLRCCALLBACKMEMBER(uint32_t,  pfnWriteMSRRC, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t u64Value));
-    /** @copydoc PDMAPICREG::pfnReadMSRR3 */
-    DECLRCCALLBACKMEMBER(uint32_t,  pfnReadMSRRC, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t *pu64Value));
-    /** @copydoc PDMAPICREG::pfnBusDeliverR3 */
-    DECLRCCALLBACKMEMBER(int,       pfnBusDeliverRC,(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
-                                                     uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode));
-    /** @copydoc PDMAPICREG::pfnLocalInterruptR3 */
-    DECLRCCALLBACKMEMBER(int,       pfnLocalInterruptRC,(PPDMDEVINS pDevIns, uint8_t u8Pin, uint8_t u8Level));
-    RTRCPTR                         RCPtrAlignment;
-
-} PDMAPIC;
-
-
-/**
- * PDM registered I/O APIC device.
- */
-typedef struct PDMIOAPIC
-{
-    /** Pointer to the APIC device instance - R3 Ptr. */
-    PPDMDEVINSR3                    pDevInsR3;
-    /** @copydoc PDMIOAPICREG::pfnSetIrqR3 */
-    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
-    /** @copydoc PDMIOAPICREG::pfnSendMsiR3 */
-    DECLR3CALLBACKMEMBER(void,      pfnSendMsiR3,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue));
-
-    /** Pointer to the PIC device instance - R0. */
-    PPDMDEVINSR0                    pDevInsR0;
-    /** @copydoc PDMIOAPICREG::pfnSetIrqR3 */
-    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
-    /** @copydoc PDMIOAPICREG::pfnSendMsiR3 */
-    DECLR0CALLBACKMEMBER(void,      pfnSendMsiR0,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue));
-
-    /** Pointer to the APIC device instance - RC Ptr. */
-    PPDMDEVINSRC                    pDevInsRC;
-    /** @copydoc PDMIOAPICREG::pfnSetIrqR3 */
-    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
-     /** @copydoc PDMIOAPICREG::pfnSendMsiR3 */
-    DECLRCCALLBACKMEMBER(void,      pfnSendMsiRC,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue));
-
-    uint8_t                         Alignment[4];
-} PDMIOAPIC;
-
-/** Maximum number of PCI busses for a VM. */
-#define PDM_PCI_BUSSES_MAX 8
-
-/**
- * PDM PCI Bus instance.
- */
-typedef struct PDMPCIBUS
-{
-    /** PCI bus number. */
-    RTUINT          iBus;
-    RTUINT          uPadding0; /**< Alignment padding.*/
-
-    /** Pointer to PCI Bus device instance. */
-    PPDMDEVINSR3                    pDevInsR3;
-    /** @copydoc PDMPCIBUSREG::pfnSetIrqR3 */
-    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iIrq, int iLevel));
-    /** @copydoc PDMPCIBUSREG::pfnRegisterR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnRegisterR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, const char *pszName, int iDev));
-    /** @copydoc PDMPCIBUSREG::pfnPCIRegisterMsiR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnRegisterMsiR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PPDMMSIREG pMsiReg));
-    /** @copydoc PDMPCIBUSREG::pfnIORegionRegisterR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnIORegionRegisterR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iRegion, uint32_t cbRegion,
-                                                           PCIADDRESSSPACE enmType, PFNPCIIOREGIONMAP pfnCallback));
-    /** @copydoc PDMPCIBUSREG::pfnSetConfigCallbacksR3 */
-    DECLR3CALLBACKMEMBER(void,      pfnSetConfigCallbacksR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PFNPCICONFIGREAD pfnRead,
-                                                             PPFNPCICONFIGREAD ppfnReadOld, PFNPCICONFIGWRITE pfnWrite, PPFNPCICONFIGWRITE ppfnWriteOld));
-    /** @copydoc PDMPCIBUSREG::pfnSaveExecR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnSaveExecR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PSSMHANDLE pSSMHandle));
-    /** @copydoc PDMPCIBUSREG::pfnLoadExecR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnLoadExecR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PSSMHANDLE pSSMHandle));
-    /** @copydoc PDMPCIBUSREG::pfnFakePCIBIOSR3 */
-    DECLR3CALLBACKMEMBER(int,       pfnFakePCIBIOSR3,(PPDMDEVINS pDevIns));
-
-    /** Pointer to the PIC device instance - R0. */
-    R0PTRTYPE(PPDMDEVINS)           pDevInsR0;
-    /** @copydoc PDMPCIBUSREG::pfnSetIrqR3 */
-    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iIrq, int iLevel));
-
-    /** Pointer to PCI Bus device instance. */
-    PPDMDEVINSRC                    pDevInsRC;
-    /** @copydoc PDMPCIBUSREG::pfnSetIrqR3 */
-    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iIrq, int iLevel));
-} PDMPCIBUS;
-
-
-#ifdef IN_RING3
-/**
- * PDM registered DMAC (DMA Controller) device.
- */
-typedef struct PDMDMAC
-{
-    /** Pointer to the DMAC device instance. */
-    PPDMDEVINSR3                    pDevIns;
-    /** Copy of the registration structure. */
-    PDMDMACREG                      Reg;
-} PDMDMAC;
-
-
-/**
- * PDM registered RTC (Real Time Clock) device.
- */
-typedef struct PDMRTC
-{
-    /** Pointer to the RTC device instance. */
-    PPDMDEVINSR3                    pDevIns;
-    /** Copy of the registration structure. */
-    PDMRTCREG                       Reg;
-} PDMRTC;
-
-#endif /* IN_RING3 */
-
-/**
- * Module type.
- */
-typedef enum PDMMODTYPE
-{
-    /** Raw-mode (RC) context module. */
-    PDMMOD_TYPE_RC,
-    /** Ring-0 (host) context module. */
-    PDMMOD_TYPE_R0,
-    /** Ring-3 (host) context module. */
-    PDMMOD_TYPE_R3
-} PDMMODTYPE;
-
-
-/** The module name length including the terminator. */
-#define PDMMOD_NAME_LEN             32
-
-/**
- * Loaded module instance.
- */
-typedef struct PDMMOD
-{
-    /** Module name. This is used for referring to
-     * the module internally, sort of like a handle. */
-    char                            szName[PDMMOD_NAME_LEN];
-    /** Module type. */
-    PDMMODTYPE                      eType;
-    /** Loader module handle. Not used for R0 modules. */
-    RTLDRMOD                        hLdrMod;
-    /** Loaded address.
-     * This is the 'handle' for R0 modules. */
-    RTUINTPTR                       ImageBase;
-    /** Old loaded address.
-     * This is used during relocation of GC modules. Not used for R0 modules. */
-    RTUINTPTR                       OldImageBase;
-    /** Where the R3 HC bits are stored.
-     * This can be equal to ImageBase but doesn't have to. Not used for R0 modules. */
-    void                           *pvBits;
-
-    /** Pointer to next module. */
-    struct PDMMOD                  *pNext;
-    /** Module filename. */
-    char                            szFilename[1];
-} PDMMOD;
-/** Pointer to loaded module instance. */
-typedef PDMMOD *PPDMMOD;
-
-
-
-/** Extra space in the free array. */
-#define PDMQUEUE_FREE_SLACK         16
-
-/**
- * Queue type.
- */
-typedef enum PDMQUEUETYPE
-{
-    /** Device consumer. */
-    PDMQUEUETYPE_DEV = 1,
-    /** Driver consumer. */
-    PDMQUEUETYPE_DRV,
-    /** Internal consumer. */
-    PDMQUEUETYPE_INTERNAL,
-    /** External consumer. */
-    PDMQUEUETYPE_EXTERNAL
-} PDMQUEUETYPE;
-
-/** Pointer to a PDM Queue. */
-typedef struct PDMQUEUE *PPDMQUEUE;
-
-/**
- * PDM Queue.
- */
-typedef struct PDMQUEUE
-{
-    /** Pointer to the next queue in the list. */
-    R3PTRTYPE(PPDMQUEUE)            pNext;
-    /** Type specific data. */
-    union
-    {
-        /** PDMQUEUETYPE_DEV */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMQUEUEDEV)   pfnCallback;
-            /** Pointer to the device instance owning the queue. */
-            R3PTRTYPE(PPDMDEVINS)       pDevIns;
-        } Dev;
-        /** PDMQUEUETYPE_DRV */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMQUEUEDRV)   pfnCallback;
-            /** Pointer to the driver instance owning the queue. */
-            R3PTRTYPE(PPDMDRVINS)       pDrvIns;
-        } Drv;
-        /** PDMQUEUETYPE_INTERNAL */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMQUEUEINT)   pfnCallback;
-        } Int;
-        /** PDMQUEUETYPE_EXTERNAL */
-        struct
-        {
-            /** Pointer to consumer function. */
-            R3PTRTYPE(PFNPDMQUEUEEXT)   pfnCallback;
-            /** Pointer to user argument. */
-            R3PTRTYPE(void *)           pvUser;
-        } Ext;
-    } u;
-    /** Queue type. */
-    PDMQUEUETYPE                    enmType;
-    /** The interval between checking the queue for events.
-     * The realtime timer below is used to do the waiting.
-     * If 0, the queue will use the VM_FF_PDM_QUEUE forced action. */
-    uint32_t                        cMilliesInterval;
-    /** Interval timer. Only used if cMilliesInterval is non-zero. */
-    PTMTIMERR3                      pTimer;
-    /** Pointer to the VM - R3. */
-    PVMR3                           pVMR3;
-    /** LIFO of pending items - R3. */
-    R3PTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingR3;
-    /** Pointer to the VM - R0. */
-    PVMR0                           pVMR0;
-    /** LIFO of pending items - R0. */
-    R0PTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingR0;
-    /** Pointer to the GC VM and indicator for GC enabled queue.
-     * If this is NULL, the queue cannot be used in GC.
-     */
-    PVMRC                           pVMRC;
-    /** LIFO of pending items - GC. */
-    RCPTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingRC;
-
-    /** Item size (bytes). */
-    uint32_t                        cbItem;
-    /** Number of items in the queue. */
-    uint32_t                        cItems;
-    /** Index to the free head (where we insert). */
-    uint32_t volatile               iFreeHead;
-    /** Index to the free tail (where we remove). */
-    uint32_t volatile               iFreeTail;
-
-    /** Unique queue name. */
-    R3PTRTYPE(const char *)         pszName;
-#if HC_ARCH_BITS == 32
-    RTR3PTR                         Alignment1;
-#endif
-    /** Stat: Times PDMQueueAlloc fails. */
-    STAMCOUNTER                     StatAllocFailures;
-    /** Stat: PDMQueueInsert calls. */
-    STAMCOUNTER                     StatInsert;
-    /** Stat: Queue flushes. */
-    STAMCOUNTER                     StatFlush;
-    /** Stat: Queue flushes with pending items left over. */
-    STAMCOUNTER                     StatFlushLeftovers;
-#ifdef VBOX_WITH_STATISTICS
-    /** State: Profiling the flushing. */
-    STAMPROFILE                     StatFlushPrf;
-    /** State: Pending items. */
-    uint32_t volatile               cStatPending;
-    uint32_t volatile               cAlignment;
-#endif
-
-    /** Array of pointers to free items. Variable size. */
-    struct PDMQUEUEFREEITEM
-    {
-        /** Pointer to the free item - HC Ptr. */
-        R3PTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemR3;
-        /** Pointer to the free item - HC Ptr. */
-        R0PTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemR0;
-        /** Pointer to the free item - GC Ptr. */
-        RCPTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemRC;
-#if HC_ARCH_BITS == 64
-        RTRCPTR                                 Alignment0;
-#endif
-    }                               aFreeItems[1];
-} PDMQUEUE;
-
-/** @name PDM::fQueueFlushing
- * @{ */
-/** Used to make sure only one EMT will flush the queues.
- * Set when an EMT is flushing queues, clear otherwise.  */
-#define PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT     0
-/** Indicating there are queues with items pending.
- * This is make sure we don't miss inserts happening during flushing.  The FF
- * cannot be used for this since it has to be cleared immediately to prevent
- * other EMTs from spinning. */
-#define PDM_QUEUE_FLUSH_FLAG_PENDING_BIT    1
-/** }@  */
-
-
-/**
- * Queue device helper task operation.
- */
-typedef enum PDMDEVHLPTASKOP
-{
-    /** The usual invalid 0 entry. */
-    PDMDEVHLPTASKOP_INVALID = 0,
-    /** ISASetIrq */
-    PDMDEVHLPTASKOP_ISA_SET_IRQ,
-    /** PCISetIrq */
-    PDMDEVHLPTASKOP_PCI_SET_IRQ,
-    /** PCISetIrq */
-    PDMDEVHLPTASKOP_IOAPIC_SET_IRQ,
-    /** The usual 32-bit hack. */
-    PDMDEVHLPTASKOP_32BIT_HACK = 0x7fffffff
-} PDMDEVHLPTASKOP;
-
-/**
- * Queued Device Helper Task.
- */
-typedef struct PDMDEVHLPTASK
-{
-    /** The queue item core (don't touch). */
-    PDMQUEUEITEMCORE                Core;
-    /** Pointer to the device instance (R3 Ptr). */
-    PPDMDEVINSR3                    pDevInsR3;
-    /** This operation to perform. */
-    PDMDEVHLPTASKOP                 enmOp;
-#if HC_ARCH_BITS == 64
-    uint32_t                        Alignment0;
-#endif
-    /** Parameters to the operation. */
-    union PDMDEVHLPTASKPARAMS
-    {
-        /**
-         * PDMDEVHLPTASKOP_ISA_SET_IRQ and PDMDEVHLPTASKOP_PCI_SET_IRQ.
-         */
-        struct PDMDEVHLPTASKSETIRQ
-        {
-            /** The IRQ */
-            int                     iIrq;
-            /** The new level. */
-            int                     iLevel;
-        } SetIRQ;
-    } u;
-} PDMDEVHLPTASK;
-/** Pointer to a queued Device Helper Task. */
-typedef PDMDEVHLPTASK *PPDMDEVHLPTASK;
-/** Pointer to a const queued Device Helper Task. */
-typedef const PDMDEVHLPTASK *PCPDMDEVHLPTASK;
-
-
-
-/**
- * An USB hub registration record.
- */
-typedef struct PDMUSBHUB
-{
-    /** The USB versions this hub support.
-     * Note that 1.1 hubs can take on 2.0 devices. */
-    uint32_t                        fVersions;
-    /** The number of ports on the hub. */
-    uint32_t                        cPorts;
-    /** The number of available ports (0..cPorts). */
-    uint32_t                        cAvailablePorts;
-    /** The driver instance of the hub. */
-    PPDMDRVINS                      pDrvIns;
-    /** Copy of the to the registration structure. */
-    PDMUSBHUBREG                    Reg;
-
-    /** Pointer to the next hub in the list. */
-    struct PDMUSBHUB               *pNext;
-} PDMUSBHUB;
-
-/** Pointer to a const USB HUB registration record. */
-typedef const PDMUSBHUB *PCPDMUSBHUB;
-
-/** Pointer to a PDM Async I/O template. */
-typedef struct PDMASYNCCOMPLETIONTEMPLATE *PPDMASYNCCOMPLETIONTEMPLATE;
-
-/** Pointer to the main PDM Async completion endpoint class. */
-typedef struct PDMASYNCCOMPLETIONEPCLASS *PPDMASYNCCOMPLETIONEPCLASS;
-
-/** Pointer to the global block cache structure. */
-typedef struct PDMBLKCACHEGLOBAL *PPDMBLKCACHEGLOBAL;
-
-/**
- * PDM VMCPU Instance data.
- * Changes to this must checked against the padding of the cfgm union in VMCPU!
- */
-typedef struct PDMCPU
-{
-    /** The number of entries in the apQueuedCritSectsLeaves table that's currently in use. */
-    uint32_t                        cQueuedCritSectLeaves;
-    uint32_t                        uPadding0; /**< Alignment padding.*/
-    /** Critical sections queued in RC/R0 because of contention preventing leave to complete. (R3 Ptrs)
-     * We will return to Ring-3 ASAP, so this queue doesn't have to be very long. */
-    R3PTRTYPE(PPDMCRITSECT)         apQueuedCritSectsLeaves[8];
-} PDMCPU;
-
-/**
- * Converts a PDM pointer into a VM pointer.
- * @returns Pointer to the VM structure the PDM is part of.
- * @param   pPDM   Pointer to PDM instance data.
- */
-#define PDM2VM(pPDM)  ( (PVM)((char*)pPDM - pPDM->offVM) )
-
-
-/**
- * PDM VM Instance data.
- * Changes to this must checked against the padding of the cfgm union in VM!
- */
-typedef struct PDM
-{
-    /** Offset to the VM structure.
-     * See PDM2VM(). */
-    RTUINT                          offVM;
-    RTUINT                          uPadding0; /**< Alignment padding.*/
-
-    /** List of registered devices. (FIFO) */
-    R3PTRTYPE(PPDMDEV)              pDevs;
-    /** List of devices instances. (FIFO) */
-    R3PTRTYPE(PPDMDEVINS)           pDevInstances;
-    /** List of registered USB devices. (FIFO) */
-    R3PTRTYPE(PPDMUSB)              pUsbDevs;
-    /** List of USB devices instances. (FIFO) */
-    R3PTRTYPE(PPDMUSBINS)           pUsbInstances;
-    /** List of registered drivers. (FIFO) */
-    R3PTRTYPE(PPDMDRV)              pDrvs;
-    /** PCI Buses. */
-    PDMPCIBUS                       aPciBuses[PDM_PCI_BUSSES_MAX];
-    /** The register PIC device. */
-    PDMPIC                          Pic;
-    /** The registered APIC device. */
-    PDMAPIC                         Apic;
-    /** The registered I/O APIC device. */
-    PDMIOAPIC                       IoApic;
-    /** The registered DMAC device. */
-    R3PTRTYPE(PPDMDMAC)             pDmac;
-    /** The registered RTC device. */
-    R3PTRTYPE(PPDMRTC)              pRtc;
-    /** The registered USB HUBs. (FIFO) */
-    R3PTRTYPE(PPDMUSBHUB)           pUsbHubs;
-
-    /** Queue in which devhlp tasks are queued for R3 execution - R3 Ptr. */
-    R3PTRTYPE(PPDMQUEUE)            pDevHlpQueueR3;
-    /** Queue in which devhlp tasks are queued for R3 execution - R0 Ptr. */
-    R0PTRTYPE(PPDMQUEUE)            pDevHlpQueueR0;
-    /** Queue in which devhlp tasks are queued for R3 execution - RC Ptr. */
-    RCPTRTYPE(PPDMQUEUE)            pDevHlpQueueRC;
-    /** Pointer to the queue which should be manually flushed - RC Ptr.
-     * Only touched by EMT. */
-    RCPTRTYPE(struct PDMQUEUE *)    pQueueFlushRC;
-    /** Pointer to the queue which should be manually flushed - R0 Ptr.
-     * Only touched by EMT. */
-    R0PTRTYPE(struct PDMQUEUE *)    pQueueFlushR0;
-    /** Bitmask controlling the queue flushing.
-     * See PDM_QUEUE_FLUSH_FLAG_ACTIVE and PDM_QUEUE_FLUSH_FLAG_PENDING. */
-    uint32_t volatile               fQueueFlushing;
-    /** Alignment padding. */
-    uint32_t                        u32Padding2;
-
-    /** @name   VMM device heap
-     * @{ */
-    /** Pointer to the heap base (MMIO2 ring-3 mapping). NULL if not registered. */
-    RTR3PTR                         pvVMMDevHeap;
-    /** The heap size. */
-    uint32_t                        cbVMMDevHeap;
-    /** Free space. */
-    uint32_t                        cbVMMDevHeapLeft;
-    /** The current mapping. NIL_RTGCPHYS if not mapped or registered. */
-    RTGCPHYS                        GCPhysVMMDevHeap;
-    /** @} */
-
-    /** The PDM lock.
-     * This is used to protect everything that deals with interrupts, i.e.
-     * the PIC, APIC, IOAPIC and PCI devices plus some PDM functions. */
-    PDMCRITSECT                     CritSect;
-
-    /** Number of times a critical section leave request needed to be queued for ring-3 execution. */
-    STAMCOUNTER                     StatQueuedCritSectLeaves;
-} PDM;
-AssertCompileMemberAlignment(PDM, GCPhysVMMDevHeap, sizeof(RTGCPHYS));
-AssertCompileMemberAlignment(PDM, CritSect, 8);
-AssertCompileMemberAlignment(PDM, StatQueuedCritSectLeaves, 8);
-/** Pointer to PDM VM instance data. */
-typedef PDM *PPDM;
-
-
-
-/**
- * PDM data kept in the UVM.
- */
-typedef struct PDMUSERPERVM
-{
-    /** @todo move more stuff over here. */
-
-    /** Linked list of timer driven PDM queues.
-     * Currently serialized by PDM::CritSect.  */
-    R3PTRTYPE(struct PDMQUEUE *)    pQueuesTimer;
-    /** Linked list of force action driven PDM queues.
-     * Currently serialized by PDM::CritSect. */
-    R3PTRTYPE(struct PDMQUEUE *)    pQueuesForced;
-
-    /** Lock protecting the lists below it. */
-    RTCRITSECT                      ListCritSect;
-    /** Pointer to list of loaded modules. */
-    PPDMMOD                         pModules;
-    /** List of initialized critical sections. (LIFO) */
-    R3PTRTYPE(PPDMCRITSECTINT)      pCritSects;
-    /** Head of the PDM Thread list. (singly linked) */
-    R3PTRTYPE(PPDMTHREAD)           pThreads;
-    /** Tail of the PDM Thread list. (singly linked) */
-    R3PTRTYPE(PPDMTHREAD)           pThreadsTail;
-
-    /** @name   PDM Async Completion
-     * @{ */
-    /** Pointer to the array of supported endpoint classes. */
-    PPDMASYNCCOMPLETIONEPCLASS      apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_MAX];
-    /** Head of the templates. Singly linked, protected by ListCritSect. */
-    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE) pAsyncCompletionTemplates;
-    /** @} */
-
-    R3PTRTYPE(PPDMBLKCACHEGLOBAL)   pBlkCacheGlobal;
-
-} PDMUSERPERVM;
-/** Pointer to the PDM data kept in the UVM. */
-typedef PDMUSERPERVM *PPDMUSERPERVM;
-
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-#ifdef IN_RING3
-extern const PDMDRVHLPR3    g_pdmR3DrvHlp;
-extern const PDMDEVHLPR3    g_pdmR3DevHlpTrusted;
-extern const PDMDEVHLPR3    g_pdmR3DevHlpUnTrusted;
-extern const PDMPICHLPR3    g_pdmR3DevPicHlp;
-extern const PDMAPICHLPR3   g_pdmR3DevApicHlp;
-extern const PDMIOAPICHLPR3 g_pdmR3DevIoApicHlp;
-extern const PDMPCIHLPR3    g_pdmR3DevPciHlp;
-extern const PDMDMACHLP     g_pdmR3DevDmacHlp;
-extern const PDMRTCHLP      g_pdmR3DevRtcHlp;
-extern const PDMHPETHLPR3   g_pdmR3DevHpetHlp;
-#endif
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** @def PDMDEV_ASSERT_DEVINS
- * Asserts the validity of the device instance.
- */
-#ifdef VBOX_STRICT
-# define PDMDEV_ASSERT_DEVINS(pDevIns)   \
-    do { \
-        AssertPtr(pDevIns); \
-        Assert(pDevIns->u32Version == PDM_DEVINS_VERSION); \
-        Assert(pDevIns->CTX_SUFF(pvInstanceData) == (void *)&pDevIns->achInstanceData[0]); \
-    } while (0)
-#else
-# define PDMDEV_ASSERT_DEVINS(pDevIns)   do { } while (0)
-#endif
-
-/** @def PDMDRV_ASSERT_DRVINS
- * Asserts the validity of the driver instance.
- */
-#ifdef VBOX_STRICT
-# define PDMDRV_ASSERT_DRVINS(pDrvIns) \
-    do { \
-        AssertPtr(pDrvIns); \
-        Assert(pDrvIns->u32Version == PDM_DRVINS_VERSION); \
-        Assert(pDrvIns->CTX_SUFF(pvInstanceData) == (void *)&pDrvIns->achInstanceData[0]); \
-    } while (0)
-#else
-# define PDMDRV_ASSERT_DRVINS(pDrvIns)   do { } while (0)
-#endif
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-#ifdef IN_RING3
-int         pdmR3CritSectInitStats(PVM pVM);
-void        pdmR3CritSectRelocate(PVM pVM);
-int         pdmR3CritSectInitDevice(PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, va_list va);
-int         pdmR3CritSectDeleteDevice(PVM pVM, PPDMDEVINS pDevIns);
-int         pdmR3CritSectInitDriver(PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, ...);
-int         pdmR3CritSectDeleteDriver(PVM pVM, PPDMDRVINS pDrvIns);
-
-int         pdmR3DevInit(PVM pVM);
-PPDMDEV     pdmR3DevLookup(PVM pVM, const char *pszName);
-int         pdmR3DevFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMLUN *ppLun);
-DECLCALLBACK(bool) pdmR3DevHlpQueueConsumer(PVM pVM, PPDMQUEUEITEMCORE pItem);
-
-int         pdmR3UsbLoadModules(PVM pVM);
-int         pdmR3UsbInstantiateDevices(PVM pVM);
-PPDMUSB     pdmR3UsbLookup(PVM pVM, const char *pszName);
-int         pdmR3UsbFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMLUN *ppLun);
-int         pdmR3UsbRegisterHub(PVM pVM, PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp);
-int         pdmR3UsbVMInitComplete(PVM pVM);
-
-int         pdmR3DrvInit(PVM pVM);
-int         pdmR3DrvInstantiate(PVM pVM, PCFGMNODE pNode, PPDMIBASE pBaseInterface, PPDMDRVINS pDrvAbove,
-                                PPDMLUN pLun, PPDMIBASE *ppBaseInterface);
-int         pdmR3DrvDetach(PPDMDRVINS pDrvIns, uint32_t fFlags);
-void        pdmR3DrvDestroyChain(PPDMDRVINS pDrvIns, uint32_t fFlags);
-PPDMDRV     pdmR3DrvLookup(PVM pVM, const char *pszName);
-
-int         pdmR3LdrInitU(PUVM pUVM);
-void        pdmR3LdrTermU(PUVM pUVM);
-char       *pdmR3FileR3(const char *pszFile, bool fShared);
-int         pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName);
-
-void        pdmR3QueueRelocate(PVM pVM, RTGCINTPTR offDelta);
-
-int         pdmR3ThreadCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
-                                    PFNPDMTHREADWAKEUPDEV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
-int         pdmR3ThreadCreateUsb(PVM pVM, PPDMDRVINS pUsbIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADUSB pfnThread,
-                                 PFNPDMTHREADWAKEUPUSB pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
-int         pdmR3ThreadCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
-                                    PFNPDMTHREADWAKEUPDRV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
-int         pdmR3ThreadDestroyDevice(PVM pVM, PPDMDEVINS pDevIns);
-int         pdmR3ThreadDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns);
-int         pdmR3ThreadDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns);
-void        pdmR3ThreadDestroyAll(PVM pVM);
-int         pdmR3ThreadResumeAll(PVM pVM);
-int         pdmR3ThreadSuspendAll(PVM pVM);
-
-#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
-int         pdmR3AsyncCompletionInit(PVM pVM);
-int         pdmR3AsyncCompletionTerm(PVM pVM);
-void        pdmR3AsyncCompletionResume(PVM pVM);
-#endif
-
-int         pdmR3BlkCacheInit(PVM pVM);
-void        pdmR3BlkCacheTerm(PVM pVM);
-int         pdmR3BlkCacheResume(PVM pVM);
-
-#endif /* IN_RING3 */
-
-void        pdmLock(PVM pVM);
-int         pdmLockEx(PVM pVM, int rc);
-void        pdmUnlock(PVM pVM);
-
-/** @} */
-
-RT_C_DECLS_END
-
-#endif
Index: unk/src/VBox/VMM/PDMLdr.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMLdr.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1630 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device Manager, module loader.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-//#define PDMLDR_FAKE_MODE
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_LDR
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/vmm.h>
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/sup.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/hwaccm.h>
-
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/ctype.h>
-#include <iprt/file.h>
-#include <iprt/ldr.h>
-#include <iprt/mem.h>
-#include <iprt/path.h>
-#include <iprt/string.h>
-
-#include <limits.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Structure which the user argument of the RTLdrGetBits() callback points to.
- * @internal
- */
-typedef struct PDMGETIMPORTARGS
-{
-    PVM         pVM;
-    PPDMMOD     pModule;
-} PDMGETIMPORTARGS, *PPDMGETIMPORTARGS;
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) pdmR3GetImportRC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser);
-static int      pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName, const char *pszSearchPath);
-static char    *pdmR3FileRC(const char *pszFile, const char *pszSearchPath);
-static char    *pdmR3FileR0(const char *pszFile, const char *pszSearchPath);
-static char    *pdmR3File(const char *pszFile, const char *pszDefaultExt, const char *pszSearchPath, bool fShared);
-
-
-
-/**
- * Loads the VMMR0.r0 module early in the init process.
- *
- * @returns VBox status code.
- * @param   pUVM            Pointer to the user mode VM structure.
- */
-VMMR3DECL(int) PDMR3LdrLoadVMMR0U(PUVM pUVM)
-{
-    return pdmR3LoadR0U(pUVM, NULL, VMMR0_MAIN_MODULE_NAME, NULL);
-}
-
-
-/**
- * Init the module loader part of PDM.
- *
- * This routine will load the Host Context Ring-0 and Guest
- * Context VMM modules.
- *
- * @returns VBox status code.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pvVMMR0Mod  The opaque returned by PDMR3LdrLoadVMMR0.
- */
-int pdmR3LdrInitU(PUVM pUVM)
-{
-#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE)
-    return VINF_SUCCESS;
-
-#else
-
-    /*
-     * Load the mandatory GC module, the VMMR0.r0 is loaded before VM creation.
-     */
-    return PDMR3LdrLoadRC(pUVM->pVM, NULL, VMMGC_MAIN_MODULE_NAME);
-#endif
-}
-
-
-/**
- * Terminate the module loader part of PDM.
- *
- * This will unload and free all modules.
- *
- * @param   pVM         The VM handle.
- *
- * @remarks This is normally called twice during termination.
- */
-void pdmR3LdrTermU(PUVM pUVM)
-{
-    /*
-     * Free the modules.
-     */
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMMOD pModule = pUVM->pdm.s.pModules;
-    pUVM->pdm.s.pModules = NULL;
-    while (pModule)
-    {
-        /* free loader item. */
-        if (pModule->hLdrMod != NIL_RTLDRMOD)
-        {
-            int rc2 = RTLdrClose(pModule->hLdrMod);
-            AssertRC(rc2);
-            pModule->hLdrMod = NIL_RTLDRMOD;
-        }
-
-        /* free bits. */
-        switch (pModule->eType)
-        {
-            case PDMMOD_TYPE_R0:
-            {
-                Assert(pModule->ImageBase);
-                int rc2 = SUPR3FreeModule((void *)(uintptr_t)pModule->ImageBase);
-                AssertRC(rc2);
-                pModule->ImageBase = 0;
-                break;
-            }
-
-#ifdef VBOX_WITH_RAW_MODE
-            case PDMMOD_TYPE_RC:
-#endif
-            case PDMMOD_TYPE_R3:
-                /* MM will free this memory for us - it's alloc only memory. :-) */
-                break;
-
-            default:
-                AssertMsgFailed(("eType=%d\n", pModule->eType));
-                break;
-        }
-        pModule->pvBits = NULL;
-
-        void    *pvFree = pModule;
-        pModule = pModule->pNext;
-        RTMemFree(pvFree);
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-}
-
-
-/**
- * Applies relocations to GC modules.
- *
- * This must be done very early in the relocation
- * process so that components can resolve GC symbols during relocation.
- *
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   offDelta    Relocation delta relative to old location.
- */
-VMMR3DECL(void) PDMR3LdrRelocateU(PUVM pUVM, RTGCINTPTR offDelta)
-{
-#ifdef VBOX_WITH_RAW_MODE
-    LogFlow(("PDMR3LdrRelocate: offDelta=%RGv\n", offDelta));
-
-    /*
-     * GC Modules.
-     */
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    if (pUVM->pdm.s.pModules)
-    {
-        /*
-         * The relocation have to be done in two passes so imports
-         * can be correctly resolved. The first pass will update
-         * the ImageBase saving the current value in OldImageBase.
-         * The second pass will do the actual relocation.
-         */
-        /* pass 1 */
-        PPDMMOD pCur;
-        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
-        {
-            if (pCur->eType == PDMMOD_TYPE_RC)
-            {
-                pCur->OldImageBase = pCur->ImageBase;
-                pCur->ImageBase = MMHyperR3ToRC(pUVM->pVM, pCur->pvBits);
-            }
-        }
-
-        /* pass 2 */
-        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
-        {
-            if (pCur->eType == PDMMOD_TYPE_RC)
-            {
-                PDMGETIMPORTARGS Args;
-                Args.pVM = pUVM->pVM;
-                Args.pModule = pCur;
-                int rc = RTLdrRelocate(pCur->hLdrMod, pCur->pvBits, pCur->ImageBase, pCur->OldImageBase,
-                                       pdmR3GetImportRC, &Args);
-                AssertFatalMsgRC(rc, ("RTLdrRelocate failed, rc=%d\n", rc));
-                DBGFR3ModuleRelocate(pUVM->pVM, pCur->OldImageBase, pCur->ImageBase, RTLdrSize(pCur->hLdrMod),
-                                     pCur->szFilename, pCur->szName);
-            }
-        }
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-#endif
-}
-
-
-/**
- * Loads a module into the host context ring-3.
- *
- * This is used by the driver and device init functions to load modules
- * containing the drivers and devices. The function can be extended to
- * load modules which are not native to the environment we're running in,
- * but at the moment this is not required.
- *
- * No reference counting is kept, since we don't implement any facilities
- * for unloading the module. But the module will naturally be released
- * when the VM terminates.
- *
- * @returns VBox status code.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   pszFilename     Filename of the module binary.
- * @param   pszName         Module name. Case sensitive and the length is limited!
- */
-int pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName)
-{
-    /*
-     * Validate input.
-     */
-    AssertMsg(pUVM->pVM->pdm.s.offVM, ("bad init order!\n"));
-    Assert(pszFilename);
-    size_t cchFilename = strlen(pszFilename);
-    Assert(pszName);
-    size_t cchName = strlen(pszName);
-    PPDMMOD  pCur;
-    if (cchName >= sizeof(pCur->szName))
-    {
-        AssertMsgFailed(("Name is too long, cchName=%d pszName='%s'\n", cchName, pszName));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Try lookup the name and see if the module exists.
-     */
-    int rc;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
-    {
-        if (!strcmp(pCur->szName, pszName))
-        {
-            if (pCur->eType == PDMMOD_TYPE_R3)
-                rc = VINF_PDM_ALREADY_LOADED;
-            else
-                rc = VERR_PDM_MODULE_NAME_CLASH;
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-
-            AssertMsgRC(rc, ("We've already got a module '%s' loaded!\n", pszName));
-            return rc;
-        }
-    }
-
-    /*
-     * Allocate the module list node and initialize it.
-     */
-    const char *pszSuff = RTLdrGetSuff();
-    size_t      cchSuff = RTPathHaveExt(pszFilename) ? 0 : strlen(pszSuff);
-    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(RT_OFFSETOF(PDMMOD, szFilename[cchFilename + cchSuff + 1]));
-    if (pModule)
-    {
-        pModule->eType = PDMMOD_TYPE_R3;
-        memcpy(pModule->szName, pszName, cchName); /* memory is zero'd, no need to copy terminator :-) */
-        memcpy(pModule->szFilename, pszFilename, cchFilename);
-        memcpy(&pModule->szFilename[cchFilename], pszSuff, cchSuff);
-
-        /*
-         * Load the loader item.
-         */
-        RTERRINFOSTATIC ErrInfo;
-        RTErrInfoInitStatic(&ErrInfo);
-        rc = SUPR3HardenedLdrLoadPlugIn(pModule->szFilename, &pModule->hLdrMod, &ErrInfo.Core);
-        if (RT_SUCCESS(rc))
-        {
-            pModule->pNext = pUVM->pdm.s.pModules;
-            pUVM->pdm.s.pModules = pModule;
-        }
-        else
-        {
-            /* Something went wrong, most likely module not found. Don't consider other unlikely errors */
-            rc = VMSetError(pUVM->pVM, rc, RT_SRC_POS,
-                            N_("Unable to load R3 module %s (%s): %s"), pModule->szFilename, pszName, ErrInfo.Core.pszMsg);
-            RTMemFree(pModule);
-        }
-    }
-    else
-        rc = VERR_NO_MEMORY;
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-#ifdef VBOX_WITH_RAW_MODE
-/**
- * Resolve an external symbol during RTLdrGetBits() of a RC module.
- *
- * @returns VBox status code.
- * @param   hLdrMod         The loader module handle.
- * @param   pszModule       Module name.
- * @param   pszSymbol       Symbol name, NULL if uSymbol should be used.
- * @param   uSymbol         Symbol ordinal, ~0 if pszSymbol should be used.
- * @param   pValue          Where to store the symbol value (address).
- * @param   pvUser          User argument.
- */
-static DECLCALLBACK(int) pdmR3GetImportRC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser)
-{
-    PVM         pVM     = ((PPDMGETIMPORTARGS)pvUser)->pVM;
-    PPDMMOD     pModule = ((PPDMGETIMPORTARGS)pvUser)->pModule;
-
-    /*
-     * Adjust input.
-     */
-    if (pszModule && !*pszModule)
-        pszModule = NULL;
-
-    /*
-     * Builtin module.
-     */
-    if (!pszModule || !strcmp(pszModule, "VMMGCBuiltin.gc"))
-    {
-        int rc = VINF_SUCCESS;
-        if (!strcmp(pszSymbol, "g_VM"))
-            *pValue = pVM->pVMRC;
-        else if (!strcmp(pszSymbol, "g_CPUM"))
-            *pValue = VM_RC_ADDR(pVM, &pVM->cpum);
-        else if (!strcmp(pszSymbol, "g_TRPM"))
-            *pValue = VM_RC_ADDR(pVM, &pVM->trpm);
-        else if (!strcmp(pszSymbol, "g_TRPMCPU"))
-            *pValue = VM_RC_ADDR(pVM, &pVM->aCpus[0].trpm);
-        else if (   !strncmp(pszSymbol, "VMM", 3)
-                 || !strcmp(pszSymbol, "g_Logger")
-                 || !strcmp(pszSymbol, "g_RelLogger"))
-        {
-            RTRCPTR RCPtr = 0;
-            rc = VMMR3GetImportRC(pVM, pszSymbol, &RCPtr);
-            if (RT_SUCCESS(rc))
-                *pValue = RCPtr;
-        }
-        else if (   !strncmp(pszSymbol, "TM", 2)
-                 || !strcmp(pszSymbol, "g_pSUPGlobalInfoPage"))
-        {
-            RTRCPTR RCPtr = 0;
-            rc = TMR3GetImportRC(pVM, pszSymbol, &RCPtr);
-            if (RT_SUCCESS(rc))
-                *pValue = RCPtr;
-        }
-        else
-        {
-            AssertMsg(!pszModule, ("Unknown builtin symbol '%s' for module '%s'!\n", pszSymbol, pModule->szName)); NOREF(pModule);
-            rc = VERR_SYMBOL_NOT_FOUND;
-        }
-        if (RT_SUCCESS(rc) || pszModule)
-        {
-            if (RT_FAILURE(rc))
-                LogRel(("PDMLdr: Couldn't find symbol '%s' in module '%s'!\n", pszSymbol, pszModule));
-            return rc;
-        }
-    }
-
-    /*
-     * Search for module.
-     */
-    PUVM     pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMMOD  pCur = pUVM->pdm.s.pModules;
-    while (pCur)
-    {
-        if (    pCur->eType == PDMMOD_TYPE_RC
-            &&  (   !pszModule
-                 || !strcmp(pCur->szName, pszModule))
-           )
-        {
-            /* Search for the symbol. */
-            int rc = RTLdrGetSymbolEx(pCur->hLdrMod, pCur->pvBits, pCur->ImageBase, pszSymbol, pValue);
-            if (RT_SUCCESS(rc))
-            {
-                AssertMsg(*pValue - pCur->ImageBase < RTLdrSize(pCur->hLdrMod),
-                          ("%RRv-%RRv %s %RRv\n", (RTRCPTR)pCur->ImageBase,
-                           (RTRCPTR)(pCur->ImageBase + RTLdrSize(pCur->hLdrMod) - 1),
-                           pszSymbol, (RTRCPTR)*pValue));
-                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-                return rc;
-            }
-            if (pszModule)
-            {
-                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-                AssertLogRelMsgFailed(("PDMLdr: Couldn't find symbol '%s' in module '%s'!\n", pszSymbol, pszModule));
-                return VERR_SYMBOL_NOT_FOUND;
-            }
-        }
-
-        /* next */
-        pCur = pCur->pNext;
-    }
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    AssertLogRelMsgFailed(("Couldn't find module '%s' for resolving symbol '%s'!\n", pszModule, pszSymbol));
-    return VERR_SYMBOL_NOT_FOUND;
-}
-
-
-/**
- * Loads a module into the guest context (i.e. into the Hypervisor memory region).
- *
- * @returns VBox status code.
- * @param   pVM             The VM to load it into.
- * @param   pszFilename     Filename of the module binary.
- * @param   pszName         Module name. Case sensitive and the length is limited!
- */
-VMMR3DECL(int) PDMR3LdrLoadRC(PVM pVM, const char *pszFilename, const char *pszName)
-{
-    /*
-     * Validate input.
-     */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-    PUVM     pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMMOD  pCur = pUVM->pdm.s.pModules;
-    while (pCur)
-    {
-        if (!strcmp(pCur->szName, pszName))
-        {
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-            AssertMsgFailed(("We've already got a module '%s' loaded!\n", pszName));
-            return VERR_PDM_MODULE_NAME_CLASH;
-        }
-        /* next */
-        pCur = pCur->pNext;
-    }
-
-    /*
-     * Find the file if not specified.
-     */
-    char *pszFile = NULL;
-    if (!pszFilename)
-        pszFilename = pszFile = pdmR3FileRC(pszName, NULL);
-
-    /*
-     * Allocate the module list node.
-     */
-    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(sizeof(*pModule) + strlen(pszFilename));
-    if (!pModule)
-    {
-        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-        RTMemTmpFree(pszFile);
-        return VERR_NO_MEMORY;
-    }
-    AssertMsg(strlen(pszName) + 1 < sizeof(pModule->szName),
-              ("pazName is too long (%d chars) max is %d chars.\n", strlen(pszName), sizeof(pModule->szName) - 1));
-    strcpy(pModule->szName, pszName);
-    pModule->eType = PDMMOD_TYPE_RC;
-    strcpy(pModule->szFilename, pszFilename);
-
-
-    /*
-     * Open the loader item.
-     */
-    RTERRINFOSTATIC ErrInfo;
-    RTErrInfoInitStatic(&ErrInfo);
-    int rc = SUPR3HardenedVerifyPlugIn(pszFilename, &ErrInfo.Core);
-    if (RT_SUCCESS(rc))
-    {
-        RTErrInfoClear(&ErrInfo.Core);
-        rc = RTLdrOpen(pszFilename, 0, RTLDRARCH_X86_32, &pModule->hLdrMod);
-    }
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Allocate space in the hypervisor.
-         */
-        size_t          cb = RTLdrSize(pModule->hLdrMod);
-        cb = RT_ALIGN_Z(cb, PAGE_SIZE);
-        uint32_t        cPages = (uint32_t)(cb >> PAGE_SHIFT);
-        if (((size_t)cPages << PAGE_SHIFT) == cb)
-        {
-            PSUPPAGE    paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(paPages[0]));
-            if (paPages)
-            {
-                rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pModule->pvBits, NULL /*pR0Ptr*/, paPages);
-                if (RT_SUCCESS(rc))
-                {
-                    RTGCPTR GCPtr;
-                    rc = MMR3HyperMapPages(pVM, pModule->pvBits, NIL_RTR0PTR,
-                                           cPages, paPages, pModule->szName, &GCPtr);
-                    if (RT_SUCCESS(rc))
-                    {
-                        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-
-                        /*
-                         * Get relocated image bits.
-                         */
-                        Assert(MMHyperR3ToRC(pVM, pModule->pvBits) == GCPtr);
-                        pModule->ImageBase = GCPtr;
-                        PDMGETIMPORTARGS Args;
-                        Args.pVM = pVM;
-                        Args.pModule = pModule;
-                        rc = RTLdrGetBits(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pdmR3GetImportRC, &Args);
-                        if (RT_SUCCESS(rc))
-                        {
-                            /*
-                             * Insert the module.
-                             */
-                            if (pUVM->pdm.s.pModules)
-                            {
-                                /* we don't expect this list to be very long, so rather save the tail pointer. */
-                                pCur = pUVM->pdm.s.pModules;
-                                while (pCur->pNext)
-                                    pCur = pCur->pNext;
-                                pCur->pNext = pModule;
-                            }
-                            else
-                                pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
-                            Log(("PDM: RC Module at %RRv %s (%s)\n", (RTRCPTR)pModule->ImageBase, pszName, pszFilename));
-                            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-                            RTMemTmpFree(pszFile);
-                            RTMemTmpFree(paPages);
-                            return VINF_SUCCESS;
-                        }
-                    }
-                    else
-                    {
-                        AssertRC(rc);
-                        SUPR3PageFreeEx(pModule->pvBits, cPages);
-                    }
-                }
-                else
-                    AssertMsgFailed(("SUPR3PageAlloc(%d,) -> %Rrc\n", cPages, rc));
-                RTMemTmpFree(paPages);
-            }
-            else
-                rc = VERR_NO_TMP_MEMORY;
-        }
-        else
-            rc = VERR_OUT_OF_RANGE;
-        int rc2 = RTLdrClose(pModule->hLdrMod);
-        AssertRC(rc2);
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-
-    /* Don't consider VERR_PDM_MODULE_NAME_CLASH and VERR_NO_MEMORY above as these are very unlikely. */
-    if (RT_FAILURE(rc) && RTErrInfoIsSet(&ErrInfo.Core))
-        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Cannot load RC module %s: %s"), pszFilename, ErrInfo.Core.pszMsg);
-    else if (RT_FAILURE(rc))
-        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Cannot load RC module %s"), pszFilename);
-
-    RTMemFree(pModule);
-    RTMemTmpFree(pszFile);
-    return rc;
-}
-#endif /* VBOX_WITH_RAW_MODE */
-
-
-/**
- * Loads a module into the ring-0 context.
- *
- * @returns VBox status code.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   pszFilename     Filename of the module binary.
- * @param   pszName         Module name. Case sensitive and the length is limited!
- * @param   pszSearchPath   List of directories to search if @a pszFilename is
- *                          not specified.  Can be NULL, in which case the arch
- *                          dependent install dir is searched.
- */
-static int pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName, const char *pszSearchPath)
-{
-    /*
-     * Validate input.
-     */
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMMOD pCur = pUVM->pdm.s.pModules;
-    while (pCur)
-    {
-        if (!strcmp(pCur->szName, pszName))
-        {
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-            AssertMsgFailed(("We've already got a module '%s' loaded!\n", pszName));
-            return VERR_PDM_MODULE_NAME_CLASH;
-        }
-        /* next */
-        pCur = pCur->pNext;
-    }
-
-    /*
-     * Find the file if not specified.
-     */
-    char *pszFile = NULL;
-    if (!pszFilename)
-        pszFilename = pszFile = pdmR3FileR0(pszName, pszSearchPath);
-
-    /*
-     * Allocate the module list node.
-     */
-    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(sizeof(*pModule) + strlen(pszFilename));
-    if (!pModule)
-    {
-        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-        RTMemTmpFree(pszFile);
-        return VERR_NO_MEMORY;
-    }
-    AssertMsg(strlen(pszName) + 1 < sizeof(pModule->szName),
-              ("pazName is too long (%d chars) max is %d chars.\n", strlen(pszName), sizeof(pModule->szName) - 1));
-    strcpy(pModule->szName, pszName);
-    pModule->eType = PDMMOD_TYPE_R0;
-    strcpy(pModule->szFilename, pszFilename);
-
-    /*
-     * Ask the support library to load it.
-     */
-    void           *pvImageBase;
-    RTERRINFOSTATIC ErrInfo;
-    RTErrInfoInitStatic(&ErrInfo);
-    int rc = SUPR3LoadModule(pszFilename, pszName, &pvImageBase, &ErrInfo.Core);
-    if (RT_SUCCESS(rc))
-    {
-        pModule->hLdrMod = NIL_RTLDRMOD;
-        pModule->ImageBase = (uintptr_t)pvImageBase;
-
-        /*
-         * Insert the module.
-         */
-        if (pUVM->pdm.s.pModules)
-        {
-            /* we don't expect this list to be very long, so rather save the tail pointer. */
-            pCur = pUVM->pdm.s.pModules;
-            while (pCur->pNext)
-                pCur = pCur->pNext;
-            pCur->pNext = pModule;
-        }
-        else
-            pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
-        Log(("PDM: R0 Module at %RHv %s (%s)\n", (RTR0PTR)pModule->ImageBase, pszName, pszFilename));
-        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-        RTMemTmpFree(pszFile);
-        return VINF_SUCCESS;
-    }
-
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    RTMemFree(pModule);
-    LogRel(("pdmR3LoadR0U: pszName=\"%s\" rc=%Rrc szErr=\"%s\"\n", pszName, rc, ErrInfo.Core.pszMsg));
-
-    /* Don't consider VERR_PDM_MODULE_NAME_CLASH and VERR_NO_MEMORY above as these are very unlikely. */
-    if (RT_FAILURE(rc) && pUVM->pVM) /** @todo VMR3SetErrorU. */
-        rc = VMSetError(pUVM->pVM, rc, RT_SRC_POS, N_("Cannot load R0 module %s: %s"), pszFilename, ErrInfo.Core.pszMsg);
-
-    RTMemTmpFree(pszFile); /* might be reference thru pszFilename in the above VMSetError call. */
-    return rc;
-}
-
-
-
-/**
- * Get the address of a symbol in a given HC ring 3 module.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pszModule       Module name.
- * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
- *                          ordinal value rather than a string pointer.
- * @param   ppvValue        Where to store the symbol value.
- */
-VMMR3DECL(int) PDMR3LdrGetSymbolR3(PVM pVM, const char *pszModule, const char *pszSymbol, void **ppvValue)
-{
-    /*
-     * Validate input.
-     */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-
-    /*
-     * Find the module.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
-    {
-        if (    pModule->eType == PDMMOD_TYPE_R3
-            &&  !strcmp(pModule->szName, pszModule))
-        {
-            RTUINTPTR Value = 0;
-            int rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pszSymbol, &Value);
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-            if (RT_SUCCESS(rc))
-            {
-                *ppvValue = (void *)(uintptr_t)Value;
-                Assert((uintptr_t)*ppvValue == Value);
-            }
-            else
-            {
-                if ((uintptr_t)pszSymbol < 0x10000)
-                    AssertMsg(rc, ("Couldn't symbol '%u' in module '%s'\n", (unsigned)(uintptr_t)pszSymbol, pszModule));
-                else
-                    AssertMsg(rc, ("Couldn't symbol '%s' in module '%s'\n", pszSymbol, pszModule));
-            }
-            return rc;
-        }
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
-    return VERR_SYMBOL_NOT_FOUND;
-}
-
-
-/**
- * Get the address of a symbol in a given HC ring 0 module.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pszModule       Module name. If NULL the main R0 module (VMMR0.r0) is assumes.
- * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
- *                          ordinal value rather than a string pointer.
- * @param   ppvValue        Where to store the symbol value.
- */
-VMMR3DECL(int) PDMR3LdrGetSymbolR0(PVM pVM, const char *pszModule, const char *pszSymbol, PRTR0PTR ppvValue)
-{
-#ifdef PDMLDR_FAKE_MODE
-    *ppvValue = 0xdeadbeef;
-    return VINF_SUCCESS;
-
-#else
-    /*
-     * Validate input.
-     */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-    if (!pszModule)
-        pszModule = "VMMR0.r0";
-
-    /*
-     * Find the module.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
-    {
-        if (    pModule->eType == PDMMOD_TYPE_R0
-            &&  !strcmp(pModule->szName, pszModule))
-        {
-            int rc = SUPR3GetSymbolR0((void *)(uintptr_t)pModule->ImageBase, pszSymbol, (void **)ppvValue);
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgRC(rc, ("Couldn't find symbol '%s' in module '%s'\n", pszSymbol, pszModule));
-                LogRel(("PDMGetSymbol: Couldn't find symbol '%s' in module '%s'\n", pszSymbol, pszModule));
-            }
-            return rc;
-        }
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
-    return VERR_SYMBOL_NOT_FOUND;
-#endif
-}
-
-
-/**
- * Same as PDMR3LdrGetSymbolR0 except that the module will be attempted loaded if not found.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pszModule       Module name. If NULL the main R0 module (VMMR0.r0) is assumed.
- * @param   pszSearchPath   List of directories to search if @a pszFile is
- *                          not qualified with a path.  Can be NULL, in which
- *                          case the arch dependent install dir is searched.
- * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
- *                          ordinal value rather than a string pointer.
- * @param   ppvValue        Where to store the symbol value.
- */
-VMMR3DECL(int) PDMR3LdrGetSymbolR0Lazy(PVM pVM, const char *pszModule, const char *pszSearchPath, const char *pszSymbol,
-                                       PRTR0PTR ppvValue)
-{
-#ifdef PDMLDR_FAKE_MODE
-    *ppvValue = 0xdeadbeef;
-    return VINF_SUCCESS;
-
-#else
-    /*
-     * Since we're lazy, we'll only check if the module is present
-     * and hand it over to PDMR3LdrGetSymbolR0 when that's done.
-     */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-    if (pszModule)
-    {
-        AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
-        PUVM    pUVM = pVM->pUVM;
-        PPDMMOD pModule;
-        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-        for (pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
-            if (    pModule->eType == PDMMOD_TYPE_R0
-                &&  !strcmp(pModule->szName, pszModule))
-                break;
-        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-        if (!pModule)
-        {
-            int rc = pdmR3LoadR0U(pUVM, NULL, pszModule, pszSearchPath);
-            AssertMsgRCReturn(rc, ("pszModule=%s rc=%Rrc\n", pszModule, rc), VERR_MODULE_NOT_FOUND);
-        }
-    }
-    return PDMR3LdrGetSymbolR0(pVM, pszModule, pszSymbol, ppvValue);
-#endif
-}
-
-
-/**
- * Get the address of a symbol in a given RC module.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pszModule       Module name. If NULL the main R0 module (VMMGC.gc) is assumes.
- * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
- *                          ordinal value rather than a string pointer.
- * @param   pRCPtrValue     Where to store the symbol value.
- */
-VMMR3DECL(int) PDMR3LdrGetSymbolRC(PVM pVM, const char *pszModule, const char *pszSymbol, PRTRCPTR pRCPtrValue)
-{
-#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE)
-    *pRCPtrValue = 0xfeedf00d;
-    return VINF_SUCCESS;
-
-#else
-    /*
-     * Validate input.
-     */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-    if (!pszModule)
-        pszModule = "VMMGC.gc";
-
-    /*
-     * Find the module.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
-    {
-        if (    pModule->eType == PDMMOD_TYPE_RC
-            &&  !strcmp(pModule->szName, pszModule))
-        {
-            RTUINTPTR Value;
-            int rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pszSymbol, &Value);
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-            if (RT_SUCCESS(rc))
-            {
-                *pRCPtrValue = (RTGCPTR)Value;
-                Assert(*pRCPtrValue == Value);
-            }
-            else
-            {
-                if ((uintptr_t)pszSymbol < 0x10000)
-                    AssertMsg(rc, ("Couldn't symbol '%u' in module '%s'\n", (unsigned)(uintptr_t)pszSymbol, pszModule));
-                else
-                    AssertMsg(rc, ("Couldn't symbol '%s' in module '%s'\n", pszSymbol, pszModule));
-            }
-            return rc;
-        }
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
-    return VERR_SYMBOL_NOT_FOUND;
-#endif
-}
-
-
-/**
- * Same as PDMR3LdrGetSymbolRC except that the module will be attempted loaded if not found.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pszModule       Module name. If NULL the main R0 module (VMMGC.gc) is assumes.
- * @param   pszSearchPath   List of directories to search if @a pszFile is
- *                          not qualified with a path.  Can be NULL, in which
- *                          case the arch dependent install dir is searched.
- * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
- *                          ordinal value rather than a string pointer.
- * @param   pRCPtrValue     Where to store the symbol value.
- */
-VMMR3DECL(int) PDMR3LdrGetSymbolRCLazy(PVM pVM, const char *pszModule, const char *pszSearchPath, const char *pszSymbol,
-                                       PRTRCPTR pRCPtrValue)
-{
-#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE)
-    *pRCPtrValue = 0xfeedf00d;
-    return VINF_SUCCESS;
-
-#else
-    /*
-     * Since we're lazy, we'll only check if the module is present
-     * and hand it over to PDMR3LdrGetSymbolRC when that's done.
-     */
-    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
-    if (pszModule)
-    {
-        AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
-        PUVM    pUVM = pVM->pUVM;
-        PPDMMOD pModule;
-        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-        for (pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
-            if (    pModule->eType == PDMMOD_TYPE_RC
-                &&  !strcmp(pModule->szName, pszModule))
-                break;
-        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-        if (!pModule)
-        {
-            char *pszFilename = pdmR3FileRC(pszModule, pszSearchPath);
-            AssertMsgReturn(pszFilename, ("pszModule=%s\n", pszModule), VERR_MODULE_NOT_FOUND);
-            int rc = PDMR3LdrLoadRC(pVM, pszFilename, pszModule);
-            RTMemTmpFree(pszFilename);
-            AssertMsgRCReturn(rc, ("pszModule=%s rc=%Rrc\n", pszModule, rc), VERR_MODULE_NOT_FOUND);
-        }
-    }
-    return PDMR3LdrGetSymbolRC(pVM, pszModule, pszSymbol, pRCPtrValue);
-#endif
-}
-
-
-/**
- * Constructs the full filename for a R3 image file.
- *
- * @returns Pointer to temporary memory containing the filename.
- *          Caller must free this using RTMemTmpFree().
- * @returns NULL on failure.
- *
- * @param   pszFile     File name (no path).
- */
-char *pdmR3FileR3(const char *pszFile, bool fShared)
-{
-    return pdmR3File(pszFile, NULL, NULL, fShared);
-}
-
-
-/**
- * Constructs the full filename for a R0 image file.
- *
- * @returns Pointer to temporary memory containing the filename.
- *          Caller must free this using RTMemTmpFree().
- * @returns NULL on failure.
- *
- * @param   pszFile         File name (no path).
- * @param   pszSearchPath   List of directories to search if @a pszFile is
- *                          not qualified with a path.  Can be NULL, in which
- *                          case the arch dependent install dir is searched.
- */
-char *pdmR3FileR0(const char *pszFile, const char *pszSearchPath)
-{
-    return pdmR3File(pszFile, NULL, pszSearchPath, /*fShared=*/false);
-}
-
-
-/**
- * Constructs the full filename for a RC image file.
- *
- * @returns Pointer to temporary memory containing the filename.
- *          Caller must free this using RTMemTmpFree().
- * @returns NULL on failure.
- *
- * @param   pszFile         File name (no path).
- * @param   pszSearchPath   List of directories to search if @a pszFile is
- *                          not qualified with a path.  Can be NULL, in which
- *                          case the arch dependent install dir is searched.
- */
-char *pdmR3FileRC(const char *pszFile, const char *pszSearchPath)
-{
-    return pdmR3File(pszFile, NULL, pszSearchPath, /*fShared=*/false);
-}
-
-
-/**
- * Worker for pdmR3File().
- *
- * @returns Pointer to temporary memory containing the filename.
- *          Caller must free this using RTMemTmpFree().
- * @returns NULL on failure.
- *
- * @param   pszDir        Directory part
- * @param   pszFile       File name part
- * @param   pszDefaultExt Extension part
- */
-static char *pdmR3FileConstruct(const char *pszDir, const char *pszFile, const char *pszDefaultExt)
-{
-    /*
-     * Allocate temp memory for return buffer.
-     */
-    size_t cchDir  = strlen(pszDir);
-    size_t cchFile = strlen(pszFile);
-    size_t cchDefaultExt;
-
-    /*
-     * Default extention?
-     */
-    if (!pszDefaultExt || strchr(pszFile, '.'))
-        cchDefaultExt = 0;
-    else
-        cchDefaultExt = strlen(pszDefaultExt);
-
-    size_t cchPath = cchDir + 1 + cchFile + cchDefaultExt + 1;
-    AssertMsgReturn(cchPath <= RTPATH_MAX, ("Path too long!\n"), NULL);
-
-    char *pszRet = (char *)RTMemTmpAlloc(cchDir + 1 + cchFile + cchDefaultExt + 1);
-    AssertMsgReturn(pszRet, ("Out of temporary memory!\n"), NULL);
-
-    /*
-     * Construct the filename.
-     */
-    memcpy(pszRet, pszDir, cchDir);
-    pszRet[cchDir++] = '/';            /* this works everywhere */
-    memcpy(pszRet + cchDir, pszFile, cchFile + 1);
-    if (cchDefaultExt)
-        memcpy(pszRet + cchDir + cchFile, pszDefaultExt, cchDefaultExt + 1);
-
-    return pszRet;
-}
-
-
-/**
- * Worker for pdmR3FileRC(), pdmR3FileR0() and pdmR3FileR3().
- *
- * @returns Pointer to temporary memory containing the filename.
- *          Caller must free this using RTMemTmpFree().
- * @returns NULL on failure.
- * @param   pszFile         File name (no path).
- * @param   pszDefaultExt   The default extention, NULL if none.
- * @param   pszSearchPath   List of directories to search if @a pszFile is
- *                          not qualified with a path.  Can be NULL, in which
- *                          case the arch dependent install dir is searched.
- * @param   fShared         If true, search in the shared directory (/usr/lib on Unix), else
- *                          search in the private directory (/usr/lib/virtualbox on Unix).
- *                          Ignored if VBOX_PATH_SHARED_LIBS is not defined.
- * @todo    We'll have this elsewhere than in the root later!
- * @todo    Remove the fShared hack again once we don't need to link against VBoxDD anymore!
- */
-static char *pdmR3File(const char *pszFile, const char *pszDefaultExt, const char *pszSearchPath, bool fShared)
-{
-    char szPath[RTPATH_MAX];
-    int  rc;
-
-    AssertLogRelReturn(!fShared || !pszSearchPath, NULL);
-    Assert(!RTPathHavePath(pszFile));
-
-    /*
-     * If there is a path, search it.
-     */
-    if (   pszSearchPath
-        && *pszSearchPath)
-    {
-        /* Check the filename length. */
-        size_t const    cchFile = strlen(pszFile);
-        if (cchFile >= sizeof(szPath))
-            return NULL;
-
-        /*
-         * Walk the search path.
-         */
-        const char *psz = pszSearchPath;
-        while (*psz)
-        {
-            /* Skip leading blanks - no directories with leading spaces, thank you. */
-            while (RT_C_IS_BLANK(*psz))
-                psz++;
-
-            /* Find the end of this element. */
-            const char *pszNext;
-            const char *pszEnd = strchr(psz, ';');
-            if (!pszEnd)
-                pszEnd = pszNext = strchr(psz, '\0');
-            else
-                pszNext = pszEnd + 1;
-            if (pszEnd != psz)
-            {
-                rc = RTPathJoinEx(szPath, sizeof(szPath), psz, pszEnd - psz, pszFile, cchFile);
-                if (RT_SUCCESS(rc))
-                {
-                    if (RTFileExists(szPath))
-                    {
-                        size_t cchPath = strlen(szPath) + 1;
-                        char *pszRet = (char *)RTMemTmpAlloc(cchPath);
-                        if (pszRet)
-                            memcpy(pszRet, szPath, cchPath);
-                        return pszRet;
-                    }
-                }
-            }
-
-            /* advance */
-            psz = pszNext;
-        }
-    }
-
-    /*
-     * Use the default location.
-     */
-    rc = fShared
-       ? RTPathSharedLibs(    szPath, sizeof(szPath))
-       : RTPathAppPrivateArch(szPath, sizeof(szPath));
-    if (!RT_SUCCESS(rc))
-    {
-        AssertMsgFailed(("RTPath[SharedLibs|AppPrivateArch](,%d) failed rc=%d!\n", sizeof(szPath), rc));
-        return NULL;
-    }
-
-    return pdmR3FileConstruct(szPath, pszFile, pszDefaultExt);
-}
-
-
-/** @internal */
-typedef struct QMFEIPARG
-{
-    RTINTPTR    uPC;
-
-    char       *pszNearSym1;
-    size_t      cchNearSym1;
-    RTINTPTR    offNearSym1;
-
-    char       *pszNearSym2;
-    size_t      cchNearSym2;
-    RTINTPTR    offNearSym2;
-} QMFEIPARG, *PQMFEIPARG;
-
-
-/**
- * Enumeration callback function used by RTLdrEnumSymbols().
- *
- * @returns VBox status code. Failure will stop the enumeration.
- * @param   hLdrMod         The loader module handle.
- * @param   pszSymbol       Symbol name. NULL if ordinal only.
- * @param   uSymbol         Symbol ordinal, ~0 if not used.
- * @param   Value           Symbol value.
- * @param   pvUser          The user argument specified to RTLdrEnumSymbols().
- */
-static DECLCALLBACK(int) pdmR3QueryModFromEIPEnumSymbols(RTLDRMOD hLdrMod, const char *pszSymbol, unsigned uSymbol, RTUINTPTR Value, void *pvUser)
-{
-    PQMFEIPARG pArgs = (PQMFEIPARG)pvUser;
-
-    RTINTPTR off = Value - pArgs->uPC;
-    if (off <= 0)   /* near1 is before or at same location. */
-    {
-        if (off > pArgs->offNearSym1)
-        {
-            pArgs->offNearSym1 = off;
-            if (pArgs->pszNearSym1 && pArgs->cchNearSym1)
-            {
-                *pArgs->pszNearSym1 = '\0';
-                if (pszSymbol)
-                    strncat(pArgs->pszNearSym1, pszSymbol, pArgs->cchNearSym1);
-                else
-                {
-                    char szOrd[32];
-                    RTStrPrintf(szOrd, sizeof(szOrd), "#%#x", uSymbol);
-                    strncat(pArgs->pszNearSym1, szOrd, pArgs->cchNearSym1);
-                }
-            }
-        }
-    }
-    else            /* near2 is after */
-    {
-        if (off < pArgs->offNearSym2)
-        {
-            pArgs->offNearSym2 = off;
-            if (pArgs->pszNearSym2 && pArgs->cchNearSym2)
-            {
-                *pArgs->pszNearSym2 = '\0';
-                if (pszSymbol)
-                    strncat(pArgs->pszNearSym2, pszSymbol, pArgs->cchNearSym2);
-                else
-                {
-                    char szOrd[32];
-                    RTStrPrintf(szOrd, sizeof(szOrd), "#%#x", uSymbol);
-                    strncat(pArgs->pszNearSym2, szOrd, pArgs->cchNearSym2);
-                }
-            }
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Internal worker for PDMR3LdrQueryRCModFromPC and PDMR3LdrQueryR0ModFromPC.
- *
- * @returns VBox status code.
- *
- * @param   pVM         VM handle
- * @param   uPC         The program counter (eip/rip) to locate the module for.
- * @param   enmType     The module type.
- * @param   pszModName  Where to store the module name.
- * @param   cchModName  Size of the module name buffer.
- * @param   pMod        Base address of the module.
- * @param   pszNearSym1 Name of the closes symbol from below.
- * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
- * @param   pNearSym1   The address of pszNearSym1.
- * @param   pszNearSym2 Name of the closes symbol from below.
- * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2.
- * @param   pNearSym2   The address of pszNearSym2.
- */
-static int pdmR3LdrQueryModFromPC(PVM pVM, RTUINTPTR uPC, PDMMODTYPE enmType,
-                                  char *pszModName,  size_t cchModName,  PRTUINTPTR pMod,
-                                  char *pszNearSym1, size_t cchNearSym1, PRTUINTPTR pNearSym1,
-                                  char *pszNearSym2, size_t cchNearSym2, PRTUINTPTR pNearSym2)
-{
-    PUVM    pUVM = pVM->pUVM;
-    int     rc   = VERR_MODULE_NOT_FOUND;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMMOD pCur= pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
-    {
-        if (pCur->eType != enmType)
-            continue;
-
-        /* The following RTLdrOpen call is a dirty hack to get ring-0 module information. */
-        RTLDRMOD hLdrMod = pCur->hLdrMod;
-        if (hLdrMod == NIL_RTLDRMOD && uPC >= pCur->ImageBase)
-        {
-            int rc2 = RTLdrOpen(pCur->szFilename, 0 /*fFlags*/, RTLDRARCH_HOST, &hLdrMod);
-            if (RT_FAILURE(rc2))
-                hLdrMod = NIL_RTLDRMOD;
-        }
-
-        if (   hLdrMod != NIL_RTLDRMOD
-            && uPC - pCur->ImageBase < RTLdrSize(hLdrMod))
-        {
-            if (pMod)
-                *pMod = pCur->ImageBase;
-            if (pszModName && cchModName)
-            {
-                *pszModName = '\0';
-                strncat(pszModName, pCur->szName, cchModName);
-            }
-            if (pNearSym1)   *pNearSym1   = 0;
-            if (pNearSym2)   *pNearSym2   = 0;
-            if (pszNearSym1) *pszNearSym1 = '\0';
-            if (pszNearSym2) *pszNearSym2 = '\0';
-
-            /*
-             * Locate the nearest symbols.
-             */
-            QMFEIPARG   Args;
-            Args.uPC         = uPC;
-            Args.pszNearSym1 = pszNearSym1;
-            Args.cchNearSym1 = cchNearSym1;
-            Args.offNearSym1 = RTINTPTR_MIN;
-            Args.pszNearSym2 = pszNearSym2;
-            Args.cchNearSym2 = cchNearSym2;
-            Args.offNearSym2 = RTINTPTR_MAX;
-
-            rc = RTLdrEnumSymbols(hLdrMod, RTLDR_ENUM_SYMBOL_FLAGS_ALL, pCur->pvBits, pCur->ImageBase,
-                                  pdmR3QueryModFromEIPEnumSymbols, &Args);
-            if (pNearSym1 && Args.offNearSym1 != RTINTPTR_MIN)
-                *pNearSym1 = Args.offNearSym1 + uPC;
-            if (pNearSym2 && Args.offNearSym2 != RTINTPTR_MAX)
-                *pNearSym2 = Args.offNearSym2 + uPC;
-
-            rc = VINF_SUCCESS;
-        }
-
-        if (hLdrMod != pCur->hLdrMod && hLdrMod != NIL_RTLDRMOD)
-            RTLdrClose(hLdrMod);
-
-        if (RT_SUCCESS(rc))
-            break;
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-/**
- * Queries raw-mode context module information from an PC (eip/rip).
- *
- * This is typically used to locate a crash address.
- *
- * @returns VBox status code.
- *
- * @param   pVM         VM handle
- * @param   uPC         The program counter (eip/rip) to locate the module for.
- * @param   pszModName  Where to store the module name.
- * @param   cchModName  Size of the module name buffer.
- * @param   pMod        Base address of the module.
- * @param   pszNearSym1 Name of the closes symbol from below.
- * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
- * @param   pNearSym1   The address of pszNearSym1.
- * @param   pszNearSym2 Name of the closes symbol from below.
- * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2.
- * @param   pNearSym2   The address of pszNearSym2.
- */
-VMMR3DECL(int) PDMR3LdrQueryRCModFromPC(PVM pVM, RTRCPTR uPC,
-                                        char *pszModName,  size_t cchModName,  PRTRCPTR pMod,
-                                        char *pszNearSym1, size_t cchNearSym1, PRTRCPTR pNearSym1,
-                                        char *pszNearSym2, size_t cchNearSym2, PRTRCPTR pNearSym2)
-{
-    RTUINTPTR AddrMod   = 0;
-    RTUINTPTR AddrNear1 = 0;
-    RTUINTPTR AddrNear2 = 0;
-    int rc = pdmR3LdrQueryModFromPC(pVM, uPC, PDMMOD_TYPE_RC,
-                                    pszModName,  cchModName,  &AddrMod,
-                                    pszNearSym1, cchNearSym1, &AddrNear1,
-                                    pszNearSym2, cchNearSym2, &AddrNear2);
-    if (RT_SUCCESS(rc))
-    {
-        if (pMod)
-            *pMod      = (RTRCPTR)AddrMod;
-        if (pNearSym1)
-            *pNearSym1 = (RTRCPTR)AddrNear1;
-        if (pNearSym2)
-            *pNearSym2 = (RTRCPTR)AddrNear2;
-    }
-    return rc;
-}
-
-
-/**
- * Queries ring-0 context module information from an PC (eip/rip).
- *
- * This is typically used to locate a crash address.
- *
- * @returns VBox status code.
- *
- * @param   pVM         VM handle
- * @param   uPC         The program counter (eip/rip) to locate the module for.
- * @param   pszModName  Where to store the module name.
- * @param   cchModName  Size of the module name buffer.
- * @param   pMod        Base address of the module.
- * @param   pszNearSym1 Name of the closes symbol from below.
- * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
- * @param   pNearSym1   The address of pszNearSym1.
- * @param   pszNearSym2 Name of the closes symbol from below.
- * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2. Optional.
- * @param   pNearSym2   The address of pszNearSym2. Optional.
- */
-VMMR3DECL(int) PDMR3LdrQueryR0ModFromPC(PVM pVM, RTR0PTR uPC,
-                                        char *pszModName,  size_t cchModName,  PRTR0PTR pMod,
-                                        char *pszNearSym1, size_t cchNearSym1, PRTR0PTR pNearSym1,
-                                        char *pszNearSym2, size_t cchNearSym2, PRTR0PTR pNearSym2)
-{
-    RTUINTPTR AddrMod   = 0;
-    RTUINTPTR AddrNear1 = 0;
-    RTUINTPTR AddrNear2 = 0;
-    int rc = pdmR3LdrQueryModFromPC(pVM, uPC, PDMMOD_TYPE_R0,
-                                    pszModName,  cchModName,  &AddrMod,
-                                    pszNearSym1, cchNearSym1, &AddrNear1,
-                                    pszNearSym2, cchNearSym2, &AddrNear2);
-    if (RT_SUCCESS(rc))
-    {
-        if (pMod)
-            *pMod      = (RTR0PTR)AddrMod;
-        if (pNearSym1)
-            *pNearSym1 = (RTR0PTR)AddrNear1;
-        if (pNearSym2)
-            *pNearSym2 = (RTR0PTR)AddrNear2;
-    }
-    return rc;
-}
-
-
-/**
- * Enumerate all PDM modules.
- *
- * @returns VBox status.
- * @param   pVM             VM Handle.
- * @param   pfnCallback     Function to call back for each of the modules.
- * @param   pvArg           User argument.
- */
-VMMR3DECL(int)  PDMR3LdrEnumModules(PVM pVM, PFNPDMR3ENUM pfnCallback, void *pvArg)
-{
-    PUVM pUVM = pVM->pUVM;
-    int  rc   = VINF_SUCCESS;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMMOD pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
-    {
-        rc = pfnCallback(pVM,
-                         pCur->szFilename,
-                         pCur->szName,
-                         pCur->ImageBase,
-                         pCur->eType == PDMMOD_TYPE_RC ? RTLdrSize(pCur->hLdrMod) : 0,
-                         pCur->eType == PDMMOD_TYPE_RC,
-                         pvArg);
-        if (RT_FAILURE(rc))
-            break;
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-/**
- * Locates a module.
- *
- * @returns Pointer to the module if found.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   pszModule       The module name.
- * @param   enmType         The module type.
- * @param   fLazy           Lazy loading the module if set.
- * @param   pszSearchPath   Search path for use when lazy loading.
- */
-static PPDMMOD pdmR3LdrFindModule(PUVM pUVM, const char *pszModule, PDMMODTYPE enmType,
-                                  bool fLazy, const char *pszSearchPath)
-{
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
-        if (    pModule->eType == enmType
-            &&  !strcmp(pModule->szName, pszModule))
-        {
-            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-            return pModule;
-        }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    if (fLazy)
-    {
-        switch (enmType)
-        {
-#ifdef VBOX_WITH_RAW_MODE
-            case PDMMOD_TYPE_RC:
-            {
-                char *pszFilename = pdmR3FileRC(pszModule, pszSearchPath);
-                if (pszFilename)
-                {
-                    int rc = PDMR3LdrLoadRC(pUVM->pVM, pszFilename, pszModule);
-                    RTMemTmpFree(pszFilename);
-                    if (RT_SUCCESS(rc))
-                        return pdmR3LdrFindModule(pUVM, pszModule, enmType, false, NULL);
-                }
-                break;
-            }
-#endif
-
-            case PDMMOD_TYPE_R0:
-            {
-                int rc = pdmR3LoadR0U(pUVM, NULL, pszModule, pszSearchPath);
-                if (RT_SUCCESS(rc))
-                    return pdmR3LdrFindModule(pUVM, pszModule, enmType, false, NULL);
-                break;
-            }
-
-            default:
-                AssertFailed();
-        }
-    }
-    return NULL;
-}
-
-
-/**
- * Resolves a ring-0 or raw-mode context interface.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pvInterface     Pointer to the interface structure.  The symbol list
- *                          describes the layout.
- * @param   cbInterface     The size of the structure pvInterface is pointing
- *                          to.  For bounds checking.
- * @param   pszModule       The module name.  If NULL we assume it's the default
- *                          R0 or RC module (@a fRing0OrRC).  We'll attempt to
- *                          load the module if it isn't found in the module
- *                          list.
- * @param   pszSearchPath   The module search path.  If NULL, search the
- *                          architecture dependent install directory.
- * @param   pszSymPrefix    What to prefix the symbols in the list with.  The
- *                          idea is that you define a list that goes with an
- *                          interface (INTERFACE_SYM_LIST) and reuse it with
- *                          each implementation.
- * @param   pszSymList      The symbol list for the interface.  This is a
- *                          semi-colon separated list of symbol base names.  As
- *                          mentioned above, each is prefixed with @a
- *                          pszSymPrefix before resolving.  There are a couple
- *                          of special symbol names that will cause us to skip
- *                          ahead a little bit:
- *                              - U8:whatever,
- *                              - U16:whatever,
- *                              - U32:whatever,
- *                              - U64:whatever,
- *                              - RCPTR:whatever,
- *                              - R3PTR:whatever,
- *                              - R0PTR:whatever,
- *                              - GCPHYS:whatever,
- *                              - HCPHYS:whatever.
- * @param   fRing0          Set if it's a ring-0 context interface, clear if
- *                          it's raw-mode context interface.
- */
-VMMR3DECL(int) PDMR3LdrGetInterfaceSymbols(PVM pVM, void *pvInterface, size_t cbInterface,
-                                           const char *pszModule, const char *pszSearchPath,
-                                           const char *pszSymPrefix, const char *pszSymList,
-                                           bool fRing0)
-{
-    /*
-     * Find the module.
-     */
-    int     rc      = VINF_SUCCESS;
-    PPDMMOD pModule = pdmR3LdrFindModule(pVM->pUVM,
-                                         pszModule ? pszModule : fRing0 ? "VMMR0.r0" : "VMMGC.gc",
-                                         fRing0 ? PDMMOD_TYPE_R0 : PDMMOD_TYPE_RC,
-                                         true /*fLazy*/, pszSearchPath);
-    if (pModule)
-    {
-        /* Prep the symbol name. */
-        char            szSymbol[256];
-        size_t const    cchSymPrefix = strlen(pszSymPrefix);
-        AssertReturn(cchSymPrefix + 5 < sizeof(szSymbol), VERR_SYMBOL_NOT_FOUND);
-        memcpy(szSymbol, pszSymPrefix, cchSymPrefix);
-
-        /*
-         * Iterate the symbol list.
-         */
-        uint32_t        offInterface = 0;
-        const char     *pszCur       = pszSymList;
-        while (pszCur)
-        {
-            /*
-             * Find the end of the current symbol name.
-             */
-            size_t      cchSym;
-            const char *pszNext = strchr(pszCur, ';');
-            if (pszNext)
-            {
-                cchSym = pszNext - pszCur;
-                pszNext++;
-            }
-            else
-                cchSym = strlen(pszCur);
-            AssertBreakStmt(cchSym > 0, rc = VERR_INVALID_PARAMETER);
-
-            /* Is it a skip instruction? */
-            const char *pszColon = (const char *)memchr(pszCur, ':', cchSym);
-            if (pszColon)
-            {
-                /*
-                 * String switch on the instruction and execute it, checking
-                 * that we didn't overshoot the interface structure.
-                 */
-#define IS_SKIP_INSTR(szInstr) \
-                (   cchSkip == sizeof(szInstr) - 1 \
-                 && !memcmp(pszCur, szInstr, sizeof(szInstr) - 1) )
-
-                size_t const cchSkip = pszColon - pszCur;
-                if (IS_SKIP_INSTR("U8"))
-                    offInterface += sizeof(uint8_t);
-                else if (IS_SKIP_INSTR("U16"))
-                    offInterface += sizeof(uint16_t);
-                else if (IS_SKIP_INSTR("U32"))
-                    offInterface += sizeof(uint32_t);
-                else if (IS_SKIP_INSTR("U64"))
-                    offInterface += sizeof(uint64_t);
-                else if (IS_SKIP_INSTR("RCPTR"))
-                    offInterface += sizeof(RTRCPTR);
-                else if (IS_SKIP_INSTR("R3PTR"))
-                    offInterface += sizeof(RTR3PTR);
-                else if (IS_SKIP_INSTR("R0PTR"))
-                    offInterface += sizeof(RTR0PTR);
-                else if (IS_SKIP_INSTR("HCPHYS"))
-                    offInterface += sizeof(RTHCPHYS);
-                else if (IS_SKIP_INSTR("GCPHYS"))
-                    offInterface += sizeof(RTGCPHYS);
-                else
-                    AssertMsgFailedBreakStmt(("Invalid skip instruction %.*s (prefix=%s)\n", cchSym, pszCur, pszSymPrefix),
-                                             rc = VERR_INVALID_PARAMETER);
-                AssertMsgBreakStmt(offInterface <= cbInterface,
-                                   ("off=%#x cb=%#x (sym=%.*s prefix=%s)\n", offInterface, cbInterface, cchSym, pszCur, pszSymPrefix),
-                                   rc = VERR_BUFFER_OVERFLOW);
-#undef IS_SKIP_INSTR
-            }
-            else
-            {
-                /*
-                 * Construct the symbol name, get its value, store it and
-                 * advance the interface cursor.
-                 */
-                AssertReturn(cchSymPrefix + cchSym < sizeof(szSymbol), VERR_SYMBOL_NOT_FOUND);
-                memcpy(&szSymbol[cchSymPrefix], pszCur, cchSym);
-                szSymbol[cchSymPrefix + cchSym] = '\0';
-
-                if (fRing0)
-                {
-                    void *pvValue;
-                    rc = SUPR3GetSymbolR0((void *)(RTR0PTR)pModule->ImageBase, szSymbol, &pvValue);
-                    AssertMsgRCBreak(rc, ("Couldn't find symbol '%s' in module '%s'\n", szSymbol, pModule->szName));
-
-                    PRTR0PTR pValue = (PRTR0PTR)((uintptr_t)pvInterface + offInterface);
-                    AssertMsgBreakStmt(offInterface + sizeof(*pValue) <= cbInterface,
-                                       ("off=%#x cb=%#x sym=%s\n", offInterface, cbInterface, szSymbol),
-                                       rc = VERR_BUFFER_OVERFLOW);
-                    *pValue = (RTR0PTR)pvValue;
-                    Assert((void *)*pValue == pvValue);
-                    offInterface += sizeof(*pValue);
-                }
-                else
-                {
-                    RTUINTPTR Value;
-                    rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, szSymbol, &Value);
-                    AssertMsgRCBreak(rc, ("Couldn't find symbol '%s' in module '%s'\n", szSymbol, pModule->szName));
-
-                    PRTRCPTR pValue = (PRTRCPTR)((uintptr_t)pvInterface + offInterface);
-                    AssertMsgBreakStmt(offInterface + sizeof(*pValue) <= cbInterface,
-                                       ("off=%#x cb=%#x sym=%s\n", offInterface, cbInterface, szSymbol),
-                                       rc = VERR_BUFFER_OVERFLOW);
-                    *pValue = (RTRCPTR)Value;
-                    Assert(*pValue == Value);
-                    offInterface += sizeof(*pValue);
-                }
-            }
-
-            /* advance */
-            pszCur = pszNext;
-        }
-
-    }
-    else
-        rc = VERR_MODULE_NOT_FOUND;
-    return rc;
-}
-
Index: unk/src/VBox/VMM/PDMQueue.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMQueue.cpp	(revision 35332)
+++ 	(revision )
@@ -1,906 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Queue - Transport data and tasks to EMT and R3.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_QUEUE
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/rem.h>
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-DECLINLINE(void)            pdmR3QueueFree(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem);
-static bool                 pdmR3QueueFlush(PPDMQUEUE pQueue);
-static DECLCALLBACK(void)   pdmR3QueueTimer(PVM pVM, PTMTIMER pTimer, void *pvUser);
-
-
-
-/**
- * Internal worker for the queue creation apis.
- *
- * @returns VBox status.
- * @param   pVM                 VM handle.
- * @param   cbItem              Item size.
- * @param   cItems              Number of items.
- * @param   cMilliesInterval    Number of milliseconds between polling the queue.
- *                              If 0 then the emulation thread will be notified whenever an item arrives.
- * @param   fRZEnabled          Set if the queue will be used from RC/R0 and need to be allocated from the hyper heap.
- * @param   pszName             The queue name. Unique. Not copied.
- * @param   ppQueue             Where to store the queue handle.
- */
-static int pdmR3QueueCreate(PVM pVM, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval, bool fRZEnabled,
-                            const char *pszName, PPDMQUEUE *ppQueue)
-{
-    PUVM pUVM = pVM->pUVM;
-
-    /*
-     * Validate input.
-     */
-    if (cbItem < sizeof(PDMQUEUEITEMCORE))
-    {
-        AssertMsgFailed(("cbItem=%d\n", cbItem));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (cItems < 1 || cItems >= 0x10000)
-    {
-        AssertMsgFailed(("cItems=%d valid:[1-65535]\n", cItems));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Align the item size and calculate the structure size.
-     */
-    cbItem = RT_ALIGN(cbItem, sizeof(RTUINTPTR));
-    size_t cb = cbItem * cItems + RT_ALIGN_Z(RT_OFFSETOF(PDMQUEUE, aFreeItems[cItems + PDMQUEUE_FREE_SLACK]), 16);
-    PPDMQUEUE pQueue;
-    int rc;
-    if (fRZEnabled)
-        rc = MMHyperAlloc(pVM, cb, 0, MM_TAG_PDM_QUEUE, (void **)&pQueue );
-    else
-        rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_QUEUE, cb, (void **)&pQueue);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Initialize the data fields.
-     */
-    pQueue->pVMR3 = pVM;
-    pQueue->pVMR0 = fRZEnabled ? pVM->pVMR0 : NIL_RTR0PTR;
-    pQueue->pVMRC = fRZEnabled ? pVM->pVMRC : NIL_RTRCPTR;
-    pQueue->pszName = pszName;
-    pQueue->cMilliesInterval = cMilliesInterval;
-    //pQueue->pTimer = NULL;
-    pQueue->cbItem = cbItem;
-    pQueue->cItems = cItems;
-    //pQueue->pPendingR3 = NULL;
-    //pQueue->pPendingR0 = NULL;
-    //pQueue->pPendingRC = NULL;
-    pQueue->iFreeHead = cItems;
-    //pQueue->iFreeTail = 0;
-    PPDMQUEUEITEMCORE pItem = (PPDMQUEUEITEMCORE)((char *)pQueue + RT_ALIGN_Z(RT_OFFSETOF(PDMQUEUE, aFreeItems[cItems + PDMQUEUE_FREE_SLACK]), 16));
-    for (unsigned i = 0; i < cItems; i++, pItem = (PPDMQUEUEITEMCORE)((char *)pItem + cbItem))
-    {
-        pQueue->aFreeItems[i].pItemR3 = pItem;
-        if (fRZEnabled)
-        {
-            pQueue->aFreeItems[i].pItemR0 = MMHyperR3ToR0(pVM, pItem);
-            pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pVM, pItem);
-        }
-    }
-
-    /*
-     * Create timer?
-     */
-    if (cMilliesInterval)
-    {
-        rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, pdmR3QueueTimer, pQueue, "Queue timer", &pQueue->pTimer);
-        if (RT_SUCCESS(rc))
-        {
-            rc = TMTimerSetMillies(pQueue->pTimer, cMilliesInterval);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("TMTimerSetMillies failed rc=%Rrc\n", rc));
-                int rc2 = TMR3TimerDestroy(pQueue->pTimer); AssertRC(rc2);
-            }
-        }
-        else
-            AssertMsgFailed(("TMR3TimerCreateInternal failed rc=%Rrc\n", rc));
-        if (RT_FAILURE(rc))
-        {
-            if (fRZEnabled)
-                MMHyperFree(pVM, pQueue);
-            else
-                MMR3HeapFree(pQueue);
-            return rc;
-        }
-
-        /*
-         * Insert into the queue list for timer driven queues.
-         */
-        pdmLock(pVM);
-        pQueue->pNext = pUVM->pdm.s.pQueuesTimer;
-        pUVM->pdm.s.pQueuesTimer = pQueue;
-        pdmUnlock(pVM);
-    }
-    else
-    {
-        /*
-         * Insert into the queue list for forced action driven queues.
-         * This is a FIFO, so insert at the end.
-         */
-        /** @todo we should add a priority to the queues so we don't have to rely on
-         * the initialization order to deal with problems like #1605 (pgm/pcnet deadlock
-         * caused by the critsect queue to be last in the chain).
-         * - Update, the critical sections are no longer using queues, so this isn't a real
-         *   problem any longer. The priority might be a nice feature for later though.
-         */
-        pdmLock(pVM);
-        if (!pUVM->pdm.s.pQueuesForced)
-            pUVM->pdm.s.pQueuesForced = pQueue;
-        else
-        {
-            PPDMQUEUE pPrev = pUVM->pdm.s.pQueuesForced;
-            while (pPrev->pNext)
-                pPrev = pPrev->pNext;
-            pPrev->pNext = pQueue;
-        }
-        pdmUnlock(pVM);
-    }
-
-    /*
-     * Register the statistics.
-     */
-    STAMR3RegisterF(pVM, &pQueue->cbItem,               STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,        "Item size.",                       "/PDM/Queue/%s/cbItem",         pQueue->pszName);
-    STAMR3RegisterF(pVM, &pQueue->cItems,               STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,        "Queue size.",                      "/PDM/Queue/%s/cItems",         pQueue->pszName);
-    STAMR3RegisterF(pVM, &pQueue->StatAllocFailures,    STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,   "PDMQueueAlloc failures.",          "/PDM/Queue/%s/AllocFailures",  pQueue->pszName);
-    STAMR3RegisterF(pVM, &pQueue->StatInsert,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Calls to PDMQueueInsert.",         "/PDM/Queue/%s/Insert",         pQueue->pszName);
-    STAMR3RegisterF(pVM, &pQueue->StatFlush,            STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Calls to pdmR3QueueFlush.",        "/PDM/Queue/%s/Flush",          pQueue->pszName);
-    STAMR3RegisterF(pVM, &pQueue->StatFlushLeftovers,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,   "Left over items after flush.",     "/PDM/Queue/%s/FlushLeftovers", pQueue->pszName);
-#ifdef VBOX_WITH_STATISTICS
-    STAMR3RegisterF(pVM, &pQueue->StatFlushPrf,         STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Profiling pdmR3QueueFlush.",       "/PDM/Queue/%s/FlushPrf",       pQueue->pszName);
-    STAMR3RegisterF(pVM, (void *)&pQueue->cStatPending, STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,        "Pending items.",                   "/PDM/Queue/%s/Pending",        pQueue->pszName);
-#endif
-
-    *ppQueue = pQueue;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Create a queue with a device owner.
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle.
- * @param   pDevIns             Device instance.
- * @param   cbItem              Size a queue item.
- * @param   cItems              Number of items in the queue.
- * @param   cMilliesInterval    Number of milliseconds between polling the queue.
- *                              If 0 then the emulation thread will be notified whenever an item arrives.
- * @param   pfnCallback         The consumer function.
- * @param   fRZEnabled          Set if the queue must be usable from RC/R0.
- * @param   pszName             The queue name. Unique. Not copied.
- * @param   ppQueue             Where to store the queue handle on success.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(int) PDMR3QueueCreateDevice(PVM pVM, PPDMDEVINS pDevIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
-                                      PFNPDMQUEUEDEV pfnCallback, bool fRZEnabled, const char *pszName, PPDMQUEUE *ppQueue)
-{
-    LogFlow(("PDMR3QueueCreateDevice: pDevIns=%p cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p fRZEnabled=%RTbool pszName=%s\n",
-             pDevIns, cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName));
-
-    /*
-     * Validate input.
-     */
-    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
-    if (!pfnCallback)
-    {
-        AssertMsgFailed(("No consumer callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the queue.
-     */
-    PPDMQUEUE pQueue;
-    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, fRZEnabled, pszName, &pQueue);
-    if (RT_SUCCESS(rc))
-    {
-        pQueue->enmType = PDMQUEUETYPE_DEV;
-        pQueue->u.Dev.pDevIns = pDevIns;
-        pQueue->u.Dev.pfnCallback = pfnCallback;
-
-        *ppQueue = pQueue;
-        Log(("PDM: Created device queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pDevIns=%p\n",
-             cbItem, cItems, cMilliesInterval, pfnCallback, pDevIns));
-    }
-    return rc;
-}
-
-
-/**
- * Create a queue with a driver owner.
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle.
- * @param   pDrvIns             Driver instance.
- * @param   cbItem              Size a queue item.
- * @param   cItems              Number of items in the queue.
- * @param   cMilliesInterval    Number of milliseconds between polling the queue.
- *                              If 0 then the emulation thread will be notified whenever an item arrives.
- * @param   pfnCallback         The consumer function.
- * @param   pszName             The queue name. Unique. Not copied.
- * @param   ppQueue             Where to store the queue handle on success.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(int) PDMR3QueueCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
-                                      PFNPDMQUEUEDRV pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
-{
-    LogFlow(("PDMR3QueueCreateDriver: pDrvIns=%p cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%s\n",
-             pDrvIns, cbItem, cItems, cMilliesInterval, pfnCallback, pszName));
-
-    /*
-     * Validate input.
-     */
-    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
-    if (!pfnCallback)
-    {
-        AssertMsgFailed(("No consumer callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the queue.
-     */
-    PPDMQUEUE pQueue;
-    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, false, pszName, &pQueue);
-    if (RT_SUCCESS(rc))
-    {
-        pQueue->enmType = PDMQUEUETYPE_DRV;
-        pQueue->u.Drv.pDrvIns = pDrvIns;
-        pQueue->u.Drv.pfnCallback = pfnCallback;
-
-        *ppQueue = pQueue;
-        Log(("PDM: Created driver queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pDrvIns=%p\n",
-             cbItem, cItems, cMilliesInterval, pfnCallback, pDrvIns));
-    }
-    return rc;
-}
-
-
-/**
- * Create a queue with an internal owner.
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle.
- * @param   cbItem              Size a queue item.
- * @param   cItems              Number of items in the queue.
- * @param   cMilliesInterval    Number of milliseconds between polling the queue.
- *                              If 0 then the emulation thread will be notified whenever an item arrives.
- * @param   pfnCallback         The consumer function.
- * @param   fRZEnabled          Set if the queue must be usable from RC/R0.
- * @param   pszName             The queue name. Unique. Not copied.
- * @param   ppQueue             Where to store the queue handle on success.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(int) PDMR3QueueCreateInternal(PVM pVM, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
-                                        PFNPDMQUEUEINT pfnCallback, bool fRZEnabled, const char *pszName, PPDMQUEUE *ppQueue)
-{
-    LogFlow(("PDMR3QueueCreateInternal: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p fRZEnabled=%RTbool pszName=%s\n",
-             cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName));
-
-    /*
-     * Validate input.
-     */
-    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
-    if (!pfnCallback)
-    {
-        AssertMsgFailed(("No consumer callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the queue.
-     */
-    PPDMQUEUE pQueue;
-    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, fRZEnabled, pszName, &pQueue);
-    if (RT_SUCCESS(rc))
-    {
-        pQueue->enmType = PDMQUEUETYPE_INTERNAL;
-        pQueue->u.Int.pfnCallback = pfnCallback;
-
-        *ppQueue = pQueue;
-        Log(("PDM: Created internal queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p\n",
-             cbItem, cItems, cMilliesInterval, pfnCallback));
-    }
-    return rc;
-}
-
-
-/**
- * Create a queue with an external owner.
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle.
- * @param   cbItem              Size a queue item.
- * @param   cItems              Number of items in the queue.
- * @param   cMilliesInterval    Number of milliseconds between polling the queue.
- *                              If 0 then the emulation thread will be notified whenever an item arrives.
- * @param   pfnCallback         The consumer function.
- * @param   pvUser              The user argument to the consumer function.
- * @param   pszName             The queue name. Unique. Not copied.
- * @param   ppQueue             Where to store the queue handle on success.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(int) PDMR3QueueCreateExternal(PVM pVM, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval, PFNPDMQUEUEEXT pfnCallback, void *pvUser, const char *pszName, PPDMQUEUE *ppQueue)
-{
-    LogFlow(("PDMR3QueueCreateExternal: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%s\n", cbItem, cItems, cMilliesInterval, pfnCallback, pszName));
-
-    /*
-     * Validate input.
-     */
-    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
-    if (!pfnCallback)
-    {
-        AssertMsgFailed(("No consumer callback!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Create the queue.
-     */
-    PPDMQUEUE pQueue;
-    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, false, pszName, &pQueue);
-    if (RT_SUCCESS(rc))
-    {
-        pQueue->enmType = PDMQUEUETYPE_EXTERNAL;
-        pQueue->u.Ext.pvUser = pvUser;
-        pQueue->u.Ext.pfnCallback = pfnCallback;
-
-        *ppQueue = pQueue;
-        Log(("PDM: Created external queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pvUser=%p\n",
-             cbItem, cItems, cMilliesInterval, pfnCallback, pvUser));
-    }
-    return rc;
-}
-
-
-/**
- * Destroy a queue.
- *
- * @returns VBox status code.
- * @param   pQueue      Queue to destroy.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(int) PDMR3QueueDestroy(PPDMQUEUE pQueue)
-{
-    LogFlow(("PDMR3QueueDestroy: pQueue=%p\n", pQueue));
-
-    /*
-     * Validate input.
-     */
-    if (!pQueue)
-        return VERR_INVALID_PARAMETER;
-    Assert(pQueue && pQueue->pVMR3);
-    PVM     pVM  = pQueue->pVMR3;
-    PUVM    pUVM = pVM->pUVM;
-
-    pdmLock(pVM);
-
-    /*
-     * Unlink it.
-     */
-    if (pQueue->pTimer)
-    {
-        if (pUVM->pdm.s.pQueuesTimer != pQueue)
-        {
-            PPDMQUEUE pCur = pUVM->pdm.s.pQueuesTimer;
-            while (pCur)
-            {
-                if (pCur->pNext == pQueue)
-                {
-                    pCur->pNext = pQueue->pNext;
-                    break;
-                }
-                pCur = pCur->pNext;
-            }
-            AssertMsg(pCur, ("Didn't find the queue!\n"));
-        }
-        else
-            pUVM->pdm.s.pQueuesTimer = pQueue->pNext;
-    }
-    else
-    {
-        if (pUVM->pdm.s.pQueuesForced != pQueue)
-        {
-            PPDMQUEUE pCur = pUVM->pdm.s.pQueuesForced;
-            while (pCur)
-            {
-                if (pCur->pNext == pQueue)
-                {
-                    pCur->pNext = pQueue->pNext;
-                    break;
-                }
-                pCur = pCur->pNext;
-            }
-            AssertMsg(pCur, ("Didn't find the queue!\n"));
-        }
-        else
-            pUVM->pdm.s.pQueuesForced = pQueue->pNext;
-    }
-    pQueue->pNext = NULL;
-    pQueue->pVMR3 = NULL;
-    pdmUnlock(pVM);
-
-    /*
-     * Deregister statistics.
-     */
-    STAMR3Deregister(pVM, &pQueue->cbItem);
-    STAMR3Deregister(pVM, &pQueue->cbItem);
-    STAMR3Deregister(pVM, &pQueue->StatAllocFailures);
-    STAMR3Deregister(pVM, &pQueue->StatInsert);
-    STAMR3Deregister(pVM, &pQueue->StatFlush);
-    STAMR3Deregister(pVM, &pQueue->StatFlushLeftovers);
-#ifdef VBOX_WITH_STATISTICS
-    STAMR3Deregister(pVM, &pQueue->StatFlushPrf);
-    STAMR3Deregister(pVM, (void *)&pQueue->cStatPending);
-#endif
-
-    /*
-     * Destroy the timer and free it.
-     */
-    if (pQueue->pTimer)
-    {
-        TMR3TimerDestroy(pQueue->pTimer);
-        pQueue->pTimer = NULL;
-    }
-    if (pQueue->pVMRC)
-    {
-        pQueue->pVMRC = NIL_RTRCPTR;
-        pQueue->pVMR0 = NIL_RTR0PTR;
-        MMHyperFree(pVM, pQueue);
-    }
-    else
-        MMR3HeapFree(pQueue);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Destroy a all queues owned by the specified device.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pDevIns     Device instance.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(int) PDMR3QueueDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
-{
-    LogFlow(("PDMR3QueueDestroyDevice: pDevIns=%p\n", pDevIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pDevIns)
-        return VERR_INVALID_PARAMETER;
-
-    PUVM pUVM = pVM->pUVM;
-    pdmLock(pVM);
-
-    /*
-     * Unlink it.
-     */
-    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
-    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
-    do
-    {
-        while (pQueue)
-        {
-            if (    pQueue->enmType == PDMQUEUETYPE_DEV
-                &&  pQueue->u.Dev.pDevIns == pDevIns)
-            {
-                PPDMQUEUE pQueueDestroy = pQueue;
-                pQueue = pQueue->pNext;
-                int rc = PDMR3QueueDestroy(pQueueDestroy);
-                AssertRC(rc);
-            }
-            else
-                pQueue = pQueue->pNext;
-        }
-
-        /* next queue list */
-        pQueue = pQueueNext;
-        pQueueNext = NULL;
-    } while (pQueue);
-
-    pdmUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Destroy a all queues owned by the specified driver.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pDrvIns     Driver instance.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(int) PDMR3QueueDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
-{
-    LogFlow(("PDMR3QueueDestroyDriver: pDrvIns=%p\n", pDrvIns));
-
-    /*
-     * Validate input.
-     */
-    if (!pDrvIns)
-        return VERR_INVALID_PARAMETER;
-
-    PUVM pUVM = pVM->pUVM;
-    pdmLock(pVM);
-
-    /*
-     * Unlink it.
-     */
-    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
-    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
-    do
-    {
-        while (pQueue)
-        {
-            if (    pQueue->enmType == PDMQUEUETYPE_DRV
-                &&  pQueue->u.Drv.pDrvIns == pDrvIns)
-            {
-                PPDMQUEUE pQueueDestroy = pQueue;
-                pQueue = pQueue->pNext;
-                int rc = PDMR3QueueDestroy(pQueueDestroy);
-                AssertRC(rc);
-            }
-            else
-                pQueue = pQueue->pNext;
-        }
-
-        /* next queue list */
-        pQueue = pQueueNext;
-        pQueueNext = NULL;
-    } while (pQueue);
-
-    pdmUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Relocate the queues.
- *
- * @param   pVM             The VM handle.
- * @param   offDelta        The relocation delta.
- */
-void pdmR3QueueRelocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    /*
-     * Process the queues.
-     */
-    PUVM      pUVM       = pVM->pUVM;
-    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
-    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
-    do
-    {
-        while (pQueue)
-        {
-            if (pQueue->pVMRC)
-            {
-                pQueue->pVMRC = pVM->pVMRC;
-
-                /* Pending RC items. */
-                if (pQueue->pPendingRC)
-                {
-                    pQueue->pPendingRC += offDelta;
-                    PPDMQUEUEITEMCORE pCur = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pVM, pQueue->pPendingRC);
-                    while (pCur->pNextRC)
-                    {
-                        pCur->pNextRC += offDelta;
-                        pCur = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pVM, pCur->pNextRC);
-                    }
-                }
-
-                /* The free items. */
-                uint32_t i = pQueue->iFreeTail;
-                while (i != pQueue->iFreeHead)
-                {
-                    pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pVM, pQueue->aFreeItems[i].pItemR3);
-                    i = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
-                }
-            }
-
-            /* next queue */
-            pQueue = pQueue->pNext;
-        }
-
-        /* next queue list */
-        pQueue = pQueueNext;
-        pQueueNext = NULL;
-    } while (pQueue);
-}
-
-
-/**
- * Flush pending queues.
- * This is a forced action callback.
- *
- * @param   pVM     VM handle.
- * @thread  Emulation thread only.
- */
-VMMR3DECL(void) PDMR3QueueFlushAll(PVM pVM)
-{
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("PDMR3QueuesFlush:\n"));
-
-    /*
-     * Only let one EMT flushing queues at any one time to preserve the order
-     * and to avoid wasting time. The FF is always cleared here, because it's
-     * only used to get someones attention. Queue inserts occurring during the
-     * flush are caught using the pending bit.
-     *
-     * Note! We must check the force action and pending flags after clearing
-     *       the active bit!
-     */
-    VM_FF_CLEAR(pVM, VM_FF_PDM_QUEUES);
-    while (!ASMAtomicBitTestAndSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT))
-    {
-        ASMAtomicBitClear(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT);
-
-        for (PPDMQUEUE pCur = pVM->pUVM->pdm.s.pQueuesForced; pCur; pCur = pCur->pNext)
-            if (    pCur->pPendingR3
-                ||  pCur->pPendingR0
-                ||  pCur->pPendingRC)
-                pdmR3QueueFlush(pCur);
-
-        ASMAtomicBitClear(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT);
-
-        /* We're done if there were no inserts while we were busy. */
-        if (   !ASMBitTest(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT)
-            && !VM_FF_ISPENDING(pVM, VM_FF_PDM_QUEUES))
-            break;
-        VM_FF_CLEAR(pVM, VM_FF_PDM_QUEUES);
-    }
-}
-
-
-/**
- * Process pending items in one queue.
- *
- * @returns Success indicator.
- *          If false the item the consumer said "enough!".
- * @param   pQueue  The queue.
- */
-static bool pdmR3QueueFlush(PPDMQUEUE pQueue)
-{
-    STAM_PROFILE_START(&pQueue->StatFlushPrf,p);
-
-    /*
-     * Get the lists.
-     */
-    PPDMQUEUEITEMCORE pItems   = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
-    RTRCPTR           pItemsRC = ASMAtomicXchgRCPtr(&pQueue->pPendingRC, NIL_RTRCPTR);
-    RTR0PTR           pItemsR0 = ASMAtomicXchgR0Ptr(&pQueue->pPendingR0, NIL_RTR0PTR);
-
-    AssertMsgReturn(   pItemsR0
-                    || pItemsRC
-                    || pItems,
-                    ("Someone is racing us? This shouldn't happen!\n"),
-                    true);
-
-    /*
-     * Reverse the list (it's inserted in LIFO order to avoid semaphores, remember).
-     */
-    PPDMQUEUEITEMCORE pCur = pItems;
-    pItems = NULL;
-    while (pCur)
-    {
-        PPDMQUEUEITEMCORE pInsert = pCur;
-        pCur = pCur->pNextR3;
-        pInsert->pNextR3 = pItems;
-        pItems = pInsert;
-    }
-
-    /*
-     * Do the same for any pending RC items.
-     */
-    while (pItemsRC)
-    {
-        PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pQueue->pVMR3, pItemsRC);
-        pItemsRC = pInsert->pNextRC;
-        pInsert->pNextRC = NIL_RTRCPTR;
-        pInsert->pNextR3 = pItems;
-        pItems = pInsert;
-    }
-
-    /*
-     * Do the same for any pending R0 items.
-     */
-    while (pItemsR0)
-    {
-        PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperR0ToR3(pQueue->pVMR3, pItemsR0);
-        pItemsR0 = pInsert->pNextR0;
-        pInsert->pNextR0 = NIL_RTR0PTR;
-        pInsert->pNextR3 = pItems;
-        pItems = pInsert;
-    }
-
-    /*
-     * Feed the items to the consumer function.
-     */
-    Log2(("pdmR3QueueFlush: pQueue=%p enmType=%d pItems=%p\n", pQueue, pQueue->enmType, pItems));
-    switch (pQueue->enmType)
-    {
-        case PDMQUEUETYPE_DEV:
-            while (pItems)
-            {
-                if (!pQueue->u.Dev.pfnCallback(pQueue->u.Dev.pDevIns, pItems))
-                    break;
-                pCur = pItems;
-                pItems = pItems->pNextR3;
-                pdmR3QueueFree(pQueue, pCur);
-            }
-            break;
-
-        case PDMQUEUETYPE_DRV:
-            while (pItems)
-            {
-                if (!pQueue->u.Drv.pfnCallback(pQueue->u.Drv.pDrvIns, pItems))
-                    break;
-                pCur = pItems;
-                pItems = pItems->pNextR3;
-                pdmR3QueueFree(pQueue, pCur);
-            }
-            break;
-
-        case PDMQUEUETYPE_INTERNAL:
-            while (pItems)
-            {
-                if (!pQueue->u.Int.pfnCallback(pQueue->pVMR3, pItems))
-                    break;
-                pCur = pItems;
-                pItems = pItems->pNextR3;
-                pdmR3QueueFree(pQueue, pCur);
-            }
-            break;
-
-        case PDMQUEUETYPE_EXTERNAL:
-            while (pItems)
-            {
-                if (!pQueue->u.Ext.pfnCallback(pQueue->u.Ext.pvUser, pItems))
-                    break;
-                pCur = pItems;
-                pItems = pItems->pNextR3;
-                pdmR3QueueFree(pQueue, pCur);
-            }
-            break;
-
-        default:
-            AssertMsgFailed(("Invalid queue type %d\n", pQueue->enmType));
-            break;
-    }
-
-    /*
-     * Success?
-     */
-    if (pItems)
-    {
-        /*
-         * Reverse the list.
-         */
-        pCur = pItems;
-        pItems = NULL;
-        while (pCur)
-        {
-            PPDMQUEUEITEMCORE pInsert = pCur;
-            pCur = pInsert->pNextR3;
-            pInsert->pNextR3 = pItems;
-            pItems = pInsert;
-        }
-
-        /*
-         * Insert the list at the tail of the pending list.
-         */
-        for (;;)
-        {
-            if (ASMAtomicCmpXchgPtr(&pQueue->pPendingR3, pItems, NULL))
-                break;
-            PPDMQUEUEITEMCORE pPending = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
-            if (pPending)
-            {
-                pCur = pPending;
-                while (pCur->pNextR3)
-                    pCur = pCur->pNextR3;
-                pCur->pNextR3 = pItems;
-                pItems = pPending;
-            }
-        }
-
-        STAM_REL_COUNTER_INC(&pQueue->StatFlushLeftovers);
-        STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
-        return false;
-    }
-
-    STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
-    return true;
-}
-
-
-/**
- * Free an item.
- *
- * @param   pQueue  The queue.
- * @param   pItem   The item.
- */
-DECLINLINE(void) pdmR3QueueFree(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem)
-{
-    VM_ASSERT_EMT(pQueue->pVMR3);
-
-    int i = pQueue->iFreeHead;
-    int iNext = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
-
-    pQueue->aFreeItems[i].pItemR3 = pItem;
-    if (pQueue->pVMRC)
-    {
-        pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pQueue->pVMR3, pItem);
-        pQueue->aFreeItems[i].pItemR0 = MMHyperR3ToR0(pQueue->pVMR3, pItem);
-    }
-
-    if (!ASMAtomicCmpXchgU32(&pQueue->iFreeHead, iNext, i))
-        AssertMsgFailed(("huh? i=%d iNext=%d iFreeHead=%d iFreeTail=%d\n", i, iNext, pQueue->iFreeHead, pQueue->iFreeTail));
-    STAM_STATS({ ASMAtomicDecU32(&pQueue->cStatPending); });
-}
-
-
-/**
- * Timer handler for PDM queues.
- * This is called by for a single queue.
- *
- * @param   pVM     VM handle.
- * @param   pTimer  Pointer to timer.
- * @param   pvUser  Pointer to the queue.
- */
-static DECLCALLBACK(void) pdmR3QueueTimer(PVM pVM, PTMTIMER pTimer, void *pvUser)
-{
-    PPDMQUEUE pQueue = (PPDMQUEUE)pvUser;
-    Assert(pTimer == pQueue->pTimer); NOREF(pTimer);
-
-    if (   pQueue->pPendingR3
-        || pQueue->pPendingR0
-        || pQueue->pPendingRC)
-        pdmR3QueueFlush(pQueue);
-    int rc = TMTimerSetMillies(pQueue->pTimer, pQueue->cMilliesInterval);
-    AssertRC(rc);
-}
-
Index: unk/src/VBox/VMM/PDMThread.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMThread.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1080 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM Thread - VM Thread Management.
- */
-
-/*
- * Copyright (C) 2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-///@todo #define LOG_GROUP LOG_GROUP_PDM_THREAD
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/mm.h>
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/semaphore.h>
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) pdmR3ThreadMain(RTTHREAD Thread, void *pvUser);
-
-
-/**
- * Wrapper around ASMAtomicCmpXchgSize.
- */
-DECLINLINE(bool) pdmR3AtomicCmpXchgState(PPDMTHREAD pThread, PDMTHREADSTATE enmNewState, PDMTHREADSTATE enmOldState)
-{
-    bool fRc;
-    ASMAtomicCmpXchgSize(&pThread->enmState, enmNewState, enmOldState, fRc);
-    return fRc;
-}
-
-
-/**
- * Does the wakeup call.
- *
- * @returns VBox status code. Already asserted on failure.
- * @param   pThread     The PDM thread.
- */
-static DECLCALLBACK(int) pdmR3ThreadWakeUp(PPDMTHREAD pThread)
-{
-    RTSemEventMultiSignal(pThread->Internal.s.SleepEvent);
-
-    int rc;
-    switch (pThread->Internal.s.enmType)
-    {
-        case PDMTHREADTYPE_DEVICE:
-            rc = pThread->u.Dev.pfnWakeUp(pThread->u.Dev.pDevIns, pThread);
-            break;
-
-        case PDMTHREADTYPE_USB:
-            rc = pThread->u.Usb.pfnWakeUp(pThread->u.Usb.pUsbIns, pThread);
-            break;
-
-        case PDMTHREADTYPE_DRIVER:
-            rc = pThread->u.Drv.pfnWakeUp(pThread->u.Drv.pDrvIns, pThread);
-            break;
-
-        case PDMTHREADTYPE_INTERNAL:
-            rc = pThread->u.Int.pfnWakeUp(pThread->Internal.s.pVM, pThread);
-            break;
-
-        case PDMTHREADTYPE_EXTERNAL:
-            rc = pThread->u.Ext.pfnWakeUp(pThread);
-            break;
-
-        default:
-            AssertMsgFailed(("%d\n", pThread->Internal.s.enmType));
-            rc = VERR_INTERNAL_ERROR;
-            break;
-    }
-    AssertRC(rc);
-    return rc;
-}
-
-
-/**
- * Allocates new thread instance.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   ppThread    Where to store the pointer to the instance.
- */
-static int pdmR3ThreadNew(PVM pVM, PPPDMTHREAD ppThread)
-{
-    PPDMTHREAD pThread;
-    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_THREAD, sizeof(*pThread), (void **)&pThread);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    pThread->u32Version     = PDMTHREAD_VERSION;
-    pThread->enmState       = PDMTHREADSTATE_INITIALIZING;
-    pThread->Thread         = NIL_RTTHREAD;
-    pThread->Internal.s.pVM = pVM;
-
-    *ppThread = pThread;
-    return VINF_SUCCESS;
-}
-
-
-
-/**
- * Initialize a new thread, this actually creates the thread.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   ppThread    Where the thread instance data handle is.
- * @param   cbStack     The stack size, see RTThreadCreate().
- * @param   enmType     The thread type, see RTThreadCreate().
- * @param   pszName     The thread name, see RTThreadCreate().
- */
-static int pdmR3ThreadInit(PVM pVM, PPPDMTHREAD ppThread, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    PPDMTHREAD  pThread = *ppThread;
-    PUVM        pUVM    = pVM->pUVM;
-
-    /*
-     * Initialize the remainder of the structure.
-     */
-    pThread->Internal.s.pVM = pVM;
-
-    int rc = RTSemEventMultiCreate(&pThread->Internal.s.BlockEvent);
-    if (RT_SUCCESS(rc))
-    {
-        rc = RTSemEventMultiCreate(&pThread->Internal.s.SleepEvent);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Create the thread and wait for it to initialize.
-             * The newly created thread will set the PDMTHREAD::Thread member.
-             */
-            RTTHREAD Thread;
-            rc = RTThreadCreate(&Thread, pdmR3ThreadMain, pThread, cbStack, enmType, RTTHREADFLAGS_WAITABLE, pszName);
-            if (RT_SUCCESS(rc))
-            {
-                rc = RTThreadUserWait(Thread, 60*1000);
-                if (    RT_SUCCESS(rc)
-                    &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
-                    rc = VERR_INTERNAL_ERROR;
-                if (RT_SUCCESS(rc))
-                {
-                    /*
-                     * Insert it into the thread list.
-                     */
-                    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-                    pThread->Internal.s.pNext = NULL;
-                    if (pUVM->pdm.s.pThreadsTail)
-                        pUVM->pdm.s.pThreadsTail->Internal.s.pNext = pThread;
-                    else
-                        pUVM->pdm.s.pThreads = pThread;
-                    pUVM->pdm.s.pThreadsTail = pThread;
-                    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-
-                    rc = RTThreadUserReset(Thread);
-                    AssertRC(rc);
-                    return rc;
-                }
-
-                /* bailout */
-                RTThreadWait(Thread, 60*1000, NULL);
-            }
-            RTSemEventMultiDestroy(pThread->Internal.s.SleepEvent);
-            pThread->Internal.s.SleepEvent = NIL_RTSEMEVENTMULTI;
-        }
-        RTSemEventMultiDestroy(pThread->Internal.s.BlockEvent);
-        pThread->Internal.s.BlockEvent = NIL_RTSEMEVENTMULTI;
-    }
-    MMHyperFree(pVM, pThread);
-    *ppThread = NULL;
-
-    return rc;
-}
-
-
-/**
- * Device Helper for creating a thread associated with a device.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pDevIns     The device instance.
- * @param   ppThread    Where to store the thread 'handle'.
- * @param   pvUser      The user argument to the thread function.
- * @param   pfnThread   The thread function.
- * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
- *                      a state change is pending.
- * @param   cbStack     See RTThreadCreate.
- * @param   enmType     See RTThreadCreate.
- * @param   pszName     See RTThreadCreate.
- */
-int pdmR3ThreadCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
-                            PFNPDMTHREADWAKEUPDEV pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    int rc = pdmR3ThreadNew(pVM, ppThread);
-    if (RT_SUCCESS(rc))
-    {
-        PPDMTHREAD pThread = *ppThread;
-        pThread->pvUser = pvUser;
-        pThread->Internal.s.enmType = PDMTHREADTYPE_DEVICE;
-        pThread->u.Dev.pDevIns = pDevIns;
-        pThread->u.Dev.pfnThread = pfnThread;
-        pThread->u.Dev.pfnWakeUp = pfnWakeUp;
-        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
-    }
-    return rc;
-}
-
-
-/**
- * USB Device Helper for creating a thread associated with an USB device.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pUsbIns     The USB device instance.
- * @param   ppThread    Where to store the thread 'handle'.
- * @param   pvUser      The user argument to the thread function.
- * @param   pfnThread   The thread function.
- * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
- *                      a state change is pending.
- * @param   cbStack     See RTThreadCreate.
- * @param   enmType     See RTThreadCreate.
- * @param   pszName     See RTThreadCreate.
- */
-int pdmR3ThreadCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADUSB pfnThread,
-                         PFNPDMTHREADWAKEUPUSB pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    int rc = pdmR3ThreadNew(pVM, ppThread);
-    if (RT_SUCCESS(rc))
-    {
-        PPDMTHREAD pThread = *ppThread;
-        pThread->pvUser = pvUser;
-        pThread->Internal.s.enmType = PDMTHREADTYPE_USB;
-        pThread->u.Usb.pUsbIns = pUsbIns;
-        pThread->u.Usb.pfnThread = pfnThread;
-        pThread->u.Usb.pfnWakeUp = pfnWakeUp;
-        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
-    }
-    return rc;
-}
-
-
-/**
- * Driver Helper for creating a thread associated with a driver.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pDrvIns     The driver instance.
- * @param   ppThread    Where to store the thread 'handle'.
- * @param   pvUser      The user argument to the thread function.
- * @param   pfnThread   The thread function.
- * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
- *                      a state change is pending.
- * @param   cbStack     See RTThreadCreate.
- * @param   enmType     See RTThreadCreate.
- * @param   pszName     See RTThreadCreate.
- */
-int pdmR3ThreadCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
-                            PFNPDMTHREADWAKEUPDRV pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    int rc = pdmR3ThreadNew(pVM, ppThread);
-    if (RT_SUCCESS(rc))
-    {
-        PPDMTHREAD pThread = *ppThread;
-        pThread->pvUser = pvUser;
-        pThread->Internal.s.enmType = PDMTHREADTYPE_DRIVER;
-        pThread->u.Drv.pDrvIns = pDrvIns;
-        pThread->u.Drv.pfnThread = pfnThread;
-        pThread->u.Drv.pfnWakeUp = pfnWakeUp;
-        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
-    }
-    return rc;
-}
-
-
-/**
- * Creates a PDM thread for internal use in the VM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   ppThread    Where to store the thread 'handle'.
- * @param   pvUser      The user argument to the thread function.
- * @param   pfnThread   The thread function.
- * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
- *                      a state change is pending.
- * @param   cbStack     See RTThreadCreate.
- * @param   enmType     See RTThreadCreate.
- * @param   pszName     See RTThreadCreate.
- */
-VMMR3DECL(int) PDMR3ThreadCreate(PVM pVM, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADINT pfnThread,
-                                 PFNPDMTHREADWAKEUPINT pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    int rc = pdmR3ThreadNew(pVM, ppThread);
-    if (RT_SUCCESS(rc))
-    {
-        PPDMTHREAD pThread = *ppThread;
-        pThread->pvUser = pvUser;
-        pThread->Internal.s.enmType = PDMTHREADTYPE_INTERNAL;
-        pThread->u.Int.pfnThread = pfnThread;
-        pThread->u.Int.pfnWakeUp = pfnWakeUp;
-        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
-    }
-    return rc;
-}
-
-
-/**
- * Creates a PDM thread for VM use by some external party.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   ppThread    Where to store the thread 'handle'.
- * @param   pvUser      The user argument to the thread function.
- * @param   pfnThread   The thread function.
- * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
- *                      a state change is pending.
- * @param   cbStack     See RTThreadCreate.
- * @param   enmType     See RTThreadCreate.
- * @param   pszName     See RTThreadCreate.
- */
-VMMR3DECL(int) PDMR3ThreadCreateExternal(PVM pVM, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADEXT pfnThread,
-                                         PFNPDMTHREADWAKEUPEXT pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
-{
-    int rc = pdmR3ThreadNew(pVM, ppThread);
-    if (RT_SUCCESS(rc))
-    {
-        PPDMTHREAD pThread = *ppThread;
-        pThread->pvUser = pvUser;
-        pThread->Internal.s.enmType = PDMTHREADTYPE_EXTERNAL;
-        pThread->u.Ext.pfnThread = pfnThread;
-        pThread->u.Ext.pfnWakeUp = pfnWakeUp;
-        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
-    }
-    return rc;
-}
-
-
-/**
- * Destroys a PDM thread.
- *
- * This will wakeup the thread, tell it to terminate, and wait for it terminate.
- *
- * @returns VBox status code.
- *          This reflects the success off destroying the thread and not the exit code
- *          of the thread as this is stored in *pRcThread.
- * @param   pThread         The thread to destroy.
- * @param   pRcThread       Where to store the thread exit code. Optional.
- * @thread  The emulation thread (EMT).
- */
-VMMR3DECL(int) PDMR3ThreadDestroy(PPDMTHREAD pThread, int *pRcThread)
-{
-    /*
-     * Assert sanity.
-     */
-    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
-    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
-    Assert(pThread->Thread != RTThreadSelf());
-    AssertPtrNullReturn(pRcThread, VERR_INVALID_POINTER);
-    PVM pVM = pThread->Internal.s.pVM;
-    VM_ASSERT_EMT(pVM);
-    PUVM pUVM = pVM->pUVM;
-
-    /*
-     * Advance the thread to the terminating state.
-     */
-    int rc = VINF_SUCCESS;
-    if (pThread->enmState <= PDMTHREADSTATE_TERMINATING)
-    {
-        for (;;)
-        {
-            PDMTHREADSTATE enmState = pThread->enmState;
-            switch (enmState)
-            {
-                case PDMTHREADSTATE_RUNNING:
-                    if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
-                        continue;
-                    rc = pdmR3ThreadWakeUp(pThread);
-                    break;
-
-                case PDMTHREADSTATE_SUSPENDED:
-                case PDMTHREADSTATE_SUSPENDING:
-                case PDMTHREADSTATE_RESUMING:
-                case PDMTHREADSTATE_INITIALIZING:
-                    if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
-                        continue;
-                    break;
-
-                case PDMTHREADSTATE_TERMINATING:
-                case PDMTHREADSTATE_TERMINATED:
-                    break;
-
-                default:
-                    AssertMsgFailed(("enmState=%d\n", enmState));
-                    rc = VERR_INTERNAL_ERROR;
-                    break;
-            }
-            break;
-        }
-    }
-    int rc2 = RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
-    AssertRC(rc2);
-
-    /*
-     * Wait for it to terminate and the do cleanups.
-     */
-    rc2 = RTThreadWait(pThread->Thread, RT_SUCCESS(rc) ? 60*1000 : 150, pRcThread);
-    if (RT_SUCCESS(rc2))
-    {
-        /* make it invalid. */
-        pThread->u32Version = 0xffffffff;
-        pThread->enmState = PDMTHREADSTATE_INVALID;
-        pThread->Thread = NIL_RTTHREAD;
-
-        /* unlink */
-        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-        if (pUVM->pdm.s.pThreads == pThread)
-        {
-            pUVM->pdm.s.pThreads = pThread->Internal.s.pNext;
-            if (!pThread->Internal.s.pNext)
-                pUVM->pdm.s.pThreadsTail = NULL;
-        }
-        else
-        {
-            PPDMTHREAD pPrev = pUVM->pdm.s.pThreads;
-            while (pPrev && pPrev->Internal.s.pNext != pThread)
-                pPrev = pPrev->Internal.s.pNext;
-            Assert(pPrev);
-            if (pPrev)
-                pPrev->Internal.s.pNext = pThread->Internal.s.pNext;
-            if (!pThread->Internal.s.pNext)
-                pUVM->pdm.s.pThreadsTail = pPrev;
-        }
-        pThread->Internal.s.pNext = NULL;
-        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-
-        /* free the resources */
-        RTSemEventMultiDestroy(pThread->Internal.s.BlockEvent);
-        pThread->Internal.s.BlockEvent = NIL_RTSEMEVENTMULTI;
-
-        RTSemEventMultiDestroy(pThread->Internal.s.SleepEvent);
-        pThread->Internal.s.SleepEvent = NIL_RTSEMEVENTMULTI;
-
-        MMR3HeapFree(pThread);
-    }
-    else if (RT_SUCCESS(rc))
-        rc = rc2;
-
-    return rc;
-}
-
-
-/**
- * Destroys all threads associated with a device.
- *
- * This function is called by PDMDevice when a device is
- * destroyed (not currently implemented).
- *
- * @returns VBox status code of the first failure.
- * @param   pVM         The VM handle.
- * @param   pDevIns     the device instance.
- */
-int pdmR3ThreadDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
-{
-    int     rc   = VINF_SUCCESS;
-    PUVM    pUVM = pVM->pUVM;
-
-    AssertPtr(pDevIns);
-
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
-    while (pThread)
-    {
-        PPDMTHREAD pNext = pThread->Internal.s.pNext;
-        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DEVICE
-            &&  pThread->u.Dev.pDevIns == pDevIns)
-        {
-            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-        }
-        pThread = pNext;
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-/**
- * Destroys all threads associated with an USB device.
- *
- * This function is called by PDMUsb when a device is destroyed.
- *
- * @returns VBox status code of the first failure.
- * @param   pVM         The VM handle.
- * @param   pUsbIns     The USB device instance.
- */
-int pdmR3ThreadDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns)
-{
-    int     rc   = VINF_SUCCESS;
-    PUVM    pUVM = pVM->pUVM;
-
-    AssertPtr(pUsbIns);
-
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
-    while (pThread)
-    {
-        PPDMTHREAD pNext = pThread->Internal.s.pNext;
-        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DEVICE
-            &&  pThread->u.Usb.pUsbIns == pUsbIns)
-        {
-            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-        }
-        pThread = pNext;
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-/**
- * Destroys all threads associated with a driver.
- *
- * This function is called by PDMDriver when a driver is destroyed.
- *
- * @returns VBox status code of the first failure.
- * @param   pVM         The VM handle.
- * @param   pDrvIns     The driver instance.
- */
-int pdmR3ThreadDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
-{
-    int     rc   = VINF_SUCCESS;
-    PUVM    pUVM = pVM->pUVM;
-
-    AssertPtr(pDrvIns);
-
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
-    while (pThread)
-    {
-        PPDMTHREAD pNext = pThread->Internal.s.pNext;
-        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DRIVER
-            &&  pThread->u.Drv.pDrvIns == pDrvIns)
-        {
-            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-        }
-        pThread = pNext;
-    }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return rc;
-}
-
-
-/**
- * Called For VM power off.
- *
- * @param   pVM         The VM handle.
- */
-void pdmR3ThreadDestroyAll(PVM pVM)
-{
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
-    while (pThread)
-    {
-        PPDMTHREAD pNext = pThread->Internal.s.pNext;
-        int rc2 = PDMR3ThreadDestroy(pThread, NULL);
-        AssertRC(rc2);
-        pThread = pNext;
-    }
-    Assert(!pUVM->pdm.s.pThreads && !pUVM->pdm.s.pThreadsTail);
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-}
-
-
-/**
- * Initiate termination of the thread (self) because something failed in a bad way.
- *
- * @param   pThread         The PDM thread.
- */
-static void pdmR3ThreadBailMeOut(PPDMTHREAD pThread)
-{
-    for (;;)
-    {
-        PDMTHREADSTATE enmState = pThread->enmState;
-        switch (enmState)
-        {
-            case PDMTHREADSTATE_SUSPENDING:
-            case PDMTHREADSTATE_SUSPENDED:
-            case PDMTHREADSTATE_RESUMING:
-            case PDMTHREADSTATE_RUNNING:
-                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
-                    continue;
-                break;
-
-            case PDMTHREADSTATE_TERMINATING:
-            case PDMTHREADSTATE_TERMINATED:
-                break;
-
-            case PDMTHREADSTATE_INITIALIZING:
-            default:
-                AssertMsgFailed(("enmState=%d\n", enmState));
-                break;
-        }
-        break;
-    }
-}
-
-
-/**
- * Called by the PDM thread in response to a wakeup call with
- * suspending as the new state.
- *
- * The thread will block in side this call until the state is changed in
- * response to a VM state change or to the device/driver/whatever calling the
- * PDMR3ThreadResume API.
- *
- * @returns VBox status code.
- *          On failure, terminate the thread.
- * @param   pThread     The PDM thread.
- */
-VMMR3DECL(int) PDMR3ThreadIAmSuspending(PPDMTHREAD pThread)
-{
-    /*
-     * Assert sanity.
-     */
-    AssertPtr(pThread);
-    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
-    Assert(pThread->Thread == RTThreadSelf() || pThread->enmState == PDMTHREADSTATE_INITIALIZING);
-    PDMTHREADSTATE enmState = pThread->enmState;
-    Assert(     enmState == PDMTHREADSTATE_SUSPENDING
-           ||   enmState == PDMTHREADSTATE_INITIALIZING);
-
-    /*
-     * Update the state, notify the control thread (the API caller) and go to sleep.
-     */
-    int rc = VERR_WRONG_ORDER;
-    if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_SUSPENDED, enmState))
-    {
-        rc = RTThreadUserSignal(pThread->Thread);
-        if (RT_SUCCESS(rc))
-        {
-            rc = RTSemEventMultiWait(pThread->Internal.s.BlockEvent, RT_INDEFINITE_WAIT);
-            if (    RT_SUCCESS(rc)
-                &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
-                return rc;
-
-            if (RT_SUCCESS(rc))
-                rc = VERR_INTERNAL_ERROR;
-        }
-    }
-
-    AssertMsgFailed(("rc=%d enmState=%d\n", rc, pThread->enmState));
-    pdmR3ThreadBailMeOut(pThread);
-    return rc;
-}
-
-
-/**
- * Called by the PDM thread in response to a resuming state.
- *
- * The purpose of this API is to tell the PDMR3ThreadResume caller that
- * the PDM thread has successfully resumed. It will also do the
- * state transition from the resuming to the running state.
- *
- * @returns VBox status code.
- *          On failure, terminate the thread.
- * @param   pThread     The PDM thread.
- */
-VMMR3DECL(int) PDMR3ThreadIAmRunning(PPDMTHREAD pThread)
-{
-    /*
-     * Assert sanity.
-     */
-    Assert(pThread->enmState == PDMTHREADSTATE_RESUMING);
-    Assert(pThread->Thread == RTThreadSelf());
-
-    /*
-     * Update the state and tell the control thread (the guy calling the resume API).
-     */
-    int rc = VERR_WRONG_ORDER;
-    if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_RUNNING, PDMTHREADSTATE_RESUMING))
-    {
-        rc = RTThreadUserSignal(pThread->Thread);
-        if (RT_SUCCESS(rc))
-            return rc;
-    }
-
-    AssertMsgFailed(("rc=%d enmState=%d\n", rc, pThread->enmState));
-    pdmR3ThreadBailMeOut(pThread);
-    return rc;
-}
-
-
-/**
- * Called by the PDM thread instead of RTThreadSleep.
- *
- * The difference is that the sleep will be interrupted on state change. The
- * thread must be in the running state, otherwise it will return immediately.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success or state change.
- * @retval  VERR_INTERRUPTED on signal or APC.
- *
- * @param   pThread     The PDM thread.
- * @param   cMillies    The number of milliseconds to sleep.
- */
-VMMR3DECL(int) PDMR3ThreadSleep(PPDMTHREAD pThread, RTMSINTERVAL cMillies)
-{
-    /*
-     * Assert sanity.
-     */
-    AssertReturn(pThread->enmState > PDMTHREADSTATE_INVALID && pThread->enmState < PDMTHREADSTATE_TERMINATED, VERR_INTERNAL_ERROR);
-    AssertReturn(pThread->Thread == RTThreadSelf(), VERR_INTERNAL_ERROR);
-
-    /*
-     * Reset the event semaphore, check the state and sleep.
-     */
-    RTSemEventMultiReset(pThread->Internal.s.SleepEvent);
-    if (pThread->enmState != PDMTHREADSTATE_RUNNING)
-        return VINF_SUCCESS;
-    return RTSemEventMultiWaitNoResume(pThread->Internal.s.SleepEvent, cMillies);
-}
-
-
-/**
- * The PDM thread function.
- *
- * @returns return from pfnThread.
- *
- * @param   Thread  The thread handle.
- * @param   pvUser  Pointer to the PDMTHREAD structure.
- */
-static DECLCALLBACK(int) pdmR3ThreadMain(RTTHREAD Thread, void *pvUser)
-{
-    PPDMTHREAD pThread = (PPDMTHREAD)pvUser;
-    Log(("PDMThread: Initializing thread %RTthrd / %p / '%s'...\n", Thread, pThread, RTThreadGetName(Thread)));
-    pThread->Thread = Thread;
-
-    /*
-     * The run loop.
-     *
-     * It handles simple thread functions which returns when they see a suspending
-     * request and leaves the PDMR3ThreadIAmSuspending and PDMR3ThreadIAmRunning
-     * parts to us.
-     */
-    int rc;
-    for (;;)
-    {
-        switch (pThread->Internal.s.enmType)
-        {
-            case PDMTHREADTYPE_DEVICE:
-                rc = pThread->u.Dev.pfnThread(pThread->u.Dev.pDevIns, pThread);
-                break;
-
-            case PDMTHREADTYPE_USB:
-                rc = pThread->u.Usb.pfnThread(pThread->u.Usb.pUsbIns, pThread);
-                break;
-
-            case PDMTHREADTYPE_DRIVER:
-                rc = pThread->u.Drv.pfnThread(pThread->u.Drv.pDrvIns, pThread);
-                break;
-
-            case PDMTHREADTYPE_INTERNAL:
-                rc = pThread->u.Int.pfnThread(pThread->Internal.s.pVM, pThread);
-                break;
-
-            case PDMTHREADTYPE_EXTERNAL:
-                rc = pThread->u.Ext.pfnThread(pThread);
-                break;
-
-            default:
-                AssertMsgFailed(("%d\n", pThread->Internal.s.enmType));
-                rc = VERR_INTERNAL_ERROR;
-                break;
-        }
-        if (RT_FAILURE(rc))
-            break;
-
-        /*
-         * If this is a simple thread function, the state will be suspending
-         * or initializing now. If it isn't we're supposed to terminate.
-         */
-        if (    pThread->enmState != PDMTHREADSTATE_SUSPENDING
-            &&  pThread->enmState != PDMTHREADSTATE_INITIALIZING)
-        {
-            Assert(pThread->enmState == PDMTHREADSTATE_TERMINATING);
-            break;
-        }
-        rc = PDMR3ThreadIAmSuspending(pThread);
-        if (RT_FAILURE(rc))
-            break;
-        if (pThread->enmState != PDMTHREADSTATE_RESUMING)
-        {
-            Assert(pThread->enmState == PDMTHREADSTATE_TERMINATING);
-            break;
-        }
-
-        rc = PDMR3ThreadIAmRunning(pThread);
-        if (RT_FAILURE(rc))
-            break;
-    }
-
-    if (RT_FAILURE(rc))
-        LogRel(("PDMThread: Thread '%s' (%RTthrd) quit unexpectedly with rc=%Rrc.\n", RTThreadGetName(Thread), Thread, rc));
-
-    /*
-     * Advance the state to terminating and then on to terminated.
-     */
-    for (;;)
-    {
-        PDMTHREADSTATE enmState = pThread->enmState;
-        if (    enmState == PDMTHREADSTATE_TERMINATING
-            ||  pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
-            break;
-    }
-
-    ASMAtomicXchgSize(&pThread->enmState, PDMTHREADSTATE_TERMINATED);
-    int rc2 = RTThreadUserSignal(Thread); AssertRC(rc2);
-
-    Log(("PDMThread: Terminating thread %RTthrd / %p / '%s': %Rrc\n", Thread, pThread, RTThreadGetName(Thread), rc));
-    return rc;
-}
-
-
-/**
- * Initiate termination of the thread because something failed in a bad way.
- *
- * @param   pThread         The PDM thread.
- */
-static void pdmR3ThreadBailOut(PPDMTHREAD pThread)
-{
-    for (;;)
-    {
-        PDMTHREADSTATE enmState = pThread->enmState;
-        switch (enmState)
-        {
-            case PDMTHREADSTATE_SUSPENDING:
-            case PDMTHREADSTATE_SUSPENDED:
-                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
-                    continue;
-                RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
-                break;
-
-            case PDMTHREADSTATE_RESUMING:
-                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
-                    continue;
-                break;
-
-            case PDMTHREADSTATE_RUNNING:
-                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
-                    continue;
-                pdmR3ThreadWakeUp(pThread);
-                break;
-
-            case PDMTHREADSTATE_TERMINATING:
-            case PDMTHREADSTATE_TERMINATED:
-                break;
-
-            case PDMTHREADSTATE_INITIALIZING:
-            default:
-                AssertMsgFailed(("enmState=%d\n", enmState));
-                break;
-        }
-        break;
-    }
-}
-
-
-/**
- * Suspends the thread.
- *
- * This can be called at the power off / suspend notifications to suspend the
- * PDM thread a bit early. The thread will be automatically suspend upon
- * completion of the device/driver notification cycle.
- *
- * The caller is responsible for serializing the control operations on the
- * thread. That basically means, always do these calls from the EMT.
- *
- * @returns VBox status code.
- * @param   pThread     The PDM thread.
- */
-VMMR3DECL(int) PDMR3ThreadSuspend(PPDMTHREAD pThread)
-{
-    /*
-     * Assert sanity.
-     */
-    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
-    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
-    Assert(pThread->Thread != RTThreadSelf());
-
-    /*
-     * This is a noop if the thread is already suspended.
-     */
-    if (pThread->enmState == PDMTHREADSTATE_SUSPENDED)
-        return VINF_SUCCESS;
-
-    /*
-     * Change the state to resuming and kick the thread.
-     */
-    int rc = RTSemEventMultiReset(pThread->Internal.s.BlockEvent);
-    if (RT_SUCCESS(rc))
-    {
-        rc = RTThreadUserReset(pThread->Thread);
-        if (RT_SUCCESS(rc))
-        {
-            rc = VERR_WRONG_ORDER;
-            if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_SUSPENDING, PDMTHREADSTATE_RUNNING))
-            {
-                rc = pdmR3ThreadWakeUp(pThread);
-                if (RT_SUCCESS(rc))
-                {
-                    /*
-                     * Wait for the thread to reach the suspended state.
-                     */
-                    if (pThread->enmState != PDMTHREADSTATE_SUSPENDED)
-                        rc = RTThreadUserWait(pThread->Thread, 60*1000);
-                    if (    RT_SUCCESS(rc)
-                        &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
-                        rc = VERR_INTERNAL_ERROR;
-                    if (RT_SUCCESS(rc))
-                        return rc;
-                }
-            }
-        }
-    }
-
-    /*
-     * Something failed, initialize termination.
-     */
-    AssertMsgFailed(("PDMR3ThreadSuspend -> rc=%Rrc enmState=%d\n", rc, pThread->enmState));
-    pdmR3ThreadBailOut(pThread);
-    return rc;
-}
-
-
-/**
- * Suspend all running threads.
- *
- * This is called by PDMR3Suspend() and PDMR3PowerOff() after all the devices
- * and drivers have been notified about the suspend / power off.
- *
- * @return VBox status code.
- * @param   pVM         The VM handle.
- */
-int pdmR3ThreadSuspendAll(PVM pVM)
-{
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect); /* This may cause deadlocks later... */
-    for (PPDMTHREAD pThread = pUVM->pdm.s.pThreads; pThread; pThread = pThread->Internal.s.pNext)
-        switch (pThread->enmState)
-        {
-            case PDMTHREADSTATE_RUNNING:
-            {
-                int rc = PDMR3ThreadSuspend(pThread);
-                AssertRCReturn(rc, rc);
-                break;
-            }
-
-            /* suspend -> power off; voluntary suspend. */
-            case PDMTHREADSTATE_SUSPENDED:
-                break;
-
-            default:
-                AssertMsgFailed(("pThread=%p enmState=%d\n", pThread, pThread->enmState));
-                break;
-        }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Resumes the thread.
- *
- * This can be called the power on / resume notifications to resume the
- * PDM thread a bit early. The thread will be automatically resumed upon
- * return from these two notification callbacks (devices/drivers).
- *
- * The caller is responsible for serializing the control operations on the
- * thread. That basically means, always do these calls from the EMT.
- *
- * @returns VBox status code.
- * @param   pThread     The PDM thread.
- */
-VMMR3DECL(int) PDMR3ThreadResume(PPDMTHREAD pThread)
-{
-    /*
-     * Assert sanity.
-     */
-    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
-    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
-    Assert(pThread->Thread != RTThreadSelf());
-
-    /*
-     * Change the state to resuming and kick the thread.
-     */
-    int rc = RTThreadUserReset(pThread->Thread);
-    if (RT_SUCCESS(rc))
-    {
-        rc = VERR_WRONG_ORDER;
-        if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_RESUMING, PDMTHREADSTATE_SUSPENDED))
-        {
-            rc = RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Wait for the thread to reach the running state.
-                 */
-                rc = RTThreadUserWait(pThread->Thread, 60*1000);
-                if (    RT_SUCCESS(rc)
-                    &&  pThread->enmState != PDMTHREADSTATE_RUNNING)
-                    rc = VERR_INTERNAL_ERROR;
-                if (RT_SUCCESS(rc))
-                    return rc;
-            }
-        }
-    }
-
-    /*
-     * Something failed, initialize termination.
-     */
-    AssertMsgFailed(("PDMR3ThreadResume -> rc=%Rrc enmState=%d\n", rc, pThread->enmState));
-    pdmR3ThreadBailOut(pThread);
-    return rc;
-}
-
-
-/**
- * Resumes all threads not running.
- *
- * This is called by PDMR3Resume() and PDMR3PowerOn() after all the devices
- * and drivers have been notified about the resume / power on .
- *
- * @return VBox status code.
- * @param   pVM         The VM handle.
- */
-int pdmR3ThreadResumeAll(PVM pVM)
-{
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
-    for (PPDMTHREAD pThread = pUVM->pdm.s.pThreads; pThread; pThread = pThread->Internal.s.pNext)
-        switch (pThread->enmState)
-        {
-            case PDMTHREADSTATE_SUSPENDED:
-            {
-                int rc = PDMR3ThreadResume(pThread);
-                AssertRCReturn(rc, rc);
-                break;
-            }
-
-            default:
-                AssertMsgFailed(("pThread=%p enmState=%d\n", pThread, pThread->enmState));
-                break;
-        }
-    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/PDMUsb.cpp
===================================================================
--- /trunk/src/VBox/VMM/PDMUsb.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1444 +1,0 @@
-/* $Id$ */
-/** @file
- * PDM - Pluggable Device and Driver Manager, USB part.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PDM_DRIVER
-#include "PDMInternal.h"
-#include <VBox/pdm.h>
-#include <VBox/vusb.h>
-#include <VBox/mm.h>
-#include <VBox/cfgm.h>
-#include <VBox/vmm.h>
-#include <VBox/sup.h>
-#include <VBox/vm.h>
-#include <VBox/version.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-#include <iprt/string.h>
-#include <iprt/asm.h>
-#include <iprt/alloc.h>
-#include <iprt/alloca.h>
-#include <iprt/path.h>
-#include <iprt/uuid.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Internal callback structure pointer.
- *
- * The main purpose is to define the extra data we associate
- * with PDMUSBREGCB so we can find the VM instance and so on.
- */
-typedef struct PDMUSBREGCBINT
-{
-    /** The callback structure. */
-    PDMUSBREGCB     Core;
-    /** A bit of padding. */
-    uint32_t        u32[4];
-    /** VM Handle. */
-    PVM             pVM;
-} PDMUSBREGCBINT, *PPDMUSBREGCBINT;
-typedef const PDMUSBREGCBINT *PCPDMUSBREGCBINT;
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** @def PDMUSB_ASSERT_USBINS
- * Asserts the validity of the USB device instance.
- */
-#ifdef VBOX_STRICT
-# define PDMUSB_ASSERT_USBINS(pUsbIns) \
-    do { \
-        AssertPtr(pUsbIns); \
-        Assert(pUsbIns->u32Version == PDM_USBINS_VERSION); \
-        Assert(pUsbIns->pvInstanceDataR3 == (void *)&pUsbIns->achInstanceData[0]); \
-    } while (0)
-#else
-# define PDMUSB_ASSERT_USBINS(pUsbIns)   do { } while (0)
-#endif
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static void pdmR3UsbDestroyDevice(PVM pVM, PPDMUSBINS pUsbIns);
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-extern const PDMUSBHLP g_pdmR3UsbHlp;
-
-
-AssertCompile(sizeof(PDMUSBINSINT) <= RT_SIZEOFMEMB(PDMUSBINS, Internal.padding));
-
-
-/**
- * Registers a USB hub driver.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pDrvIns         The driver instance of the hub.
- * @param   fVersions       Indicates the kinds of USB devices that can be attached to this HUB.
- * @param   cPorts          The number of ports.
- * @param   pUsbHubReg      The hub callback structure that PDMUsb uses to interact with it.
- * @param   ppUsbHubHlp     The helper callback structure that the hub uses to talk to PDMUsb.
- * @thread  EMT
- */
-int pdmR3UsbRegisterHub(PVM pVM, PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp)
-{
-    /*
-     * Validate input.
-     */
-    /* The driver must be in the USB class. */
-    if (!(pDrvIns->pReg->fClass & PDM_DRVREG_CLASS_USB))
-    {
-        LogRel(("pdmR3UsbRegisterHub: fClass=%#x expected %#x to be set\n", pDrvIns->pReg->fClass, PDM_DRVREG_CLASS_USB));
-        return VERR_INVALID_PARAMETER;
-    }
-    AssertMsgReturn(!(fVersions & ~(VUSB_STDVER_11 | VUSB_STDVER_20)), ("%#x\n", fVersions), VERR_INVALID_PARAMETER);
-    AssertPtrReturn(ppUsbHubHlp, VERR_INVALID_POINTER);
-    AssertPtrReturn(pUsbHubReg, VERR_INVALID_POINTER);
-    AssertReturn(pUsbHubReg->u32Version == PDM_USBHUBREG_VERSION, VERR_INVALID_MAGIC);
-    AssertReturn(pUsbHubReg->u32TheEnd == PDM_USBHUBREG_VERSION, VERR_INVALID_MAGIC);
-    AssertPtrReturn(pUsbHubReg->pfnAttachDevice, VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pUsbHubReg->pfnDetachDevice, VERR_INVALID_PARAMETER);
-
-    /*
-     * Check for duplicate registration and find the last hub for FIFO registration.
-     */
-    PPDMUSBHUB pPrev = NULL;
-    for (PPDMUSBHUB pCur = pVM->pdm.s.pUsbHubs; pCur; pCur = pCur->pNext)
-    {
-        if (pCur->pDrvIns == pDrvIns)
-            return VERR_PDM_USB_HUB_EXISTS;
-        pPrev = pCur;
-    }
-
-    /*
-     * Create an internal USB hub structure.
-     */
-    PPDMUSBHUB pHub = (PPDMUSBHUB)MMR3HeapAlloc(pVM, MM_TAG_PDM_DRIVER, sizeof(*pHub));
-    if (!pHub)
-        return VERR_NO_MEMORY;
-
-    pHub->fVersions = fVersions;
-    pHub->cPorts = cPorts;
-    pHub->cAvailablePorts = cPorts;
-    pHub->pDrvIns = pDrvIns;
-    pHub->Reg = *pUsbHubReg;
-    pHub->pNext = NULL;
-
-    /* link it */
-    if (pPrev)
-        pPrev->pNext = pHub;
-    else
-        pVM->pdm.s.pUsbHubs = pHub;
-
-    Log(("PDM: Registered USB hub %p/%s\n", pDrvIns, pDrvIns->pReg->szName));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Loads one device module and call the registration entry point.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pRegCB          The registration callback stuff.
- * @param   pszFilename     Module filename.
- * @param   pszName         Module name.
- */
-static int pdmR3UsbLoad(PVM pVM, PCPDMUSBREGCBINT pRegCB, const char *pszFilename, const char *pszName)
-{
-    /*
-     * Load it.
-     */
-    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Get the registration export and call it.
-         */
-        FNPDMVBOXUSBREGISTER *pfnVBoxUsbRegister;
-        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxUsbRegister", (void **)&pfnVBoxUsbRegister);
-        if (RT_SUCCESS(rc))
-        {
-            Log(("PDM: Calling VBoxUsbRegister (%p) of %s (%s)\n", pfnVBoxUsbRegister, pszName, pszFilename));
-            rc = pfnVBoxUsbRegister(&pRegCB->Core, VBOX_VERSION);
-            if (RT_SUCCESS(rc))
-                Log(("PDM: Successfully loaded device module %s (%s).\n", pszName, pszFilename));
-            else
-                AssertMsgFailed(("VBoxDevicesRegister failed with rc=%Rrc for module %s (%s)\n", rc, pszName, pszFilename));
-        }
-        else
-        {
-            AssertMsgFailed(("Failed to locate 'VBoxUsbRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
-            if (rc == VERR_SYMBOL_NOT_FOUND)
-                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
-        }
-    }
-    else
-        AssertMsgFailed(("Failed to load VBoxDD!\n"));
-    return rc;
-}
-
-
-
-/**
- * @interface_method_impl{PDMUSBREGCB,pfnRegister}
- */
-static DECLCALLBACK(int) pdmR3UsbReg_Register(PCPDMUSBREGCB pCallbacks, PCPDMUSBREG pReg)
-{
-    /*
-     * Validate the registration structure.
-     */
-    Assert(pReg);
-    AssertMsgReturn(pReg->u32Version == PDM_USBREG_VERSION,
-                    ("Unknown struct version %#x!\n", pReg->u32Version),
-                    VERR_PDM_UNKNOWN_USBREG_VERSION);
-    AssertMsgReturn(    pReg->szName[0]
-                    &&  strlen(pReg->szName) < sizeof(pReg->szName),
-                    ("Invalid name '%s'\n", pReg->szName),
-                    VERR_PDM_INVALID_USB_REGISTRATION);
-    AssertMsgReturn(pReg->fFlags == 0, ("fFlags=%#x\n", pReg->fFlags), VERR_PDM_INVALID_USB_REGISTRATION);
-    AssertMsgReturn(pReg->cMaxInstances > 0,
-                    ("Max instances %u! (USB Device %s)\n", pReg->cMaxInstances, pReg->szName),
-                    VERR_PDM_INVALID_USB_REGISTRATION);
-    AssertMsgReturn(pReg->cbInstance <= _1M,
-                    ("Instance size %d bytes! (USB Device %s)\n", pReg->cbInstance, pReg->szName),
-                    VERR_PDM_INVALID_USB_REGISTRATION);
-    AssertMsgReturn(pReg->pfnConstruct, ("No constructor! (USB Device %s)\n", pReg->szName),
-                    VERR_PDM_INVALID_USB_REGISTRATION);
-
-    /*
-     * Check for duplicate and find FIFO entry at the same time.
-     */
-    PCPDMUSBREGCBINT pRegCB = (PCPDMUSBREGCBINT)pCallbacks;
-    PPDMUSB pUsbPrev = NULL;
-    PPDMUSB pUsb = pRegCB->pVM->pdm.s.pUsbDevs;
-    for (; pUsb; pUsbPrev = pUsb, pUsb = pUsb->pNext)
-        AssertMsgReturn(strcmp(pUsb->pReg->szName, pReg->szName),
-                        ("USB Device '%s' already exists\n", pReg->szName),
-                        VERR_PDM_USB_NAME_CLASH);
-
-    /*
-     * Allocate new device structure and insert it into the list.
-     */
-    pUsb = (PPDMUSB)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DEVICE, sizeof(*pUsb));
-    if (pUsb)
-    {
-        pUsb->pNext = NULL;
-        pUsb->iNextInstance = 0;
-        pUsb->pInstances = NULL;
-        pUsb->pReg = pReg;
-        pUsb->cchName = (RTUINT)strlen(pReg->szName);
-
-        if (pUsbPrev)
-            pUsbPrev->pNext = pUsb;
-        else
-            pRegCB->pVM->pdm.s.pUsbDevs = pUsb;
-        Log(("PDM: Registered USB device '%s'\n", pReg->szName));
-        return VINF_SUCCESS;
-    }
-    return VERR_NO_MEMORY;
-}
-
-
-/**
- * Load USB Device modules.
- *
- * This is called by pdmR3DevInit() after it has loaded it's device modules.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- */
-int pdmR3UsbLoadModules(PVM pVM)
-{
-    LogFlow(("pdmR3UsbLoadModules:\n"));
-
-    AssertRelease(!(RT_OFFSETOF(PDMUSBINS, achInstanceData) & 15));
-    AssertRelease(sizeof(pVM->pdm.s.pUsbInstances->Internal.s) <= sizeof(pVM->pdm.s.pUsbInstances->Internal.padding));
-
-    /*
-     * Initialize the callback structure.
-     */
-    PDMUSBREGCBINT RegCB;
-    RegCB.Core.u32Version = PDM_USBREG_CB_VERSION;
-    RegCB.Core.pfnRegister = pdmR3UsbReg_Register;
-    RegCB.pVM = pVM;
-
-    /*
-     * Load the builtin module
-     */
-    PCFGMNODE pUsbNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/USB/");
-    bool fLoadBuiltin;
-    int rc = CFGMR3QueryBool(pUsbNode, "LoadBuiltin", &fLoadBuiltin);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
-        fLoadBuiltin = true;
-    else if (RT_FAILURE(rc))
-    {
-        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
-        return rc;
-    }
-    if (fLoadBuiltin)
-    {
-        /* make filename */
-        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
-        if (!pszFilename)
-            return VERR_NO_TMP_MEMORY;
-        rc = pdmR3UsbLoad(pVM, &RegCB, pszFilename, "VBoxDD");
-        RTMemTmpFree(pszFilename);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Load additional device modules.
-     */
-    PCFGMNODE pCur;
-    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
-    {
-        /*
-         * Get the name and path.
-         */
-        char szName[PDMMOD_NAME_LEN];
-        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
-        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
-        {
-            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
-            return VERR_PDM_MODULE_NAME_TOO_LONG;
-        }
-        else if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
-            return rc;
-        }
-
-        /* the path is optional, if no path the module name + path is used. */
-        char szFilename[RTPATH_MAX];
-        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-            strcpy(szFilename, szName);
-        else if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
-            return rc;
-        }
-
-        /* prepend path? */
-        if (!RTPathHavePath(szFilename))
-        {
-            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
-            if (!psz)
-                return VERR_NO_TMP_MEMORY;
-            size_t cch = strlen(psz) + 1;
-            if (cch > sizeof(szFilename))
-            {
-                RTMemTmpFree(psz);
-                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
-                return VERR_FILENAME_TOO_LONG;
-            }
-            memcpy(szFilename, psz, cch);
-            RTMemTmpFree(psz);
-        }
-
-        /*
-         * Load the module and register it's devices.
-         */
-        rc = pdmR3UsbLoad(pVM, &RegCB, szFilename, szName);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Send the init-complete notification to all the USB devices.
- *
- * This is called from pdmR3DevInit() after it has do its notification round.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- */
-int pdmR3UsbVMInitComplete(PVM pVM)
-{
-    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-    {
-        if (pUsbIns->pReg->pfnVMInitComplete)
-        {
-            int rc = pUsbIns->pReg->pfnVMInitComplete(pUsbIns);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("InitComplete on USB device '%s'/%d failed with rc=%Rrc\n",
-                                 pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-                return rc;
-            }
-        }
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Lookups a device structure by name.
- * @internal
- */
-PPDMUSB pdmR3UsbLookup(PVM pVM, const char *pszName)
-{
-    size_t cchName = strlen(pszName);
-    for (PPDMUSB pUsb = pVM->pdm.s.pUsbDevs; pUsb; pUsb = pUsb->pNext)
-        if (    pUsb->cchName == cchName
-            &&  !strcmp(pUsb->pReg->szName, pszName))
-            return pUsb;
-    return NULL;
-}
-
-
-/**
- * Locates a suitable hub for the specified kind of device.
- *
- * @returns VINF_SUCCESS and *ppHub on success.
- *          VERR_PDM_NO_USB_HUBS or VERR_PDM_NO_USB_PORTS on failure.
- * @param   pVM             The VM handle.
- * @param   iUsbVersion     The USB device version.
- * @param   ppHub           Where to store the pointer to the USB hub.
- */
-static int pdmR3UsbFindHub(PVM pVM, uint32_t iUsbVersion, PPDMUSBHUB *ppHub)
-{
-    *ppHub = NULL;
-    if (!pVM->pdm.s.pUsbHubs)
-        return VERR_PDM_NO_USB_HUBS;
-
-    for (PPDMUSBHUB pCur = pVM->pdm.s.pUsbHubs; pCur; pCur = pCur->pNext)
-        if (    pCur->cAvailablePorts > 0
-            &&  (   (pCur->fVersions & iUsbVersion)
-                 || pCur->fVersions == VUSB_STDVER_11))
-        {
-            *ppHub = pCur;
-            if (pCur->fVersions & iUsbVersion)
-                break;
-        }
-    if (*ppHub)
-        return VINF_SUCCESS;
-    return VERR_PDM_NO_USB_PORTS;
-}
-
-
-/**
- * Creates the device.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pUsbDev         The USB device emulation.
- * @param   iInstance       -1 if not called by pdmR3UsbInstantiateDevices().
- * @param   pUuid           The UUID for this device.
- * @param   pInstanceNode   The instance CFGM node. NULL if not called by pdmR3UsbInstantiateDevices().
- * @param   ppConfig        Pointer to the device configuration pointer. This is set to NULL if inserted
- *                          into the tree or cleaned up.
- *
- *                          In the pdmR3UsbInstantiateDevices() case (pInstanceNode != NULL) this is
- *                          the actual config node and will not be cleaned up.
- *
- * @parma   iUsbVersion     The USB version preferred by the device.
- */
-static int pdmR3UsbCreateDevice(PVM pVM, PPDMUSBHUB pHub, PPDMUSB pUsbDev, int iInstance, PCRTUUID pUuid, PCFGMNODE pInstanceNode, PCFGMNODE *ppConfig, uint32_t iUsbVersion)
-{
-    const bool fAtRuntime = pInstanceNode == NULL;
-    int rc;
-
-    /*
-     * If not called by pdmR3UsbInstantiateDevices(), we'll have to fix
-     * the configuration now.
-     */
-    /* USB device node. */
-    PCFGMNODE pDevNode = CFGMR3GetChildF(CFGMR3GetRoot(pVM), "USB/%s/", pUsbDev->pReg->szName);
-    if (!pDevNode)
-    {
-        rc = CFGMR3InsertNodeF(CFGMR3GetRoot(pVM), &pDevNode, "USB/%s/", pUsbDev->pReg->szName);
-        AssertRCReturn(rc, rc);
-    }
-
-    /* The instance node and number. */
-    if (!pInstanceNode)
-    {
-        for (unsigned c = 0; c < _2M; c++)
-        {
-            iInstance = pUsbDev->iNextInstance++;
-            rc = CFGMR3InsertNodeF(pDevNode, &pInstanceNode, "%d/", iInstance);
-            if (rc != VERR_CFGM_NODE_EXISTS)
-                break;
-        }
-        AssertRCReturn(rc, rc);
-    }
-    else
-    {
-        Assert(iInstance >= 0);
-        if (iInstance >= (int)pUsbDev->iNextInstance)
-            pUsbDev->iNextInstance = iInstance + 1;
-    }
-
-    /* The instance config node. */
-    PCFGMNODE pConfigToDelete = NULL;
-    PCFGMNODE pConfig = NULL;
-    if (!ppConfig || !*ppConfig)
-    {
-        rc = CFGMR3InsertNode(pInstanceNode, "Config", &pConfig);
-        AssertRCReturn(rc, rc);
-    }
-    else if (fAtRuntime)
-    {
-        rc = CFGMR3InsertSubTree(pInstanceNode, "Config", *ppConfig, &pConfig);
-        AssertRCReturn(rc, rc);
-        *ppConfig = NULL;
-        pConfigToDelete = pConfig;
-    }
-    else
-        pConfig = *ppConfig;
-    Assert(CFGMR3GetChild(pInstanceNode, "Config") == pConfig);
-
-    /* The global device config node. */
-    PCFGMNODE pGlobalConfig = CFGMR3GetChild(pDevNode, "GlobalConfig");
-    if (!pGlobalConfig)
-    {
-        rc = CFGMR3InsertNode(pDevNode, "GlobalConfig", &pGlobalConfig);
-        if (RT_FAILURE(rc))
-        {
-            CFGMR3RemoveNode(pConfigToDelete);
-            AssertRCReturn(rc, rc);
-        }
-    }
-
-    /*
-     * Allocate the device instance.
-     */
-    size_t cb = RT_OFFSETOF(PDMUSBINS, achInstanceData[pUsbDev->pReg->cbInstance]);
-    cb = RT_ALIGN_Z(cb, 16);
-    PPDMUSBINS pUsbIns;
-    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_USB, cb, (void **)&pUsbIns);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsgFailed(("Failed to allocate %d bytes of instance data for USB device '%s'. rc=%Rrc\n",
-                         cb, pUsbDev->pReg->szName, rc));
-        CFGMR3RemoveNode(pConfigToDelete);
-        return rc;
-    }
-
-    /*
-     * Initialize it.
-     */
-    pUsbIns->u32Version                     = PDM_USBINS_VERSION;
-    //pUsbIns->Internal.s.pNext               = NULL;
-    //pUsbIns->Internal.s.pPerDeviceNext      = NULL;
-    pUsbIns->Internal.s.pUsbDev             = pUsbDev;
-    pUsbIns->Internal.s.pVM                 = pVM;
-    //pUsbIns->Internal.s.pLuns               = NULL;
-    pUsbIns->Internal.s.pCfg                = pInstanceNode;
-    pUsbIns->Internal.s.pCfgDelete          = pConfigToDelete;
-    pUsbIns->Internal.s.pCfgGlobal          = pGlobalConfig;
-    pUsbIns->Internal.s.Uuid                = *pUuid;
-    //pUsbIns->Internal.s.pHub                = NULL;
-    pUsbIns->Internal.s.iPort               = UINT32_MAX; /* to be determined. */
-    pUsbIns->Internal.s.fVMSuspended        = true;
-    //pUsbIns->Internal.s.pfnAsyncNotify      = NULL;
-    pUsbIns->pHlpR3                         = &g_pdmR3UsbHlp;
-    pUsbIns->pReg                           = pUsbDev->pReg;
-    pUsbIns->pCfg                           = pConfig;
-    pUsbIns->pCfgGlobal                     = pGlobalConfig;
-    pUsbIns->iInstance                      = iInstance;
-    pUsbIns->pvInstanceDataR3               = &pUsbIns->achInstanceData[0];
-    pUsbIns->pszName                        = RTStrDup(pUsbDev->pReg->szName);
-
-    /*
-     * Link it into all the lists.
-     */
-    /* The global instance FIFO. */
-    PPDMUSBINS pPrev1 = pVM->pdm.s.pUsbInstances;
-    if (!pPrev1)
-        pVM->pdm.s.pUsbInstances = pUsbIns;
-    else
-    {
-        while (pPrev1->Internal.s.pNext)
-        {
-            Assert(pPrev1->u32Version == PDM_USBINS_VERSION);
-            pPrev1 = pPrev1->Internal.s.pNext;
-        }
-        pPrev1->Internal.s.pNext = pUsbIns;
-    }
-
-    /* The per device instance FIFO. */
-    PPDMUSBINS pPrev2 = pUsbDev->pInstances;
-    if (!pPrev2)
-        pUsbDev->pInstances = pUsbIns;
-    else
-    {
-        while (pPrev2->Internal.s.pPerDeviceNext)
-        {
-            Assert(pPrev2->u32Version == PDM_USBINS_VERSION);
-            pPrev2 = pPrev2->Internal.s.pPerDeviceNext;
-        }
-        pPrev2->Internal.s.pPerDeviceNext = pUsbIns;
-    }
-
-    /*
-     * Call the constructor.
-     */
-    Log(("PDM: Constructing USB device '%s' instance %d...\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-    rc = pUsbIns->pReg->pfnConstruct(pUsbIns, pUsbIns->iInstance, pUsbIns->pCfg, pUsbIns->pCfgGlobal);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Attach it to the hub.
-         */
-        Log(("PDM: Attaching it...\n"));
-        rc = pHub->Reg.pfnAttachDevice(pHub->pDrvIns, pUsbIns, &pUsbIns->Internal.s.iPort);
-        if (RT_SUCCESS(rc))
-        {
-            pHub->cAvailablePorts--;
-            Assert((int32_t)pHub->cAvailablePorts >= 0 && pHub->cAvailablePorts < pHub->cPorts);
-            pUsbIns->Internal.s.pHub = pHub;
-
-            /* Send the hot-plugged notification if applicable. */
-            if (fAtRuntime && pUsbIns->pReg->pfnHotPlugged)
-                pUsbIns->pReg->pfnHotPlugged(pUsbIns);
-
-            Log(("PDM: Successfully attached USB device '%s' instance %d to hub %p\n",
-                 pUsbIns->pReg->szName, pUsbIns->iInstance, pHub));
-            return VINF_SUCCESS;
-        }
-
-        LogRel(("PDM: Failed to attach USB device '%s' instance %d to hub %p: %Rrc\n",
-                pUsbIns->pReg->szName, pUsbIns->iInstance, pHub, rc));
-    }
-    else
-        AssertMsgFailed(("Failed to construct '%s'/%d! %Rra\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-    if (fAtRuntime)
-        pdmR3UsbDestroyDevice(pVM, pUsbIns);
-    /* else: destructors are invoked later. */
-    return rc;
-}
-
-
-/**
- * Instantiate USB devices.
- *
- * This is called by pdmR3DevInit() after it has instantiated the
- * other devices and their drivers. If there aren't any hubs
- * around, we'll silently skip the USB devices.
- *
- * @returns VBox status code.
- * @param   pVM
- */
-int pdmR3UsbInstantiateDevices(PVM pVM)
-{
-    /*
-     * Any hubs?
-     */
-    if (!pVM->pdm.s.pUsbHubs)
-    {
-        Log(("PDM: No USB hubs, skipping USB device instantiation.\n"));
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Count the device instances.
-     */
-    PCFGMNODE pCur;
-    PCFGMNODE pUsbNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "USB/");
-    PCFGMNODE pInstanceNode;
-    unsigned cUsbDevs = 0;
-    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
-    {
-        PCFGMNODE pGlobal = CFGMR3GetChild(pCur, "GlobalConfig/");
-        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
-            if (pInstanceNode != pGlobal)
-                cUsbDevs++;
-    }
-    if (!cUsbDevs)
-    {
-        Log(("PDM: No USB devices were configured!\n"));
-        return VINF_SUCCESS;
-    }
-    Log2(("PDM: cUsbDevs=%d!\n", cUsbDevs));
-
-    /*
-     * Collect info on each USB device instance.
-     */
-    struct USBDEVORDER
-    {
-        /** Configuration node. */
-        PCFGMNODE   pNode;
-        /** Pointer to the USB device. */
-        PPDMUSB     pUsbDev;
-        /** Init order. */
-        uint32_t    u32Order;
-        /** VBox instance number. */
-        uint32_t    iInstance;
-    } *paUsbDevs = (struct USBDEVORDER *)alloca(sizeof(paUsbDevs[0]) * (cUsbDevs + 1)); /* (One extra for swapping) */
-    Assert(paUsbDevs);
-    int rc;
-    unsigned i = 0;
-    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
-    {
-        /* Get the device name. */
-        char szName[sizeof(paUsbDevs[0].pUsbDev->pReg->szName)];
-        rc = CFGMR3GetName(pCur, szName, sizeof(szName));
-        AssertMsgRCReturn(rc, ("Configuration error: device name is too long (or something)! rc=%Rrc\n", rc), rc);
-
-        /* Find the device. */
-        PPDMUSB pUsbDev = pdmR3UsbLookup(pVM, szName);
-        AssertMsgReturn(pUsbDev, ("Configuration error: device '%s' not found!\n", szName), VERR_PDM_DEVICE_NOT_FOUND);
-
-        /* Configured priority or use default? */
-        uint32_t u32Order;
-        rc = CFGMR3QueryU32(pCur, "Priority", &u32Order);
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-            u32Order = i << 4;
-        else
-            AssertMsgRCReturn(rc, ("Configuration error: reading \"Priority\" for the '%s' USB device failed rc=%Rrc!\n", szName, rc), rc);
-
-        /* Global config. */
-        PCFGMNODE pGlobal = CFGMR3GetChild(pCur, "GlobalConfig/");
-        if (!pGlobal)
-        {
-            rc = CFGMR3InsertNode(pCur, "GlobalConfig/", &pGlobal);
-            AssertMsgRCReturn(rc, ("Failed to create GlobalConfig node! rc=%Rrc\n", rc), rc);
-            CFGMR3SetRestrictedRoot(pGlobal);
-        }
-
-        /* Enumerate the device instances. */
-        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
-        {
-            if (pInstanceNode == pGlobal)
-                continue;
-
-            paUsbDevs[i].pNode = pInstanceNode;
-            paUsbDevs[i].pUsbDev = pUsbDev;
-            paUsbDevs[i].u32Order = u32Order;
-
-            /* Get the instance number. */
-            char szInstance[32];
-            rc = CFGMR3GetName(pInstanceNode, szInstance, sizeof(szInstance));
-            AssertMsgRCReturn(rc, ("Configuration error: instance name is too long (or something)! rc=%Rrc\n", rc), rc);
-            char *pszNext = NULL;
-            rc = RTStrToUInt32Ex(szInstance, &pszNext, 0, &paUsbDevs[i].iInstance);
-            AssertMsgRCReturn(rc, ("Configuration error: RTStrToInt32Ex failed on the instance name '%s'! rc=%Rrc\n", szInstance, rc), rc);
-            AssertMsgReturn(!*pszNext, ("Configuration error: the instance name '%s' isn't all digits. (%s)\n", szInstance, pszNext), VERR_INVALID_PARAMETER);
-
-            /* next instance */
-            i++;
-        }
-    } /* devices */
-    Assert(i == cUsbDevs);
-
-    /*
-     * Sort the device array ascending on u32Order. (bubble)
-     */
-    unsigned c = cUsbDevs - 1;
-    while (c)
-    {
-        unsigned j = 0;
-        for (i = 0; i < c; i++)
-            if (paUsbDevs[i].u32Order > paUsbDevs[i + 1].u32Order)
-            {
-                paUsbDevs[cUsbDevs] = paUsbDevs[i + 1];
-                paUsbDevs[i + 1] = paUsbDevs[i];
-                paUsbDevs[i] = paUsbDevs[cUsbDevs];
-                j = i;
-            }
-        c = j;
-    }
-
-    /*
-     * Instantiate the devices.
-     */
-    for (i = 0; i < cUsbDevs; i++)
-    {
-        /*
-         * Make sure there is a config node and mark it as restricted.
-         */
-        PCFGMNODE pConfigNode = CFGMR3GetChild(paUsbDevs[i].pNode, "Config/");
-        if (!pConfigNode)
-        {
-            rc = CFGMR3InsertNode(paUsbDevs[i].pNode, "Config", &pConfigNode);
-            AssertMsgRCReturn(rc, ("Failed to create Config node! rc=%Rrc\n", rc), rc);
-        }
-        CFGMR3SetRestrictedRoot(pConfigNode);
-
-        /** @todo
-         * Figure out the USB version from the USB device registration and the configuration.
-         */
-        uint32_t iUsbVersion = VUSB_STDVER_11;
-
-        /*
-         * Find a suitable hub with free ports.
-         */
-        PPDMUSBHUB pHub;
-        rc = pdmR3UsbFindHub(pVM, iUsbVersion, &pHub);
-        if (RT_FAILURE(rc))
-        {
-            Log(("pdmR3UsbFindHub failed %Rrc\n", rc));
-            return rc;
-        }
-
-        /*
-         * Create and attach the device.
-         */
-        RTUUID Uuid;
-        rc = RTUuidCreate(&Uuid);
-        AssertRCReturn(rc, rc);
-        rc = pdmR3UsbCreateDevice(pVM, pHub, paUsbDevs[i].pUsbDev, paUsbDevs[i].iInstance, &Uuid, paUsbDevs[i].pNode, &pConfigNode, iUsbVersion);
-        if (RT_FAILURE(rc))
-            return rc;
-    } /* for device instances */
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Creates a USB proxy device instance.
- *
- * This will find an appropriate HUB for the USB device, create the necessary CFGM stuff
- * and try instantiate the proxy device.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pUuid           The UUID to be associated with the device.
- * @param   fRemote         Whether it's a remove or local device.
- * @param   pszAddress      The address string.
- * @param   pvBackend       Pointer to the backend.
- * @param   iUsbVersion     The preferred USB version.
- * @param   fMaskedIfs      The interfaces to hide from the guest.
- */
-VMMR3DECL(int) PDMR3USBCreateProxyDevice(PVM pVM, PCRTUUID pUuid, bool fRemote, const char *pszAddress, void *pvBackend,
-                                         uint32_t iUsbVersion, uint32_t fMaskedIfs)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT(pVM);
-    AssertPtrReturn(pUuid, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszAddress, VERR_INVALID_POINTER);
-    AssertReturn(    iUsbVersion == VUSB_STDVER_20
-                 ||  iUsbVersion == VUSB_STDVER_11, VERR_INVALID_PARAMETER);
-
-    /*
-     * Find the USBProxy driver.
-     */
-    PPDMUSB pUsbDev = pdmR3UsbLookup(pVM, "USBProxy");
-    if (!pUsbDev)
-    {
-        LogRel(("PDMR3USBCreateProxyDevice: The USBProxy device class wasn't found\n"));
-        return VERR_PDM_NO_USBPROXY;
-    }
-
-    /*
-     * Find a suitable hub with free ports.
-     */
-    PPDMUSBHUB pHub;
-    int rc = pdmR3UsbFindHub(pVM, iUsbVersion, &pHub);
-    if (RT_FAILURE(rc))
-    {
-        Log(("pdmR3UsbFindHub: failed %Rrc\n", rc));
-        return rc;
-    }
-
-    /*
-     * Create the CFGM configuration node.
-     */
-    PCFGMNODE pConfig = CFGMR3CreateTree(pVM);
-    AssertReturn(pConfig, VERR_NO_MEMORY);
-    do /* break loop */
-    {
-        rc = CFGMR3InsertString(pConfig,  "Address", pszAddress);               AssertRCBreak(rc);
-        char szUuid[RTUUID_STR_LENGTH];
-        rc = RTUuidToStr(pUuid, &szUuid[0], sizeof(szUuid));                    AssertRCBreak(rc);
-        rc = CFGMR3InsertString(pConfig,  "UUID", szUuid);                      AssertRCBreak(rc);
-        rc = CFGMR3InsertInteger(pConfig, "Remote", fRemote);                   AssertRCBreak(rc);
-        rc = CFGMR3InsertInteger(pConfig, "USBVersion", iUsbVersion);           AssertRCBreak(rc);
-        rc = CFGMR3InsertInteger(pConfig, "pvBackend", (uintptr_t)pvBackend);   AssertRCBreak(rc);
-        rc = CFGMR3InsertInteger(pConfig, "MaskedIfs", fMaskedIfs);             AssertRCBreak(rc);
-        rc = CFGMR3InsertInteger(pConfig, "Force11Device", !(pHub->fVersions & iUsbVersion)); AssertRCBreak(rc);
-    } while (0); /* break loop */
-    if (RT_FAILURE(rc))
-    {
-        CFGMR3RemoveNode(pConfig);
-        LogRel(("PDMR3USBCreateProxyDevice: failed to setup CFGM config, rc=%Rrc\n", rc));
-        return rc;
-    }
-
-    /*
-     * Finally, try create it.
-     */
-    rc = pdmR3UsbCreateDevice(pVM, pHub, pUsbDev, -1, pUuid, NULL, &pConfig, iUsbVersion);
-    if (RT_FAILURE(rc) && pConfig)
-        CFGMR3RemoveNode(pConfig);
-    return rc;
-}
-
-
-/**
- * Destroys a hot-plugged USB device.
- *
- * The device must be detached from the HUB at this point.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pUsbIns         The USB device instance to destroy.
- * @thread  EMT
- */
-static void pdmR3UsbDestroyDevice(PVM pVM, PPDMUSBINS pUsbIns)
-{
-    Assert(!pUsbIns->Internal.s.pHub);
-
-    /*
-     * Do the unplug notification.
-     */
-    /** @todo what about the drivers? */
-    if (pUsbIns->pReg->pfnHotUnplugged)
-        pUsbIns->pReg->pfnHotUnplugged(pUsbIns);
-
-    /*
-     * Destroy the luns with their driver chains and call the device destructor.
-     */
-    while (pUsbIns->Internal.s.pLuns)
-    {
-        PPDMLUN pLun = pUsbIns->Internal.s.pLuns;
-        pUsbIns->Internal.s.pLuns = pLun->pNext;
-        if (pLun->pTop)
-            pdmR3DrvDestroyChain(pLun->pTop, PDM_TACH_FLAGS_NOT_HOT_PLUG); /* Hotplugging is handled differently here atm. */
-        MMR3HeapFree(pLun);
-    }
-
-    /* finally, the device. */
-    if (pUsbIns->pReg->pfnDestruct)
-    {
-        Log(("PDM: Destructing USB device '%s' instance %d...\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-        pUsbIns->pReg->pfnDestruct(pUsbIns);
-    }
-    //TMR3TimerDestroyUsb(pVM, pUsbIns);
-    //SSMR3DeregisterUsb(pVM, pUsbIns, NULL, 0);
-    pdmR3ThreadDestroyUsb(pVM, pUsbIns);
-
-    /*
-     * Unlink it.
-     */
-    /* The global instance FIFO. */
-    if (pVM->pdm.s.pUsbInstances == pUsbIns)
-        pVM->pdm.s.pUsbInstances = pUsbIns->Internal.s.pNext;
-    else
-    {
-        PPDMUSBINS pPrev = pVM->pdm.s.pUsbInstances;
-        while (pPrev && pPrev->Internal.s.pNext != pUsbIns)
-        {
-            Assert(pPrev->u32Version == PDM_USBINS_VERSION);
-            pPrev = pPrev->Internal.s.pNext;
-        }
-        Assert(pPrev); Assert(pPrev != pUsbIns);
-        if (pPrev)
-            pPrev->Internal.s.pNext = pUsbIns->Internal.s.pNext;
-    }
-
-    /* The per device instance FIFO. */
-    PPDMUSB pUsbDev = pUsbIns->Internal.s.pUsbDev;
-    if (pUsbDev->pInstances == pUsbIns)
-        pUsbDev->pInstances = pUsbIns->Internal.s.pPerDeviceNext;
-    else
-    {
-        PPDMUSBINS pPrev = pUsbDev->pInstances;
-        while (pPrev && pPrev->Internal.s.pPerDeviceNext != pUsbIns)
-        {
-            Assert(pPrev->u32Version == PDM_USBINS_VERSION);
-            pPrev = pPrev->Internal.s.pPerDeviceNext;
-        }
-        Assert(pPrev); Assert(pPrev != pUsbIns);
-        if (pPrev)
-            pPrev->Internal.s.pPerDeviceNext = pUsbIns->Internal.s.pPerDeviceNext;
-    }
-
-    /*
-     * Trash it.
-     */
-    pUsbIns->u32Version = 0;
-    pUsbIns->pReg = NULL;
-    if (pUsbIns->pszName)
-    {
-        RTStrFree(pUsbIns->pszName);
-        pUsbIns->pszName = NULL;
-    }
-    CFGMR3RemoveNode(pUsbIns->Internal.s.pCfgDelete);
-    MMR3HeapFree(pUsbIns);
-}
-
-
-/**
- * Detaches and destroys a USB device.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pUuid           The UUID associated with the device to detach.
- * @thread  EMT
- */
-VMMR3DECL(int) PDMR3USBDetachDevice(PVM pVM, PCRTUUID pUuid)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT(pVM);
-    AssertPtrReturn(pUuid, VERR_INVALID_POINTER);
-    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
-
-    /*
-     * Search the global list for it.
-     */
-    PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;
-    for ( ; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
-        if (!RTUuidCompare(&pUsbIns->Internal.s.Uuid, pUuid))
-            break;
-    if (!pUsbIns)
-        return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND; /** @todo VERR_PDM_USB_INSTANCE_NOT_FOUND */
-
-    /*
-     * Detach it from the HUB (if it's actually attached to one).
-     */
-    PPDMUSBHUB pHub = pUsbIns->Internal.s.pHub;
-    if (pHub)
-    {
-        int rc = pHub->Reg.pfnDetachDevice(pHub->pDrvIns, pUsbIns, pUsbIns->Internal.s.iPort);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("PDM: Failed to detach USB device '%s' instance %d from %p: %Rrc\n",
-                    pUsbIns->pReg->szName, pUsbIns->iInstance, pHub, rc));
-            return rc;
-        }
-
-        pHub->cAvailablePorts++;
-        Assert(pHub->cAvailablePorts > 0 && pHub->cAvailablePorts <= pHub->cPorts);
-        pUsbIns->Internal.s.pHub = NULL;
-    }
-
-    /*
-     * Notify about unplugging and destroy the device with it's drivers.
-     */
-    pdmR3UsbDestroyDevice(pVM, pUsbIns);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Checks if there are any USB hubs attached.
- *
- * @returns true / false accordingly.
- * @param   pVM     Pointer to the shared VM structure.
- */
-VMMR3DECL(bool) PDMR3USBHasHub(PVM pVM)
-{
-    return pVM->pdm.s.pUsbHubs != NULL;
-}
-
-
-/** @name USB Device Helpers
- * @{
- */
-
-/** @interface_method_impl{PDMUSBHLPR3,pfnDriverAttach} */
-static DECLCALLBACK(int) pdmR3UsbHlp_DriverAttach(PPDMUSBINS pUsbIns, RTUINT iLun, PPDMIBASE pBaseInterface, PPDMIBASE *ppBaseInterface, const char *pszDesc)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    PVM pVM = pUsbIns->Internal.s.pVM;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: iLun=%d pBaseInterface=%p ppBaseInterface=%p pszDesc=%p:{%s}\n",
-             pUsbIns->pReg->szName, pUsbIns->iInstance, iLun, pBaseInterface, ppBaseInterface, pszDesc, pszDesc));
-
-    /*
-     * Lookup the LUN, it might already be registered.
-     */
-    PPDMLUN pLunPrev = NULL;
-    PPDMLUN pLun = pUsbIns->Internal.s.pLuns;
-    for (; pLun; pLunPrev = pLun, pLun = pLun->pNext)
-        if (pLun->iLun == iLun)
-            break;
-
-    /*
-     * Create the LUN if if wasn't found, else check if driver is already attached to it.
-     */
-    if (!pLun)
-    {
-        if (    !pBaseInterface
-            ||  !pszDesc
-            ||  !*pszDesc)
-        {
-            Assert(pBaseInterface);
-            Assert(pszDesc || *pszDesc);
-            return VERR_INVALID_PARAMETER;
-        }
-
-        pLun = (PPDMLUN)MMR3HeapAlloc(pVM, MM_TAG_PDM_LUN, sizeof(*pLun));
-        if (!pLun)
-            return VERR_NO_MEMORY;
-
-        pLun->iLun      = iLun;
-        pLun->pNext     = pLunPrev ? pLunPrev->pNext : NULL;
-        pLun->pTop      = NULL;
-        pLun->pBottom   = NULL;
-        pLun->pDevIns   = NULL;
-        pLun->pUsbIns   = pUsbIns;
-        pLun->pszDesc   = pszDesc;
-        pLun->pBase     = pBaseInterface;
-        if (!pLunPrev)
-            pUsbIns->Internal.s.pLuns = pLun;
-        else
-            pLunPrev->pNext = pLun;
-        Log(("pdmR3UsbHlp_DriverAttach: Registered LUN#%d '%s' with device '%s'/%d.\n",
-             iLun, pszDesc, pUsbIns->pReg->szName, pUsbIns->iInstance));
-    }
-    else if (pLun->pTop)
-    {
-        AssertMsgFailed(("Already attached! The device should keep track of such things!\n"));
-        LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, VERR_PDM_DRIVER_ALREADY_ATTACHED));
-        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
-    }
-    Assert(pLun->pBase == pBaseInterface);
-
-
-    /*
-     * Get the attached driver configuration.
-     */
-    int rc;
-    PCFGMNODE pNode = CFGMR3GetChildF(pUsbIns->Internal.s.pCfg, "LUN#%u", iLun);
-    if (pNode)
-        rc = pdmR3DrvInstantiate(pVM, pNode, pBaseInterface, NULL /*pDrvAbove*/, pLun, ppBaseInterface);
-    else
-        rc = VERR_PDM_NO_ATTACHED_DRIVER;
-
-
-    LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnAssertEMT} */
-static DECLCALLBACK(bool) pdmR3UsbHlp_AssertEMT(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    if (VM_IS_EMT(pUsbIns->Internal.s.pVM))
-        return true;
-
-    char szMsg[100];
-    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance);
-    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
-    AssertBreakpoint();
-    return false;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnAssertOther} */
-static DECLCALLBACK(bool) pdmR3UsbHlp_AssertOther(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    if (!VM_IS_EMT(pUsbIns->Internal.s.pVM))
-        return true;
-
-    char szMsg[100];
-    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance);
-    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
-    AssertBreakpoint();
-    return false;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnDBGFStopV} */
-static DECLCALLBACK(int) pdmR3UsbHlp_DBGFStopV(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction, const char *pszFormat, va_list args)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-#ifdef LOG_ENABLED
-    va_list va2;
-    va_copy(va2, args);
-    LogFlow(("pdmR3UsbHlp_DBGFStopV: caller='%s'/%d: pszFile=%p:{%s} iLine=%d pszFunction=%p:{%s} pszFormat=%p:{%s} (%N)\n",
-             pUsbIns->pReg->szName, pUsbIns->iInstance, pszFile, pszFile, iLine, pszFunction, pszFunction, pszFormat, pszFormat, pszFormat, &va2));
-    va_end(va2);
-#endif
-
-    PVM pVM = pUsbIns->Internal.s.pVM;
-    VM_ASSERT_EMT(pVM);
-    int rc = DBGFR3EventSrcV(pVM, DBGFEVENT_DEV_STOP, pszFile, iLine, pszFunction, pszFormat, args);
-    if (rc == VERR_DBGF_NOT_ATTACHED)
-        rc = VINF_SUCCESS;
-
-    LogFlow(("pdmR3UsbHlp_DBGFStopV: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnDBGFInfoRegister} */
-static DECLCALLBACK(int) pdmR3UsbHlp_DBGFInfoRegister(PPDMUSBINS pUsbIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERUSB pfnHandler)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    LogFlow(("pdmR3UsbHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
-             pUsbIns->pReg->szName, pUsbIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
-
-    PVM pVM = pUsbIns->Internal.s.pVM;
-    VM_ASSERT_EMT(pVM);
-    /** @todo int rc = DBGFR3InfoRegisterUsb(pVM, pszName, pszDesc, pfnHandler, pUsbIns); */
-    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
-
-    LogFlow(("pdmR3UsbHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnMMHeapAlloc} */
-static DECLCALLBACK(void *) pdmR3UsbHlp_MMHeapAlloc(PPDMUSBINS pUsbIns, size_t cb)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    LogFlow(("pdmR3UsbHlp_MMHeapAlloc: caller='%s'/%d: cb=%#x\n", pUsbIns->pReg->szName, pUsbIns->iInstance, cb));
-
-    void *pv = MMR3HeapAlloc(pUsbIns->Internal.s.pVM, MM_TAG_PDM_USB_USER, cb);
-
-    LogFlow(("pdmR3UsbHlp_MMHeapAlloc: caller='%s'/%d: returns %p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pv));
-    return pv;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnMMHeapAllocZ} */
-static DECLCALLBACK(void *) pdmR3UsbHlp_MMHeapAllocZ(PPDMUSBINS pUsbIns, size_t cb)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    LogFlow(("pdmR3UsbHlp_MMHeapAllocZ: caller='%s'/%d: cb=%#x\n", pUsbIns->pReg->szName, pUsbIns->iInstance, cb));
-
-    void *pv = MMR3HeapAllocZ(pUsbIns->Internal.s.pVM, MM_TAG_PDM_USB_USER, cb);
-
-    LogFlow(("pdmR3UsbHlp_MMHeapAllocZ: caller='%s'/%d: returns %p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pv));
-    return pv;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnPDMQueueCreate} */
-static DECLCALLBACK(int) pdmR3UsbHlp_PDMQueueCreate(PPDMUSBINS pUsbIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
-                                                    PFNPDMQUEUEUSB pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    LogFlow(("pdmR3UsbHlp_PDMQueueCreate: caller='%s'/%d: cbItem=%#x cItems=%#x cMilliesInterval=%u pfnCallback=%p pszName=%p:{%s} ppQueue=%p\n",
-             pUsbIns->pReg->szName, pUsbIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, pszName, ppQueue));
-
-    PVM pVM = pUsbIns->Internal.s.pVM;
-    VM_ASSERT_EMT(pVM);
-
-    if (pUsbIns->iInstance > 0)
-    {
-        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s_%u", pszName, pUsbIns->iInstance);
-        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
-    }
-
-    /** @todo int rc = PDMR3QueueCreateUsb(pVM, pUsbIns, cbItem, cItems, cMilliesInterval, pfnCallback, fGCEnabled, pszName, ppQueue); */
-    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
-
-    LogFlow(("pdmR3UsbHlp_PDMQueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc, *ppQueue));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnSSMRegister} */
-static DECLCALLBACK(int) pdmR3UsbHlp_SSMRegister(PPDMUSBINS pUsbIns, uint32_t uVersion, size_t cbGuess,
-                                                 PFNSSMUSBLIVEPREP pfnLivePrep, PFNSSMUSBLIVEEXEC pfnLiveExec, PFNSSMUSBLIVEVOTE pfnLiveVote,
-                                                 PFNSSMUSBSAVEPREP pfnSavePrep, PFNSSMUSBSAVEEXEC pfnSaveExec, PFNSSMUSBSAVEDONE pfnSaveDone,
-                                                 PFNSSMUSBLOADPREP pfnLoadPrep, PFNSSMUSBLOADEXEC pfnLoadExec, PFNSSMUSBLOADDONE pfnLoadDone)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    VM_ASSERT_EMT(pUsbIns->Internal.s.pVM);
-    LogFlow(("pdmR3UsbHlp_SSMRegister: caller='%s'/%d: uVersion=#x cbGuess=%#x\n"
-             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoadDone=%p\n",
-             pUsbIns->pReg->szName, pUsbIns->iInstance, uVersion, cbGuess,
-             pfnLivePrep, pfnLiveExec, pfnLiveVote,
-             pfnSavePrep, pfnSaveExec, pfnSaveDone,
-             pfnLoadPrep, pfnLoadExec, pfnLoadDone));
-
-    /** @todo
-    int rc = SSMR3RegisterUsb(pUsbIns->Internal.s.pVM, pUsbIns, pUsbIns->pReg->szName, pUsbIns->iInstance,
-                              uVersion, cbGuess,
-                              pfnLivePrep, pfnLiveExec, pfnLiveVote,
-                              pfnSavePrep, pfnSaveExec, pfnSaveDone,
-                              pfnLoadPrep, pfnLoadExec, pfnLoadDone); */
-    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
-
-    LogFlow(("pdmR3UsbHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnSTAMRegisterV} */
-static DECLCALLBACK(void) pdmR3UsbHlp_STAMRegisterV(PPDMUSBINS pUsbIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
-                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    PVM pVM = pUsbIns->Internal.s.pVM;
-    VM_ASSERT_EMT(pVM);
-
-    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-    AssertRC(rc);
-
-    NOREF(pVM);
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnTMTimerCreate} */
-static DECLCALLBACK(int) pdmR3UsbHlp_TMTimerCreate(PPDMUSBINS pUsbIns, TMCLOCK enmClock, PFNTMTIMERUSB pfnCallback, void *pvUser,
-                                                   uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    PVM pVM = pUsbIns->Internal.s.pVM;
-    VM_ASSERT_EMT(pVM);
-    LogFlow(("pdmR3UsbHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
-             pUsbIns->pReg->szName, pUsbIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
-
-    if (pUsbIns->iInstance > 0) /** @todo use a string cache here later. */
-    {
-         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_USB_DESC, "%s [%u]", pszDesc, pUsbIns->iInstance);
-         if (pszDesc2)
-             pszDesc = pszDesc2;
-    }
-
-    /** @todo
-    int rc = TMR3TimerCreateUsb(pVM, pUsbIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer); */
-    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
-
-    LogFlow(("pdmR3UsbHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnVMSetErrorV} */
-static DECLCALLBACK(int) pdmR3UsbHlp_VMSetErrorV(PPDMUSBINS pUsbIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    int rc2 = VMSetErrorV(pUsbIns->Internal.s.pVM, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnVMSetRuntimeErrorV} */
-static DECLCALLBACK(int) pdmR3UsbHlp_VMSetRuntimeErrorV(PPDMUSBINS pUsbIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    int rc = VMSetRuntimeErrorV(pUsbIns->Internal.s.pVM, fFlags, pszErrorId, pszFormat, va);
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnVMState} */
-static DECLCALLBACK(VMSTATE) pdmR3UsbHlp_VMState(PPDMUSBINS pUsbIns)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-
-    VMSTATE enmVMState = VMR3GetState(pUsbIns->Internal.s.pVM);
-
-    LogFlow(("pdmR3UsbHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pUsbIns->pReg->szName, pUsbIns->iInstance,
-             enmVMState, VMR3GetStateName(enmVMState)));
-    return enmVMState;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnSetAsyncNotification} */
-static DECLCALLBACK(int) pdmR3UsbHlp_SetAsyncNotification(PPDMUSBINS pUsbIns, PFNPDMUSBASYNCNOTIFY pfnAsyncNotify)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    VM_ASSERT_EMT0(pUsbIns->Internal.s.pVM);
-    LogFlow(("pdmR3UsbHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pfnAsyncNotify));
-
-    int rc = VINF_SUCCESS;
-    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
-    AssertStmt(!pUsbIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
-    AssertStmt(pUsbIns->Internal.s.fVMSuspended || pUsbIns->Internal.s.fVMReset, rc = VERR_WRONG_ORDER);
-    VMSTATE enmVMState = VMR3GetState(pUsbIns->Internal.s.pVM);
-    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
-               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
-               || enmVMState == VMSTATE_SUSPENDING_LS
-               || enmVMState == VMSTATE_RESETTING
-               || enmVMState == VMSTATE_RESETTING_LS
-               || enmVMState == VMSTATE_POWERING_OFF
-               || enmVMState == VMSTATE_POWERING_OFF_LS,
-               rc = VERR_INVALID_STATE);
-
-    if (RT_SUCCESS(rc))
-        pUsbIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
-
-    LogFlow(("pdmR3UsbHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
-    return rc;
-}
-
-
-/** @interface_method_impl{PDMUSBHLP,pfnAsyncNotificationCompleted} */
-static DECLCALLBACK(void) pdmR3UsbHlp_AsyncNotificationCompleted(PPDMUSBINS pUsbIns)
-{
-    PDMUSB_ASSERT_USBINS(pUsbIns);
-    PVM pVM = pUsbIns->Internal.s.pVM;
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    if (   enmVMState == VMSTATE_SUSPENDING
-        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
-        || enmVMState == VMSTATE_SUSPENDING_LS
-        || enmVMState == VMSTATE_RESETTING
-        || enmVMState == VMSTATE_RESETTING_LS
-        || enmVMState == VMSTATE_POWERING_OFF
-        || enmVMState == VMSTATE_POWERING_OFF_LS)
-    {
-        LogFlow(("pdmR3UsbHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
-        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
-    }
-    else
-        LogFlow(("pdmR3UsbHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance, enmVMState));
-}
-
-
-/**
- * The USB device helper structure.
- */
-const PDMUSBHLP g_pdmR3UsbHlp =
-{
-    PDM_USBHLP_VERSION,
-    pdmR3UsbHlp_DriverAttach,
-    pdmR3UsbHlp_AssertEMT,
-    pdmR3UsbHlp_AssertOther,
-    pdmR3UsbHlp_DBGFStopV,
-    pdmR3UsbHlp_DBGFInfoRegister,
-    pdmR3UsbHlp_MMHeapAlloc,
-    pdmR3UsbHlp_MMHeapAllocZ,
-    pdmR3UsbHlp_PDMQueueCreate,
-    pdmR3UsbHlp_SSMRegister,
-    pdmR3UsbHlp_STAMRegisterV,
-    pdmR3UsbHlp_TMTimerCreate,
-    pdmR3UsbHlp_VMSetErrorV,
-    pdmR3UsbHlp_VMSetRuntimeErrorV,
-    pdmR3UsbHlp_VMState,
-    pdmR3UsbHlp_SetAsyncNotification,
-    pdmR3UsbHlp_AsyncNotificationCompleted,
-    PDM_USBHLP_VERSION
-};
-
-/** @}  */
Index: unk/src/VBox/VMM/PGM.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,4011 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager and Monitor. (Mixing stuff here, not good?)
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page pg_pgm PGM - The Page Manager and Monitor
- *
- * @see grp_pgm,
- * @ref pg_pgm_pool,
- * @ref pg_pgm_phys.
- *
- *
- * @section         sec_pgm_modes           Paging Modes
- *
- * There are three memory contexts: Host Context (HC), Guest Context (GC)
- * and intermediate context. When talking about paging HC can also be referred to
- * as "host paging", and GC referred to as "shadow paging".
- *
- * We define three basic paging modes: 32-bit, PAE and AMD64. The host paging mode
- * is defined by the host operating system. The mode used in the shadow paging mode
- * depends on the host paging mode and what the mode the guest is currently in. The
- * following relation between the two is defined:
- *
- * @verbatim
-     Host > 32-bit |  PAE   | AMD64  |
-   Guest  |        |        |        |
-   ==v================================
-   32-bit   32-bit    PAE     PAE
-   -------|--------|--------|--------|
-   PAE       PAE      PAE     PAE
-   -------|--------|--------|--------|
-   AMD64    AMD64    AMD64    AMD64
-   -------|--------|--------|--------| @endverbatim
- *
- * All configuration except those in the diagonal (upper left) are expected to
- * require special effort from the switcher (i.e. a bit slower).
- *
- *
- *
- *
- * @section         sec_pgm_shw             The Shadow Memory Context
- *
- *
- *  [..]
- *
- * Because of guest context mappings requires PDPT and PML4 entries to allow
- * writing on AMD64, the two upper levels will have fixed flags whatever the
- * guest is thinking of using there. So, when shadowing the PD level we will
- * calculate the effective flags of PD and all the higher levels. In legacy
- * PAE mode this only applies to the PWT and PCD bits (the rest are
- * ignored/reserved/MBZ). We will ignore those bits for the present.
- *
- *
- *
- * @section         sec_pgm_int             The Intermediate Memory Context
- *
- * The world switch goes thru an intermediate memory context which purpose it is
- * to provide different mappings of the switcher code. All guest mappings are also
- * present in this context.
- *
- * The switcher code is mapped at the same location as on the host, at an
- * identity mapped location (physical equals virtual address), and at the
- * hypervisor location. The identity mapped location is for when the world
- * switches that involves disabling paging.
- *
- * PGM maintain page tables for 32-bit, PAE and AMD64 paging modes. This
- * simplifies switching guest CPU mode and consistency at the cost of more
- * code to do the work. All memory use for those page tables is located below
- * 4GB (this includes page tables for guest context mappings).
- *
- *
- * @subsection      subsec_pgm_int_gc       Guest Context Mappings
- *
- * During assignment and relocation of a guest context mapping the intermediate
- * memory context is used to verify the new location.
- *
- * Guest context mappings are currently restricted to below 4GB, for reasons
- * of simplicity. This may change when we implement AMD64 support.
- *
- *
- *
- *
- * @section         sec_pgm_misc            Misc
- *
- * @subsection      subsec_pgm_misc_diff    Differences Between Legacy PAE and Long Mode PAE
- *
- * The differences between legacy PAE and long mode PAE are:
- *      -# PDPE bits 1, 2, 5 and 6 are defined differently. In leagcy mode they are
- *         all marked down as must-be-zero, while in long mode 1, 2 and 5 have the
- *         usual meanings while 6 is ignored (AMD). This means that upon switching to
- *         legacy PAE mode we'll have to clear these bits and when going to long mode
- *         they must be set. This applies to both intermediate and shadow contexts,
- *         however we don't need to do it for the intermediate one since we're
- *         executing with CR0.WP at that time.
- *      -# CR3 allows a 32-byte aligned address in legacy mode, while in long mode
- *         a page aligned one is required.
- *
- *
- * @section         sec_pgm_handlers        Access Handlers
- *
- * Placeholder.
- *
- *
- * @subsection      sec_pgm_handlers_virt   Virtual Access Handlers
- *
- * Placeholder.
- *
- *
- * @subsection      sec_pgm_handlers_virt   Virtual Access Handlers
- *
- * We currently implement three types of virtual access handlers:  ALL, WRITE
- * and HYPERVISOR (WRITE). See PGMVIRTHANDLERTYPE for some more details.
- *
- * The HYPERVISOR access handlers is kept in a separate tree since it doesn't apply
- * to physical pages (PGMTREES::HyperVirtHandlers) and only needs to be consulted in
- * a special \#PF case. The ALL and WRITE are in the PGMTREES::VirtHandlers tree, the
- * rest of this section is going to be about these handlers.
- *
- * We'll go thru the life cycle of a handler and try make sense of it all, don't know
- * how successful this is gonna be...
- *
- * 1. A handler is registered thru the PGMR3HandlerVirtualRegister and
- * PGMHandlerVirtualRegisterEx APIs. We check for conflicting virtual handlers
- * and create a new node that is inserted into the AVL tree (range key). Then
- * a full PGM resync is flagged (clear pool, sync cr3, update virtual bit of PGMPAGE).
- *
- * 2. The following PGMSyncCR3/SyncCR3 operation will first make invoke HandlerVirtualUpdate.
- *
- * 2a. HandlerVirtualUpdate will will lookup all the pages covered by virtual handlers
- * via the current guest CR3 and update the physical page -> virtual handler
- * translation. Needless to say, this doesn't exactly scale very well. If any changes
- * are detected, it will flag a virtual bit update just like we did on registration.
- * PGMPHYS pages with changes will have their virtual handler state reset to NONE.
- *
- * 2b. The virtual bit update process will iterate all the pages covered by all the
- * virtual handlers and update the PGMPAGE virtual handler state to the max of all
- * virtual handlers on that page.
- *
- * 2c. Back in SyncCR3 we will now flush the entire shadow page cache to make sure
- * we don't miss any alias mappings of the monitored pages.
- *
- * 2d. SyncCR3 will then proceed with syncing the CR3 table.
- *
- * 3. \#PF(np,read) on a page in the range. This will cause it to be synced
- * read-only and resumed if it's a WRITE handler. If it's an ALL handler we
- * will call the handlers like in the next step. If the physical mapping has
- * changed we will - some time in the future - perform a handler callback
- * (optional) and update the physical -> virtual handler cache.
- *
- * 4. \#PF(,write) on a page in the range. This will cause the handler to
- * be invoked.
- *
- * 5. The guest invalidates the page and changes the physical backing or
- * unmaps it. This should cause the invalidation callback to be invoked
- * (it might not yet be 100% perfect). Exactly what happens next... is
- * this where we mess up and end up out of sync for a while?
- *
- * 6. The handler is deregistered by the client via PGMHandlerVirtualDeregister.
- * We will then set all PGMPAGEs in the physical -> virtual handler cache for
- * this handler to NONE and trigger a full PGM resync (basically the same
- * as int step 1). Which means 2 is executed again.
- *
- *
- * @subsubsection   sub_sec_pgm_handler_virt_todo   TODOs
- *
- * There is a bunch of things that needs to be done to make the virtual handlers
- * work 100% correctly and work more efficiently.
- *
- * The first bit hasn't been implemented yet because it's going to slow the
- * whole mess down even more, and besides it seems to be working reliably for
- * our current uses. OTOH, some of the optimizations might end up more or less
- * implementing the missing bits, so we'll see.
- *
- * On the optimization side, the first thing to do is to try avoid unnecessary
- * cache flushing. Then try team up with the shadowing code to track changes
- * in mappings by means of access to them (shadow in), updates to shadows pages,
- * invlpg, and shadow PT discarding (perhaps).
- *
- * Some idea that have popped up for optimization for current and new features:
- *    - bitmap indicating where there are virtual handlers installed.
- *      (4KB => 2**20 pages, page 2**12 => covers 32-bit address space 1:1!)
- *    - Further optimize this by min/max (needs min/max avl getters).
- *    - Shadow page table entry bit (if any left)?
- *
- */
-
-
-/** @page pg_pgm_phys   PGM Physical Guest Memory Management
- *
- *
- * Objectives:
- *      - Guest RAM over-commitment using memory ballooning,
- *        zero pages and general page sharing.
- *      - Moving or mirroring a VM onto a different physical machine.
- *
- *
- * @subsection subsec_pgmPhys_Definitions       Definitions
- *
- * Allocation chunk - A RTR0MemObjAllocPhysNC object and the tracking
- * machinery associated with it.
- *
- *
- *
- *
- * @subsection subsec_pgmPhys_AllocPage         Allocating a page.
- *
- * Initially we map *all* guest memory to the (per VM) zero page, which
- * means that none of the read functions will cause pages to be allocated.
- *
- * Exception, access bit in page tables that have been shared. This must
- * be handled, but we must also make sure PGMGst*Modify doesn't make
- * unnecessary modifications.
- *
- * Allocation points:
- *      - PGMPhysSimpleWriteGCPhys and PGMPhysWrite.
- *      - Replacing a zero page mapping at \#PF.
- *      - Replacing a shared page mapping at \#PF.
- *      - ROM registration (currently MMR3RomRegister).
- *      - VM restore (pgmR3Load).
- *
- * For the first three it would make sense to keep a few pages handy
- * until we've reached the max memory commitment for the VM.
- *
- * For the ROM registration, we know exactly how many pages we need
- * and will request these from ring-0. For restore, we will save
- * the number of non-zero pages in the saved state and allocate
- * them up front. This would allow the ring-0 component to refuse
- * the request if the isn't sufficient memory available for VM use.
- *
- * Btw. for both ROM and restore allocations we won't be requiring
- * zeroed pages as they are going to be filled instantly.
- *
- *
- * @subsection subsec_pgmPhys_FreePage          Freeing a page
- *
- * There are a few points where a page can be freed:
- *      - After being replaced by the zero page.
- *      - After being replaced by a shared page.
- *      - After being ballooned by the guest additions.
- *      - At reset.
- *      - At restore.
- *
- * When freeing one or more pages they will be returned to the ring-0
- * component and replaced by the zero page.
- *
- * The reasoning for clearing out all the pages on reset is that it will
- * return us to the exact same state as on power on, and may thereby help
- * us reduce the memory load on the system. Further it might have a
- * (temporary) positive influence on memory fragmentation (@see subsec_pgmPhys_Fragmentation).
- *
- * On restore, as mention under the allocation topic, pages should be
- * freed / allocated depending on how many is actually required by the
- * new VM state. The simplest approach is to do like on reset, and free
- * all non-ROM pages and then allocate what we need.
- *
- * A measure to prevent some fragmentation, would be to let each allocation
- * chunk have some affinity towards the VM having allocated the most pages
- * from it. Also, try make sure to allocate from allocation chunks that
- * are almost full. Admittedly, both these measures might work counter to
- * our intentions and its probably not worth putting a lot of effort,
- * cpu time or memory into this.
- *
- *
- * @subsection subsec_pgmPhys_SharePage         Sharing a page
- *
- * The basic idea is that there there will be a idle priority kernel
- * thread walking the non-shared VM pages hashing them and looking for
- * pages with the same checksum. If such pages are found, it will compare
- * them byte-by-byte to see if they actually are identical. If found to be
- * identical it will allocate a shared page, copy the content, check that
- * the page didn't change while doing this, and finally request both the
- * VMs to use the shared page instead. If the page is all zeros (special
- * checksum and byte-by-byte check) it will request the VM that owns it
- * to replace it with the zero page.
- *
- * To make this efficient, we will have to make sure not to try share a page
- * that will change its contents soon. This part requires the most work.
- * A simple idea would be to request the VM to write monitor the page for
- * a while to make sure it isn't modified any time soon. Also, it may
- * make sense to skip pages that are being write monitored since this
- * information is readily available to the thread if it works on the
- * per-VM guest memory structures (presently called PGMRAMRANGE).
- *
- *
- * @subsection subsec_pgmPhys_Fragmentation     Fragmentation Concerns and Counter Measures
- *
- * The pages are organized in allocation chunks in ring-0, this is a necessity
- * if we wish to have an OS agnostic approach to this whole thing. (On Linux we
- * could easily work on a page-by-page basis if we liked. Whether this is possible
- * or efficient on NT I don't quite know.) Fragmentation within these chunks may
- * become a problem as part of the idea here is that we wish to return memory to
- * the host system.
- *
- * For instance, starting two VMs at the same time, they will both allocate the
- * guest memory on-demand and if permitted their page allocations will be
- * intermixed. Shut down one of the two VMs and it will be difficult to return
- * any memory to the host system because the page allocation for the two VMs are
- * mixed up in the same allocation chunks.
- *
- * To further complicate matters, when pages are freed because they have been
- * ballooned or become shared/zero the whole idea is that the page is supposed
- * to be reused by another VM or returned to the host system. This will cause
- * allocation chunks to contain pages belonging to different VMs and prevent
- * returning memory to the host when one of those VM shuts down.
- *
- * The only way to really deal with this problem is to move pages. This can
- * either be done at VM shutdown and or by the idle priority worker thread
- * that will be responsible for finding sharable/zero pages. The mechanisms
- * involved for coercing a VM to move a page (or to do it for it) will be
- * the same as when telling it to share/zero a page.
- *
- *
- * @subsection subsec_pgmPhys_Tracking      Tracking Structures And Their Cost
- *
- * There's a difficult balance between keeping the per-page tracking structures
- * (global and guest page) easy to use and keeping them from eating too much
- * memory. We have limited virtual memory resources available when operating in
- * 32-bit kernel space (on 64-bit there'll it's quite a different story). The
- * tracking structures will be attempted designed such that we can deal with up
- * to 32GB of memory on a 32-bit system and essentially unlimited on 64-bit ones.
- *
- *
- * @subsubsection subsubsec_pgmPhys_Tracking_Kernel     Kernel Space
- *
- * @see pg_GMM
- *
- * @subsubsection subsubsec_pgmPhys_Tracking_PerVM      Per-VM
- *
- * Fixed info is the physical address of the page (HCPhys) and the page id
- * (described above). Theoretically we'll need 48(-12) bits for the HCPhys part.
- * Today we've restricting ourselves to 40(-12) bits because this is the current
- * restrictions of all AMD64 implementations (I think Barcelona will up this
- * to 48(-12) bits, not that it really matters) and I needed the bits for
- * tracking mappings of a page. 48-12 = 36. That leaves 28 bits, which means a
- * decent range for the page id: 2^(28+12) = 1024TB.
- *
- * In additions to these, we'll have to keep maintaining the page flags as we
- * currently do. Although it wouldn't harm to optimize these quite a bit, like
- * for instance the ROM shouldn't depend on having a write handler installed
- * in order for it to become read-only. A RO/RW bit should be considered so
- * that the page syncing code doesn't have to mess about checking multiple
- * flag combinations (ROM || RW handler || write monitored) in order to
- * figure out how to setup a shadow PTE. But this of course, is second
- * priority at present. Current this requires 12 bits, but could probably
- * be optimized to ~8.
- *
- * Then there's the 24 bits used to track which shadow page tables are
- * currently mapping a page for the purpose of speeding up physical
- * access handlers, and thereby the page pool cache. More bit for this
- * purpose wouldn't hurt IIRC.
- *
- * Then there is a new bit in which we need to record what kind of page
- * this is, shared, zero, normal or write-monitored-normal. This'll
- * require 2 bits. One bit might be needed for indicating whether a
- * write monitored page has been written to. And yet another one or
- * two for tracking migration status. 3-4 bits total then.
- *
- * Whatever is left will can be used to record the sharabilitiy of a
- * page. The page checksum will not be stored in the per-VM table as
- * the idle thread will not be permitted to do modifications to it.
- * It will instead have to keep its own working set of potentially
- * shareable pages and their check sums and stuff.
- *
- * For the present we'll keep the current packing of the
- * PGMRAMRANGE::aHCPhys to keep the changes simple, only of course,
- * we'll have to change it to a struct with a total of 128-bits at
- * our disposal.
- *
- * The initial layout will be like this:
- * @verbatim
-    RTHCPHYS HCPhys;            The current stuff.
-        63:40                   Current shadow PT tracking stuff.
-        39:12                   The physical page frame number.
-        11:0                    The current flags.
-    uint32_t u28PageId : 28;    The page id.
-    uint32_t u2State : 2;       The page state { zero, shared, normal, write monitored }.
-    uint32_t fWrittenTo : 1;    Whether a write monitored page was written to.
-    uint32_t u1Reserved : 1;    Reserved for later.
-    uint32_t u32Reserved;       Reserved for later, mostly sharing stats.
- @endverbatim
- *
- * The final layout will be something like this:
- * @verbatim
-    RTHCPHYS HCPhys;            The current stuff.
-        63:48                   High page id (12+).
-        47:12                   The physical page frame number.
-        11:0                    Low page id.
-    uint32_t fReadOnly : 1;     Whether it's readonly page (rom or monitored in some way).
-    uint32_t u3Type : 3;        The page type {RESERVED, MMIO, MMIO2, ROM, shadowed ROM, RAM}.
-    uint32_t u2PhysMon : 2;     Physical access handler type {none, read, write, all}.
-    uint32_t u2VirtMon : 2;     Virtual access handler type {none, read, write, all}..
-    uint32_t u2State : 2;       The page state { zero, shared, normal, write monitored }.
-    uint32_t fWrittenTo : 1;    Whether a write monitored page was written to.
-    uint32_t u20Reserved : 20;  Reserved for later, mostly sharing stats.
-    uint32_t u32Tracking;       The shadow PT tracking stuff, roughly.
- @endverbatim
- *
- * Cost wise, this means we'll double the cost for guest memory. There isn't anyway
- * around that I'm afraid. It means that the cost of dealing out 32GB of memory
- * to one or more VMs is: (32GB >> PAGE_SHIFT) * 16 bytes, or 128MBs. Or another
- * example, the VM heap cost when assigning 1GB to a VM will be: 4MB.
- *
- * A couple of cost examples for the total cost per-VM + kernel.
- * 32-bit Windows and 32-bit linux:
- *      1GB guest ram, 256K pages:  4MB +  2MB(+) =   6MB
- *      4GB guest ram, 1M pages:   16MB +  8MB(+) =  24MB
- *     32GB guest ram, 8M pages:  128MB + 64MB(+) = 192MB
- * 64-bit Windows and 64-bit linux:
- *      1GB guest ram, 256K pages:  4MB +  3MB(+) =   7MB
- *      4GB guest ram, 1M pages:   16MB + 12MB(+) =  28MB
- *     32GB guest ram, 8M pages:  128MB + 96MB(+) = 224MB
- *
- * UPDATE - 2007-09-27:
- * Will need a ballooned flag/state too because we cannot
- * trust the guest 100% and reporting the same page as ballooned more
- * than once will put the GMM off balance.
- *
- *
- * @subsection subsec_pgmPhys_Serializing       Serializing Access
- *
- * Initially, we'll try a simple scheme:
- *
- *      - The per-VM RAM tracking structures (PGMRAMRANGE) is only modified
- *        by the EMT thread of that VM while in the pgm critsect.
- *      - Other threads in the VM process that needs to make reliable use of
- *        the per-VM RAM tracking structures will enter the critsect.
- *      - No process external thread or kernel thread will ever try enter
- *        the pgm critical section, as that just won't work.
- *      - The idle thread (and similar threads) doesn't not need 100% reliable
- *        data when performing it tasks as the EMT thread will be the one to
- *        do the actual changes later anyway. So, as long as it only accesses
- *        the main ram range, it can do so by somehow preventing the VM from
- *        being destroyed while it works on it...
- *
- *      - The over-commitment management, including the allocating/freeing
- *        chunks, is serialized by a ring-0 mutex lock (a fast one since the
- *        more mundane mutex implementation is broken on Linux).
- *      - A separate mutex is protecting the set of allocation chunks so
- *        that pages can be shared or/and freed up while some other VM is
- *        allocating more chunks. This mutex can be take from under the other
- *        one, but not the other way around.
- *
- *
- * @subsection subsec_pgmPhys_Request           VM Request interface
- *
- * When in ring-0 it will become necessary to send requests to a VM so it can
- * for instance move a page while defragmenting during VM destroy. The idle
- * thread will make use of this interface to request VMs to setup shared
- * pages and to perform write monitoring of pages.
- *
- * I would propose an interface similar to the current VMReq interface, similar
- * in that it doesn't require locking and that the one sending the request may
- * wait for completion if it wishes to. This shouldn't be very difficult to
- * realize.
- *
- * The requests themselves are also pretty simple. They are basically:
- *      -# Check that some precondition is still true.
- *      -# Do the update.
- *      -# Update all shadow page tables involved with the page.
- *
- * The 3rd step is identical to what we're already doing when updating a
- * physical handler, see pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs.
- *
- *
- *
- * @section sec_pgmPhys_MappingCaches   Mapping Caches
- *
- * In order to be able to map in and out memory and to be able to support
- * guest with more RAM than we've got virtual address space, we'll employing
- * a mapping cache.  Normally ring-0 and ring-3 can share the same cache,
- * however on 32-bit darwin the ring-0 code is running in a different memory
- * context and therefore needs a separate cache.  In raw-mode context we also
- * need a separate cache.  The 32-bit darwin mapping cache and the one for
- * raw-mode context share a lot of code, see PGMRZDYNMAP.
- *
- *
- * @subsection subsec_pgmPhys_MappingCaches_R3  Ring-3
- *
- * We've considered implementing the ring-3 mapping cache page based but found
- * that this was bother some when one had to take into account TLBs+SMP and
- * portability (missing the necessary APIs on several platforms). There were
- * also some performance concerns with this approach which hadn't quite been
- * worked out.
- *
- * Instead, we'll be mapping allocation chunks into the VM process. This simplifies
- * matters greatly quite a bit since we don't need to invent any new ring-0 stuff,
- * only some minor RTR0MEMOBJ mapping stuff. The main concern here is that mapping
- * compared to the previous idea is that mapping or unmapping a 1MB chunk is more
- * costly than a single page, although how much more costly is uncertain. We'll
- * try address this by using a very big cache, preferably bigger than the actual
- * VM RAM size if possible. The current VM RAM sizes should give some idea for
- * 32-bit boxes, while on 64-bit we can probably get away with employing an
- * unlimited cache.
- *
- * The cache have to parts, as already indicated, the ring-3 side and the
- * ring-0 side.
- *
- * The ring-0 will be tied to the page allocator since it will operate on the
- * memory objects it contains. It will therefore require the first ring-0 mutex
- * discussed in @ref subsec_pgmPhys_Serializing. We
- * some double house keeping wrt to who has mapped what I think, since both
- * VMMR0.r0 and RTR0MemObj will keep track of mapping relations
- *
- * The ring-3 part will be protected by the pgm critsect. For simplicity, we'll
- * require anyone that desires to do changes to the mapping cache to do that
- * from within this critsect. Alternatively, we could employ a separate critsect
- * for serializing changes to the mapping cache as this would reduce potential
- * contention with other threads accessing mappings unrelated to the changes
- * that are in process. We can see about this later, contention will show
- * up in the statistics anyway, so it'll be simple to tell.
- *
- * The organization of the ring-3 part will be very much like how the allocation
- * chunks are organized in ring-0, that is in an AVL tree by chunk id. To avoid
- * having to walk the tree all the time, we'll have a couple of lookaside entries
- * like in we do for I/O ports and MMIO in IOM.
- *
- * The simplified flow of a PGMPhysRead/Write function:
- *      -# Enter the PGM critsect.
- *      -# Lookup GCPhys in the ram ranges and get the Page ID.
- *      -# Calc the Allocation Chunk ID from the Page ID.
- *      -# Check the lookaside entries and then the AVL tree for the Chunk ID.
- *         If not found in cache:
- *              -# Call ring-0 and request it to be mapped and supply
- *                 a chunk to be unmapped if the cache is maxed out already.
- *              -# Insert the new mapping into the AVL tree (id + R3 address).
- *      -# Update the relevant lookaside entry and return the mapping address.
- *      -# Do the read/write according to monitoring flags and everything.
- *      -# Leave the critsect.
- *
- *
- * @section sec_pgmPhys_Fallback            Fallback
- *
- * Current all the "second tier" hosts will not support the RTR0MemObjAllocPhysNC
- * API and thus require a fallback.
- *
- * So, when RTR0MemObjAllocPhysNC returns VERR_NOT_SUPPORTED the page allocator
- * will return to the ring-3 caller (and later ring-0) and asking it to seed
- * the page allocator with some fresh pages (VERR_GMM_SEED_ME). Ring-3 will
- * then perform an SUPR3PageAlloc(cbChunk >> PAGE_SHIFT) call and make a
- * "SeededAllocPages" call to ring-0.
- *
- * The first time ring-0 sees the VERR_NOT_SUPPORTED failure it will disable
- * all page sharing (zero page detection will continue). It will also force
- * all allocations to come from the VM which seeded the page. Both these
- * measures are taken to make sure that there will never be any need for
- * mapping anything into ring-3 - everything will be mapped already.
- *
- * Whether we'll continue to use the current MM locked memory management
- * for this I don't quite know (I'd prefer not to and just ditch that all
- * together), we'll see what's simplest to do.
- *
- *
- *
- * @section sec_pgmPhys_Changes             Changes
- *
- * Breakdown of the changes involved?
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/cpum.h>
-#include <VBox/iom.h>
-#include <VBox/sup.h>
-#include <VBox/mm.h>
-#include <VBox/em.h>
-#include <VBox/stam.h>
-#include <VBox/rem.h>
-#include <VBox/selm.h>
-#include <VBox/ssm.h>
-#include <VBox/hwaccm.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include "PGMInline.h"
-
-#include <VBox/dbg.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-
-#include <iprt/asm.h>
-#include <iprt/asm-amd64-x86.h>
-#include <iprt/assert.h>
-#include <iprt/env.h>
-#include <iprt/mem.h>
-#include <iprt/file.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int                pgmR3InitPaging(PVM pVM);
-static int                pgmR3InitStats(PVM pVM);
-static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(int)  pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser);
-static DECLCALLBACK(int)  pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
-static DECLCALLBACK(int)  pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
-#ifdef VBOX_STRICT
-static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser);
-#endif
-static int                pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0);
-static void               pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst);
-static PGMMODE            pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher);
-
-#ifdef VBOX_WITH_DEBUGGER
-/** @todo Convert the first two commands to 'info' items. */
-static DECLCALLBACK(int)  pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-static DECLCALLBACK(int)  pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-static DECLCALLBACK(int)  pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-static DECLCALLBACK(int)  pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-# ifdef VBOX_STRICT
-static DECLCALLBACK(int)  pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-# endif
-static DECLCALLBACK(int)  pgmR3CmdPhysToFile(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-#endif
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-#ifdef VBOX_WITH_DEBUGGER
-/** Argument descriptors for '.pgmerror' and '.pgmerroroff'. */
-static const DBGCVARDESC g_aPgmErrorArgs[] =
-{
-    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
-    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "where",        "Error injection location." },
-};
-
-static const DBGCVARDESC g_aPgmPhysToFileArgs[] =
-{
-    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
-    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "file",         "The file name." },
-    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "nozero",       "If present, zero pages are skipped." },
-};
-
-# ifdef DEBUG_sandervl
-static const DBGCVARDESC g_aPgmCountPhysWritesArgs[] =
-{
-    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
-    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "enabled",      "on/off." },
-    {  1,           1,          DBGCVAR_CAT_NUMBER_NO_RANGE, 0,                         "interval",     "Interval in ms." },
-};
-# endif
-
-/** Command descriptors. */
-static const DBGCCMD    g_aCmds[] =
-{
-    /* pszCmd,  cArgsMin, cArgsMax, paArgDesc,                cArgDescs,    pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
-    { "pgmram",        0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdRam,        "",                     "Display the ram ranges." },
-    { "pgmsync",       0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdSync,       "",                     "Sync the CR3 page." },
-    { "pgmerror",      0, 1,        &g_aPgmErrorArgs[0],      1,            NULL,               0,          pgmR3CmdError,      "",                     "Enables inject runtime of errors into parts of PGM." },
-    { "pgmerroroff",   0, 1,        &g_aPgmErrorArgs[0],      1,            NULL,               0,          pgmR3CmdError,      "",                     "Disables inject runtime errors into parts of PGM." },
-# ifdef VBOX_STRICT
-    { "pgmassertcr3",  0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdAssertCR3,  "",                     "Check the shadow CR3 mapping." },
-#  if HC_ARCH_BITS == 64
-    { "pgmcheckduppages", 0, 0,     NULL,                     0,            NULL,               0,          pgmR3CmdCheckDuplicatePages,  "",           "Check for duplicate pages in all running VMs." },
-    { "pgmsharedmodules", 0, 0,     NULL,                     0,            NULL,               0,          pgmR3CmdShowSharedModules,  "",             "Print shared modules info." },
-#  endif
-# endif
-    { "pgmsyncalways", 0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdSyncAlways, "",                     "Toggle permanent CR3 syncing." },
-    { "pgmphystofile", 1, 2,        &g_aPgmPhysToFileArgs[0], 2,            NULL,               0,          pgmR3CmdPhysToFile, "",                     "Save the physical memory to file." },
-};
-#endif
-
-
-
-
-/*
- * Shadow - 32-bit mode
- */
-#define PGM_SHW_TYPE                PGM_TYPE_32BIT
-#define PGM_SHW_NAME(name)          PGM_SHW_NAME_32BIT(name)
-#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_32BIT_STR(name)
-#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_32BIT_STR(name)
-#include "PGMShw.h"
-
-/* Guest - real mode */
-#define PGM_GST_TYPE                PGM_TYPE_REAL
-#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_REAL(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_32BIT_REAL_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_32BIT_REAL_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
-#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
-#include "PGMBth.h"
-#include "PGMGstDefs.h"
-#include "PGMGst.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef BTH_PGMPOOLKIND_ROOT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - protected mode */
-#define PGM_GST_TYPE                PGM_TYPE_PROT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_PROT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_32BIT_PROT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_32BIT_PROT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
-#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
-#include "PGMBth.h"
-#include "PGMGstDefs.h"
-#include "PGMGst.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef BTH_PGMPOOLKIND_ROOT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - 32-bit mode */
-#define PGM_GST_TYPE                PGM_TYPE_32BIT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_32BIT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_32BIT_32BIT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_32BIT_32BIT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
-#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
-#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD
-#include "PGMBth.h"
-#include "PGMGstDefs.h"
-#include "PGMGst.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_BIG
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef BTH_PGMPOOLKIND_ROOT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-#undef PGM_SHW_TYPE
-#undef PGM_SHW_NAME
-#undef PGM_SHW_NAME_RC_STR
-#undef PGM_SHW_NAME_R0_STR
-
-
-/*
- * Shadow - PAE mode
- */
-#define PGM_SHW_TYPE                PGM_TYPE_PAE
-#define PGM_SHW_NAME(name)          PGM_SHW_NAME_PAE(name)
-#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_PAE_STR(name)
-#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_PAE_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
-#include "PGMShw.h"
-
-/* Guest - real mode */
-#define PGM_GST_TYPE                PGM_TYPE_REAL
-#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_REAL_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_REAL_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
-#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef BTH_PGMPOOLKIND_ROOT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - protected mode */
-#define PGM_GST_TYPE                PGM_TYPE_PROT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PROT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_PROT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_PROT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
-#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef BTH_PGMPOOLKIND_ROOT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - 32-bit mode */
-#define PGM_GST_TYPE                PGM_TYPE_32BIT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_32BIT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_32BIT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_32BIT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
-#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
-#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_FOR_32BIT
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_BIG
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef BTH_PGMPOOLKIND_ROOT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - PAE mode */
-#define PGM_GST_TYPE                PGM_TYPE_PAE
-#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PAE_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PAE_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PAE(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_PAE_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_PAE_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
-#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
-#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT
-#include "PGMBth.h"
-#include "PGMGstDefs.h"
-#include "PGMGst.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_BIG
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef BTH_PGMPOOLKIND_ROOT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-#undef PGM_SHW_TYPE
-#undef PGM_SHW_NAME
-#undef PGM_SHW_NAME_RC_STR
-#undef PGM_SHW_NAME_R0_STR
-
-
-/*
- * Shadow - AMD64 mode
- */
-#define PGM_SHW_TYPE                PGM_TYPE_AMD64
-#define PGM_SHW_NAME(name)          PGM_SHW_NAME_AMD64(name)
-#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_AMD64_STR(name)
-#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_AMD64_STR(name)
-#include "PGMShw.h"
-
-#ifdef VBOX_WITH_64_BITS_GUESTS
-/* Guest - AMD64 mode */
-# define PGM_GST_TYPE               PGM_TYPE_AMD64
-# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
-# define PGM_GST_NAME_RC_STR(name)  PGM_GST_NAME_RC_AMD64_STR(name)
-# define PGM_GST_NAME_R0_STR(name)  PGM_GST_NAME_R0_AMD64_STR(name)
-# define PGM_BTH_NAME(name)         PGM_BTH_NAME_AMD64_AMD64(name)
-# define PGM_BTH_NAME_RC_STR(name)  PGM_BTH_NAME_RC_AMD64_AMD64_STR(name)
-# define PGM_BTH_NAME_R0_STR(name)  PGM_BTH_NAME_R0_AMD64_AMD64_STR(name)
-# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PAE_PT
-# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
-# define BTH_PGMPOOLKIND_ROOT       PGMPOOLKIND_64BIT_PML4
-# include "PGMBth.h"
-# include "PGMGstDefs.h"
-# include "PGMGst.h"
-# undef BTH_PGMPOOLKIND_PT_FOR_BIG
-# undef BTH_PGMPOOLKIND_PT_FOR_PT
-# undef BTH_PGMPOOLKIND_ROOT
-# undef PGM_BTH_NAME
-# undef PGM_BTH_NAME_RC_STR
-# undef PGM_BTH_NAME_R0_STR
-# undef PGM_GST_TYPE
-# undef PGM_GST_NAME
-# undef PGM_GST_NAME_RC_STR
-# undef PGM_GST_NAME_R0_STR
-#endif /* VBOX_WITH_64_BITS_GUESTS */
-
-#undef PGM_SHW_TYPE
-#undef PGM_SHW_NAME
-#undef PGM_SHW_NAME_RC_STR
-#undef PGM_SHW_NAME_R0_STR
-
-
-/*
- * Shadow - Nested paging mode
- */
-#define PGM_SHW_TYPE                PGM_TYPE_NESTED
-#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED(name)
-#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_NESTED_STR(name)
-#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_NESTED_STR(name)
-#include "PGMShw.h"
-
-/* Guest - real mode */
-#define PGM_GST_TYPE                PGM_TYPE_REAL
-#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_REAL(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_REAL_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_REAL_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - protected mode */
-#define PGM_GST_TYPE                PGM_TYPE_PROT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PROT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_PROT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_PROT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - 32-bit mode */
-#define PGM_GST_TYPE                PGM_TYPE_32BIT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_32BIT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_32BIT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
-#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_BIG
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - PAE mode */
-#define PGM_GST_TYPE                PGM_TYPE_PAE
-#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PAE_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PAE_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_PAE_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_PAE_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
-#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_BIG
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-#ifdef VBOX_WITH_64_BITS_GUESTS
-/* Guest - AMD64 mode */
-# define PGM_GST_TYPE               PGM_TYPE_AMD64
-# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
-# define PGM_GST_NAME_RC_STR(name)  PGM_GST_NAME_RC_AMD64_STR(name)
-# define PGM_GST_NAME_R0_STR(name)  PGM_GST_NAME_R0_AMD64_STR(name)
-# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_AMD64(name)
-# define PGM_BTH_NAME_RC_STR(name)  PGM_BTH_NAME_RC_NESTED_AMD64_STR(name)
-# define PGM_BTH_NAME_R0_STR(name)  PGM_BTH_NAME_R0_NESTED_AMD64_STR(name)
-# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PAE_PT
-# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
-# include "PGMGstDefs.h"
-# include "PGMBth.h"
-# undef BTH_PGMPOOLKIND_PT_FOR_BIG
-# undef BTH_PGMPOOLKIND_PT_FOR_PT
-# undef PGM_BTH_NAME
-# undef PGM_BTH_NAME_RC_STR
-# undef PGM_BTH_NAME_R0_STR
-# undef PGM_GST_TYPE
-# undef PGM_GST_NAME
-# undef PGM_GST_NAME_RC_STR
-# undef PGM_GST_NAME_R0_STR
-#endif /* VBOX_WITH_64_BITS_GUESTS */
-
-#undef PGM_SHW_TYPE
-#undef PGM_SHW_NAME
-#undef PGM_SHW_NAME_RC_STR
-#undef PGM_SHW_NAME_R0_STR
-
-
-/*
- * Shadow - EPT
- */
-#define PGM_SHW_TYPE                PGM_TYPE_EPT
-#define PGM_SHW_NAME(name)          PGM_SHW_NAME_EPT(name)
-#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_EPT_STR(name)
-#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_EPT_STR(name)
-#include "PGMShw.h"
-
-/* Guest - real mode */
-#define PGM_GST_TYPE                PGM_TYPE_REAL
-#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_REAL(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_REAL_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_REAL_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - protected mode */
-#define PGM_GST_TYPE                PGM_TYPE_PROT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PROT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_PROT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_PROT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - 32-bit mode */
-#define PGM_GST_TYPE                PGM_TYPE_32BIT
-#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_32BIT(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_32BIT_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_32BIT_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
-#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_BIG
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-/* Guest - PAE mode */
-#define PGM_GST_TYPE                PGM_TYPE_PAE
-#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
-#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PAE_STR(name)
-#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PAE_STR(name)
-#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PAE(name)
-#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_PAE_STR(name)
-#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_PAE_STR(name)
-#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
-#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
-#include "PGMGstDefs.h"
-#include "PGMBth.h"
-#undef BTH_PGMPOOLKIND_PT_FOR_BIG
-#undef BTH_PGMPOOLKIND_PT_FOR_PT
-#undef PGM_BTH_NAME
-#undef PGM_BTH_NAME_RC_STR
-#undef PGM_BTH_NAME_R0_STR
-#undef PGM_GST_TYPE
-#undef PGM_GST_NAME
-#undef PGM_GST_NAME_RC_STR
-#undef PGM_GST_NAME_R0_STR
-
-#ifdef VBOX_WITH_64_BITS_GUESTS
-/* Guest - AMD64 mode */
-# define PGM_GST_TYPE               PGM_TYPE_AMD64
-# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
-# define PGM_GST_NAME_RC_STR(name)  PGM_GST_NAME_RC_AMD64_STR(name)
-# define PGM_GST_NAME_R0_STR(name)  PGM_GST_NAME_R0_AMD64_STR(name)
-# define PGM_BTH_NAME(name)         PGM_BTH_NAME_EPT_AMD64(name)
-# define PGM_BTH_NAME_RC_STR(name)  PGM_BTH_NAME_RC_EPT_AMD64_STR(name)
-# define PGM_BTH_NAME_R0_STR(name)  PGM_BTH_NAME_R0_EPT_AMD64_STR(name)
-# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PAE_PT
-# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
-# include "PGMGstDefs.h"
-# include "PGMBth.h"
-# undef BTH_PGMPOOLKIND_PT_FOR_BIG
-# undef BTH_PGMPOOLKIND_PT_FOR_PT
-# undef PGM_BTH_NAME
-# undef PGM_BTH_NAME_RC_STR
-# undef PGM_BTH_NAME_R0_STR
-# undef PGM_GST_TYPE
-# undef PGM_GST_NAME
-# undef PGM_GST_NAME_RC_STR
-# undef PGM_GST_NAME_R0_STR
-#endif /* VBOX_WITH_64_BITS_GUESTS */
-
-#undef PGM_SHW_TYPE
-#undef PGM_SHW_NAME
-#undef PGM_SHW_NAME_RC_STR
-#undef PGM_SHW_NAME_R0_STR
-
-
-
-/**
- * Initiates the paging of VM.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to VM structure.
- */
-VMMR3DECL(int) PGMR3Init(PVM pVM)
-{
-    LogFlow(("PGMR3Init:\n"));
-    PCFGMNODE pCfgPGM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM");
-    int rc;
-
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompile(sizeof(pVM->pgm.s) <= sizeof(pVM->pgm.padding));
-    AssertCompile(sizeof(pVM->aCpus[0].pgm.s) <= sizeof(pVM->aCpus[0].pgm.padding));
-    AssertCompileMemberAlignment(PGM, CritSect, sizeof(uintptr_t));
-
-    /*
-     * Init the structure.
-     */
-#ifdef PGM_WITHOUT_MAPPINGS
-    pVM->pgm.s.fMappingsDisabled = true;
-#endif
-    pVM->pgm.s.offVM       = RT_OFFSETOF(VM, pgm.s);
-    pVM->pgm.s.offVCpuPGM  = RT_OFFSETOF(VMCPU, pgm.s);
-
-    /* Init the per-CPU part. */
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[idCpu];
-        PPGMCPU pPGM = &pVCpu->pgm.s;
-
-        pPGM->offVM      = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)pVM;
-        pPGM->offVCpu    = RT_OFFSETOF(VMCPU, pgm.s);
-        pPGM->offPGM     = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)&pVM->pgm.s;
-
-        pPGM->enmShadowMode    = PGMMODE_INVALID;
-        pPGM->enmGuestMode     = PGMMODE_INVALID;
-
-        pPGM->GCPhysCR3        = NIL_RTGCPHYS;
-
-        pPGM->pGst32BitPdR3    = NULL;
-        pPGM->pGstPaePdptR3    = NULL;
-        pPGM->pGstAmd64Pml4R3  = NULL;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-        pPGM->pGst32BitPdR0    = NIL_RTR0PTR;
-        pPGM->pGstPaePdptR0    = NIL_RTR0PTR;
-        pPGM->pGstAmd64Pml4R0  = NIL_RTR0PTR;
-#endif
-        pPGM->pGst32BitPdRC    = NIL_RTRCPTR;
-        pPGM->pGstPaePdptRC    = NIL_RTRCPTR;
-        for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.apGstPaePDsR3); i++)
-        {
-            pPGM->apGstPaePDsR3[i]             = NULL;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-            pPGM->apGstPaePDsR0[i]             = NIL_RTR0PTR;
-#endif
-            pPGM->apGstPaePDsRC[i]             = NIL_RTRCPTR;
-            pPGM->aGCPhysGstPaePDs[i]          = NIL_RTGCPHYS;
-            pPGM->aGCPhysGstPaePDsMonitored[i] = NIL_RTGCPHYS;
-        }
-
-        pPGM->fA20Enabled      = true;
-    }
-
-    pVM->pgm.s.enmHostMode      = SUPPAGINGMODE_INVALID;
-    pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1; /* default; checked later */
-    pVM->pgm.s.GCPtrPrevRamRangeMapping = MM_HYPER_AREA_ADDRESS;
-
-    rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "RamPreAlloc", &pVM->pgm.s.fRamPreAlloc,
-#ifdef VBOX_WITH_PREALLOC_RAM_BY_DEFAULT
-                            true
-#else
-                            false
-#endif
-                           );
-    AssertLogRelRCReturn(rc, rc);
-
-#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
-    rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, _1G / GMM_CHUNK_SIZE);
-#else
-    rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, UINT32_MAX);
-#endif
-    AssertLogRelRCReturn(rc, rc);
-    for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
-        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
-
-    /*
-     * Get the configured RAM size - to estimate saved state size.
-     */
-    uint64_t    cbRam;
-    rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        cbRam = 0;
-    else if (RT_SUCCESS(rc))
-    {
-        if (cbRam < PAGE_SIZE)
-            cbRam = 0;
-        cbRam = RT_ALIGN_64(cbRam, PAGE_SIZE);
-    }
-    else
-    {
-        AssertMsgFailed(("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc));
-        return rc;
-    }
-
-#ifdef VBOX_WITH_STATISTICS
-    /*
-     * Allocate memory for the statistics before someone tries to use them.
-     */
-    size_t cbTotalStats = RT_ALIGN_Z(sizeof(PGMSTATS), 64) + RT_ALIGN_Z(sizeof(PGMCPUSTATS), 64) * pVM->cCpus;
-    void *pv;
-    rc = MMHyperAlloc(pVM, RT_ALIGN_Z(cbTotalStats, PAGE_SIZE), PAGE_SIZE, MM_TAG_PGM, &pv);
-    AssertRCReturn(rc, rc);
-
-    pVM->pgm.s.pStatsR3 = (PGMSTATS *)pv;
-    pVM->pgm.s.pStatsR0 = MMHyperCCToR0(pVM, pv);
-    pVM->pgm.s.pStatsRC = MMHyperCCToRC(pVM, pv);
-    pv = (uint8_t *)pv + RT_ALIGN_Z(sizeof(PGMSTATS), 64);
-
-    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
-    {
-        pVM->aCpus[iCpu].pgm.s.pStatsR3 = (PGMCPUSTATS *)pv;
-        pVM->aCpus[iCpu].pgm.s.pStatsR0 = MMHyperCCToR0(pVM, pv);
-        pVM->aCpus[iCpu].pgm.s.pStatsRC = MMHyperCCToRC(pVM, pv);
-
-        pv = (uint8_t *)pv + RT_ALIGN_Z(sizeof(PGMCPUSTATS), 64);
-    }
-#endif /* VBOX_WITH_STATISTICS */
-
-    /*
-     * Register callbacks, string formatters and the saved state data unit.
-     */
-#ifdef VBOX_STRICT
-    VMR3AtStateRegister(pVM, pgmR3ResetNoMorePhysWritesFlag, NULL);
-#endif
-    PGMRegisterStringFormatTypes();
-
-    rc = pgmR3InitSavedState(pVM, cbRam);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Initialize the PGM critical section and flush the phys TLBs
-     */
-    rc = PDMR3CritSectInit(pVM, &pVM->pgm.s.CritSect, RT_SRC_POS, "PGM");
-    AssertRCReturn(rc, rc);
-
-    PGMR3PhysChunkInvalidateTLB(pVM);
-    PGMPhysInvalidatePageMapTLB(pVM);
-
-    /*
-     * For the time being we sport a full set of handy pages in addition to the base
-     * memory to simplify things.
-     */
-    rc = MMR3ReserveHandyPages(pVM, RT_ELEMENTS(pVM->pgm.s.aHandyPages)); /** @todo this should be changed to PGM_HANDY_PAGES_MIN but this needs proper testing... */
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Trees
-     */
-    rc = MMHyperAlloc(pVM, sizeof(PGMTREES), 0, MM_TAG_PGM, (void **)&pVM->pgm.s.pTreesR3);
-    if (RT_SUCCESS(rc))
-    {
-        pVM->pgm.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pTreesR3);
-        pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
-
-        /*
-         * Allocate the zero page.
-         */
-        rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvZeroPgR3);
-    }
-    if (RT_SUCCESS(rc))
-    {
-        pVM->pgm.s.pvZeroPgRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pvZeroPgR3);
-        pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
-        pVM->pgm.s.HCPhysZeroPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvZeroPgR3);
-        AssertRelease(pVM->pgm.s.HCPhysZeroPg != NIL_RTHCPHYS);
-
-        /*
-         * Allocate the invalid MMIO page.
-         * (The invalid bits in HCPhysInvMmioPg are set later on init complete.)
-         */
-        rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvMmioPgR3);
-    }
-    if (RT_SUCCESS(rc))
-    {
-        ASMMemFill32(pVM->pgm.s.pvMmioPgR3, PAGE_SIZE, 0xfeedface);
-        pVM->pgm.s.HCPhysMmioPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvMmioPgR3);
-        AssertRelease(pVM->pgm.s.HCPhysMmioPg != NIL_RTHCPHYS);
-        pVM->pgm.s.HCPhysInvMmioPg = pVM->pgm.s.HCPhysMmioPg;
-
-        /*
-         * Init the paging.
-         */
-        rc = pgmR3InitPaging(pVM);
-    }
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Init the page pool.
-         */
-        rc = pgmR3PoolInit(pVM);
-    }
-    if (RT_SUCCESS(rc))
-    {
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            PVMCPU pVCpu = &pVM->aCpus[i];
-            rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
-            if (RT_FAILURE(rc))
-                break;
-        }
-    }
-
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Info & statistics
-         */
-        DBGFR3InfoRegisterInternal(pVM, "mode",
-                                   "Shows the current paging mode. "
-                                   "Recognizes 'all', 'guest', 'shadow' and 'host' as arguments, defaulting to 'all' if nothing is given.",
-                                   pgmR3InfoMode);
-        DBGFR3InfoRegisterInternal(pVM, "pgmcr3",
-                                   "Dumps all the entries in the top level paging table. No arguments.",
-                                   pgmR3InfoCr3);
-        DBGFR3InfoRegisterInternal(pVM, "phys",
-                                   "Dumps all the physical address ranges. No arguments.",
-                                   pgmR3PhysInfo);
-        DBGFR3InfoRegisterInternal(pVM, "handlers",
-                                   "Dumps physical, virtual and hyper virtual handlers. "
-                                   "Pass 'phys', 'virt', 'hyper' as argument if only one kind is wanted."
-                                   "Add 'nost' if the statistics are unwanted, use together with 'all' or explicit selection.",
-                                   pgmR3InfoHandlers);
-        DBGFR3InfoRegisterInternal(pVM, "mappings",
-                                   "Dumps guest mappings.",
-                                   pgmR3MapInfo);
-
-        pgmR3InitStats(pVM);
-
-#ifdef VBOX_WITH_DEBUGGER
-        /*
-         * Debugger commands.
-         */
-        static bool s_fRegisteredCmds = false;
-        if (!s_fRegisteredCmds)
-        {
-            int rc2 = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
-            if (RT_SUCCESS(rc2))
-                s_fRegisteredCmds = true;
-        }
-#endif
-        return VINF_SUCCESS;
-    }
-
-    /* Almost no cleanup necessary, MM frees all memory. */
-    PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
-
-    return rc;
-}
-
-
-/**
- * Init paging.
- *
- * Since we need to check what mode the host is operating in before we can choose
- * the right paging functions for the host we have to delay this until R0 has
- * been initialized.
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle.
- */
-static int pgmR3InitPaging(PVM pVM)
-{
-    /*
-     * Force a recalculation of modes and switcher so everyone gets notified.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
-        pVCpu->pgm.s.enmGuestMode  = PGMMODE_INVALID;
-    }
-
-    pVM->pgm.s.enmHostMode   = SUPPAGINGMODE_INVALID;
-
-    /*
-     * Allocate static mapping space for whatever the cr3 register
-     * points to and in the case of PAE mode to the 4 PDs.
-     */
-    int rc = MMR3HyperReserve(pVM, PAGE_SIZE * 5, "CR3 mapping", &pVM->pgm.s.GCPtrCR3Mapping);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsgFailed(("Failed to reserve two pages for cr mapping in HMA, rc=%Rrc\n", rc));
-        return rc;
-    }
-    MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-
-    /*
-     * Allocate pages for the three possible intermediate contexts
-     * (AMD64, PAE and plain 32-Bit). We maintain all three contexts
-     * for the sake of simplicity. The AMD64 uses the PAE for the
-     * lower levels, making the total number of pages 11 (3 + 7 + 1).
-     *
-     * We assume that two page tables will be enought for the core code
-     * mappings (HC virtual and identity).
-     */
-    pVM->pgm.s.pInterPD         = (PX86PD)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.pInterPD,         VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPTs[0]    = (PX86PT)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.apInterPTs[0],    VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPTs[1]    = (PX86PT)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.apInterPTs[1],    VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPaePTs[0] = (PX86PTPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePTs[0], VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPaePTs[1] = (PX86PTPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePTs[1], VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPaePDs[0] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[0], VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPaePDs[1] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[1], VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPaePDs[2] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[2], VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.apInterPaePDs[3] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[3], VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.pInterPaePDPT    = (PX86PDPT)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePDPT,    VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.pInterPaePDPT64  = (PX86PDPT)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePDPT64,  VERR_NO_PAGE_MEMORY);
-    pVM->pgm.s.pInterPaePML4    = (PX86PML4)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePML4,    VERR_NO_PAGE_MEMORY);
-
-    pVM->pgm.s.HCPhysInterPD = MMPage2Phys(pVM, pVM->pgm.s.pInterPD);
-    AssertRelease(pVM->pgm.s.HCPhysInterPD != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPD & PAGE_OFFSET_MASK));
-    pVM->pgm.s.HCPhysInterPaePDPT = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT);
-    AssertRelease(pVM->pgm.s.HCPhysInterPaePDPT != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePDPT & PAGE_OFFSET_MASK));
-    pVM->pgm.s.HCPhysInterPaePML4 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePML4);
-    AssertRelease(pVM->pgm.s.HCPhysInterPaePML4 != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePML4 & PAGE_OFFSET_MASK) && pVM->pgm.s.HCPhysInterPaePML4 < 0xffffffff);
-
-    /*
-     * Initialize the pages, setting up the PML4 and PDPT for repetitive 4GB action.
-     */
-    ASMMemZeroPage(pVM->pgm.s.pInterPD);
-    ASMMemZeroPage(pVM->pgm.s.apInterPTs[0]);
-    ASMMemZeroPage(pVM->pgm.s.apInterPTs[1]);
-
-    ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[0]);
-    ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[1]);
-
-    ASMMemZeroPage(pVM->pgm.s.pInterPaePDPT);
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.apInterPaePDs); i++)
-    {
-        ASMMemZeroPage(pVM->pgm.s.apInterPaePDs[i]);
-        pVM->pgm.s.pInterPaePDPT->a[i].u = X86_PDPE_P | PGM_PLXFLAGS_PERMANENT
-                                          | MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[i]);
-    }
-
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePDPT64->a); i++)
-    {
-        const unsigned iPD = i % RT_ELEMENTS(pVM->pgm.s.apInterPaePDs);
-        pVM->pgm.s.pInterPaePDPT64->a[i].u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A | PGM_PLXFLAGS_PERMANENT
-                                            | MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[iPD]);
-    }
-
-    RTHCPHYS HCPhysInterPaePDPT64 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64);
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePML4->a); i++)
-        pVM->pgm.s.pInterPaePML4->a[i].u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A | PGM_PLXFLAGS_PERMANENT
-                                         | HCPhysInterPaePDPT64;
-
-    /*
-     * Initialize paging workers and mode from current host mode
-     * and the guest running in real mode.
-     */
-    pVM->pgm.s.enmHostMode = SUPR3GetPagingMode();
-    switch (pVM->pgm.s.enmHostMode)
-    {
-        case SUPPAGINGMODE_32_BIT:
-        case SUPPAGINGMODE_32_BIT_GLOBAL:
-        case SUPPAGINGMODE_PAE:
-        case SUPPAGINGMODE_PAE_GLOBAL:
-        case SUPPAGINGMODE_PAE_NX:
-        case SUPPAGINGMODE_PAE_GLOBAL_NX:
-            break;
-
-        case SUPPAGINGMODE_AMD64:
-        case SUPPAGINGMODE_AMD64_GLOBAL:
-        case SUPPAGINGMODE_AMD64_NX:
-        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
-#ifndef VBOX_WITH_HYBRID_32BIT_KERNEL
-            if (ARCH_BITS != 64)
-            {
-                AssertMsgFailed(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
-                LogRel(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
-                return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
-            }
-#endif
-            break;
-        default:
-            AssertMsgFailed(("Host mode %d is not supported\n", pVM->pgm.s.enmHostMode));
-            return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
-    }
-    rc = pgmR3ModeDataInit(pVM, false /* don't resolve GC and R0 syms yet */);
-    if (RT_SUCCESS(rc))
-    {
-        LogFlow(("pgmR3InitPaging: returns successfully\n"));
-#if HC_ARCH_BITS == 64
-        LogRel(("Debug: HCPhysInterPD=%RHp HCPhysInterPaePDPT=%RHp HCPhysInterPaePML4=%RHp\n",
-                pVM->pgm.s.HCPhysInterPD, pVM->pgm.s.HCPhysInterPaePDPT, pVM->pgm.s.HCPhysInterPaePML4));
-        LogRel(("Debug: apInterPTs={%RHp,%RHp} apInterPaePTs={%RHp,%RHp} apInterPaePDs={%RHp,%RHp,%RHp,%RHp} pInterPaePDPT64=%RHp\n",
-                MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]),    MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]),
-                MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[1]),
-                MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[1]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[2]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[3]),
-                MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64)));
-#endif
-        return VINF_SUCCESS;
-    }
-
-    LogFlow(("pgmR3InitPaging: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Init statistics
- * @returns VBox status code.
- */
-static int pgmR3InitStats(PVM pVM)
-{
-    PPGM pPGM = &pVM->pgm.s;
-    int  rc;
-
-    /*
-     * Release statistics.
-     */
-    /* Common - misc variables */
-    STAM_REL_REG(pVM, &pPGM->cAllPages,                          STAMTYPE_U32,     "/PGM/Page/cAllPages",                STAMUNIT_COUNT,     "The total number of pages.");
-    STAM_REL_REG(pVM, &pPGM->cPrivatePages,                      STAMTYPE_U32,     "/PGM/Page/cPrivatePages",            STAMUNIT_COUNT,     "The number of private pages.");
-    STAM_REL_REG(pVM, &pPGM->cSharedPages,                       STAMTYPE_U32,     "/PGM/Page/cSharedPages",             STAMUNIT_COUNT,     "The number of shared pages.");
-    STAM_REL_REG(pVM, &pPGM->cReusedSharedPages,                 STAMTYPE_U32,     "/PGM/Page/cReusedSharedPages",       STAMUNIT_COUNT,     "The number of reused shared pages.");
-    STAM_REL_REG(pVM, &pPGM->cZeroPages,                         STAMTYPE_U32,     "/PGM/Page/cZeroPages",               STAMUNIT_COUNT,     "The number of zero backed pages.");
-    STAM_REL_REG(pVM, &pPGM->cPureMmioPages,                     STAMTYPE_U32,     "/PGM/Page/cPureMmioPages",           STAMUNIT_COUNT,     "The number of pure MMIO pages.");
-    STAM_REL_REG(pVM, &pPGM->cMonitoredPages,                    STAMTYPE_U32,     "/PGM/Page/cMonitoredPages",          STAMUNIT_COUNT,     "The number of write monitored pages.");
-    STAM_REL_REG(pVM, &pPGM->cWrittenToPages,                    STAMTYPE_U32,     "/PGM/Page/cWrittenToPages",          STAMUNIT_COUNT,     "The number of previously write monitored pages that have been written to.");
-    STAM_REL_REG(pVM, &pPGM->cWriteLockedPages,                  STAMTYPE_U32,     "/PGM/Page/cWriteLockedPages",        STAMUNIT_COUNT,     "The number of write(/read) locked pages.");
-    STAM_REL_REG(pVM, &pPGM->cReadLockedPages,                   STAMTYPE_U32,     "/PGM/Page/cReadLockedPages",         STAMUNIT_COUNT,     "The number of read (only) locked pages.");
-    STAM_REL_REG(pVM, &pPGM->cBalloonedPages,                    STAMTYPE_U32,     "/PGM/Page/cBalloonedPages",          STAMUNIT_COUNT,     "The number of ballooned pages.");
-    STAM_REL_REG(pVM, &pPGM->cHandyPages,                        STAMTYPE_U32,     "/PGM/Page/cHandyPages",              STAMUNIT_COUNT,     "The number of handy pages (not included in cAllPages).");
-    STAM_REL_REG(pVM, &pPGM->cRelocations,                       STAMTYPE_COUNTER, "/PGM/cRelocations",                  STAMUNIT_OCCURENCES,"Number of hypervisor relocations.");
-    STAM_REL_REG(pVM, &pPGM->ChunkR3Map.c,                       STAMTYPE_U32,     "/PGM/ChunkR3Map/c",                  STAMUNIT_COUNT,     "Number of mapped chunks.");
-    STAM_REL_REG(pVM, &pPGM->ChunkR3Map.cMax,                    STAMTYPE_U32,     "/PGM/ChunkR3Map/cMax",               STAMUNIT_COUNT,     "Maximum number of mapped chunks.");
-    STAM_REL_REG(pVM, &pPGM->cMappedChunks,                      STAMTYPE_U32,     "/PGM/ChunkR3Map/Mapped",             STAMUNIT_COUNT,     "Number of times we mapped a chunk.");
-    STAM_REL_REG(pVM, &pPGM->cUnmappedChunks,                    STAMTYPE_U32,     "/PGM/ChunkR3Map/Unmapped",           STAMUNIT_COUNT,     "Number of times we unmapped a chunk.");
-
-    STAM_REL_REG(pVM, &pPGM->StatLargePageAlloc,                 STAMTYPE_COUNTER, "/PGM/LargePage/Alloc",               STAMUNIT_OCCURENCES, "The number of large pages we've used.");
-    STAM_REL_REG(pVM, &pPGM->StatLargePageReused,                STAMTYPE_COUNTER, "/PGM/LargePage/Reused",              STAMUNIT_OCCURENCES, "The number of times we've reused a large page.");
-    STAM_REL_REG(pVM, &pPGM->StatLargePageRefused,               STAMTYPE_COUNTER, "/PGM/LargePage/Refused",             STAMUNIT_OCCURENCES, "The number of times we couldn't use a large page.");
-    STAM_REL_REG(pVM, &pPGM->StatLargePageRecheck,               STAMTYPE_COUNTER, "/PGM/LargePage/Recheck",             STAMUNIT_OCCURENCES, "The number of times we've rechecked a disabled large page.");
-
-    /* Live save */
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.fActive,              STAMTYPE_U8,      "/PGM/LiveSave/fActive",              STAMUNIT_COUNT,     "Active or not.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cIgnoredPages,        STAMTYPE_U32,     "/PGM/LiveSave/cIgnoredPages",        STAMUNIT_COUNT,     "The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM).");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesLong,      STAMTYPE_U32,     "/PGM/LiveSave/cDirtyPagesLong",      STAMUNIT_COUNT,     "Longer term dirty page average.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesShort,     STAMTYPE_U32,     "/PGM/LiveSave/cDirtyPagesShort",     STAMUNIT_COUNT,     "Short term dirty page average.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cPagesPerSecond,      STAMTYPE_U32,     "/PGM/LiveSave/cPagesPerSecond",      STAMUNIT_COUNT,     "Pages per second.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cSavedPages,          STAMTYPE_U64,     "/PGM/LiveSave/cSavedPages",          STAMUNIT_COUNT,     "The total number of saved pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cReadyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Ram/cReadPages",       STAMUNIT_COUNT,     "RAM: Ready pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cDirtyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Ram/cDirtyPages",      STAMUNIT_COUNT,     "RAM: Dirty pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cZeroPages,       STAMTYPE_U32,     "/PGM/LiveSave/Ram/cZeroPages",       STAMUNIT_COUNT,     "RAM: Ready zero pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cMonitoredPages,  STAMTYPE_U32,     "/PGM/LiveSave/Ram/cMonitoredPages",  STAMUNIT_COUNT,     "RAM: Write monitored pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cReadyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Rom/cReadPages",       STAMUNIT_COUNT,     "ROM: Ready pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cDirtyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Rom/cDirtyPages",      STAMUNIT_COUNT,     "ROM: Dirty pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cZeroPages,       STAMTYPE_U32,     "/PGM/LiveSave/Rom/cZeroPages",       STAMUNIT_COUNT,     "ROM: Ready zero pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cMonitoredPages,  STAMTYPE_U32,     "/PGM/LiveSave/Rom/cMonitoredPages",  STAMUNIT_COUNT,     "ROM: Write monitored pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cReadyPages,    STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cReadPages",     STAMUNIT_COUNT,     "MMIO2: Ready pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cDirtyPages,    STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cDirtyPages",    STAMUNIT_COUNT,     "MMIO2: Dirty pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cZeroPages,     STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cZeroPages",     STAMUNIT_COUNT,     "MMIO2: Ready zero pages.");
-    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cMonitoredPages,STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cMonitoredPages",STAMUNIT_COUNT,     "MMIO2: Write monitored pages.");
-
-#ifdef VBOX_WITH_STATISTICS
-
-# define PGM_REG_COUNTER(a, b, c) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b); \
-        AssertRC(rc);
-
-# define PGM_REG_COUNTER_BYTES(a, b, c) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, c, b); \
-        AssertRC(rc);
-
-# define PGM_REG_PROFILE(a, b, c) \
-        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b); \
-        AssertRC(rc);
-
-    PGMSTATS *pStats = pVM->pgm.s.pStatsR3;
-
-    PGM_REG_PROFILE(&pStats->StatAllocLargePage,                "/PGM/LargePage/Prof/Alloc",          "Time spent by the host OS for large page allocation.");
-    PGM_REG_PROFILE(&pStats->StatClearLargePage,                "/PGM/LargePage/Prof/Clear",          "Time spent clearing the newly allocated large pages.");
-    PGM_REG_COUNTER(&pStats->StatLargePageOverflow,             "/PGM/LargePage/Overflow",            "The number of times allocating a large page took too long.");
-    PGM_REG_PROFILE(&pStats->StatR3IsValidLargePage,            "/PGM/LargePage/Prof/R3/IsValid",     "pgmPhysIsValidLargePage profiling - R3.");
-    PGM_REG_PROFILE(&pStats->StatRZIsValidLargePage,            "/PGM/LargePage/Prof/RZ/IsValid",     "pgmPhysIsValidLargePage profiling - RZ.");
-
-    PGM_REG_COUNTER(&pStats->StatR3DetectedConflicts,           "/PGM/R3/DetectedConflicts",          "The number of times PGMR3CheckMappingConflicts() detected a conflict.");
-    PGM_REG_PROFILE(&pStats->StatR3ResolveConflict,             "/PGM/R3/ResolveConflict",            "pgmR3SyncPTResolveConflict() profiling (includes the entire relocation).");
-    PGM_REG_COUNTER(&pStats->StatR3PhysRead,                    "/PGM/R3/Phys/Read",                  "The number of times PGMPhysRead was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysReadBytes,         "/PGM/R3/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
-    PGM_REG_COUNTER(&pStats->StatR3PhysWrite,                   "/PGM/R3/Phys/Write",                 "The number of times PGMPhysWrite was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysWriteBytes,        "/PGM/R3/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
-    PGM_REG_COUNTER(&pStats->StatR3PhysSimpleRead,              "/PGM/R3/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysSimpleReadBytes,   "/PGM/R3/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
-    PGM_REG_COUNTER(&pStats->StatR3PhysSimpleWrite,             "/PGM/R3/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysSimpleWriteBytes,  "/PGM/R3/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
-
-    PGM_REG_COUNTER(&pStats->StatRZChunkR3MapTlbHits,           "/PGM/ChunkR3Map/TlbHitsRZ",          "TLB hits.");
-    PGM_REG_COUNTER(&pStats->StatRZChunkR3MapTlbMisses,         "/PGM/ChunkR3Map/TlbMissesRZ",        "TLB misses.");
-    PGM_REG_PROFILE(&pStats->StatChunkAging,                    "/PGM/ChunkR3Map/Map/Aging",          "Chunk aging profiling.");
-    PGM_REG_PROFILE(&pStats->StatChunkFindCandidate,            "/PGM/ChunkR3Map/Map/Find",           "Chunk unmap find profiling.");
-    PGM_REG_PROFILE(&pStats->StatChunkUnmap,                    "/PGM/ChunkR3Map/Map/Unmap",          "Chunk unmap of address space profiling.");
-    PGM_REG_PROFILE(&pStats->StatChunkMap,                      "/PGM/ChunkR3Map/Map/Map",            "Chunk map of address space profiling.");
-
-    PGM_REG_COUNTER(&pStats->StatRZPageMapTlbHits,              "/PGM/RZ/Page/MapTlbHits",            "TLB hits.");
-    PGM_REG_COUNTER(&pStats->StatRZPageMapTlbMisses,            "/PGM/RZ/Page/MapTlbMisses",          "TLB misses.");
-    PGM_REG_COUNTER(&pStats->StatR3ChunkR3MapTlbHits,           "/PGM/ChunkR3Map/TlbHitsR3",          "TLB hits.");
-    PGM_REG_COUNTER(&pStats->StatR3ChunkR3MapTlbMisses,         "/PGM/ChunkR3Map/TlbMissesR3",        "TLB misses.");
-    PGM_REG_COUNTER(&pStats->StatR3PageMapTlbHits,              "/PGM/R3/Page/MapTlbHits",            "TLB hits.");
-    PGM_REG_COUNTER(&pStats->StatR3PageMapTlbMisses,            "/PGM/R3/Page/MapTlbMisses",          "TLB misses.");
-    PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushes,             "/PGM/R3/Page/MapTlbFlushes",         "TLB flushes (all contexts).");
-    PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushEntry,          "/PGM/R3/Page/MapTlbFlushEntry",      "TLB entry flushes (all contexts).");
-
-    PGM_REG_PROFILE(&pStats->StatRZSyncCR3HandlerVirtualUpdate, "/PGM/RZ/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
-    PGM_REG_PROFILE(&pStats->StatRZSyncCR3HandlerVirtualReset,  "/PGM/RZ/SyncCR3/Handlers/VirtualReset",  "Profiling of the virtual handler resets.");
-    PGM_REG_PROFILE(&pStats->StatR3SyncCR3HandlerVirtualUpdate, "/PGM/R3/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
-    PGM_REG_PROFILE(&pStats->StatR3SyncCR3HandlerVirtualReset,  "/PGM/R3/SyncCR3/Handlers/VirtualReset",  "Profiling of the virtual handler resets.");
-
-    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerReset,            "/PGM/RZ/PhysHandlerReset",           "The number of times PGMHandlerPhysicalReset is called.");
-    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerReset,            "/PGM/R3/PhysHandlerReset",           "The number of times PGMHandlerPhysicalReset is called.");
-    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerLookupHits,       "/PGM/RZ/PhysHandlerLookupHits",      "The number of cache hits when looking up physical handlers.");
-    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerLookupHits,       "/PGM/R3/PhysHandlerLookupHits",      "The number of cache hits when looking up physical handlers.");
-    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerLookupMisses,     "/PGM/RZ/PhysHandlerLookupMisses",    "The number of cache misses when looking up physical handlers.");
-    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerLookupMisses,     "/PGM/R3/PhysHandlerLookupMisses",    "The number of cache misses when looking up physical handlers.");
-    PGM_REG_PROFILE(&pStats->StatRZVirtHandlerSearchByPhys,     "/PGM/RZ/VirtHandlerSearchByPhys",    "Profiling of pgmHandlerVirtualFindByPhysAddr.");
-    PGM_REG_PROFILE(&pStats->StatR3VirtHandlerSearchByPhys,     "/PGM/R3/VirtHandlerSearchByPhys",    "Profiling of pgmHandlerVirtualFindByPhysAddr.");
-
-    PGM_REG_COUNTER(&pStats->StatRZPageReplaceShared,           "/PGM/RZ/Page/ReplacedShared",        "Times a shared page was replaced.");
-    PGM_REG_COUNTER(&pStats->StatRZPageReplaceZero,             "/PGM/RZ/Page/ReplacedZero",          "Times the zero page was replaced.");
-/// @todo    PGM_REG_COUNTER(&pStats->StatRZPageHandyAllocs,             "/PGM/RZ/Page/HandyAllocs",               "Number of times we've allocated more handy pages.");
-    PGM_REG_COUNTER(&pStats->StatR3PageReplaceShared,           "/PGM/R3/Page/ReplacedShared",        "Times a shared page was replaced.");
-    PGM_REG_COUNTER(&pStats->StatR3PageReplaceZero,             "/PGM/R3/Page/ReplacedZero",          "Times the zero page was replaced.");
-/// @todo    PGM_REG_COUNTER(&pStats->StatR3PageHandyAllocs,             "/PGM/R3/Page/HandyAllocs",               "Number of times we've allocated more handy pages.");
-
-    PGM_REG_COUNTER(&pStats->StatRZPhysRead,                    "/PGM/RZ/Phys/Read",                  "The number of times PGMPhysRead was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysReadBytes,         "/PGM/RZ/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
-    PGM_REG_COUNTER(&pStats->StatRZPhysWrite,                   "/PGM/RZ/Phys/Write",                 "The number of times PGMPhysWrite was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysWriteBytes,        "/PGM/RZ/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
-    PGM_REG_COUNTER(&pStats->StatRZPhysSimpleRead,              "/PGM/RZ/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysSimpleReadBytes,   "/PGM/RZ/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
-    PGM_REG_COUNTER(&pStats->StatRZPhysSimpleWrite,             "/PGM/RZ/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysSimpleWriteBytes,  "/PGM/RZ/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
-
-    /* GC only: */
-    PGM_REG_COUNTER(&pStats->StatRCInvlPgConflict,              "/PGM/RC/InvlPgConflict",             "Number of times PGMInvalidatePage() detected a mapping conflict.");
-    PGM_REG_COUNTER(&pStats->StatRCInvlPgSyncMonCR3,            "/PGM/RC/InvlPgSyncMonitorCR3",       "Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3.");
-
-    PGM_REG_COUNTER(&pStats->StatRCPhysRead,                    "/PGM/RC/Phys/Read",                  "The number of times PGMPhysRead was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysReadBytes,         "/PGM/RC/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
-    PGM_REG_COUNTER(&pStats->StatRCPhysWrite,                   "/PGM/RC/Phys/Write",                 "The number of times PGMPhysWrite was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysWriteBytes,        "/PGM/RC/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
-    PGM_REG_COUNTER(&pStats->StatRCPhysSimpleRead,              "/PGM/RC/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysSimpleReadBytes,   "/PGM/RC/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
-    PGM_REG_COUNTER(&pStats->StatRCPhysSimpleWrite,             "/PGM/RC/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
-    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysSimpleWriteBytes,  "/PGM/RC/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
-
-    PGM_REG_COUNTER(&pStats->StatTrackVirgin,                   "/PGM/Track/Virgin",                  "The number of first time shadowings");
-    PGM_REG_COUNTER(&pStats->StatTrackAliased,                  "/PGM/Track/Aliased",                 "The number of times switching to cRef2, i.e. the page is being shadowed by two PTs.");
-    PGM_REG_COUNTER(&pStats->StatTrackAliasedMany,              "/PGM/Track/AliasedMany",             "The number of times we're tracking using cRef2.");
-    PGM_REG_COUNTER(&pStats->StatTrackAliasedLots,              "/PGM/Track/AliasedLots",             "The number of times we're hitting pages which has overflowed cRef2");
-    PGM_REG_COUNTER(&pStats->StatTrackOverflows,                "/PGM/Track/Overflows",               "The number of times the extent list grows too long.");
-    PGM_REG_COUNTER(&pStats->StatTrackNoExtentsLeft,            "/PGM/Track/NoExtentLeft",            "The number of times the extent list was exhausted.");
-    PGM_REG_PROFILE(&pStats->StatTrackDeref,                    "/PGM/Track/Deref",                   "Profiling of SyncPageWorkerTrackDeref (expensive).");
-
-# undef PGM_REG_COUNTER
-# undef PGM_REG_PROFILE
-#endif
-
-    /*
-     * Note! The layout below matches the member layout exactly!
-     */
-
-    /*
-     * Common - stats
-     */
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        PPGMCPU pPgmCpu = &pVM->aCpus[idCpu].pgm.s;
-
-#define PGM_REG_COUNTER(a, b, c) \
-    rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, idCpu); \
-    AssertRC(rc);
-#define PGM_REG_PROFILE(a, b, c) \
-    rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, idCpu); \
-    AssertRC(rc);
-
-        PGM_REG_COUNTER(&pPgmCpu->cGuestModeChanges, "/PGM/CPU%u/cGuestModeChanges",  "Number of guest mode changes.");
-
-#ifdef VBOX_WITH_STATISTICS
-        PGMCPUSTATS *pCpuStats = pVM->aCpus[idCpu].pgm.s.pStatsR3;
-
-# if 0 /* rarely useful; leave for debugging. */
-        for (unsigned j = 0; j < RT_ELEMENTS(pPgmCpu->StatSyncPtPD); j++)
-            STAMR3RegisterF(pVM, &pCpuStats->StatSyncPtPD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
-                            "The number of SyncPT per PD n.", "/PGM/CPU%u/PDSyncPT/%04X", i, j);
-        for (unsigned j = 0; j < RT_ELEMENTS(pCpuStats->StatSyncPagePD); j++)
-            STAMR3RegisterF(pVM, &pCpuStats->StatSyncPagePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
-                            "The number of SyncPage per PD n.", "/PGM/CPU%u/PDSyncPage/%04X", i, j);
-# endif
-        /* R0 only: */
-        PGM_REG_PROFILE(&pCpuStats->StatR0NpMiscfg,                      "/PGM/CPU%u/R0/NpMiscfg",                     "PGMR0Trap0eHandlerNPMisconfig() profiling.");
-        PGM_REG_COUNTER(&pCpuStats->StatR0NpMiscfgSyncPage,              "/PGM/CPU%u/R0/NpMiscfgSyncPage",             "SyncPage calls from PGMR0Trap0eHandlerNPMisconfig().");
-
-        /* RZ only: */
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0e,                      "/PGM/CPU%u/RZ/Trap0e",                     "Profiling of the PGMTrap0eHandler() body.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Ballooned,        "/PGM/CPU%u/RZ/Trap0e/Time2/Ballooned",         "Profiling of the Trap0eHandler body when the cause is read access to a ballooned page.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2CSAM,             "/PGM/CPU%u/RZ/Trap0e/Time2/CSAM",              "Profiling of the Trap0eHandler body when the cause is CSAM.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2DirtyAndAccessed, "/PGM/CPU%u/RZ/Trap0e/Time2/DirtyAndAccessedBits", "Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2GuestTrap,        "/PGM/CPU%u/RZ/Trap0e/Time2/GuestTrap",         "Profiling of the Trap0eHandler body when the cause is a guest trap.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndPhys,          "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerPhysical",   "Profiling of the Trap0eHandler body when the cause is a physical handler.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndVirt,          "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerVirtual",    "Profiling of the Trap0eHandler body when the cause is a virtual handler.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndUnhandled,     "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerUnhandled",  "Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2InvalidPhys,      "/PGM/CPU%u/RZ/Trap0e/Time2/InvalidPhys",       "Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2MakeWritable,     "/PGM/CPU%u/RZ/Trap0e/Time2/MakeWritable",      "Profiling of the Trap0eHandler body when the cause is that a page needed to be made writeable.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Mapping,          "/PGM/CPU%u/RZ/Trap0e/Time2/Mapping",           "Profiling of the Trap0eHandler body when the cause is related to the guest mappings.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Misc,             "/PGM/CPU%u/RZ/Trap0e/Time2/Misc",              "Profiling of the Trap0eHandler body when the cause is not known.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSync,        "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSync",         "Profiling of the Trap0eHandler body when the cause is an out-of-sync page.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndPhys, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncHndPhys",  "Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndVirt, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncHndVirt",  "Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndObs,  "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncObsHnd",   "Profiling of the Trap0eHandler body when the cause is an obsolete handler page.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2SyncPT,           "/PGM/CPU%u/RZ/Trap0e/Time2/SyncPT",            "Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2WPEmulation,      "/PGM/CPU%u/RZ/Trap0e/Time2/WPEmulation",       "Profiling of the Trap0eHandler body when the cause is CR0.WP emulation.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eConflicts,             "/PGM/CPU%u/RZ/Trap0e/Conflicts",               "The number of times #PF was caused by an undetected conflict.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersMapping,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Mapping",        "Number of traps due to access handlers in mappings.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersOutOfSync,     "/PGM/CPU%u/RZ/Trap0e/Handlers/OutOfSync",      "Number of traps due to out-of-sync handled pages.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysAll,       "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysAll",        "Number of traps due to physical all-access handlers.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysAllOpt,    "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysAllOpt",     "Number of the physical all-access handler traps using the optimization.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysWrite,     "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysWrite",      "Number of traps due to physical write-access handlers.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersVirtual,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Virtual",        "Number of traps due to virtual access handlers.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersVirtualByPhys, "/PGM/CPU%u/RZ/Trap0e/Handlers/VirtualByPhys",  "Number of traps due to virtual access handlers by physical address.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersVirtualUnmarked,"/PGM/CPU%u/RZ/Trap0e/Handlers/VirtualUnmarked","Number of traps due to virtual access handlers by virtual address (without proper physical flags).");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersUnhandled,     "/PGM/CPU%u/RZ/Trap0e/Handlers/Unhandled",      "Number of traps due to access outside range of monitored page(s).");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersInvalid,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Invalid",        "Number of traps due to access to invalid physical memory.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNotPresentRead,      "/PGM/CPU%u/RZ/Trap0e/Err/User/NPRead",         "Number of user mode not present read page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNotPresentWrite,     "/PGM/CPU%u/RZ/Trap0e/Err/User/NPWrite",        "Number of user mode not present write page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSWrite,               "/PGM/CPU%u/RZ/Trap0e/Err/User/Write",          "Number of user mode write page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSReserved,            "/PGM/CPU%u/RZ/Trap0e/Err/User/Reserved",       "Number of user mode reserved bit page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNXE,                 "/PGM/CPU%u/RZ/Trap0e/Err/User/NXE",            "Number of user mode NXE page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSRead,                "/PGM/CPU%u/RZ/Trap0e/Err/User/Read",           "Number of user mode read page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVNotPresentRead,      "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPRead",   "Number of supervisor mode not present read page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVNotPresentWrite,     "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPWrite",  "Number of supervisor mode not present write page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVWrite,               "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Write",    "Number of supervisor mode write page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVReserved,            "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Reserved", "Number of supervisor mode reserved bit page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSNXE,                  "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NXE",      "Number of supervisor mode NXE page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eGuestPF,               "/PGM/CPU%u/RZ/Trap0e/GuestPF",                 "Number of real guest page faults.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eGuestPFMapping,        "/PGM/CPU%u/RZ/Trap0e/GuestPF/InMapping",       "Number of real guest page faults in a mapping.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eWPEmulInRZ,            "/PGM/CPU%u/RZ/Trap0e/WP/InRZ",                 "Number of guest page faults due to X86_CR0_WP emulation.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eWPEmulToR3,            "/PGM/CPU%u/RZ/Trap0e/WP/ToR3",                 "Number of guest page faults due to X86_CR0_WP emulation (forward to R3 for emulation).");
-#if 0 /* rarely useful; leave for debugging. */
-        for (unsigned j = 0; j < RT_ELEMENTS(pCpuStats->StatRZTrap0ePD); j++)
-            STAMR3RegisterF(pVM, &pCpuStats->StatRZTrap0ePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
-                            "The number of traps in page directory n.", "/PGM/CPU%u/RZ/Trap0e/PD/%04X", i, j);
-#endif
-        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteHandled,        "/PGM/CPU%u/RZ/CR3WriteHandled",                "The number of times the Guest CR3 change was successfully handled.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteUnhandled,      "/PGM/CPU%u/RZ/CR3WriteUnhandled",              "The number of times the Guest CR3 change was passed back to the recompiler.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteConflict,       "/PGM/CPU%u/RZ/CR3WriteConflict",               "The number of times the Guest CR3 monitoring detected a conflict.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZGuestROMWriteHandled,        "/PGM/CPU%u/RZ/ROMWriteHandled",                "The number of times the Guest ROM change was successfully handled.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZGuestROMWriteUnhandled,      "/PGM/CPU%u/RZ/ROMWriteUnhandled",              "The number of times the Guest ROM change was passed back to the recompiler.");
-
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapMigrateInvlPg,         "/PGM/CPU%u/RZ/DynMap/MigrateInvlPg",            "invlpg count in PGMR0DynMapMigrateAutoSet.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapGCPageInl,             "/PGM/CPU%u/RZ/DynMap/PageGCPageInl",            "Calls to pgmR0DynMapGCPageInlined.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlHits,         "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/Hits",       "Hash table lookup hits.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlMisses,       "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/Misses",     "Misses that falls back to the code common.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlRamHits,      "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/RamHits",    "1st ram range hits.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlRamMisses,    "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/RamMisses",  "1st ram range misses, takes slow path.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapHCPageInl,             "/PGM/CPU%u/RZ/DynMap/PageHCPageInl",            "Calls to pgmRZDynMapHCPageInlined.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapHCPageInlHits,         "/PGM/CPU%u/RZ/DynMap/PageHCPageInl/Hits",       "Hash table lookup hits.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapHCPageInlMisses,       "/PGM/CPU%u/RZ/DynMap/PageHCPageInl/Misses",     "Misses that falls back to the code common.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPage,                  "/PGM/CPU%u/RZ/DynMap/Page",                     "Calls to pgmR0DynMapPage");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetOptimize,           "/PGM/CPU%u/RZ/DynMap/Page/SetOptimize",         "Calls to pgmRZDynMapOptimizeAutoSet.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchFlushes,      "/PGM/CPU%u/RZ/DynMap/Page/SetSearchFlushes",    "Set search restoring to subset flushes.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchHits,         "/PGM/CPU%u/RZ/DynMap/Page/SetSearchHits",       "Set search hits.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchMisses,       "/PGM/CPU%u/RZ/DynMap/Page/SetSearchMisses",     "Set search misses.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapHCPage,                "/PGM/CPU%u/RZ/DynMap/Page/HCPage",              "Calls to pgmRZDynMapHCPageCommon (ring-0).");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits0,             "/PGM/CPU%u/RZ/DynMap/Page/Hits0",               "Hits at iPage+0");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits1,             "/PGM/CPU%u/RZ/DynMap/Page/Hits1",               "Hits at iPage+1");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits2,             "/PGM/CPU%u/RZ/DynMap/Page/Hits2",               "Hits at iPage+2");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageInvlPg,            "/PGM/CPU%u/RZ/DynMap/Page/InvlPg",              "invlpg count in pgmR0DynMapPageSlow.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlow,              "/PGM/CPU%u/RZ/DynMap/Page/Slow",                "Calls to pgmR0DynMapPageSlow - subtract this from pgmR0DynMapPage to get 1st level hits.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLoopHits,      "/PGM/CPU%u/RZ/DynMap/Page/SlowLoopHits" ,       "Hits in the loop path.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLoopMisses,    "/PGM/CPU%u/RZ/DynMap/Page/SlowLoopMisses",      "Misses in the loop path. NonLoopMisses = Slow - SlowLoopHit - SlowLoopMisses");
-        //PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLostHits,      "/PGM/CPU%u/R0/DynMap/Page/SlowLostHits",        "Lost hits.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSubsets,               "/PGM/CPU%u/RZ/DynMap/Subsets",                  "Times PGMRZDynMapPushAutoSubset was called.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPopFlushes,            "/PGM/CPU%u/RZ/DynMap/SubsetPopFlushes",         "Times PGMRZDynMapPopAutoSubset flushes the subset.");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[0],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct000..09",      "00-09% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[1],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct010..19",      "10-19% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[2],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct020..29",      "20-29% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[3],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct030..39",      "30-39% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[4],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct040..49",      "40-49% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[5],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct050..59",      "50-59% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[6],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct060..69",      "60-69% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[7],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct070..79",      "70-79% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[8],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct080..89",      "80-89% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[9],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct090..99",      "90-99% filled (RC: min(set-size, dynmap-size))");
-        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[10],     "/PGM/CPU%u/RZ/DynMap/SetFilledPct100",          "100% filled (RC: min(set-size, dynmap-size))");
-
-        /* HC only: */
-
-        /* RZ & R3: */
-        PGM_REG_PROFILE(&pCpuStats->StatRZSyncCR3,                     "/PGM/CPU%u/RZ/SyncCR3",                        "Profiling of the PGMSyncCR3() body.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZSyncCR3Handlers,             "/PGM/CPU%u/RZ/SyncCR3/Handlers",               "Profiling of the PGMSyncCR3() update handler section.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3Global,               "/PGM/CPU%u/RZ/SyncCR3/Global",                 "The number of global CR3 syncs.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3NotGlobal,            "/PGM/CPU%u/RZ/SyncCR3/NotGlobal",              "The number of non-global CR3 syncs.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstCacheHit,          "/PGM/CPU%u/RZ/SyncCR3/DstChacheHit",           "The number of times we got some kind of a cache hit.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstFreed,             "/PGM/CPU%u/RZ/SyncCR3/DstFreed",               "The number of times we've had to free a shadow entry.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstFreedSrcNP,        "/PGM/CPU%u/RZ/SyncCR3/DstFreedSrcNP",          "The number of times we've had to free a shadow entry for which the source entry was not present.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstNotPresent,        "/PGM/CPU%u/RZ/SyncCR3/DstNotPresent",          "The number of times we've encountered a not present shadow entry for a present guest entry.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstSkippedGlobalPD,   "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPD",     "The number of times a global page directory wasn't flushed.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstSkippedGlobalPT,   "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPT",     "The number of times a page table with only global entries wasn't flushed.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZSyncPT,                      "/PGM/CPU%u/RZ/SyncPT",                         "Profiling of the pfnSyncPT() body.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPTFailed,                "/PGM/CPU%u/RZ/SyncPT/Failed",                  "The number of times pfnSyncPT() failed.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPT4K,                    "/PGM/CPU%u/RZ/SyncPT/4K",                      "Nr of 4K PT syncs");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPT4M,                    "/PGM/CPU%u/RZ/SyncPT/4M",                      "Nr of 4M PT syncs");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPagePDNAs,               "/PGM/CPU%u/RZ/SyncPagePDNAs",                  "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPagePDOutOfSync,         "/PGM/CPU%u/RZ/SyncPagePDOutOfSync",            "The number of time we've encountered an out-of-sync PD in SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZAccessedPage,                "/PGM/CPU%u/RZ/AccessedPage",               "The number of pages marked not present for accessed bit emulation.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZDirtyBitTracking,            "/PGM/CPU%u/RZ/DirtyPage",                  "Profiling the dirty bit tracking in CheckPageFault().");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPage,                   "/PGM/CPU%u/RZ/DirtyPage/Mark",             "The number of pages marked read-only for dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageBig,                "/PGM/CPU%u/RZ/DirtyPage/MarkBig",          "The number of 4MB pages marked read-only for dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageSkipped,            "/PGM/CPU%u/RZ/DirtyPage/Skipped",          "The number of pages already dirty or readonly.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageTrap,               "/PGM/CPU%u/RZ/DirtyPage/Trap",             "The number of traps generated for dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageStale,              "/PGM/CPU%u/RZ/DirtyPage/Stale",            "The number of traps generated for dirty bit tracking (stale tlb entries).");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDirtiedPage,                 "/PGM/CPU%u/RZ/DirtyPage/SetDirty",         "The number of pages marked dirty because of write accesses.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyTrackRealPF,            "/PGM/CPU%u/RZ/DirtyPage/RealPF",           "The number of real pages faults during dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZPageAlreadyDirty,            "/PGM/CPU%u/RZ/DirtyPage/AlreadySet",       "The number of pages already marked dirty because of write accesses.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZInvalidatePage,              "/PGM/CPU%u/RZ/InvalidatePage",             "PGMInvalidatePage() profiling.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4KBPages,      "/PGM/CPU%u/RZ/InvalidatePage/4KBPages",    "The number of times PGMInvalidatePage() was called for a 4KB page.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4MBPages,      "/PGM/CPU%u/RZ/InvalidatePage/4MBPages",    "The number of times PGMInvalidatePage() was called for a 4MB page.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4MBPagesSkip,  "/PGM/CPU%u/RZ/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDMappings,    "/PGM/CPU%u/RZ/InvalidatePage/PDMappings",  "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDNAs,         "/PGM/CPU%u/RZ/InvalidatePage/PDNAs",       "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDNPs,         "/PGM/CPU%u/RZ/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDOutOfSync,   "/PGM/CPU%u/RZ/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePageSkipped,       "/PGM/CPU%u/RZ/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncSupervisor,     "/PGM/CPU%u/RZ/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncUser,           "/PGM/CPU%u/RZ/OutOfSync/User",             "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncSupervisorWrite,"/PGM/CPU%u/RZ/OutOfSync/SuperVisorWrite",  "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncUserWrite,      "/PGM/CPU%u/RZ/OutOfSync/UserWrite",        "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncBallloon,       "/PGM/CPU%u/RZ/OutOfSync/Balloon",          "The number of times a ballooned page was accessed (read).");
-        PGM_REG_PROFILE(&pCpuStats->StatRZPrefetch,                    "/PGM/CPU%u/RZ/Prefetch",                   "PGMPrefetchPage profiling.");
-        PGM_REG_PROFILE(&pCpuStats->StatRZFlushTLB,                    "/PGM/CPU%u/RZ/FlushTLB",                   "Profiling of the PGMFlushTLB() body.");
-        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBNewCR3,              "/PGM/CPU%u/RZ/FlushTLB/NewCR3",            "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
-        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBNewCR3Global,        "/PGM/CPU%u/RZ/FlushTLB/NewCR3Global",      "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
-        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBSameCR3,             "/PGM/CPU%u/RZ/FlushTLB/SameCR3",           "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
-        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBSameCR3Global,       "/PGM/CPU%u/RZ/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
-        PGM_REG_PROFILE(&pCpuStats->StatRZGstModifyPage,               "/PGM/CPU%u/RZ/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
-
-        PGM_REG_PROFILE(&pCpuStats->StatR3SyncCR3,                     "/PGM/CPU%u/R3/SyncCR3",                        "Profiling of the PGMSyncCR3() body.");
-        PGM_REG_PROFILE(&pCpuStats->StatR3SyncCR3Handlers,             "/PGM/CPU%u/R3/SyncCR3/Handlers",               "Profiling of the PGMSyncCR3() update handler section.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3Global,               "/PGM/CPU%u/R3/SyncCR3/Global",                 "The number of global CR3 syncs.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3NotGlobal,            "/PGM/CPU%u/R3/SyncCR3/NotGlobal",              "The number of non-global CR3 syncs.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstCacheHit,          "/PGM/CPU%u/R3/SyncCR3/DstChacheHit",           "The number of times we got some kind of a cache hit.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstFreed,             "/PGM/CPU%u/R3/SyncCR3/DstFreed",               "The number of times we've had to free a shadow entry.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstFreedSrcNP,        "/PGM/CPU%u/R3/SyncCR3/DstFreedSrcNP",          "The number of times we've had to free a shadow entry for which the source entry was not present.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstNotPresent,        "/PGM/CPU%u/R3/SyncCR3/DstNotPresent",          "The number of times we've encountered a not present shadow entry for a present guest entry.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstSkippedGlobalPD,   "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPD",     "The number of times a global page directory wasn't flushed.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstSkippedGlobalPT,   "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPT",     "The number of times a page table with only global entries wasn't flushed.");
-        PGM_REG_PROFILE(&pCpuStats->StatR3SyncPT,                      "/PGM/CPU%u/R3/SyncPT",                         "Profiling of the pfnSyncPT() body.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPTFailed,                "/PGM/CPU%u/R3/SyncPT/Failed",                  "The number of times pfnSyncPT() failed.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPT4K,                    "/PGM/CPU%u/R3/SyncPT/4K",                      "Nr of 4K PT syncs");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPT4M,                    "/PGM/CPU%u/R3/SyncPT/4M",                      "Nr of 4M PT syncs");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPagePDNAs,               "/PGM/CPU%u/R3/SyncPagePDNAs",                  "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPagePDOutOfSync,         "/PGM/CPU%u/R3/SyncPagePDOutOfSync",            "The number of time we've encountered an out-of-sync PD in SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3AccessedPage,                "/PGM/CPU%u/R3/AccessedPage",               "The number of pages marked not present for accessed bit emulation.");
-        PGM_REG_PROFILE(&pCpuStats->StatR3DirtyBitTracking,            "/PGM/CPU%u/R3/DirtyPage",                  "Profiling the dirty bit tracking in CheckPageFault().");
-        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPage,                   "/PGM/CPU%u/R3/DirtyPage/Mark",             "The number of pages marked read-only for dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageBig,                "/PGM/CPU%u/R3/DirtyPage/MarkBig",          "The number of 4MB pages marked read-only for dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageSkipped,            "/PGM/CPU%u/R3/DirtyPage/Skipped",          "The number of pages already dirty or readonly.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageTrap,               "/PGM/CPU%u/R3/DirtyPage/Trap",             "The number of traps generated for dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3DirtiedPage,                 "/PGM/CPU%u/R3/DirtyPage/SetDirty",         "The number of pages marked dirty because of write accesses.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyTrackRealPF,            "/PGM/CPU%u/R3/DirtyPage/RealPF",           "The number of real pages faults during dirty bit tracking.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3PageAlreadyDirty,            "/PGM/CPU%u/R3/DirtyPage/AlreadySet",       "The number of pages already marked dirty because of write accesses.");
-        PGM_REG_PROFILE(&pCpuStats->StatR3InvalidatePage,              "/PGM/CPU%u/R3/InvalidatePage",             "PGMInvalidatePage() profiling.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4KBPages,      "/PGM/CPU%u/R3/InvalidatePage/4KBPages",    "The number of times PGMInvalidatePage() was called for a 4KB page.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4MBPages,      "/PGM/CPU%u/R3/InvalidatePage/4MBPages",    "The number of times PGMInvalidatePage() was called for a 4MB page.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4MBPagesSkip,  "/PGM/CPU%u/R3/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDMappings,    "/PGM/CPU%u/R3/InvalidatePage/PDMappings",  "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDNAs,         "/PGM/CPU%u/R3/InvalidatePage/PDNAs",       "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDNPs,         "/PGM/CPU%u/R3/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDOutOfSync,   "/PGM/CPU%u/R3/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePageSkipped,       "/PGM/CPU%u/R3/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncSupervisor,     "/PGM/CPU%u/R3/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncUser,           "/PGM/CPU%u/R3/OutOfSync/User",             "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncBallloon,       "/PGM/CPU%u/R3/OutOfSync/Balloon",          "The number of times a ballooned page was accessed (read).");
-        PGM_REG_PROFILE(&pCpuStats->StatR3Prefetch,                    "/PGM/CPU%u/R3/Prefetch",                   "PGMPrefetchPage profiling.");
-        PGM_REG_PROFILE(&pCpuStats->StatR3FlushTLB,                    "/PGM/CPU%u/R3/FlushTLB",                   "Profiling of the PGMFlushTLB() body.");
-        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBNewCR3,              "/PGM/CPU%u/R3/FlushTLB/NewCR3",            "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
-        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBNewCR3Global,        "/PGM/CPU%u/R3/FlushTLB/NewCR3Global",      "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
-        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBSameCR3,             "/PGM/CPU%u/R3/FlushTLB/SameCR3",           "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
-        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBSameCR3Global,       "/PGM/CPU%u/R3/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
-        PGM_REG_PROFILE(&pCpuStats->StatR3GstModifyPage,               "/PGM/CPU%u/R3/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
-#endif /* VBOX_WITH_STATISTICS */
-
-#undef PGM_REG_PROFILE
-#undef PGM_REG_COUNTER
-
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Init the PGM bits that rely on VMMR0 and MM to be fully initialized.
- *
- * The dynamic mapping area will also be allocated and initialized at this
- * time. We could allocate it during PGMR3Init of course, but the mapping
- * wouldn't be allocated at that time preventing us from setting up the
- * page table entries with the dummy page.
- *
- * @returns VBox status code.
- * @param   pVM     VM handle.
- */
-VMMR3DECL(int) PGMR3InitDynMap(PVM pVM)
-{
-    RTGCPTR GCPtr;
-    int     rc;
-
-    /*
-     * Reserve space for the dynamic mappings.
-     */
-    rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping", &GCPtr);
-    if (RT_SUCCESS(rc))
-        pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
-
-    if (    RT_SUCCESS(rc)
-        &&  (pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) != ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT))
-    {
-        rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping not crossing", &GCPtr);
-        if (RT_SUCCESS(rc))
-            pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
-    }
-    if (RT_SUCCESS(rc))
-    {
-        AssertRelease((pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) == ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT));
-        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-    }
-    return rc;
-}
-
-
-/**
- * Ring-3 init finalizing.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(int) PGMR3InitFinalize(PVM pVM)
-{
-    int rc;
-
-    /*
-     * Reserve space for the dynamic mappings.
-     * Initialize the dynamic mapping pages with dummy pages to simply the cache.
-     */
-    /* get the pointer to the page table entries. */
-    PPGMMAPPING pMapping = pgmGetMapping(pVM, pVM->pgm.s.pbDynPageMapBaseGC);
-    AssertRelease(pMapping);
-    const uintptr_t off = pVM->pgm.s.pbDynPageMapBaseGC - pMapping->GCPtr;
-    const unsigned iPT =  off >> X86_PD_SHIFT;
-    const unsigned iPG = (off >> X86_PT_SHIFT) & X86_PT_MASK;
-    pVM->pgm.s.paDynPageMap32BitPTEsGC = pMapping->aPTs[iPT].pPTRC      + iPG * sizeof(pMapping->aPTs[0].pPTR3->a[0]);
-    pVM->pgm.s.paDynPageMapPaePTEsGC   = pMapping->aPTs[iPT].paPaePTsRC + iPG * sizeof(pMapping->aPTs[0].paPaePTsR3->a[0]);
-
-    /* init cache area */
-    RTHCPHYS HCPhysDummy = MMR3PageDummyHCPhys(pVM);
-    for (uint32_t offDynMap = 0; offDynMap < MM_HYPER_DYNAMIC_SIZE; offDynMap += PAGE_SIZE)
-    {
-        rc = PGMMap(pVM, pVM->pgm.s.pbDynPageMapBaseGC + offDynMap, HCPhysDummy, PAGE_SIZE, 0);
-        AssertRCReturn(rc, rc);
-    }
-
-    /*
-     * Determine the max physical address width (MAXPHYADDR) and apply it to
-     * all the mask members and stuff.
-     */
-    uint32_t cMaxPhysAddrWidth;
-    uint32_t uMaxExtLeaf = ASMCpuId_EAX(0x80000000);
-    if (   uMaxExtLeaf >= 0x80000008
-        && uMaxExtLeaf <= 0x80000fff)
-    {
-        cMaxPhysAddrWidth = ASMCpuId_EAX(0x80000008) & 0xff;
-        LogRel(("PGM: The CPU physical address width is %u bits\n", cMaxPhysAddrWidth));
-        cMaxPhysAddrWidth = RT_MIN(52, cMaxPhysAddrWidth);
-        pVM->pgm.s.fLessThan52PhysicalAddressBits = cMaxPhysAddrWidth < 52;
-        for (uint32_t iBit = cMaxPhysAddrWidth; iBit < 52; iBit++)
-            pVM->pgm.s.HCPhysInvMmioPg |= RT_BIT_64(iBit);
-    }
-    else
-    {
-        LogRel(("PGM: ASSUMING CPU physical address width of 48 bits (uMaxExtLeaf=%#x)\n", uMaxExtLeaf));
-        cMaxPhysAddrWidth = 48;
-        pVM->pgm.s.fLessThan52PhysicalAddressBits = true;
-        pVM->pgm.s.HCPhysInvMmioPg |= UINT64_C(0x000f0000000000);
-    }
-
-    pVM->pgm.s.GCPhysInvAddrMask = 0;
-    for (uint32_t iBit = cMaxPhysAddrWidth; iBit < 64; iBit++)
-        pVM->pgm.s.GCPhysInvAddrMask |= RT_BIT_64(iBit);
-
-    /*
-     * Initialize the invalid paging entry masks, assuming NX is disabled.
-     */
-    uint64_t fMbzPageFrameMask = pVM->pgm.s.GCPhysInvAddrMask & UINT64_C(0x000ffffffffff000);
-    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[iCpu];
-
-        /** @todo The manuals are not entirely clear whether the physical
-         *        address width is relevant.  See table 5-9 in the intel
-         *        manual vs the PDE4M descriptions.  Write testcase (NP). */
-        pVCpu->pgm.s.fGst32BitMbzBigPdeMask   = ((uint32_t)(fMbzPageFrameMask >> (32 - 13)) & X86_PDE4M_PG_HIGH_MASK)
-                                              | X86_PDE4M_MBZ_MASK;
-
-        pVCpu->pgm.s.fGstPaeMbzPteMask        = fMbzPageFrameMask | X86_PTE_PAE_MBZ_MASK_NO_NX;
-        pVCpu->pgm.s.fGstPaeMbzPdeMask        = fMbzPageFrameMask | X86_PDE_PAE_MBZ_MASK_NO_NX;
-        pVCpu->pgm.s.fGstPaeMbzBigPdeMask     = fMbzPageFrameMask | X86_PDE2M_PAE_MBZ_MASK_NO_NX;
-        pVCpu->pgm.s.fGstPaeMbzPdpeMask       = fMbzPageFrameMask | X86_PDPE_PAE_MBZ_MASK;
-
-        pVCpu->pgm.s.fGstAmd64MbzPteMask      = fMbzPageFrameMask | X86_PTE_LM_MBZ_MASK_NO_NX;
-        pVCpu->pgm.s.fGstAmd64MbzPdeMask      = fMbzPageFrameMask | X86_PDE_LM_MBZ_MASK_NX;
-        pVCpu->pgm.s.fGstAmd64MbzBigPdeMask   = fMbzPageFrameMask | X86_PDE2M_LM_MBZ_MASK_NX;
-        pVCpu->pgm.s.fGstAmd64MbzPdpeMask     = fMbzPageFrameMask | X86_PDPE_LM_MBZ_MASK_NO_NX;
-        pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask  = fMbzPageFrameMask | X86_PDPE1G_LM_MBZ_MASK_NO_NX;
-        pVCpu->pgm.s.fGstAmd64MbzPml4eMask    = fMbzPageFrameMask | X86_PML4E_MBZ_MASK_NO_NX;
-
-        pVCpu->pgm.s.fGst64ShadowedPteMask    = X86_PTE_P   | X86_PTE_RW   | X86_PTE_US   | X86_PTE_G | X86_PTE_A | X86_PTE_D;
-        pVCpu->pgm.s.fGst64ShadowedPdeMask    = X86_PDE_P   | X86_PDE_RW   | X86_PDE_US   | X86_PDE_A;
-        pVCpu->pgm.s.fGst64ShadowedBigPdeMask = X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A;
-        pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask =
-            X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_G | X86_PDE4M_A | X86_PDE4M_D;
-        pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask  = X86_PDPE_P  | X86_PDPE_RW  | X86_PDPE_US  | X86_PDPE_A;
-        pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
-    }
-
-    /*
-     * Note that AMD uses all the 8 reserved bits for the address (so 40 bits in total);
-     * Intel only goes up to 36 bits, so we stick to 36 as well.
-     * Update: More recent intel manuals specifies 40 bits just like AMD.
-     */
-    uint32_t u32Dummy, u32Features;
-    CPUMGetGuestCpuId(VMMGetCpu(pVM), 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
-    if (u32Features & X86_CPUID_FEATURE_EDX_PSE36)
-        pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(RT_MAX(36, cMaxPhysAddrWidth)) - 1;
-    else
-        pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1;
-
-    /*
-     * Allocate memory if we're supposed to do that.
-     */
-    if (pVM->pgm.s.fRamPreAlloc)
-        rc = pgmR3PhysRamPreAllocate(pVM);
-
-    LogRel(("PGMR3InitFinalize: 4 MB PSE mask %RGp\n", pVM->pgm.s.GCPhys4MBPSEMask));
-    return rc;
-}
-
-
-/**
- * Applies relocations to data and code managed by this component.
- *
- * This function will be called at init and whenever the VMM need to relocate it
- * self inside the GC.
- *
- * @param   pVM     The VM.
- * @param   offDelta    Relocation delta relative to old location.
- */
-VMMR3DECL(void) PGMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    LogFlow(("PGMR3Relocate %RGv to %RGv\n", pVM->pgm.s.GCPtrCR3Mapping, pVM->pgm.s.GCPtrCR3Mapping + offDelta));
-
-    /*
-     * Paging stuff.
-     */
-    pVM->pgm.s.GCPtrCR3Mapping += offDelta;
-
-    pgmR3ModeDataInit(pVM, true /* resolve GC/R0 symbols */);
-
-    /* Shadow, guest and both mode switch & relocation for each VCPU. */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU  pVCpu = &pVM->aCpus[i];
-
-        pgmR3ModeDataSwitch(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
-
-        PGM_SHW_PFN(Relocate, pVCpu)(pVCpu, offDelta);
-        PGM_GST_PFN(Relocate, pVCpu)(pVCpu, offDelta);
-        PGM_BTH_PFN(Relocate, pVCpu)(pVCpu, offDelta);
-    }
-
-    /*
-     * Trees.
-     */
-    pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
-
-    /*
-     * Ram ranges.
-     */
-    if (pVM->pgm.s.pRamRangesR3)
-    {
-        /* Update the pSelfRC pointers and relink them. */
-        for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
-            if (!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
-                pCur->pSelfRC = MMHyperCCToRC(pVM, pCur);
-        pgmR3PhysRelinkRamRanges(pVM);
-    }
-
-    /*
-     * Update the pSelfRC pointer of the MMIO2 ram ranges since they might not
-     * be mapped and thus not included in the above exercise.
-     */
-    for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
-        if (!(pCur->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
-            pCur->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pCur->RamRange);
-
-    /*
-     * Update the two page directories with all page table mappings.
-     * (One or more of them have changed, that's why we're here.)
-     */
-    pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pMappingsR3);
-    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur->pNextR3; pCur = pCur->pNextR3)
-        pCur->pNextRC = MMHyperR3ToRC(pVM, pCur->pNextR3);
-
-    /* Relocate GC addresses of Page Tables. */
-    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
-    {
-        for (RTHCUINT i = 0; i < pCur->cPTs; i++)
-        {
-            pCur->aPTs[i].pPTRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].pPTR3);
-            pCur->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].paPaePTsR3);
-        }
-    }
-
-    /*
-     * Dynamic page mapping area.
-     */
-    pVM->pgm.s.paDynPageMap32BitPTEsGC += offDelta;
-    pVM->pgm.s.paDynPageMapPaePTEsGC   += offDelta;
-    pVM->pgm.s.pbDynPageMapBaseGC      += offDelta;
-
-    if (pVM->pgm.s.pRCDynMap)
-    {
-        pVM->pgm.s.pRCDynMap += offDelta;
-        PPGMRCDYNMAP pDynMap = (PPGMRCDYNMAP)MMHyperRCToCC(pVM, pVM->pgm.s.pRCDynMap);
-
-        pDynMap->paPages     += offDelta;
-        PPGMRCDYNMAPENTRY paPages = (PPGMRCDYNMAPENTRY)MMHyperRCToCC(pVM, pDynMap->paPages);
-
-        for (uint32_t iPage = 0; iPage < pDynMap->cPages; iPage++)
-        {
-            paPages[iPage].pvPage  += offDelta;
-            paPages[iPage].uPte.pv += offDelta;
-        }
-    }
-
-    /*
-     * The Zero page.
-     */
-    pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR || !VMMIsHwVirtExtForced(pVM));
-#else
-    AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR);
-#endif
-
-    /*
-     * Physical and virtual handlers.
-     */
-    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,     true, pgmR3RelocatePhysHandler,      &offDelta);
-    pVM->pgm.s.pLastPhysHandlerRC = NIL_RTRCPTR;
-    RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->VirtHandlers,      true, pgmR3RelocateVirtHandler,      &offDelta);
-    RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3RelocateHyperVirtHandler, &offDelta);
-
-    /*
-     * The page pool.
-     */
-    pgmR3PoolRelocate(pVM);
-
-#ifdef VBOX_WITH_STATISTICS
-    /*
-     * Statistics.
-     */
-    pVM->pgm.s.pStatsRC = MMHyperCCToRC(pVM, pVM->pgm.s.pStatsR3);
-    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
-        pVM->aCpus[iCpu].pgm.s.pStatsRC = MMHyperCCToRC(pVM, pVM->aCpus[iCpu].pgm.s.pStatsR3);
-#endif
-}
-
-
-/**
- * Callback function for relocating a physical access handler.
- *
- * @returns 0 (continue enum)
- * @param   pNode       Pointer to a PGMPHYSHANDLER node.
- * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
- *                      not certain the delta will fit in a void pointer for all possible configs.
- */
-static DECLCALLBACK(int) pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PPGMPHYSHANDLER pHandler = (PPGMPHYSHANDLER)pNode;
-    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
-    if (pHandler->pfnHandlerRC)
-        pHandler->pfnHandlerRC += offDelta;
-    if (pHandler->pvUserRC >= 0x10000)
-        pHandler->pvUserRC += offDelta;
-    return 0;
-}
-
-
-/**
- * Callback function for relocating a virtual access handler.
- *
- * @returns 0 (continue enum)
- * @param   pNode       Pointer to a PGMVIRTHANDLER node.
- * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
- *                      not certain the delta will fit in a void pointer for all possible configs.
- */
-static DECLCALLBACK(int) pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
-{
-    PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
-    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
-    Assert(     pHandler->enmType == PGMVIRTHANDLERTYPE_ALL
-           ||   pHandler->enmType == PGMVIRTHANDLERTYPE_WRITE);
-    Assert(pHandler->pfnHandlerRC);
-    pHandler->pfnHandlerRC += offDelta;
-    return 0;
-}
-
-
-/**
- * Callback function for relocating a virtual access handler for the hypervisor mapping.
- *
- * @returns 0 (continue enum)
- * @param   pNode       Pointer to a PGMVIRTHANDLER node.
- * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
- *                      not certain the delta will fit in a void pointer for all possible configs.
- */
-static DECLCALLBACK(int) pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
-{
-    PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
-    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
-    Assert(pHandler->enmType == PGMVIRTHANDLERTYPE_HYPERVISOR);
-    Assert(pHandler->pfnHandlerRC);
-    pHandler->pfnHandlerRC  += offDelta;
-    return 0;
-}
-
-
-/**
- * Resets a virtual CPU when unplugged.
- *
- * @param   pVM                 The VM handle.
- * @param   pVCpu               The virtual CPU handle.
- */
-VMMR3DECL(void) PGMR3ResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
-{
-    int rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
-    AssertRC(rc);
-
-    rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
-    AssertRC(rc);
-
-    STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
-
-    pgmR3PoolResetUnpluggedCpu(pVM, pVCpu);
-
-    /*
-     * Re-init other members.
-     */
-    pVCpu->pgm.s.fA20Enabled = true;
-
-    /*
-     * Clear the FFs PGM owns.
-     */
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
-}
-
-
-/**
- * The VM is being reset.
- *
- * For the PGM component this means that any PD write monitors
- * needs to be removed.
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) PGMR3Reset(PVM pVM)
-{
-    int rc;
-
-    LogFlow(("PGMR3Reset:\n"));
-    VM_ASSERT_EMT(pVM);
-
-    pgmLock(pVM);
-
-    /*
-     * Unfix any fixed mappings and disable CR3 monitoring.
-     */
-    pVM->pgm.s.fMappingsFixed         = false;
-    pVM->pgm.s.fMappingsFixedRestored = false;
-    pVM->pgm.s.GCPtrMappingFixed      = NIL_RTGCPTR;
-    pVM->pgm.s.cbMappingFixed         = 0;
-
-    /*
-     * Exit the guest paging mode before the pgm pool gets reset.
-     * Important to clean up the amd64 case.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU  pVCpu = &pVM->aCpus[i];
-        rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
-        AssertRC(rc);
-    }
-
-#ifdef DEBUG
-    DBGFR3InfoLog(pVM, "mappings", NULL);
-    DBGFR3InfoLog(pVM, "handlers", "all nostat");
-#endif
-
-    /*
-     * Switch mode back to real mode. (before resetting the pgm pool!)
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU  pVCpu = &pVM->aCpus[i];
-
-        rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
-        AssertRC(rc);
-
-        STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
-    }
-
-    /*
-     * Reset the shadow page pool.
-     */
-    pgmR3PoolReset(pVM);
-
-    /*
-     * Re-init various other members and clear the FFs that PGM owns.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        pVCpu->pgm.s.fA20Enabled = true;
-        pVCpu->pgm.s.fGst32BitPageSizeExtension = false;
-        PGMNotifyNxeChanged(pVCpu, false);
-
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
-    }
-
-    /*
-     * Reset (zero) RAM and shadow ROM pages.
-     */
-    rc = pgmR3PhysRamReset(pVM);
-    if (RT_SUCCESS(rc))
-        rc = pgmR3PhysRomReset(pVM);
-
-
-    pgmUnlock(pVM);
-    AssertReleaseRC(rc);
-}
-
-
-#ifdef VBOX_STRICT
-/**
- * VM state change callback for clearing fNoMorePhysWrites after
- * a snapshot has been created.
- */
-static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser)
-{
-    if (   enmState == VMSTATE_RUNNING
-        || enmState == VMSTATE_RESUMING)
-        pVM->pgm.s.fNoMorePhysWrites = false;
-}
-#endif
-
-/**
- * Private API to reset fNoMorePhysWrites.
- */
-VMMR3DECL(void) PGMR3ResetNoMorePhysWritesFlag(PVM pVM)
-{
-    pVM->pgm.s.fNoMorePhysWrites = false;
-}
-
-/**
- * Terminates the PGM.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to VM structure.
- */
-VMMR3DECL(int) PGMR3Term(PVM pVM)
-{
-    /* Must free shared pages here. */
-    pgmLock(pVM);
-    pgmR3PhysRamTerm(pVM);
-    pgmR3PhysRomTerm(pVM);
-    pgmUnlock(pVM);
-
-    PGMDeregisterStringFormatTypes();
-    return PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
-}
-
-
-/**
- * Show paging mode.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     "all" (default), "guest", "shadow" or "host".
- */
-static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    /* digest argument. */
-    bool fGuest, fShadow, fHost;
-    if (pszArgs)
-        pszArgs = RTStrStripL(pszArgs);
-    if (!pszArgs || !*pszArgs || strstr(pszArgs, "all"))
-        fShadow = fHost = fGuest = true;
-    else
-    {
-        fShadow = fHost = fGuest = false;
-        if (strstr(pszArgs, "guest"))
-            fGuest = true;
-        if (strstr(pszArgs, "shadow"))
-            fShadow = true;
-        if (strstr(pszArgs, "host"))
-            fHost = true;
-    }
-
-    /** @todo SMP support! */
-    /* print info. */
-    if (fGuest)
-        pHlp->pfnPrintf(pHlp, "Guest paging mode:  %s, changed %RU64 times, A20 %s\n",
-                        PGMGetModeName(pVM->aCpus[0].pgm.s.enmGuestMode), pVM->aCpus[0].pgm.s.cGuestModeChanges.c,
-                        pVM->aCpus[0].pgm.s.fA20Enabled ? "enabled" : "disabled");
-    if (fShadow)
-        pHlp->pfnPrintf(pHlp, "Shadow paging mode: %s\n", PGMGetModeName(pVM->aCpus[0].pgm.s.enmShadowMode));
-    if (fHost)
-    {
-        const char *psz;
-        switch (pVM->pgm.s.enmHostMode)
-        {
-            case SUPPAGINGMODE_INVALID:             psz = "invalid"; break;
-            case SUPPAGINGMODE_32_BIT:              psz = "32-bit"; break;
-            case SUPPAGINGMODE_32_BIT_GLOBAL:       psz = "32-bit+G"; break;
-            case SUPPAGINGMODE_PAE:                 psz = "PAE"; break;
-            case SUPPAGINGMODE_PAE_GLOBAL:          psz = "PAE+G"; break;
-            case SUPPAGINGMODE_PAE_NX:              psz = "PAE+NX"; break;
-            case SUPPAGINGMODE_PAE_GLOBAL_NX:       psz = "PAE+G+NX"; break;
-            case SUPPAGINGMODE_AMD64:               psz = "AMD64"; break;
-            case SUPPAGINGMODE_AMD64_GLOBAL:        psz = "AMD64+G"; break;
-            case SUPPAGINGMODE_AMD64_NX:            psz = "AMD64+NX"; break;
-            case SUPPAGINGMODE_AMD64_GLOBAL_NX:     psz = "AMD64+G+NX"; break;
-            default:                                psz = "unknown"; break;
-        }
-        pHlp->pfnPrintf(pHlp, "Host paging mode:   %s\n", psz);
-    }
-}
-
-
-/**
- * Dump registered MMIO ranges to the log.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-    pHlp->pfnPrintf(pHlp,
-                    "RAM ranges (pVM=%p)\n"
-                    "%.*s %.*s\n",
-                    pVM,
-                    sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range                    ",
-                    sizeof(RTHCPTR) * 2,      "pvHC            ");
-
-    for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
-        pHlp->pfnPrintf(pHlp,
-                        "%RGp-%RGp %RHv %s\n",
-                        pCur->GCPhys,
-                        pCur->GCPhysLast,
-                        pCur->pvR3,
-                        pCur->pszDesc);
-}
-
-/**
- * Dump the page directory to the log.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    /** @todo SMP support!! */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-/** @todo fix this! Convert the PGMR3DumpHierarchyHC functions to do guest stuff. */
-    /* Big pages supported? */
-    const bool fPSE  = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
-
-    /* Global pages supported? */
-    const bool fPGE = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PGE);
-
-    NOREF(pszArgs);
-
-    /*
-     * Get page directory addresses.
-     */
-    PX86PD     pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
-    Assert(pPDSrc);
-    Assert(PGMPhysGCPhys2R3PtrAssert(pVM, (RTGCPHYS)(CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
-
-    /*
-     * Iterate the page directory.
-     */
-    for (unsigned iPD = 0; iPD < RT_ELEMENTS(pPDSrc->a); iPD++)
-    {
-        X86PDE PdeSrc = pPDSrc->a[iPD];
-        if (PdeSrc.n.u1Present)
-        {
-            if (PdeSrc.b.u1Size && fPSE)
-                pHlp->pfnPrintf(pHlp,
-                                "%04X - %RGp P=%d U=%d RW=%d G=%d - BIG\n",
-                                iPD,
-                                pgmGstGet4MBPhysPage(&pVM->pgm.s, PdeSrc),
-                                PdeSrc.b.u1Present, PdeSrc.b.u1User, PdeSrc.b.u1Write, PdeSrc.b.u1Global && fPGE);
-            else
-                pHlp->pfnPrintf(pHlp,
-                                "%04X - %RGp P=%d U=%d RW=%d [G=%d]\n",
-                                iPD,
-                                (RTGCPHYS)(PdeSrc.u & X86_PDE_PG_MASK),
-                                PdeSrc.n.u1Present, PdeSrc.n.u1User, PdeSrc.n.u1Write, PdeSrc.b.u1Global && fPGE);
-        }
-    }
-}
-
-
-/**
- * Service a VMMCALLRING3_PGM_LOCK call.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-VMMR3DECL(int) PGMR3LockCall(PVM pVM)
-{
-    int rc = PDMR3CritSectEnterEx(&pVM->pgm.s.CritSect, true /* fHostCall */);
-    AssertRC(rc);
-    return rc;
-}
-
-
-/**
- * Converts a PGMMODE value to a PGM_TYPE_* \#define.
- *
- * @returns PGM_TYPE_*.
- * @param   pgmMode     The mode value to convert.
- */
-DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
-{
-    switch (pgmMode)
-    {
-        case PGMMODE_REAL:      return PGM_TYPE_REAL;
-        case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
-        case PGMMODE_32_BIT:    return PGM_TYPE_32BIT;
-        case PGMMODE_PAE:
-        case PGMMODE_PAE_NX:    return PGM_TYPE_PAE;
-        case PGMMODE_AMD64:
-        case PGMMODE_AMD64_NX:  return PGM_TYPE_AMD64;
-        case PGMMODE_NESTED:    return PGM_TYPE_NESTED;
-        case PGMMODE_EPT:       return PGM_TYPE_EPT;
-        default:
-            AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
-    }
-}
-
-
-/**
- * Gets the index into the paging mode data array of a SHW+GST mode.
- *
- * @returns PGM::paPagingData index.
- * @param   uShwType      The shadow paging mode type.
- * @param   uGstType      The guest paging mode type.
- */
-DECLINLINE(unsigned) pgmModeDataIndex(unsigned uShwType, unsigned uGstType)
-{
-    Assert(uShwType >= PGM_TYPE_32BIT && uShwType <= PGM_TYPE_MAX);
-    Assert(uGstType >= PGM_TYPE_REAL  && uGstType <= PGM_TYPE_AMD64);
-    return (uShwType - PGM_TYPE_32BIT) * (PGM_TYPE_AMD64 - PGM_TYPE_REAL + 1)
-         + (uGstType - PGM_TYPE_REAL);
-}
-
-
-/**
- * Gets the index into the paging mode data array of a SHW+GST mode.
- *
- * @returns PGM::paPagingData index.
- * @param   enmShw      The shadow paging mode.
- * @param   enmGst      The guest paging mode.
- */
-DECLINLINE(unsigned) pgmModeDataIndexByMode(PGMMODE enmShw, PGMMODE enmGst)
-{
-    Assert(enmShw >= PGMMODE_32_BIT && enmShw <= PGMMODE_MAX);
-    Assert(enmGst > PGMMODE_INVALID && enmGst < PGMMODE_MAX);
-    return pgmModeDataIndex(pgmModeToType(enmShw), pgmModeToType(enmGst));
-}
-
-
-/**
- * Calculates the max data index.
- * @returns The number of entries in the paging data array.
- */
-DECLINLINE(unsigned) pgmModeDataMaxIndex(void)
-{
-    return pgmModeDataIndex(PGM_TYPE_MAX, PGM_TYPE_AMD64) + 1;
-}
-
-
-/**
- * Initializes the paging mode data kept in PGM::paModeData.
- *
- * @param   pVM             The VM handle.
- * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
- *                          This is used early in the init process to avoid trouble with PDM
- *                          not being initialized yet.
- */
-static int pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0)
-{
-    PPGMMODEDATA pModeData;
-    int rc;
-
-    /*
-     * Allocate the array on the first call.
-     */
-    if (!pVM->pgm.s.paModeData)
-    {
-        pVM->pgm.s.paModeData = (PPGMMODEDATA)MMR3HeapAllocZ(pVM, MM_TAG_PGM, sizeof(PGMMODEDATA) * pgmModeDataMaxIndex());
-        AssertReturn(pVM->pgm.s.paModeData, VERR_NO_MEMORY);
-    }
-
-    /*
-     * Initialize the array entries.
-     */
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_REAL)];
-    pModeData->uShwType = PGM_TYPE_32BIT;
-    pModeData->uGstType = PGM_TYPE_REAL;
-    rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_REAL(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_32BIT_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGMMODE_PROTECTED)];
-    pModeData->uShwType = PGM_TYPE_32BIT;
-    pModeData->uGstType = PGM_TYPE_PROT;
-    rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_PROT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_32BIT_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_32BIT)];
-    pModeData->uShwType = PGM_TYPE_32BIT;
-    pModeData->uGstType = PGM_TYPE_32BIT;
-    rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_32BIT_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_REAL)];
-    pModeData->uShwType = PGM_TYPE_PAE;
-    pModeData->uGstType = PGM_TYPE_REAL;
-    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_REAL(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_PAE_REAL(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PROT)];
-    pModeData->uShwType = PGM_TYPE_PAE;
-    pModeData->uGstType = PGM_TYPE_PROT;
-    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_PROT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_PAE_PROT(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_32BIT)];
-    pModeData->uShwType = PGM_TYPE_PAE;
-    pModeData->uGstType = PGM_TYPE_32BIT;
-    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_PAE_32BIT(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PAE)];
-    pModeData->uShwType = PGM_TYPE_PAE;
-    pModeData->uGstType = PGM_TYPE_PAE;
-    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_PAE_PAE(InitData)(    pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-#ifdef VBOX_WITH_64_BITS_GUESTS
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_AMD64, PGM_TYPE_AMD64)];
-    pModeData->uShwType = PGM_TYPE_AMD64;
-    pModeData->uGstType = PGM_TYPE_AMD64;
-    rc = PGM_SHW_NAME_AMD64(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_AMD64(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_AMD64_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-#endif
-
-    /* The nested paging mode. */
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_REAL)];
-    pModeData->uShwType = PGM_TYPE_NESTED;
-    pModeData->uGstType = PGM_TYPE_REAL;
-    rc = PGM_GST_NAME_REAL(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_NESTED_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGMMODE_PROTECTED)];
-    pModeData->uShwType = PGM_TYPE_NESTED;
-    pModeData->uGstType = PGM_TYPE_PROT;
-    rc = PGM_GST_NAME_PROT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_NESTED_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_32BIT)];
-    pModeData->uShwType = PGM_TYPE_NESTED;
-    pModeData->uGstType = PGM_TYPE_32BIT;
-    rc = PGM_GST_NAME_32BIT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_NESTED_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_PAE)];
-    pModeData->uShwType = PGM_TYPE_NESTED;
-    pModeData->uGstType = PGM_TYPE_PAE;
-    rc = PGM_GST_NAME_PAE(InitData)(         pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_NESTED_PAE(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-#ifdef VBOX_WITH_64_BITS_GUESTS
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
-    pModeData->uShwType = PGM_TYPE_NESTED;
-    pModeData->uGstType = PGM_TYPE_AMD64;
-    rc = PGM_GST_NAME_AMD64(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_NESTED_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-#endif
-
-    /* The shadow part of the nested callback mode depends on the host paging mode (AMD-V only). */
-    switch (pVM->pgm.s.enmHostMode)
-    {
-#if HC_ARCH_BITS == 32
-    case SUPPAGINGMODE_32_BIT:
-    case SUPPAGINGMODE_32_BIT_GLOBAL:
-        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
-        {
-            pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
-            rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-        }
-# ifdef VBOX_WITH_64_BITS_GUESTS
-        pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
-        rc = PGM_SHW_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-# endif
-        break;
-
-    case SUPPAGINGMODE_PAE:
-    case SUPPAGINGMODE_PAE_NX:
-    case SUPPAGINGMODE_PAE_GLOBAL:
-    case SUPPAGINGMODE_PAE_GLOBAL_NX:
-        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
-        {
-            pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
-            rc = PGM_SHW_NAME_PAE(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-        }
-# ifdef VBOX_WITH_64_BITS_GUESTS
-        pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
-        rc = PGM_SHW_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-# endif
-        break;
-#endif /* HC_ARCH_BITS == 32 */
-
-#if HC_ARCH_BITS == 64 || defined(RT_OS_DARWIN)
-    case SUPPAGINGMODE_AMD64:
-    case SUPPAGINGMODE_AMD64_GLOBAL:
-    case SUPPAGINGMODE_AMD64_NX:
-    case SUPPAGINGMODE_AMD64_GLOBAL_NX:
-# ifdef VBOX_WITH_64_BITS_GUESTS
-        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_AMD64; i++)
-# else
-        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
-# endif
-        {
-            pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
-            rc = PGM_SHW_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-        }
-        break;
-#endif /* HC_ARCH_BITS == 64 || RT_OS_DARWIN */
-
-    default:
-        AssertFailed();
-        break;
-    }
-
-    /* Extended paging (EPT) / Intel VT-x */
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_REAL)];
-    pModeData->uShwType = PGM_TYPE_EPT;
-    pModeData->uGstType = PGM_TYPE_REAL;
-    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_REAL(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_EPT_REAL(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PROT)];
-    pModeData->uShwType = PGM_TYPE_EPT;
-    pModeData->uGstType = PGM_TYPE_PROT;
-    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_PROT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_EPT_PROT(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_32BIT)];
-    pModeData->uShwType = PGM_TYPE_EPT;
-    pModeData->uGstType = PGM_TYPE_32BIT;
-    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_EPT_32BIT(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PAE)];
-    pModeData->uShwType = PGM_TYPE_EPT;
-    pModeData->uGstType = PGM_TYPE_PAE;
-    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_EPT_PAE(InitData)(    pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-
-#ifdef VBOX_WITH_64_BITS_GUESTS
-    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_AMD64)];
-    pModeData->uShwType = PGM_TYPE_EPT;
-    pModeData->uGstType = PGM_TYPE_AMD64;
-    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_GST_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-    rc = PGM_BTH_NAME_EPT_AMD64(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
-#endif
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Switch to different (or relocated in the relocate case) mode data.
- *
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   enmShw      The the shadow paging mode.
- * @param   enmGst      The the guest paging mode.
- */
-static void pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst)
-{
-    PPGMMODEDATA pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndexByMode(enmShw, enmGst)];
-
-    Assert(pModeData->uGstType == pgmModeToType(enmGst));
-    Assert(pModeData->uShwType == pgmModeToType(enmShw));
-
-    /* shadow */
-    pVCpu->pgm.s.pfnR3ShwRelocate             = pModeData->pfnR3ShwRelocate;
-    pVCpu->pgm.s.pfnR3ShwExit                 = pModeData->pfnR3ShwExit;
-    pVCpu->pgm.s.pfnR3ShwGetPage              = pModeData->pfnR3ShwGetPage;
-    Assert(pVCpu->pgm.s.pfnR3ShwGetPage);
-    pVCpu->pgm.s.pfnR3ShwModifyPage           = pModeData->pfnR3ShwModifyPage;
-
-    pVCpu->pgm.s.pfnRCShwGetPage              = pModeData->pfnRCShwGetPage;
-    pVCpu->pgm.s.pfnRCShwModifyPage           = pModeData->pfnRCShwModifyPage;
-
-    pVCpu->pgm.s.pfnR0ShwGetPage              = pModeData->pfnR0ShwGetPage;
-    pVCpu->pgm.s.pfnR0ShwModifyPage           = pModeData->pfnR0ShwModifyPage;
-
-
-    /* guest */
-    pVCpu->pgm.s.pfnR3GstRelocate             = pModeData->pfnR3GstRelocate;
-    pVCpu->pgm.s.pfnR3GstExit                 = pModeData->pfnR3GstExit;
-    pVCpu->pgm.s.pfnR3GstGetPage              = pModeData->pfnR3GstGetPage;
-    Assert(pVCpu->pgm.s.pfnR3GstGetPage);
-    pVCpu->pgm.s.pfnR3GstModifyPage           = pModeData->pfnR3GstModifyPage;
-    pVCpu->pgm.s.pfnR3GstGetPDE               = pModeData->pfnR3GstGetPDE;
-    pVCpu->pgm.s.pfnRCGstGetPage              = pModeData->pfnRCGstGetPage;
-    pVCpu->pgm.s.pfnRCGstModifyPage           = pModeData->pfnRCGstModifyPage;
-    pVCpu->pgm.s.pfnRCGstGetPDE               = pModeData->pfnRCGstGetPDE;
-    pVCpu->pgm.s.pfnR0GstGetPage              = pModeData->pfnR0GstGetPage;
-    pVCpu->pgm.s.pfnR0GstModifyPage           = pModeData->pfnR0GstModifyPage;
-    pVCpu->pgm.s.pfnR0GstGetPDE               = pModeData->pfnR0GstGetPDE;
-
-    /* both */
-    pVCpu->pgm.s.pfnR3BthRelocate             = pModeData->pfnR3BthRelocate;
-    pVCpu->pgm.s.pfnR3BthInvalidatePage       = pModeData->pfnR3BthInvalidatePage;
-    pVCpu->pgm.s.pfnR3BthSyncCR3              = pModeData->pfnR3BthSyncCR3;
-    Assert(pVCpu->pgm.s.pfnR3BthSyncCR3);
-    pVCpu->pgm.s.pfnR3BthPrefetchPage         = pModeData->pfnR3BthPrefetchPage;
-    pVCpu->pgm.s.pfnR3BthVerifyAccessSyncPage = pModeData->pfnR3BthVerifyAccessSyncPage;
-#ifdef VBOX_STRICT
-    pVCpu->pgm.s.pfnR3BthAssertCR3            = pModeData->pfnR3BthAssertCR3;
-#endif
-    pVCpu->pgm.s.pfnR3BthMapCR3               = pModeData->pfnR3BthMapCR3;
-    pVCpu->pgm.s.pfnR3BthUnmapCR3             = pModeData->pfnR3BthUnmapCR3;
-
-    pVCpu->pgm.s.pfnRCBthTrap0eHandler        = pModeData->pfnRCBthTrap0eHandler;
-    pVCpu->pgm.s.pfnRCBthInvalidatePage       = pModeData->pfnRCBthInvalidatePage;
-    pVCpu->pgm.s.pfnRCBthSyncCR3              = pModeData->pfnRCBthSyncCR3;
-    pVCpu->pgm.s.pfnRCBthPrefetchPage         = pModeData->pfnRCBthPrefetchPage;
-    pVCpu->pgm.s.pfnRCBthVerifyAccessSyncPage = pModeData->pfnRCBthVerifyAccessSyncPage;
-#ifdef VBOX_STRICT
-    pVCpu->pgm.s.pfnRCBthAssertCR3            = pModeData->pfnRCBthAssertCR3;
-#endif
-    pVCpu->pgm.s.pfnRCBthMapCR3               = pModeData->pfnRCBthMapCR3;
-    pVCpu->pgm.s.pfnRCBthUnmapCR3             = pModeData->pfnRCBthUnmapCR3;
-
-    pVCpu->pgm.s.pfnR0BthTrap0eHandler        = pModeData->pfnR0BthTrap0eHandler;
-    pVCpu->pgm.s.pfnR0BthInvalidatePage       = pModeData->pfnR0BthInvalidatePage;
-    pVCpu->pgm.s.pfnR0BthSyncCR3              = pModeData->pfnR0BthSyncCR3;
-    pVCpu->pgm.s.pfnR0BthPrefetchPage         = pModeData->pfnR0BthPrefetchPage;
-    pVCpu->pgm.s.pfnR0BthVerifyAccessSyncPage = pModeData->pfnR0BthVerifyAccessSyncPage;
-#ifdef VBOX_STRICT
-    pVCpu->pgm.s.pfnR0BthAssertCR3            = pModeData->pfnR0BthAssertCR3;
-#endif
-    pVCpu->pgm.s.pfnR0BthMapCR3               = pModeData->pfnR0BthMapCR3;
-    pVCpu->pgm.s.pfnR0BthUnmapCR3             = pModeData->pfnR0BthUnmapCR3;
-}
-
-
-/**
- * Calculates the shadow paging mode.
- *
- * @returns The shadow paging mode.
- * @param   pVM             VM handle.
- * @param   enmGuestMode    The guest mode.
- * @param   enmHostMode     The host mode.
- * @param   enmShadowMode   The current shadow mode.
- * @param   penmSwitcher    Where to store the switcher to use.
- *                          VMMSWITCHER_INVALID means no change.
- */
-static PGMMODE pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher)
-{
-    VMMSWITCHER enmSwitcher = VMMSWITCHER_INVALID;
-    switch (enmGuestMode)
-    {
-        /*
-         * When switching to real or protected mode we don't change
-         * anything since it's likely that we'll switch back pretty soon.
-         *
-         * During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
-         * and is supposed to determine which shadow paging and switcher to
-         * use during init.
-         */
-        case PGMMODE_REAL:
-        case PGMMODE_PROTECTED:
-            if (    enmShadowMode != PGMMODE_INVALID
-                && !HWACCMIsEnabled(pVM) /* always switch in hwaccm mode! */)
-                break; /* (no change) */
-
-            switch (enmHostMode)
-            {
-                case SUPPAGINGMODE_32_BIT:
-                case SUPPAGINGMODE_32_BIT_GLOBAL:
-                    enmShadowMode = PGMMODE_32_BIT;
-                    enmSwitcher = VMMSWITCHER_32_TO_32;
-                    break;
-
-                case SUPPAGINGMODE_PAE:
-                case SUPPAGINGMODE_PAE_NX:
-                case SUPPAGINGMODE_PAE_GLOBAL:
-                case SUPPAGINGMODE_PAE_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_PAE;
-                    enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
-#ifdef DEBUG_bird
-                    if (RTEnvExist("VBOX_32BIT"))
-                    {
-                        enmShadowMode = PGMMODE_32_BIT;
-                        enmSwitcher = VMMSWITCHER_PAE_TO_32;
-                    }
-#endif
-                    break;
-
-                case SUPPAGINGMODE_AMD64:
-                case SUPPAGINGMODE_AMD64_GLOBAL:
-                case SUPPAGINGMODE_AMD64_NX:
-                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_PAE;
-                    enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
-#ifdef DEBUG_bird
-                    if (RTEnvExist("VBOX_32BIT"))
-                    {
-                        enmShadowMode = PGMMODE_32_BIT;
-                        enmSwitcher = VMMSWITCHER_AMD64_TO_32;
-                    }
-#endif
-                    break;
-
-                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
-            }
-            break;
-
-        case PGMMODE_32_BIT:
-            switch (enmHostMode)
-            {
-                case SUPPAGINGMODE_32_BIT:
-                case SUPPAGINGMODE_32_BIT_GLOBAL:
-                    enmShadowMode = PGMMODE_32_BIT;
-                    enmSwitcher = VMMSWITCHER_32_TO_32;
-                    break;
-
-                case SUPPAGINGMODE_PAE:
-                case SUPPAGINGMODE_PAE_NX:
-                case SUPPAGINGMODE_PAE_GLOBAL:
-                case SUPPAGINGMODE_PAE_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_PAE;
-                    enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
-#ifdef DEBUG_bird
-                    if (RTEnvExist("VBOX_32BIT"))
-                    {
-                        enmShadowMode = PGMMODE_32_BIT;
-                        enmSwitcher = VMMSWITCHER_PAE_TO_32;
-                    }
-#endif
-                    break;
-
-                case SUPPAGINGMODE_AMD64:
-                case SUPPAGINGMODE_AMD64_GLOBAL:
-                case SUPPAGINGMODE_AMD64_NX:
-                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_PAE;
-                    enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
-#ifdef DEBUG_bird
-                    if (RTEnvExist("VBOX_32BIT"))
-                    {
-                        enmShadowMode = PGMMODE_32_BIT;
-                        enmSwitcher = VMMSWITCHER_AMD64_TO_32;
-                    }
-#endif
-                    break;
-
-                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
-            }
-            break;
-
-        case PGMMODE_PAE:
-        case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
-            switch (enmHostMode)
-            {
-                case SUPPAGINGMODE_32_BIT:
-                case SUPPAGINGMODE_32_BIT_GLOBAL:
-                    enmShadowMode = PGMMODE_PAE;
-                    enmSwitcher = VMMSWITCHER_32_TO_PAE;
-                    break;
-
-                case SUPPAGINGMODE_PAE:
-                case SUPPAGINGMODE_PAE_NX:
-                case SUPPAGINGMODE_PAE_GLOBAL:
-                case SUPPAGINGMODE_PAE_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_PAE;
-                    enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
-                    break;
-
-                case SUPPAGINGMODE_AMD64:
-                case SUPPAGINGMODE_AMD64_GLOBAL:
-                case SUPPAGINGMODE_AMD64_NX:
-                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_PAE;
-                    enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
-                    break;
-
-                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
-            }
-            break;
-
-        case PGMMODE_AMD64:
-        case PGMMODE_AMD64_NX:
-            switch (enmHostMode)
-            {
-                case SUPPAGINGMODE_32_BIT:
-                case SUPPAGINGMODE_32_BIT_GLOBAL:
-                    enmShadowMode = PGMMODE_AMD64;
-                    enmSwitcher = VMMSWITCHER_32_TO_AMD64;
-                    break;
-
-                case SUPPAGINGMODE_PAE:
-                case SUPPAGINGMODE_PAE_NX:
-                case SUPPAGINGMODE_PAE_GLOBAL:
-                case SUPPAGINGMODE_PAE_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_AMD64;
-                    enmSwitcher = VMMSWITCHER_PAE_TO_AMD64;
-                    break;
-
-                case SUPPAGINGMODE_AMD64:
-                case SUPPAGINGMODE_AMD64_GLOBAL:
-                case SUPPAGINGMODE_AMD64_NX:
-                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
-                    enmShadowMode = PGMMODE_AMD64;
-                    enmSwitcher = VMMSWITCHER_AMD64_TO_AMD64;
-                    break;
-
-                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
-            }
-            break;
-
-
-        default:
-            AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
-            *penmSwitcher = VMMSWITCHER_INVALID;
-            return PGMMODE_INVALID;
-    }
-    /* Override the shadow mode is nested paging is active. */
-    pVM->pgm.s.fNestedPaging = HWACCMIsNestedPagingActive(pVM);
-    if (pVM->pgm.s.fNestedPaging)
-        enmShadowMode = HWACCMGetShwPagingMode(pVM);
-
-    *penmSwitcher = enmSwitcher;
-    return enmShadowMode;
-}
-
-
-/**
- * Performs the actual mode change.
- * This is called by PGMChangeMode and pgmR3InitPaging().
- *
- * @returns VBox status code. May suspend or power off the VM on error, but this
- *          will trigger using FFs and not status codes.
- *
- * @param   pVM             VM handle.
- * @param   pVCpu           The VMCPU to operate on.
- * @param   enmGuestMode    The new guest mode. This is assumed to be different from
- *                          the current mode.
- */
-VMMR3DECL(int) PGMR3ChangeMode(PVM pVM, PVMCPU pVCpu, PGMMODE enmGuestMode)
-{
-    bool fIsOldGuestPagingMode64Bits = (pVCpu->pgm.s.enmGuestMode >= PGMMODE_AMD64);
-    bool fIsNewGuestPagingMode64Bits = (enmGuestMode >= PGMMODE_AMD64);
-
-    Log(("PGMR3ChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
-    STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
-
-    /*
-     * Calc the shadow mode and switcher.
-     */
-    VMMSWITCHER enmSwitcher;
-    PGMMODE     enmShadowMode = pgmR3CalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode, &enmSwitcher);
-
-#ifdef VBOX_WITH_RAW_MODE
-    if (enmSwitcher != VMMSWITCHER_INVALID)
-    {
-        /*
-         * Select new switcher.
-         */
-        int rc = VMMR3SelectSwitcher(pVM, enmSwitcher);
-        if (RT_FAILURE(rc))
-        {
-            AssertReleaseMsgFailed(("VMMR3SelectSwitcher(%d) -> %Rrc\n", enmSwitcher, rc));
-            return rc;
-        }
-    }
-#endif
-
-    /*
-     * Exit old mode(s).
-     */
-#if HC_ARCH_BITS == 32
-    /* The nested shadow paging mode for AMD-V does change when running 64 bits guests on 32 bits hosts; typically PAE <-> AMD64 */
-    const bool fForceShwEnterExit = (    fIsOldGuestPagingMode64Bits != fIsNewGuestPagingMode64Bits
-                                     &&  enmShadowMode == PGMMODE_NESTED);
-#else
-    const bool fForceShwEnterExit = false;
-#endif
-    /* shadow */
-    if (    enmShadowMode != pVCpu->pgm.s.enmShadowMode
-        ||  fForceShwEnterExit)
-    {
-        LogFlow(("PGMR3ChangeMode: Shadow mode: %s -> %s\n",  PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
-        if (PGM_SHW_PFN(Exit, pVCpu))
-        {
-            int rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
-            if (RT_FAILURE(rc))
-            {
-                AssertMsgFailed(("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc));
-                return rc;
-            }
-        }
-
-    }
-    else
-        LogFlow(("PGMR3ChangeMode: Shadow mode remains: %s\n",  PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
-
-    /* guest */
-    if (PGM_GST_PFN(Exit, pVCpu))
-    {
-        int rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
-        if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc));
-            return rc;
-        }
-    }
-
-    /*
-     * Load new paging mode data.
-     */
-    pgmR3ModeDataSwitch(pVM, pVCpu, enmShadowMode, enmGuestMode);
-
-    /*
-     * Enter new shadow mode (if changed).
-     */
-    if (    enmShadowMode != pVCpu->pgm.s.enmShadowMode
-        ||  fForceShwEnterExit)
-    {
-        int rc;
-        pVCpu->pgm.s.enmShadowMode = enmShadowMode;
-        switch (enmShadowMode)
-        {
-            case PGMMODE_32_BIT:
-                rc = PGM_SHW_NAME_32BIT(Enter)(pVCpu, false);
-                break;
-            case PGMMODE_PAE:
-            case PGMMODE_PAE_NX:
-                rc = PGM_SHW_NAME_PAE(Enter)(pVCpu, false);
-                break;
-            case PGMMODE_AMD64:
-            case PGMMODE_AMD64_NX:
-                rc = PGM_SHW_NAME_AMD64(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
-                break;
-            case PGMMODE_NESTED:
-                rc = PGM_SHW_NAME_NESTED(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
-                break;
-            case PGMMODE_EPT:
-                rc = PGM_SHW_NAME_EPT(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
-                break;
-            case PGMMODE_REAL:
-            case PGMMODE_PROTECTED:
-            default:
-                AssertReleaseMsgFailed(("enmShadowMode=%d\n", enmShadowMode));
-                return VERR_INTERNAL_ERROR;
-        }
-        if (RT_FAILURE(rc))
-        {
-            AssertReleaseMsgFailed(("Entering enmShadowMode=%d failed: %Rrc\n", enmShadowMode, rc));
-            pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
-            return rc;
-        }
-    }
-
-    /*
-     * Always flag the necessary updates
-     */
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-
-    /*
-     * Enter the new guest and shadow+guest modes.
-     */
-    int rc = -1;
-    int rc2 = -1;
-    RTGCPHYS GCPhysCR3 = NIL_RTGCPHYS;
-    pVCpu->pgm.s.enmGuestMode = enmGuestMode;
-    switch (enmGuestMode)
-    {
-        case PGMMODE_REAL:
-            rc = PGM_GST_NAME_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
-            switch (pVCpu->pgm.s.enmShadowMode)
-            {
-                case PGMMODE_32_BIT:
-                    rc2 = PGM_BTH_NAME_32BIT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_PAE:
-                case PGMMODE_PAE_NX:
-                    rc2 = PGM_BTH_NAME_PAE_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_NESTED:
-                    rc2 = PGM_BTH_NAME_NESTED_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_EPT:
-                    rc2 = PGM_BTH_NAME_EPT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_AMD64:
-                case PGMMODE_AMD64_NX:
-                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
-                default: AssertFailed(); break;
-            }
-            break;
-
-        case PGMMODE_PROTECTED:
-            rc = PGM_GST_NAME_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
-            switch (pVCpu->pgm.s.enmShadowMode)
-            {
-                case PGMMODE_32_BIT:
-                    rc2 = PGM_BTH_NAME_32BIT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_PAE:
-                case PGMMODE_PAE_NX:
-                    rc2 = PGM_BTH_NAME_PAE_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_NESTED:
-                    rc2 = PGM_BTH_NAME_NESTED_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_EPT:
-                    rc2 = PGM_BTH_NAME_EPT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
-                    break;
-                case PGMMODE_AMD64:
-                case PGMMODE_AMD64_NX:
-                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
-                default: AssertFailed(); break;
-            }
-            break;
-
-        case PGMMODE_32_BIT:
-            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
-            rc = PGM_GST_NAME_32BIT(Enter)(pVCpu, GCPhysCR3);
-            switch (pVCpu->pgm.s.enmShadowMode)
-            {
-                case PGMMODE_32_BIT:
-                    rc2 = PGM_BTH_NAME_32BIT_32BIT(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_PAE:
-                case PGMMODE_PAE_NX:
-                    rc2 = PGM_BTH_NAME_PAE_32BIT(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_NESTED:
-                    rc2 = PGM_BTH_NAME_NESTED_32BIT(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_EPT:
-                    rc2 = PGM_BTH_NAME_EPT_32BIT(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_AMD64:
-                case PGMMODE_AMD64_NX:
-                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
-                default: AssertFailed(); break;
-            }
-            break;
-
-        case PGMMODE_PAE_NX:
-        case PGMMODE_PAE:
-        {
-            uint32_t u32Dummy, u32Features;
-
-            CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
-            if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
-                return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
-                                         N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (General/Advanced)"));
-
-            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
-            rc = PGM_GST_NAME_PAE(Enter)(pVCpu, GCPhysCR3);
-            switch (pVCpu->pgm.s.enmShadowMode)
-            {
-                case PGMMODE_PAE:
-                case PGMMODE_PAE_NX:
-                    rc2 = PGM_BTH_NAME_PAE_PAE(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_NESTED:
-                    rc2 = PGM_BTH_NAME_NESTED_PAE(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_EPT:
-                    rc2 = PGM_BTH_NAME_EPT_PAE(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_32_BIT:
-                case PGMMODE_AMD64:
-                case PGMMODE_AMD64_NX:
-                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
-                default: AssertFailed(); break;
-            }
-            break;
-        }
-
-#ifdef VBOX_WITH_64_BITS_GUESTS
-        case PGMMODE_AMD64_NX:
-        case PGMMODE_AMD64:
-            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & UINT64_C(0xfffffffffffff000); /** @todo define this mask! */
-            rc = PGM_GST_NAME_AMD64(Enter)(pVCpu, GCPhysCR3);
-            switch (pVCpu->pgm.s.enmShadowMode)
-            {
-                case PGMMODE_AMD64:
-                case PGMMODE_AMD64_NX:
-                    rc2 = PGM_BTH_NAME_AMD64_AMD64(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_NESTED:
-                    rc2 = PGM_BTH_NAME_NESTED_AMD64(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_EPT:
-                    rc2 = PGM_BTH_NAME_EPT_AMD64(Enter)(pVCpu, GCPhysCR3);
-                    break;
-                case PGMMODE_32_BIT:
-                case PGMMODE_PAE:
-                case PGMMODE_PAE_NX:
-                    AssertMsgFailed(("Should use AMD64 shadow mode!\n"));
-                default: AssertFailed(); break;
-            }
-            break;
-#endif
-
-        default:
-            AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
-            rc = VERR_NOT_IMPLEMENTED;
-            break;
-    }
-
-    /* status codes. */
-    AssertRC(rc);
-    AssertRC(rc2);
-    if (RT_SUCCESS(rc))
-    {
-        rc = rc2;
-        if (RT_SUCCESS(rc)) /* no informational status codes. */
-            rc = VINF_SUCCESS;
-    }
-
-    /* Notify HWACCM as well. */
-    HWACCMR3PagingModeChanged(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
-    return rc;
-}
-
-
-/**
- * Called by pgmPoolFlushAllInt prior to flushing the pool.
- *
- * @returns VBox status code, fully asserted.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU to operate on.
- */
-int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu)
-{
-    /* Unmap the old CR3 value before flushing everything. */
-    int rc = PGM_BTH_PFN(UnmapCR3, pVCpu)(pVCpu);
-    AssertRC(rc);
-
-    /* Exit the current shadow paging mode as well; nested paging and EPT use a root CR3 which will get flushed here. */
-    rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
-    AssertRC(rc);
-    Assert(pVCpu->pgm.s.pShwPageCR3R3 == NULL);
-    return rc;
-}
-
-
-/**
- * Called by pgmPoolFlushAllInt after flushing the pool.
- *
- * @returns VBox status code, fully asserted.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU to operate on.
- */
-int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu)
-{
-    pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
-    int rc = PGMR3ChangeMode(pVM, pVCpu, PGMGetGuestMode(pVCpu));
-    Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
-    AssertRCReturn(rc, rc);
-    AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
-
-    Assert(pVCpu->pgm.s.pShwPageCR3R3 != NULL);
-    AssertMsg(   pVCpu->pgm.s.enmShadowMode >= PGMMODE_NESTED
-              || CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu),
-              ("%RHp != %RHp %s\n", (RTHCPHYS)CPUMGetHyperCR3(pVCpu), PGMGetHyperCR3(pVCpu), PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
-    return rc;
-}
-
-#ifdef VBOX_WITH_DEBUGGER
-
-/**
- * The '.pgmram' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-    if (!pVM->pgm.s.pRamRangesRC)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no Ram is registered.\n");
-
-    /*
-     * Dump the ranges.
-     */
-    int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "From     - To (incl) pvHC\n");
-    PPGMRAMRANGE pRam;
-    for (pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
-    {
-        rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL,
-            "%RGp - %RGp  %p\n",
-            pRam->GCPhys, pRam->GCPhysLast, pRam->pvR3);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * The '.pgmerror' and '.pgmerroroff' commands.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int)  pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-    AssertReturn(cArgs == 0 || (cArgs == 1 && paArgs[0].enmType == DBGCVAR_TYPE_STRING),
-                 pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Hit bug in the parser.\n"));
-
-    if (!cArgs)
-    {
-        /*
-         * Print the list of error injection locations with status.
-         */
-        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "PGM error inject locations:\n");
-        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "  handy - %RTbool\n", pVM->pgm.s.fErrInjHandyPages);
-    }
-    else
-    {
-
-        /*
-         * String switch on where to inject the error.
-         */
-        bool const  fNewState = !strcmp(pCmd->pszCmd, "pgmerror");
-        const char *pszWhere = paArgs[0].u.pszString;
-        if (!strcmp(pszWhere, "handy"))
-            ASMAtomicWriteBool(&pVM->pgm.s.fErrInjHandyPages, fNewState);
-        else
-            return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Invalid 'where' value: %s.\n", pszWhere);
-        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "done\n");
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * The '.pgmsync' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /** @todo SMP support */
-
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    /*
-     * Force page directory sync.
-     */
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-
-    int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Forcing page directory sync.\n");
-    if (RT_FAILURE(rc))
-        return rc;
-
-    return VINF_SUCCESS;
-}
-
-
-#ifdef VBOX_STRICT
-/**
- * The '.pgmassertcr3' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /** @todo SMP support!! */
-
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Checking shadow CR3 page tables for consistency.\n");
-    if (RT_FAILURE(rc))
-        return rc;
-
-    PGMAssertCR3(pVM, pVCpu, CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu));
-
-    return VINF_SUCCESS;
-}
-#endif /* VBOX_STRICT */
-
-
-/**
- * The '.pgmsyncalways' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /** @todo SMP support!! */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-
-    /*
-     * Force page directory sync.
-     */
-    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS)
-    {
-        ASMAtomicAndU32(&pVCpu->pgm.s.fSyncFlags, ~PGM_SYNC_ALWAYS);
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Disabled permanent forced page directory syncing.\n");
-    }
-    else
-    {
-        ASMAtomicOrU32(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_ALWAYS);
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Enabled permanent forced page directory syncing.\n");
-    }
-}
-
-
-/**
- * The '.pgmphystofile' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) pgmR3CmdPhysToFile(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-    if (    cArgs < 1
-        ||  cArgs > 2
-        ||  paArgs[0].enmType != DBGCVAR_TYPE_STRING
-        ||  (   cArgs > 1
-             && paArgs[1].enmType != DBGCVAR_TYPE_STRING))
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: parser error, invalid arguments.\n");
-    if (    cArgs >= 2
-        &&  strcmp(paArgs[1].u.pszString, "nozero"))
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Invalid 2nd argument '%s', must be 'nozero'.\n", paArgs[1].u.pszString);
-    bool fIncZeroPgs = cArgs < 2;
-
-    /*
-     * Open the output file and get the ram parameters.
-     */
-    RTFILE hFile;
-    int rc = RTFileOpen(&hFile, paArgs[0].u.pszString, RTFILE_O_WRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE);
-    if (RT_FAILURE(rc))
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileOpen(,'%s',) -> %Rrc.\n", paArgs[0].u.pszString, rc);
-
-    uint32_t cbRamHole = 0;
-    CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
-    uint64_t cbRam     = 0;
-    CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
-    RTGCPHYS GCPhysEnd = cbRam + cbRamHole;
-
-    /*
-     * Dump the physical memory, page by page.
-     */
-    RTGCPHYS    GCPhys = 0;
-    char        abZeroPg[PAGE_SIZE];
-    RT_ZERO(abZeroPg);
-
-    pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
-          pRam && pRam->GCPhys < GCPhysEnd && RT_SUCCESS(rc);
-          pRam = pRam->pNextR3)
-    {
-        /* fill the gap */
-        if (pRam->GCPhys > GCPhys && fIncZeroPgs)
-        {
-            while (pRam->GCPhys > GCPhys && RT_SUCCESS(rc))
-            {
-                rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
-                GCPhys += PAGE_SIZE;
-            }
-        }
-
-        PCPGMPAGE pPage = &pRam->aPages[0];
-        while (GCPhys < pRam->GCPhysLast && RT_SUCCESS(rc))
-        {
-            if (    PGM_PAGE_IS_ZERO(pPage)
-                ||  PGM_PAGE_IS_BALLOONED(pPage))
-            {
-                if (fIncZeroPgs)
-                {
-                    rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
-                    if (RT_FAILURE(rc))
-                        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
-                }
-            }
-            else
-            {
-                switch (PGM_PAGE_GET_TYPE(pPage))
-                {
-                    case PGMPAGETYPE_RAM:
-                    case PGMPAGETYPE_ROM_SHADOW: /* trouble?? */
-                    case PGMPAGETYPE_ROM:
-                    case PGMPAGETYPE_MMIO2:
-                    {
-                        void const     *pvPage;
-                        PGMPAGEMAPLOCK  Lock;
-                        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
-                        if (RT_SUCCESS(rc))
-                        {
-                            rc = RTFileWrite(hFile, pvPage, PAGE_SIZE, NULL);
-                            PGMPhysReleasePageMappingLock(pVM, &Lock);
-                            if (RT_FAILURE(rc))
-                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
-                        }
-                        else
-                            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: PGMPhysGCPhys2CCPtrReadOnly -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
-                        break;
-                    }
-
-                    default:
-                        AssertFailed();
-                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
-                    case PGMPAGETYPE_MMIO:
-                        if (fIncZeroPgs)
-                        {
-                            rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
-                            if (RT_FAILURE(rc))
-                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
-                        }
-                        break;
-                }
-            }
-
-
-            /* advance */
-            GCPhys += PAGE_SIZE;
-            pPage++;
-        }
-    }
-    pgmUnlock(pVM);
-
-    RTFileClose(hFile);
-    if (RT_SUCCESS(rc))
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Successfully saved physical memory to '%s'.\n", paArgs[0].u.pszString);
-    return VINF_SUCCESS;
-}
-
-#endif /* VBOX_WITH_DEBUGGER */
-
-/**
- * pvUser argument of the pgmR3CheckIntegrity*Node callbacks.
- */
-typedef struct PGMCHECKINTARGS
-{
-    bool                    fLeftToRight;    /**< true: left-to-right; false: right-to-left. */
-    PPGMPHYSHANDLER         pPrevPhys;
-    PPGMVIRTHANDLER         pPrevVirt;
-    PPGMPHYS2VIRTHANDLER    pPrevPhys2Virt;
-    PVM                     pVM;
-} PGMCHECKINTARGS, *PPGMCHECKINTARGS;
-
-/**
- * Validate a node in the physical handler tree.
- *
- * @returns 0 on if ok, other wise 1.
- * @param   pNode       The handler node.
- * @param   pvUser      pVM.
- */
-static DECLCALLBACK(int) pgmR3CheckIntegrityPhysHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
-    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
-    AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
-    AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
-    AssertReleaseMsg(   !pArgs->pPrevPhys
-                     || (pArgs->fLeftToRight ? pArgs->pPrevPhys->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys->Core.KeyLast > pCur->Core.Key),
-                     ("pPrevPhys=%p %RGp-%RGp %s\n"
-                      "     pCur=%p %RGp-%RGp %s\n",
-                      pArgs->pPrevPhys, pArgs->pPrevPhys->Core.Key, pArgs->pPrevPhys->Core.KeyLast, pArgs->pPrevPhys->pszDesc,
-                      pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
-    pArgs->pPrevPhys = pCur;
-    return 0;
-}
-
-
-/**
- * Validate a node in the virtual handler tree.
- *
- * @returns 0 on if ok, other wise 1.
- * @param   pNode       The handler node.
- * @param   pvUser      pVM.
- */
-static DECLCALLBACK(int) pgmR3CheckIntegrityVirtHandlerNode(PAVLROGCPTRNODECORE pNode, void *pvUser)
-{
-    PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
-    PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)pNode;
-    AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
-    AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGv-%RGv %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
-    AssertReleaseMsg(   !pArgs->pPrevVirt
-                     || (pArgs->fLeftToRight ? pArgs->pPrevVirt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevVirt->Core.KeyLast > pCur->Core.Key),
-                     ("pPrevVirt=%p %RGv-%RGv %s\n"
-                      "     pCur=%p %RGv-%RGv %s\n",
-                      pArgs->pPrevVirt, pArgs->pPrevVirt->Core.Key, pArgs->pPrevVirt->Core.KeyLast, pArgs->pPrevVirt->pszDesc,
-                      pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
-    for (unsigned iPage = 0; iPage < pCur->cPages; iPage++)
-    {
-        AssertReleaseMsg(pCur->aPhysToVirt[iPage].offVirtHandler == -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage]),
-                         ("pCur=%p %RGv-%RGv %s\n"
-                          "iPage=%d offVirtHandle=%#x expected %#x\n",
-                          pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc,
-                          iPage, pCur->aPhysToVirt[iPage].offVirtHandler, -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage])));
-    }
-    pArgs->pPrevVirt = pCur;
-    return 0;
-}
-
-
-/**
- * Validate a node in the virtual handler tree.
- *
- * @returns 0 on if ok, other wise 1.
- * @param   pNode       The handler node.
- * @param   pvUser      pVM.
- */
-static DECLCALLBACK(int) pgmR3CheckIntegrityPhysToVirtHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
-    PPGMPHYS2VIRTHANDLER pCur = (PPGMPHYS2VIRTHANDLER)pNode;
-    AssertReleaseMsgReturn(!((uintptr_t)pCur & 3),      ("\n"), 1);
-    AssertReleaseMsgReturn(!(pCur->offVirtHandler & 3), ("\n"), 1);
-    AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp\n", pCur, pCur->Core.Key, pCur->Core.KeyLast));
-    AssertReleaseMsg(   !pArgs->pPrevPhys2Virt
-                     || (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
-                     ("pPrevPhys2Virt=%p %RGp-%RGp\n"
-                      "          pCur=%p %RGp-%RGp\n",
-                      pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
-                      pCur, pCur->Core.Key, pCur->Core.KeyLast));
-    AssertReleaseMsg(   !pArgs->pPrevPhys2Virt
-                     || (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
-                     ("pPrevPhys2Virt=%p %RGp-%RGp\n"
-                      "          pCur=%p %RGp-%RGp\n",
-                      pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
-                      pCur, pCur->Core.Key, pCur->Core.KeyLast));
-    AssertReleaseMsg((pCur->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE | PGMPHYS2VIRTHANDLER_IS_HEAD)) == (PGMPHYS2VIRTHANDLER_IN_TREE | PGMPHYS2VIRTHANDLER_IS_HEAD),
-                     ("pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                      pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
-    if (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
-    {
-        PPGMPHYS2VIRTHANDLER pCur2 = pCur;
-        for (;;)
-        {
-            pCur2 = (PPGMPHYS2VIRTHANDLER)((intptr_t)pCur + (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
-            AssertReleaseMsg(pCur2 != pCur,
-                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
-            AssertReleaseMsg((pCur2->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE | PGMPHYS2VIRTHANDLER_IS_HEAD)) == PGMPHYS2VIRTHANDLER_IN_TREE,
-                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
-                              "pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
-                              pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
-            AssertReleaseMsg((pCur2->Core.Key ^ pCur->Core.Key) < PAGE_SIZE,
-                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
-                              "pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
-                              pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
-            AssertReleaseMsg((pCur2->Core.KeyLast ^ pCur->Core.KeyLast) < PAGE_SIZE,
-                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
-                              "pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
-                              pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
-            if (!(pCur2->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
-                break;
-        }
-    }
-
-    pArgs->pPrevPhys2Virt = pCur;
-    return 0;
-}
-
-
-/**
- * Perform an integrity check on the PGM component.
- *
- * @returns VINF_SUCCESS if everything is fine.
- * @returns VBox error status after asserting on integrity breach.
- * @param   pVM     The VM handle.
- */
-VMMR3DECL(int) PGMR3CheckIntegrity(PVM pVM)
-{
-    AssertReleaseReturn(pVM->pgm.s.offVM, VERR_INTERNAL_ERROR);
-
-    /*
-     * Check the trees.
-     */
-    int cErrors = 0;
-    const static PGMCHECKINTARGS s_LeftToRight = {  true, NULL, NULL, NULL, pVM };
-    const static PGMCHECKINTARGS s_RightToLeft = { false, NULL, NULL, NULL, pVM };
-    PGMCHECKINTARGS Args = s_LeftToRight;
-    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,       true,  pgmR3CheckIntegrityPhysHandlerNode, &Args);
-    Args = s_RightToLeft;
-    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,       false, pgmR3CheckIntegrityPhysHandlerNode, &Args);
-    Args = s_LeftToRight;
-    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers,       true,  pgmR3CheckIntegrityVirtHandlerNode, &Args);
-    Args = s_RightToLeft;
-    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers,       false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
-    Args = s_LeftToRight;
-    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers,  true,  pgmR3CheckIntegrityVirtHandlerNode, &Args);
-    Args = s_RightToLeft;
-    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers,  false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
-    Args = s_LeftToRight;
-    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, true,  pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
-    Args = s_RightToLeft;
-    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, false, pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
-
-    return !cErrors ? VINF_SUCCESS : VERR_INTERNAL_ERROR;
-}
-
Index: unk/src/VBox/VMM/PGMBth.h
===================================================================
--- /trunk/src/VBox/VMM/PGMBth.h	(revision 35332)
+++ 	(revision )
@@ -1,208 +1,0 @@
-/* $Id$ */
-/** @file
- * VBox - Page Manager / Monitor, Shadow+Guest Paging Template.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-RT_C_DECLS_BEGIN
-PGM_BTH_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0);
-PGM_BTH_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
-PGM_BTH_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta);
-
-PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
-PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
-PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uError);
-PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
-PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
-PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
-PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
-PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
-RT_C_DECLS_END
-
-
-/**
- * Initializes the both bit of the paging mode data.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
- *                          This is used early in the init process to avoid trouble with PDM
- *                          not being initialized yet.
- */
-PGM_BTH_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0)
-{
-    Assert(pModeData->uShwType == PGM_SHW_TYPE); Assert(pModeData->uGstType == PGM_GST_TYPE);
-
-    /* Ring 3 */
-    pModeData->pfnR3BthRelocate             = PGM_BTH_NAME(Relocate);
-    pModeData->pfnR3BthSyncCR3              = PGM_BTH_NAME(SyncCR3);
-    pModeData->pfnR3BthInvalidatePage       = PGM_BTH_NAME(InvalidatePage);
-    pModeData->pfnR3BthPrefetchPage         = PGM_BTH_NAME(PrefetchPage);
-    pModeData->pfnR3BthVerifyAccessSyncPage = PGM_BTH_NAME(VerifyAccessSyncPage);
-#ifdef VBOX_STRICT
-    pModeData->pfnR3BthAssertCR3            = PGM_BTH_NAME(AssertCR3);
-#endif
-    pModeData->pfnR3BthMapCR3               = PGM_BTH_NAME(MapCR3);
-    pModeData->pfnR3BthUnmapCR3             = PGM_BTH_NAME(UnmapCR3);
-
-    if (fResolveGCAndR0)
-    {
-        int rc;
-
-#if PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT /* No AMD64 for traditional virtualization, only VT-x and AMD-V. */
-        /* GC */
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(Trap0eHandler),       &pModeData->pfnRCBthTrap0eHandler);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(Trap0eHandler),  rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(InvalidatePage),      &pModeData->pfnRCBthInvalidatePage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(InvalidatePage), rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(SyncCR3),             &pModeData->pfnRCBthSyncCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(SyncCR3), rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(PrefetchPage),        &pModeData->pfnRCBthPrefetchPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(PrefetchPage), rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(VerifyAccessSyncPage),&pModeData->pfnRCBthVerifyAccessSyncPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(VerifyAccessSyncPage), rc), rc);
-# ifdef VBOX_STRICT
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(AssertCR3),           &pModeData->pfnRCBthAssertCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(AssertCR3), rc), rc);
-# endif
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(MapCR3),              &pModeData->pfnRCBthMapCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(MapCR3), rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(UnmapCR3),            &pModeData->pfnRCBthUnmapCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(UnmapCR3), rc), rc);
-#endif /* Not AMD64 shadow paging. */
-
-        /* Ring 0 */
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(Trap0eHandler),       &pModeData->pfnR0BthTrap0eHandler);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(Trap0eHandler),  rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(InvalidatePage),      &pModeData->pfnR0BthInvalidatePage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(InvalidatePage), rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(SyncCR3),             &pModeData->pfnR0BthSyncCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(SyncCR3), rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(PrefetchPage),        &pModeData->pfnR0BthPrefetchPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(PrefetchPage), rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(VerifyAccessSyncPage),&pModeData->pfnR0BthVerifyAccessSyncPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(VerifyAccessSyncPage), rc), rc);
-#ifdef VBOX_STRICT
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(AssertCR3),           &pModeData->pfnR0BthAssertCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(AssertCR3), rc), rc);
-#endif
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(MapCR3),              &pModeData->pfnR0BthMapCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(MapCR3), rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(UnmapCR3),            &pModeData->pfnR0BthUnmapCR3);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(UnmapCR3), rc), rc);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Enters the shadow+guest mode.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   GCPhysCR3   The physical address from the CR3 register.
- */
-PGM_BTH_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
-{
-    /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
-     * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
-     */
-#if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
-          || PGM_SHW_TYPE == PGM_TYPE_PAE    \
-          || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
-      && (   PGM_GST_TYPE == PGM_TYPE_REAL   \
-          || PGM_GST_TYPE == PGM_TYPE_PROT))
-
-    PVM pVM = pVCpu->pVMR3;
-
-    Assert(HWACCMIsNestedPagingActive(pVM) == pVM->pgm.s.fNestedPaging);
-    Assert(!pVM->pgm.s.fNestedPaging);
-
-    pgmLock(pVM);
-    /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
-     *       but any calls to GC need a proper shadow page setup as well.
-     */
-    /* Free the previous root mapping if still active. */
-    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
-    if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
-    {
-        Assert(pVCpu->pgm.s.pShwPageCR3R3->enmKind != PGMPOOLKIND_FREE);
-
-        /* Mark the page as unlocked; allow flushing again. */
-        pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
-
-        /* Remove the hypervisor mappings from the shadow page table. */
-        pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
-
-        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.pShwPageCR3R3, pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
-        pVCpu->pgm.s.pShwPageCR3R3 = 0;
-        pVCpu->pgm.s.pShwPageCR3RC = 0;
-        pVCpu->pgm.s.pShwPageCR3R0 = 0;
-        pVCpu->pgm.s.iShwUser      = 0;
-        pVCpu->pgm.s.iShwUserTable = 0;
-    }
-
-    /* construct a fake address. */
-    GCPhysCR3 = RT_BIT_64(63);
-    pVCpu->pgm.s.iShwUser      = SHW_POOL_ROOT_IDX;
-    pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
-    int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable,
-                          &pVCpu->pgm.s.pShwPageCR3R3);
-    if (rc == VERR_PGM_POOL_FLUSHED)
-    {
-        Log(("Bth-Enter: PGM pool flushed -> signal sync cr3\n"));
-        Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
-        pgmUnlock(pVM);
-        return VINF_PGM_SYNC_CR3;
-    }
-    AssertRCReturn(rc, rc);
-
-    /* Mark the page as locked; disallow flushing. */
-    pgmPoolLockPage(pPool, pVCpu->pgm.s.pShwPageCR3R3);
-
-    pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.pShwPageCR3R3);
-    pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.pShwPageCR3R3);
-
-    /* Set the current hypervisor CR3. */
-    CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
-
-    /* Apply all hypervisor mappings to the new CR3. */
-    rc = pgmMapActivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
-    pgmUnlock(pVM);
-    return rc;
-#else
-    return VINF_SUCCESS;
-#endif
-}
-
-
-/**
- * Relocate any GC pointers related to shadow mode paging.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   offDelta    The relocation offset.
- */
-PGM_BTH_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta)
-{
-    /* nothing special to do here - InitData does the job. */
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/PGMDbg.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGMDbg.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2329 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager and Monitor - Debugger & Debugging APIs.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM
-#include <VBox/pgm.h>
-#include <VBox/stam.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include "PGMInline.h"
-#include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <iprt/string.h>
-#include <VBox/log.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The max needle size that we will bother searching for
- * This must not be more than half a page! */
-#define MAX_NEEDLE_SIZE     256
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * State structure for the paging hierarchy dumpers.
- */
-typedef struct PGMR3DUMPHIERARCHYSTATE
-{
-    /** The VM handle. */
-    PVM             pVM;
-    /** Output helpers. */
-    PCDBGFINFOHLP   pHlp;
-    /** Set if PSE, PAE or long mode is enabled. */
-    bool            fPse;
-    /** Set if PAE or long mode is enabled. */
-    bool            fPae;
-    /** Set if long mode is enabled. */
-    bool            fLme;
-    /** Set if nested paging. */
-    bool            fNp;
-    /** Set if EPT. */
-    bool            fEpt;
-    /** Set if NXE is enabled. */
-    bool            fNxe;
-    /** The number or chars the address needs. */
-    uint8_t         cchAddress;
-    /** The last reserved bit. */
-    uint8_t         uLastRsvdBit;
-    /** Dump the page info as well (shadow page summary / guest physical
-     *  page summary). */
-    bool            fDumpPageInfo;
-    /** Whether or not to print the header. */
-    bool            fPrintHeader;
-    /** Whether to print the CR3 value */
-    bool            fPrintCr3;
-    /** Padding*/
-    bool            afReserved[5];
-    /** The current address.  */
-    uint64_t        u64Address;
-    /** The last address to dump structures for. */
-    uint64_t        u64FirstAddress;
-    /** The last address to dump structures for. */
-    uint64_t        u64LastAddress;
-    /** Mask with the high reserved bits set. */
-    uint64_t        u64HighReservedBits;
-    /** The number of leaf entries that we've printed. */
-    uint64_t        cLeaves;
-} PGMR3DUMPHIERARCHYSTATE;
-/** Pointer to the paging hierarchy dumper state. */
-typedef PGMR3DUMPHIERARCHYSTATE *PPGMR3DUMPHIERARCHYSTATE;
-
-
-
-/**
- * Converts a R3 pointer to a GC physical address.
- *
- * Only for the debugger.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success, *pGCPhys is set.
- * @retval  VERR_INVALID_POINTER if the pointer is not within the GC physical memory.
- *
- * @param   pVM         The VM handle.
- * @param   R3Ptr       The R3 pointer to convert.
- * @param   pGCPhys     Where to store the GC physical address on success.
- */
-VMMR3DECL(int) PGMR3DbgR3Ptr2GCPhys(PVM pVM, RTR3PTR R3Ptr, PRTGCPHYS pGCPhys)
-{
-    *pGCPhys = NIL_RTGCPHYS;
-    return VERR_NOT_IMPLEMENTED;
-}
-
-
-/**
- * Converts a R3 pointer to a HC physical address.
- *
- * Only for the debugger.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success, *pHCPhys is set.
- * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical page but has no physical backing.
- * @retval  VERR_INVALID_POINTER if the pointer is not within the GC physical memory.
- *
- * @param   pVM         The VM handle.
- * @param   R3Ptr       The R3 pointer to convert.
- * @param   pHCPhys     Where to store the HC physical address on success.
- */
-VMMR3DECL(int) PGMR3DbgR3Ptr2HCPhys(PVM pVM, RTR3PTR R3Ptr, PRTHCPHYS pHCPhys)
-{
-    *pHCPhys = NIL_RTHCPHYS;
-    return VERR_NOT_IMPLEMENTED;
-}
-
-
-/**
- * Converts a HC physical address to a GC physical address.
- *
- * Only for the debugger.
- *
- * @returns VBox status code
- * @retval  VINF_SUCCESS on success, *pGCPhys is set.
- * @retval  VERR_INVALID_POINTER if the HC physical address is not within the GC physical memory.
- *
- * @param   pVM     The VM handle.
- * @param   HCPhys  The HC physical address to convert.
- * @param   pGCPhys Where to store the GC physical address on success.
- */
-VMMR3DECL(int) PGMR3DbgHCPhys2GCPhys(PVM pVM, RTHCPHYS HCPhys, PRTGCPHYS pGCPhys)
-{
-    /*
-     * Validate and adjust the input a bit.
-     */
-    if (HCPhys == NIL_RTHCPHYS)
-        return VERR_INVALID_POINTER;
-    unsigned off = HCPhys & PAGE_OFFSET_MASK;
-    HCPhys &= X86_PTE_PAE_PG_MASK;
-    if (HCPhys == 0)
-        return VERR_INVALID_POINTER;
-
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-         pRam;
-         pRam = pRam->CTX_SUFF(pNext))
-    {
-        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
-        while (iPage-- > 0)
-            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
-            {
-                *pGCPhys = pRam->GCPhys + (iPage << PAGE_SHIFT) + off;
-                return VINF_SUCCESS;
-            }
-    }
-    return VERR_INVALID_POINTER;
-}
-
-
-/**
- * Read physical memory API for the debugger, similar to
- * PGMPhysSimpleReadGCPhys.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle.
- * @param   pvDst       Where to store what's read.
- * @param   GCPhysDst   Where to start reading from.
- * @param   cb          The number of bytes to attempt reading.
- * @param   fFlags      Flags, MBZ.
- * @param   pcbRead     For store the actual number of bytes read, pass NULL if
- *                      partial reads are unwanted.
- * @todo    Unused?
- */
-VMMR3DECL(int) PGMR3DbgReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb, uint32_t fFlags, size_t *pcbRead)
-{
-    /* validate */
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
-    AssertReturn(pVM, VERR_INVALID_PARAMETER);
-
-    /* try simple first. */
-    int rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc, cb);
-    if (RT_SUCCESS(rc) || !pcbRead)
-        return rc;
-
-    /* partial read that failed, chop it up in pages. */
-    *pcbRead = 0;
-    size_t const cbReq = cb;
-    rc = VINF_SUCCESS;
-    while (cb > 0)
-    {
-        size_t cbChunk = PAGE_SIZE;
-        cbChunk -= GCPhysSrc & PAGE_OFFSET_MASK;
-        if (cbChunk > cb)
-            cbChunk = cb;
-
-        rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc, cbChunk);
-
-        /* advance */
-        if (RT_FAILURE(rc))
-            break;
-        *pcbRead  += cbChunk;
-        cb        -= cbChunk;
-        GCPhysSrc += cbChunk;
-        pvDst = (uint8_t *)pvDst + cbChunk;
-    }
-
-    return *pcbRead && RT_FAILURE(rc) ? -rc : rc;
-}
-
-
-/**
- * Write physical memory API for the debugger, similar to
- * PGMPhysSimpleWriteGCPhys.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle.
- * @param   GCPhysDst   Where to start writing.
- * @param   pvSrc       What to write.
- * @param   cb          The number of bytes to attempt writing.
- * @param   fFlags      Flags, MBZ.
- * @param   pcbWritten  For store the actual number of bytes written, pass NULL
- *                      if partial writes are unwanted.
- * @todo    Unused?
- */
-VMMR3DECL(int) PGMR3DbgWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb, uint32_t fFlags, size_t *pcbWritten)
-{
-    /* validate */
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
-    AssertReturn(pVM, VERR_INVALID_PARAMETER);
-
-    /* try simple first. */
-    int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysDst, pvSrc, cb);
-    if (RT_SUCCESS(rc) || !pcbWritten)
-        return rc;
-
-    /* partial write that failed, chop it up in pages. */
-    *pcbWritten = 0;
-    rc = VINF_SUCCESS;
-    while (cb > 0)
-    {
-        size_t cbChunk = PAGE_SIZE;
-        cbChunk -= GCPhysDst & PAGE_OFFSET_MASK;
-        if (cbChunk > cb)
-            cbChunk = cb;
-
-        rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysDst, pvSrc, cbChunk);
-
-        /* advance */
-        if (RT_FAILURE(rc))
-            break;
-        *pcbWritten += cbChunk;
-        cb          -= cbChunk;
-        GCPhysDst   += cbChunk;
-        pvSrc = (uint8_t const *)pvSrc + cbChunk;
-    }
-
-    return *pcbWritten && RT_FAILURE(rc) ? -rc : rc;
-
-}
-
-
-/**
- * Read virtual memory API for the debugger, similar to PGMPhysSimpleReadGCPtr.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle.
- * @param   pvDst       Where to store what's read.
- * @param   GCPtrDst    Where to start reading from.
- * @param   cb          The number of bytes to attempt reading.
- * @param   fFlags      Flags, MBZ.
- * @param   pcbRead     For store the actual number of bytes read, pass NULL if
- *                      partial reads are unwanted.
- * @todo    Unused?
- */
-VMMR3DECL(int) PGMR3DbgReadGCPtr(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, uint32_t fFlags, size_t *pcbRead)
-{
-    /* validate */
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
-    AssertReturn(pVM, VERR_INVALID_PARAMETER);
-
-    /* @todo SMP support! */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-/** @todo deal with HMA */
-    /* try simple first. */
-    int rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCPtrSrc, cb);
-    if (RT_SUCCESS(rc) || !pcbRead)
-        return rc;
-
-    /* partial read that failed, chop it up in pages. */
-    *pcbRead = 0;
-    rc = VINF_SUCCESS;
-    while (cb > 0)
-    {
-        size_t cbChunk = PAGE_SIZE;
-        cbChunk -= GCPtrSrc & PAGE_OFFSET_MASK;
-        if (cbChunk > cb)
-            cbChunk = cb;
-
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCPtrSrc, cbChunk);
-
-        /* advance */
-        if (RT_FAILURE(rc))
-            break;
-        *pcbRead  += cbChunk;
-        cb        -= cbChunk;
-        GCPtrSrc  += cbChunk;
-        pvDst = (uint8_t *)pvDst + cbChunk;
-    }
-
-    return *pcbRead && RT_FAILURE(rc) ? -rc : rc;
-
-}
-
-
-/**
- * Write virtual memory API for the debugger, similar to
- * PGMPhysSimpleWriteGCPtr.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle.
- * @param   GCPtrDst    Where to start writing.
- * @param   pvSrc       What to write.
- * @param   cb          The number of bytes to attempt writing.
- * @param   fFlags      Flags, MBZ.
- * @param   pcbWritten  For store the actual number of bytes written, pass NULL
- *                      if partial writes are unwanted.
- * @todo    Unused?
- */
-VMMR3DECL(int) PGMR3DbgWriteGCPtr(PVM pVM, RTGCPTR GCPtrDst, void const *pvSrc, size_t cb, uint32_t fFlags, size_t *pcbWritten)
-{
-    /* validate */
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
-    AssertReturn(pVM, VERR_INVALID_PARAMETER);
-
-    /* @todo SMP support! */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-/** @todo deal with HMA */
-    /* try simple first. */
-    int rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCPtrDst, pvSrc, cb);
-    if (RT_SUCCESS(rc) || !pcbWritten)
-        return rc;
-
-    /* partial write that failed, chop it up in pages. */
-    *pcbWritten = 0;
-    rc = VINF_SUCCESS;
-    while (cb > 0)
-    {
-        size_t cbChunk = PAGE_SIZE;
-        cbChunk -= GCPtrDst & PAGE_OFFSET_MASK;
-        if (cbChunk > cb)
-            cbChunk = cb;
-
-        rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCPtrDst, pvSrc, cbChunk);
-
-        /* advance */
-        if (RT_FAILURE(rc))
-            break;
-        *pcbWritten += cbChunk;
-        cb          -= cbChunk;
-        GCPtrDst    += cbChunk;
-        pvSrc = (uint8_t const *)pvSrc + cbChunk;
-    }
-
-    return *pcbWritten && RT_FAILURE(rc) ? -rc : rc;
-
-}
-
-
-/**
- * memchr() with alignment considerations.
- *
- * @returns Pointer to matching byte, NULL if none found.
- * @param   pb                  Where to search. Aligned.
- * @param   b                   What to search for.
- * @param   cb                  How much to search .
- * @param   uAlign              The alignment restriction of the result.
- */
-static const uint8_t *pgmR3DbgAlignedMemChr(const uint8_t *pb, uint8_t b, size_t cb, uint32_t uAlign)
-{
-    const uint8_t *pbRet;
-    if (uAlign <= 32)
-    {
-        pbRet = (const uint8_t *)memchr(pb, b, cb);
-        if ((uintptr_t)pbRet & (uAlign - 1))
-        {
-            do
-            {
-                pbRet++;
-                size_t cbLeft = cb - (pbRet - pb);
-                if (!cbLeft)
-                {
-                    pbRet = NULL;
-                    break;
-                }
-                pbRet = (const uint8_t *)memchr(pbRet, b, cbLeft);
-            } while ((uintptr_t)pbRet & (uAlign - 1));
-        }
-    }
-    else
-    {
-        pbRet = NULL;
-        if (cb)
-        {
-            for (;;)
-            {
-                if (*pb == b)
-                {
-                    pbRet = pb;
-                    break;
-                }
-                if (cb <= uAlign)
-                    break;
-                cb -= uAlign;
-                pb += uAlign;
-            }
-        }
-    }
-    return pbRet;
-}
-
-
-/**
- * Scans a page for a byte string, keeping track of potential
- * cross page matches.
- *
- * @returns true and *poff on match.
- *          false on mismatch.
- * @param   pbPage          Pointer to the current page.
- * @param   poff            Input: The offset into the page (aligned).
- *                          Output: The page offset of the match on success.
- * @param   cb              The number of bytes to search, starting of *poff.
- * @param   uAlign          The needle alignment. This is of course less than a page.
- * @param   pabNeedle       The byte string to search for.
- * @param   cbNeedle        The length of the byte string.
- * @param   pabPrev         The buffer that keeps track of a partial match that we
- *                          bring over from the previous page. This buffer must be
- *                          at least cbNeedle - 1 big.
- * @param   pcbPrev         Input: The number of partial matching bytes from the previous page.
- *                          Output: The number of partial matching bytes from this page.
- *                          Initialize to 0 before the first call to this function.
- */
-static bool pgmR3DbgScanPage(const uint8_t *pbPage, int32_t *poff, uint32_t cb, uint32_t uAlign,
-                             const uint8_t *pabNeedle, size_t cbNeedle,
-                             uint8_t *pabPrev, size_t *pcbPrev)
-{
-    /*
-     * Try complete any partial match from the previous page.
-     */
-    if (*pcbPrev > 0)
-    {
-        size_t cbPrev = *pcbPrev;
-        Assert(!*poff);
-        Assert(cbPrev < cbNeedle);
-        if (!memcmp(pbPage, pabNeedle + cbPrev, cbNeedle - cbPrev))
-        {
-            if (cbNeedle - cbPrev > cb)
-                return false;
-            *poff = -(int32_t)cbPrev;
-            return true;
-        }
-
-        /* check out the remainder of the previous page. */
-        const uint8_t *pb = pabPrev;
-        for (;;)
-        {
-            if (cbPrev <= uAlign)
-                break;
-            cbPrev -= uAlign;
-            pb = pgmR3DbgAlignedMemChr(pb + uAlign, *pabNeedle, cbPrev, uAlign);
-            if (!pb)
-                break;
-            cbPrev = *pcbPrev - (pb - pabPrev);
-            if (    !memcmp(pb + 1, &pabNeedle[1], cbPrev - 1)
-                &&  !memcmp(pbPage, pabNeedle + cbPrev, cbNeedle - cbPrev))
-            {
-                if (cbNeedle - cbPrev > cb)
-                    return false;
-                *poff = -(int32_t)cbPrev;
-                return true;
-            }
-        }
-
-        *pcbPrev = 0;
-    }
-
-    /*
-     * Match the body of the page.
-     */
-    const uint8_t *pb = pbPage + *poff;
-    const uint8_t *pbEnd = pb + cb;
-    for (;;)
-    {
-        pb = pgmR3DbgAlignedMemChr(pb, *pabNeedle, cb, uAlign);
-        if (!pb)
-            break;
-        cb = pbEnd - pb;
-        if (cb >= cbNeedle)
-        {
-            /* match? */
-            if (!memcmp(pb + 1, &pabNeedle[1], cbNeedle - 1))
-            {
-                *poff = pb - pbPage;
-                return true;
-            }
-        }
-        else
-        {
-            /* partial match at the end of the page? */
-            if (!memcmp(pb + 1, &pabNeedle[1], cb - 1))
-            {
-                /* We're copying one byte more that we really need here, but wtf. */
-                memcpy(pabPrev, pb, cb);
-                *pcbPrev = cb;
-                return false;
-            }
-        }
-
-        /* no match, skip ahead. */
-        if (cb <= uAlign)
-            break;
-        pb += uAlign;
-        cb -= uAlign;
-    }
-
-    return false;
-}
-
-
-/**
- * Scans guest physical memory for a byte string.
- *
- * @returns VBox status codes:
- * @retval  VINF_SUCCESS and *pGCPtrHit on success.
- * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
- * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
- * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   GCPhys          Where to start searching.
- * @param   cbRange         The number of bytes to search.
- * @param   GCPhysAlign     The alignment of the needle. Must be a power of two
- *                          and less or equal to 4GB.
- * @param   pabNeedle       The byte string to search for.
- * @param   cbNeedle        The length of the byte string. Max 256 bytes.
- * @param   pGCPhysHit      Where to store the address of the first occurrence on success.
- */
-VMMR3DECL(int) PGMR3DbgScanPhysical(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cbRange, RTGCPHYS GCPhysAlign,
-                                    const uint8_t *pabNeedle, size_t cbNeedle, PRTGCPHYS pGCPhysHit)
-{
-    /*
-     * Validate and adjust the input a bit.
-     */
-    if (!VALID_PTR(pGCPhysHit))
-        return VERR_INVALID_POINTER;
-    *pGCPhysHit = NIL_RTGCPHYS;
-
-    if (    !VALID_PTR(pabNeedle)
-        ||  GCPhys == NIL_RTGCPHYS)
-        return VERR_INVALID_POINTER;
-    if (!cbNeedle)
-        return VERR_INVALID_PARAMETER;
-    if (cbNeedle > MAX_NEEDLE_SIZE)
-        return VERR_INVALID_PARAMETER;
-
-    if (!cbRange)
-        return VERR_DBGF_MEM_NOT_FOUND;
-    if (GCPhys + cbNeedle - 1 < GCPhys)
-        return VERR_DBGF_MEM_NOT_FOUND;
-
-    if (!GCPhysAlign)
-        return VERR_INVALID_PARAMETER;
-    if (GCPhysAlign > UINT32_MAX)
-        return VERR_NOT_POWER_OF_TWO;
-    if (GCPhysAlign & (GCPhysAlign - 1))
-        return VERR_INVALID_PARAMETER;
-
-    if (GCPhys & (GCPhysAlign - 1))
-    {
-        RTGCPHYS Adj = GCPhysAlign - (GCPhys & (GCPhysAlign - 1));
-        if (    cbRange <= Adj
-            ||  GCPhys + Adj < GCPhys)
-            return VERR_DBGF_MEM_NOT_FOUND;
-        GCPhys  += Adj;
-        cbRange -= Adj;
-    }
-
-    const bool      fAllZero   = ASMMemIsAll8(pabNeedle, cbNeedle, 0) == NULL;
-    const uint32_t  cIncPages  = GCPhysAlign <= PAGE_SIZE
-                               ? 1
-                               : GCPhysAlign >> PAGE_SHIFT;
-    const RTGCPHYS  GCPhysLast = GCPhys + cbRange - 1 >= GCPhys
-                               ? GCPhys + cbRange - 1
-                               : ~(RTGCPHYS)0;
-
-    /*
-     * Search the memory - ignore MMIO and zero pages, also don't
-     * bother to match across ranges.
-     */
-    pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-         pRam;
-         pRam = pRam->CTX_SUFF(pNext))
-    {
-        /*
-         * If the search range starts prior to the current ram range record,
-         * adjust the search range and possibly conclude the search.
-         */
-        RTGCPHYS off;
-        if (GCPhys < pRam->GCPhys)
-        {
-            if (GCPhysLast < pRam->GCPhys)
-                break;
-            GCPhys = pRam->GCPhys;
-            off = 0;
-        }
-        else
-            off = GCPhys - pRam->GCPhys;
-        if (off < pRam->cb)
-        {
-            /*
-             * Iterate the relevant pages.
-             */
-            uint8_t         abPrev[MAX_NEEDLE_SIZE];
-            size_t          cbPrev   = 0;
-            const uint32_t  cPages   = pRam->cb >> PAGE_SHIFT;
-            uint32_t        iPage    = off >> PAGE_SHIFT;
-            uint32_t        offPage  = GCPhys & PAGE_OFFSET_MASK;
-            GCPhys &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
-            for (;; offPage = 0)
-            {
-                PPGMPAGE pPage = &pRam->aPages[iPage];
-                if (    (   !PGM_PAGE_IS_ZERO(pPage)
-                         || fAllZero)
-                    &&  !PGM_PAGE_IS_BALLOONED(pPage)
-                    &&  !PGM_PAGE_IS_MMIO(pPage))
-                {
-                    void const     *pvPage;
-                    PGMPAGEMAPLOCK  Lock;
-                    int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
-                    if (RT_SUCCESS(rc))
-                    {
-                        int32_t     offHit = offPage;
-                        bool        fRc;
-                        if (GCPhysAlign < PAGE_SIZE)
-                        {
-                            uint32_t cbSearch = (GCPhys ^ GCPhysLast) & ~(RTGCPHYS)PAGE_OFFSET_MASK
-                                              ? PAGE_SIZE                           - (uint32_t)offPage
-                                              : (GCPhysLast & PAGE_OFFSET_MASK) + 1 - (uint32_t)offPage;
-                            fRc = pgmR3DbgScanPage((uint8_t const *)pvPage, &offHit, cbSearch, (uint32_t)GCPhysAlign,
-                                                   pabNeedle, cbNeedle, &abPrev[0], &cbPrev);
-                        }
-                        else
-                            fRc = memcmp(pvPage, pabNeedle, cbNeedle) == 0
-                               && (GCPhysLast - GCPhys) >= cbNeedle;
-                        PGMPhysReleasePageMappingLock(pVM, &Lock);
-                        if (fRc)
-                        {
-                            *pGCPhysHit = GCPhys + offHit;
-                            pgmUnlock(pVM);
-                            return VINF_SUCCESS;
-                        }
-                    }
-                    else
-                        cbPrev = 0; /* ignore error. */
-                }
-                else
-                    cbPrev = 0;
-
-                /* advance to the next page. */
-                GCPhys += (RTGCPHYS)cIncPages << PAGE_SHIFT;
-                if (GCPhys >= GCPhysLast) /* (may not always hit, but we're run out of ranges.) */
-                {
-                    pgmUnlock(pVM);
-                    return VERR_DBGF_MEM_NOT_FOUND;
-                }
-                iPage += cIncPages;
-                if (    iPage < cIncPages
-                    ||  iPage >= cPages)
-                    break;
-            }
-        }
-    }
-    pgmUnlock(pVM);
-    return VERR_DBGF_MEM_NOT_FOUND;
-}
-
-
-/**
- * Scans (guest) virtual memory for a byte string.
- *
- * @returns VBox status codes:
- * @retval  VINF_SUCCESS and *pGCPtrHit on success.
- * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
- * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
- * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pVCpu           The CPU context to search in.
- * @param   GCPtr           Where to start searching.
- * @param   GCPtrAlign      The alignment of the needle. Must be a power of two
- *                          and less or equal to 4GB.
- * @param   cbRange         The number of bytes to search. Max 256 bytes.
- * @param   pabNeedle       The byte string to search for.
- * @param   cbNeedle        The length of the byte string.
- * @param   pGCPtrHit       Where to store the address of the first occurrence on success.
- */
-VMMR3DECL(int) PGMR3DbgScanVirtual(PVM pVM, PVMCPU pVCpu, RTGCPTR GCPtr, RTGCPTR cbRange, RTGCPTR GCPtrAlign,
-                                   const uint8_t *pabNeedle, size_t cbNeedle, PRTGCUINTPTR pGCPtrHit)
-{
-    VMCPU_ASSERT_EMT(pVCpu);
-
-    /*
-     * Validate and adjust the input a bit.
-     */
-    if (!VALID_PTR(pGCPtrHit))
-        return VERR_INVALID_POINTER;
-    *pGCPtrHit = 0;
-
-    if (!VALID_PTR(pabNeedle))
-        return VERR_INVALID_POINTER;
-    if (!cbNeedle)
-        return VERR_INVALID_PARAMETER;
-    if (cbNeedle > MAX_NEEDLE_SIZE)
-        return VERR_INVALID_PARAMETER;
-
-    if (!cbRange)
-        return VERR_DBGF_MEM_NOT_FOUND;
-    if (GCPtr + cbNeedle - 1 < GCPtr)
-        return VERR_DBGF_MEM_NOT_FOUND;
-
-    if (!GCPtrAlign)
-        return VERR_INVALID_PARAMETER;
-    if (GCPtrAlign > UINT32_MAX)
-        return VERR_NOT_POWER_OF_TWO;
-    if (GCPtrAlign & (GCPtrAlign - 1))
-        return VERR_INVALID_PARAMETER;
-
-    if (GCPtr & (GCPtrAlign - 1))
-    {
-        RTGCPTR Adj = GCPtrAlign - (GCPtr & (GCPtrAlign - 1));
-        if (    cbRange <= Adj
-            ||  GCPtr + Adj < GCPtr)
-            return VERR_DBGF_MEM_NOT_FOUND;
-        GCPtr   += Adj;
-        cbRange -= Adj;
-    }
-
-    /*
-     * Search the memory - ignore MMIO, zero and not-present pages.
-     */
-    const bool      fAllZero  = ASMMemIsAll8(pabNeedle, cbNeedle, 0) == NULL;
-    PGMMODE         enmMode   = PGMGetGuestMode(pVCpu);
-    RTGCPTR         GCPtrMask = PGMMODE_IS_LONG_MODE(enmMode) ? UINT64_MAX : UINT32_MAX;
-    uint8_t         abPrev[MAX_NEEDLE_SIZE];
-    size_t          cbPrev    = 0;
-    const uint32_t  cIncPages = GCPtrAlign <= PAGE_SIZE
-                              ? 1
-                              : GCPtrAlign >> PAGE_SHIFT;
-    const RTGCPTR   GCPtrLast = GCPtr + cbRange - 1 >= GCPtr
-                              ? (GCPtr + cbRange - 1) & GCPtrMask
-                              : GCPtrMask;
-    RTGCPTR         cPages    = (((GCPtrLast - GCPtr) + (GCPtr & PAGE_OFFSET_MASK)) >> PAGE_SHIFT) + 1;
-    uint32_t        offPage   = GCPtr & PAGE_OFFSET_MASK;
-    GCPtr &= ~(RTGCPTR)PAGE_OFFSET_MASK;
-    for (;; offPage = 0)
-    {
-        RTGCPHYS GCPhys;
-        int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
-        if (RT_SUCCESS(rc))
-        {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
-            if (    pPage
-                &&  (   !PGM_PAGE_IS_ZERO(pPage)
-                     || fAllZero)
-                &&  !PGM_PAGE_IS_BALLOONED(pPage)
-                &&  !PGM_PAGE_IS_MMIO(pPage))
-            {
-                void const *pvPage;
-                PGMPAGEMAPLOCK Lock;
-                rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
-                if (RT_SUCCESS(rc))
-                {
-                    int32_t offHit = offPage;
-                    bool    fRc;
-                    if (GCPtrAlign < PAGE_SIZE)
-                    {
-                        uint32_t cbSearch = cPages > 0
-                                          ? PAGE_SIZE                          - (uint32_t)offPage
-                                          : (GCPtrLast & PAGE_OFFSET_MASK) + 1 - (uint32_t)offPage;
-                        fRc = pgmR3DbgScanPage((uint8_t const *)pvPage, &offHit, cbSearch, (uint32_t)GCPtrAlign,
-                                               pabNeedle, cbNeedle, &abPrev[0], &cbPrev);
-                    }
-                    else
-                        fRc = memcmp(pvPage, pabNeedle, cbNeedle) == 0
-                           && (GCPtrLast - GCPtr) >= cbNeedle;
-                    PGMPhysReleasePageMappingLock(pVM, &Lock);
-                    if (fRc)
-                    {
-                        *pGCPtrHit = GCPtr + offHit;
-                        return VINF_SUCCESS;
-                    }
-                }
-                else
-                    cbPrev = 0; /* ignore error. */
-            }
-            else
-                cbPrev = 0;
-        }
-        else
-            cbPrev = 0; /* ignore error. */
-
-        /* advance to the next page. */
-        if (cPages <= cIncPages)
-            break;
-        cPages -= cIncPages;
-        GCPtr += (RTGCPTR)cIncPages << PAGE_SHIFT;
-    }
-    return VERR_DBGF_MEM_NOT_FOUND;
-}
-
-
-/**
- * Initializes the dumper state.
- *
- * @param   pState          The state to initialize.
- * @param   pVM             The VM handle.
- * @param   fFlags          The flags.
- * @param   u64FirstAddr    The first address.
- * @param   u64LastAddr     The last address.
- * @param   pHlp            The output helpers.
- */
-static void pgmR3DumpHierarchyInitState(PPGMR3DUMPHIERARCHYSTATE pState, PVM pVM, uint32_t fFlags,
-                                        uint64_t u64FirstAddr, uint64_t u64LastAddr, PCDBGFINFOHLP pHlp)
-{
-    pState->pVM                 = pVM;
-    pState->pHlp                = pHlp ? pHlp : DBGFR3InfoLogHlp();
-    pState->fPse                = !!(fFlags & (DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME));
-    pState->fPae                = !!(fFlags & (DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME));
-    pState->fLme                = !!(fFlags & DBGFPGDMP_FLAGS_LME);
-    pState->fNp                 = !!(fFlags & DBGFPGDMP_FLAGS_NP);
-    pState->fEpt                = !!(fFlags & DBGFPGDMP_FLAGS_EPT);
-    pState->fNxe                = !!(fFlags & DBGFPGDMP_FLAGS_NXE);
-    pState->cchAddress          = pState->fLme ? 16 : 8;
-    pState->uLastRsvdBit        = pState->fNxe ? 62 : 63;
-    pState->fDumpPageInfo       = !!(fFlags & DBGFPGDMP_FLAGS_PAGE_INFO);
-    pState->fPrintHeader        = !!(fFlags & DBGFPGDMP_FLAGS_HEADER);
-    pState->fPrintCr3           = !!(fFlags & DBGFPGDMP_FLAGS_PRINT_CR3);
-    pState->afReserved[0]       = false;
-    pState->afReserved[1]       = false;
-    pState->afReserved[2]       = false;
-    pState->afReserved[3]       = false;
-    pState->afReserved[4]       = false;
-    pState->u64Address          = u64FirstAddr;
-    pState->u64FirstAddress     = u64FirstAddr;
-    pState->u64LastAddress      = u64LastAddr;
-    pState->u64HighReservedBits = pState->uLastRsvdBit == 62 ? UINT64_C(0x7ff) << 52 : UINT64_C(0xfff) << 52;
-    pState->cLeaves             = 0;
-}
-
-
-/**
- * The simple way out, too tired to think of a more elegant solution.
- *
- * @returns The base address of this page table/directory/whatever.
- * @param   pState              The state where we get the current address.
- * @param   cShift              The shift count for the table entries.
- * @param   cEntries            The number of table entries.
- * @param   piFirst             Where to return the table index of the first
- *                              entry to dump.
- * @param   piLast              Where to return the table index of the last
- *                              entry.
- */
-static uint64_t pgmR3DumpHierarchyCalcRange(PPGMR3DUMPHIERARCHYSTATE pState, uint32_t cShift, uint32_t cEntries,
-                                            uint32_t *piFirst, uint32_t *piLast)
-{
-    const uint64_t iBase  = (pState->u64Address     >> cShift) & ~(uint64_t)(cEntries - 1);
-    const uint64_t iFirst = pState->u64FirstAddress >> cShift;
-    const uint64_t iLast  = pState->u64LastAddress  >> cShift;
-
-    if (   iBase                >= iFirst
-        && iBase + cEntries - 1 <= iLast)
-    {
-        /* full range. */
-        *piFirst = 0;
-        *piLast  = cEntries - 1;
-    }
-    else if (   iBase + cEntries - 1 < iFirst
-             || iBase                > iLast)
-    {
-        /* no match */
-        *piFirst = cEntries;
-        *piLast  = 0;
-    }
-    else
-    {
-        /* partial overlap */
-        *piFirst = iBase <= iFirst
-                 ? iFirst - iBase
-                 : 0;
-        *piLast  = iBase + cEntries - 1 <= iLast
-                 ? cEntries - 1
-                 : iLast - iBase;
-    }
-
-    return iBase << cShift;
-}
-
-
-/**
- * Maps/finds the shadow page.
- *
- * @returns VBox status code.
- * @param   pState              The dumper state.
- * @param   HCPhys              The physical address of the shadow page.
- * @param   pszDesc             The description.
- * @param   fIsMapping          Set if it's a mapping.
- * @param   ppv                 Where to return the pointer.
- */
-static int pgmR3DumpHierarchyShwMapPage(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, const char *pszDesc,
-                                        bool fIsMapping, void const **ppv)
-{
-    void *pvPage;
-    if (!fIsMapping)
-    {
-        int rc = MMPagePhys2PageTry(pState->pVM, HCPhys, &pvPage);
-        if (RT_FAILURE(rc))
-        {
-            pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! %s at HCPhys=%RHp was not found in the page pool!\n",
-                                    pState->cchAddress, pState->u64Address, pszDesc, HCPhys);
-            return rc;
-        }
-    }
-    else
-    {
-        pvPage = NULL;
-        for (PPGMMAPPING pMap = pState->pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
-        {
-            uint64_t off = pState->u64Address - pMap->GCPtr;
-            if (off < pMap->cb)
-            {
-                const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
-                const int iSub = (int)((off >> X86_PD_PAE_SHIFT) & 1); /* MSC is a pain sometimes */
-                if ((iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0) != HCPhys)
-                    pState->pHlp->pfnPrintf(pState->pHlp,
-                                            "%0*llx error! Mapping error! PT %d has HCPhysPT=%RHp not %RHp is in the PD.\n",
-                                            pState->cchAddress, pState->u64Address, iPDE,
-                                            iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0, HCPhys);
-                pvPage = &pMap->aPTs[iPDE].paPaePTsR3[iSub];
-                break;
-            }
-        }
-        if (!pvPage)
-        {
-            pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! PT mapping %s at HCPhys=%RHp was not found in the page pool!\n",
-                                    pState->cchAddress, pState->u64Address, HCPhys);
-            return VERR_INVALID_PARAMETER;
-        }
-    }
-    *ppv = pvPage;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dumps the a shadow page summary or smth.
- *
- * @param   pState              The dumper state.
- * @param   HCPhys              The page address.
- */
-static void pgmR3DumpHierarchyShwTablePageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys)
-{
-    pgmLock(pState->pVM);
-    char            szPage[80];
-    PPGMPOOLPAGE    pPage = pgmPoolQueryPageForDbg(pState->pVM->pgm.s.CTX_SUFF(pPool), HCPhys);
-    if (pPage)
-        RTStrPrintf(szPage, sizeof(szPage), " idx=0i%u", pPage->idx);
-    else
-    {
-        /* probably a mapping */
-        strcpy(szPage, " not found");
-        for (PPGMMAPPING pMap = pState->pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
-        {
-            uint64_t off = pState->u64Address - pMap->GCPtr;
-            if (off < pMap->cb)
-            {
-                const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
-                if (pMap->aPTs[iPDE].HCPhysPT == HCPhys)
-                    RTStrPrintf(szPage, sizeof(szPage), " #%u: %s", iPDE, pMap->pszDesc);
-                else if (pMap->aPTs[iPDE].HCPhysPaePT0 == HCPhys)
-                    RTStrPrintf(szPage, sizeof(szPage), " #%u/0: %s", iPDE, pMap->pszDesc);
-                else if (pMap->aPTs[iPDE].HCPhysPaePT1 == HCPhys)
-                    RTStrPrintf(szPage, sizeof(szPage), " #%u/1: %s", iPDE, pMap->pszDesc);
-                else
-                    continue;
-                break;
-            }
-        }
-    }
-    pgmUnlock(pState->pVM);
-    pState->pHlp->pfnPrintf(pState->pHlp, "%s", szPage);
-}
-
-
-/**
- * Figures out which guest page this is and dumps a summary.
- *
- * @param   pState              The dumper state.
- * @param   HCPhys              The page address.
- * @param   cbPage              The page size.
- */
-static void pgmR3DumpHierarchyShwGuestPageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, uint32_t cbPage)
-{
-    char        szPage[80];
-    RTGCPHYS    GCPhys;
-    int rc = PGMR3DbgHCPhys2GCPhys(pState->pVM, HCPhys, &GCPhys);
-    if (RT_SUCCESS(rc))
-    {
-        pgmLock(pState->pVM);
-        PCPGMPAGE pPage = pgmPhysGetPage(&pState->pVM->pgm.s, GCPhys);
-        if (pPage)
-            RTStrPrintf(szPage, sizeof(szPage), "%R[pgmpage]", pPage);
-        else
-            strcpy(szPage, "not found");
-        pgmUnlock(pState->pVM);
-        pState->pHlp->pfnPrintf(pState->pHlp, " -> %RGp %s", GCPhys, szPage);
-    }
-    else
-    {
-        /* check the heap */
-        uint32_t cbAlloc;
-        rc = MMR3HyperQueryInfoFromHCPhys(pState->pVM, HCPhys, szPage, sizeof(szPage), &cbAlloc);
-        if (RT_SUCCESS(rc))
-            pState->pHlp->pfnPrintf(pState->pHlp, " %s %#x bytes", szPage, cbAlloc);
-        else
-            pState->pHlp->pfnPrintf(pState->pHlp, " not found");
-    }
-}
-
-
-/**
- * Dumps a PAE shadow page table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState              The dumper state.
- * @param   HCPhys              The page table address.
- * @param   fIsMapping          Whether it is a mapping.
- */
-static int pgmR3DumpHierarchyShwPaePT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, bool fIsMapping)
-{
-    PCPGMSHWPTPAE pPT;
-    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page table", fIsMapping, (void const **)&pPT);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-        if (PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_P)
-        {
-            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PT_PAE_SHIFT);
-            if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
-            {
-                X86PTEPAE Pte;
-                Pte.u = PGMSHWPTEPAE_GET_U(pPT->a[i]);
-                pState->pHlp->pfnPrintf(pState->pHlp,
-                                        pState->fLme  /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
-                                        ? "%016llx 3    | P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx"
-                                        :  "%08llx 2   |  P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pte.n.u1Write       ? 'W'  : 'R',
-                                        Pte.n.u1User        ? 'U'  : 'S',
-                                        Pte.n.u1Accessed    ? 'A'  : '-',
-                                        Pte.n.u1Dirty       ? 'D'  : '-',
-                                        Pte.n.u1Global      ? 'G'  : '-',
-                                        Pte.n.u1WriteThru   ? "WT" : "--",
-                                        Pte.n.u1CacheDisable? "CD" : "--",
-                                        Pte.n.u1PAT         ? "AT" : "--",
-                                        Pte.n.u1NoExecute   ? "NX" : "--",
-                                        Pte.u & PGM_PTFLAGS_TRACK_DIRTY   ? 'd' : '-',
-                                        Pte.u & RT_BIT(10)                ? '1' : '0',
-                                        Pte.u & PGM_PTFLAGS_CSAM_VALIDATED? 'v' : '-',
-                                        Pte.u & X86_PTE_PAE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyShwGuestPageInfo(pState, Pte.u & X86_PTE_PAE_PG_MASK, _4K);
-                if ((Pte.u >> 52) & 0x7ff)
-                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx%s", (Pte.u >> 52) & 0x7ff, pState->fLme ? "" : "!");
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-            }
-            else if (   (PGMSHWPTEPAE_GET_U(pPT->a[i]) & (pState->pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX))
-                     ==                                  (pState->pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX))
-                pState->pHlp->pfnPrintf(pState->pHlp,
-                                        pState->fLme
-                                        ? "%016llx 3    | invalid / MMIO optimization\n"
-                                        :  "%08llx 2   |  invalid / MMIO optimization\n",
-                                        pState->u64Address);
-            else
-                pState->pHlp->pfnPrintf(pState->pHlp,
-                                        pState->fLme
-                                        ? "%016llx 3    | invalid: %RX64\n"
-                                        :  "%08llx 2   |  invalid: %RX64\n",
-                                        pState->u64Address, PGMSHWPTEPAE_GET_U(pPT->a[i]));
-            pState->cLeaves++;
-        }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dumps a PAE shadow page directory table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState      The dumper state.
- * @param   HCPhys      The physical address of the page directory table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int  pgmR3DumpHierarchyShwPaePD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
-{
-    PCX86PDPAE pPD;
-    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory", false, (void const **)&pPD);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth > 0);
-    cMaxDepth--;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PDEPAE Pde = pPD->a[i];
-        if (Pde.n.u1Present)
-        {
-            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PD_PAE_SHIFT);
-            if (Pde.b.u1Size)
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,
-                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 2M a  p ?  phys*/
-                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx"
-                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pde.b.u1Write       ? 'W'  : 'R',
-                                        Pde.b.u1User        ? 'U'  : 'S',
-                                        Pde.b.u1Accessed    ? 'A'  : '-',
-                                        Pde.b.u1Dirty       ? 'D'  : '-',
-                                        Pde.b.u1Global      ? 'G'  : '-',
-                                        Pde.b.u1WriteThru   ? "WT" : "--",
-                                        Pde.b.u1CacheDisable? "CD" : "--",
-                                        Pde.b.u1PAT         ? "AT" : "--",
-                                        Pde.b.u1NoExecute   ? "NX" : "--",
-                                        Pde.u & RT_BIT_64(9)            ? '1' : '0',
-                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
-                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
-                                        Pde.u & X86_PDE2M_PAE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyShwGuestPageInfo(pState, Pde.u & X86_PDE2M_PAE_PG_MASK, _2M);
-                if ((Pde.u >> 52) & 0x7ff)
-                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx%s", (Pde.u >> 52) & 0x7ff, pState->fLme ? "" : "!");
-                if ((Pde.u >> 13) & 0xff)
-                    pState->pHlp->pfnPrintf(pState->pHlp, " 20:13=%02llx%s", (Pde.u >> 13) & 0x0ff, pState->fLme ? "" : "!");
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-                pState->cLeaves++;
-            }
-            else
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,
-                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 4M a  p ?  phys */
-                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx"
-                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pde.n.u1Write       ? 'W'  : 'R',
-                                        Pde.n.u1User        ? 'U'  : 'S',
-                                        Pde.n.u1Accessed    ? 'A'  : '-',
-                                        Pde.n.u1Reserved0   ? '?'  : '.', /* ignored */
-                                        Pde.n.u1Reserved1   ? '?'  : '.', /* ignored */
-                                        Pde.n.u1WriteThru   ? "WT" : "--",
-                                        Pde.n.u1CacheDisable? "CD" : "--",
-                                        Pde.n.u1NoExecute   ? "NX" : "--",
-                                        Pde.u & RT_BIT_64(9)            ? '1' : '0',
-                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
-                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
-                                        Pde.u & X86_PDE_PAE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pde.u & X86_PDE_PAE_PG_MASK);
-                if ((Pde.u >> 52) & 0x7ff)
-                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx!", (Pde.u >> 52) & 0x7ff);
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-                if (cMaxDepth)
-                {
-                    int rc2 = pgmR3DumpHierarchyShwPaePT(pState, Pde.u & X86_PDE_PAE_PG_MASK, !!(Pde.u & PGM_PDFLAGS_MAPPING));
-                    if (rc2 < rc && RT_SUCCESS(rc))
-                        rc = rc2;
-                }
-                else
-                    pState->cLeaves++;
-            }
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Dumps a PAE shadow page directory pointer table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState      The dumper state.
- * @param   HCPhys      The physical address of the page directory pointer table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int  pgmR3DumpHierarchyShwPaePDPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
-{
-    /* Fend of addresses that are out of range in PAE mode - simplifies the code below. */
-    if (!pState->fLme && pState->u64Address >= _4G)
-        return VINF_SUCCESS;
-
-    PCX86PDPT pPDPT;
-    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory pointer table", false, (void const **)&pPDPT);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth > 0);
-    cMaxDepth--;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PDPT_SHIFT,
-                                                          pState->fLme ? X86_PG_AMD64_PDPE_ENTRIES : X86_PG_PAE_PDPE_ENTRIES,
-                                                          &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PDPE Pdpe = pPDPT->a[i];
-        if (Pdpe.n.u1Present)
-        {
-            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PDPT_SHIFT);
-            if (pState->fLme)
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX .. a p ?  */
-                                        "%016llx 1  |   P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pdpe.lm.u1Write       ? 'W'  : 'R',
-                                        Pdpe.lm.u1User        ? 'U'  : 'S',
-                                        Pdpe.lm.u1Accessed    ? 'A'  : '-',
-                                        Pdpe.lm.u3Reserved & 1? '?'  : '.', /* ignored */
-                                        Pdpe.lm.u3Reserved & 4? '!'  : '.', /* mbz */
-                                        Pdpe.lm.u1WriteThru   ? "WT" : "--",
-                                        Pdpe.lm.u1CacheDisable? "CD" : "--",
-                                        Pdpe.lm.u3Reserved & 2? "!"  : "..",/* mbz */
-                                        Pdpe.lm.u1NoExecute   ? "NX" : "--",
-                                        Pdpe.u & RT_BIT(9)                ? '1' : '0',
-                                        Pdpe.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
-                                        Pdpe.u & RT_BIT(11)               ? '1' : '0',
-                                        Pdpe.u & X86_PDPE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK);
-                if ((Pdpe.u >> 52) & 0x7ff)
-                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx", (Pdpe.u >> 52) & 0x7ff);
-            }
-            else
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX .. a p ?  */
-                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pdpe.n.u2Reserved & 1? '!'  : '.', /* mbz */
-                                        Pdpe.n.u2Reserved & 2? '!'  : '.', /* mbz */
-                                        Pdpe.n.u4Reserved & 1? '!'  : '.', /* mbz */
-                                        Pdpe.n.u4Reserved & 2? '!'  : '.', /* mbz */
-                                        Pdpe.n.u4Reserved & 8? '!'  : '.', /* mbz */
-                                        Pdpe.n.u1WriteThru   ? "WT" : "--",
-                                        Pdpe.n.u1CacheDisable? "CD" : "--",
-                                        Pdpe.n.u4Reserved & 2? "!"  : "..",/* mbz */
-                                        Pdpe.lm.u1NoExecute  ? "!!"  : "..",/* mbz */
-                                        Pdpe.u & RT_BIT(9)                ? '1' : '0',
-                                        Pdpe.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
-                                        Pdpe.u & RT_BIT(11)               ? '1' : '0',
-                                        Pdpe.u & X86_PDPE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK);
-                if ((Pdpe.u >> 52) & 0xfff)
-                    pState->pHlp->pfnPrintf(pState->pHlp, " 63:52=%03llx!", (Pdpe.u >> 52) & 0xfff);
-            }
-            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-            if (cMaxDepth)
-            {
-                int rc2 = pgmR3DumpHierarchyShwPaePD(pState, Pdpe.u & X86_PDPE_PG_MASK, cMaxDepth);
-                if (rc2 < rc && RT_SUCCESS(rc))
-                    rc = rc2;
-            }
-            else
-                pState->cLeaves++;
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Dumps a 32-bit shadow page table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pVM         The VM handle.
- * @param   HCPhys      The physical address of the table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int pgmR3DumpHierarchyShwPaePML4(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
-{
-    PCX86PML4 pPML4;
-    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page map level 4", false, (void const **)&pPML4);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth);
-    cMaxDepth--;
-
-    /*
-     * This is a bit tricky as we're working on unsigned addresses while the
-     * AMD64 spec uses signed tricks.
-     */
-    uint32_t iFirst = (pState->u64FirstAddress >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    uint32_t iLast  = (pState->u64LastAddress  >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    if (   pState->u64LastAddress  <= UINT64_C(0x00007fffffffffff)
-        || pState->u64FirstAddress >= UINT64_C(0xffff800000000000))
-    { /* Simple, nothing to adjust */ }
-    else if (pState->u64FirstAddress <= UINT64_C(0x00007fffffffffff))
-        iLast = X86_PG_AMD64_ENTRIES / 2 - 1;
-    else if (pState->u64LastAddress  >= UINT64_C(0xffff800000000000))
-        iFirst = X86_PG_AMD64_ENTRIES / 2;
-    else
-        iFirst = X86_PG_AMD64_ENTRIES; /* neither address is canonical */
-
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PML4E Pml4e = pPML4->a[i];
-        if (Pml4e.n.u1Present)
-        {
-            pState->u64Address = ((uint64_t)i << X86_PML4_SHIFT)
-                               | (i >= RT_ELEMENTS(pPML4->a) / 2 ? UINT64_C(0xffff000000000000) : 0);
-            pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
-                                    "%016llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
-                                    pState->u64Address,
-                                    Pml4e.n.u1Write       ? 'W'  : 'R',
-                                    Pml4e.n.u1User        ? 'U'  : 'S',
-                                    Pml4e.n.u1Accessed    ? 'A'  : '-',
-                                    Pml4e.n.u3Reserved & 1? '?'  : '.', /* ignored */
-                                    Pml4e.n.u3Reserved & 4? '!'  : '.', /* mbz */
-                                    Pml4e.n.u1WriteThru   ? "WT" : "--",
-                                    Pml4e.n.u1CacheDisable? "CD" : "--",
-                                    Pml4e.n.u3Reserved & 2? "!"  : "..",/* mbz */
-                                    Pml4e.n.u1NoExecute   ? "NX" : "--",
-                                    Pml4e.u & RT_BIT(9)                ? '1' : '0',
-                                    Pml4e.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
-                                    Pml4e.u & RT_BIT(11)               ? '1' : '0',
-                                    Pml4e.u & X86_PML4E_PG_MASK);
-            if (pState->fDumpPageInfo)
-                pgmR3DumpHierarchyShwTablePageInfo(pState, Pml4e.u & X86_PML4E_PG_MASK);
-            if ((Pml4e.u >> 52) & 0x7ff)
-                pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx!", (Pml4e.u >> 52) & 0x7ff);
-            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-            if (cMaxDepth)
-            {
-                int rc2 = pgmR3DumpHierarchyShwPaePDPT(pState, Pml4e.u & X86_PML4E_PG_MASK, cMaxDepth);
-                if (rc2 < rc && RT_SUCCESS(rc))
-                    rc = rc2;
-            }
-            else
-                pState->cLeaves++;
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Dumps a 32-bit shadow page table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pVM         The VM handle.
- * @param   pPT         Pointer to the page table.
- * @param   fMapping    Set if it's a guest mapping.
- */
-static int pgmR3DumpHierarchyShw32BitPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, bool fMapping)
-{
-    PCX86PT pPT;
-    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page table", fMapping, (void const **)&pPT);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PTE Pte = pPT->a[i];
-        if (Pte.n.u1Present)
-        {
-            pState->u64Address = u64BaseAddress + (i << X86_PT_SHIFT);
-            pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d  */
-                                    "%08llx 1  |   P %c %c %c %c %c %s %s %s .. 4K %c%c%c  %08x",
-                                    pState->u64Address,
-                                    Pte.n.u1Write       ? 'W'  : 'R',
-                                    Pte.n.u1User        ? 'U'  : 'S',
-                                    Pte.n.u1Accessed    ? 'A'  : '-',
-                                    Pte.n.u1Dirty       ? 'D'  : '-',
-                                    Pte.n.u1Global      ? 'G'  : '-',
-                                    Pte.n.u1WriteThru   ? "WT" : "--",
-                                    Pte.n.u1CacheDisable? "CD" : "--",
-                                    Pte.n.u1PAT         ? "AT" : "--",
-                                    Pte.u & PGM_PTFLAGS_TRACK_DIRTY     ? 'd' : '-',
-                                    Pte.u & RT_BIT(10)                  ? '1' : '0',
-                                    Pte.u & PGM_PTFLAGS_CSAM_VALIDATED  ? 'v' : '-',
-                                    Pte.u & X86_PDE_PG_MASK);
-            if (pState->fDumpPageInfo)
-                pgmR3DumpHierarchyShwGuestPageInfo(pState, Pte.u & X86_PDE_PG_MASK, _4K);
-            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-        }
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dumps a 32-bit shadow page directory and page tables.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState      The dumper state.
- * @param   HCPhys      The physical address of the table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int pgmR3DumpHierarchyShw32BitPD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
-{
-    if (pState->u64Address >= _4G)
-        return VINF_SUCCESS;
-
-    PCX86PD pPD;
-    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory", false, (void const **)&pPD);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth > 0);
-    cMaxDepth--;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PDE Pde = pPD->a[i];
-        if (Pde.n.u1Present)
-        {
-            pState->u64Address = (uint32_t)i << X86_PD_SHIFT;
-            if (Pde.b.u1Size && pState->fPse)
-            {
-                uint64_t u64Phys = ((uint64_t)(Pde.u & X86_PDE4M_PG_HIGH_MASK) << X86_PDE4M_PG_HIGH_SHIFT)
-                                 | (Pde.u & X86_PDE4M_PG_MASK);
-                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
-                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s .. 4M %c%c%c  %08llx",
-                                        pState->u64Address,
-                                        Pde.b.u1Write       ? 'W'  : 'R',
-                                        Pde.b.u1User        ? 'U'  : 'S',
-                                        Pde.b.u1Accessed    ? 'A'  : '-',
-                                        Pde.b.u1Dirty       ? 'D'  : '-',
-                                        Pde.b.u1Global      ? 'G'  : '-',
-                                        Pde.b.u1WriteThru   ? "WT" : "--",
-                                        Pde.b.u1CacheDisable? "CD" : "--",
-                                        Pde.b.u1PAT         ? "AT" : "--",
-                                        Pde.u & RT_BIT_32(9)            ? '1' : '0',
-                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
-                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
-                                        u64Phys);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyShwGuestPageInfo(pState, u64Phys, _4M);
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-                pState->cLeaves++;
-            }
-            else
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
-                                        "%08llx 0 |    P %c %c %c %c %c %s %s .. .. 4K %c%c%c  %08x",
-                                        pState->u64Address,
-                                        Pde.n.u1Write       ? 'W'  : 'R',
-                                        Pde.n.u1User        ? 'U'  : 'S',
-                                        Pde.n.u1Accessed    ? 'A'  : '-',
-                                        Pde.n.u1Reserved0   ? '?'  : '.', /* ignored */
-                                        Pde.n.u1Reserved1   ? '?'  : '.', /* ignored */
-                                        Pde.n.u1WriteThru   ? "WT" : "--",
-                                        Pde.n.u1CacheDisable? "CD" : "--",
-                                        Pde.u & RT_BIT_32(9)            ? '1' : '0',
-                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
-                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
-                                        Pde.u & X86_PDE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pde.u & X86_PDE_PG_MASK);
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-                if (cMaxDepth)
-                {
-                    int rc2 = pgmR3DumpHierarchyShw32BitPT(pState, Pde.u & X86_PDE_PG_MASK, !!(Pde.u & PGM_PDFLAGS_MAPPING));
-                    if (rc2 < rc && RT_SUCCESS(rc))
-                        rc = rc2;
-                }
-                else
-                    pState->cLeaves++;
-            }
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * Internal worker that initiates the actual dump.
- *
- * @returns VBox status code.
- * @param   pState              The dumper state.
- * @param   cr3                 The CR3 value.
- * @param   cMaxDepth           The max depth.
- */
-static int pgmR3DumpHierarchyShwDoIt(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t cr3, unsigned cMaxDepth)
-{
-    int             rc;
-    unsigned const  cch     = pState->cchAddress;
-    uint64_t const  cr3Mask = pState->fEpt ? X86_CR3_AMD64_PAGE_MASK
-                            : pState->fLme ? X86_CR3_AMD64_PAGE_MASK
-                            : pState->fPae ? X86_CR3_PAE_PAGE_MASK
-                            :                X86_CR3_PAGE_MASK;
-    if (pState->fPrintCr3)
-    {
-        const char * const  pszMode = pState->fEpt ? "Extended Page Tables"
-                                    : pState->fLme ? "Long Mode"
-                                    : pState->fPae ? "PAE Mode"
-                                    : pState->fPse ? "32-bit w/ PSE"
-                                    :                "32-bit";
-        pState->pHlp->pfnPrintf(pState->pHlp, "cr3=%0*llx", cch, cr3);
-        if (pState->fDumpPageInfo)
-            pgmR3DumpHierarchyShwTablePageInfo(pState, cr3 & X86_CR3_AMD64_PAGE_MASK);
-        pState->pHlp->pfnPrintf(pState->pHlp, " %s%s\n",
-                                pszMode,
-                                pState->fNp  ? " + Nested Paging" : "",
-                                pState->fNxe ? " + NX" : "");
-    }
-
-
-    if (pState->fEpt)
-    {
-        if (pState->fPrintHeader)
-            pState->pHlp->pfnPrintf(pState->pHlp,
-                                    "%-*s        R - Readable\n"
-                                    "%-*s        | W - Writeable\n"
-                                    "%-*s        | | X - Executable\n"
-                                    "%-*s        | | | EMT - EPT memory type\n"
-                                    "%-*s        | | | |   PAT - Ignored PAT?\n"
-                                    "%-*s        | | | |   |    AVL1 - 4 available bits\n"
-                                    "%-*s        | | | |   |    |   AVL2 - 12 available bits\n"
-                                    "%-*s Level  | | | |   |    |    |     page  \n"
-                                  /* xxxx n **** R W X EMT PAT AVL1 AVL2   xxxxxxxxxxxxx
-                                                 R W X  7   0   f   fff    0123456701234567 */
-                                    ,
-                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
-
-        pState->pHlp->pfnPrintf(pState->pHlp, "EPT dumping is not yet implemented, sorry.\n");
-        /** @todo implemented EPT dumping. */
-        rc = VERR_NOT_IMPLEMENTED;
-    }
-    else
-    {
-        if (pState->fPrintHeader)
-            pState->pHlp->pfnPrintf(pState->pHlp,
-                                    "%-*s        P - Present\n"
-                                    "%-*s        | R/W - Read (0) / Write (1)\n"
-                                    "%-*s        | | U/S - User (1) / Supervisor (0)\n"
-                                    "%-*s        | | | A - Accessed\n"
-                                    "%-*s        | | | | D - Dirty\n"
-                                    "%-*s        | | | | | G - Global\n"
-                                    "%-*s        | | | | | | WT - Write thru\n"
-                                    "%-*s        | | | | | | |  CD - Cache disable\n"
-                                    "%-*s        | | | | | | |  |  AT - Attribute table (PAT)\n"
-                                    "%-*s        | | | | | | |  |  |  NX - No execute (K8)\n"
-                                    "%-*s        | | | | | | |  |  |  |  4K/4M/2M - Page size.\n"
-                                    "%-*s        | | | | | | |  |  |  |  |  AVL - a=allocated; m=mapping; d=track dirty;\n"
-                                    "%-*s        | | | | | | |  |  |  |  |  |     p=permanent; v=validated;\n"
-                                    "%-*s Level  | | | | | | |  |  |  |  |  |    Page\n"
-                                  /* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
-                                                 - W U - - - -- -- -- -- -- 010 */
-                                    ,
-                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
-                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
-        if (pState->fLme)
-            rc = pgmR3DumpHierarchyShwPaePML4(pState, cr3 & cr3Mask, cMaxDepth);
-        else if (pState->fPae)
-            rc = pgmR3DumpHierarchyShwPaePDPT(pState, cr3 & cr3Mask, cMaxDepth);
-        else
-            rc = pgmR3DumpHierarchyShw32BitPD(pState, cr3 & cr3Mask, cMaxDepth);
-    }
-
-    if (!pState->cLeaves)
-        pState->pHlp->pfnPrintf(pState->pHlp, "not present\n");
-    return rc;
-}
-
-
-/**
- * dbgfR3PagingDumpEx worker.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   cr3             The CR3 register value.
- * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
- * @param   u64FirstAddr    The start address.
- * @param   u64LastAddr     The address to stop after.
- * @param   cMaxDepth       The max depth.
- * @param   pHlp            The output callbacks.  Defaults to log if NULL.
- *
- * @internal
- */
-VMMR3_INT_DECL(int) PGMR3DumpHierarchyShw(PVM pVM, uint64_t cr3, uint32_t fFlags, uint64_t u64FirstAddr, uint64_t u64LastAddr,
-                                          uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
-{
-    /* Minimal validation as we're only supposed to service DBGF. */
-    AssertReturn(~(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(!(fFlags & (DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3)), VERR_INVALID_PARAMETER);
-    AssertReturn(fFlags & DBGFPGDMP_FLAGS_SHADOW, VERR_INVALID_PARAMETER);
-
-    PGMR3DUMPHIERARCHYSTATE State;
-    pgmR3DumpHierarchyInitState(&State, pVM, fFlags, u64FirstAddr, u64LastAddr, pHlp);
-    return pgmR3DumpHierarchyShwDoIt(&State, cr3, cMaxDepth);
-}
-
-
-/**
- * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pVM         The VM handle.
- * @param   cr3         The root of the hierarchy.
- * @param   cr4         The cr4, only PAE and PSE is currently used.
- * @param   fLongMode   Set if long mode, false if not long mode.
- * @param   cMaxDepth   Number of levels to dump.
- * @param   pHlp        Pointer to the output functions.
- *
- * @deprecated Use DBGFR3PagingDumpEx.
- */
-VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint64_t cr3, uint64_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
-{
-    if (!cMaxDepth)
-        return VINF_SUCCESS;
-
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    if (!pVCpu)
-        pVCpu = &pVM->aCpus[0];
-
-    uint32_t fFlags = DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3 | DBGFPGDMP_FLAGS_PAGE_INFO | DBGFPGDMP_FLAGS_SHADOW;
-    fFlags |= cr4 & (X86_CR4_PAE | X86_CR4_PSE);
-    if (fLongMode)
-        fFlags |= DBGFPGDMP_FLAGS_LME;
-
-    return DBGFR3PagingDumpEx(pVM, pVCpu->idCpu, fFlags, cr3, 0, fLongMode ? UINT64_MAX : UINT32_MAX, cMaxDepth, pHlp);
-}
-
-
-/**
- * Maps the guest page.
- *
- * @returns VBox status code.
- * @param   pState              The dumper state.
- * @param   GCPhys              The physical address of the guest page.
- * @param   pszDesc             The description.
- * @param   ppv                 Where to return the pointer.
- * @param   pLock               Where to return the mapping lock.  Hand this to
- *                              PGMPhysReleasePageMappingLock when done.
- */
-static int pgmR3DumpHierarchyGstMapPage(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, const char *pszDesc,
-                                        void const **ppv, PPGMPAGEMAPLOCK pLock)
-{
-    int rc = PGMPhysGCPhys2CCPtrReadOnly(pState->pVM, GCPhys, ppv, pLock);
-    if (RT_FAILURE(rc))
-    {
-        pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! Failed to map %s at GCPhys=%RGp: %Rrc!\n",
-                                pState->cchAddress, pState->u64Address, pszDesc, GCPhys, rc);
-        return rc;
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Figures out which guest page this is and dumps a summary.
- *
- * @param   pState              The dumper state.
- * @param   GCPhys              The page address.
- * @param   cbPage              The page size.
- */
-static void pgmR3DumpHierarchyGstPageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, uint32_t cbPage)
-{
-    char szPage[80];
-    pgmLock(pState->pVM);
-    PCPGMPAGE pPage = pgmPhysGetPage(&pState->pVM->pgm.s, GCPhys);
-    if (pPage)
-        RTStrPrintf(szPage, sizeof(szPage), " %R[pgmpage]", pPage);
-    else
-        strcpy(szPage, " not found");
-    pgmUnlock(pState->pVM);
-    pState->pHlp->pfnPrintf(pState->pHlp, "%s", szPage);
-}
-
-
-/**
- * Checks the entry for reserved bits.
- *
- * @param   pState              The dumper state.
- * @param   u64Entry            The entry to check.
- */
-static void pgmR3DumpHierarchyGstCheckReservedHighBits(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t u64Entry)
-{
-    uint32_t uRsvd = (u64Entry & pState->u64HighReservedBits) >> 52;
-    if (uRsvd)
-        pState->pHlp->pfnPrintf(pState->pHlp, " %u:52=%03x%s",
-                                pState->uLastRsvdBit, uRsvd, pState->fLme ? "" : "!");
-    /** @todo check the valid physical bits as well. */
-}
-
-
-/**
- * Dumps a PAE shadow page table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState              The dumper state.
- * @param   GCPhys              The page table address.
- */
-static int pgmR3DumpHierarchyGstPaePT(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys)
-{
-    PCX86PTPAE      pPT;
-    PGMPAGEMAPLOCK  Lock;
-    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page table", (void const **)&pPT, &Lock);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PTEPAE Pte = pPT->a[i];
-        if (Pte.n.u1Present)
-        {
-            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PT_PAE_SHIFT);
-            pState->pHlp->pfnPrintf(pState->pHlp,
-                                    pState->fLme  /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
-                                    ? "%016llx 3    | P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx"
-                                    :  "%08llx 2   |  P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx",
-                                    pState->u64Address,
-                                    Pte.n.u1Write       ? 'W'  : 'R',
-                                    Pte.n.u1User        ? 'U'  : 'S',
-                                    Pte.n.u1Accessed    ? 'A'  : '-',
-                                    Pte.n.u1Dirty       ? 'D'  : '-',
-                                    Pte.n.u1Global      ? 'G'  : '-',
-                                    Pte.n.u1WriteThru   ? "WT" : "--",
-                                    Pte.n.u1CacheDisable? "CD" : "--",
-                                    Pte.n.u1PAT         ? "AT" : "--",
-                                    Pte.n.u1NoExecute   ? "NX" : "--",
-                                    Pte.u & RT_BIT(9)   ? '1' : '0',
-                                    Pte.u & RT_BIT(10)  ? '1' : '0',
-                                    Pte.u & RT_BIT(11)  ? '1' : '0',
-                                    Pte.u & X86_PTE_PAE_PG_MASK);
-            if (pState->fDumpPageInfo)
-                pgmR3DumpHierarchyGstPageInfo(pState, Pte.u & X86_PTE_PAE_PG_MASK, _4K);
-            pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pte.u);
-            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-            pState->cLeaves++;
-        }
-    }
-
-    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dumps a PAE shadow page directory table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState      The dumper state.
- * @param   GCPhys      The physical address of the table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int  pgmR3DumpHierarchyGstPaePD(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, unsigned cMaxDepth)
-{
-    PCX86PDPAE      pPD;
-    PGMPAGEMAPLOCK  Lock;
-    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory", (void const **)&pPD, &Lock);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth > 0);
-    cMaxDepth--;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PDEPAE Pde = pPD->a[i];
-        if (Pde.n.u1Present)
-        {
-            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PD_PAE_SHIFT);
-            if (Pde.b.u1Size)
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,
-                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 2M a  p ?  phys*/
-                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx"
-                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pde.b.u1Write         ? 'W'  : 'R',
-                                        Pde.b.u1User          ? 'U'  : 'S',
-                                        Pde.b.u1Accessed      ? 'A'  : '-',
-                                        Pde.b.u1Dirty         ? 'D'  : '-',
-                                        Pde.b.u1Global        ? 'G'  : '-',
-                                        Pde.b.u1WriteThru     ? "WT" : "--",
-                                        Pde.b.u1CacheDisable  ? "CD" : "--",
-                                        Pde.b.u1PAT           ? "AT" : "--",
-                                        Pde.b.u1NoExecute     ? "NX" : "--",
-                                        Pde.u & RT_BIT_64(9)  ? '1' : '0',
-                                        Pde.u & RT_BIT_64(10) ? '1' : '0',
-                                        Pde.u & RT_BIT_64(11) ? '1' : '0',
-                                        Pde.u & X86_PDE2M_PAE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE2M_PAE_PG_MASK, _2M);
-                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pde.u);
-                if ((Pde.u >> 13) & 0xff)
-                    pState->pHlp->pfnPrintf(pState->pHlp, " 20:13=%02llx%s", (Pde.u >> 13) & 0x0ff, pState->fLme ? "" : "!");
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-                pState->cLeaves++;
-            }
-            else
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,
-                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 4M a  p ?  phys */
-                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx"
-                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pde.n.u1Write         ? 'W'  : 'R',
-                                        Pde.n.u1User          ? 'U'  : 'S',
-                                        Pde.n.u1Accessed      ? 'A'  : '-',
-                                        Pde.n.u1Reserved0     ? '?'  : '.', /* ignored */
-                                        Pde.n.u1Reserved1     ? '?'  : '.', /* ignored */
-                                        Pde.n.u1WriteThru     ? "WT" : "--",
-                                        Pde.n.u1CacheDisable  ? "CD" : "--",
-                                        Pde.n.u1NoExecute     ? "NX" : "--",
-                                        Pde.u & RT_BIT_64(9)  ? '1' : '0',
-                                        Pde.u & RT_BIT_64(10) ? '1' : '0',
-                                        Pde.u & RT_BIT_64(11) ? '1' : '0',
-                                        Pde.u & X86_PDE_PAE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE_PAE_PG_MASK, _4K);
-                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pde.u);
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-                if (cMaxDepth)
-                {
-                    int rc2 = pgmR3DumpHierarchyGstPaePT(pState, Pde.u & X86_PDE_PAE_PG_MASK);
-                    if (rc2 < rc && RT_SUCCESS(rc))
-                        rc = rc2;
-                }
-                else
-                    pState->cLeaves++;
-            }
-        }
-    }
-
-    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
-    return rc;
-}
-
-
-/**
- * Dumps a PAE shadow page directory pointer table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState      The dumper state.
- * @param   GCPhys      The physical address of the table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int  pgmR3DumpHierarchyGstPaePDPT(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, unsigned cMaxDepth)
-{
-    /* Fend of addresses that are out of range in PAE mode - simplifies the code below. */
-    if (!pState->fLme && pState->u64Address >= _4G)
-        return VINF_SUCCESS;
-
-    PCX86PDPT       pPDPT;
-    PGMPAGEMAPLOCK  Lock;
-    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory pointer table", (void const **)&pPDPT, &Lock);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth > 0);
-    cMaxDepth--;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PDPT_SHIFT,
-                                                          pState->fLme ? X86_PG_AMD64_PDPE_ENTRIES : X86_PG_PAE_PDPE_ENTRIES,
-                                                          &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PDPE Pdpe = pPDPT->a[i];
-        if (Pdpe.n.u1Present)
-        {
-            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PDPT_SHIFT);
-            if (pState->fLme)
-            {
-                /** @todo Do 1G pages.  */
-                pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX .. a p ?  */
-                                        "%016llx 1  |   P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pdpe.lm.u1Write        ? 'W'  : 'R',
-                                        Pdpe.lm.u1User         ? 'U'  : 'S',
-                                        Pdpe.lm.u1Accessed     ? 'A'  : '-',
-                                        Pdpe.lm.u3Reserved & 1 ? '?'  : '.', /* ignored */
-                                        Pdpe.lm.u3Reserved & 4 ? '!'  : '.', /* mbz */
-                                        Pdpe.lm.u1WriteThru    ? "WT" : "--",
-                                        Pdpe.lm.u1CacheDisable ? "CD" : "--",
-                                        Pdpe.lm.u3Reserved & 2 ? "!"  : "..",/* mbz */
-                                        Pdpe.lm.u1NoExecute    ? "NX" : "--",
-                                        Pdpe.u & RT_BIT_64(9)  ? '1' : '0',
-                                        Pdpe.u & RT_BIT_64(10) ? '1' : '0',
-                                        Pdpe.u & RT_BIT_64(11) ? '1' : '0',
-                                        Pdpe.u & X86_PDPE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyGstPageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK, _4K);
-                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pdpe.u);
-            }
-            else
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX .. a p ?  */
-                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
-                                        pState->u64Address,
-                                        Pdpe.n.u2Reserved & 1  ? '!'  : '.',  /* mbz */
-                                        Pdpe.n.u2Reserved & 2  ? '!'  : '.',  /* mbz */
-                                        Pdpe.n.u4Reserved & 1  ? '!'  : '.',  /* mbz */
-                                        Pdpe.n.u4Reserved & 2  ? '!'  : '.',  /* mbz */
-                                        Pdpe.n.u4Reserved & 8  ? '!'  : '.',  /* mbz */
-                                        Pdpe.n.u1WriteThru     ? "WT" : "--",
-                                        Pdpe.n.u1CacheDisable  ? "CD" : "--",
-                                        Pdpe.n.u4Reserved & 2  ? "!"  : "..", /* mbz */
-                                        Pdpe.lm.u1NoExecute    ? "!!"  : "..",/* mbz */
-                                        Pdpe.u & RT_BIT_64(9)  ? '1' : '0',
-                                        Pdpe.u & RT_BIT_64(10) ? '1' : '0',
-                                        Pdpe.u & RT_BIT_64(11) ? '1' : '0',
-                                        Pdpe.u & X86_PDPE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyGstPageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK, _4K);
-                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pdpe.u);
-            }
-            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-            if (cMaxDepth)
-            {
-                int rc2 = pgmR3DumpHierarchyGstPaePD(pState, Pdpe.u & X86_PDPE_PG_MASK, cMaxDepth);
-                if (rc2 < rc && RT_SUCCESS(rc))
-                    rc = rc2;
-            }
-            else
-                pState->cLeaves++;
-        }
-    }
-
-    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
-    return rc;
-}
-
-
-/**
- * Dumps a 32-bit shadow page table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pVM         The VM handle.
- * @param   GCPhys      The physical address of the table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int pgmR3DumpHierarchyGstPaePML4(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys, unsigned cMaxDepth)
-{
-    PCX86PML4       pPML4;
-    PGMPAGEMAPLOCK  Lock;
-    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page map level 4", (void const **)&pPML4, &Lock);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth);
-    cMaxDepth--;
-
-    /*
-     * This is a bit tricky as we're working on unsigned addresses while the
-     * AMD64 spec uses signed tricks.
-     */
-    uint32_t iFirst = (pState->u64FirstAddress >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    uint32_t iLast  = (pState->u64LastAddress  >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    if (   pState->u64LastAddress  <= UINT64_C(0x00007fffffffffff)
-        || pState->u64FirstAddress >= UINT64_C(0xffff800000000000))
-    { /* Simple, nothing to adjust */ }
-    else if (pState->u64FirstAddress <= UINT64_C(0x00007fffffffffff))
-        iLast = X86_PG_AMD64_ENTRIES / 2 - 1;
-    else if (pState->u64LastAddress  >= UINT64_C(0xffff800000000000))
-        iFirst = X86_PG_AMD64_ENTRIES / 2;
-    else
-        iFirst = X86_PG_AMD64_ENTRIES; /* neither address is canonical */
-
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PML4E Pml4e = pPML4->a[i];
-        if (Pml4e.n.u1Present)
-        {
-            pState->u64Address = ((uint64_t)i << X86_PML4_SHIFT)
-                               | (i >= RT_ELEMENTS(pPML4->a) / 2 ? UINT64_C(0xffff000000000000) : 0);
-            pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
-                                    "%016llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
-                                    pState->u64Address,
-                                    Pml4e.n.u1Write         ? 'W'  : 'R',
-                                    Pml4e.n.u1User          ? 'U'  : 'S',
-                                    Pml4e.n.u1Accessed      ? 'A'  : '-',
-                                    Pml4e.n.u3Reserved & 1  ? '?'  : '.', /* ignored */
-                                    Pml4e.n.u3Reserved & 4  ? '!'  : '.', /* mbz */
-                                    Pml4e.n.u1WriteThru     ? "WT" : "--",
-                                    Pml4e.n.u1CacheDisable  ? "CD" : "--",
-                                    Pml4e.n.u3Reserved & 2  ? "!"  : "..",/* mbz */
-                                    Pml4e.n.u1NoExecute     ? "NX" : "--",
-                                    Pml4e.u & RT_BIT_64(9)  ? '1' : '0',
-                                    Pml4e.u & RT_BIT_64(10) ? '1' : '0',
-                                    Pml4e.u & RT_BIT_64(11) ? '1' : '0',
-                                    Pml4e.u & X86_PML4E_PG_MASK);
-            if (pState->fDumpPageInfo)
-                pgmR3DumpHierarchyGstPageInfo(pState, Pml4e.u & X86_PML4E_PG_MASK, _4K);
-            pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pml4e.u);
-            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-            if (cMaxDepth)
-            {
-                int rc2 = pgmR3DumpHierarchyGstPaePDPT(pState, Pml4e.u & X86_PML4E_PG_MASK, cMaxDepth);
-                if (rc2 < rc && RT_SUCCESS(rc))
-                    rc = rc2;
-            }
-            else
-                pState->cLeaves++;
-        }
-    }
-
-    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
-    return rc;
-}
-
-
-/**
- * Dumps a 32-bit shadow page table.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState      The dumper state.
- * @param   GCPhys      The physical address of the table.
- */
-static int pgmR3DumpHierarchyGst32BitPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys)
-{
-    PCX86PT         pPT;
-    PGMPAGEMAPLOCK  Lock;
-    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page table", (void const **)&pPT, &Lock);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PTE Pte = pPT->a[i];
-        if (Pte.n.u1Present)
-        {
-            pState->u64Address = u64BaseAddress + (i << X86_PT_SHIFT);
-            pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d  */
-                                    "%08llx 1  |   P %c %c %c %c %c %s %s %s .. 4K %c%c%c  %08x",
-                                    pState->u64Address,
-                                    Pte.n.u1Write         ? 'W'  : 'R',
-                                    Pte.n.u1User          ? 'U'  : 'S',
-                                    Pte.n.u1Accessed      ? 'A'  : '-',
-                                    Pte.n.u1Dirty         ? 'D'  : '-',
-                                    Pte.n.u1Global        ? 'G'  : '-',
-                                    Pte.n.u1WriteThru     ? "WT" : "--",
-                                    Pte.n.u1CacheDisable  ? "CD" : "--",
-                                    Pte.n.u1PAT           ? "AT" : "--",
-                                    Pte.u & RT_BIT_32(9)  ? '1' : '0',
-                                    Pte.u & RT_BIT_32(10) ? '1' : '0',
-                                    Pte.u & RT_BIT_32(11) ? '1' : '0',
-                                    Pte.u & X86_PDE_PG_MASK);
-            if (pState->fDumpPageInfo)
-                pgmR3DumpHierarchyGstPageInfo(pState, Pte.u & X86_PDE_PG_MASK, _4K);
-            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-        }
-    }
-
-    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dumps a 32-bit shadow page directory and page tables.
- *
- * @returns VBox status code (VINF_SUCCESS).
- * @param   pState      The dumper state.
- * @param   GCPhys      The physical address of the table.
- * @param   cMaxDepth   The maximum depth.
- */
-static int pgmR3DumpHierarchyGst32BitPD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys, unsigned cMaxDepth)
-{
-    if (pState->u64Address >= _4G)
-        return VINF_SUCCESS;
-
-    PCX86PD         pPD;
-    PGMPAGEMAPLOCK  Lock;
-    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory", (void const **)&pPD, &Lock);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    Assert(cMaxDepth > 0);
-    cMaxDepth--;
-
-    uint32_t iFirst, iLast;
-    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
-    for (uint32_t i = iFirst; i <= iLast; i++)
-    {
-        X86PDE Pde = pPD->a[i];
-        if (Pde.n.u1Present)
-        {
-            pState->u64Address = (uint32_t)i << X86_PD_SHIFT;
-            if (Pde.b.u1Size && pState->fPse)
-            {
-                uint64_t u64Phys = ((uint64_t)(Pde.u & X86_PDE4M_PG_HIGH_MASK) << X86_PDE4M_PG_HIGH_SHIFT)
-                                 | (Pde.u & X86_PDE4M_PG_MASK);
-                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
-                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s .. 4M %c%c%c  %08llx",
-                                        pState->u64Address,
-                                        Pde.b.u1Write         ? 'W'  : 'R',
-                                        Pde.b.u1User          ? 'U'  : 'S',
-                                        Pde.b.u1Accessed      ? 'A'  : '-',
-                                        Pde.b.u1Dirty         ? 'D'  : '-',
-                                        Pde.b.u1Global        ? 'G'  : '-',
-                                        Pde.b.u1WriteThru     ? "WT" : "--",
-                                        Pde.b.u1CacheDisable  ? "CD" : "--",
-                                        Pde.b.u1PAT           ? "AT" : "--",
-                                        Pde.u & RT_BIT_32(9)  ? '1' : '0',
-                                        Pde.u & RT_BIT_32(10) ? '1' : '0',
-                                        Pde.u & RT_BIT_32(11) ? '1' : '0',
-                                        u64Phys);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyGstPageInfo(pState, u64Phys, _4M);
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-                pState->cLeaves++;
-            }
-            else
-            {
-                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
-                                        "%08llx 0 |    P %c %c %c %c %c %s %s .. .. .. %c%c%c  %08x",
-                                        pState->u64Address,
-                                        Pde.n.u1Write         ? 'W'  : 'R',
-                                        Pde.n.u1User          ? 'U'  : 'S',
-                                        Pde.n.u1Accessed      ? 'A'  : '-',
-                                        Pde.n.u1Reserved0     ? '?'  : '.', /* ignored */
-                                        Pde.n.u1Reserved1     ? '?'  : '.', /* ignored */
-                                        Pde.n.u1WriteThru     ? "WT" : "--",
-                                        Pde.n.u1CacheDisable  ? "CD" : "--",
-                                        Pde.u & RT_BIT_32(9)  ? '1' : '0',
-                                        Pde.u & RT_BIT_32(10) ? '1' : '0',
-                                        Pde.u & RT_BIT_32(11) ? '1' : '0',
-                                        Pde.u & X86_PDE_PG_MASK);
-                if (pState->fDumpPageInfo)
-                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE_PG_MASK, _4K);
-                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
-
-                if (cMaxDepth)
-                {
-                    int rc2 = pgmR3DumpHierarchyGst32BitPT(pState, Pde.u & X86_PDE_PG_MASK);
-                    if (rc2 < rc && RT_SUCCESS(rc))
-                        rc = rc2;
-                }
-                else
-                    pState->cLeaves++;
-            }
-        }
-    }
-
-    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
-    return rc;
-}
-
-
-/**
- * Internal worker that initiates the actual dump.
- *
- * @returns VBox status code.
- * @param   pState              The dumper state.
- * @param   cr3                 The CR3 value.
- * @param   cMaxDepth           The max depth.
- */
-static int pgmR3DumpHierarchyGstDoIt(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t cr3, unsigned cMaxDepth)
-{
-    int             rc;
-    unsigned const  cch     = pState->cchAddress;
-    uint64_t const  cr3Mask = pState->fEpt ? X86_CR3_AMD64_PAGE_MASK
-                            : pState->fLme ? X86_CR3_AMD64_PAGE_MASK
-                            : pState->fPae ? X86_CR3_PAE_PAGE_MASK
-                            :                X86_CR3_PAGE_MASK;
-    if (pState->fPrintCr3)
-    {
-        const char * const  pszMode = pState->fEpt ? "Extended Page Tables"
-                                    : pState->fLme ? "Long Mode"
-                                    : pState->fPae ? "PAE Mode"
-                                    : pState->fPse ? "32-bit w/ PSE"
-                                    :                "32-bit";
-        pState->pHlp->pfnPrintf(pState->pHlp, "cr3=%0*llx", cch, cr3);
-        if (pState->fDumpPageInfo)
-            pgmR3DumpHierarchyGstPageInfo(pState, cr3 & X86_CR3_AMD64_PAGE_MASK, _4K);
-        pState->pHlp->pfnPrintf(pState->pHlp, " %s%s%s\n",
-                                pszMode,
-                                pState->fNp  ? " + Nested Paging" : "",
-                                pState->fNxe ? " + NX" : "");
-    }
-
-
-    if (pState->fEpt)
-    {
-        if (pState->fPrintHeader)
-            pState->pHlp->pfnPrintf(pState->pHlp,
-                                    "%-*s        R - Readable\n"
-                                    "%-*s        | W - Writeable\n"
-                                    "%-*s        | | X - Executable\n"
-                                    "%-*s        | | | EMT - EPT memory type\n"
-                                    "%-*s        | | | |   PAT - Ignored PAT?\n"
-                                    "%-*s        | | | |   |    AVL1 - 4 available bits\n"
-                                    "%-*s        | | | |   |    |   AVL2 - 12 available bits\n"
-                                    "%-*s Level  | | | |   |    |    |     page  \n"
-                                  /* xxxx n **** R W X EMT PAT AVL1 AVL2   xxxxxxxxxxxxx
-                                                 R W X  7   0   f   fff    0123456701234567 */
-                                    ,
-                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
-
-        pState->pHlp->pfnPrintf(pState->pHlp, "EPT dumping is not yet implemented, sorry.\n");
-        /** @todo implemented EPT dumping. */
-        rc = VERR_NOT_IMPLEMENTED;
-    }
-    else
-    {
-        if (pState->fPrintHeader)
-            pState->pHlp->pfnPrintf(pState->pHlp,
-                                    "%-*s        P - Present\n"
-                                    "%-*s        | R/W - Read (0) / Write (1)\n"
-                                    "%-*s        | | U/S - User (1) / Supervisor (0)\n"
-                                    "%-*s        | | | A - Accessed\n"
-                                    "%-*s        | | | | D - Dirty\n"
-                                    "%-*s        | | | | | G - Global\n"
-                                    "%-*s        | | | | | | WT - Write thru\n"
-                                    "%-*s        | | | | | | |  CD - Cache disable\n"
-                                    "%-*s        | | | | | | |  |  AT - Attribute table (PAT)\n"
-                                    "%-*s        | | | | | | |  |  |  NX - No execute (K8)\n"
-                                    "%-*s        | | | | | | |  |  |  |  4K/4M/2M - Page size.\n"
-                                    "%-*s        | | | | | | |  |  |  |  |  AVL - 3 available bits.\n"
-                                    "%-*s Level  | | | | | | |  |  |  |  |  |    Page\n"
-                                  /* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
-                                                 - W U - - - -- -- -- -- -- 010 */
-                                    ,
-                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
-                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
-        if (pState->fLme)
-            rc = pgmR3DumpHierarchyGstPaePML4(pState, cr3 & cr3Mask, cMaxDepth);
-        else if (pState->fPae)
-            rc = pgmR3DumpHierarchyGstPaePDPT(pState, cr3 & cr3Mask, cMaxDepth);
-        else
-            rc = pgmR3DumpHierarchyGst32BitPD(pState, cr3 & cr3Mask, cMaxDepth);
-    }
-
-    if (!pState->cLeaves)
-        pState->pHlp->pfnPrintf(pState->pHlp, "not present\n");
-    return rc;
-}
-
-
-/**
- * dbgfR3PagingDumpEx worker.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   cr3             The CR3 register value.
- * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
- * @param   FirstAddr       The start address.
- * @param   LastAddr        The address to stop after.
- * @param   cMaxDepth       The max depth.
- * @param   pHlp            The output callbacks.  Defaults to log if NULL.
- *
- * @internal
- */
-VMMR3_INT_DECL(int) PGMR3DumpHierarchyGst(PVM pVM, uint64_t cr3, uint32_t fFlags, RTGCPTR FirstAddr, RTGCPTR LastAddr,
-                                          uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
-{
-    /* Minimal validation as we're only supposed to service DBGF. */
-    AssertReturn(~(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(!(fFlags & (DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3)), VERR_INVALID_PARAMETER);
-    AssertReturn(fFlags & DBGFPGDMP_FLAGS_GUEST, VERR_INVALID_PARAMETER);
-
-    PGMR3DUMPHIERARCHYSTATE State;
-    pgmR3DumpHierarchyInitState(&State, pVM, fFlags, FirstAddr, LastAddr, pHlp);
-    return pgmR3DumpHierarchyGstDoIt(&State, cr3, cMaxDepth);
-}
-
Index: unk/src/VBox/VMM/PGMGst.h
===================================================================
--- /trunk/src/VBox/VMM/PGMGst.h	(revision 35332)
+++ 	(revision )
@@ -1,134 +1,0 @@
-/* $Id$ */
-/** @file
- * VBox - Page Manager / Monitor, Guest Paging Template.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-RT_C_DECLS_BEGIN
-/* r3 */
-PGM_GST_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0);
-PGM_GST_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
-PGM_GST_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta);
-PGM_GST_DECL(int, Exit)(PVMCPU pVCpu);
-
-/* all */
-PGM_GST_DECL(int, GetPage)(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys);
-PGM_GST_DECL(int, ModifyPage)(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask);
-PGM_GST_DECL(int, GetPDE)(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPDE);
-RT_C_DECLS_END
-
-
-/**
- * Initializes the guest bit of the paging mode data.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
- *                          This is used early in the init process to avoid trouble with PDM
- *                          not being initialized yet.
- */
-PGM_GST_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0)
-{
-    Assert(pModeData->uGstType == PGM_GST_TYPE);
-
-    /* Ring-3 */
-    pModeData->pfnR3GstRelocate           = PGM_GST_NAME(Relocate);
-    pModeData->pfnR3GstExit               = PGM_GST_NAME(Exit);
-    pModeData->pfnR3GstGetPDE             = PGM_GST_NAME(GetPDE);
-    pModeData->pfnR3GstGetPage            = PGM_GST_NAME(GetPage);
-    pModeData->pfnR3GstModifyPage         = PGM_GST_NAME(ModifyPage);
-
-    if (fResolveGCAndR0)
-    {
-        int rc;
-
-#if PGM_SHW_TYPE != PGM_TYPE_AMD64 /* No AMD64 for traditional virtualization, only VT-x and AMD-V. */
-        /* GC */
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_GST_NAME_RC_STR(GetPage),          &pModeData->pfnRCGstGetPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_RC_STR(GetPage),  rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_GST_NAME_RC_STR(ModifyPage),       &pModeData->pfnRCGstModifyPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_RC_STR(ModifyPage),  rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_GST_NAME_RC_STR(GetPDE),           &pModeData->pfnRCGstGetPDE);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_RC_STR(GetPDE), rc), rc);
-#endif /* Not AMD64 shadow paging. */
-
-        /* Ring-0 */
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_GST_NAME_R0_STR(GetPage),          &pModeData->pfnR0GstGetPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_R0_STR(GetPage),  rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_GST_NAME_R0_STR(ModifyPage),       &pModeData->pfnR0GstModifyPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_R0_STR(ModifyPage),  rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_GST_NAME_R0_STR(GetPDE),           &pModeData->pfnR0GstGetPDE);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_R0_STR(GetPDE), rc), rc);
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Enters the guest mode.
- *
- * @returns VBox status code.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   GCPhysCR3   The physical address from the CR3 register.
- */
-PGM_GST_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
-{
-    /*
-     * Map and monitor CR3
-     */
-    int rc = PGM_BTH_PFN(MapCR3, pVCpu)(pVCpu, GCPhysCR3);
-    return rc;
-}
-
-
-/**
- * Relocate any GC pointers related to guest mode paging.
- *
- * @returns VBox status code.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   offDelta    The relocation offset.
- */
-PGM_GST_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta)
-{
-    pVCpu->pgm.s.pGst32BitPdRC += offDelta;
-
-    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.apGstPaePDsRC); i++)
-    {
-        pVCpu->pgm.s.apGstPaePDsRC[i] += offDelta;
-    }
-    pVCpu->pgm.s.pGstPaePdptRC += offDelta;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Exits the guest mode.
- *
- * @returns VBox status code.
- * @param   pVCpu       The VMCPU to operate on.
- */
-PGM_GST_DECL(int, Exit)(PVMCPU pVCpu)
-{
-    int rc;
-
-    rc = PGM_BTH_PFN(UnmapCR3, pVCpu)(pVCpu);
-    return rc;
-}
-
Index: unk/src/VBox/VMM/PGMGstDefs.h
===================================================================
--- /trunk/src/VBox/VMM/PGMGstDefs.h	(revision 35332)
+++ 	(revision )
@@ -1,231 +1,0 @@
-/* $Id$ */
-/** @file
- * VBox - Page Manager, Guest Paging Template - All context code.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-#undef GSTPT
-#undef PGSTPT
-#undef GSTPTE
-#undef PGSTPTE
-#undef GSTPD
-#undef PGSTPD
-#undef GSTPDE
-#undef PGSTPDE
-#undef GSTPTWALK
-#undef PGSTPTWALK
-#undef PCGSTPTWALK
-#undef GST_BIG_PAGE_SIZE
-#undef GST_BIG_PAGE_OFFSET_MASK
-#undef GST_PDE_PG_MASK
-#undef GST_PDE_BIG_PG_MASK
-#undef GST_PD_SHIFT
-#undef GST_PD_MASK
-#undef GST_PTE_PG_MASK
-#undef GST_GET_PTE_SHW_FLAGS
-#undef GST_PT_SHIFT
-#undef GST_PT_MASK
-#undef GST_TOTAL_PD_ENTRIES
-#undef GST_CR3_PAGE_MASK
-#undef GST_PDPE_ENTRIES
-#undef GST_PDPT_SHIFT
-#undef GST_PDPT_MASK
-#undef GST_PDPE_PG_MASK
-#undef GST_GET_PTE_GCPHYS
-#undef GST_GET_PDE_GCPHYS
-#undef GST_GET_BIG_PDE_GCPHYS
-#undef GST_GET_PDE_SHW_FLAGS
-#undef GST_GET_BIG_PDE_SHW_FLAGS
-#undef GST_GET_BIG_PDE_SHW_FLAGS_4_PTE
-#undef GST_IS_PTE_VALID
-#undef GST_IS_PDE_VALID
-#undef GST_IS_BIG_PDE_VALID
-#undef GST_IS_PDPE_VALID
-#undef GST_IS_BIG_PDPE_VALID
-#undef GST_IS_PML4E_VALID
-#undef GST_IS_PSE_ACTIVE
-#undef GST_IS_NX_ACTIVE
-#undef BTH_IS_NP_ACTIVE
-
-#if PGM_GST_TYPE == PGM_TYPE_REAL \
- || PGM_GST_TYPE == PGM_TYPE_PROT
-
-# if PGM_SHW_TYPE == PGM_TYPE_EPT
-#  define GSTPT                                 X86PTPAE
-#  define PGSTPT                                PX86PTPAE
-#  define GSTPTE                                X86PTEPAE
-#  define PGSTPTE                               PX86PTEPAE
-#  define GSTPD                                 X86PDPAE
-#  define PGSTPD                                PX86PDPAE
-#  define GSTPDE                                X86PDEPAE
-#  define PGSTPDE                               PX86PDEPAE
-#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
-#  define GST_IS_NX_ACTIVE(pVCpu)               (true && This_should_perhaps_not_be_used_in_this_context)
-#  define BTH_IS_NP_ACTIVE(pVM)                 (true)
-# else
-#  if PGM_SHW_TYPE == PGM_TYPE_32BIT /* Same as shadow paging, but no PGMSHWPTEPAE. */
-#   define GSTPT                                X86PT
-#   define PGSTPT                               PX86PT
-#   define GSTPTE                               X86PTE
-#   define PGSTPTE                              PX86PTE
-#   define GSTPD                                X86PD
-#   define PGSTPD                               PX86PD
-#   define GSTPDE                               X86PDE
-#   define PGSTPDE                              PX86PDE
-#   define GST_PTE_PG_MASK                      X86_PTE_PG_MASK
-#  else
-#   define GSTPT                                X86PTPAE
-#   define PGSTPT                               PX86PTPAE
-#   define GSTPTE                               X86PTEPAE
-#   define PGSTPTE                              PX86PTEPAE
-#   define GSTPD                                X86PDPAE
-#   define PGSTPD                               PX86PDPAE
-#   define GSTPDE                               X86PDEPAE
-#   define PGSTPDE                              PX86PDEPAE
-#   define GST_PTE_PG_MASK                      X86_PTE_PAE_PG_MASK
-#  endif
-#  define GST_IS_NX_ACTIVE(pVCpu)               (pgmGstIsNoExecuteActive(pVCpu))
-#  if PGM_GST_TYPE == PGM_TYPE_PROT             /* (comment at top of PGMAllBth.h) */
-#   define BTH_IS_NP_ACTIVE(pVM)                (pVM->pgm.s.fNestedPaging)
-#  else
-#   define BTH_IS_NP_ACTIVE(pVM)                (false)
-#  endif
-# endif
-# define GST_GET_PTE_GCPHYS(Pte)                ((Pte).u & GST_PTE_PG_MASK)
-# define GST_GET_PDE_GCPHYS(Pde)                (true && This_should_perhaps_not_be_used_in_this_context) //??
-# define GST_GET_BIG_PDE_GCPHYS(Pde)            (true && This_should_perhaps_not_be_used_in_this_context) //??
-# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G)) /**< @todo Could return P|RW|US|A|D here without consulting the PTE. */
-# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      (true && This_should_perhaps_not_be_used_in_this_context) //??
-# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  (true && This_should_perhaps_not_be_used_in_this_context) //??
-# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde) (true && This_should_perhaps_not_be_used_in_this_context) //??
-# define GST_IS_PTE_VALID(pVCpu, Pte)           (true)
-# define GST_IS_PDE_VALID(pVCpu, Pde)           (true)
-# define GST_IS_BIG_PDE_VALID(pVCpu, Pde)       (true)
-# define GST_IS_PDPE_VALID(pVCpu, Pdpe)         (true)
-# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)     (true)
-# define GST_IS_PML4E_VALID(pVCpu, Pml4e)       (true)
-# define GST_IS_PSE_ACTIVE(pVCpu)               (false && This_should_not_be_used_in_this_context)
-
-#elif PGM_GST_TYPE == PGM_TYPE_32BIT
-# define GSTPT                                  X86PT
-# define PGSTPT                                 PX86PT
-# define GSTPTE                                 X86PTE
-# define PGSTPTE                                PX86PTE
-# define GSTPD                                  X86PD
-# define PGSTPD                                 PX86PD
-# define GSTPDE                                 X86PDE
-# define PGSTPDE                                PX86PDE
-# define GSTPTWALK                              PGMPTWALKGST32BIT
-# define PGSTPTWALK                             PPGMPTWALKGST32BIT
-# define PCGSTPTWALK                            PCPGMPTWALKGST32BIT
-# define GST_BIG_PAGE_SIZE                      X86_PAGE_4M_SIZE
-# define GST_BIG_PAGE_OFFSET_MASK               X86_PAGE_4M_OFFSET_MASK
-# define GST_PDE_PG_MASK                        X86_PDE_PG_MASK
-# define GST_PDE_BIG_PG_MASK                    X86_PDE4M_PG_MASK
-# define GST_GET_PTE_GCPHYS(Pte)                ((Pte).u & GST_PDE_PG_MASK)
-# define GST_GET_PDE_GCPHYS(Pde)                ((Pde).u & GST_PDE_PG_MASK)
-# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       pgmGstGet4MBPhysPage(&(pVM)->pgm.s, Pde)
-# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A))
-# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde) \
-    ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A))
-# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde) \
-    ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A | X86_PDE4M_D | X86_PDE4M_G))
-# define GST_PD_SHIFT                           X86_PD_SHIFT
-# define GST_PD_MASK                            X86_PD_MASK
-# define GST_TOTAL_PD_ENTRIES                   X86_PG_ENTRIES
-# define GST_PTE_PG_MASK                        X86_PTE_PG_MASK
-# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G))
-# define GST_PT_SHIFT                           X86_PT_SHIFT
-# define GST_PT_MASK                            X86_PT_MASK
-# define GST_CR3_PAGE_MASK                      X86_CR3_PAGE_MASK
-# define GST_IS_PTE_VALID(pVCpu, Pte)           (true)
-# define GST_IS_PDE_VALID(pVCpu, Pde)           (true)
-# define GST_IS_BIG_PDE_VALID(pVCpu, Pde)       (!( (Pde).u & (pVCpu)->pgm.s.fGst32BitMbzBigPdeMask ))
-//# define GST_IS_PDPE_VALID(pVCpu, Pdpe)         (false)
-//# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)     (false)
-//# define GST_IS_PML4E_VALID(pVCpu, Pml4e)       (false)
-# define GST_IS_PSE_ACTIVE(pVCpu)               pgmGst32BitIsPageSizeExtActive(pVCpu)
-# define GST_IS_NX_ACTIVE(pVCpu)                (false)
-# define BTH_IS_NP_ACTIVE(pVM)                  (false)
-
-#elif   PGM_GST_TYPE == PGM_TYPE_PAE \
-     || PGM_GST_TYPE == PGM_TYPE_AMD64
-# define GSTPT                                  X86PTPAE
-# define PGSTPT                                 PX86PTPAE
-# define GSTPTE                                 X86PTEPAE
-# define PGSTPTE                                PX86PTEPAE
-# define GSTPD                                  X86PDPAE
-# define PGSTPD                                 PX86PDPAE
-# define GSTPDE                                 X86PDEPAE
-# define PGSTPDE                                PX86PDEPAE
-# define GST_BIG_PAGE_SIZE                      X86_PAGE_2M_SIZE
-# define GST_BIG_PAGE_OFFSET_MASK               X86_PAGE_2M_OFFSET_MASK
-# define GST_PDE_PG_MASK                        X86_PDE_PAE_PG_MASK
-# define GST_PDE_BIG_PG_MASK                    X86_PDE2M_PAE_PG_MASK
-# define GST_GET_PTE_GCPHYS(Pte)                ((Pte).u & GST_PTE_PG_MASK)
-# define GST_GET_PDE_GCPHYS(Pde)                ((Pde).u & GST_PDE_PG_MASK)
-# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       ((Pde).u & GST_PDE_BIG_PG_MASK)
-# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (pVCpu)->pgm.s.fGst64ShadowedPteMask )
-# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedPdeMask )
-# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPdeMask )
-# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde)  ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPde4PteMask )
-
-# define GST_PD_SHIFT                           X86_PD_PAE_SHIFT
-# define GST_PD_MASK                            X86_PD_PAE_MASK
-# if PGM_GST_TYPE == PGM_TYPE_PAE
-#  define GSTPTWALK                             PGMPTWALKGSTPAE
-#  define PGSTPTWALK                            PPGMPTWALKGSTPAE
-#  define PCGSTPTWALK                           PCPGMPTWALKGSTPAE
-#  define GST_TOTAL_PD_ENTRIES                  (X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES)
-#  define GST_PDPE_ENTRIES                      X86_PG_PAE_PDPE_ENTRIES
-#  define GST_PDPE_PG_MASK                      X86_PDPE_PG_MASK
-#  define GST_PDPT_SHIFT                        X86_PDPT_SHIFT
-#  define GST_PDPT_MASK                         X86_PDPT_MASK_PAE
-#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
-#  define GST_CR3_PAGE_MASK                     X86_CR3_PAE_PAGE_MASK
-#  define GST_IS_PTE_VALID(pVCpu, Pte)          (!( (Pte).u   & (pVCpu)->pgm.s.fGstPaeMbzPteMask ))
-#  define GST_IS_PDE_VALID(pVCpu, Pde)          (!( (Pde).u   & (pVCpu)->pgm.s.fGstPaeMbzPdeMask ))
-#  define GST_IS_BIG_PDE_VALID(pVCpu, Pde)      (!( (Pde).u   & (pVCpu)->pgm.s.fGstPaeMbzBigPdeMask ))
-#  define GST_IS_PDPE_VALID(pVCpu, Pdpe)        (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstPaeMbzPdpeMask ))
-//# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)    (false)
-//# define GST_IS_PML4E_VALID(pVCpu, Pml4e)      (false)
-# else
-#  define GSTPTWALK                             PGMPTWALKGSTAMD64
-#  define PGSTPTWALK                            PPGMPTWALKGSTAMD64
-#  define PCGSTPTWALK                           PCPGMPTWALKGSTAMD64
-#  define GST_TOTAL_PD_ENTRIES                  (X86_PG_AMD64_ENTRIES * X86_PG_AMD64_PDPE_ENTRIES)
-#  define GST_PDPE_ENTRIES                      X86_PG_AMD64_PDPE_ENTRIES
-#  define GST_PDPT_SHIFT                        X86_PDPT_SHIFT
-#  define GST_PDPE_PG_MASK                      X86_PDPE_PG_MASK
-#  define GST_PDPT_MASK                         X86_PDPT_MASK_AMD64
-#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
-#  define GST_CR3_PAGE_MASK                     X86_CR3_AMD64_PAGE_MASK
-#  define GST_IS_PTE_VALID(pVCpu, Pte)          (!( (Pte).u   & (pVCpu)->pgm.s.fGstAmd64MbzPteMask ))
-#  define GST_IS_PDE_VALID(pVCpu, Pde)          (!( (Pde).u   & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask ))
-#  define GST_IS_BIG_PDE_VALID(pVCpu, Pde)      (!( (Pde).u   & (pVCpu)->pgm.s.fGstAmd64MbzBigPdeMask ))
-#  define GST_IS_PDPE_VALID(pVCpu, Pdpe)        (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstAmd64MbzPdpeMask ))
-#  define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)    (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstAmd64MbzBigPdpeMask ))
-#  define GST_IS_PML4E_VALID(pVCpu, Pml4e)      (!( (Pml4e).u & (pVCpu)->pgm.s.fGstAmd64MbzPml4eMask ))
-# endif
-# define GST_PT_SHIFT                           X86_PT_PAE_SHIFT
-# define GST_PT_MASK                            X86_PT_PAE_MASK
-# define GST_IS_PSE_ACTIVE(pVCpu)               (true)
-# define GST_IS_NX_ACTIVE(pVCpu)                (pgmGstIsNoExecuteActive(pVCpu))
-# define BTH_IS_NP_ACTIVE(pVM)                  (false)
-#endif
-
Index: unk/src/VBox/VMM/PGMHandler.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGMHandler.cpp	(revision 35332)
+++ 	(revision )
@@ -1,640 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager / Monitor, Access Handlers.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/cpum.h>
-#include <VBox/iom.h>
-#include <VBox/sup.h>
-#include <VBox/mm.h>
-#include <VBox/em.h>
-#include <VBox/stam.h>
-#include <VBox/csam.h>
-#include <VBox/rem.h>
-#include <VBox/dbgf.h>
-#include <VBox/rem.h>
-#include <VBox/selm.h>
-#include <VBox/ssm.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include "PGMInline.h"
-#include <VBox/dbg.h>
-
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-#include <iprt/asm.h>
-#include <iprt/thread.h>
-#include <iprt/string.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/hwaccm.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) pgmR3HandlerPhysicalOneClear(PAVLROGCPHYSNODECORE pNode, void *pvUser);
-static DECLCALLBACK(int) pgmR3HandlerPhysicalOneSet(PAVLROGCPHYSNODECORE pNode, void *pvUser);
-static DECLCALLBACK(int) pgmR3InfoHandlersPhysicalOne(PAVLROGCPHYSNODECORE pNode, void *pvUser);
-static DECLCALLBACK(int) pgmR3InfoHandlersVirtualOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
-
-
-
-/**
- * Register a access handler for a physical range.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   enmType         Handler type. Any of the PGMPHYSHANDLERTYPE_PHYSICAL* enums.
- * @param   GCPhys          Start physical address.
- * @param   GCPhysLast      Last physical address. (inclusive)
- * @param   pfnHandlerR3    The R3 handler.
- * @param   pvUserR3        User argument to the R3 handler.
- * @param   pszModR0        The R0 handler module. NULL means the default R0 module.
- * @param   pszHandlerR0    The R0 handler symbol name.
- * @param   pvUserR0        User argument to the R0 handler.
- * @param   pszModRC        The RC handler module. NULL means the default RC
- *                          module.
- * @param   pszHandlerRC    The RC handler symbol name.
- * @param   pvUserRC        User argument to the RC handler. Values less than
- *                          0x10000 will not be relocated.
- * @param   pszDesc         Pointer to description string. This must not be freed.
- */
-VMMR3DECL(int) PGMR3HandlerPhysicalRegister(PVM pVM, PGMPHYSHANDLERTYPE enmType, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
-                                            PFNPGMR3PHYSHANDLER pfnHandlerR3, void *pvUserR3,
-                                            const char *pszModR0, const char *pszHandlerR0, RTR0PTR pvUserR0,
-                                            const char *pszModRC, const char *pszHandlerRC, RTRCPTR pvUserRC, const char *pszDesc)
-{
-    LogFlow(("PGMR3HandlerPhysicalRegister: enmType=%d GCPhys=%RGp GCPhysLast=%RGp pfnHandlerR3=%RHv pvUserHC=%RHv pszModR0=%s pszHandlerR0=%s pvUserR0=%RHv pszModRC=%s pszHandlerRC=%s pvUser=%RRv pszDesc=%s\n",
-             enmType, GCPhys, GCPhysLast, pfnHandlerR3, pvUserR3, pszModR0, pszHandlerR0, pvUserR0, pszHandlerRC, pszModRC, pvUserRC, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    if (!pszModRC)
-        pszModRC = VMMGC_MAIN_MODULE_NAME;
-    if (!pszModR0)
-        pszModR0 = VMMR0_MAIN_MODULE_NAME;
-    if (!pszHandlerR0)
-        pszHandlerR0 = "pgmPhysHandlerRedirectToHC";
-    if (!pszHandlerRC)
-        pszHandlerRC = "pgmPhysHandlerRedirectToHC";
-    AssertPtrReturn(pfnHandlerR3, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszHandlerR0, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszHandlerRC, VERR_INVALID_POINTER);
-
-    /*
-     * Resolve the R0 handler.
-     */
-    R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0 = NIL_RTR0PTR;
-    int rc = VINF_SUCCESS;
-    rc = PDMR3LdrGetSymbolR0Lazy(pVM, pszModR0, NULL /*pszSearchPath*/, pszHandlerR0, &pfnHandlerR0);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Resolve the GC handler.
-         */
-        RTRCPTR pfnHandlerRC = NIL_RTRCPTR;
-        rc = PDMR3LdrGetSymbolRCLazy(pVM, pszModRC, NULL /*pszSearchPath*/, pszHandlerRC, &pfnHandlerRC);
-        if (RT_SUCCESS(rc))
-            return PGMHandlerPhysicalRegisterEx(pVM, enmType, GCPhys, GCPhysLast, pfnHandlerR3, pvUserR3,
-                                                pfnHandlerR0, pvUserR0, pfnHandlerRC, pvUserRC, pszDesc);
-
-        AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszModRC, pszHandlerRC, rc));
-    }
-    else
-        AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszModR0, pszHandlerR0, rc));
-
-    return rc;
-}
-
-
-/**
- * Updates the physical page access handlers.
- *
- * @param   pVM     VM handle.
- * @remark  Only used when restoring a saved state.
- */
-void pgmR3HandlerPhysicalUpdateAll(PVM pVM)
-{
-    LogFlow(("pgmHandlerPhysicalUpdateAll:\n"));
-
-    /*
-     * Clear and set.
-     * (the right -> left on the setting pass is just bird speculating on cache hits)
-     */
-    pgmLock(pVM);
-    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,  true, pgmR3HandlerPhysicalOneClear, pVM);
-    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, false, pgmR3HandlerPhysicalOneSet, pVM);
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Clears all the page level flags for one physical handler range.
- *
- * @returns 0
- * @param   pNode   Pointer to a PGMPHYSHANDLER.
- * @param   pvUser  VM handle.
- */
-static DECLCALLBACK(int) pgmR3HandlerPhysicalOneClear(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
-    PPGMRAMRANGE    pRamHint = NULL;
-    RTGCPHYS        GCPhys = pCur->Core.Key;
-    RTUINT          cPages = pCur->cPages;
-    PPGM            pPGM = &((PVM)pvUser)->pgm.s;
-    for (;;)
-    {
-        PPGMPAGE pPage;
-        int rc = pgmPhysGetPageWithHintEx(pPGM, GCPhys, &pPage, &pRamHint);
-        if (RT_SUCCESS(rc))
-            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE);
-        else
-            AssertRC(rc);
-
-        if (--cPages == 0)
-            return 0;
-        GCPhys += PAGE_SIZE;
-    }
-}
-
-
-/**
- * Sets all the page level flags for one physical handler range.
- *
- * @returns 0
- * @param   pNode   Pointer to a PGMPHYSHANDLER.
- * @param   pvUser  VM handle.
- */
-static DECLCALLBACK(int) pgmR3HandlerPhysicalOneSet(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
-    unsigned        uState = pgmHandlerPhysicalCalcState(pCur);
-    PPGMRAMRANGE    pRamHint = NULL;
-    RTGCPHYS        GCPhys = pCur->Core.Key;
-    RTUINT          cPages = pCur->cPages;
-    PPGM            pPGM = &((PVM)pvUser)->pgm.s;
-    for (;;)
-    {
-        PPGMPAGE pPage;
-        int rc = pgmPhysGetPageWithHintEx(pPGM, GCPhys, &pPage, &pRamHint);
-        if (RT_SUCCESS(rc))
-            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState);
-        else
-            AssertRC(rc);
-
-        if (--cPages == 0)
-            return 0;
-        GCPhys += PAGE_SIZE;
-    }
-}
-
-
-/**
- * Register a access handler for a virtual range.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   enmType         Handler type. Any of the PGMVIRTHANDLERTYPE_* enums.
- * @param   GCPtr           Start address.
- * @param   GCPtrLast       Last address (inclusive).
- * @param   pfnInvalidateR3 The R3 invalidate callback (can be 0)
- * @param   pfnHandlerR3    The R3 handler.
- * @param   pszHandlerRC    The RC handler symbol name.
- * @param   pszModRC        The RC handler module.
- * @param   pszDesc         Pointer to description string. This must not be freed.
- */
-VMMR3DECL(int) PGMR3HandlerVirtualRegister(PVM pVM, PGMVIRTHANDLERTYPE enmType, RTGCPTR GCPtr, RTGCPTR GCPtrLast,
-                                           PFNPGMR3VIRTINVALIDATE pfnInvalidateR3,
-                                           PFNPGMR3VIRTHANDLER pfnHandlerR3,
-                                           const char *pszHandlerRC, const char *pszModRC,
-                                           const char *pszDesc)
-{
-    LogFlow(("PGMR3HandlerVirtualRegisterEx: enmType=%d GCPtr=%RGv GCPtrLast=%RGv pszHandlerRC=%p:{%s} pszModRC=%p:{%s} pszDesc=%s\n",
-             enmType, GCPtr, GCPtrLast, pszHandlerRC, pszHandlerRC, pszModRC, pszModRC, pszDesc));
-
-    /* Not supported/relevant for VT-x and AMD-V. */
-    if (HWACCMIsEnabled(pVM))
-        return VERR_NOT_IMPLEMENTED;
-
-    /*
-     * Validate input.
-     */
-    if (!pszModRC)
-        pszModRC = VMMGC_MAIN_MODULE_NAME;
-    if (!pszModRC || !*pszModRC || !pszHandlerRC || !*pszHandlerRC)
-    {
-        AssertMsgFailed(("pfnHandlerGC or/and pszModRC is missing\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Resolve the GC handler.
-     */
-    RTRCPTR pfnHandlerRC;
-    int rc = PDMR3LdrGetSymbolRCLazy(pVM, pszModRC, NULL /*pszSearchPath*/, pszHandlerRC, &pfnHandlerRC);
-    if (RT_SUCCESS(rc))
-        return PGMR3HandlerVirtualRegisterEx(pVM, enmType, GCPtr, GCPtrLast, pfnInvalidateR3, pfnHandlerR3, pfnHandlerRC, pszDesc);
-
-    AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszModRC, pszHandlerRC, rc));
-    return rc;
-}
-
-
-/**
- * Register an access handler for a virtual range.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   enmType         Handler type. Any of the PGMVIRTHANDLERTYPE_* enums.
- * @param   GCPtr           Start address.
- * @param   GCPtrLast       Last address (inclusive).
- * @param   pfnInvalidateR3 The R3 invalidate callback (can be 0)
- * @param   pfnHandlerR3    The R3 handler.
- * @param   pfnHandlerRC    The RC handler.
- * @param   pszDesc         Pointer to description string. This must not be freed.
- * @thread  EMT
- */
-/** @todo create a template for virtual handlers (see async i/o), we're wasting space
- * duplicating the function pointers now. (Or we will once we add the missing callbacks.) */
-VMMDECL(int) PGMR3HandlerVirtualRegisterEx(PVM pVM, PGMVIRTHANDLERTYPE enmType, RTGCPTR GCPtr, RTGCPTR GCPtrLast,
-                                           R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3,
-                                           R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3,
-                                           RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC,
-                                           R3PTRTYPE(const char *) pszDesc)
-{
-    Log(("PGMR3HandlerVirtualRegister: enmType=%d GCPtr=%RGv GCPtrLast=%RGv pfnInvalidateR3=%RHv pfnHandlerR3=%RHv pfnHandlerRC=%RRv pszDesc=%s\n",
-         enmType, GCPtr, GCPtrLast, pfnInvalidateR3, pfnHandlerR3, pfnHandlerRC, pszDesc));
-
-    /* Not supported/relevant for VT-x and AMD-V. */
-    if (HWACCMIsEnabled(pVM))
-        return VERR_NOT_IMPLEMENTED;
-
-    /*
-     * Validate input.
-     */
-    switch (enmType)
-    {
-        case PGMVIRTHANDLERTYPE_ALL:
-            AssertReleaseMsgReturn(   (GCPtr     & PAGE_OFFSET_MASK) == 0
-                                   && (GCPtrLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK,
-                                   ("PGMVIRTHANDLERTYPE_ALL: GCPtr=%RGv GCPtrLast=%RGv\n", GCPtr, GCPtrLast),
-                                   VERR_NOT_IMPLEMENTED);
-            break;
-        case PGMVIRTHANDLERTYPE_WRITE:
-            if (!pfnHandlerR3)
-            {
-                AssertMsgFailed(("No HC handler specified!!\n"));
-                return VERR_INVALID_PARAMETER;
-            }
-            break;
-
-        case PGMVIRTHANDLERTYPE_HYPERVISOR:
-            if (pfnHandlerR3)
-            {
-                AssertMsgFailed(("R3 handler specified for hypervisor range!?!\n"));
-                return VERR_INVALID_PARAMETER;
-            }
-            break;
-        default:
-            AssertMsgFailed(("Invalid enmType! enmType=%d\n", enmType));
-            return VERR_INVALID_PARAMETER;
-    }
-    if (GCPtrLast < GCPtr)
-    {
-        AssertMsgFailed(("GCPtrLast < GCPtr (%#x < %#x)\n", GCPtrLast, GCPtr));
-        return VERR_INVALID_PARAMETER;
-    }
-    if (!pfnHandlerRC)
-    {
-        AssertMsgFailed(("pfnHandlerRC is missing\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Allocate and initialize a new entry.
-     */
-    unsigned cPages = (RT_ALIGN(GCPtrLast + 1, PAGE_SIZE) - (GCPtr & PAGE_BASE_GC_MASK)) >> PAGE_SHIFT;
-    PPGMVIRTHANDLER pNew;
-    int rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[cPages]), 0, MM_TAG_PGM_HANDLERS, (void **)&pNew); /** @todo r=bird: incorrect member name PhysToVirt? */
-    if (RT_FAILURE(rc))
-        return rc;
-
-    pNew->Core.Key      = GCPtr;
-    pNew->Core.KeyLast  = GCPtrLast;
-
-    pNew->enmType       = enmType;
-    pNew->pfnInvalidateR3 = pfnInvalidateR3;
-    pNew->pfnHandlerRC  = pfnHandlerRC;
-    pNew->pfnHandlerR3  = pfnHandlerR3;
-    pNew->pszDesc       = pszDesc;
-    pNew->cb            = GCPtrLast - GCPtr + 1;
-    pNew->cPages        = cPages;
-    /* Will be synced at next guest execution attempt. */
-    while (cPages-- > 0)
-    {
-        pNew->aPhysToVirt[cPages].Core.Key = NIL_RTGCPHYS;
-        pNew->aPhysToVirt[cPages].Core.KeyLast = NIL_RTGCPHYS;
-        pNew->aPhysToVirt[cPages].offVirtHandler = -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[cPages]);
-        pNew->aPhysToVirt[cPages].offNextAlias = 0;
-    }
-
-    /*
-     * Try to insert it into the tree.
-     *
-     * The current implementation doesn't allow multiple handlers for
-     * the same range this makes everything much simpler and faster.
-     */
-    AVLROGCPTRTREE *pRoot = enmType != PGMVIRTHANDLERTYPE_HYPERVISOR
-                          ? &pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers
-                          : &pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers;
-    pgmLock(pVM);
-    if (*pRoot != 0)
-    {
-        PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrGetBestFit(pRoot, pNew->Core.Key, true);
-        if (    !pCur
-            ||  GCPtr     > pCur->Core.KeyLast
-            ||  GCPtrLast < pCur->Core.Key)
-            pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrGetBestFit(pRoot, pNew->Core.Key, false);
-        if (    pCur
-            &&  GCPtr     <= pCur->Core.KeyLast
-            &&  GCPtrLast >= pCur->Core.Key)
-        {
-            /*
-             * The LDT sometimes conflicts with the IDT and LDT ranges while being
-             * updated on linux. So, we don't assert simply log it.
-             */
-            Log(("PGMR3HandlerVirtualRegister: Conflict with existing range %RGv-%RGv (%s), req. %RGv-%RGv (%s)\n",
-                 pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc, GCPtr, GCPtrLast, pszDesc));
-            MMHyperFree(pVM, pNew);
-            pgmUnlock(pVM);
-            return VERR_PGM_HANDLER_VIRTUAL_CONFLICT;
-        }
-    }
-    if (RTAvlroGCPtrInsert(pRoot, &pNew->Core))
-    {
-        if (enmType != PGMVIRTHANDLERTYPE_HYPERVISOR)
-        {
-            PVMCPU pVCpu = VMMGetCpu(pVM);
-
-            pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL | PGM_SYNC_CLEAR_PGM_POOL;
-            VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-        }
-        pgmUnlock(pVM);
-
-#ifdef VBOX_WITH_STATISTICS
-        char szPath[256];
-        RTStrPrintf(szPath, sizeof(szPath), "/PGM/VirtHandler/Calls/%RGv-%RGv", pNew->Core.Key, pNew->Core.KeyLast);
-        rc = STAMR3Register(pVM, &pNew->Stat, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szPath, STAMUNIT_TICKS_PER_CALL, pszDesc);
-        AssertRC(rc);
-#endif
-        return VINF_SUCCESS;
-    }
-
-    pgmUnlock(pVM);
-    AssertFailed();
-    MMHyperFree(pVM, pNew);
-    return VERR_PGM_HANDLER_VIRTUAL_CONFLICT;
-}
-
-
-/**
- * Modify the page invalidation callback handler for a registered virtual range.
- * (add more when needed)
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   GCPtr           Start address.
- * @param   pfnInvalidateR3 The R3 invalidate callback (can be 0)
- * @remarks Doesn't work with the hypervisor access handler type.
- */
-VMMDECL(int) PGMHandlerVirtualChangeInvalidateCallback(PVM pVM, RTGCPTR GCPtr, PFNPGMR3VIRTINVALIDATE pfnInvalidateR3)
-{
-    pgmLock(pVM);
-    PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrGet(&pVM->pgm.s.pTreesR3->VirtHandlers, GCPtr);
-    if (pCur)
-    {
-        pCur->pfnInvalidateR3 = pfnInvalidateR3;
-        pgmUnlock(pVM);
-        return VINF_SUCCESS;
-    }
-    pgmUnlock(pVM);
-    AssertMsgFailed(("Range %#x not found!\n", GCPtr));
-    return VERR_INVALID_PARAMETER;
-}
-
-/**
- * Deregister an access handler for a virtual range.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   GCPtr       Start address.
- * @thread  EMT
- */
-VMMDECL(int) PGMHandlerVirtualDeregister(PVM pVM, RTGCPTR GCPtr)
-{
-    pgmLock(pVM);
-
-    /*
-     * Find the handler.
-     * We naturally assume GCPtr is a unique specification.
-     */
-    PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtr);
-    if (RT_LIKELY(pCur))
-    {
-        Log(("PGMHandlerVirtualDeregister: Removing Virtual (%d) Range %RGv-%RGv %s\n", pCur->enmType,
-             pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
-        Assert(pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR);
-
-        /*
-         * Reset the flags and remove phys2virt nodes.
-         */
-        PPGM pPGM = &pVM->pgm.s;
-        for (unsigned iPage = 0; iPage < pCur->cPages; iPage++)
-            if (pCur->aPhysToVirt[iPage].offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE)
-                pgmHandlerVirtualClearPage(pPGM, pCur, iPage);
-
-        /*
-         * Schedule CR3 sync.
-         */
-        PVMCPU pVCpu = VMMGetCpu(pVM);
-
-        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL | PGM_SYNC_CLEAR_PGM_POOL;
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-    }
-    else
-    {
-        /* must be a hypervisor one then. */
-        pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, GCPtr);
-        if (RT_UNLIKELY(!pCur))
-        {
-            pgmUnlock(pVM);
-            AssertMsgFailed(("Range %#x not found!\n", GCPtr));
-            return VERR_INVALID_PARAMETER;
-        }
-
-        Log(("PGMHandlerVirtualDeregister: Removing Hyper Virtual (%d) Range %RGv-%RGv %s\n", pCur->enmType,
-             pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
-        Assert(pCur->enmType == PGMVIRTHANDLERTYPE_HYPERVISOR);
-    }
-
-    pgmUnlock(pVM);
-
-    STAM_DEREG(pVM, &pCur->Stat);
-    MMHyperFree(pVM, pCur);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Arguments for pgmR3InfoHandlersPhysicalOne and pgmR3InfoHandlersVirtualOne.
- */
-typedef struct PGMHANDLERINFOARG
-{
-    /** The output helpers.*/
-    PCDBGFINFOHLP   pHlp;
-    /** Set if statistics should be dumped. */
-    bool            fStats;
-} PGMHANDLERINFOARG, *PPGMHANDLERINFOARG;
-
-
-/**
- * Info callback for 'pgmhandlers'.
- *
- * @param   pHlp        The output helpers.
- * @param   pszArgs     The arguments. phys or virt.
- */
-DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    /*
-     * Test input.
-     */
-    PGMHANDLERINFOARG Args = { pHlp, /* .fStats = */ true };
-    bool fPhysical = !pszArgs  || !*pszArgs;
-    bool fVirtual = fPhysical;
-    bool fHyper = fPhysical;
-    if (!fPhysical)
-    {
-        bool fAll = strstr(pszArgs, "all") != NULL;
-        fPhysical   = fAll || strstr(pszArgs, "phys") != NULL;
-        fVirtual    = fAll || strstr(pszArgs, "virt") != NULL;
-        fHyper      = fAll || strstr(pszArgs, "hyper")!= NULL;
-        Args.fStats = strstr(pszArgs, "nost") == NULL;
-    }
-
-    /*
-     * Dump the handlers.
-     */
-    if (fPhysical)
-    {
-        pHlp->pfnPrintf(pHlp,
-            "Physical handlers: (PhysHandlers=%d (%#x))\n"
-            "From     - To (incl) HandlerHC UserHC    HandlerGC UserGC    Type     Description\n",
-            pVM->pgm.s.pTreesR3->PhysHandlers, pVM->pgm.s.pTreesR3->PhysHandlers);
-        RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, true, pgmR3InfoHandlersPhysicalOne, &Args);
-    }
-
-    if (fVirtual)
-    {
-        pHlp->pfnPrintf(pHlp,
-            "Virtual handlers:\n"
-            "From     - To (excl) HandlerHC HandlerGC Type     Description\n");
-        RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->VirtHandlers, true, pgmR3InfoHandlersVirtualOne, &Args);
-    }
-
-    if (fHyper)
-    {
-        pHlp->pfnPrintf(pHlp,
-            "Hypervisor Virtual handlers:\n"
-            "From     - To (excl) HandlerHC HandlerGC Type     Description\n");
-        RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3InfoHandlersVirtualOne, &Args);
-    }
-}
-
-
-/**
- * Displays one physical handler range.
- *
- * @returns 0
- * @param   pNode   Pointer to a PGMPHYSHANDLER.
- * @param   pvUser  Pointer to command helper functions.
- */
-static DECLCALLBACK(int) pgmR3InfoHandlersPhysicalOne(PAVLROGCPHYSNODECORE pNode, void *pvUser)
-{
-    PPGMPHYSHANDLER     pCur = (PPGMPHYSHANDLER)pNode;
-    PPGMHANDLERINFOARG  pArgs= (PPGMHANDLERINFOARG)pvUser;
-    PCDBGFINFOHLP       pHlp = pArgs->pHlp;
-    const char *pszType;
-    switch (pCur->enmType)
-    {
-        case PGMPHYSHANDLERTYPE_MMIO:           pszType = "MMIO   "; break;
-        case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE: pszType = "Write  "; break;
-        case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:   pszType = "All    "; break;
-        default:                                pszType = "????"; break;
-    }
-    pHlp->pfnPrintf(pHlp,
-        "%RGp - %RGp  %RHv  %RHv  %RRv  %RRv  %s  %s\n",
-        pCur->Core.Key, pCur->Core.KeyLast, pCur->pfnHandlerR3, pCur->pvUserR3, pCur->pfnHandlerRC, pCur->pvUserRC, pszType, pCur->pszDesc);
-#ifdef VBOX_WITH_STATISTICS
-    if (pArgs->fStats)
-        pHlp->pfnPrintf(pHlp, "   cPeriods: %9RU64  cTicks: %11RU64  Min: %11RU64  Avg: %11RU64 Max: %11RU64\n",
-                        pCur->Stat.cPeriods, pCur->Stat.cTicks, pCur->Stat.cTicksMin,
-                        pCur->Stat.cPeriods ? pCur->Stat.cTicks / pCur->Stat.cPeriods : 0, pCur->Stat.cTicksMax);
-#endif
-    return 0;
-}
-
-
-/**
- * Displays one virtual handler range.
- *
- * @returns 0
- * @param   pNode   Pointer to a PGMVIRTHANDLER.
- * @param   pvUser  Pointer to command helper functions.
- */
-static DECLCALLBACK(int) pgmR3InfoHandlersVirtualOne(PAVLROGCPTRNODECORE pNode, void *pvUser)
-{
-    PPGMVIRTHANDLER     pCur = (PPGMVIRTHANDLER)pNode;
-    PPGMHANDLERINFOARG  pArgs= (PPGMHANDLERINFOARG)pvUser;
-    PCDBGFINFOHLP       pHlp = pArgs->pHlp;
-    const char *pszType;
-    switch (pCur->enmType)
-    {
-        case PGMVIRTHANDLERTYPE_WRITE:      pszType = "Write  "; break;
-        case PGMVIRTHANDLERTYPE_ALL:        pszType = "All    "; break;
-        case PGMVIRTHANDLERTYPE_HYPERVISOR: pszType = "WriteHyp "; break;
-        default:                            pszType = "????"; break;
-    }
-    pHlp->pfnPrintf(pHlp, "%RGv - %RGv  %RHv  %RRv  %s  %s\n",
-        pCur->Core.Key, pCur->Core.KeyLast, pCur->pfnHandlerR3, pCur->pfnHandlerRC, pszType, pCur->pszDesc);
-#ifdef VBOX_WITH_STATISTICS
-    if (pArgs->fStats)
-        pHlp->pfnPrintf(pHlp, "   cPeriods: %9RU64  cTicks: %11RU64  Min: %11RU64  Avg: %11RU64 Max: %11RU64\n",
-                        pCur->Stat.cPeriods, pCur->Stat.cTicks, pCur->Stat.cTicksMin,
-                        pCur->Stat.cPeriods ? pCur->Stat.cTicks / pCur->Stat.cPeriods : 0, pCur->Stat.cTicksMax);
-#endif
-    return 0;
-}
-
Index: unk/src/VBox/VMM/PGMInline.h
===================================================================
--- /trunk/src/VBox/VMM/PGMInline.h	(revision 35332)
+++ 	(revision )
@@ -1,1542 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Inlined functions.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___PGMInline_h
-#define ___PGMInline_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/err.h>
-#include <VBox/stam.h>
-#include <VBox/param.h>
-#include <VBox/vmm.h>
-#include <VBox/mm.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/pdmapi.h>
-#include <VBox/dis.h>
-#include <VBox/dbgf.h>
-#include <VBox/log.h>
-#include <VBox/gmm.h>
-#include <VBox/hwaccm.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/avl.h>
-#include <iprt/critsect.h>
-#include <iprt/sha.h>
-
-
-
-/** @addtogroup grp_pgm_int   Internals
- * @internal
- * @{
- */
-
-/** @todo Split out all the inline stuff into a separate file.  Then we can
- *        include it later when VM and VMCPU are defined and so avoid all that
- *        &pVM->pgm.s and &pVCpu->pgm.s stuff.  It also chops ~1600 lines off
- *        this file and will make it somewhat easier to navigate... */
-
-/**
- * Gets the PGMRAMRANGE structure for a guest page.
- *
- * @returns Pointer to the RAM range on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- */
-DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-                break;
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    return pRam;
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page.
- *
- * @returns Pointer to the page on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- */
-DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-                return NULL;
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    return &pRam->aPages[off >> PAGE_SHIFT];
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page.
- *
- * Old Phys code: Will make sure the page is present.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS and a valid *ppPage on success.
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppPage      Where to store the page pointer on success.
- */
-DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-            {
-                *ppPage = NULL; /* avoid incorrect and very annoying GCC warnings */
-                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-            }
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
-    return VINF_SUCCESS;
-}
-
-
-
-
-/**
- * Gets the PGMPAGE structure for a guest page.
- *
- * Old Phys code: Will make sure the page is present.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS and a valid *ppPage on success.
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppPage      Where to store the page pointer on success.
- * @param   ppRamHint   Where to read and store the ram list hint.
- *                      The caller initializes this to NULL before the call.
- */
-DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
-{
-    RTGCPHYS off;
-    PPGMRAMRANGE pRam = *ppRamHint;
-    if (    !pRam
-        ||  RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
-    {
-        pRam = pPGM->CTX_SUFF(pRamRanges);
-        off = GCPhys - pRam->GCPhys;
-        if (RT_UNLIKELY(off >= pRam->cb))
-        {
-            do
-            {
-                pRam = pRam->CTX_SUFF(pNext);
-                if (RT_UNLIKELY(!pRam))
-                {
-                    *ppPage = NULL; /* Kill the incorrect and extremely annoying GCC warnings. */
-                    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-                }
-                off = GCPhys - pRam->GCPhys;
-            } while (off >= pRam->cb);
-        }
-        *ppRamHint = pRam;
-    }
-    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
- *
- * @returns Pointer to the page on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppRam       Where to store the pointer to the PGMRAMRANGE.
- */
-DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-                return NULL;
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    *ppRam = pRam;
-    return &pRam->aPages[off >> PAGE_SHIFT];
-}
-
-
-/**
- * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
- *
- * @returns Pointer to the page on success.
- * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
- *
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   ppPage      Where to store the pointer to the PGMPAGE structure.
- * @param   ppRam       Where to store the pointer to the PGMRAMRANGE structure.
- */
-DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
-{
-    /*
-     * Optimize for the first range.
-     */
-    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
-    RTGCPHYS off = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb))
-    {
-        do
-        {
-            pRam = pRam->CTX_SUFF(pNext);
-            if (RT_UNLIKELY(!pRam))
-            {
-                *ppRam = NULL;  /* Shut up silly GCC warnings. */
-                *ppPage = NULL; /* ditto */
-                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-            }
-            off = GCPhys - pRam->GCPhys;
-        } while (off >= pRam->cb);
-    }
-    *ppRam = pRam;
-    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Convert GC Phys to HC Phys.
- *
- * @returns VBox status.
- * @param   pPGM        PGM handle.
- * @param   GCPhys      The GC physical address.
- * @param   pHCPhys     Where to store the corresponding HC physical address.
- *
- * @deprecated  Doesn't deal with zero, shared or write monitored pages.
- *              Avoid when writing new code!
- */
-DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
-{
-    PPGMPAGE pPage;
-    int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
-    if (RT_FAILURE(rc))
-        return rc;
-    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
-    return VINF_SUCCESS;
-}
-
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
-
-/**
- * Inlined version of the ring-0 version of the host page mapping code
- * that optimizes access to pages already in the set.
- *
- * @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
- * @param   pVCpu       The current CPU.
- * @param   HCPhys      The physical address of the page.
- * @param   ppv         Where to store the mapping address.
- */
-DECLINLINE(int) pgmRZDynMapHCPageInlined(PVMCPU pVCpu, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
-{
-    PPGMMAPSET  pSet    = &pVCpu->pgm.s.AutoSet;
-
-    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
-    Assert(!(HCPhys & PAGE_OFFSET_MASK));
-    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
-
-    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
-    unsigned    iEntry  = pSet->aiHashTable[iHash];
-    if (    iEntry < pSet->cEntries
-        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
-        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
-    {
-        pSet->aEntries[iEntry].cInlinedRefs++;
-        *ppv = pSet->aEntries[iEntry].pvPage;
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlHits);
-    }
-    else
-    {
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlMisses);
-        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
-    }
-
-    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Inlined version of the guest page mapping code that optimizes access to pages
- * already in the set.
- *
- * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The current CPU.
- * @param   GCPhys      The guest physical address of the page.
- * @param   ppv         Where to store the mapping address.
- */
-DECLINLINE(int) pgmRZDynMapGCPageV2Inlined(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
-{
-    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
-    AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys));
-
-    /*
-     * Get the ram range.
-     */
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-    RTGCPHYS        off  = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb
-        /** @todo   || page state stuff */))
-    {
-        /* This case is not counted into StatRZDynMapGCPageInl. */
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
-        return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
-    }
-
-    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
-
-    /*
-     * pgmRZDynMapHCPageInlined with out stats.
-     */
-    PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
-    Assert(!(HCPhys & PAGE_OFFSET_MASK));
-    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
-
-    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
-    unsigned    iEntry  = pSet->aiHashTable[iHash];
-    if (    iEntry < pSet->cEntries
-        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
-        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
-    {
-        pSet->aEntries[iEntry].cInlinedRefs++;
-        *ppv = pSet->aEntries[iEntry].pvPage;
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
-    }
-    else
-    {
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
-        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
-    }
-
-    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Inlined version of the ring-0 version of guest page mapping that optimizes
- * access to pages already in the set.
- *
- * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
- * @param   pVCpu       The current CPU.
- * @param   GCPhys      The guest physical address of the page.
- * @param   ppv         Where to store the mapping address.
- */
-DECLINLINE(int) pgmRZDynMapGCPageInlined(PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
-{
-    return pgmRZDynMapGCPageV2Inlined(pVCpu->CTX_SUFF(pVM), pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
-}
-
-
-/**
- * Inlined version of the ring-0 version of the guest byte mapping code
- * that optimizes access to pages already in the set.
- *
- * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
- * @param   pVCpu       The current CPU.
- * @param   HCPhys      The physical address of the page.
- * @param   ppv         Where to store the mapping address. The offset is
- *                      preserved.
- */
-DECLINLINE(int) pgmRZDynMapGCPageOffInlined(PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
-{
-    STAM_PROFILE_START(&pVCpu->pgm.s.StatRZDynMapGCPageInl, a);
-
-    /*
-     * Get the ram range.
-     */
-    PVM             pVM  = pVCpu->CTX_SUFF(pVM);
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-    RTGCPHYS        off  = GCPhys - pRam->GCPhys;
-    if (RT_UNLIKELY(off >= pRam->cb
-        /** @todo   || page state stuff */))
-    {
-        /* This case is not counted into StatRZDynMapGCPageInl. */
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
-        return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
-    }
-
-    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
-    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
-
-    /*
-     * pgmRZDynMapHCPageInlined with out stats.
-     */
-    PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
-    Assert(!(HCPhys & PAGE_OFFSET_MASK));
-    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
-
-    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
-    unsigned    iEntry  = pSet->aiHashTable[iHash];
-    if (    iEntry < pSet->cEntries
-        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
-        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
-    {
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
-        pSet->aEntries[iEntry].cInlinedRefs++;
-        *ppv = (void *)((uintptr_t)pSet->aEntries[iEntry].pvPage | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
-    }
-    else
-    {
-        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
-        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
-        *ppv = (void *)((uintptr_t)*ppv | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
-    }
-
-    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
-    return VINF_SUCCESS;
-}
-
-#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
-
-/**
- * Maps the page into current context (RC and maybe R0).
- *
- * @returns pointer to the mapping.
- * @param   pVM         Pointer to the PGM instance data.
- * @param   pPage       The page.
- */
-DECLINLINE(void *) pgmPoolMapPageInlined(PVM pVM, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
-{
-    if (pPage->idx >= PGMPOOL_IDX_FIRST)
-    {
-        Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
-        void *pv;
-        pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
-        return pv;
-    }
-    AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
-}
-
-/**
- * Maps the page into current context (RC and maybe R0).
- *
- * @returns pointer to the mapping.
- * @param   pVM         Pointer to the PGM instance data.
- * @param   pVCpu       The current CPU.
- * @param   pPage       The page.
- */
-DECLINLINE(void *) pgmPoolMapPageV2Inlined(PVM pVM, PVMCPU pVCpu, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
-{
-    if (pPage->idx >= PGMPOOL_IDX_FIRST)
-    {
-        Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
-        void *pv;
-        Assert(pVCpu == VMMGetCpu(pVM));
-        pgmRZDynMapHCPageInlined(pVCpu, pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
-        return pv;
-    }
-    AssertFatalMsgFailed(("pgmPoolMapPageV2Inlined invalid page index %x\n", pPage->idx));
-}
-
-#endif /*  VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 || IN_RC */
-#ifndef IN_RC
-
-/**
- * Queries the Physical TLB entry for a physical guest page,
- * attempting to load the TLB entry if necessary.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
- *
- * @param   pPGM        The PGM instance handle.
- * @param   GCPhys      The address of the guest page.
- * @param   ppTlbe      Where to store the pointer to the TLB entry.
- */
-DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
-{
-    int rc;
-    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
-    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
-    {
-        STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
-        rc = VINF_SUCCESS;
-    }
-    else
-        rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
-    *ppTlbe = pTlbe;
-    return rc;
-}
-
-
-/**
- * Queries the Physical TLB entry for a physical guest page,
- * attempting to load the TLB entry if necessary.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
- *
- * @param   pPGM        The PGM instance handle.
- * @param   pPage       Pointer to the PGMPAGE structure corresponding to
- *                      GCPhys.
- * @param   GCPhys      The address of the guest page.
- * @param   ppTlbe      Where to store the pointer to the TLB entry.
- */
-DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
-{
-    int rc;
-    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
-    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
-    {
-        STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
-        rc = VINF_SUCCESS;
-    }
-    else
-        rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
-    *ppTlbe = pTlbe;
-    return rc;
-}
-
-#endif /* !IN_RC */
-
-
-/**
- * Checks if the no-execute (NX) feature is active (EFER.NXE=1).
- *
- * Only used when the guest is in PAE or long mode.  This is inlined so that we
- * can perform consistency checks in debug builds.
- *
- * @returns true if it is, false if it isn't.
- * @param   pVCpu       The current CPU.
- */
-DECL_FORCE_INLINE(bool) pgmGstIsNoExecuteActive(PVMCPU pVCpu)
-{
-    Assert(pVCpu->pgm.s.fNoExecuteEnabled == CPUMIsGuestNXEnabled(pVCpu));
-    Assert(CPUMIsGuestInPAEMode(pVCpu) || CPUMIsGuestInLongMode(pVCpu));
-    return pVCpu->pgm.s.fNoExecuteEnabled;
-}
-
-
-/**
- * Checks if the page size extension (PSE) is currently enabled (CR4.PSE=1).
- *
- * Only used when the guest is in paged 32-bit mode.  This is inlined so that
- * we can perform consistency checks in debug builds.
- *
- * @returns true if it is, false if it isn't.
- * @param   pVCpu       The current CPU.
- */
-DECL_FORCE_INLINE(bool) pgmGst32BitIsPageSizeExtActive(PVMCPU pVCpu)
-{
-    Assert(pVCpu->pgm.s.fGst32BitPageSizeExtension == CPUMIsGuestPageSizeExtEnabled(pVCpu));
-    Assert(!CPUMIsGuestInPAEMode(pVCpu));
-    Assert(!CPUMIsGuestInLongMode(pVCpu));
-    return pVCpu->pgm.s.fGst32BitPageSizeExtension;
-}
-
-
-/**
- * Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
- * Takes PSE-36 into account.
- *
- * @returns guest physical address
- * @param   pPGM        Pointer to the PGM instance data.
- * @param   Pde         Guest Pde
- */
-DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
-{
-    RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
-    GCPhys |= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
-
-    return GCPhys & pPGM->GCPhys4MBPSEMask;
-}
-
-
-/**
- * Gets the address the guest page directory (32-bit paging).
- *
- * @returns VBox status code.
- * @param   pVCpu       The current CPU.
- * @param   ppPd        Where to return the mapping. This is always set.
- */
-DECLINLINE(int) pgmGstGet32bitPDPtrEx(PVMCPU pVCpu, PX86PD *ppPd)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPd RTLOG_COMMA_SRC_POS);
-    if (RT_FAILURE(rc))
-    {
-        *ppPd = NULL;
-        return rc;
-    }
-#else
-    *ppPd = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
-    if (RT_UNLIKELY(!*ppPd))
-        return pgmGstLazyMap32BitPD(pVCpu, ppPd);
-#endif
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets the address the guest page directory (32-bit paging).
- *
- * @returns Pointer the page directory entry in question.
- * @param   pVCpu       The current CPU.
- */
-DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PVMCPU pVCpu)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PD pGuestPD = NULL;
-    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPD RTLOG_COMMA_SRC_POS);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
-        return NULL;
-    }
-#else
-    PX86PD pGuestPD = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
-    if (RT_UNLIKELY(!pGuestPD))
-    {
-        int rc = pgmGstLazyMap32BitPD(pVCpu, &pGuestPD);
-        if (RT_FAILURE(rc))
-            return NULL;
-    }
-#endif
-    return pGuestPD;
-}
-
-
-/**
- * Gets the guest page directory pointer table.
- *
- * @returns VBox status code.
- * @param   pVCpu       The current CPU.
- * @param   ppPdpt      Where to return the mapping.  This is always set.
- */
-DECLINLINE(int) pgmGstGetPaePDPTPtrEx(PVMCPU pVCpu, PX86PDPT *ppPdpt)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPdpt RTLOG_COMMA_SRC_POS);
-    if (RT_FAILURE(rc))
-    {
-        *ppPdpt = NULL;
-        return rc;
-    }
-#else
-    *ppPdpt = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
-    if (RT_UNLIKELY(!*ppPdpt))
-        return pgmGstLazyMapPaePDPT(pVCpu, ppPdpt);
-#endif
-    return VINF_SUCCESS;
-}
-
-/**
- * Gets the guest page directory pointer table.
- *
- * @returns Pointer to the page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pVCpu       The current CPU.
- */
-DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PVMCPU pVCpu)
-{
-    PX86PDPT pGuestPdpt;
-    int rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGuestPdpt);
-    AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
-    return pGuestPdpt;
-}
-
-
-/**
- * Gets the guest page directory pointer table entry for the specified address.
- *
- * @returns Pointer to the page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pVCpu       The current CPU
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    AssertGCPtr32(GCPtr);
-
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PDPT pGuestPDPT = NULL;
-    int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPDPT RTLOG_COMMA_SRC_POS);
-    AssertRCReturn(rc, NULL);
-#else
-    PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
-    if (RT_UNLIKELY(!pGuestPDPT))
-    {
-        int rc = pgmGstLazyMapPaePDPT(pVCpu, &pGuestPDPT);
-        if (RT_FAILURE(rc))
-            return NULL;
-    }
-#endif
-    return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
-}
-
-
-/**
- * Gets the page directory entry for the specified address.
- *
- * @returns The page directory entry in question.
- * @returns A non-present entry if the page directory is not present or on an invalid page.
- * @param   pVCpu       The handle of the virtual CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    AssertGCPtr32(GCPtr);
-    PX86PDPT    pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
-    if (RT_LIKELY(pGuestPDPT))
-    {
-        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-        if (    pGuestPDPT->a[iPdpt].n.u1Present
-            &&  !(pGuestPDPT->a[iPdpt].u & pVCpu->pgm.s.fGstPaeMbzPdpeMask) )
-        {
-            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-            PX86PDPAE   pGuestPD = NULL;
-            int rc = pgmRZDynMapGCPageInlined(pVCpu,
-                                              pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
-                                              (void **)&pGuestPD
-                                              RTLOG_COMMA_SRC_POS);
-            if (RT_SUCCESS(rc))
-                return pGuestPD->a[iPD];
-            AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
-#else
-            PX86PDPAE   pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
-            if (    !pGuestPD
-                ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
-                pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
-            if (pGuestPD)
-                return pGuestPD->a[iPD];
-#endif
-        }
-    }
-
-    X86PDEPAE ZeroPde = {0};
-    return ZeroPde;
-}
-
-
-/**
- * Gets the page directory pointer table entry for the specified address
- * and returns the index into the page directory
- *
- * @returns Pointer to the page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- * @param   piPD        Receives the index into the returned page directory
- * @param   pPdpe       Receives the page directory pointer entry. Optional.
- */
-DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
-{
-    AssertGCPtr32(GCPtr);
-
-    /* The PDPE. */
-    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
-    if (RT_UNLIKELY(!pGuestPDPT))
-        return NULL;
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-    if (pPdpe)
-        *pPdpe = pGuestPDPT->a[iPdpt];
-    if (!pGuestPDPT->a[iPdpt].n.u1Present)
-        return NULL;
-    if (RT_UNLIKELY(pVCpu->pgm.s.fGstPaeMbzPdpeMask & pGuestPDPT->a[iPdpt].u))
-        return NULL;
-
-    /* The PDE. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PDPAE   pGuestPD = NULL;
-    int rc = pgmRZDynMapGCPageInlined(pVCpu,
-                                      pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
-                                      (void **)&pGuestPD
-                                      RTLOG_COMMA_SRC_POS);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
-        return NULL;
-    }
-#else
-    PX86PDPAE   pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
-    if (    !pGuestPD
-        ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
-        pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
-#endif
-
-    *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-    return pGuestPD;
-}
-
-#ifndef IN_RC
-
-/**
- * Gets the page map level-4 pointer for the guest.
- *
- * @returns VBox status code.
- * @param   pVCpu       The current CPU.
- * @param   ppPml4      Where to return the mapping.  Always set.
- */
-DECLINLINE(int) pgmGstGetLongModePML4PtrEx(PVMCPU pVCpu, PX86PML4 *ppPml4)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPml4 RTLOG_COMMA_SRC_POS);
-    if (RT_FAILURE(rc))
-    {
-        *ppPml4 = NULL;
-        return rc;
-    }
-#else
-    *ppPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
-    if (RT_UNLIKELY(!*ppPml4))
-        return pgmGstLazyMapPml4(pVCpu, ppPml4);
-#endif
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets the page map level-4 pointer for the guest.
- *
- * @returns Pointer to the PML4 page.
- * @param   pVCpu       The current CPU.
- */
-DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PVMCPU pVCpu)
-{
-    PX86PML4 pGuestPml4;
-    int rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGuestPml4);
-    AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc);
-    return pGuestPml4;
-}
-
-
-/**
- * Gets the pointer to a page map level-4 entry.
- *
- * @returns Pointer to the PML4 entry.
- * @param   pVCpu       The current CPU.
- * @param   iPml4       The index.
- * @remarks Only used by AssertCR3.
- */
-DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PVMCPU pVCpu, unsigned int iPml4)
-{
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
-    PX86PML4 pGuestPml4;
-    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPml4 RTLOG_COMMA_SRC_POS);
-    AssertRCReturn(rc, NULL);
-#else
-    PX86PML4 pGuestPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
-    if (RT_UNLIKELY(!pGuestPml4))
-    {
-         int rc = pgmGstLazyMapPml4(pVCpu, &pGuestPml4);
-         AssertRCReturn(rc, NULL);
-    }
-#endif
-    return &pGuestPml4->a[iPml4];
-}
-
-
-/**
- * Gets the page directory entry for the specified address.
- *
- * @returns The page directory entry in question.
- * @returns A non-present entry if the page directory is not present or on an invalid page.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PVMCPU pVCpu, RTGCPTR64 GCPtr)
-{
-    /*
-     * Note! To keep things simple, ASSUME invalid physical addresses will
-     *       cause X86_TRAP_PF_RSVD.  This isn't a problem until we start
-     *       supporting 52-bit wide physical guest addresses.
-     */
-    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
-    const unsigned  iPml4      = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    if (    RT_LIKELY(pGuestPml4)
-        &&  pGuestPml4->a[iPml4].n.u1Present
-        &&  !(pGuestPml4->a[iPml4].u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask) )
-    {
-        PCX86PDPT   pPdptTemp;
-        int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
-        if (RT_SUCCESS(rc))
-        {
-            const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-            if (    pPdptTemp->a[iPdpt].n.u1Present
-                &&  !(pPdptTemp->a[iPdpt].u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask) )
-            {
-                PCX86PDPAE pPD;
-                rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
-                if (RT_SUCCESS(rc))
-                {
-                    const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-                    return pPD->a[iPD];
-                }
-            }
-        }
-        AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
-    }
-
-    X86PDEPAE ZeroPde = {0};
-    return ZeroPde;
-}
-
-
-/**
- * Gets the GUEST page directory pointer for the specified address.
- *
- * @returns The page directory in question.
- * @returns NULL if the page directory is not present or on an invalid page.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- * @param   ppPml4e     Page Map Level-4 Entry (out)
- * @param   pPdpe       Page directory pointer table entry (out)
- * @param   piPD        Receives the index into the returned page directory
- */
-DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe, unsigned *piPD)
-{
-    /* The PMLE4. */
-    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
-    if (RT_UNLIKELY(!pGuestPml4))
-        return NULL;
-    const unsigned  iPml4      = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    PCX86PML4E      pPml4e     = *ppPml4e = &pGuestPml4->a[iPml4];
-    if (!pPml4e->n.u1Present)
-        return NULL;
-    if (RT_UNLIKELY(pPml4e->u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask))
-        return NULL;
-
-    /* The PDPE. */
-    PCX86PDPT       pPdptTemp;
-    int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
-        return NULL;
-    }
-    const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
-    *pPdpe = pPdptTemp->a[iPdpt];
-    if (!pPdpe->n.u1Present)
-        return NULL;
-    if (RT_UNLIKELY(pPdpe->u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask))
-        return NULL;
-
-    /* The PDE. */
-    PX86PDPAE pPD;
-    rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
-        return NULL;
-    }
-
-    *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-    return pPD;
-}
-
-#endif /* !IN_RC */
-
-/**
- * Gets the shadow page directory, 32-bit.
- *
- * @returns Pointer to the shadow 32-bit PD.
- * @param   pVCpu       The current CPU.
- */
-DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PVMCPU pVCpu)
-{
-    return (PX86PD)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
-}
-
-
-/**
- * Gets the shadow page directory entry for the specified address, 32-bit.
- *
- * @returns Shadow 32-bit PDE.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDE) pgmShwGet32BitPDE(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
-
-    PX86PD pShwPde = pgmShwGet32BitPDPtr(pVCpu);
-    if (!pShwPde)
-    {
-        X86PDE ZeroPde = {0};
-        return ZeroPde;
-    }
-    return pShwPde->a[iPd];
-}
-
-
-/**
- * Gets the pointer to the shadow page directory entry for the specified
- * address, 32-bit.
- *
- * @returns Pointer to the shadow 32-bit PDE.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
-
-    PX86PD pPde = pgmShwGet32BitPDPtr(pVCpu);
-    AssertReturn(pPde, NULL);
-    return &pPde->a[iPd];
-}
-
-
-/**
- * Gets the shadow page pointer table, PAE.
- *
- * @returns Pointer to the shadow PAE PDPT.
- * @param   pVCpu       The current CPU.
- */
-DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PVMCPU pVCpu)
-{
-    return (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
-}
-
-
-/**
- * Gets the shadow page directory for the specified address, PAE.
- *
- * @returns Pointer to the shadow PD.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-    PX86PDPT        pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
-
-    if (!pPdpt->a[iPdpt].n.u1Present)
-        return NULL;
-
-    /* Fetch the pgm pool shadow descriptor. */
-    PVM pVM = pVCpu->CTX_SUFF(pVM);
-    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
-    AssertReturn(pShwPde, NULL);
-
-    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
-}
-
-
-/**
- * Gets the shadow page directory for the specified address, PAE.
- *
- * @returns Pointer to the shadow PD.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPU pVCpu, PX86PDPT pPdpt, RTGCPTR GCPtr)
-{
-    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-
-    if (!pPdpt->a[iPdpt].n.u1Present)
-        return NULL;
-
-    /* Fetch the pgm pool shadow descriptor. */
-    PVM             pVM     = pVCpu->CTX_SUFF(pVM);
-    PPGMPOOLPAGE    pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
-    AssertReturn(pShwPde, NULL);
-
-    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
-}
-
-
-/**
- * Gets the shadow page directory entry, PAE.
- *
- * @returns PDE.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-
-    PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
-    if (!pShwPde)
-    {
-        X86PDEPAE ZeroPde = {0};
-        return ZeroPde;
-    }
-    return pShwPde->a[iPd];
-}
-
-
-/**
- * Gets the pointer to the shadow page directory entry for an address, PAE.
- *
- * @returns Pointer to the PDE.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- * @remarks Only used by AssertCR3.
- */
-DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-
-    PX86PDPAE pPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
-    AssertReturn(pPde, NULL);
-    return &pPde->a[iPd];
-}
-
-#ifndef IN_RC
-
-/**
- * Gets the shadow page map level-4 pointer.
- *
- * @returns Pointer to the shadow PML4.
- * @param   pVCpu       The current CPU.
- */
-DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PVMCPU pVCpu)
-{
-    return (PX86PML4)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
-}
-
-
-/**
- * Gets the shadow page map level-4 entry for the specified address.
- *
- * @returns The entry.
- * @param   pVCpu       The current CPU.
- * @param   GCPtr       The address.
- */
-DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PVMCPU pVCpu, RTGCPTR GCPtr)
-{
-    const unsigned  iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
-    PX86PML4        pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
-
-    if (!pShwPml4)
-    {
-        X86PML4E ZeroPml4e = {0};
-        return ZeroPml4e;
-    }
-    return pShwPml4->a[iPml4];
-}
-
-
-/**
- * Gets the pointer to the specified shadow page map level-4 entry.
- *
- * @returns The entry.
- * @param   pVCpu       The current CPU.
- * @param   iPml4       The PML4 index.
- */
-DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PVMCPU pVCpu, unsigned int iPml4)
-{
-    PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
-    if (!pShwPml4)
-        return NULL;
-    return &pShwPml4->a[iPml4];
-}
-
-#endif /* !IN_RC */
-
-
-/**
- * Cached physical handler lookup.
- *
- * @returns Physical handler covering @a GCPhys.
- * @param   pVM                 The VM handle.
- * @param   GCPhys              The lookup address.
- */
-DECLINLINE(PPGMPHYSHANDLER) pgmHandlerPhysicalLookup(PVM pVM, RTGCPHYS GCPhys)
-{
-    PPGMPHYSHANDLER pHandler = pVM->pgm.s.CTX_SUFF(pLastPhysHandler);
-    if (   pHandler
-        && GCPhys >= pHandler->Core.Key
-        && GCPhys < pHandler->Core.KeyLast)
-    {
-        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupHits));
-        return pHandler;
-    }
-
-    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupMisses));
-    pHandler = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
-    if (pHandler)
-        pVM->pgm.s.CTX_SUFF(pLastPhysHandler) = pHandler;
-    return pHandler;
-}
-
-
-/**
- * Gets the page state for a physical handler.
- *
- * @returns The physical handler page state.
- * @param   pCur    The physical handler in question.
- */
-DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
-{
-    switch (pCur->enmType)
-    {
-        case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
-            return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
-
-        case PGMPHYSHANDLERTYPE_MMIO:
-        case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
-            return PGM_PAGE_HNDL_PHYS_STATE_ALL;
-
-        default:
-            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
-    }
-}
-
-
-/**
- * Gets the page state for a virtual handler.
- *
- * @returns The virtual handler page state.
- * @param   pCur    The virtual handler in question.
- * @remarks This should never be used on a hypervisor access handler.
- */
-DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
-{
-    switch (pCur->enmType)
-    {
-        case PGMVIRTHANDLERTYPE_WRITE:
-            return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
-        case PGMVIRTHANDLERTYPE_ALL:
-            return PGM_PAGE_HNDL_VIRT_STATE_ALL;
-        default:
-            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
-    }
-}
-
-
-/**
- * Clears one physical page of a virtual handler
- *
- * @param   pPGM    Pointer to the PGM instance.
- * @param   pCur    Virtual handler structure
- * @param   iPage   Physical page index
- *
- * @remark  Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
- *          need to care about other handlers in the same page.
- */
-DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
-{
-    const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
-
-    /*
-     * Remove the node from the tree (it's supposed to be in the tree if we get here!).
-     */
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-    AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
-                     ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                      pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
-#endif
-    if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
-    {
-        /* We're the head of the alias chain. */
-        PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-        AssertReleaseMsg(pRemove != NULL,
-                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
-        AssertReleaseMsg(pRemove == pPhys2Virt,
-                         ("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
-                          "   got:    pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
-                          pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
-#endif
-        if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
-        {
-            /* Insert the next list in the alias chain into the tree. */
-            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-            AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
-                             ("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
-                             pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
-#endif
-            pNext->offNextAlias |= PGMPHYS2VIRTHANDLER_IS_HEAD;
-            bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
-            AssertRelease(fRc);
-        }
-    }
-    else
-    {
-        /* Locate the previous node in the alias chain. */
-        PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-        AssertReleaseMsg(pPrev != pPhys2Virt,
-                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
-                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
-#endif
-        for (;;)
-        {
-            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
-            if (pNext == pPhys2Virt)
-            {
-                /* unlink. */
-                LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
-                         pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
-                if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
-                    pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
-                else
-                {
-                    PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
-                    pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
-                                        | (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
-                }
-                break;
-            }
-
-            /* next */
-            if (pNext == pPrev)
-            {
-#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
-                AssertReleaseMsg(pNext != pPrev,
-                                 ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
-                                  pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
-#endif
-                break;
-            }
-            pPrev = pNext;
-        }
-    }
-    Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
-          pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
-    pPhys2Virt->offNextAlias = 0;
-    pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
-
-    /*
-     * Clear the ram flags for this page.
-     */
-    PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
-    AssertReturnVoid(pPage);
-    PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
-}
-
-
-/**
- * Internal worker for finding a 'in-use' shadow page give by it's physical address.
- *
- * @returns Pointer to the shadow page structure.
- * @param   pPool       The pool.
- * @param   idx         The pool page index.
- */
-DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
-{
-    AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
-    return &pPool->aPages[idx];
-}
-
-
-/**
- * Clear references to guest physical memory.
- *
- * @param   pPool       The pool.
- * @param   pPoolPage   The pool page.
- * @param   pPhysPage   The physical guest page tracking structure.
- * @param   iPte        Shadow PTE index
- */
-DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte)
-{
-    /*
-     * Just deal with the simple case here.
-     */
-# ifdef LOG_ENABLED
-    const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
-# endif
-    const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
-    if (cRefs == 1)
-    {
-        Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
-        Assert(iPte == PGM_PAGE_GET_PTE_INDEX(pPhysPage));
-        /* Invalidate the tracking data. */
-        PGM_PAGE_SET_TRACKING(pPhysPage, 0);
-    }
-    else
-        pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage, iPte);
-    Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
-}
-
-
-/**
- * Moves the page to the head of the age list.
- *
- * This is done when the cached page is used in one way or another.
- *
- * @param   pPool       The pool.
- * @param   pPage       The cached page.
- */
-DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
-{
-    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
-
-    /*
-     * Move to the head of the age list.
-     */
-    if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
-    {
-        /* unlink */
-        pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
-        if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
-            pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
-        else
-            pPool->iAgeTail = pPage->iAgePrev;
-
-        /* insert at head */
-        pPage->iAgePrev = NIL_PGMPOOL_IDX;
-        pPage->iAgeNext = pPool->iAgeHead;
-        Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
-        pPool->iAgeHead = pPage->idx;
-        pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
-    }
-}
-
-/**
- * Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
- *
- * @param   pVM         VM Handle.
- * @param   pPage       PGM pool page
- */
-DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
-{
-    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
-    ASMAtomicIncU32(&pPage->cLocked);
-}
-
-
-/**
- * Unlocks a page to allow flushing again
- *
- * @param   pVM         VM Handle.
- * @param   pPage       PGM pool page
- */
-DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
-{
-    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
-    Assert(pPage->cLocked);
-    ASMAtomicDecU32(&pPage->cLocked);
-}
-
-
-/**
- * Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
- *
- * @returns VBox status code.
- * @param   pPage       PGM pool page
- */
-DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
-{
-    if (pPage->cLocked)
-    {
-        LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
-        if (pPage->cModifications)
-            pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
-        return true;
-    }
-    return false;
-}
-
-
-/**
- * Tells if mappings are to be put into the shadow page table or not.
- *
- * @returns boolean result
- * @param   pVM         VM handle.
- */
-DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
-{
-#ifdef PGM_WITHOUT_MAPPINGS
-    /* There are no mappings in VT-x and AMD-V mode. */
-    Assert(pPGM->fMappingsDisabled);
-    return false;
-#else
-    return !pPGM->fMappingsDisabled;
-#endif
-}
-
-
-/**
- * Checks if the mappings are floating and enabled.
- *
- * @returns true / false.
- * @param   pVM         The VM handle.
- */
-DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PPGM pPGM)
-{
-#ifdef PGM_WITHOUT_MAPPINGS
-    /* There are no mappings in VT-x and AMD-V mode. */
-    Assert(pPGM->fMappingsDisabled);
-    return false;
-#else
-    return !pPGM->fMappingsDisabled
-        && !pPGM->fMappingsFixed;
-#endif
-}
-
-/** @} */
-
-#endif
-
Index: unk/src/VBox/VMM/PGMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/PGMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,3866 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___PGMInternal_h
-#define ___PGMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/err.h>
-#include <VBox/dbg.h>
-#include <VBox/stam.h>
-#include <VBox/param.h>
-#include <VBox/vmm.h>
-#include <VBox/mm.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/pdmapi.h>
-#include <VBox/dis.h>
-#include <VBox/dbgf.h>
-#include <VBox/log.h>
-#include <VBox/gmm.h>
-#include <VBox/hwaccm.h>
-#include <VBox/hwacc_vmx.h>
-#include <include/internal/pgm.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/avl.h>
-#include <iprt/critsect.h>
-#include <iprt/sha.h>
-
-
-
-/** @defgroup grp_pgm_int   Internals
- * @ingroup grp_pgm
- * @internal
- * @{
- */
-
-
-/** @name PGM Compile Time Config
- * @{
- */
-
-/**
- * Indicates that there are no guest mappings to care about.
- * Currently on raw-mode related code uses mappings, i.e. RC and R3 code.
- */
-#if defined(IN_RING0) || !defined(VBOX_WITH_RAW_MODE)
-# define PGM_WITHOUT_MAPPINGS
-#endif
-
-/**
- * Check and skip global PDEs for non-global flushes
- */
-#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
-
-/**
- * Optimization for PAE page tables that are modified often
- */
-//#if 0 /* disabled again while debugging */
-#ifndef IN_RC
-# define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
-#endif
-//#endif
-
-/**
- * Large page support enabled only on 64 bits hosts; applies to nested paging only.
- */
-#if (HC_ARCH_BITS == 64) && !defined(IN_RC)
-# define PGM_WITH_LARGE_PAGES
-#endif
-
-/**
- * Enables optimizations for MMIO handlers that exploits X86_TRAP_PF_RSVD and
- * VMX_EXIT_EPT_MISCONFIG.
- */
-#if 1 /* testing */
-# define PGM_WITH_MMIO_OPTIMIZATIONS
-#endif
-
-/**
- * Chunk unmapping code activated on 32-bit hosts for > 1.5/2 GB guest memory support
- */
-#if (HC_ARCH_BITS == 32) && !defined(RT_OS_DARWIN)
-# define PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
-#endif
-
-/**
- * Sync N pages instead of a whole page table
- */
-#define PGM_SYNC_N_PAGES
-
-/**
- * Number of pages to sync during a page fault
- *
- * When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
- * causes a lot of unnecessary extents and also is slower than taking more \#PFs.
- *
- * Note that \#PFs are much more expensive in the VT-x/AMD-V case due to
- * world switch overhead, so let's sync more.
- */
-# ifdef IN_RING0
-/* Chose 32 based on the compile test in #4219; 64 shows worse stats.
- * 32 again shows better results than 16; slightly more overhead in the \#PF handler,
- * but ~5% fewer faults.
- */
-# define PGM_SYNC_NR_PAGES               32
-#else
-# define PGM_SYNC_NR_PAGES               8
-#endif
-
-/**
- * Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
- */
-#define PGM_MAX_PHYSCACHE_ENTRIES       64
-#define PGM_MAX_PHYSCACHE_ENTRIES_MASK  (PGM_MAX_PHYSCACHE_ENTRIES-1)
-
-
-/** @def PGMPOOL_CFG_MAX_GROW
- * The maximum number of pages to add to the pool in one go.
- */
-#define PGMPOOL_CFG_MAX_GROW            (_256K >> PAGE_SHIFT)
-
-/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
- * Enables some extra assertions for virtual handlers (mainly phys2virt related).
- */
-#ifdef VBOX_STRICT
-# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
-#endif
-
-/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
- * Enables the experimental lazy page allocation code. */
-/*#define VBOX_WITH_NEW_LAZY_PAGE_ALLOC */
-
-/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES
- * Enables real write monitoring of pages, i.e. mapping them read-only and
- * only making them writable when getting a write access #PF. */
-#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES
-
-/** @} */
-
-
-/** @name PDPT and PML4 flags.
- * These are placed in the three bits available for system programs in
- * the PDPT and PML4 entries.
- * @{ */
-/** The entry is a permanent one and it's must always be present.
- * Never free such an entry. */
-#define PGM_PLXFLAGS_PERMANENT          RT_BIT_64(10)
-/** Mapping (hypervisor allocated pagetable). */
-#define PGM_PLXFLAGS_MAPPING            RT_BIT_64(11)
-/** @} */
-
-/** @name Page directory flags.
- * These are placed in the three bits available for system programs in
- * the page directory entries.
- * @{ */
-/** Mapping (hypervisor allocated pagetable). */
-#define PGM_PDFLAGS_MAPPING             RT_BIT_64(10)
-/** Made read-only to facilitate dirty bit tracking. */
-#define PGM_PDFLAGS_TRACK_DIRTY         RT_BIT_64(11)
-/** @} */
-
-/** @name Page flags.
- * These are placed in the three bits available for system programs in
- * the page entries.
- * @{ */
-/** Made read-only to facilitate dirty bit tracking. */
-#define PGM_PTFLAGS_TRACK_DIRTY         RT_BIT_64(9)
-
-#ifndef PGM_PTFLAGS_CSAM_VALIDATED
-/** Scanned and approved by CSAM (tm).
- * NOTE: Must be identical to the one defined in CSAMInternal.h!!
- * @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
-#define PGM_PTFLAGS_CSAM_VALIDATED      RT_BIT_64(11)
-#endif
-
-/** @} */
-
-/** @name Defines used to indicate the shadow and guest paging in the templates.
- * @{ */
-#define PGM_TYPE_REAL                   1
-#define PGM_TYPE_PROT                   2
-#define PGM_TYPE_32BIT                  3
-#define PGM_TYPE_PAE                    4
-#define PGM_TYPE_AMD64                  5
-#define PGM_TYPE_NESTED                 6
-#define PGM_TYPE_EPT                    7
-#define PGM_TYPE_MAX                    PGM_TYPE_EPT
-/** @} */
-
-/** Macro for checking if the guest is using paging.
- * @param uGstType     PGM_TYPE_*
- * @param uShwType     PGM_TYPE_*
- * @remark  ASSUMES certain order of the PGM_TYPE_* values.
- */
-#define PGM_WITH_PAGING(uGstType, uShwType)  \
-    (   (uGstType) >= PGM_TYPE_32BIT \
-     && (uShwType) != PGM_TYPE_NESTED \
-     && (uShwType) != PGM_TYPE_EPT)
-
-/** Macro for checking if the guest supports the NX bit.
- * @param uGstType     PGM_TYPE_*
- * @param uShwType     PGM_TYPE_*
- * @remark  ASSUMES certain order of the PGM_TYPE_* values.
- */
-#define PGM_WITH_NX(uGstType, uShwType)  \
-    (   (uGstType) >= PGM_TYPE_PAE \
-     && (uShwType) != PGM_TYPE_NESTED \
-     && (uShwType) != PGM_TYPE_EPT)
-
-
-/** @def PGM_HCPHYS_2_PTR
- * Maps a HC physical page pool address to a virtual address.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The current CPU.
- * @param   HCPhys      The HC physical address to map to a virtual one.
- * @param   ppv         Where to store the virtual address. No need to cast
- *                      this.
- *
- * @remark  Use with care as we don't have so much dynamic mapping space in
- *          ring-0 on 32-bit darwin and in RC.
- * @remark  There is no need to assert on the result.
- */
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
-# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
-     pgmRZDynMapHCPageInlined(pVCpu, HCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
-#else
-# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
-     MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
-#endif
-
-/** @def PGM_GCPHYS_2_PTR_V2
- * Maps a GC physical page address to a virtual address.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The current CPU.
- * @param   GCPhys  The GC physical address to map to a virtual one.
- * @param   ppv     Where to store the virtual address. No need to cast this.
- *
- * @remark  Use with care as we don't have so much dynamic mapping space in
- *          ring-0 on 32-bit darwin and in RC.
- * @remark  There is no need to assert on the result.
- */
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
-# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
-     pgmRZDynMapGCPageV2Inlined(pVM, pVCpu, GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
-#else
-# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
-     PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only */, (PRTR3PTR)(ppv)) /** @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
-#endif
-
-/** @def PGM_GCPHYS_2_PTR
- * Maps a GC physical page address to a virtual address.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   GCPhys  The GC physical address to map to a virtual one.
- * @param   ppv     Where to store the virtual address. No need to cast this.
- *
- * @remark  Use with care as we don't have so much dynamic mapping space in
- *          ring-0 on 32-bit darwin and in RC.
- * @remark  There is no need to assert on the result.
- */
-#define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2(pVM, VMMGetCpu(pVM), GCPhys, ppv)
-
-/** @def PGM_GCPHYS_2_PTR_BY_VMCPU
- * Maps a GC physical page address to a virtual address.
- *
- * @returns VBox status code.
- * @param   pVCpu   The current CPU.
- * @param   GCPhys  The GC physical address to map to a virtual one.
- * @param   ppv     Where to store the virtual address. No need to cast this.
- *
- * @remark  Use with care as we don't have so much dynamic mapping space in
- *          ring-0 on 32-bit darwin and in RC.
- * @remark  There is no need to assert on the result.
- */
-#define PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2((pVCpu)->CTX_SUFF(pVM), pVCpu, GCPhys, ppv)
-
-/** @def PGM_GCPHYS_2_PTR_EX
- * Maps a unaligned GC physical page address to a virtual address.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   GCPhys  The GC physical address to map to a virtual one.
- * @param   ppv     Where to store the virtual address. No need to cast this.
- *
- * @remark  Use with care as we don't have so much dynamic mapping space in
- *          ring-0 on 32-bit darwin and in RC.
- * @remark  There is no need to assert on the result.
- */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
-# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
-     pgmRZDynMapGCPageOffInlined(VMMGetCpu(pVM), GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
-#else
-# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
-     PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only */, (PRTR3PTR)(ppv)) /** @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
-#endif
-
-/** @def PGM_DYNMAP_UNUSED_HINT
- * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
- * is no longer used.
- *
- * For best effect only apply this to the page that was mapped most recently.
- *
- * @param   pVCpu   The current CPU.
- * @param   pPage   The pool page.
- */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
-# ifdef LOG_ENABLED
-#  define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  pgmRZDynMapUnusedHint(pVCpu, pvPage, RT_SRC_POS)
-# else
-#  define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  pgmRZDynMapUnusedHint(pVCpu, pvPage)
-# endif
-#else
-# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  do {} while (0)
-#endif
-
-/** @def PGM_DYNMAP_UNUSED_HINT_VM
- * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
- * is no longer used.
- *
- * For best effect only apply this to the page that was mapped most recently.
- *
- * @param   pVM     The VM handle.
- * @param   pPage   The pool page.
- */
-#define PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvPage)  PGM_DYNMAP_UNUSED_HINT(VMMGetCpu(pVM), pvPage)
-
-
-/** @def PGM_INVL_PG
- * Invalidates a page.
- *
- * @param   pVCpu       The VMCPU handle.
- * @param   GCVirt      The virtual address of the page to invalidate.
- */
-#ifdef IN_RC
-# define PGM_INVL_PG(pVCpu, GCVirt)             ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
-#elif defined(IN_RING0)
-# define PGM_INVL_PG(pVCpu, GCVirt)             HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
-#else
-# define PGM_INVL_PG(pVCpu, GCVirt)             HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
-#endif
-
-/** @def PGM_INVL_PG_ALL_VCPU
- * Invalidates a page on all VCPUs
- *
- * @param   pVM         The VM handle.
- * @param   GCVirt      The virtual address of the page to invalidate.
- */
-#ifdef IN_RC
-# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
-#elif defined(IN_RING0)
-# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
-#else
-# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
-#endif
-
-/** @def PGM_INVL_BIG_PG
- * Invalidates a 4MB page directory entry.
- *
- * @param   pVCpu       The VMCPU handle.
- * @param   GCVirt      The virtual address within the page directory to invalidate.
- */
-#ifdef IN_RC
-# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         ASMReloadCR3()
-#elif defined(IN_RING0)
-# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         HWACCMFlushTLB(pVCpu)
-#else
-# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         HWACCMFlushTLB(pVCpu)
-#endif
-
-/** @def PGM_INVL_VCPU_TLBS()
- * Invalidates the TLBs of the specified VCPU
- *
- * @param   pVCpu       The VMCPU handle.
- */
-#ifdef IN_RC
-# define PGM_INVL_VCPU_TLBS(pVCpu)             ASMReloadCR3()
-#elif defined(IN_RING0)
-# define PGM_INVL_VCPU_TLBS(pVCpu)             HWACCMFlushTLB(pVCpu)
-#else
-# define PGM_INVL_VCPU_TLBS(pVCpu)             HWACCMFlushTLB(pVCpu)
-#endif
-
-/** @def PGM_INVL_ALL_VCPU_TLBS()
- * Invalidates the TLBs of all VCPUs
- *
- * @param   pVM         The VM handle.
- */
-#ifdef IN_RC
-# define PGM_INVL_ALL_VCPU_TLBS(pVM)            ASMReloadCR3()
-#elif defined(IN_RING0)
-# define PGM_INVL_ALL_VCPU_TLBS(pVM)            HWACCMFlushTLBOnAllVCpus(pVM)
-#else
-# define PGM_INVL_ALL_VCPU_TLBS(pVM)            HWACCMFlushTLBOnAllVCpus(pVM)
-#endif
-
-
-/** @name Safer Shadow PAE PT/PTE
- * For helping avoid misinterpreting invalid PAE/AMD64 page table entries as
- * present.
- *
- * @{
- */
-#if 1
-/**
- * For making sure that u1Present and X86_PTE_P checks doesn't mistake
- * invalid entries for present.
- * @sa X86PTEPAE.
- */
-typedef union PGMSHWPTEPAE
-{
-    /** Unsigned integer view */
-    X86PGPAEUINT    uCareful;
-    /* Not other views. */
-} PGMSHWPTEPAE;
-
-# define PGMSHWPTEPAE_IS_P(Pte)                 ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == X86_PTE_P )
-# define PGMSHWPTEPAE_IS_RW(Pte)                ( !!((Pte).uCareful & X86_PTE_RW))
-# define PGMSHWPTEPAE_IS_US(Pte)                ( !!((Pte).uCareful & X86_PTE_US))
-# define PGMSHWPTEPAE_IS_A(Pte)                 ( !!((Pte).uCareful & X86_PTE_A))
-# define PGMSHWPTEPAE_IS_D(Pte)                 ( !!((Pte).uCareful & X86_PTE_D))
-# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)       ( !!((Pte).uCareful & PGM_PTFLAGS_TRACK_DIRTY) )
-# define PGMSHWPTEPAE_IS_P_RW(Pte)              ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_RW | X86_PTE_PAE_MBZ_MASK_NX)) == (X86_PTE_P | X86_PTE_RW) )
-# define PGMSHWPTEPAE_GET_LOG(Pte)              ( (Pte).uCareful )
-# define PGMSHWPTEPAE_GET_HCPHYS(Pte)           ( (Pte).uCareful & X86_PTE_PAE_PG_MASK )
-# define PGMSHWPTEPAE_GET_U(Pte)                ( (Pte).uCareful ) /**< Use with care. */
-# define PGMSHWPTEPAE_SET(Pte, uVal)            do { (Pte).uCareful = (uVal); } while (0)
-# define PGMSHWPTEPAE_SET2(Pte, Pte2)           do { (Pte).uCareful = (Pte2).uCareful; } while (0)
-# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal)     do { ASMAtomicWriteU64(&(Pte).uCareful, (uVal)); } while (0)
-# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)    do { ASMAtomicWriteU64(&(Pte).uCareful, (Pte2).uCareful); } while (0)
-# define PGMSHWPTEPAE_SET_RO(Pte)               do { (Pte).uCareful &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
-# define PGMSHWPTEPAE_SET_RW(Pte)               do { (Pte).uCareful |= X86_PTE_RW; } while (0)
-
-/**
- * For making sure that u1Present and X86_PTE_P checks doesn't mistake
- * invalid entries for present.
- * @sa X86PTPAE.
- */
-typedef struct PGMSHWPTPAE
-{
-    PGMSHWPTEPAE  a[X86_PG_PAE_ENTRIES];
-} PGMSHWPTPAE;
-
-#else
-typedef X86PTEPAE           PGMSHWPTEPAE;
-typedef X86PTPAE            PGMSHWPTPAE;
-# define PGMSHWPTEPAE_IS_P(Pte)                 ( (Pte).n.u1Present )
-# define PGMSHWPTEPAE_IS_RW(Pte)                ( (Pte).n.u1Write )
-# define PGMSHWPTEPAE_IS_US(Pte)                ( (Pte).n.u1User )
-# define PGMSHWPTEPAE_IS_A(Pte)                 ( (Pte).n.u1Accessed )
-# define PGMSHWPTEPAE_IS_D(Pte)                 ( (Pte).n.u1Dirty )
-# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)       ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
-# define PGMSHWPTEPAE_IS_P_RW(Pte)              ( ((Pte).u & (X86_PTE_P | X86_PTE_RW)) == (X86_PTE_P | X86_PTE_RW) )
-# define PGMSHWPTEPAE_GET_LOG(Pte)              ( (Pte).u )
-# define PGMSHWPTEPAE_GET_HCPHYS(Pte)           ( (Pte).u & X86_PTE_PAE_PG_MASK )
-# define PGMSHWPTEPAE_GET_U(Pte)                ( (Pte).u ) /**< Use with care. */
-# define PGMSHWPTEPAE_SET(Pte, uVal)            do { (Pte).u = (uVal); } while (0)
-# define PGMSHWPTEPAE_SET2(Pte, Pte2)           do { (Pte).u = (Pte2).u; } while (0)
-# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal)     do { ASMAtomicWriteU64(&(Pte).u, (uVal)); } while (0)
-# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)    do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
-# define PGMSHWPTEPAE_SET_RO(Pte)               do { (Pte).u &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
-# define PGMSHWPTEPAE_SET_RW(Pte)               do { (Pte).u |= X86_PTE_RW; } while (0)
-
-#endif
-
-/** Pointer to a shadow PAE PTE.  */
-typedef PGMSHWPTEPAE       *PPGMSHWPTEPAE;
-/** Pointer to a const shadow PAE PTE.  */
-typedef PGMSHWPTEPAE const *PCPGMSHWPTEPAE;
-
-/** Pointer to a shadow PAE page table.  */
-typedef PGMSHWPTPAE        *PPGMSHWPTPAE;
-/** Pointer to a const shadow PAE page table.  */
-typedef PGMSHWPTPAE const  *PCPGMSHWPTPAE;
-/** @}  */
-
-
-/** Size of the GCPtrConflict array in PGMMAPPING.
- * @remarks Must be a power of two. */
-#define PGMMAPPING_CONFLICT_MAX         8
-
-/**
- * Structure for tracking GC Mappings.
- *
- * This structure is used by linked list in both GC and HC.
- */
-typedef struct PGMMAPPING
-{
-    /** Pointer to next entry. */
-    R3PTRTYPE(struct PGMMAPPING *)      pNextR3;
-    /** Pointer to next entry. */
-    R0PTRTYPE(struct PGMMAPPING *)      pNextR0;
-    /** Pointer to next entry. */
-    RCPTRTYPE(struct PGMMAPPING *)      pNextRC;
-    /** Indicate whether this entry is finalized. */
-    bool                                fFinalized;
-    /** Start Virtual address. */
-    RTGCPTR                             GCPtr;
-    /** Last Virtual address (inclusive). */
-    RTGCPTR                             GCPtrLast;
-    /** Range size (bytes). */
-    RTGCPTR                             cb;
-    /** Pointer to relocation callback function. */
-    R3PTRTYPE(PFNPGMRELOCATE)           pfnRelocate;
-    /** User argument to the callback. */
-    R3PTRTYPE(void *)                   pvUser;
-    /** Mapping description / name. For easing debugging. */
-    R3PTRTYPE(const char *)             pszDesc;
-    /** Last 8 addresses that caused conflicts. */
-    RTGCPTR                             aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
-    /** Number of conflicts for this hypervisor mapping. */
-    uint32_t                            cConflicts;
-    /** Number of page tables. */
-    uint32_t                            cPTs;
-
-    /** Array of page table mapping data. Each entry
-     * describes one page table. The array can be longer
-     * than the declared length.
-     */
-    struct
-    {
-        /** The HC physical address of the page table. */
-        RTHCPHYS                        HCPhysPT;
-        /** The HC physical address of the first PAE page table. */
-        RTHCPHYS                        HCPhysPaePT0;
-        /** The HC physical address of the second PAE page table. */
-        RTHCPHYS                        HCPhysPaePT1;
-        /** The HC virtual address of the 32-bit page table. */
-        R3PTRTYPE(PX86PT)               pPTR3;
-        /** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
-        R3PTRTYPE(PPGMSHWPTPAE)         paPaePTsR3;
-        /** The RC virtual address of the 32-bit page table. */
-        RCPTRTYPE(PX86PT)               pPTRC;
-        /** The RC virtual address of the two PAE page table. */
-        RCPTRTYPE(PPGMSHWPTPAE)         paPaePTsRC;
-        /** The R0 virtual address of the 32-bit page table. */
-        R0PTRTYPE(PX86PT)               pPTR0;
-        /** The R0 virtual address of the two PAE page table. */
-        R0PTRTYPE(PPGMSHWPTPAE)         paPaePTsR0;
-    } aPTs[1];
-} PGMMAPPING;
-/** Pointer to structure for tracking GC Mappings. */
-typedef struct PGMMAPPING *PPGMMAPPING;
-
-
-/**
- * Physical page access handler structure.
- *
- * This is used to keep track of physical address ranges
- * which are being monitored in some kind of way.
- */
-typedef struct PGMPHYSHANDLER
-{
-    AVLROGCPHYSNODECORE                 Core;
-    /** Access type. */
-    PGMPHYSHANDLERTYPE                  enmType;
-    /** Number of pages to update. */
-    uint32_t                            cPages;
-    /** Set if we have pages that have been aliased. */
-    uint32_t                            cAliasedPages;
-    /** Set if we have pages that have temporarily been disabled. */
-    uint32_t                            cTmpOffPages;
-    /** Pointer to R3 callback function. */
-    R3PTRTYPE(PFNPGMR3PHYSHANDLER)      pfnHandlerR3;
-    /** User argument for R3 handlers. */
-    R3PTRTYPE(void *)                   pvUserR3;
-    /** Pointer to R0 callback function. */
-    R0PTRTYPE(PFNPGMR0PHYSHANDLER)      pfnHandlerR0;
-    /** User argument for R0 handlers. */
-    R0PTRTYPE(void *)                   pvUserR0;
-    /** Pointer to RC callback function. */
-    RCPTRTYPE(PFNPGMRCPHYSHANDLER)      pfnHandlerRC;
-    /** User argument for RC handlers. */
-    RCPTRTYPE(void *)                   pvUserRC;
-    /** Description / Name. For easing debugging. */
-    R3PTRTYPE(const char *)             pszDesc;
-#ifdef VBOX_WITH_STATISTICS
-    /** Profiling of this handler. */
-    STAMPROFILE                         Stat;
-#endif
-} PGMPHYSHANDLER;
-/** Pointer to a physical page access handler structure. */
-typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
-
-
-/**
- * Cache node for the physical addresses covered by a virtual handler.
- */
-typedef struct PGMPHYS2VIRTHANDLER
-{
-    /** Core node for the tree based on physical ranges. */
-    AVLROGCPHYSNODECORE                 Core;
-    /** Offset from this struct to the PGMVIRTHANDLER structure. */
-    int32_t                             offVirtHandler;
-    /** Offset of the next alias relative to this one.
-     * Bit 0 is used for indicating whether we're in the tree.
-     * Bit 1 is used for indicating that we're the head node.
-     */
-    int32_t                             offNextAlias;
-} PGMPHYS2VIRTHANDLER;
-/** Pointer to a phys to virtual handler structure. */
-typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
-
-/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
- * node is in the tree. */
-#define PGMPHYS2VIRTHANDLER_IN_TREE     RT_BIT(0)
-/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
- * node is in the head of an alias chain.
- * The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
-#define PGMPHYS2VIRTHANDLER_IS_HEAD     RT_BIT(1)
-/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
-#define PGMPHYS2VIRTHANDLER_OFF_MASK    (~(int32_t)3)
-
-
-/**
- * Virtual page access handler structure.
- *
- * This is used to keep track of virtual address ranges
- * which are being monitored in some kind of way.
- */
-typedef struct PGMVIRTHANDLER
-{
-    /** Core node for the tree based on virtual ranges. */
-    AVLROGCPTRNODECORE                  Core;
-    /** Size of the range (in bytes). */
-    RTGCPTR                             cb;
-    /** Number of cache pages. */
-    uint32_t                            cPages;
-    /** Access type. */
-    PGMVIRTHANDLERTYPE                  enmType;
-    /** Pointer to the RC callback function. */
-    RCPTRTYPE(PFNPGMRCVIRTHANDLER)      pfnHandlerRC;
-#if HC_ARCH_BITS == 64
-    RTRCPTR                             padding;
-#endif
-    /** Pointer to the R3 callback function for invalidation. */
-    R3PTRTYPE(PFNPGMR3VIRTINVALIDATE)   pfnInvalidateR3;
-    /** Pointer to the R3 callback function. */
-    R3PTRTYPE(PFNPGMR3VIRTHANDLER)      pfnHandlerR3;
-    /** Description / Name. For easing debugging. */
-    R3PTRTYPE(const char *)             pszDesc;
-#ifdef VBOX_WITH_STATISTICS
-    /** Profiling of this handler. */
-    STAMPROFILE                         Stat;
-#endif
-    /** Array of cached physical addresses for the monitored ranged.  */
-    PGMPHYS2VIRTHANDLER                 aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
-} PGMVIRTHANDLER;
-/** Pointer to a virtual page access handler structure. */
-typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
-
-
-/** @name Page type predicates.
- * @{ */
-#define PGMPAGETYPE_IS_READABLE(type)   ( (type) <= PGMPAGETYPE_ROM )
-#define PGMPAGETYPE_IS_WRITEABLE(type)  ( (type) <= PGMPAGETYPE_ROM_SHADOW )
-#define PGMPAGETYPE_IS_RWX(type)        ( (type) <= PGMPAGETYPE_ROM_SHADOW )
-#define PGMPAGETYPE_IS_ROX(type)        ( (type) == PGMPAGETYPE_ROM )
-#define PGMPAGETYPE_IS_NP(type)         ( (type) == PGMPAGETYPE_MMIO )
-/** @} */
-
-
-/**
- * A Physical Guest Page tracking structure.
- *
- * The format of this structure is complicated because we have to fit a lot
- * of information into as few bits as possible. The format is also subject
- * to change (there is one coming up soon). Which means that for we'll be
- * using PGM_PAGE_GET_*, PGM_PAGE_IS_ and PGM_PAGE_SET_* macros for *all*
- * accesses to the structure.
- */
-typedef struct PGMPAGE
-{
-    /** The physical address and the Page ID. */
-    RTHCPHYS    HCPhysAndPageID;
-    /** Combination of:
-     *  - [0-7]: u2HandlerPhysStateY - the physical handler state
-     *    (PGM_PAGE_HNDL_PHYS_STATE_*).
-     *  - [8-9]: u2HandlerVirtStateY - the virtual handler state
-     *    (PGM_PAGE_HNDL_VIRT_STATE_*).
-     *  - [10]:  u1FTDirty - indicator of dirty page for fault tolerance tracking
-     *  - [13-14]: u2PDEType  - paging structure needed to map the page (PGM_PAGE_PDE_TYPE_*)
-     *  - [15]:  fWrittenToY - flag indicating that a write monitored page was
-     *    written to when set.
-     *  - [11-13]: 3 unused bits.
-     * @remarks Warning! All accesses to the bits are hardcoded.
-     *
-     * @todo    Change this to a union with both bitfields, u8 and u accessors.
-     *          That'll help deal with some of the hardcoded accesses.
-     *
-     * @todo    Include uStateY and uTypeY as well so it becomes 32-bit.  This
-     *          will make it possible to turn some of the 16-bit accesses into
-     *          32-bit ones, which may be efficient (stalls).
-     */
-    RTUINT16U   u16MiscY;
-    /** The page state.
-     * Only 3 bits are really needed for this. */
-    uint16_t    uStateY   : 3;
-    /** The page type (PGMPAGETYPE).
-     * Only 3 bits are really needed for this. */
-    uint16_t    uTypeY    : 3;
-    /** PTE index for usage tracking (page pool). */
-    uint16_t    uPteIdx   : 10;
-    /** Usage tracking (page pool). */
-    uint16_t    u16TrackingY;
-    /** The number of read locks on this page. */
-    uint8_t     cReadLocksY;
-    /** The number of write locks on this page. */
-    uint8_t     cWriteLocksY;
-} PGMPAGE;
-AssertCompileSize(PGMPAGE, 16);
-/** Pointer to a physical guest page. */
-typedef PGMPAGE *PPGMPAGE;
-/** Pointer to a const physical guest page. */
-typedef const PGMPAGE *PCPGMPAGE;
-/** Pointer to a physical guest page pointer. */
-typedef PPGMPAGE *PPPGMPAGE;
-
-
-/**
- * Clears the page structure.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_CLEAR(pPage) \
-    do { \
-        (pPage)->HCPhysAndPageID     = 0; \
-        (pPage)->uStateY             = 0; \
-        (pPage)->uTypeY              = 0; \
-        (pPage)->uPteIdx             = 0; \
-        (pPage)->u16MiscY.u          = 0; \
-        (pPage)->u16TrackingY        = 0; \
-        (pPage)->cReadLocksY         = 0; \
-        (pPage)->cWriteLocksY        = 0; \
-    } while (0)
-
-/**
- * Initializes the page structure.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
-    do { \
-        RTHCPHYS SetHCPhysTmp = (_HCPhys); \
-        AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
-        (pPage)->HCPhysAndPageID     = (SetHCPhysTmp << (28-12)) | ((_idPage) & UINT32_C(0x0fffffff)); \
-        (pPage)->uStateY             = (_uState); \
-        (pPage)->uTypeY              = (_uType); \
-        (pPage)->uPteIdx             = 0; \
-        (pPage)->u16MiscY.u          = 0; \
-        (pPage)->u16TrackingY        = 0; \
-        (pPage)->cReadLocksY         = 0; \
-        (pPage)->cWriteLocksY        = 0; \
-    } while (0)
-
-/**
- * Initializes the page structure of a ZERO page.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   pVM         The VM handle (for getting the zero page address).
- * @param   uType       The page type (PGMPAGETYPE).
- */
-#define PGM_PAGE_INIT_ZERO(pPage, pVM, uType)  \
-    PGM_PAGE_INIT((pPage), (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (uType), PGM_PAGE_STATE_ZERO)
-
-
-/** @name The Page state, PGMPAGE::uStateY.
- * @{ */
-/** The zero page.
- * This is a per-VM page that's never ever mapped writable. */
-#define PGM_PAGE_STATE_ZERO             0
-/** A allocated page.
- * This is a per-VM page allocated from the page pool (or wherever
- * we get MMIO2 pages from if the type is MMIO2).
- */
-#define PGM_PAGE_STATE_ALLOCATED        1
-/** A allocated page that's being monitored for writes.
- * The shadow page table mappings are read-only. When a write occurs, the
- * fWrittenTo member is set, the page remapped as read-write and the state
- * moved back to allocated. */
-#define PGM_PAGE_STATE_WRITE_MONITORED  2
-/** The page is shared, aka. copy-on-write.
- * This is a page that's shared with other VMs. */
-#define PGM_PAGE_STATE_SHARED           3
-/** The page is ballooned, so no longer available for this VM. */
-#define PGM_PAGE_STATE_BALLOONED        4
-/** @} */
-
-
-/**
- * Gets the page state.
- * @returns page state (PGM_PAGE_STATE_*).
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_STATE(pPage)           ( (pPage)->uStateY )
-
-/**
- * Sets the page state.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   _uState     The new page state.
- */
-#define PGM_PAGE_SET_STATE(pPage, _uState)  do { (pPage)->uStateY = (_uState); } while (0)
-
-
-/**
- * Gets the host physical address of the guest page.
- * @returns host physical address (RTHCPHYS).
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_HCPHYS(pPage)          ( ((pPage)->HCPhysAndPageID >> 28) << 12 )
-
-/**
- * Sets the host physical address of the guest page.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   _HCPhys     The new host physical address.
- */
-#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
-    do { \
-        RTHCPHYS SetHCPhysTmp = (_HCPhys); \
-        AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
-        (pPage)->HCPhysAndPageID = ((pPage)->HCPhysAndPageID & UINT32_C(0x0fffffff)) \
-                                 | (SetHCPhysTmp << (28-12)); \
-    } while (0)
-
-/**
- * Get the Page ID.
- * @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_PAGEID(pPage)          (  (uint32_t)((pPage)->HCPhysAndPageID & UINT32_C(0x0fffffff)) )
-
-/**
- * Sets the Page ID.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_SET_PAGEID(pPage, _idPage) \
-    do { \
-        (pPage)->HCPhysAndPageID = (((pPage)->HCPhysAndPageID) & UINT64_C(0xfffffffff0000000)) \
-                                 | ((_idPage) & UINT32_C(0x0fffffff)); \
-    } while (0)
-
-/**
- * Get the Chunk ID.
- * @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_CHUNKID(pPage)         ( PGM_PAGE_GET_PAGEID(pPage) >> GMM_CHUNKID_SHIFT )
-
-/**
- * Get the index of the page within the allocation chunk.
- * @returns The page index.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage)   ( (uint32_t)((pPage)->HCPhysAndPageID & GMM_PAGEID_IDX_MASK) )
-
-/**
- * Gets the page type.
- * @returns The page type.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_TYPE(pPage)            (pPage)->uTypeY
-
-/**
- * Sets the page type.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   _enmType    The new page type (PGMPAGETYPE).
- */
-#define PGM_PAGE_SET_TYPE(pPage, _enmType)  do { (pPage)->uTypeY = (_enmType); } while (0)
-
-/**
- * Gets the page table index
- * @returns The page table index.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_PTE_INDEX(pPage)            (pPage)->uPteIdx
-
-/**
- * Sets the page table index
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   iPte        New page table index.
- */
-#define PGM_PAGE_SET_PTE_INDEX(pPage, _iPte)  do { (pPage)->uPteIdx = (_iPte); } while (0)
-
-/**
- * Checks if the page is marked for MMIO.
- * @returns true/false.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_IS_MMIO(pPage)             ( (pPage)->uTypeY == PGMPAGETYPE_MMIO )
-
-/**
- * Checks if the page is backed by the ZERO page.
- * @returns true/false.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_IS_ZERO(pPage)             ( (pPage)->uStateY == PGM_PAGE_STATE_ZERO )
-
-/**
- * Checks if the page is backed by a SHARED page.
- * @returns true/false.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_IS_SHARED(pPage)           ( (pPage)->uStateY == PGM_PAGE_STATE_SHARED )
-
-/**
- * Checks if the page is ballooned.
- * @returns true/false.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_IS_BALLOONED(pPage)        ( (pPage)->uStateY == PGM_PAGE_STATE_BALLOONED )
-
-/**
- * Marks the page as written to (for GMM change monitoring).
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_SET_WRITTEN_TO(pPage)      do { (pPage)->u16MiscY.au8[1] |= UINT8_C(0x80); } while (0)
-
-/**
- * Clears the written-to indicator.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage)    do { (pPage)->u16MiscY.au8[1] &= UINT8_C(0x7f); } while (0)
-
-/**
- * Checks if the page was marked as written-to.
- * @returns true/false.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_IS_WRITTEN_TO(pPage)       ( !!((pPage)->u16MiscY.au8[1] & UINT8_C(0x80)) )
-
-/**
- * Marks the page as dirty for FTM
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_SET_FT_DIRTY(pPage)        do { (pPage)->u16MiscY.au8[1] |= UINT8_C(0x04); } while (0)
-
-/**
- * Clears the FTM dirty indicator
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_CLEAR_FT_DIRTY(pPage)      do { (pPage)->u16MiscY.au8[1] &= UINT8_C(0xfb); } while (0)
-
-/**
- * Checks if the page was marked as dirty for FTM
- * @returns true/false.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_IS_FT_DIRTY(pPage)         ( !!((pPage)->u16MiscY.au8[1] & UINT8_C(0x04)) )
-
-
-/** @name PT usage values (PGMPAGE::u2PDEType).
- *
- * @{ */
-/** Either as a PT or PDE. */
-#define PGM_PAGE_PDE_TYPE_DONTCARE             0
-/** Must use a page table to map the range. */
-#define PGM_PAGE_PDE_TYPE_PT                   1
-/** Can use a page directory entry to map the continuous range. */
-#define PGM_PAGE_PDE_TYPE_PDE                  2
-/** Can use a page directory entry to map the continuous range - temporarily disabled (by page monitoring). */
-#define PGM_PAGE_PDE_TYPE_PDE_DISABLED         3
-/** @} */
-
-/**
- * Set the PDE type of the page
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   uType       PGM_PAGE_PDE_TYPE_*
- */
-#define PGM_PAGE_SET_PDE_TYPE(pPage, uType) \
-    do { \
-        (pPage)->u16MiscY.au8[1] = ((pPage)->u16MiscY.au8[1] & UINT8_C(0x9f)) \
-                                 | (((uType)                 & UINT8_C(0x03)) << 5); \
-    } while (0)
-
-/**
- * Checks if the page was marked being part of a large page
- * @returns true/false.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_PDE_TYPE(pPage)       ( ((pPage)->u16MiscY.au8[1] & UINT8_C(0x60)) >> 5)
-
-/** Enabled optimized access handler tests.
- * These optimizations makes ASSUMPTIONS about the state values and the u16MiscY
- * layout.  When enabled, the compiler should normally generate more compact
- * code.
- */
-#define PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS 1
-
-/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY).
- *
- * @remarks The values are assigned in order of priority, so we can calculate
- *          the correct state for a page with different handlers installed.
- * @{ */
-/** No handler installed. */
-#define PGM_PAGE_HNDL_PHYS_STATE_NONE       0
-/** Monitoring is temporarily disabled. */
-#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED   1
-/** Write access is monitored. */
-#define PGM_PAGE_HNDL_PHYS_STATE_WRITE      2
-/** All access is monitored. */
-#define PGM_PAGE_HNDL_PHYS_STATE_ALL        3
-/** @} */
-
-/**
- * Gets the physical access handler state of a page.
- * @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)  \
-    ( (pPage)->u16MiscY.au8[0] )
-
-/**
- * Sets the physical access handler state of a page.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   _uState     The new state value.
- */
-#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
-    do { (pPage)->u16MiscY.au8[0] = (_uState); } while (0)
-
-/**
- * Checks if the page has any physical access handlers, including temporarily disabled ones.
- * @returns true/false
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) \
-    ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
-
-/**
- * Checks if the page has any active physical access handlers.
- * @returns true/false
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) \
-    ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
-
-
-/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateY).
- *
- * @remarks The values are assigned in order of priority, so we can calculate
- *          the correct state for a page with different handlers installed.
- * @{ */
-/** No handler installed. */
-#define PGM_PAGE_HNDL_VIRT_STATE_NONE       0
-/* 1 is reserved so the lineup is identical with the physical ones. */
-/** Write access is monitored. */
-#define PGM_PAGE_HNDL_VIRT_STATE_WRITE      2
-/** All access is monitored. */
-#define PGM_PAGE_HNDL_VIRT_STATE_ALL        3
-/** @} */
-
-/**
- * Gets the virtual access handler state of a page.
- * @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ((uint8_t)( (pPage)->u16MiscY.au8[1] & UINT8_C(0x03) ))
-
-/**
- * Sets the virtual access handler state of a page.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- * @param   _uState     The new state value.
- */
-#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
-    do { \
-        (pPage)->u16MiscY.au8[1] = ((pPage)->u16MiscY.au8[1] & UINT8_C(0xfc)) \
-                                 | ((_uState)                & UINT8_C(0x03)); \
-    } while (0)
-
-/**
- * Checks if the page has any virtual access handlers.
- * @returns true/false
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) \
-    ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
-
-/**
- * Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
- * virtual handlers.
- * @returns true/false
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) \
-    PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
-
-
-/**
- * Checks if the page has any access handlers, including temporarily disabled ones.
- * @returns true/false
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
-# define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
-    ( ((pPage)->u16MiscY.u & UINT16_C(0x0303)) != 0 )
-#else
-# define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
-    (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE \
-     || PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
-#endif
-
-/**
- * Checks if the page has any active access handlers.
- * @returns true/false
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
-# define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
-    ( ((pPage)->u16MiscY.u & UINT16_C(0x0202)) != 0 )
-#else
-# define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
-    (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
-     || PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
-#endif
-
-/**
- * Checks if the page has any active access handlers catching all accesses.
- * @returns true/false
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
-# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
-    (   ( ((pPage)->u16MiscY.au8[0] | (pPage)->u16MiscY.au8[1]) & UINT8_C(0x3) ) \
-     == PGM_PAGE_HNDL_PHYS_STATE_ALL )
-#else
-# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
-    (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL \
-     || PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL )
-#endif
-
-
-/** @def PGM_PAGE_GET_TRACKING
- * Gets the packed shadow page pool tracking data associated with a guest page.
- * @returns uint16_t containing the data.
- * @param   pPage       Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_TRACKING(pPage)        ( (pPage)->u16TrackingY )
-
-/** @def PGM_PAGE_SET_TRACKING
- * Sets the packed shadow page pool tracking data associated with a guest page.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- * @param   u16TrackingData     The tracking data to store.
- */
-#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
-    do { (pPage)->u16TrackingY = (u16TrackingData); } while (0)
-
-/** @def PGM_PAGE_GET_TD_CREFS
- * Gets the @a cRefs tracking data member.
- * @returns cRefs.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_TD_CREFS(pPage) \
-    ((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
-
-/** @def PGM_PAGE_GET_TD_IDX
- * Gets the @a idx tracking data member.
- * @returns idx.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_TD_IDX(pPage) \
-    ((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT)   & PGMPOOL_TD_IDX_MASK)
-
-
-/** Max number of locks on a page. */
-#define PGM_PAGE_MAX_LOCKS                  UINT8_C(254)
-
-/** Get the read lock count.
- * @returns count.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_READ_LOCKS(pPage)      ( (pPage)->cReadLocksY )
-
-/** Get the write lock count.
- * @returns count.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_GET_WRITE_LOCKS(pPage)     ( (pPage)->cWriteLocksY )
-
-/** Decrement the read lock counter.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_DEC_READ_LOCKS(pPage)      do { --(pPage)->cReadLocksY; } while (0)
-
-/** Decrement the write lock counter.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_DEC_WRITE_LOCKS(pPage)     do { --(pPage)->cWriteLocksY; } while (0)
-
-/** Increment the read lock counter.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_INC_READ_LOCKS(pPage)      do { ++(pPage)->cReadLocksY; } while (0)
-
-/** Increment the write lock counter.
- * @param   pPage               Pointer to the physical guest page tracking structure.
- */
-#define PGM_PAGE_INC_WRITE_LOCKS(pPage)     do { ++(pPage)->cWriteLocksY; } while (0)
-
-
-#if 0
-/** Enables sanity checking of write monitoring using CRC-32.  */
-# define PGMLIVESAVERAMPAGE_WITH_CRC32
-#endif
-
-/**
- * Per page live save tracking data.
- */
-typedef struct PGMLIVESAVERAMPAGE
-{
-    /** Number of times it has been dirtied. */
-    uint32_t    cDirtied : 24;
-    /** Whether it is currently dirty. */
-    uint32_t    fDirty : 1;
-    /** Ignore the page.
-     *  This is used for pages that has been MMIO, MMIO2 or ROM pages once.  We will
-     *  deal with these after pausing the VM and DevPCI have said it bit about
-     *  remappings. */
-    uint32_t    fIgnore : 1;
-    /** Was a ZERO page last time around. */
-    uint32_t    fZero : 1;
-    /** Was a SHARED page last time around. */
-    uint32_t    fShared : 1;
-    /** Whether the page is/was write monitored in a previous pass. */
-    uint32_t    fWriteMonitored : 1;
-    /** Whether the page is/was write monitored earlier in this pass. */
-    uint32_t    fWriteMonitoredJustNow : 1;
-    /** Bits reserved for future use.  */
-    uint32_t    u2Reserved : 2;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-    /** CRC-32 for the page. This is for internal consistency checks.  */
-    uint32_t    u32Crc;
-#endif
-} PGMLIVESAVERAMPAGE;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-AssertCompileSize(PGMLIVESAVERAMPAGE, 8);
-#else
-AssertCompileSize(PGMLIVESAVERAMPAGE, 4);
-#endif
-/** Pointer to the per page live save tracking data. */
-typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE;
-
-/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */
-#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0
-
-
-/**
- * Ram range for GC Phys to HC Phys conversion.
- *
- * Can be used for HC Virt to GC Phys and HC Virt to HC Phys
- * conversions too, but we'll let MM handle that for now.
- *
- * This structure is used by linked lists in both GC and HC.
- */
-typedef struct PGMRAMRANGE
-{
-    /** Start of the range. Page aligned. */
-    RTGCPHYS                            GCPhys;
-    /** Size of the range. (Page aligned of course). */
-    RTGCPHYS                            cb;
-    /** Pointer to the next RAM range - for R3. */
-    R3PTRTYPE(struct PGMRAMRANGE *)     pNextR3;
-    /** Pointer to the next RAM range - for R0. */
-    R0PTRTYPE(struct PGMRAMRANGE *)     pNextR0;
-    /** Pointer to the next RAM range - for RC. */
-    RCPTRTYPE(struct PGMRAMRANGE *)     pNextRC;
-    /** PGM_RAM_RANGE_FLAGS_* flags. */
-    uint32_t                            fFlags;
-    /** Last address in the range (inclusive). Page aligned (-1). */
-    RTGCPHYS                            GCPhysLast;
-    /** Start of the HC mapping of the range. This is only used for MMIO2. */
-    R3PTRTYPE(void *)                   pvR3;
-    /** Live save per page tracking data. */
-    R3PTRTYPE(PPGMLIVESAVERAMPAGE)      paLSPages;
-    /** The range description. */
-    R3PTRTYPE(const char *)             pszDesc;
-    /** Pointer to self - R0 pointer. */
-    R0PTRTYPE(struct PGMRAMRANGE *)     pSelfR0;
-    /** Pointer to self - RC pointer. */
-    RCPTRTYPE(struct PGMRAMRANGE *)     pSelfRC;
-    /** Padding to make aPage aligned on sizeof(PGMPAGE). */
-    uint32_t                            au32Alignment2[HC_ARCH_BITS == 32 ? 1 : 3];
-    /** Array of physical guest page tracking structures. */
-    PGMPAGE                             aPages[1];
-} PGMRAMRANGE;
-/** Pointer to Ram range for GC Phys to HC Phys conversion. */
-typedef PGMRAMRANGE *PPGMRAMRANGE;
-
-/** @name PGMRAMRANGE::fFlags
- * @{ */
-/** The RAM range is floating around as an independent guest mapping. */
-#define PGM_RAM_RANGE_FLAGS_FLOATING        RT_BIT(20)
-/** Ad hoc RAM range for an ROM mapping. */
-#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM      RT_BIT(21)
-/** Ad hoc RAM range for an MMIO mapping. */
-#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO     RT_BIT(22)
-/** Ad hoc RAM range for an MMIO2 mapping. */
-#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2    RT_BIT(23)
-/** @} */
-
-/** Tests if a RAM range is an ad hoc one or not.
- * @returns true/false.
- * @param   pRam    The RAM range.
- */
-#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
-    (!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2) ) )
-
-
-/**
- * Per page tracking structure for ROM image.
- *
- * A ROM image may have a shadow page, in which case we may have two pages
- * backing it.  This structure contains the PGMPAGE for both while
- * PGMRAMRANGE have a copy of the active one.  It is important that these
- * aren't out of sync in any regard other than page pool tracking data.
- */
-typedef struct PGMROMPAGE
-{
-    /** The page structure for the virgin ROM page. */
-    PGMPAGE     Virgin;
-    /** The page structure for the shadow RAM page. */
-    PGMPAGE     Shadow;
-    /** The current protection setting. */
-    PGMROMPROT  enmProt;
-    /** Live save status information. Makes use of unused alignment space. */
-    struct
-    {
-        /** The previous protection value. */
-        uint8_t u8Prot;
-        /** Written to flag set by the handler. */
-        bool    fWrittenTo;
-        /** Whether the shadow page is dirty or not. */
-        bool    fDirty;
-        /** Whether it was dirtied in the recently. */
-        bool    fDirtiedRecently;
-    } LiveSave;
-} PGMROMPAGE;
-AssertCompileSizeAlignment(PGMROMPAGE, 8);
-/** Pointer to a ROM page tracking structure. */
-typedef PGMROMPAGE *PPGMROMPAGE;
-
-
-/**
- * A registered ROM image.
- *
- * This is needed to keep track of ROM image since they generally intrude
- * into a PGMRAMRANGE.  It also keeps track of additional info like the
- * two page sets (read-only virgin and read-write shadow), the current
- * state of each page.
- *
- * Because access handlers cannot easily be executed in a different
- * context, the ROM ranges needs to be accessible and in all contexts.
- */
-typedef struct PGMROMRANGE
-{
-    /** Pointer to the next range - R3. */
-    R3PTRTYPE(struct PGMROMRANGE *)     pNextR3;
-    /** Pointer to the next range - R0. */
-    R0PTRTYPE(struct PGMROMRANGE *)     pNextR0;
-    /** Pointer to the next range - RC. */
-    RCPTRTYPE(struct PGMROMRANGE *)     pNextRC;
-    /** Pointer alignment */
-    RTRCPTR                             RCPtrAlignment;
-    /** Address of the range. */
-    RTGCPHYS                            GCPhys;
-    /** Address of the last byte in the range. */
-    RTGCPHYS                            GCPhysLast;
-    /** Size of the range. */
-    RTGCPHYS                            cb;
-    /** The flags (PGMPHYS_ROM_FLAGS_*). */
-    uint32_t                            fFlags;
-    /** The saved state range ID. */
-    uint8_t                             idSavedState;
-    /** Alignment padding. */
-    uint8_t                             au8Alignment[3];
-    /** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
-    uint32_t                            au32Alignemnt[HC_ARCH_BITS == 32 ? 5 : 1];
-    /** The size bits pvOriginal points to. */
-    uint32_t                            cbOriginal;
-    /** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
-     * This is used for strictness checks. */
-    R3PTRTYPE(const void *)             pvOriginal;
-    /** The ROM description. */
-    R3PTRTYPE(const char *)             pszDesc;
-    /** The per page tracking structures. */
-    PGMROMPAGE                          aPages[1];
-} PGMROMRANGE;
-/** Pointer to a ROM range. */
-typedef PGMROMRANGE *PPGMROMRANGE;
-
-
-/**
- * Live save per page data for an MMIO2 page.
- *
- * Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring
- * of MMIO2 pages.  The current approach is using some optimistic SHA-1 +
- * CRC-32 for detecting changes as well as special handling of zero pages.  This
- * is a TEMPORARY measure which isn't perfect, but hopefully it is good enough
- * for speeding things up.  (We're using SHA-1 and not SHA-256 or SHA-512
- * because of speed (2.5x and 6x slower).)
- *
- * @todo Implement dirty MMIO2 page reporting that can be enabled during live
- *       save but normally is disabled.  Since we can write monitor guest
- *       accesses on our own, we only need this for host accesses.  Shouldn't be
- *       too difficult for DevVGA, VMMDev might be doable, the planned
- *       networking fun will be fun since it involves ring-0.
- */
-typedef struct PGMLIVESAVEMMIO2PAGE
-{
-    /** Set if the page is considered dirty. */
-    bool        fDirty;
-    /** The number of scans this page has remained unchanged for.
-     * Only updated for dirty pages. */
-    uint8_t     cUnchangedScans;
-    /** Whether this page was zero at the last scan. */
-    bool        fZero;
-    /** Alignment padding. */
-    bool        fReserved;
-    /** CRC-32 for the first half of the page.
-     * This is used together with u32CrcH2 to quickly detect changes in the page
-     * during the non-final passes.  */
-    uint32_t    u32CrcH1;
-    /** CRC-32 for the second half of the page. */
-    uint32_t    u32CrcH2;
-    /** SHA-1 for the saved page.
-     * This is used in the final pass to skip pages without changes. */
-    uint8_t     abSha1Saved[RTSHA1_HASH_SIZE];
-} PGMLIVESAVEMMIO2PAGE;
-/** Pointer to a live save status data for an MMIO2 page. */
-typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE;
-
-/**
- * A registered MMIO2 (= Device RAM) range.
- *
- * There are a few reason why we need to keep track of these
- * registrations.  One of them is the deregistration & cleanup stuff,
- * while another is that the PGMRAMRANGE associated with such a region may
- * have to be removed from the ram range list.
- *
- * Overlapping with a RAM range has to be 100% or none at all.  The pages
- * in the existing RAM range must not be ROM nor MMIO.  A guru meditation
- * will be raised if a partial overlap or an overlap of ROM pages is
- * encountered.  On an overlap we will free all the existing RAM pages and
- * put in the ram range pages instead.
- */
-typedef struct PGMMMIO2RANGE
-{
-    /** The owner of the range. (a device) */
-    PPDMDEVINSR3                        pDevInsR3;
-    /** Pointer to the ring-3 mapping of the allocation. */
-    RTR3PTR                             pvR3;
-    /** Pointer to the next range - R3. */
-    R3PTRTYPE(struct PGMMMIO2RANGE *)   pNextR3;
-    /** Whether it's mapped or not. */
-    bool                                fMapped;
-    /** Whether it's overlapping or not. */
-    bool                                fOverlapping;
-    /** The PCI region number.
-     * @remarks This ASSUMES that nobody will ever really need to have multiple
-     *          PCI devices with matching MMIO region numbers on a single device. */
-    uint8_t                             iRegion;
-    /** The saved state range ID. */
-    uint8_t                             idSavedState;
-    /** Alignment padding for putting the ram range on a PGMPAGE alignment boundary. */
-    uint8_t                             abAlignemnt[HC_ARCH_BITS == 32 ? 12 : 12];
-    /** Live save per page tracking data. */
-    R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE)    paLSPages;
-    /** The associated RAM range. */
-    PGMRAMRANGE                         RamRange;
-} PGMMMIO2RANGE;
-/** Pointer to a MMIO2 range. */
-typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
-
-
-
-
-/**
- * PGMPhysRead/Write cache entry
- */
-typedef struct PGMPHYSCACHEENTRY
-{
-    /** R3 pointer to physical page. */
-    R3PTRTYPE(uint8_t *)                pbR3;
-    /** GC Physical address for cache entry */
-    RTGCPHYS                            GCPhys;
-#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
-    RTGCPHYS                            u32Padding0; /**< alignment padding. */
-#endif
-} PGMPHYSCACHEENTRY;
-
-/**
- * PGMPhysRead/Write cache to reduce REM memory access overhead
- */
-typedef struct PGMPHYSCACHE
-{
-    /** Bitmap of valid cache entries */
-    uint64_t                            aEntries;
-    /** Cache entries */
-    PGMPHYSCACHEENTRY                   Entry[PGM_MAX_PHYSCACHE_ENTRIES];
-} PGMPHYSCACHE;
-
-
-/** Pointer to an allocation chunk ring-3 mapping. */
-typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
-/** Pointer to an allocation chunk ring-3 mapping pointer. */
-typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
-
-/**
- * Ring-3 tracking structore for an allocation chunk ring-3 mapping.
- *
- * The primary tree (Core) uses the chunk id as key.
- */
-typedef struct PGMCHUNKR3MAP
-{
-    /** The key is the chunk id. */
-    AVLU32NODECORE                      Core;
-    /** The current age thingy. */
-    uint32_t                            iAge;
-    /** The current reference count. */
-    uint32_t volatile                   cRefs;
-    /** The current permanent reference count. */
-    uint32_t volatile                   cPermRefs;
-    /** The mapping address. */
-    void                               *pv;
-} PGMCHUNKR3MAP;
-
-/**
- * Allocation chunk ring-3 mapping TLB entry.
- */
-typedef struct PGMCHUNKR3MAPTLBE
-{
-    /** The chunk id. */
-    uint32_t volatile                   idChunk;
-#if HC_ARCH_BITS == 64
-    uint32_t                            u32Padding; /**< alignment padding. */
-#endif
-    /** The chunk map. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(PPGMCHUNKR3MAP) volatile  pChunk;
-#else
-    R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile  pChunk;
-#endif
-} PGMCHUNKR3MAPTLBE;
-/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
-typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
-
-/** The number of TLB entries in PGMCHUNKR3MAPTLB.
- * @remark Must be a power of two value. */
-#define PGM_CHUNKR3MAPTLB_ENTRIES   64
-
-/**
- * Allocation chunk ring-3 mapping TLB.
- *
- * @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
- *          At first glance this might look kinda odd since AVL trees are
- *          supposed to give the most optimal lookup times of all trees
- *          due to their balancing. However, take a tree with 1023 nodes
- *          in it, that's 10 levels, meaning that most searches has to go
- *          down 9 levels before they find what they want. This isn't fast
- *          compared to a TLB hit. There is the factor of cache misses,
- *          and of course the problem with trees and branch prediction.
- *          This is why we use TLBs in front of most of the trees.
- *
- * @todo    Generalize this TLB + AVL stuff, shouldn't be all that
- *          difficult when we switch to the new inlined AVL trees (from kStuff).
- */
-typedef struct PGMCHUNKR3MAPTLB
-{
-    /** The TLB entries. */
-    PGMCHUNKR3MAPTLBE   aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
-} PGMCHUNKR3MAPTLB;
-
-/**
- * Calculates the index of a guest page in the Ring-3 Chunk TLB.
- * @returns Chunk TLB index.
- * @param   idChunk         The Chunk ID.
- */
-#define PGM_CHUNKR3MAPTLB_IDX(idChunk)     ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
-
-
-/**
- * Ring-3 guest page mapping TLB entry.
- * @remarks used in ring-0 as well at the moment.
- */
-typedef struct PGMPAGER3MAPTLBE
-{
-    /** Address of the page. */
-    RTGCPHYS volatile                   GCPhys;
-    /** The guest page. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(PPGMPAGE) volatile        pPage;
-#else
-    R3R0PTRTYPE(PPGMPAGE) volatile      pPage;
-#endif
-    /** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(PPGMCHUNKR3MAP) volatile  pMap;
-#else
-    R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
-#endif
-    /** The address */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(void *) volatile          pv;
-#else
-    R3R0PTRTYPE(void *) volatile        pv;
-#endif
-#if HC_ARCH_BITS == 32
-    uint32_t                            u32Padding; /**< alignment padding. */
-#endif
-} PGMPAGER3MAPTLBE;
-/** Pointer to an entry in the HC physical TLB. */
-typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
-
-
-/** The number of entries in the ring-3 guest page mapping TLB.
- * @remarks The value must be a power of two. */
-#define PGM_PAGER3MAPTLB_ENTRIES 256
-
-/**
- * Ring-3 guest page mapping TLB.
- * @remarks used in ring-0 as well at the moment.
- */
-typedef struct PGMPAGER3MAPTLB
-{
-    /** The TLB entries. */
-    PGMPAGER3MAPTLBE            aEntries[PGM_PAGER3MAPTLB_ENTRIES];
-} PGMPAGER3MAPTLB;
-/** Pointer to the ring-3 guest page mapping TLB. */
-typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
-
-/**
- * Calculates the index of the TLB entry for the specified guest page.
- * @returns Physical TLB index.
- * @param   GCPhys      The guest physical address.
- */
-#define PGM_PAGER3MAPTLB_IDX(GCPhys)    ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
-
-
-/**
- * Raw-mode context dynamic mapping cache entry.
- *
- * Because of raw-mode context being reloctable and all relocations are applied
- * in ring-3, this has to be defined here and be RC specific.
- *
- * @sa PGMRZDYNMAPENTRY, PGMR0DYNMAPENTRY.
- */
-typedef struct PGMRCDYNMAPENTRY
-{
-    /** The physical address of the currently mapped page.
-     * This is duplicate for three reasons: cache locality, cache policy of the PT
-     * mappings and sanity checks.   */
-    RTHCPHYS                    HCPhys;
-    /** Pointer to the page. */
-    RTRCPTR                     pvPage;
-    /** The number of references. */
-    int32_t volatile            cRefs;
-    /** PTE pointer union. */
-    union PGMRCDYNMAPENTRY_PPTE
-    {
-        /** PTE pointer, 32-bit legacy version. */
-        RCPTRTYPE(PX86PTE)      pLegacy;
-        /** PTE pointer, PAE version. */
-        RCPTRTYPE(PX86PTEPAE)   pPae;
-        /** PTE pointer, the void version. */
-        RTRCPTR                 pv;
-    } uPte;
-    /** Alignment padding. */
-    RTRCPTR                     RCPtrAlignment;
-} PGMRCDYNMAPENTRY;
-/** Pointer to a dynamic mapping cache entry for the raw-mode context. */
-typedef PGMRCDYNMAPENTRY *PPGMRCDYNMAPENTRY;
-
-
-/**
- * Dynamic mapping cache for the raw-mode context.
- *
- * This is initialized during VMMRC init based upon the pbDynPageMapBaseGC and
- * paDynPageMap* PGM members.  However, it has to be defined in PGMInternal.h
- * so that we can perform relocations from PGMR3Relocate.  This has the
- * consequence that we must have separate ring-0 and raw-mode context versions
- * of this struct even if they share the basic elements.
- *
- * @sa PPGMRZDYNMAP, PGMR0DYNMAP.
- */
-typedef struct PGMRCDYNMAP
-{
-    /** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
-    uint32_t                        u32Magic;
-    /** Array for tracking and managing the pages.  */
-    RCPTRTYPE(PPGMRCDYNMAPENTRY)    paPages;
-    /** The cache size given as a number of pages. */
-    uint32_t                        cPages;
-    /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
-    bool                            fLegacyMode;
-    /** The current load.
-     * This does not include guard pages. */
-    uint32_t                        cLoad;
-    /** The max load ever.
-     * This is maintained to get trigger adding of more mapping space. */
-    uint32_t                        cMaxLoad;
-    /** The number of guard pages. */
-    uint32_t                        cGuardPages;
-    /** The number of users (protected by hInitLock). */
-    uint32_t                        cUsers;
-} PGMRCDYNMAP;
-/** Pointer to the dynamic cache for the raw-mode context. */
-typedef PGMRCDYNMAP *PPGMRCDYNMAP;
-
-
-/**
- * Mapping cache usage set entry.
- *
- * @remarks 16-bit ints was chosen as the set is not expected to be used beyond
- *          the dynamic ring-0 and (to some extent) raw-mode context mapping
- *          cache.  If it's extended to include ring-3, well, then something
- *          will have be changed here...
- */
-typedef struct PGMMAPSETENTRY
-{
-    /** Pointer to the page. */
-#ifndef IN_RC
-    RTR0PTR                     pvPage;
-#else
-    RTRCPTR                     pvPage;
-# if HC_ARCH_BITS == 64
-    uint32_t                    u32Alignment2;
-# endif
-#endif
-    /** The mapping cache index. */
-    uint16_t                    iPage;
-    /** The number of references.
-     * The max is UINT16_MAX - 1. */
-    uint16_t                    cRefs;
-    /** The number inlined references.
-     * The max is UINT16_MAX - 1. */
-    uint16_t                    cInlinedRefs;
-    /** Unreferences.  */
-    uint16_t                    cUnrefs;
-
-#if HC_ARCH_BITS == 32
-    uint32_t                    u32Alignment1;
-#endif
-    /** The physical address for this entry. */
-    RTHCPHYS                    HCPhys;
-} PGMMAPSETENTRY;
-AssertCompileMemberOffset(PGMMAPSETENTRY, iPage, RT_MAX(sizeof(RTR0PTR), sizeof(RTRCPTR)));
-AssertCompileMemberAlignment(PGMMAPSETENTRY, HCPhys, sizeof(RTHCPHYS));
-/** Pointer to a mapping cache usage set entry. */
-typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
-
-/**
- * Mapping cache usage set.
- *
- * This is used in ring-0 and the raw-mode context to track dynamic mappings
- * done during exits / traps.  The set is
- */
-typedef struct PGMMAPSET
-{
-    /** The number of occupied entries.
-     * This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
-     * dynamic mappings. */
-    uint32_t                    cEntries;
-    /** The start of the current subset.
-     * This is UINT32_MAX if no subset is currently open. */
-    uint32_t                    iSubset;
-    /** The index of the current CPU, only valid if the set is open. */
-    int32_t                     iCpu;
-    uint32_t                    alignment;
-    /** The entries. */
-    PGMMAPSETENTRY              aEntries[64];
-    /** HCPhys -> iEntry fast lookup table.
-     * Use PGMMAPSET_HASH for hashing.
-     * The entries may or may not be valid, check against cEntries. */
-    uint8_t                     aiHashTable[128];
-} PGMMAPSET;
-AssertCompileSizeAlignment(PGMMAPSET, 8);
-/** Pointer to the mapping cache set. */
-typedef PGMMAPSET *PPGMMAPSET;
-
-/** PGMMAPSET::cEntries value for a closed set. */
-#define PGMMAPSET_CLOSED            UINT32_C(0xdeadc0fe)
-
-/** Hash function for aiHashTable. */
-#define PGMMAPSET_HASH(HCPhys)      (((HCPhys) >> PAGE_SHIFT) & 127)
-
-
-/** @name Context neutral page mapper TLB.
- *
- * Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
- * code is writting in a kind of context neutral way. Time will show whether
- * this actually makes sense or not...
- *
- * @todo this needs to be reconsidered and dropped/redone since the ring-0
- *       context ends up using a global mapping cache on some platforms
- *       (darwin).
- *
- * @{ */
-/** @typedef PPGMPAGEMAPTLB
- * The page mapper TLB pointer type for the current context. */
-/** @typedef PPGMPAGEMAPTLB
- * The page mapper TLB entry pointer type for the current context. */
-/** @typedef PPGMPAGEMAPTLB
- * The page mapper TLB entry pointer pointer type for the current context. */
-/** @def PGM_PAGEMAPTLB_ENTRIES
- * The number of TLB entries in the page mapper TLB for the current context. */
-/** @def PGM_PAGEMAPTLB_IDX
- * Calculate the TLB index for a guest physical address.
- * @returns The TLB index.
- * @param   GCPhys      The guest physical address. */
-/** @typedef PPGMPAGEMAP
- * Pointer to a page mapper unit for current context. */
-/** @typedef PPPGMPAGEMAP
- * Pointer to a page mapper unit pointer for current context. */
-#ifdef IN_RC
-// typedef PPGMPAGEGCMAPTLB               PPGMPAGEMAPTLB;
-// typedef PPGMPAGEGCMAPTLBE              PPGMPAGEMAPTLBE;
-// typedef PPGMPAGEGCMAPTLBE             *PPPGMPAGEMAPTLBE;
-# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGEGCMAPTLB_ENTRIES
-# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGEGCMAPTLB_IDX(GCPhys)
- typedef void *                         PPGMPAGEMAP;
- typedef void **                        PPPGMPAGEMAP;
-//#elif IN_RING0
-// typedef PPGMPAGER0MAPTLB               PPGMPAGEMAPTLB;
-// typedef PPGMPAGER0MAPTLBE              PPGMPAGEMAPTLBE;
-// typedef PPGMPAGER0MAPTLBE             *PPPGMPAGEMAPTLBE;
-//# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGER0MAPTLB_ENTRIES
-//# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGER0MAPTLB_IDX(GCPhys)
-// typedef PPGMCHUNKR0MAP                 PPGMPAGEMAP;
-// typedef PPPGMCHUNKR0MAP                PPPGMPAGEMAP;
-#else
- typedef PPGMPAGER3MAPTLB               PPGMPAGEMAPTLB;
- typedef PPGMPAGER3MAPTLBE              PPGMPAGEMAPTLBE;
- typedef PPGMPAGER3MAPTLBE             *PPPGMPAGEMAPTLBE;
-# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGER3MAPTLB_ENTRIES
-# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGER3MAPTLB_IDX(GCPhys)
- typedef PPGMCHUNKR3MAP                 PPGMPAGEMAP;
- typedef PPPGMCHUNKR3MAP                PPPGMPAGEMAP;
-#endif
-/** @} */
-
-
-/** @name PGM Pool Indexes.
- * Aka. the unique shadow page identifier.
- * @{ */
-/** NIL page pool IDX. */
-#define NIL_PGMPOOL_IDX                 0
-/** The first normal index. */
-#define PGMPOOL_IDX_FIRST_SPECIAL       1
-/** Page directory (32-bit root). */
-#define PGMPOOL_IDX_PD                  1
-/** Page Directory Pointer Table (PAE root). */
-#define PGMPOOL_IDX_PDPT                2
-/** AMD64 CR3 level index.*/
-#define PGMPOOL_IDX_AMD64_CR3           3
-/** Nested paging root.*/
-#define PGMPOOL_IDX_NESTED_ROOT         4
-/** The first normal index. */
-#define PGMPOOL_IDX_FIRST               5
-/** The last valid index. (inclusive, 14 bits) */
-#define PGMPOOL_IDX_LAST                0x3fff
-/** @} */
-
-/** The NIL index for the parent chain. */
-#define NIL_PGMPOOL_USER_INDEX          ((uint16_t)0xffff)
-#define NIL_PGMPOOL_PRESENT_INDEX       ((uint16_t)0xffff)
-
-/**
- * Node in the chain linking a shadowed page to it's parent (user).
- */
-#pragma pack(1)
-typedef struct PGMPOOLUSER
-{
-    /** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
-    uint16_t            iNext;
-    /** The user page index. */
-    uint16_t            iUser;
-    /** Index into the user table. */
-    uint32_t            iUserTable;
-} PGMPOOLUSER, *PPGMPOOLUSER;
-typedef const PGMPOOLUSER *PCPGMPOOLUSER;
-#pragma pack()
-
-
-/** The NIL index for the phys ext chain. */
-#define NIL_PGMPOOL_PHYSEXT_INDEX       ((uint16_t)0xffff)
-/** The NIL pte index for a phys ext chain slot. */
-#define NIL_PGMPOOL_PHYSEXT_IDX_PTE     ((uint16_t)0xffff)
-
-/**
- * Node in the chain of physical cross reference extents.
- * @todo Calling this an 'extent' is not quite right, find a better name.
- * @todo find out the optimal size of the aidx array
- */
-#pragma pack(1)
-typedef struct PGMPOOLPHYSEXT
-{
-    /** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
-    uint16_t            iNext;
-    /** Alignment. */
-    uint16_t            u16Align;
-    /** The user page index. */
-    uint16_t            aidx[3];
-    /** The page table index or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown. */
-    uint16_t            apte[3];
-} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
-typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
-#pragma pack()
-
-
-/**
- * The kind of page that's being shadowed.
- */
-typedef enum PGMPOOLKIND
-{
-    /** The virtual invalid 0 entry. */
-    PGMPOOLKIND_INVALID = 0,
-    /** The entry is free (=unused). */
-    PGMPOOLKIND_FREE,
-
-    /** Shw: 32-bit page table;     Gst: no paging  */
-    PGMPOOLKIND_32BIT_PT_FOR_PHYS,
-    /** Shw: 32-bit page table;     Gst: 32-bit page table.  */
-    PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
-    /** Shw: 32-bit page table;     Gst: 4MB page.  */
-    PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
-    /** Shw: PAE page table;        Gst: no paging  */
-    PGMPOOLKIND_PAE_PT_FOR_PHYS,
-    /** Shw: PAE page table;        Gst: 32-bit page table. */
-    PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
-    /** Shw: PAE page table;        Gst: Half of a 4MB page.  */
-    PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
-    /** Shw: PAE page table;        Gst: PAE page table. */
-    PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
-    /** Shw: PAE page table;        Gst: 2MB page.  */
-    PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
-
-    /** Shw: 32-bit page directory. Gst: 32-bit page directory. */
-    PGMPOOLKIND_32BIT_PD,
-    /** Shw: 32-bit page directory. Gst: no paging. */
-    PGMPOOLKIND_32BIT_PD_PHYS,
-    /** Shw: PAE page directory 0;  Gst: 32-bit page directory. */
-    PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
-    /** Shw: PAE page directory 1;  Gst: 32-bit page directory. */
-    PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
-    /** Shw: PAE page directory 2;  Gst: 32-bit page directory. */
-    PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
-    /** Shw: PAE page directory 3;  Gst: 32-bit page directory. */
-    PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
-    /** Shw: PAE page directory;    Gst: PAE page directory. */
-    PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
-    /** Shw: PAE page directory;    Gst: no paging.             Note: +NP. */
-    PGMPOOLKIND_PAE_PD_PHYS,
-
-    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst 32 bits paging. */
-    PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
-    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst PAE PDPT. */
-    PGMPOOLKIND_PAE_PDPT,
-    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst: no paging. */
-    PGMPOOLKIND_PAE_PDPT_PHYS,
-
-    /** Shw: 64-bit page directory pointer table;   Gst: 64-bit page directory pointer table. */
-    PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
-    /** Shw: 64-bit page directory pointer table;   Gst: no paging  */
-    PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
-    /** Shw: 64-bit page directory table;           Gst: 64-bit page directory table. */
-    PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
-    /** Shw: 64-bit page directory table;           Gst: no paging  */
-    PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
-
-    /** Shw: 64-bit PML4;                           Gst: 64-bit PML4. */
-    PGMPOOLKIND_64BIT_PML4,
-
-    /** Shw: EPT page directory pointer table;      Gst: no paging  */
-    PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
-    /** Shw: EPT page directory table;              Gst: no paging  */
-    PGMPOOLKIND_EPT_PD_FOR_PHYS,
-    /** Shw: EPT page table;                        Gst: no paging  */
-    PGMPOOLKIND_EPT_PT_FOR_PHYS,
-
-    /** Shw: Root Nested paging table. */
-    PGMPOOLKIND_ROOT_NESTED,
-
-    /** The last valid entry. */
-    PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
-} PGMPOOLKIND;
-
-/**
- * The access attributes of the page; only applies to big pages.
- */
-typedef enum
-{
-    PGMPOOLACCESS_DONTCARE = 0,
-    PGMPOOLACCESS_USER_RW,
-    PGMPOOLACCESS_USER_R,
-    PGMPOOLACCESS_USER_RW_NX,
-    PGMPOOLACCESS_USER_R_NX,
-    PGMPOOLACCESS_SUPERVISOR_RW,
-    PGMPOOLACCESS_SUPERVISOR_R,
-    PGMPOOLACCESS_SUPERVISOR_RW_NX,
-    PGMPOOLACCESS_SUPERVISOR_R_NX
-} PGMPOOLACCESS;
-
-/**
- * The tracking data for a page in the pool.
- */
-typedef struct PGMPOOLPAGE
-{
-    /** AVL node code with the (R3) physical address of this page. */
-    AVLOHCPHYSNODECORE  Core;
-    /** Pointer to the R3 mapping of the page. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R3PTRTYPE(void *)   pvPageR3;
-#else
-    R3R0PTRTYPE(void *) pvPageR3;
-#endif
-    /** The guest physical address. */
-#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
-    uint32_t            Alignment0;
-#endif
-    RTGCPHYS            GCPhys;
-
-    /** Access handler statistics to determine whether the guest is (re)initializing a page table. */
-    RTGCPTR             pvLastAccessHandlerRip;
-    RTGCPTR             pvLastAccessHandlerFault;
-    uint64_t            cLastAccessHandlerCount;
-
-    /** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
-    uint8_t             enmKind;
-    /** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */
-    uint8_t             enmAccess;
-    /** The index of this page. */
-    uint16_t            idx;
-    /** The next entry in the list this page currently resides in.
-     * It's either in the free list or in the GCPhys hash. */
-    uint16_t            iNext;
-    /** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
-    uint16_t            iUserHead;
-    /** The number of present entries. */
-    uint16_t            cPresent;
-    /** The first entry in the table which is present. */
-    uint16_t            iFirstPresent;
-    /** The number of modifications to the monitored page. */
-    uint16_t            cModifications;
-    /** The next modified page. NIL_PGMPOOL_IDX if tail. */
-    uint16_t            iModifiedNext;
-    /** The previous modified page. NIL_PGMPOOL_IDX if head. */
-    uint16_t            iModifiedPrev;
-    /** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
-    uint16_t            iMonitoredNext;
-    /** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
-    uint16_t            iMonitoredPrev;
-    /** The next page in the age list. */
-    uint16_t            iAgeNext;
-    /** The previous page in the age list. */
-    uint16_t            iAgePrev;
-    /** Used to indicate that the page is zeroed. */
-    bool                fZeroed;
-    /** Used to indicate that a PT has non-global entries. */
-    bool                fSeenNonGlobal;
-    /** Used to indicate that we're monitoring writes to the guest page. */
-    bool                fMonitored;
-    /** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
-     * (All pages are in the age list.) */
-    bool                fCached;
-    /** This is used by the R3 access handlers when invoked by an async thread.
-     * It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
-    bool volatile       fReusedFlushPending;
-    /** Used to mark the page as dirty (write monitoring is temporarily
-     *  off). */
-    bool                fDirty;
-
-    /** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
-    uint32_t            cLocked;
-    uint32_t            idxDirty;
-    RTGCPTR             pvDirtyFault;
-} PGMPOOLPAGE, *PPGMPOOLPAGE, **PPPGMPOOLPAGE;
-/** Pointer to a const pool page. */
-typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
-
-
-/** The hash table size. */
-# define PGMPOOL_HASH_SIZE      0x40
-/** The hash function. */
-# define PGMPOOL_HASH(GCPhys)   ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
-
-
-/**
- * The shadow page pool instance data.
- *
- * It's all one big allocation made at init time, except for the
- * pages that is. The user nodes follows immediately after the
- * page structures.
- */
-typedef struct PGMPOOL
-{
-    /** The VM handle - R3 Ptr. */
-    PVMR3                       pVMR3;
-    /** The VM handle - R0 Ptr. */
-    PVMR0                       pVMR0;
-    /** The VM handle - RC Ptr. */
-    PVMRC                       pVMRC;
-    /** The max pool size. This includes the special IDs.  */
-    uint16_t                    cMaxPages;
-    /** The current pool size. */
-    uint16_t                    cCurPages;
-    /** The head of the free page list. */
-    uint16_t                    iFreeHead;
-    /* Padding. */
-    uint16_t                    u16Padding;
-    /** Head of the chain of free user nodes. */
-    uint16_t                    iUserFreeHead;
-    /** The number of user nodes we've allocated. */
-    uint16_t                    cMaxUsers;
-    /** The number of present page table entries in the entire pool. */
-    uint32_t                    cPresent;
-    /** Pointer to the array of user nodes - RC pointer. */
-    RCPTRTYPE(PPGMPOOLUSER)     paUsersRC;
-    /** Pointer to the array of user nodes - R3 pointer. */
-    R3PTRTYPE(PPGMPOOLUSER)     paUsersR3;
-    /** Pointer to the array of user nodes - R0 pointer. */
-    R0PTRTYPE(PPGMPOOLUSER)     paUsersR0;
-    /** Head of the chain of free phys ext nodes. */
-    uint16_t                    iPhysExtFreeHead;
-    /** The number of user nodes we've allocated. */
-    uint16_t                    cMaxPhysExts;
-    /** Pointer to the array of physical xref extent - RC pointer. */
-    RCPTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsRC;
-    /** Pointer to the array of physical xref extent nodes - R3 pointer. */
-    R3PTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsR3;
-    /** Pointer to the array of physical xref extent nodes - R0 pointer. */
-    R0PTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsR0;
-    /** Hash table for GCPhys addresses. */
-    uint16_t                    aiHash[PGMPOOL_HASH_SIZE];
-    /** The head of the age list. */
-    uint16_t                    iAgeHead;
-    /** The tail of the age list. */
-    uint16_t                    iAgeTail;
-    /** Set if the cache is enabled. */
-    bool                        fCacheEnabled;
-    /** Alignment padding. */
-    bool                        afPadding1[3];
-    /** Head of the list of modified pages. */
-    uint16_t                    iModifiedHead;
-    /** The current number of modified pages. */
-    uint16_t                    cModifiedPages;
-    /** Access handler, RC. */
-    RCPTRTYPE(PFNPGMRCPHYSHANDLER)  pfnAccessHandlerRC;
-    /** Access handler, R0. */
-    R0PTRTYPE(PFNPGMR0PHYSHANDLER)  pfnAccessHandlerR0;
-    /** Access handler, R3. */
-    R3PTRTYPE(PFNPGMR3PHYSHANDLER)  pfnAccessHandlerR3;
-    /** The access handler description (R3 ptr). */
-    R3PTRTYPE(const char *)         pszAccessHandler;
-# if HC_ARCH_BITS == 32
-    /** Alignment padding. */
-    uint32_t                    u32Padding2;
-# endif
-    /* Next available slot. */
-    uint32_t                    idxFreeDirtyPage;
-    /* Number of active dirty pages. */
-    uint32_t                    cDirtyPages;
-    /* Array of current dirty pgm pool page indices. */
-    struct
-    {
-        uint16_t                    uIdx;
-        uint16_t                    Alignment[3];
-        uint64_t                    aPage[512];
-    } aDirtyPages[16];
-    /** The number of pages currently in use. */
-    uint16_t                    cUsedPages;
-#ifdef VBOX_WITH_STATISTICS
-    /** The high water mark for cUsedPages. */
-    uint16_t                    cUsedPagesHigh;
-    uint32_t                    Alignment1;         /**< Align the next member on a 64-bit boundary. */
-    /** Profiling pgmPoolAlloc(). */
-    STAMPROFILEADV              StatAlloc;
-    /** Profiling pgmR3PoolClearDoIt(). */
-    STAMPROFILE                 StatClearAll;
-    /** Profiling pgmR3PoolReset(). */
-    STAMPROFILE                 StatR3Reset;
-    /** Profiling pgmPoolFlushPage(). */
-    STAMPROFILE                 StatFlushPage;
-    /** Profiling pgmPoolFree(). */
-    STAMPROFILE                 StatFree;
-    /** Counting explicit flushes by PGMPoolFlushPage(). */
-    STAMCOUNTER                 StatForceFlushPage;
-    /** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */
-    STAMCOUNTER                 StatForceFlushDirtyPage;
-    /** Counting flushes for reused pages. */
-    STAMCOUNTER                 StatForceFlushReused;
-    /** Profiling time spent zeroing pages. */
-    STAMPROFILE                 StatZeroPage;
-    /** Profiling of pgmPoolTrackDeref. */
-    STAMPROFILE                 StatTrackDeref;
-    /** Profiling pgmTrackFlushGCPhysPT. */
-    STAMPROFILE                 StatTrackFlushGCPhysPT;
-    /** Profiling pgmTrackFlushGCPhysPTs. */
-    STAMPROFILE                 StatTrackFlushGCPhysPTs;
-    /** Profiling pgmTrackFlushGCPhysPTsSlow. */
-    STAMPROFILE                 StatTrackFlushGCPhysPTsSlow;
-    /** Number of times we've been out of user records. */
-    STAMCOUNTER                 StatTrackFreeUpOneUser;
-    /** Nr of flushed entries. */
-    STAMCOUNTER                 StatTrackFlushEntry;
-    /** Nr of updated entries. */
-    STAMCOUNTER                 StatTrackFlushEntryKeep;
-    /** Profiling deref activity related tracking GC physical pages. */
-    STAMPROFILE                 StatTrackDerefGCPhys;
-    /** Number of linear searches for a HCPhys in the ram ranges. */
-    STAMCOUNTER                 StatTrackLinearRamSearches;
-    /** The number of failing pgmPoolTrackPhysExtAlloc calls. */
-    STAMCOUNTER                 StamTrackPhysExtAllocFailures;
-    /** Profiling the RC/R0 access handler. */
-    STAMPROFILE                 StatMonitorRZ;
-    /** Times we've failed interpreting the instruction. */
-    STAMCOUNTER                 StatMonitorRZEmulateInstr;
-    /** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
-    STAMPROFILE                 StatMonitorRZFlushPage;
-    /* Times we've detected a page table reinit. */
-    STAMCOUNTER                 StatMonitorRZFlushReinit;
-    /** Counting flushes for pages that are modified too often. */
-    STAMCOUNTER                 StatMonitorRZFlushModOverflow;
-    /** Times we've detected fork(). */
-    STAMCOUNTER                 StatMonitorRZFork;
-    /** Profiling the RC/R0 access we've handled (except REP STOSD). */
-    STAMPROFILE                 StatMonitorRZHandled;
-    /** Times we've failed interpreting a patch code instruction. */
-    STAMCOUNTER                 StatMonitorRZIntrFailPatch1;
-    /** Times we've failed interpreting a patch code instruction during flushing. */
-    STAMCOUNTER                 StatMonitorRZIntrFailPatch2;
-    /** The number of times we've seen rep prefixes we can't handle. */
-    STAMCOUNTER                 StatMonitorRZRepPrefix;
-    /** Profiling the REP STOSD cases we've handled. */
-    STAMPROFILE                 StatMonitorRZRepStosd;
-    /** Nr of handled PT faults. */
-    STAMCOUNTER                 StatMonitorRZFaultPT;
-    /** Nr of handled PD faults. */
-    STAMCOUNTER                 StatMonitorRZFaultPD;
-    /** Nr of handled PDPT faults. */
-    STAMCOUNTER                 StatMonitorRZFaultPDPT;
-    /** Nr of handled PML4 faults. */
-    STAMCOUNTER                 StatMonitorRZFaultPML4;
-
-    /** Profiling the R3 access handler. */
-    STAMPROFILE                 StatMonitorR3;
-    /** Times we've failed interpreting the instruction. */
-    STAMCOUNTER                 StatMonitorR3EmulateInstr;
-    /** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
-    STAMPROFILE                 StatMonitorR3FlushPage;
-    /* Times we've detected a page table reinit. */
-    STAMCOUNTER                 StatMonitorR3FlushReinit;
-    /** Counting flushes for pages that are modified too often. */
-    STAMCOUNTER                 StatMonitorR3FlushModOverflow;
-    /** Times we've detected fork(). */
-    STAMCOUNTER                 StatMonitorR3Fork;
-    /** Profiling the R3 access we've handled (except REP STOSD). */
-    STAMPROFILE                 StatMonitorR3Handled;
-    /** The number of times we've seen rep prefixes we can't handle. */
-    STAMCOUNTER                 StatMonitorR3RepPrefix;
-    /** Profiling the REP STOSD cases we've handled. */
-    STAMPROFILE                 StatMonitorR3RepStosd;
-    /** Nr of handled PT faults. */
-    STAMCOUNTER                 StatMonitorR3FaultPT;
-    /** Nr of handled PD faults. */
-    STAMCOUNTER                 StatMonitorR3FaultPD;
-    /** Nr of handled PDPT faults. */
-    STAMCOUNTER                 StatMonitorR3FaultPDPT;
-    /** Nr of handled PML4 faults. */
-    STAMCOUNTER                 StatMonitorR3FaultPML4;
-    /** The number of times we're called in an async thread an need to flush. */
-    STAMCOUNTER                 StatMonitorR3Async;
-    /** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
-    STAMCOUNTER                 StatResetDirtyPages;
-    /** Times we've called pgmPoolAddDirtyPage. */
-    STAMCOUNTER                 StatDirtyPage;
-    /** Times we've had to flush duplicates for dirty page management. */
-    STAMCOUNTER                 StatDirtyPageDupFlush;
-    /** Times we've had to flush because of overflow. */
-    STAMCOUNTER                 StatDirtyPageOverFlowFlush;
-
-    /** The high water mark for cModifiedPages. */
-    uint16_t                    cModifiedPagesHigh;
-    uint16_t                    Alignment2[3];      /**< Align the next member on a 64-bit boundary. */
-
-    /** The number of cache hits. */
-    STAMCOUNTER                 StatCacheHits;
-    /** The number of cache misses. */
-    STAMCOUNTER                 StatCacheMisses;
-    /** The number of times we've got a conflict of 'kind' in the cache. */
-    STAMCOUNTER                 StatCacheKindMismatches;
-    /** Number of times we've been out of pages. */
-    STAMCOUNTER                 StatCacheFreeUpOne;
-    /** The number of cacheable allocations. */
-    STAMCOUNTER                 StatCacheCacheable;
-    /** The number of uncacheable allocations. */
-    STAMCOUNTER                 StatCacheUncacheable;
-#else
-    uint32_t                    Alignment3;         /**< Align the next member on a 64-bit boundary. */
-#endif
-    /** The AVL tree for looking up a page by its HC physical address. */
-    AVLOHCPHYSTREE              HCPhysTree;
-    uint32_t                    Alignment4;         /**< Align the next member on a 64-bit boundary. */
-    /** Array of pages. (cMaxPages in length)
-     * The Id is the index into thist array.
-     */
-    PGMPOOLPAGE                 aPages[PGMPOOL_IDX_FIRST];
-} PGMPOOL, *PPGMPOOL, **PPPGMPOOL;
-AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8);
-AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8);
-AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8);
-#ifdef VBOX_WITH_STATISTICS
-AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8);
-#endif
-AssertCompileMemberAlignment(PGMPOOL, aPages, 8);
-
-
-/** @def PGMPOOL_PAGE_2_PTR
- * Maps a pool page pool into the current context.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   pPage   The pool page.
- *
- * @remark  In RC this uses PGMGCDynMapHCPage(), so it will consume of the
- *          small page window employeed by that function. Be careful.
- * @remark  There is no need to assert on the result.
- */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
-# define PGMPOOL_PAGE_2_PTR(pVM, pPage)  pgmPoolMapPageInlined((pVM), (pPage) RTLOG_COMMA_SRC_POS)
-#elif defined(VBOX_STRICT)
-# define PGMPOOL_PAGE_2_PTR(pVM, pPage)  pgmPoolMapPageStrict(pPage)
-DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
-{
-    Assert(pPage && pPage->pvPageR3);
-    return pPage->pvPageR3;
-}
-#else
-# define PGMPOOL_PAGE_2_PTR(pVM, pPage)  ((pPage)->pvPageR3)
-#endif
-
-
-/** @def PGMPOOL_PAGE_2_PTR_V2
- * Maps a pool page pool into the current context, taking both VM and VMCPU.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- * @param   pVCpu   The current CPU.
- * @param   pPage   The pool page.
- *
- * @remark  In RC this uses PGMGCDynMapHCPage(), so it will consume of the
- *          small page window employeed by that function. Be careful.
- * @remark  There is no need to assert on the result.
- */
-#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
-# define PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPage)   pgmPoolMapPageV2Inlined((pVM), (pVCpu), (pPage) RTLOG_COMMA_SRC_POS)
-#else
-# define PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPage)   PGMPOOL_PAGE_2_PTR((pVM), (pPage))
-#endif
-
-
-/** @name Per guest page tracking data.
- * This is currently as a 16-bit word in the PGMPAGE structure, the idea though
- * is to use more bits for it and split it up later on. But for now we'll play
- * safe and change as little as possible.
- *
- * The 16-bit word has two parts:
- *
- * The first 14-bit forms the @a idx field. It is either the index of a page in
- * the shadow page pool, or and index into the extent list.
- *
- * The 2 topmost bits makes up the @a cRefs field, which counts the number of
- * shadow page pool references to the page. If cRefs equals
- * PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
- * (misnomer) table and not the shadow page pool.
- *
- * See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
- * the 16-bit word.
- *
- * @{ */
-/** The shift count for getting to the cRefs part. */
-#define PGMPOOL_TD_CREFS_SHIFT          14
-/** The mask applied after shifting the tracking data down by
- * PGMPOOL_TD_CREFS_SHIFT. */
-#define PGMPOOL_TD_CREFS_MASK           0x3
-/** The cRefs value used to indicate that the idx is the head of a
- * physical cross reference list. */
-#define PGMPOOL_TD_CREFS_PHYSEXT        PGMPOOL_TD_CREFS_MASK
-/** The shift used to get idx. */
-#define PGMPOOL_TD_IDX_SHIFT            0
-/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
-#define PGMPOOL_TD_IDX_MASK             0x3fff
-/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
- * simply too many mappings of this page. */
-#define PGMPOOL_TD_IDX_OVERFLOWED       PGMPOOL_TD_IDX_MASK
-
-/** @def PGMPOOL_TD_MAKE
- * Makes a 16-bit tracking data word.
- *
- * @returns tracking data.
- * @param   cRefs       The @a cRefs field. Must be within bounds!
- * @param   idx         The @a idx field. Must also be within bounds! */
-#define PGMPOOL_TD_MAKE(cRefs, idx)     ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) | (idx) )
-
-/** @def PGMPOOL_TD_GET_CREFS
- * Get the @a cRefs field from a tracking data word.
- *
- * @returns The @a cRefs field
- * @param   u16         The tracking data word.
- * @remarks This will only return 1 or PGMPOOL_TD_CREFS_PHYSEXT for a
- *          non-zero @a u16. */
-#define PGMPOOL_TD_GET_CREFS(u16)       ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
-
-/** @def PGMPOOL_TD_GET_IDX
- * Get the @a idx field from a tracking data word.
- *
- * @returns The @a idx field
- * @param   u16         The tracking data word. */
-#define PGMPOOL_TD_GET_IDX(u16)         ( ((u16) >> PGMPOOL_TD_IDX_SHIFT)   & PGMPOOL_TD_IDX_MASK   )
-/** @} */
-
-
-/**
- * Trees are using self relative offsets as pointers.
- * So, all its data, including the root pointer, must be in the heap for HC and GC
- * to have the same layout.
- */
-typedef struct PGMTREES
-{
-    /** Physical access handlers (AVL range+offsetptr tree). */
-    AVLROGCPHYSTREE                 PhysHandlers;
-    /** Virtual access handlers (AVL range + GC ptr tree). */
-    AVLROGCPTRTREE                  VirtHandlers;
-    /** Virtual access handlers (Phys range AVL range + offsetptr tree). */
-    AVLROGCPHYSTREE                 PhysToVirtHandlers;
-    /** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
-    AVLROGCPTRTREE                  HyperVirtHandlers;
-} PGMTREES;
-/** Pointer to PGM trees. */
-typedef PGMTREES *PPGMTREES;
-
-
-/**
- * Page fault guest state for the AMD64 paging mode.
- */
-typedef struct PGMPTWALKCORE
-{
-    /** The guest virtual address that is being resolved by the walk
-     *  (input). */
-    RTGCPTR         GCPtr;
-
-    /** The guest physical address that is the result of the walk.
-     * @remarks only valid if fSucceeded is set.  */
-    RTGCPHYS        GCPhys;
-
-    /** Set if the walk succeeded, i.d. GCPhys is valid. */
-    bool            fSucceeded;
-    /** The level problem arrised at.
-     * PTE is level 1, PDE is level 2, PDPE is level 3, PML4 is level 4, CR3 is
-     * level 8.  This is 0 on success. */
-    uint8_t         uLevel;
-    /** Set if the page isn't present. */
-    bool            fNotPresent;
-    /** Encountered a bad physical address. */
-    bool            fBadPhysAddr;
-    /** Set if there was reserved bit violations. */
-    bool            fRsvdError;
-    /** Set if it involves a big page (2/4 MB). */
-    bool            fBigPage;
-    /** Set if it involves a gigantic page (1 GB). */
-    bool            fGigantPage;
-    /** The effect X86_PTE_US flag for the address. */
-    bool            fEffectiveUS;
-    /** The effect X86_PTE_RW flag for the address. */
-    bool            fEffectiveRW;
-    /** The effect X86_PTE_NX flag for the address. */
-    bool            fEffectiveNX;
-} PGMPTWALKCORE;
-
-
-/**
- * Guest page table walk for the AMD64 mode.
- */
-typedef struct PGMPTWALKGSTAMD64
-{
-    /** The common core. */
-    PGMPTWALKCORE   Core;
-
-    PX86PML4        pPml4;
-    PX86PML4E       pPml4e;
-    X86PML4E        Pml4e;
-
-    PX86PDPT        pPdpt;
-    PX86PDPE        pPdpe;
-    X86PDPE         Pdpe;
-
-    PX86PDPAE       pPd;
-    PX86PDEPAE      pPde;
-    X86PDEPAE       Pde;
-
-    PX86PTPAE       pPt;
-    PX86PTEPAE      pPte;
-    X86PTEPAE       Pte;
-} PGMPTWALKGSTAMD64;
-/** Pointer to a AMD64 guest page table walk. */
-typedef PGMPTWALKGSTAMD64 *PPGMPTWALKGSTAMD64;
-/** Pointer to a const AMD64 guest page table walk. */
-typedef PGMPTWALKGSTAMD64 const *PCPGMPTWALKGSTAMD64;
-
-/**
- * Guest page table walk for the PAE mode.
- */
-typedef struct PGMPTWALKGSTPAE
-{
-    /** The common core. */
-    PGMPTWALKCORE   Core;
-
-    PX86PDPT        pPdpt;
-    PX86PDPE        pPdpe;
-    X86PDPE         Pdpe;
-
-    PX86PDPAE       pPd;
-    PX86PDEPAE      pPde;
-    X86PDEPAE       Pde;
-
-    PX86PTPAE       pPt;
-    PX86PTEPAE      pPte;
-    X86PTEPAE       Pte;
-} PGMPTWALKGSTPAE;
-/** Pointer to a PAE guest page table walk. */
-typedef PGMPTWALKGSTPAE *PPGMPTWALKGSTPAE;
-/** Pointer to a const AMD64 guest page table walk. */
-typedef PGMPTWALKGSTPAE const *PCPGMPTWALKGSTPAE;
-
-/**
- * Guest page table walk for the 32-bit mode.
- */
-typedef struct PGMPTWALKGST32BIT
-{
-    /** The common core. */
-    PGMPTWALKCORE   Core;
-
-    PX86PD          pPd;
-    PX86PDE         pPde;
-    X86PDE          Pde;
-
-    PX86PT          pPt;
-    PX86PTE         pPte;
-    X86PTE          Pte;
-} PGMPTWALKGST32BIT;
-/** Pointer to a 32-bit guest page table walk. */
-typedef PGMPTWALKGST32BIT *PPGMPTWALKGST32BIT;
-/** Pointer to a const 32-bit guest page table walk. */
-typedef PGMPTWALKGST32BIT const *PCPGMPTWALKGST32BIT;
-
-
-/** @name Paging mode macros
- * @{
- */
-#ifdef IN_RC
-# define PGM_CTX(a,b)                   a##RC##b
-# define PGM_CTX_STR(a,b)               a "GC" b
-# define PGM_CTX_DECL(type)             VMMRCDECL(type)
-#else
-# ifdef IN_RING3
-#  define PGM_CTX(a,b)                   a##R3##b
-#  define PGM_CTX_STR(a,b)               a "R3" b
-#  define PGM_CTX_DECL(type)             DECLCALLBACK(type)
-# else
-#  define PGM_CTX(a,b)                   a##R0##b
-#  define PGM_CTX_STR(a,b)               a "R0" b
-#  define PGM_CTX_DECL(type)             VMMDECL(type)
-# endif
-#endif
-
-#define PGM_GST_NAME_REAL(name)         PGM_CTX(pgm,GstReal##name)
-#define PGM_GST_NAME_RC_REAL_STR(name)  "pgmRCGstReal" #name
-#define PGM_GST_NAME_R0_REAL_STR(name)  "pgmR0GstReal" #name
-#define PGM_GST_NAME_PROT(name)         PGM_CTX(pgm,GstProt##name)
-#define PGM_GST_NAME_RC_PROT_STR(name)  "pgmRCGstProt" #name
-#define PGM_GST_NAME_R0_PROT_STR(name)  "pgmR0GstProt" #name
-#define PGM_GST_NAME_32BIT(name)        PGM_CTX(pgm,Gst32Bit##name)
-#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
-#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
-#define PGM_GST_NAME_PAE(name)          PGM_CTX(pgm,GstPAE##name)
-#define PGM_GST_NAME_RC_PAE_STR(name)   "pgmRCGstPAE" #name
-#define PGM_GST_NAME_R0_PAE_STR(name)   "pgmR0GstPAE" #name
-#define PGM_GST_NAME_AMD64(name)        PGM_CTX(pgm,GstAMD64##name)
-#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
-#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
-#define PGM_GST_PFN(name, pVCpu)        ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
-#define PGM_GST_DECL(type, name)        PGM_CTX_DECL(type) PGM_GST_NAME(name)
-
-#define PGM_SHW_NAME_32BIT(name)        PGM_CTX(pgm,Shw32Bit##name)
-#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
-#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
-#define PGM_SHW_NAME_PAE(name)          PGM_CTX(pgm,ShwPAE##name)
-#define PGM_SHW_NAME_RC_PAE_STR(name)   "pgmRCShwPAE" #name
-#define PGM_SHW_NAME_R0_PAE_STR(name)   "pgmR0ShwPAE" #name
-#define PGM_SHW_NAME_AMD64(name)        PGM_CTX(pgm,ShwAMD64##name)
-#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
-#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
-#define PGM_SHW_NAME_NESTED(name)        PGM_CTX(pgm,ShwNested##name)
-#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
-#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
-#define PGM_SHW_NAME_EPT(name)          PGM_CTX(pgm,ShwEPT##name)
-#define PGM_SHW_NAME_RC_EPT_STR(name)   "pgmRCShwEPT" #name
-#define PGM_SHW_NAME_R0_EPT_STR(name)   "pgmR0ShwEPT" #name
-#define PGM_SHW_DECL(type, name)        PGM_CTX_DECL(type) PGM_SHW_NAME(name)
-#define PGM_SHW_PFN(name, pVCpu)        ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
-
-/*                   Shw_Gst */
-#define PGM_BTH_NAME_32BIT_REAL(name)   PGM_CTX(pgm,Bth32BitReal##name)
-#define PGM_BTH_NAME_32BIT_PROT(name)   PGM_CTX(pgm,Bth32BitProt##name)
-#define PGM_BTH_NAME_32BIT_32BIT(name)  PGM_CTX(pgm,Bth32Bit32Bit##name)
-#define PGM_BTH_NAME_PAE_REAL(name)     PGM_CTX(pgm,BthPAEReal##name)
-#define PGM_BTH_NAME_PAE_PROT(name)     PGM_CTX(pgm,BthPAEProt##name)
-#define PGM_BTH_NAME_PAE_32BIT(name)    PGM_CTX(pgm,BthPAE32Bit##name)
-#define PGM_BTH_NAME_PAE_PAE(name)      PGM_CTX(pgm,BthPAEPAE##name)
-#define PGM_BTH_NAME_AMD64_PROT(name)   PGM_CTX(pgm,BthAMD64Prot##name)
-#define PGM_BTH_NAME_AMD64_AMD64(name)  PGM_CTX(pgm,BthAMD64AMD64##name)
-#define PGM_BTH_NAME_NESTED_REAL(name)  PGM_CTX(pgm,BthNestedReal##name)
-#define PGM_BTH_NAME_NESTED_PROT(name)  PGM_CTX(pgm,BthNestedProt##name)
-#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
-#define PGM_BTH_NAME_NESTED_PAE(name)   PGM_CTX(pgm,BthNestedPAE##name)
-#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
-#define PGM_BTH_NAME_EPT_REAL(name)     PGM_CTX(pgm,BthEPTReal##name)
-#define PGM_BTH_NAME_EPT_PROT(name)     PGM_CTX(pgm,BthEPTProt##name)
-#define PGM_BTH_NAME_EPT_32BIT(name)    PGM_CTX(pgm,BthEPT32Bit##name)
-#define PGM_BTH_NAME_EPT_PAE(name)      PGM_CTX(pgm,BthEPTPAE##name)
-#define PGM_BTH_NAME_EPT_AMD64(name)    PGM_CTX(pgm,BthEPTAMD64##name)
-
-#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name)    "pgmRCBth32BitReal" #name
-#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name)    "pgmRCBth32BitProt" #name
-#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name)   "pgmRCBth32Bit32Bit" #name
-#define PGM_BTH_NAME_RC_PAE_REAL_STR(name)      "pgmRCBthPAEReal" #name
-#define PGM_BTH_NAME_RC_PAE_PROT_STR(name)      "pgmRCBthPAEProt" #name
-#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name)     "pgmRCBthPAE32Bit" #name
-#define PGM_BTH_NAME_RC_PAE_PAE_STR(name)       "pgmRCBthPAEPAE" #name
-#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name)   "pgmRCBthAMD64AMD64" #name
-#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name)   "pgmRCBthNestedReal" #name
-#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name)   "pgmRCBthNestedProt" #name
-#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name)  "pgmRCBthNested32Bit" #name
-#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name)    "pgmRCBthNestedPAE" #name
-#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name)  "pgmRCBthNestedAMD64" #name
-#define PGM_BTH_NAME_RC_EPT_REAL_STR(name)      "pgmRCBthEPTReal" #name
-#define PGM_BTH_NAME_RC_EPT_PROT_STR(name)      "pgmRCBthEPTProt" #name
-#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name)     "pgmRCBthEPT32Bit" #name
-#define PGM_BTH_NAME_RC_EPT_PAE_STR(name)       "pgmRCBthEPTPAE" #name
-#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name)     "pgmRCBthEPTAMD64" #name
-#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name)    "pgmR0Bth32BitReal" #name
-#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name)    "pgmR0Bth32BitProt" #name
-#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name)   "pgmR0Bth32Bit32Bit" #name
-#define PGM_BTH_NAME_R0_PAE_REAL_STR(name)      "pgmR0BthPAEReal" #name
-#define PGM_BTH_NAME_R0_PAE_PROT_STR(name)      "pgmR0BthPAEProt" #name
-#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name)     "pgmR0BthPAE32Bit" #name
-#define PGM_BTH_NAME_R0_PAE_PAE_STR(name)       "pgmR0BthPAEPAE" #name
-#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name)    "pgmR0BthAMD64Prot" #name
-#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name)   "pgmR0BthAMD64AMD64" #name
-#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name)   "pgmR0BthNestedReal" #name
-#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name)   "pgmR0BthNestedProt" #name
-#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name)  "pgmR0BthNested32Bit" #name
-#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name)    "pgmR0BthNestedPAE" #name
-#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name)  "pgmR0BthNestedAMD64" #name
-#define PGM_BTH_NAME_R0_EPT_REAL_STR(name)      "pgmR0BthEPTReal" #name
-#define PGM_BTH_NAME_R0_EPT_PROT_STR(name)      "pgmR0BthEPTProt" #name
-#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name)     "pgmR0BthEPT32Bit" #name
-#define PGM_BTH_NAME_R0_EPT_PAE_STR(name)       "pgmR0BthEPTPAE" #name
-#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name)     "pgmR0BthEPTAMD64" #name
-
-#define PGM_BTH_DECL(type, name)        PGM_CTX_DECL(type) PGM_BTH_NAME(name)
-#define PGM_BTH_PFN(name, pVCpu)        ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
-/** @} */
-
-/**
- * Data for each paging mode.
- */
-typedef struct PGMMODEDATA
-{
-    /** The guest mode type. */
-    uint32_t                        uGstType;
-    /** The shadow mode type. */
-    uint32_t                        uShwType;
-
-    /** @name Function pointers for Shadow paging.
-     * @{
-     */
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwExit,(PVMCPU pVCpu));
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
-
-    DECLRCCALLBACKMEMBER(int,       pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
-    DECLRCCALLBACKMEMBER(int,       pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
-
-    DECLR0CALLBACKMEMBER(int,       pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
-    DECLR0CALLBACKMEMBER(int,       pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
-    /** @} */
-
-    /** @name Function pointers for Guest paging.
-     * @{
-     */
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstExit,(PVMCPU pVCpu));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
-    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
-    DECLRCCALLBACKMEMBER(int,       pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
-    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
-    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
-    DECLR0CALLBACKMEMBER(int,       pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
-    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
-    /** @} */
-
-    /** @name Function pointers for Both Shadow and Guest paging.
-     * @{
-     */
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
-    /*                           no pfnR3BthTrap0eHandler */
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
-#ifdef VBOX_STRICT
-    DECLR3CALLBACKMEMBER(unsigned,  pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
-#endif
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthUnmapCR3,(PVMCPU pVCpu));
-
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
-#ifdef VBOX_STRICT
-    DECLRCCALLBACKMEMBER(unsigned,  pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
-#endif
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthUnmapCR3,(PVMCPU pVCpu));
-
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
-#ifdef VBOX_STRICT
-    DECLR0CALLBACKMEMBER(unsigned,  pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
-#endif
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthUnmapCR3,(PVMCPU pVCpu));
-    /** @} */
-} PGMMODEDATA, *PPGMMODEDATA;
-
-
-#ifdef VBOX_WITH_STATISTICS
-/**
- * PGM statistics.
- *
- * These lives on the heap when compiled in as they would otherwise waste
- * unnecessary space in release builds.
- */
-typedef struct PGMSTATS
-{
-    /* R3 only: */
-    STAMCOUNTER StatR3DetectedConflicts;            /**< R3: Number of times PGMR3MapHasConflicts() detected a conflict. */
-    STAMPROFILE StatR3ResolveConflict;              /**< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). */
-
-    /* R3+RZ */
-    STAMCOUNTER StatRZChunkR3MapTlbHits;            /**< RC/R0: Ring-3/0 chunk mapper TLB hits. */
-    STAMCOUNTER StatRZChunkR3MapTlbMisses;          /**< RC/R0: Ring-3/0 chunk mapper TLB misses. */
-    STAMCOUNTER StatRZPageMapTlbHits;               /**< RC/R0: Ring-3/0 page mapper TLB hits. */
-    STAMCOUNTER StatRZPageMapTlbMisses;             /**< RC/R0: Ring-3/0 page mapper TLB misses. */
-    STAMCOUNTER StatPageMapTlbFlushes;              /**< ALL: Ring-3/0 page mapper TLB flushes. */
-    STAMCOUNTER StatPageMapTlbFlushEntry;           /**< ALL: Ring-3/0 page mapper TLB flushes. */
-    STAMCOUNTER StatR3ChunkR3MapTlbHits;            /**< R3: Ring-3/0 chunk mapper TLB hits. */
-    STAMCOUNTER StatR3ChunkR3MapTlbMisses;          /**< R3: Ring-3/0 chunk mapper TLB misses. */
-    STAMCOUNTER StatR3PageMapTlbHits;               /**< R3: Ring-3/0 page mapper TLB hits. */
-    STAMCOUNTER StatR3PageMapTlbMisses;             /**< R3: Ring-3/0 page mapper TLB misses. */
-    STAMPROFILE StatRZSyncCR3HandlerVirtualReset;   /**< RC/R0: Profiling of the virtual handler resets. */
-    STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate;  /**< RC/R0: Profiling of the virtual handler updates. */
-    STAMPROFILE StatR3SyncCR3HandlerVirtualReset;   /**< R3: Profiling of the virtual handler resets. */
-    STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate;  /**< R3: Profiling of the virtual handler updates. */
-    STAMCOUNTER StatR3PhysHandlerReset;             /**< R3: The number of times PGMHandlerPhysicalReset is called. */
-    STAMCOUNTER StatRZPhysHandlerReset;             /**< RC/R0: The number of times PGMHandlerPhysicalReset is called. */
-    STAMCOUNTER StatR3PhysHandlerLookupHits;        /**< R3: Number of cache hits when looking up physical handlers. */
-    STAMCOUNTER StatR3PhysHandlerLookupMisses;      /**< R3: Number of cache misses when looking up physical handlers. */
-    STAMCOUNTER StatRZPhysHandlerLookupHits;        /**< RC/R0: Number of cache hits when lookup up physical handlers. */
-    STAMCOUNTER StatRZPhysHandlerLookupMisses;      /**< RC/R0: Number of cache misses when looking up physical handlers */
-    STAMPROFILE StatRZVirtHandlerSearchByPhys;      /**< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. */
-    STAMPROFILE StatR3VirtHandlerSearchByPhys;      /**< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. */
-    STAMCOUNTER StatRZPageReplaceShared;            /**< RC/R0: Times a shared page has been replaced by a private one. */
-    STAMCOUNTER StatRZPageReplaceZero;              /**< RC/R0: Times the zero page has been replaced by a private one. */
-/// @todo    STAMCOUNTER StatRZPageHandyAllocs;              /**< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. */
-    STAMCOUNTER StatR3PageReplaceShared;            /**< R3: Times a shared page has been replaced by a private one. */
-    STAMCOUNTER StatR3PageReplaceZero;              /**< R3: Times the zero page has been replaced by a private one. */
-/// @todo    STAMCOUNTER StatR3PageHandyAllocs;              /**< R3: The number of times we've executed GMMR3AllocateHandyPages. */
-
-    /* RC only: */
-    STAMCOUNTER StatRCInvlPgConflict;               /**< RC: Number of times PGMInvalidatePage() detected a mapping conflict. */
-    STAMCOUNTER StatRCInvlPgSyncMonCR3;             /**< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. */
-
-    STAMCOUNTER StatRZPhysRead;
-    STAMCOUNTER StatRZPhysReadBytes;
-    STAMCOUNTER StatRZPhysWrite;
-    STAMCOUNTER StatRZPhysWriteBytes;
-    STAMCOUNTER StatR3PhysRead;
-    STAMCOUNTER StatR3PhysReadBytes;
-    STAMCOUNTER StatR3PhysWrite;
-    STAMCOUNTER StatR3PhysWriteBytes;
-    STAMCOUNTER StatRCPhysRead;
-    STAMCOUNTER StatRCPhysReadBytes;
-    STAMCOUNTER StatRCPhysWrite;
-    STAMCOUNTER StatRCPhysWriteBytes;
-
-    STAMCOUNTER StatRZPhysSimpleRead;
-    STAMCOUNTER StatRZPhysSimpleReadBytes;
-    STAMCOUNTER StatRZPhysSimpleWrite;
-    STAMCOUNTER StatRZPhysSimpleWriteBytes;
-    STAMCOUNTER StatR3PhysSimpleRead;
-    STAMCOUNTER StatR3PhysSimpleReadBytes;
-    STAMCOUNTER StatR3PhysSimpleWrite;
-    STAMCOUNTER StatR3PhysSimpleWriteBytes;
-    STAMCOUNTER StatRCPhysSimpleRead;
-    STAMCOUNTER StatRCPhysSimpleReadBytes;
-    STAMCOUNTER StatRCPhysSimpleWrite;
-    STAMCOUNTER StatRCPhysSimpleWriteBytes;
-
-    STAMCOUNTER StatTrackVirgin;                    /**< The number of first time shadowings. */
-    STAMCOUNTER StatTrackAliased;                   /**< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. */
-    STAMCOUNTER StatTrackAliasedMany;               /**< The number of times we're tracking using cRef2. */
-    STAMCOUNTER StatTrackAliasedLots;               /**< The number of times we're hitting pages which has overflowed cRef2. */
-    STAMCOUNTER StatTrackNoExtentsLeft;             /**< The number of times the extent list was exhausted. */
-    STAMCOUNTER StatTrackOverflows;                 /**< The number of times the extent list grows to long. */
-    STAMPROFILE StatTrackDeref;                     /**< Profiling of SyncPageWorkerTrackDeref (expensive). */
-
-    /** Time spent by the host OS for large page allocation. */
-    STAMPROFILE                 StatAllocLargePage;
-    /** Time spent clearing the newly allocated large pages. */
-    STAMPROFILE                 StatClearLargePage;
-    /** The number of times allocating a large pages takes more than the allowed period. */
-    STAMCOUNTER                 StatLargePageOverflow;
-    /** pgmPhysIsValidLargePage profiling - R3 */
-    STAMPROFILE                 StatR3IsValidLargePage;
-    /** pgmPhysIsValidLargePage profiling - RZ*/
-    STAMPROFILE                 StatRZIsValidLargePage;
-
-    STAMPROFILE                 StatChunkAging;
-    STAMPROFILE                 StatChunkFindCandidate;
-    STAMPROFILE                 StatChunkUnmap;
-    STAMPROFILE                 StatChunkMap;
-} PGMSTATS;
-#endif /* VBOX_WITH_STATISTICS */
-
-
-/**
- * Converts a PGM pointer into a VM pointer.
- * @returns Pointer to the VM structure the PGM is part of.
- * @param   pPGM   Pointer to PGM instance data.
- */
-#define PGM2VM(pPGM)  ( (PVM)((char*)pPGM - pPGM->offVM) )
-
-/**
- * PGM Data (part of VM)
- */
-typedef struct PGM
-{
-    /** Offset to the VM structure. */
-    int32_t                         offVM;
-    /** Offset of the PGMCPU structure relative to VMCPU. */
-    int32_t                         offVCpuPGM;
-
-    /** @cfgm{RamPreAlloc, boolean, false}
-     * Indicates whether the base RAM should all be allocated before starting
-     * the VM (default), or if it should be allocated when first written to.
-     */
-    bool                            fRamPreAlloc;
-    /** Indicates whether write monitoring is currently in use.
-     * This is used to prevent conflicts between live saving and page sharing
-     * detection. */
-    bool                            fPhysWriteMonitoringEngaged;
-    /** Set if the CPU has less than 52-bit physical address width.
-     * This is used  */
-    bool                            fLessThan52PhysicalAddressBits;
-    /** Set when nested paging is active.
-     * This is meant to save calls to HWACCMIsNestedPagingActive and let the
-     * compilers optimize the code better.  Whether we use nested paging or
-     * not is something we find out during VMM initialization and we won't
-     * change this later on. */
-    bool                            fNestedPaging;
-    /** The host paging mode. (This is what SUPLib reports.) */
-    SUPPAGINGMODE                   enmHostMode;
-    /** We're not in a state which permits writes to guest memory.
-     * (Only used in strict builds.) */
-    bool                            fNoMorePhysWrites;
-    /** Alignment padding that makes the next member start on a 8 byte boundary. */
-    bool                            afAlignment1[3];
-
-    /** Indicates that PGMR3FinalizeMappings has been called and that further
-     * PGMR3MapIntermediate calls will be rejected. */
-    bool                            fFinalizedMappings;
-    /** If set no conflict checks are required. */
-    bool                            fMappingsFixed;
-    /** If set if restored as fixed but we were unable to re-fixate at the old
-     *  location because of room or address incompatibilities. */
-    bool                            fMappingsFixedRestored;
-    /** If set, then no mappings are put into the shadow page table.
-     * Use pgmMapAreMappingsEnabled() instead of direct access. */
-    bool                            fMappingsDisabled;
-    /** Size of fixed mapping.
-     * This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
-    uint32_t                        cbMappingFixed;
-    /** Generation ID for the RAM ranges. This member is incremented everytime
-     * a RAM range is linked or unlinked. */
-    uint32_t volatile               idRamRangesGen;
-
-    /** Base address (GC) of fixed mapping.
-     * This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
-    RTGCPTR                         GCPtrMappingFixed;
-    /** The address of the previous RAM range mapping. */
-    RTGCPTR                         GCPtrPrevRamRangeMapping;
-
-    /** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
-    RTGCPHYS                        GCPhys4MBPSEMask;
-    /** Mask containing the invalid bits of a guest physical address.
-     * @remarks this does not stop at bit 52.  */
-    RTGCPHYS                        GCPhysInvAddrMask;
-
-
-    /** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
-     * This is sorted by physical address and contains no overlapping ranges. */
-    R3PTRTYPE(PPGMRAMRANGE)         pRamRangesR3;
-    /** PGM offset based trees - R3 Ptr. */
-    R3PTRTYPE(PPGMTREES)            pTreesR3;
-    /** Caching the last physical handler we looked up in R3. */
-    R3PTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerR3;
-    /** Shadow Page Pool - R3 Ptr. */
-    R3PTRTYPE(PPGMPOOL)             pPoolR3;
-    /** Linked list of GC mappings - for HC.
-     * The list is sorted ascending on address. */
-    R3PTRTYPE(PPGMMAPPING)          pMappingsR3;
-    /** Pointer to the list of ROM ranges - for R3.
-     * This is sorted by physical address and contains no overlapping ranges. */
-    R3PTRTYPE(PPGMROMRANGE)         pRomRangesR3;
-    /** Pointer to the list of MMIO2 ranges - for R3.
-     * Registration order. */
-    R3PTRTYPE(PPGMMMIO2RANGE)       pMmio2RangesR3;
-    /** Pointer to SHW+GST mode data (function pointers).
-     * The index into this table is made up from */
-    R3PTRTYPE(PPGMMODEDATA)         paModeData;
-    /*RTR3PTR                         R3PtrAlignment0;*/
-
-
-    /** R0 pointer corresponding to PGM::pRamRangesR3. */
-    R0PTRTYPE(PPGMRAMRANGE)         pRamRangesR0;
-    /** PGM offset based trees - R0 Ptr. */
-    R0PTRTYPE(PPGMTREES)            pTreesR0;
-    /** Caching the last physical handler we looked up in R0. */
-    R0PTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerR0;
-    /** Shadow Page Pool - R0 Ptr. */
-    R0PTRTYPE(PPGMPOOL)             pPoolR0;
-    /** Linked list of GC mappings - for R0.
-     * The list is sorted ascending on address. */
-    R0PTRTYPE(PPGMMAPPING)          pMappingsR0;
-    /** R0 pointer corresponding to PGM::pRomRangesR3. */
-    R0PTRTYPE(PPGMROMRANGE)         pRomRangesR0;
-    /*RTR0PTR                         R0PtrAlignment0;*/
-
-
-    /** RC pointer corresponding to PGM::pRamRangesR3. */
-    RCPTRTYPE(PPGMRAMRANGE)         pRamRangesRC;
-    /** PGM offset based trees - RC Ptr. */
-    RCPTRTYPE(PPGMTREES)            pTreesRC;
-    /** Caching the last physical handler we looked up in RC. */
-    RCPTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerRC;
-    /** Shadow Page Pool - RC Ptr. */
-    RCPTRTYPE(PPGMPOOL)             pPoolRC;
-    /** Linked list of GC mappings - for RC.
-     * The list is sorted ascending on address. */
-    RCPTRTYPE(PPGMMAPPING)          pMappingsRC;
-    /** RC pointer corresponding to PGM::pRomRangesR3. */
-    RCPTRTYPE(PPGMROMRANGE)         pRomRangesRC;
-    /*RTRCPTR                         RCPtrAlignment0;*/
-    /** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
-    RCPTRTYPE(PX86PTE)              paDynPageMap32BitPTEsGC;
-    /** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
-    RCPTRTYPE(PPGMSHWPTEPAE)        paDynPageMapPaePTEsGC;
-
-
-    /** Pointer to the 5 page CR3 content mapping.
-     * The first page is always the CR3 (in some form) while the 4 other pages
-     * are used of the PDs in PAE mode. */
-    RTGCPTR                         GCPtrCR3Mapping;
-
-    /** @name Intermediate Context
-     * @{ */
-    /** Pointer to the intermediate page directory - Normal. */
-    R3PTRTYPE(PX86PD)               pInterPD;
-    /** Pointer to the intermediate page tables - Normal.
-     * There are two page tables, one for the identity mapping and one for
-     * the host context mapping (of the core code). */
-    R3PTRTYPE(PX86PT)               apInterPTs[2];
-    /** Pointer to the intermediate page tables - PAE. */
-    R3PTRTYPE(PX86PTPAE)            apInterPaePTs[2];
-    /** Pointer to the intermediate page directory - PAE. */
-    R3PTRTYPE(PX86PDPAE)            apInterPaePDs[4];
-    /** Pointer to the intermediate page directory - PAE. */
-    R3PTRTYPE(PX86PDPT)             pInterPaePDPT;
-    /** Pointer to the intermediate page-map level 4 - AMD64. */
-    R3PTRTYPE(PX86PML4)             pInterPaePML4;
-    /** Pointer to the intermediate page directory - AMD64. */
-    R3PTRTYPE(PX86PDPT)             pInterPaePDPT64;
-    /** The Physical Address (HC) of the intermediate Page Directory - Normal. */
-    RTHCPHYS                        HCPhysInterPD;
-    /** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
-    RTHCPHYS                        HCPhysInterPaePDPT;
-    /** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
-    RTHCPHYS                        HCPhysInterPaePML4;
-    /** @} */
-
-    /** Base address of the dynamic page mapping area.
-     * The array is MM_HYPER_DYNAMIC_SIZE bytes big.
-     *
-     * @todo The plan of keeping PGMRCDYNMAP private to PGMRZDynMap.cpp didn't
-     *       work out.  Some cleaning up of the initialization that would
-     *       remove this memory is yet to be done...
-     */
-    RCPTRTYPE(uint8_t *)            pbDynPageMapBaseGC;
-    /** The address of the raw-mode context mapping cache. */
-    RCPTRTYPE(PPGMRCDYNMAP)         pRCDynMap;
-    /** The address of the ring-0 mapping cache if we're making use of it.  */
-    RTR0PTR                         pvR0DynMapUsed;
-#if HC_ARCH_BITS == 32
-    /** Alignment padding that makes the next member start on a 8 byte boundary. */
-    uint32_t                        u32Alignment2;
-#endif
-
-    /** PGM critical section.
-     * This protects the physical & virtual access handlers, ram ranges,
-     * and the page flag updating (some of it anyway).
-     */
-    PDMCRITSECT                     CritSect;
-
-    /**
-     * Data associated with managing the ring-3 mappings of the allocation chunks.
-     */
-    struct
-    {
-        /** The chunk tree, ordered by chunk id. */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-        R3PTRTYPE(PAVLU32NODECORE)  pTree;
-#else
-        R3R0PTRTYPE(PAVLU32NODECORE) pTree;
-#endif
-#if HC_ARCH_BITS == 32
-        uint32_t                    u32Alignment;
-#endif
-        /** The chunk mapping TLB. */
-        PGMCHUNKR3MAPTLB            Tlb;
-        /** The number of mapped chunks. */
-        uint32_t                    c;
-        /** The maximum number of mapped chunks.
-         * @cfgm    PGM/MaxRing3Chunks */
-        uint32_t                    cMax;
-        /** The current time. */
-        uint32_t                    iNow;
-        /** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
-        uint32_t                    AgeingCountdown;
-    } ChunkR3Map;
-
-    /**
-     * The page mapping TLB for ring-3 and (for the time being) ring-0.
-     */
-    PGMPAGER3MAPTLB                 PhysTlbHC;
-
-    /** @name   The zero page.
-     * @{ */
-    /** The host physical address of the zero page. */
-    RTHCPHYS                        HCPhysZeroPg;
-    /** The ring-3 mapping of the zero page. */
-    RTR3PTR                         pvZeroPgR3;
-    /** The ring-0 mapping of the zero page. */
-    RTR0PTR                         pvZeroPgR0;
-    /** The GC mapping of the zero page. */
-    RTRCPTR                         pvZeroPgRC;
-    RTRCPTR                         RCPtrAlignment3;
-    /** @}*/
-
-    /** @name   The Invalid MMIO page.
-     * This page is filled with 0xfeedface.
-     * @{ */
-    /** The host physical address of the invalid MMIO page. */
-    RTHCPHYS                        HCPhysMmioPg;
-    /** The host pysical address of the invalid MMIO page plus all invalid
-     * physical address bits set.  This is used to trigger X86_TRAP_PF_RSVD.
-     * @remarks Check fLessThan52PhysicalAddressBits before use. */
-    RTHCPHYS                        HCPhysInvMmioPg;
-    /** The ring-3 mapping of the invalid MMIO page. */
-    RTR3PTR                         pvMmioPgR3;
-#if HC_ARCH_BITS == 32
-    RTR3PTR                         R3PtrAlignment4;
-#endif
-    /** @} */
-
-
-    /** The number of handy pages. */
-    uint32_t                        cHandyPages;
-
-    /** The number of large handy pages. */
-    uint32_t                        cLargeHandyPages;
-
-    /**
-     * Array of handy pages.
-     *
-     * This array is used in a two way communication between pgmPhysAllocPage
-     * and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
-     * an intermediary.
-     *
-     * The size of this array is important, see pgmPhysEnsureHandyPage for details.
-     * (The current size of 32 pages, means 128 KB of handy memory.)
-     */
-    GMMPAGEDESC                     aHandyPages[PGM_HANDY_PAGES];
-
-    /**
-     * Array of large handy pages. (currently size 1)
-     *
-     * This array is used in a two way communication between pgmPhysAllocLargePage
-     * and GMMR0AllocateLargePage, with PGMR3PhysAllocateLargePage serving as
-     * an intermediary.
-     */
-    GMMPAGEDESC                     aLargeHandyPage[1];
-
-    /**
-     * Live save data.
-     */
-    struct
-    {
-        /** Per type statistics. */
-        struct
-        {
-            /** The number of ready pages.  */
-            uint32_t                cReadyPages;
-            /** The number of dirty pages. */
-            uint32_t                cDirtyPages;
-            /** The number of ready zero pages.  */
-            uint32_t                cZeroPages;
-            /** The number of write monitored pages. */
-            uint32_t                cMonitoredPages;
-        }                           Rom,
-                                    Mmio2,
-                                    Ram;
-        /** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */
-        uint32_t                    cIgnoredPages;
-        /** Indicates that a live save operation is active.  */
-        bool                        fActive;
-        /** Padding. */
-        bool                        afReserved[2];
-        /** The next history index. */
-        uint8_t                     iDirtyPagesHistory;
-        /** History of the total amount of dirty pages. */
-        uint32_t                    acDirtyPagesHistory[64];
-        /** Short term dirty page average. */
-        uint32_t                    cDirtyPagesShort;
-        /** Long term dirty page average. */
-        uint32_t                    cDirtyPagesLong;
-        /** The number of saved pages.  This is used to get some kind of estimate of the
-         * link speed so we can decide when we're done.  It is reset after the first
-         * 7 passes so the speed estimate doesn't get inflated by the initial set of
-         * zero pages.   */
-        uint64_t                    cSavedPages;
-        /** The nanosecond timestamp when cSavedPages was 0. */
-        uint64_t                    uSaveStartNS;
-        /** Pages per second (for statistics). */
-        uint32_t                    cPagesPerSecond;
-        uint32_t                    cAlignment;
-    } LiveSave;
-
-    /** @name   Error injection.
-     * @{ */
-    /** Inject handy page allocation errors pretending we're completely out of
-     * memory. */
-    bool volatile                   fErrInjHandyPages;
-    /** Padding. */
-    bool                            afReserved[3];
-    /** @} */
-
-    /** @name Release Statistics
-     * @{ */
-    uint32_t                        cAllPages;          /**< The total number of pages. (Should be Private + Shared + Zero + Pure MMIO.) */
-    uint32_t                        cPrivatePages;      /**< The number of private pages. */
-    uint32_t                        cSharedPages;       /**< The number of shared pages. */
-    uint32_t                        cReusedSharedPages; /**< The number of reused shared pages. */
-    uint32_t                        cZeroPages;         /**< The number of zero backed pages. */
-    uint32_t                        cPureMmioPages;     /**< The number of pure MMIO pages. */
-    uint32_t                        cMonitoredPages;    /**< The number of write monitored pages. */
-    uint32_t                        cWrittenToPages;    /**< The number of previously write monitored pages. */
-    uint32_t                        cWriteLockedPages;  /**< The number of write locked pages. */
-    uint32_t                        cReadLockedPages;   /**< The number of read locked pages. */
-    uint32_t                        cBalloonedPages;    /**< The number of ballooned pages. */
-    uint32_t                        cMappedChunks;      /**< Number of times we mapped a chunk. */
-    uint32_t                        cUnmappedChunks;    /**< Number of times we unmapped a chunk. */
-/*    uint32_t                        aAlignment4[1]; */
-
-    /** The number of times we were forced to change the hypervisor region location. */
-    STAMCOUNTER                     cRelocations;
-
-    STAMCOUNTER                     StatLargePageAlloc;                 /**< The number of large pages we've allocated.*/
-    STAMCOUNTER                     StatLargePageReused;                /**< The number of large pages we've reused.*/
-    STAMCOUNTER                     StatLargePageRefused;               /**< The number of times we couldn't use a large page.*/
-    STAMCOUNTER                     StatLargePageRecheck;               /**< The number of times we rechecked a disabled large page.*/
-    /** @} */
-
-#ifdef VBOX_WITH_STATISTICS
-    /** @name Statistics on the heap.
-     * @{ */
-    R3PTRTYPE(PGMSTATS *)           pStatsR3;
-    R0PTRTYPE(PGMSTATS *)           pStatsR0;
-    RCPTRTYPE(PGMSTATS *)           pStatsRC;
-    RTRCPTR                         RCPtrAlignment;
-    /** @}  */
-#endif
-} PGM;
-#ifndef IN_TSTVMSTRUCTGC /* HACK */
-AssertCompileMemberAlignment(PGM, paDynPageMap32BitPTEsGC, 8);
-AssertCompileMemberAlignment(PGM, GCPtrMappingFixed, sizeof(RTGCPTR));
-AssertCompileMemberAlignment(PGM, HCPhysInterPD, 8);
-AssertCompileMemberAlignment(PGM, CritSect, 8);
-AssertCompileMemberAlignment(PGM, ChunkR3Map, 8);
-AssertCompileMemberAlignment(PGM, PhysTlbHC, 8);
-AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8);
-AssertCompileMemberAlignment(PGM, aHandyPages, 8);
-AssertCompileMemberAlignment(PGM, cRelocations, 8);
-#endif /* !IN_TSTVMSTRUCTGC */
-/** Pointer to the PGM instance data. */
-typedef PGM *PPGM;
-
-
-
-typedef struct PGMCPUSTATS
-{
-    /* Common */
-    STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES];       /**< SyncPT - PD distribution. */
-    STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES];     /**< SyncPage - PD distribution. */
-
-    /* R0 only: */
-    STAMPROFILE StatR0NpMiscfg;                     /**< R0: PGMR0Trap0eHandlerNPMisconfig() profiling. */
-    STAMCOUNTER StatR0NpMiscfgSyncPage;             /**< R0: SyncPage calls from PGMR0Trap0eHandlerNPMisconfig(). */
-
-    /* RZ only: */
-    STAMPROFILE StatRZTrap0e;                       /**< RC/R0: PGMTrap0eHandler() profiling. */
-    STAMPROFILE StatRZTrap0eTime2Ballooned;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is read access to a ballooned page. */
-    STAMPROFILE StatRZTrap0eTime2CSAM;              /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. */
-    STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. */
-    STAMPROFILE StatRZTrap0eTime2GuestTrap;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. */
-    STAMPROFILE StatRZTrap0eTime2HndPhys;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. */
-    STAMPROFILE StatRZTrap0eTime2HndVirt;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. */
-    STAMPROFILE StatRZTrap0eTime2HndUnhandled;      /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. */
-    STAMPROFILE StatRZTrap0eTime2InvalidPhys;       /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address. */
-    STAMPROFILE StatRZTrap0eTime2MakeWritable;      /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a page that needed to be made writable. */
-    STAMPROFILE StatRZTrap0eTime2Mapping;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is the guest mappings. */
-    STAMPROFILE StatRZTrap0eTime2Misc;              /**< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. */
-    STAMPROFILE StatRZTrap0eTime2OutOfSync;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. */
-    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. */
-    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. */
-    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs;   /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. */
-    STAMPROFILE StatRZTrap0eTime2SyncPT;            /**< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. */
-    STAMPROFILE StatRZTrap0eTime2WPEmulation;       /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP emulation. */
-    STAMCOUNTER StatRZTrap0eConflicts;              /**< RC/R0: The number of times \#PF was caused by an undetected conflict. */
-    STAMCOUNTER StatRZTrap0eHandlersMapping;        /**< RC/R0: Number of traps due to access handlers in mappings. */
-    STAMCOUNTER StatRZTrap0eHandlersOutOfSync;      /**< RC/R0: Number of out-of-sync handled pages. */
-    STAMCOUNTER StatRZTrap0eHandlersPhysAll;        /**< RC/R0: Number of traps due to physical all-access handlers. */
-    STAMCOUNTER StatRZTrap0eHandlersPhysAllOpt;     /**< RC/R0: Number of the physical all-access handler traps using the optimization. */
-    STAMCOUNTER StatRZTrap0eHandlersPhysWrite;      /**< RC/R0: Number of traps due to write-physical access handlers. */
-    STAMCOUNTER StatRZTrap0eHandlersVirtual;        /**< RC/R0: Number of traps due to virtual access handlers. */
-    STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys;  /**< RC/R0: Number of traps due to virtual access handlers found by physical address. */
-    STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/**< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). */
-    STAMCOUNTER StatRZTrap0eHandlersUnhandled;      /**< RC/R0: Number of traps due to access outside range of monitored page(s). */
-    STAMCOUNTER StatRZTrap0eHandlersInvalid;        /**< RC/R0: Number of traps due to access to invalid physical memory. */
-    STAMCOUNTER StatRZTrap0eUSNotPresentRead;       /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eUSNotPresentWrite;      /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eUSWrite;                /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eUSReserved;             /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eUSNXE;                  /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eUSRead;                 /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eSVNotPresentRead;       /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eSVNotPresentWrite;      /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eSVWrite;                /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eSVReserved;             /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eSNXE;                   /**< RC/R0: \#PF err kind */
-    STAMCOUNTER StatRZTrap0eGuestPF;                /**< RC/R0: Real guest \#PFs. */
-    STAMCOUNTER StatRZTrap0eGuestPFMapping;         /**< RC/R0: Real guest \#PF to HMA or other mapping. */
-    STAMCOUNTER StatRZTrap0eWPEmulInRZ;             /**< RC/R0: WP=0 virtualization trap, handled. */
-    STAMCOUNTER StatRZTrap0eWPEmulToR3;             /**< RC/R0: WP=0 virtualization trap, chickened out. */
-    STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES];     /**< RC/R0: PD distribution of the \#PFs. */
-    STAMCOUNTER StatRZGuestCR3WriteHandled;         /**< RC/R0: The number of times WriteHandlerCR3() was successfully called. */
-    STAMCOUNTER StatRZGuestCR3WriteUnhandled;       /**< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. */
-    STAMCOUNTER StatRZGuestCR3WriteConflict;        /**< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. */
-    STAMCOUNTER StatRZGuestROMWriteHandled;         /**< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. */
-    STAMCOUNTER StatRZGuestROMWriteUnhandled;       /**< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler */
-    STAMCOUNTER StatRZDynMapMigrateInvlPg;          /**< RZ: invlpg in PGMR0DynMapMigrateAutoSet. */
-    STAMPROFILE StatRZDynMapGCPageInl;              /**< RZ: Calls to pgmRZDynMapGCPageInlined. */
-    STAMCOUNTER StatRZDynMapGCPageInlHits;          /**< RZ: Hash table lookup hits. */
-    STAMCOUNTER StatRZDynMapGCPageInlMisses;        /**< RZ: Misses that falls back to the code common. */
-    STAMCOUNTER StatRZDynMapGCPageInlRamHits;       /**< RZ: 1st ram range hits. */
-    STAMCOUNTER StatRZDynMapGCPageInlRamMisses;     /**< RZ: 1st ram range misses, takes slow path. */
-    STAMPROFILE StatRZDynMapHCPageInl;              /**< RZ: Calls to pgmRZDynMapHCPageInlined. */
-    STAMCOUNTER StatRZDynMapHCPageInlHits;          /**< RZ: Hash table lookup hits. */
-    STAMCOUNTER StatRZDynMapHCPageInlMisses;        /**< RZ: Misses that falls back to the code common. */
-    STAMPROFILE StatRZDynMapHCPage;                 /**< RZ: Calls to pgmRZDynMapHCPageCommon. */
-    STAMCOUNTER StatRZDynMapSetOptimize;            /**< RZ: Calls to pgmRZDynMapOptimizeAutoSet. */
-    STAMCOUNTER StatRZDynMapSetSearchFlushes;       /**< RZ: Set search restoring to subset flushes. */
-    STAMCOUNTER StatRZDynMapSetSearchHits;          /**< RZ: Set search hits. */
-    STAMCOUNTER StatRZDynMapSetSearchMisses;        /**< RZ: Set search misses. */
-    STAMCOUNTER StatRZDynMapPage;                   /**< RZ: Calls to pgmR0DynMapPage. */
-    STAMCOUNTER StatRZDynMapPageHits0;              /**< RZ: Hits at iPage+0. */
-    STAMCOUNTER StatRZDynMapPageHits1;              /**< RZ: Hits at iPage+1. */
-    STAMCOUNTER StatRZDynMapPageHits2;              /**< RZ: Hits at iPage+2. */
-    STAMCOUNTER StatRZDynMapPageInvlPg;             /**< RZ: invlpg. */
-    STAMCOUNTER StatRZDynMapPageSlow;               /**< RZ: Calls to pgmR0DynMapPageSlow. */
-    STAMCOUNTER StatRZDynMapPageSlowLoopHits;       /**< RZ: Hits in the pgmR0DynMapPageSlow search loop. */
-    STAMCOUNTER StatRZDynMapPageSlowLoopMisses;     /**< RZ: Misses in the pgmR0DynMapPageSlow search loop. */
-    //STAMCOUNTER StatRZDynMapPageSlowLostHits;       /**< RZ: Lost hits. */
-    STAMCOUNTER StatRZDynMapSubsets;                /**< RZ: Times PGMDynMapPushAutoSubset was called. */
-    STAMCOUNTER StatRZDynMapPopFlushes;             /**< RZ: Times PGMDynMapPopAutoSubset flushes the subset. */
-    STAMCOUNTER aStatRZDynMapSetFilledPct[11];      /**< RZ: Set fill distribution, percent. */
-
-    /* HC - R3 and (maybe) R0: */
-
-    /* RZ & R3: */
-    STAMPROFILE StatRZSyncCR3;                      /**< RC/R0: PGMSyncCR3() profiling. */
-    STAMPROFILE StatRZSyncCR3Handlers;              /**< RC/R0: Profiling of the PGMSyncCR3() update handler section. */
-    STAMCOUNTER StatRZSyncCR3Global;                /**< RC/R0: The number of global CR3 syncs. */
-    STAMCOUNTER StatRZSyncCR3NotGlobal;             /**< RC/R0: The number of non-global CR3 syncs. */
-    STAMCOUNTER StatRZSyncCR3DstCacheHit;           /**< RC/R0: The number of times we got some kind of cache hit on a page table. */
-    STAMCOUNTER StatRZSyncCR3DstFreed;              /**< RC/R0: The number of times we've had to free a shadow entry. */
-    STAMCOUNTER StatRZSyncCR3DstFreedSrcNP;         /**< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. */
-    STAMCOUNTER StatRZSyncCR3DstNotPresent;         /**< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. */
-    STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD;    /**< RC/R0: The number of times a global page directory wasn't flushed. */
-    STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT;    /**< RC/R0: The number of times a page table with only global entries wasn't flushed. */
-    STAMPROFILE StatRZSyncPT;                       /**< RC/R0: PGMSyncPT() profiling. */
-    STAMCOUNTER StatRZSyncPTFailed;                 /**< RC/R0: The number of times PGMSyncPT() failed. */
-    STAMCOUNTER StatRZSyncPT4K;                     /**< RC/R0: Number of 4KB syncs. */
-    STAMCOUNTER StatRZSyncPT4M;                     /**< RC/R0: Number of 4MB syncs. */
-    STAMCOUNTER StatRZSyncPagePDNAs;                /**< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
-    STAMCOUNTER StatRZSyncPagePDOutOfSync;          /**< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. */
-    STAMCOUNTER StatRZAccessedPage;                 /**< RC/R0: The number of pages marked not present for accessed bit emulation. */
-    STAMPROFILE StatRZDirtyBitTracking;             /**< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. */
-    STAMCOUNTER StatRZDirtyPage;                    /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
-    STAMCOUNTER StatRZDirtyPageBig;                 /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
-    STAMCOUNTER StatRZDirtyPageSkipped;             /**< RC/R0: The number of pages already dirty or readonly. */
-    STAMCOUNTER StatRZDirtyPageTrap;                /**< RC/R0: The number of traps generated for dirty bit tracking. */
-    STAMCOUNTER StatRZDirtyPageStale;               /**< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) */
-    STAMCOUNTER StatRZDirtyTrackRealPF;             /**< RC/R0: The number of real pages faults during dirty bit tracking. */
-    STAMCOUNTER StatRZDirtiedPage;                  /**< RC/R0: The number of pages marked dirty because of write accesses. */
-    STAMCOUNTER StatRZPageAlreadyDirty;             /**< RC/R0: The number of pages already marked dirty because of write accesses. */
-    STAMPROFILE StatRZInvalidatePage;               /**< RC/R0: PGMInvalidatePage() profiling. */
-    STAMCOUNTER StatRZInvalidatePage4KBPages;       /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. */
-    STAMCOUNTER StatRZInvalidatePage4MBPages;       /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. */
-    STAMCOUNTER StatRZInvalidatePage4MBPagesSkip;   /**< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. */
-    STAMCOUNTER StatRZInvalidatePagePDMappings;     /**< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). */
-    STAMCOUNTER StatRZInvalidatePagePDNAs;          /**< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
-    STAMCOUNTER StatRZInvalidatePagePDNPs;          /**< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. */
-    STAMCOUNTER StatRZInvalidatePagePDOutOfSync;    /**< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
-    STAMCOUNTER StatRZInvalidatePageSkipped;        /**< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
-    STAMCOUNTER StatRZPageOutOfSyncUser;            /**< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
-    STAMCOUNTER StatRZPageOutOfSyncSupervisor;      /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
-    STAMCOUNTER StatRZPageOutOfSyncUserWrite;       /**< RC/R0: The number of times user page is out of sync was detected in \#PF. */
-    STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF. */
-    STAMCOUNTER StatRZPageOutOfSyncBallloon;        /**< RC/R0: The number of times a ballooned page was accessed (read). */
-    STAMPROFILE StatRZPrefetch;                     /**< RC/R0: PGMPrefetchPage. */
-    STAMPROFILE StatRZFlushTLB;                     /**< RC/R0: Profiling of the PGMFlushTLB() body. */
-    STAMCOUNTER StatRZFlushTLBNewCR3;               /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
-    STAMCOUNTER StatRZFlushTLBNewCR3Global;         /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
-    STAMCOUNTER StatRZFlushTLBSameCR3;              /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
-    STAMCOUNTER StatRZFlushTLBSameCR3Global;        /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
-    STAMPROFILE StatRZGstModifyPage;                /**< RC/R0: Profiling of the PGMGstModifyPage() body */
-
-    STAMPROFILE StatR3SyncCR3;                      /**< R3: PGMSyncCR3() profiling. */
-    STAMPROFILE StatR3SyncCR3Handlers;              /**< R3: Profiling of the PGMSyncCR3() update handler section. */
-    STAMCOUNTER StatR3SyncCR3Global;                /**< R3: The number of global CR3 syncs. */
-    STAMCOUNTER StatR3SyncCR3NotGlobal;             /**< R3: The number of non-global CR3 syncs. */
-    STAMCOUNTER StatR3SyncCR3DstFreed;              /**< R3: The number of times we've had to free a shadow entry. */
-    STAMCOUNTER StatR3SyncCR3DstFreedSrcNP;         /**< R3: The number of times we've had to free a shadow entry for which the source entry was not present. */
-    STAMCOUNTER StatR3SyncCR3DstNotPresent;         /**< R3: The number of times we've encountered a not present shadow entry for a present guest entry. */
-    STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD;    /**< R3: The number of times a global page directory wasn't flushed. */
-    STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT;    /**< R3: The number of times a page table with only global entries wasn't flushed. */
-    STAMCOUNTER StatR3SyncCR3DstCacheHit;           /**< R3: The number of times we got some kind of cache hit on a page table. */
-    STAMPROFILE StatR3SyncPT;                       /**< R3: PGMSyncPT() profiling. */
-    STAMCOUNTER StatR3SyncPTFailed;                 /**< R3: The number of times PGMSyncPT() failed. */
-    STAMCOUNTER StatR3SyncPT4K;                     /**< R3: Number of 4KB syncs. */
-    STAMCOUNTER StatR3SyncPT4M;                     /**< R3: Number of 4MB syncs. */
-    STAMCOUNTER StatR3SyncPagePDNAs;                /**< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
-    STAMCOUNTER StatR3SyncPagePDOutOfSync;          /**< R3: The number of time we've encountered an out-of-sync PD in SyncPage. */
-    STAMCOUNTER StatR3AccessedPage;                 /**< R3: The number of pages marked not present for accessed bit emulation. */
-    STAMPROFILE StatR3DirtyBitTracking;             /**< R3: Profiling the dirty bit tracking in CheckPageFault(). */
-    STAMCOUNTER StatR3DirtyPage;                    /**< R3: The number of pages marked read-only for dirty bit tracking. */
-    STAMCOUNTER StatR3DirtyPageBig;                 /**< R3: The number of pages marked read-only for dirty bit tracking. */
-    STAMCOUNTER StatR3DirtyPageSkipped;             /**< R3: The number of pages already dirty or readonly. */
-    STAMCOUNTER StatR3DirtyPageTrap;                /**< R3: The number of traps generated for dirty bit tracking. */
-    STAMCOUNTER StatR3DirtyTrackRealPF;             /**< R3: The number of real pages faults during dirty bit tracking. */
-    STAMCOUNTER StatR3DirtiedPage;                  /**< R3: The number of pages marked dirty because of write accesses. */
-    STAMCOUNTER StatR3PageAlreadyDirty;             /**< R3: The number of pages already marked dirty because of write accesses. */
-    STAMPROFILE StatR3InvalidatePage;               /**< R3: PGMInvalidatePage() profiling. */
-    STAMCOUNTER StatR3InvalidatePage4KBPages;       /**< R3: The number of times PGMInvalidatePage() was called for a 4KB page. */
-    STAMCOUNTER StatR3InvalidatePage4MBPages;       /**< R3: The number of times PGMInvalidatePage() was called for a 4MB page. */
-    STAMCOUNTER StatR3InvalidatePage4MBPagesSkip;   /**< R3: The number of times PGMInvalidatePage() skipped a 4MB page. */
-    STAMCOUNTER StatR3InvalidatePagePDNAs;          /**< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
-    STAMCOUNTER StatR3InvalidatePagePDNPs;          /**< R3: The number of times PGMInvalidatePage() was called for a not present page directory. */
-    STAMCOUNTER StatR3InvalidatePagePDMappings;     /**< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). */
-    STAMCOUNTER StatR3InvalidatePagePDOutOfSync;    /**< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
-    STAMCOUNTER StatR3InvalidatePageSkipped;        /**< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
-    STAMCOUNTER StatR3PageOutOfSyncUser;            /**< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
-    STAMCOUNTER StatR3PageOutOfSyncSupervisor;      /**< R3: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
-    STAMCOUNTER StatR3PageOutOfSyncUserWrite;       /**< R3: The number of times user page is out of sync was detected in \#PF. */
-    STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /**< R3: The number of times supervisor page is out of sync was detected in in \#PF. */
-    STAMCOUNTER StatR3PageOutOfSyncBallloon;        /**< R3: The number of times a ballooned page was accessed (read). */
-    STAMPROFILE StatR3Prefetch;                     /**< R3: PGMPrefetchPage. */
-    STAMPROFILE StatR3FlushTLB;                     /**< R3: Profiling of the PGMFlushTLB() body. */
-    STAMCOUNTER StatR3FlushTLBNewCR3;               /**< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
-    STAMCOUNTER StatR3FlushTLBNewCR3Global;         /**< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
-    STAMCOUNTER StatR3FlushTLBSameCR3;              /**< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
-    STAMCOUNTER StatR3FlushTLBSameCR3Global;        /**< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
-    STAMPROFILE StatR3GstModifyPage;                /**< R3: Profiling of the PGMGstModifyPage() body */
-    /** @} */
-} PGMCPUSTATS;
-
-
-/**
- * Converts a PGMCPU pointer into a VM pointer.
- * @returns Pointer to the VM structure the PGM is part of.
- * @param   pPGM   Pointer to PGMCPU instance data.
- */
-#define PGMCPU2VM(pPGM)         ( (PVM)((char*)(pPGM) - (pPGM)->offVM) )
-
-/**
- * Converts a PGMCPU pointer into a PGM pointer.
- * @returns Pointer to the VM structure the PGM is part of.
- * @param   pPGM   Pointer to PGMCPU instance data.
- */
-#define PGMCPU2PGM(pPGMCpu)     ( (PPGM)((char *)(pPGMCpu) - (pPGMCpu)->offPGM) )
-
-/**
- * PGMCPU Data (part of VMCPU).
- */
-typedef struct PGMCPU
-{
-    /** Offset to the VM structure. */
-    int32_t                         offVM;
-    /** Offset to the VMCPU structure. */
-    int32_t                         offVCpu;
-    /** Offset of the PGM structure relative to VMCPU. */
-    int32_t                         offPGM;
-    uint32_t                        uPadding0;      /**< structure size alignment. */
-
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAW_MODE)
-    /** Automatically tracked physical memory mapping set.
-     * Ring-0 and strict raw-mode builds. */
-    PGMMAPSET                       AutoSet;
-#endif
-
-    /** A20 gate mask.
-     * Our current approach to A20 emulation is to let REM do it and don't bother
-     * anywhere else. The interesting Guests will be operating with it enabled anyway.
-     * But whould need arrise, we'll subject physical addresses to this mask. */
-    RTGCPHYS                        GCPhysA20Mask;
-    /** A20 gate state - boolean! */
-    bool                            fA20Enabled;
-    /** Mirror of the EFER.NXE bit.  Managed by PGMNotifyNxeChanged. */
-    bool                            fNoExecuteEnabled;
-    /** Unused bits. */
-    bool                            afUnused[2];
-
-    /** What needs syncing (PGM_SYNC_*).
-     * This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
-     * PGMFlushTLB, and PGMR3Load. */
-    RTUINT                          fSyncFlags;
-
-    /** The shadow paging mode. */
-    PGMMODE                         enmShadowMode;
-    /** The guest paging mode. */
-    PGMMODE                         enmGuestMode;
-
-    /** The current physical address representing in the guest CR3 register. */
-    RTGCPHYS                        GCPhysCR3;
-
-    /** @name 32-bit Guest Paging.
-     * @{ */
-    /** The guest's page directory, R3 pointer. */
-    R3PTRTYPE(PX86PD)               pGst32BitPdR3;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-    /** The guest's page directory, R0 pointer. */
-    R0PTRTYPE(PX86PD)               pGst32BitPdR0;
-#endif
-    /** The guest's page directory, static RC mapping. */
-    RCPTRTYPE(PX86PD)               pGst32BitPdRC;
-    /** Mask containing the MBZ bits of a big page PDE. */
-    uint32_t                        fGst32BitMbzBigPdeMask;
-    /** Set if the page size extension (PSE) is enabled. */
-    bool                            fGst32BitPageSizeExtension;
-    /** Alignment padding. */
-    bool                            afAlignment2[3];
-    /** @} */
-
-    /** @name PAE Guest Paging.
-     * @{ */
-    /** The guest's page directory pointer table, static RC mapping. */
-    RCPTRTYPE(PX86PDPT)             pGstPaePdptRC;
-    /** The guest's page directory pointer table, R3 pointer. */
-    R3PTRTYPE(PX86PDPT)             pGstPaePdptR3;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-    /** The guest's page directory pointer table, R0 pointer. */
-    R0PTRTYPE(PX86PDPT)             pGstPaePdptR0;
-#endif
-
-    /** The guest's page directories, R3 pointers.
-     * These are individual pointers and don't have to be adjacent.
-     * These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
-    R3PTRTYPE(PX86PDPAE)            apGstPaePDsR3[4];
-    /** The guest's page directories, R0 pointers.
-     * Same restrictions as apGstPaePDsR3. */
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-    R0PTRTYPE(PX86PDPAE)            apGstPaePDsR0[4];
-#endif
-    /** The guest's page directories, static GC mapping.
-     * Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
-     * These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
-    RCPTRTYPE(PX86PDPAE)            apGstPaePDsRC[4];
-    /** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
-    RTGCPHYS                        aGCPhysGstPaePDs[4];
-    /** The physical addresses of the monitored guest page directories (PAE). */
-    RTGCPHYS                        aGCPhysGstPaePDsMonitored[4];
-    /** Mask containing the MBZ PTE bits. */
-    uint64_t                        fGstPaeMbzPteMask;
-    /** Mask containing the MBZ PDE bits. */
-    uint64_t                        fGstPaeMbzPdeMask;
-    /** Mask containing the MBZ big page PDE bits. */
-    uint64_t                        fGstPaeMbzBigPdeMask;
-    /** Mask containing the MBZ PDPE bits. */
-    uint64_t                        fGstPaeMbzPdpeMask;
-    /** @} */
-
-    /** @name AMD64 Guest Paging.
-     * @{ */
-    /** The guest's page directory pointer table, R3 pointer. */
-    R3PTRTYPE(PX86PML4)             pGstAmd64Pml4R3;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-    /** The guest's page directory pointer table, R0 pointer. */
-    R0PTRTYPE(PX86PML4)             pGstAmd64Pml4R0;
-#else
-    RTR0PTR                         alignment6b; /**< alignment equalizer. */
-#endif
-    /** Mask containing the MBZ PTE bits. */
-    uint64_t                        fGstAmd64MbzPteMask;
-    /** Mask containing the MBZ PDE bits. */
-    uint64_t                        fGstAmd64MbzPdeMask;
-    /** Mask containing the MBZ big page PDE bits. */
-    uint64_t                        fGstAmd64MbzBigPdeMask;
-    /** Mask containing the MBZ PDPE bits. */
-    uint64_t                        fGstAmd64MbzPdpeMask;
-    /** Mask containing the MBZ big page PDPE bits. */
-    uint64_t                        fGstAmd64MbzBigPdpeMask;
-    /** Mask containing the MBZ PML4E bits. */
-    uint64_t                        fGstAmd64MbzPml4eMask;
-    /** Mask containing the PDPE bits that we shadow. */
-    uint64_t                        fGstAmd64ShadowedPdpeMask;
-    /** Mask containing the PML4E bits that we shadow. */
-    uint64_t                        fGstAmd64ShadowedPml4eMask;
-    /** @} */
-
-    /** @name PAE and AMD64 Guest Paging.
-     * @{ */
-    /** Mask containing the PTE bits that we shadow. */
-    uint64_t                        fGst64ShadowedPteMask;
-    /** Mask containing the PDE bits that we shadow. */
-    uint64_t                        fGst64ShadowedPdeMask;
-    /** Mask containing the big page PDE bits that we shadow in the PDE. */
-    uint64_t                        fGst64ShadowedBigPdeMask;
-    /** Mask containing the big page PDE bits that we shadow in the PTE. */
-    uint64_t                        fGst64ShadowedBigPde4PteMask;
-    /** @}  */
-
-    /** Pointer to the page of the current active CR3 - R3 Ptr. */
-    R3PTRTYPE(PPGMPOOLPAGE)         pShwPageCR3R3;
-    /** Pointer to the page of the current active CR3 - R0 Ptr. */
-    R0PTRTYPE(PPGMPOOLPAGE)         pShwPageCR3R0;
-    /** Pointer to the page of the current active CR3 - RC Ptr. */
-    RCPTRTYPE(PPGMPOOLPAGE)         pShwPageCR3RC;
-    /* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
-    uint32_t                        iShwUser;
-    /* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
-    uint32_t                        iShwUserTable;
-# if HC_ARCH_BITS == 64
-    RTRCPTR                         alignment6; /**< structure size alignment. */
-# endif
-    /** @} */
-
-    /** @name Function pointers for Shadow paging.
-     * @{
-     */
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwExit,(PVMCPU pVCpu));
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
-    DECLR3CALLBACKMEMBER(int,       pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
-
-    DECLRCCALLBACKMEMBER(int,       pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
-    DECLRCCALLBACKMEMBER(int,       pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
-
-    DECLR0CALLBACKMEMBER(int,       pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
-    DECLR0CALLBACKMEMBER(int,       pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
-
-    /** @} */
-
-    /** @name Function pointers for Guest paging.
-     * @{
-     */
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstExit,(PVMCPU pVCpu));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
-    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
-    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
-    DECLRCCALLBACKMEMBER(int,       pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
-    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
-#if HC_ARCH_BITS == 64
-    RTRCPTR                         alignment3; /**< structure size alignment. */
-#endif
-
-    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
-    DECLR0CALLBACKMEMBER(int,       pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
-    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
-    /** @} */
-
-    /** @name Function pointers for Both Shadow and Guest paging.
-     * @{
-     */
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
-    /*                           no pfnR3BthTrap0eHandler */
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
-    DECLR3CALLBACKMEMBER(unsigned,  pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
-    DECLR3CALLBACKMEMBER(int,       pfnR3BthUnmapCR3,(PVMCPU pVCpu));
-
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
-    DECLR0CALLBACKMEMBER(unsigned,  pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
-    DECLR0CALLBACKMEMBER(int,       pfnR0BthUnmapCR3,(PVMCPU pVCpu));
-
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
-    DECLRCCALLBACKMEMBER(unsigned,  pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
-    DECLRCCALLBACKMEMBER(int,       pfnRCBthUnmapCR3,(PVMCPU pVCpu));
-#if 0
-    RTRCPTR                         alignment2; /**< structure size alignment. */
-#endif
-    /** @} */
-
-    /** For saving stack space, the disassembler state is allocated here instead of
-     * on the stack.
-     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
-    union
-    {
-        /** The disassembler scratch space. */
-        DISCPUSTATE                 DisState;
-        /** Padding. */
-        uint8_t                     abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
-    };
-
-    /** Count the number of pgm pool access handler calls. */
-    uint64_t                        cPoolAccessHandler;
-
-    /** @name Release Statistics
-     * @{ */
-    /** The number of times the guest has switched mode since last reset or statistics reset. */
-    STAMCOUNTER                     cGuestModeChanges;
-    /** @} */
-
-#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap.  */
-    /** @name Statistics
-     * @{ */
-    /** RC: Pointer to the statistics. */
-    RCPTRTYPE(PGMCPUSTATS *)        pStatsRC;
-    /** RC: Which statistic this \#PF should be attributed to. */
-    RCPTRTYPE(PSTAMPROFILE)         pStatTrap0eAttributionRC;
-    /** R0: Pointer to the statistics. */
-    R0PTRTYPE(PGMCPUSTATS *)        pStatsR0;
-    /** R0: Which statistic this \#PF should be attributed to. */
-    R0PTRTYPE(PSTAMPROFILE)         pStatTrap0eAttributionR0;
-    /** R3: Pointer to the statistics. */
-    R3PTRTYPE(PGMCPUSTATS *)        pStatsR3;
-    /** Alignment padding. */
-    RTR3PTR                         pPaddingR3;
-    /** @}  */
-#endif /* VBOX_WITH_STATISTICS */
-} PGMCPU;
-/** Pointer to the per-cpu PGM data. */
-typedef PGMCPU *PPGMCPU;
-
-
-/** @name PGM::fSyncFlags Flags
- * @{
- */
-/** Updates the virtual access handler state bit in PGMPAGE. */
-#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL        RT_BIT(0)
-/** Always sync CR3. */
-#define PGM_SYNC_ALWAYS                         RT_BIT(1)
-/** Check monitoring on next CR3 (re)load and invalidate page.
- * @todo This is obsolete now. Remove after 2.2.0 is branched off. */
-#define PGM_SYNC_MONITOR_CR3                    RT_BIT(2)
-/** Check guest mapping in SyncCR3. */
-#define PGM_SYNC_MAP_CR3                        RT_BIT(3)
-/** Clear the page pool (a light weight flush). */
-#define PGM_SYNC_CLEAR_PGM_POOL_BIT             8
-#define PGM_SYNC_CLEAR_PGM_POOL                 RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT)
-/** @} */
-
-
-RT_C_DECLS_BEGIN
-
-int             pgmLock(PVM pVM);
-void            pgmUnlock(PVM pVM);
-
-int             pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb);
-int             pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
-int             pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
-PPGMMAPPING     pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
-int             pgmMapResolveConflicts(PVM pVM);
-DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-
-void            pgmR3HandlerPhysicalUpdateAll(PVM pVM);
-bool            pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
-void            pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, bool fDoAccounting);
-int             pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER *ppVirt, unsigned *piPage);
-DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
-#if defined(VBOX_STRICT) || defined(LOG_ENABLED)
-void            pgmHandlerVirtualDumpPhysPages(PVM pVM);
-#else
-# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
-#endif
-DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-int             pgmR3InitSavedState(PVM pVM, uint64_t cbRam);
-
-int             pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
-int             pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys);
-int             pgmPhysIsValidLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage);
-int             pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
-int             pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
-void            pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage);
-int             pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
-int             pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
-int             pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
-int             pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv);
-int             pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
-int             pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
-int             pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
-VMMDECL(int)    pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
-VMMDECL(int)    pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
-int             pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys);
-
-#ifdef IN_RING3
-void            pgmR3PhysRelinkRamRanges(PVM pVM);
-int             pgmR3PhysRamPreAllocate(PVM pVM);
-int             pgmR3PhysRamReset(PVM pVM);
-int             pgmR3PhysRomReset(PVM pVM);
-int             pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
-int             pgmR3PhysRamTerm(PVM pVM);
-void            pgmR3PhysRomTerm(PVM pVM);
-
-int             pgmR3PoolInit(PVM pVM);
-void            pgmR3PoolRelocate(PVM pVM);
-void            pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu);
-void            pgmR3PoolReset(PVM pVM);
-void            pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb);
-DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl);
-void            pgmR3PoolWriteProtectPages(PVM pVM);
-
-#endif /* IN_RING3 */
-#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
-int             pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
-int             pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
-# ifdef LOG_ENABLED
-void            pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL);
-# else
-void            pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint);
-# endif
-#endif
-int             pgmPoolAllocEx(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser,
-                               uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage);
-
-DECLINLINE(int) pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable,
-                             PPPGMPOOLPAGE ppPage)
-{
-    return pgmPoolAllocEx(pVM, GCPhys, enmKind, PGMPOOLACCESS_DONTCARE, iUser, iUserTable, false, ppPage);
-}
-
-void            pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
-void            pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
-int             pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush = true /* DO NOT USE false UNLESS YOU KNOWN WHAT YOU'RE DOING!! */);
-void            pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys);
-PPGMPOOLPAGE    pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
-PPGMPOOLPAGE    pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys);
-int             pgmPoolSyncCR3(PVMCPU pVCpu);
-bool            pgmPoolIsDirtyPage(PVM pVM, RTGCPHYS GCPhys);
-void            pgmPoolInvalidateDirtyPage(PVM pVM, RTGCPHYS GCPhysPT);
-int             pgmPoolTrackUpdateGCPhys(PVM pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs);
-void            pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte);
-uint16_t        pgmPoolTrackPhysExtAddref(PVM pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte);
-void            pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte);
-void            pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, unsigned cbWrite);
-int             pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
-void            pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
-
-void            pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage);
-void            pgmPoolResetDirtyPages(PVM pVM);
-void            pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage);
-
-int             pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
-int             pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
-
-void            pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
-void            pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
-int             pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
-int             pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
-
-int             pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
-int             pgmShwSyncNestedPageLocked(PVMCPU pVCpu, RTGCPHYS GCPhysFault, uint32_t cPages, PGMMODE enmShwPagingMode);
-
-int             pgmGstLazyMap32BitPD(PVMCPU pVCpu, PX86PD *ppPd);
-int             pgmGstLazyMapPaePDPT(PVMCPU pVCpu, PX86PDPT *ppPdpt);
-int             pgmGstLazyMapPaePD(PVMCPU pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd);
-int             pgmGstLazyMapPml4(PVMCPU pVCpu, PX86PML4 *ppPml4);
-
-# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
-DECLCALLBACK(int)  pgmR3CmdCheckDuplicatePages(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-DECLCALLBACK(int)  pgmR3CmdShowSharedModules(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-# endif
-
-RT_C_DECLS_END
-
-/** @} */
-
-#endif
-
Index: unk/src/VBox/VMM/PGMMap.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGMMap.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1469 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager, Guest Context Mappings.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include "PGMInline.h"
-
-#include <VBox/log.h>
-#include <VBox/err.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static void pgmR3MapClearPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iOldPDE);
-static void pgmR3MapSetPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
-static int  pgmR3MapIntermediateCheckOne(PVM pVM, uintptr_t uAddress, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault);
-static void pgmR3MapIntermediateDoOne(PVM pVM, uintptr_t uAddress, RTHCPHYS HCPhys, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault);
-
-
-/**
- * Creates a page table based mapping in GC.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   GCPtr           Virtual Address. (Page table aligned!)
- * @param   cb              Size of the range. Must be a 4MB aligned!
- * @param   fFlags          PGMR3MAPPT_FLAGS_UNMAPPABLE or 0.
- * @param   pfnRelocate     Relocation callback function.
- * @param   pvUser          User argument to the callback.
- * @param   pszDesc         Pointer to description string. This must not be freed.
- */
-VMMR3DECL(int) PGMR3MapPT(PVM pVM, RTGCPTR GCPtr, uint32_t cb, uint32_t fFlags, PFNPGMRELOCATE pfnRelocate, void *pvUser, const char *pszDesc)
-{
-    LogFlow(("PGMR3MapPT: GCPtr=%#x cb=%d fFlags=%#x pfnRelocate=%p pvUser=%p pszDesc=%s\n", GCPtr, cb, fFlags, pfnRelocate, pvUser, pszDesc));
-    AssertMsg(pVM->pgm.s.pInterPD, ("Paging isn't initialized, init order problems!\n"));
-
-    /*
-     * Validate input.
-     */
-    Assert(!fFlags || fFlags == PGMR3MAPPT_FLAGS_UNMAPPABLE);
-    if (cb < _2M || cb > 64 * _1M)
-    {
-        AssertMsgFailed(("Serious? cb=%d\n", cb));
-        return VERR_INVALID_PARAMETER;
-    }
-    cb = RT_ALIGN_32(cb, _4M);
-    RTGCPTR GCPtrLast = GCPtr + cb - 1;
-
-    AssertMsgReturn(GCPtrLast >= GCPtr, ("Range wraps! GCPtr=%x GCPtrLast=%x\n", GCPtr, GCPtrLast),
-                    VERR_INVALID_PARAMETER);
-    AssertMsgReturn(!pVM->pgm.s.fMappingsFixed, ("Mappings are fixed! It's not possible to add new mappings at this time!\n"),
-                    VERR_PGM_MAPPINGS_FIXED);
-    AssertPtrReturn(pfnRelocate, VERR_INVALID_PARAMETER);
-
-    /*
-     * Find list location.
-     */
-    PPGMMAPPING pPrev = NULL;
-    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
-    while (pCur)
-    {
-        if (pCur->GCPtrLast >= GCPtr && pCur->GCPtr <= GCPtrLast)
-        {
-            AssertMsgFailed(("Address is already in use by %s. req %#x-%#x take %#x-%#x\n",
-                             pCur->pszDesc, GCPtr, GCPtrLast, pCur->GCPtr, pCur->GCPtrLast));
-            LogRel(("VERR_PGM_MAPPING_CONFLICT: Address is already in use by %s. req %#x-%#x take %#x-%#x\n",
-                    pCur->pszDesc, GCPtr, GCPtrLast, pCur->GCPtr, pCur->GCPtrLast));
-            return VERR_PGM_MAPPING_CONFLICT;
-        }
-        if (pCur->GCPtr > GCPtr)
-            break;
-        pPrev = pCur;
-        pCur = pCur->pNextR3;
-    }
-
-    /*
-     * Check for conflicts with intermediate mappings.
-     */
-    const unsigned iPageDir = GCPtr >> X86_PD_SHIFT;
-    const unsigned cPTs     = cb >> X86_PD_SHIFT;
-    if (pVM->pgm.s.fFinalizedMappings)
-    {
-        for (unsigned i = 0; i < cPTs; i++)
-            if (pVM->pgm.s.pInterPD->a[iPageDir + i].n.u1Present)
-            {
-                AssertMsgFailed(("Address %#x is already in use by an intermediate mapping.\n", GCPtr + (i << PAGE_SHIFT)));
-                LogRel(("VERR_PGM_MAPPING_CONFLICT: Address %#x is already in use by an intermediate mapping.\n", GCPtr + (i << PAGE_SHIFT)));
-                return VERR_PGM_MAPPING_CONFLICT;
-            }
-        /** @todo AMD64: add check in PAE structures too, so we can remove all the 32-Bit paging stuff there. */
-    }
-
-    /*
-     * Allocate and initialize the new list node.
-     */
-    PPGMMAPPING pNew;
-    int rc;
-    if (fFlags & PGMR3MAPPT_FLAGS_UNMAPPABLE)
-        rc = MMHyperAlloc(           pVM, RT_OFFSETOF(PGMMAPPING, aPTs[cPTs]), 0, MM_TAG_PGM_MAPPINGS, (void **)&pNew);
-    else
-        rc = MMR3HyperAllocOnceNoRel(pVM, RT_OFFSETOF(PGMMAPPING, aPTs[cPTs]), 0, MM_TAG_PGM_MAPPINGS, (void **)&pNew);
-    if (RT_FAILURE(rc))
-        return rc;
-    pNew->GCPtr         = GCPtr;
-    pNew->GCPtrLast     = GCPtrLast;
-    pNew->cb            = cb;
-    pNew->pfnRelocate   = pfnRelocate;
-    pNew->pvUser        = pvUser;
-    pNew->pszDesc       = pszDesc;
-    pNew->cPTs          = cPTs;
-
-    /*
-     * Allocate page tables and insert them into the page directories.
-     * (One 32-bit PT and two PAE PTs.)
-     */
-    uint8_t *pbPTs;
-    if (fFlags & PGMR3MAPPT_FLAGS_UNMAPPABLE)
-        rc = MMHyperAlloc(           pVM, PAGE_SIZE * 3 * cPTs, PAGE_SIZE, MM_TAG_PGM_MAPPINGS, (void **)&pbPTs);
-    else
-        rc = MMR3HyperAllocOnceNoRel(pVM, PAGE_SIZE * 3 * cPTs, PAGE_SIZE, MM_TAG_PGM_MAPPINGS, (void **)&pbPTs);
-    if (RT_FAILURE(rc))
-    {
-        MMHyperFree(pVM, pNew);
-        return VERR_NO_MEMORY;
-    }
-
-    /*
-     * Init the page tables and insert them into the page directories.
-     */
-    Log4(("PGMR3MapPT: GCPtr=%RGv cPTs=%u pbPTs=%p\n", GCPtr, cPTs, pbPTs));
-    for (unsigned i = 0; i < cPTs; i++)
-    {
-        /*
-         * 32-bit.
-         */
-        pNew->aPTs[i].pPTR3    = (PX86PT)pbPTs;
-        pNew->aPTs[i].pPTRC    = MMHyperR3ToRC(pVM, pNew->aPTs[i].pPTR3);
-        pNew->aPTs[i].pPTR0    = MMHyperR3ToR0(pVM, pNew->aPTs[i].pPTR3);
-        pNew->aPTs[i].HCPhysPT = MMR3HyperHCVirt2HCPhys(pVM, pNew->aPTs[i].pPTR3);
-        pbPTs += PAGE_SIZE;
-        Log4(("PGMR3MapPT: i=%d: pPTR3=%RHv pPTRC=%RRv pPRTR0=%RHv HCPhysPT=%RHp\n",
-              i, pNew->aPTs[i].pPTR3, pNew->aPTs[i].pPTRC, pNew->aPTs[i].pPTR0, pNew->aPTs[i].HCPhysPT));
-
-        /*
-         * PAE.
-         */
-        pNew->aPTs[i].HCPhysPaePT0 = MMR3HyperHCVirt2HCPhys(pVM, pbPTs);
-        pNew->aPTs[i].HCPhysPaePT1 = MMR3HyperHCVirt2HCPhys(pVM, pbPTs + PAGE_SIZE);
-        pNew->aPTs[i].paPaePTsR3 = (PPGMSHWPTPAE)pbPTs;
-        pNew->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pbPTs);
-        pNew->aPTs[i].paPaePTsR0 = MMHyperR3ToR0(pVM, pbPTs);
-        pbPTs += PAGE_SIZE * 2;
-        Log4(("PGMR3MapPT: i=%d: paPaePTsR#=%RHv paPaePTsRC=%RRv paPaePTsR#=%RHv HCPhysPaePT0=%RHp HCPhysPaePT1=%RHp\n",
-              i, pNew->aPTs[i].paPaePTsR3, pNew->aPTs[i].paPaePTsRC, pNew->aPTs[i].paPaePTsR0, pNew->aPTs[i].HCPhysPaePT0, pNew->aPTs[i].HCPhysPaePT1));
-    }
-    if (pVM->pgm.s.fFinalizedMappings)
-        pgmR3MapSetPDEs(pVM, pNew, iPageDir);
-    /* else PGMR3FinalizeMappings() */
-
-    /*
-     * Insert the new mapping.
-     */
-    pNew->pNextR3 = pCur;
-    pNew->pNextRC = pCur ? MMHyperR3ToRC(pVM, pCur) : NIL_RTRCPTR;
-    pNew->pNextR0 = pCur ? MMHyperR3ToR0(pVM, pCur) : NIL_RTR0PTR;
-    if (pPrev)
-    {
-        pPrev->pNextR3 = pNew;
-        pPrev->pNextRC = MMHyperR3ToRC(pVM, pNew);
-        pPrev->pNextR0 = MMHyperR3ToR0(pVM, pNew);
-    }
-    else
-    {
-        pVM->pgm.s.pMappingsR3 = pNew;
-        pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pNew);
-        pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pNew);
-    }
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Removes a page table based mapping.
- *
- * @returns VBox status code.
- * @param   pVM     VM Handle.
- * @param   GCPtr   Virtual Address. (Page table aligned!)
- *
- * @remarks Don't call this without passing PGMR3MAPPT_FLAGS_UNMAPPABLE to
- *          PGMR3MapPT or you'll burn in the heap.
- */
-VMMR3DECL(int)  PGMR3UnmapPT(PVM pVM, RTGCPTR GCPtr)
-{
-    LogFlow(("PGMR3UnmapPT: GCPtr=%#x\n", GCPtr));
-    AssertReturn(pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
-
-    /*
-     * Find it.
-     */
-    PPGMMAPPING pPrev = NULL;
-    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
-    while (pCur)
-    {
-        if (pCur->GCPtr == GCPtr)
-        {
-            /*
-             * Unlink it.
-             */
-            if (pPrev)
-            {
-                pPrev->pNextR3 = pCur->pNextR3;
-                pPrev->pNextRC = pCur->pNextRC;
-                pPrev->pNextR0 = pCur->pNextR0;
-            }
-            else
-            {
-                pVM->pgm.s.pMappingsR3 = pCur->pNextR3;
-                pVM->pgm.s.pMappingsRC = pCur->pNextRC;
-                pVM->pgm.s.pMappingsR0 = pCur->pNextR0;
-            }
-
-            /*
-             * Free the page table memory, clear page directory entries
-             * and free the page tables and node memory.
-             */
-            MMHyperFree(pVM, pCur->aPTs[0].pPTR3);
-            if (pCur->GCPtr != NIL_RTGCPTR)
-                pgmR3MapClearPDEs(pVM, pCur, pCur->GCPtr >> X86_PD_SHIFT);
-            MMHyperFree(pVM, pCur);
-
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-            {
-                PVMCPU pVCpu = &pVM->aCpus[i];
-                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-            }
-            return VINF_SUCCESS;
-        }
-
-        /* done? */
-        if (pCur->GCPtr > GCPtr)
-            break;
-
-        /* next */
-        pPrev = pCur;
-        pCur = pCur->pNextR3;
-    }
-
-    AssertMsgFailed(("No mapping for %#x found!\n", GCPtr));
-    return VERR_INVALID_PARAMETER;
-}
-
-
-/**
- * Checks whether a range of PDEs in the intermediate
- * memory context are unused.
- *
- * We're talking 32-bit PDEs here.
- *
- * @returns true/false.
- * @param   pVM         Pointer to the shared VM structure.
- * @param   iPD         The first PDE in the range.
- * @param   cPTs        The number of PDEs in the range.
- */
-DECLINLINE(bool) pgmR3AreIntermediatePDEsUnused(PVM pVM, unsigned iPD, unsigned cPTs)
-{
-    if (pVM->pgm.s.pInterPD->a[iPD].n.u1Present)
-        return false;
-    while (cPTs > 1)
-    {
-        iPD++;
-        if (pVM->pgm.s.pInterPD->a[iPD].n.u1Present)
-            return false;
-        cPTs--;
-    }
-    return true;
-}
-
-
-/**
- * Unlinks the mapping.
- *
- * The mapping *must* be in the list.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pMapping        The mapping to unlink.
- */
-static void pgmR3MapUnlink(PVM pVM, PPGMMAPPING pMapping)
-{
-    PPGMMAPPING pAfterThis = pVM->pgm.s.pMappingsR3;
-    if (pAfterThis == pMapping)
-    {
-        /* head */
-        pVM->pgm.s.pMappingsR3 = pMapping->pNextR3;
-        pVM->pgm.s.pMappingsRC = pMapping->pNextRC;
-        pVM->pgm.s.pMappingsR0 = pMapping->pNextR0;
-    }
-    else
-    {
-        /* in the list */
-        while (pAfterThis->pNextR3 != pMapping)
-        {
-            pAfterThis = pAfterThis->pNextR3;
-            AssertReleaseReturnVoid(pAfterThis);
-        }
-
-        pAfterThis->pNextR3 = pMapping->pNextR3;
-        pAfterThis->pNextRC = pMapping->pNextRC;
-        pAfterThis->pNextR0 = pMapping->pNextR0;
-    }
-}
-
-
-/**
- * Links the mapping.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pMapping        The mapping to linked.
- */
-static void pgmR3MapLink(PVM pVM, PPGMMAPPING pMapping)
-{
-    /*
-     * Find the list location (it's sorted by GCPhys) and link it in.
-     */
-    if (    !pVM->pgm.s.pMappingsR3
-        ||  pVM->pgm.s.pMappingsR3->GCPtr > pMapping->GCPtr)
-    {
-        /* head */
-        pMapping->pNextR3 = pVM->pgm.s.pMappingsR3;
-        pMapping->pNextRC = pVM->pgm.s.pMappingsRC;
-        pMapping->pNextR0 = pVM->pgm.s.pMappingsR0;
-        pVM->pgm.s.pMappingsR3 = pMapping;
-        pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pMapping);
-        pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pMapping);
-    }
-    else
-    {
-        /* in the list */
-        PPGMMAPPING pAfterThis  = pVM->pgm.s.pMappingsR3;
-        PPGMMAPPING pBeforeThis = pAfterThis->pNextR3;
-        while (pBeforeThis && pBeforeThis->GCPtr <= pMapping->GCPtr)
-        {
-            pAfterThis = pBeforeThis;
-            pBeforeThis = pBeforeThis->pNextR3;
-        }
-
-        pMapping->pNextR3 = pAfterThis->pNextR3;
-        pMapping->pNextRC = pAfterThis->pNextRC;
-        pMapping->pNextR0 = pAfterThis->pNextR0;
-        pAfterThis->pNextR3 = pMapping;
-        pAfterThis->pNextRC = MMHyperR3ToRC(pVM, pMapping);
-        pAfterThis->pNextR0 = MMHyperR3ToR0(pVM, pMapping);
-    }
-}
-
-
-/**
- * Finalizes the intermediate context.
- *
- * This is called at the end of the ring-3 init and will construct the
- * intermediate paging structures, relocating all the mappings in the process.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- * @thread  EMT(0)
- */
-VMMR3DECL(int) PGMR3FinalizeMappings(PVM pVM)
-{
-    AssertReturn(!pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
-    pVM->pgm.s.fFinalizedMappings = true;
-
-    /*
-     * Loop until all mappings have been finalized.
-     */
-#if 0
-    unsigned    iPDNext = UINT32_C(0xc0000000) >> X86_PD_SHIFT; /* makes CSAM/PATM freak out booting linux. :-/ */
-#elif 0
-    unsigned    iPDNext = MM_HYPER_AREA_ADDRESS >> X86_PD_SHIFT;
-#else
-    unsigned    iPDNext = 1 << X86_PD_SHIFT; /* no hint, map them from the top. */
-#endif
-
-    PPGMMAPPING pCur;
-    do
-    {
-        pCur = pVM->pgm.s.pMappingsR3;
-        while (pCur)
-        {
-            if (!pCur->fFinalized)
-            {
-                /*
-                 * Find a suitable location.
-                 */
-                RTGCPTR const   GCPtrOld = pCur->GCPtr;
-                const unsigned  cPTs     = pCur->cPTs;
-                unsigned        iPDNew   = iPDNext;
-                if (    iPDNew + cPTs >= X86_PG_ENTRIES /* exclude the last PD */
-                    ||  !pgmR3AreIntermediatePDEsUnused(pVM, iPDNew, cPTs)
-                    ||  !pCur->pfnRelocate(pVM, GCPtrOld, (RTGCPTR)iPDNew << X86_PD_SHIFT, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
-                {
-                    /* No luck, just scan down from 4GB-4MB, giving up at 4MB. */
-                    iPDNew = X86_PG_ENTRIES - cPTs - 1;
-                    while (     iPDNew > 0
-                           &&   (   !pgmR3AreIntermediatePDEsUnused(pVM, iPDNew, cPTs)
-                                 || !pCur->pfnRelocate(pVM, GCPtrOld, (RTGCPTR)iPDNew << X86_PD_SHIFT, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
-                           )
-                        iPDNew--;
-                    AssertLogRelReturn(iPDNew != 0, VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
-                }
-
-                /*
-                 * Relocate it (something akin to pgmR3MapRelocate).
-                 */
-                pgmR3MapSetPDEs(pVM, pCur, iPDNew);
-
-                /* unlink the mapping, update the entry and relink it. */
-                pgmR3MapUnlink(pVM, pCur);
-
-                RTGCPTR const GCPtrNew = (RTGCPTR)iPDNew << X86_PD_SHIFT;
-                pCur->GCPtr      = GCPtrNew;
-                pCur->GCPtrLast  = GCPtrNew + pCur->cb - 1;
-                pCur->fFinalized = true;
-
-                pgmR3MapLink(pVM, pCur);
-
-                /* Finally work the callback. */
-                pCur->pfnRelocate(pVM, GCPtrOld, GCPtrNew, PGMRELOCATECALL_RELOCATE, pCur->pvUser);
-
-                /*
-                 * The list order might have changed, start from the beginning again.
-                 */
-                iPDNext = iPDNew + cPTs;
-                break;
-            }
-
-            /* next */
-            pCur = pCur->pNextR3;
-        }
-    } while (pCur);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets the size of the current guest mappings if they were to be
- * put next to one another.
- *
- * @returns VBox status code.
- * @param   pVM     The VM.
- * @param   pcb     Where to store the size.
- */
-VMMR3DECL(int) PGMR3MappingsSize(PVM pVM, uint32_t *pcb)
-{
-    RTGCPTR cb = 0;
-    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
-        cb += pCur->cb;
-
-    *pcb = cb;
-    AssertReturn(*pcb == cb, VERR_NUMBER_TOO_BIG);
-    Log(("PGMR3MappingsSize: return %d (%#x) bytes\n", cb, cb));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Fixates the guest context mappings in a range reserved from the Guest OS.
- *
- * @returns VBox status code.
- * @param   pVM         The VM.
- * @param   GCPtrBase   The address of the reserved range of guest memory.
- * @param   cb          The size of the range starting at GCPtrBase.
- */
-VMMR3DECL(int) PGMR3MappingsFix(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb)
-{
-    Log(("PGMR3MappingsFix: GCPtrBase=%RGv cb=%#x (fMappingsFixed=%RTbool fMappingsDisabled=%RTbool)\n",
-         GCPtrBase, cb, pVM->pgm.s.fMappingsFixed, pVM->pgm.s.fMappingsDisabled));
-
-    /*
-     * Ignore the additions mapping fix call if disabled.
-     */
-    if (!pgmMapAreMappingsEnabled(&pVM->pgm.s))
-    {
-        Assert(HWACCMIsEnabled(pVM));
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Only applies to VCPU 0 as we don't support SMP guests with raw mode.
-     */
-    Assert(pVM->cCpus == 1);
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    /*
-     * Before we do anything we'll do a forced PD sync to try make sure any
-     * pending relocations because of these mappings have been resolved.
-     */
-    PGMSyncCR3(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu), true);
-
-    return pgmR3MappingsFixInternal(pVM, GCPtrBase, cb);
-}
-
-
-/**
- * Internal worker for PGMR3MappingsFix and pgmR3Load.
- *
- * (This does not perform a SyncCR3 before the fixation like PGMR3MappingsFix.)
- *
- * @returns VBox status code.
- * @param   pVM         The VM.
- * @param   GCPtrBase   The address of the reserved range of guest memory.
- * @param   cb          The size of the range starting at GCPtrBase.
- */
-int pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb)
-{
-    /*
-     * Check input arguments and pre-conditions.
-     */
-    AssertMsgReturn(!(GCPtrBase & X86_PAGE_4M_OFFSET_MASK), ("GCPtrBase (%#x) has to be aligned on a 4MB address!\n", GCPtrBase),
-                    VERR_INVALID_PARAMETER);
-    AssertMsgReturn(cb && !(cb & X86_PAGE_4M_OFFSET_MASK), ("cb (%#x) is 0 or not aligned on a 4MB address!\n", cb),
-                    VERR_INVALID_PARAMETER);
-    AssertReturn(pgmMapAreMappingsEnabled(&pVM->pgm.s), VERR_INTERNAL_ERROR_3);
-    AssertReturn(pVM->cCpus == 1, VERR_INTERNAL_ERROR_4);
-
-    /*
-     * Check that it's not conflicting with a core code mapping in the intermediate page table.
-     */
-    unsigned    iPDNew = GCPtrBase >> X86_PD_SHIFT;
-    unsigned    i = cb >> X86_PD_SHIFT;
-    while (i-- > 0)
-    {
-        if (pVM->pgm.s.pInterPD->a[iPDNew + i].n.u1Present)
-        {
-            /* Check that it's not one or our mappings. */
-            PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
-            while (pCur)
-            {
-                if (iPDNew + i - (pCur->GCPtr >> X86_PD_SHIFT) < (pCur->cb >> X86_PD_SHIFT))
-                    break;
-                pCur = pCur->pNextR3;
-            }
-            if (!pCur)
-            {
-                LogRel(("PGMR3MappingsFix: Conflicts with intermediate PDE %#x (GCPtrBase=%RGv cb=%#zx). The guest should retry.\n",
-                        iPDNew + i, GCPtrBase, cb));
-                return VERR_PGM_MAPPINGS_FIX_CONFLICT;
-            }
-        }
-    }
-
-    /*
-     * In PAE / PAE mode, make sure we don't cross page directories.
-     */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-    if (    (   pVCpu->pgm.s.enmGuestMode  == PGMMODE_PAE
-             || pVCpu->pgm.s.enmGuestMode  == PGMMODE_PAE_NX)
-        &&  (   pVCpu->pgm.s.enmShadowMode == PGMMODE_PAE
-             || pVCpu->pgm.s.enmShadowMode == PGMMODE_PAE_NX))
-    {
-        unsigned iPdptBase = GCPtrBase >> X86_PDPT_SHIFT;
-        unsigned iPdptLast = (GCPtrBase + cb - 1) >> X86_PDPT_SHIFT;
-        if (iPdptBase != iPdptLast)
-        {
-            LogRel(("PGMR3MappingsFix: Crosses PD boundary; iPdptBase=%#x iPdptLast=%#x (GCPtrBase=%RGv cb=%#zx). The guest should retry.\n",
-                    iPdptBase, iPdptLast, GCPtrBase, cb));
-            return VERR_PGM_MAPPINGS_FIX_CONFLICT;
-        }
-    }
-
-    /*
-     * Loop the mappings and check that they all agree on their new locations.
-     */
-    RTGCPTR     GCPtrCur = GCPtrBase;
-    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
-    while (pCur)
-    {
-        if (!pCur->pfnRelocate(pVM, pCur->GCPtr, GCPtrCur, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
-        {
-            AssertMsgFailed(("The suggested fixed address %#x was rejected by '%s'!\n", GCPtrCur, pCur->pszDesc));
-            return VERR_PGM_MAPPINGS_FIX_REJECTED;
-        }
-        /* next */
-        GCPtrCur += pCur->cb;
-        pCur = pCur->pNextR3;
-    }
-    if (GCPtrCur > GCPtrBase + cb)
-    {
-        AssertMsgFailed(("cb (%#x) is less than the required range %#x!\n", cb, GCPtrCur - GCPtrBase));
-        return VERR_PGM_MAPPINGS_FIX_TOO_SMALL;
-    }
-
-    /*
-     * Loop the table assigning the mappings to the passed in memory
-     * and call their relocator callback.
-     */
-    GCPtrCur = GCPtrBase;
-    pCur = pVM->pgm.s.pMappingsR3;
-    while (pCur)
-    {
-        RTGCPTR const GCPtrOld = pCur->GCPtr;
-
-        /*
-         * Relocate the page table(s).
-         */
-        if (pCur->GCPtr != NIL_RTGCPTR)
-            pgmR3MapClearPDEs(pVM, pCur, GCPtrOld >> X86_PD_SHIFT);
-        pgmR3MapSetPDEs(pVM, pCur, GCPtrCur >> X86_PD_SHIFT);
-
-        /*
-         * Update the entry.
-         */
-        pCur->GCPtr = GCPtrCur;
-        pCur->GCPtrLast = GCPtrCur + pCur->cb - 1;
-
-        /*
-         * Callback to execute the relocation.
-         */
-        pCur->pfnRelocate(pVM, GCPtrOld, GCPtrCur, PGMRELOCATECALL_RELOCATE, pCur->pvUser);
-
-        /*
-         * Advance.
-         */
-        GCPtrCur += pCur->cb;
-        pCur = pCur->pNextR3;
-    }
-
-    /*
-     * Mark the mappings as fixed at this new location and return.
-     */
-    pVM->pgm.s.fMappingsFixed           = true;
-    pVM->pgm.s.fMappingsFixedRestored   = false;
-    pVM->pgm.s.GCPtrMappingFixed        = GCPtrBase;
-    pVM->pgm.s.cbMappingFixed           = cb;
-
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
-        VMCPU_FF_SET(&pVM->aCpus[idCpu], VMCPU_FF_PGM_SYNC_CR3);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Interface for disabling the guest mappings when switching to HWACCM mode
- * during VM creation and VM reset.
- *
- * (This doesn't touch the intermediate table!)
- *
- * @returns VBox status code.
- * @param   pVM         The VM.
- */
-VMMR3DECL(int) PGMR3MappingsDisable(PVM pVM)
-{
-    AssertReturn(!pVM->pgm.s.fMappingsFixed,            VERR_INTERNAL_ERROR_4);
-    AssertReturn(!pVM->pgm.s.fMappingsFixedRestored,    VERR_INTERNAL_ERROR_4);
-    if (pVM->pgm.s.fMappingsDisabled)
-        return VINF_SUCCESS;
-
-    /*
-     * Deactivate (only applies to Virtual CPU #0).
-     */
-    if (pVM->aCpus[0].pgm.s.pShwPageCR3R3)
-    {
-        pgmLock(pVM);                           /* to avoid assertions */
-        int rc = pgmMapDeactivateCR3(pVM, pVM->aCpus[0].pgm.s.pShwPageCR3R3);
-        pgmUnlock(pVM);
-        AssertRCReturn(rc, rc);
-    }
-
-    /*
-     * Mark the mappings as disabled and trigger a CR3 re-sync.
-     */
-    pVM->pgm.s.fMappingsDisabled = true;
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
-        VMCPU_FF_SET(&pVM->aCpus[idCpu], VMCPU_FF_PGM_SYNC_CR3);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Unfixes the mappings.
- *
- * Unless PGMR3MappingsDisable is in effect, mapping conflict detection will be
- * enabled after this call.  If the mappings are fixed, a full CR3 resync will
- * take place afterwards.
- *
- * @returns VBox status code.
- * @param   pVM         The VM.
- */
-VMMR3DECL(int) PGMR3MappingsUnfix(PVM pVM)
-{
-    Log(("PGMR3MappingsUnfix: fMappingsFixed=%RTbool fMappingsDisabled=%RTbool\n", pVM->pgm.s.fMappingsFixed, pVM->pgm.s.fMappingsDisabled));
-    if (   pgmMapAreMappingsEnabled(&pVM->pgm.s)
-        && (    pVM->pgm.s.fMappingsFixed
-            ||  pVM->pgm.s.fMappingsFixedRestored)
-       )
-    {
-        bool const fResyncCR3 = pVM->pgm.s.fMappingsFixed;
-
-        pVM->pgm.s.fMappingsFixed           = false;
-        pVM->pgm.s.fMappingsFixedRestored   = false;
-        pVM->pgm.s.GCPtrMappingFixed        = 0;
-        pVM->pgm.s.cbMappingFixed           = 0;
-
-        if (fResyncCR3)
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-                VMCPU_FF_SET(&pVM->aCpus[i], VMCPU_FF_PGM_SYNC_CR3);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Checks if the mappings needs re-fixing after a restore.
- *
- * @returns true if they need, false if not.
- * @param   pVM                 The VM handle.
- */
-VMMR3DECL(bool) PGMR3MappingsNeedReFixing(PVM pVM)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
-    return pVM->pgm.s.fMappingsFixedRestored;
-}
-
-
-/**
- * Map pages into the intermediate context (switcher code).
- *
- * These pages are mapped at both the give virtual address and at the physical
- * address (for identity mapping).
- *
- * @returns VBox status code.
- * @param   pVM         The virtual machine.
- * @param   Addr        Intermediate context address of the mapping.
- * @param   HCPhys      Start of the range of physical pages. This must be entriely below 4GB!
- * @param   cbPages     Number of bytes to map.
- *
- * @remark  This API shall not be used to anything but mapping the switcher code.
- */
-VMMR3DECL(int) PGMR3MapIntermediate(PVM pVM, RTUINTPTR Addr, RTHCPHYS HCPhys, unsigned cbPages)
-{
-    LogFlow(("PGMR3MapIntermediate: Addr=%RTptr HCPhys=%RHp cbPages=%#x\n", Addr, HCPhys, cbPages));
-
-    /*
-     * Adjust input.
-     */
-    cbPages += (uint32_t)HCPhys & PAGE_OFFSET_MASK;
-    cbPages  = RT_ALIGN(cbPages, PAGE_SIZE);
-    HCPhys  &= X86_PTE_PAE_PG_MASK;
-    Addr    &= PAGE_BASE_MASK;
-    /* We only care about the first 4GB, because on AMD64 we'll be repeating them all over the address space. */
-    uint32_t uAddress = (uint32_t)Addr;
-
-    /*
-     * Assert input and state.
-     */
-    AssertMsg(pVM->pgm.s.offVM, ("Bad init order\n"));
-    AssertMsg(pVM->pgm.s.pInterPD, ("Bad init order, paging.\n"));
-    AssertMsg(cbPages <= (512 << PAGE_SHIFT), ("The mapping is too big %d bytes\n", cbPages));
-    AssertMsg(HCPhys < _4G && HCPhys + cbPages < _4G, ("Addr=%RTptr HCPhys=%RHp cbPages=%d\n", Addr, HCPhys, cbPages));
-    AssertReturn(!pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
-
-    /*
-     * Check for internal conflicts between the virtual address and the physical address.
-     * A 1:1 mapping is fine, but partial overlapping is a no-no.
-     */
-    if (    uAddress != HCPhys
-        &&  (   uAddress < HCPhys
-                ? HCPhys - uAddress < cbPages
-                : uAddress - HCPhys < cbPages
-            )
-       )
-        AssertLogRelMsgFailedReturn(("Addr=%RTptr HCPhys=%RHp cbPages=%d\n", Addr, HCPhys, cbPages),
-                                    VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
-
-    const unsigned cPages = cbPages >> PAGE_SHIFT;
-    int rc = pgmR3MapIntermediateCheckOne(pVM, uAddress, cPages, pVM->pgm.s.apInterPTs[0], pVM->pgm.s.apInterPaePTs[0]);
-    if (RT_FAILURE(rc))
-        return rc;
-    rc = pgmR3MapIntermediateCheckOne(pVM, (uintptr_t)HCPhys, cPages, pVM->pgm.s.apInterPTs[1], pVM->pgm.s.apInterPaePTs[1]);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Everythings fine, do the mapping.
-     */
-    pgmR3MapIntermediateDoOne(pVM, uAddress, HCPhys, cPages, pVM->pgm.s.apInterPTs[0], pVM->pgm.s.apInterPaePTs[0]);
-    pgmR3MapIntermediateDoOne(pVM, (uintptr_t)HCPhys, HCPhys, cPages, pVM->pgm.s.apInterPTs[1], pVM->pgm.s.apInterPaePTs[1]);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Validates that there are no conflicts for this mapping into the intermediate context.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   uAddress    Address of the mapping.
- * @param   cPages      Number of pages.
- * @param   pPTDefault      Pointer to the default page table for this mapping.
- * @param   pPTPaeDefault   Pointer to the default page table for this mapping.
- */
-static int pgmR3MapIntermediateCheckOne(PVM pVM, uintptr_t uAddress, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault)
-{
-    AssertMsg((uAddress >> X86_PD_SHIFT) + cPages <= 1024, ("64-bit fixme\n"));
-
-    /*
-     * Check that the ranges are available.
-     * (This code doesn't have to be fast.)
-     */
-    while (cPages > 0)
-    {
-        /*
-         * 32-Bit.
-         */
-        unsigned iPDE = (uAddress >> X86_PD_SHIFT) & X86_PD_MASK;
-        unsigned iPTE = (uAddress >> X86_PT_SHIFT) & X86_PT_MASK;
-        PX86PT pPT = pPTDefault;
-        if (pVM->pgm.s.pInterPD->a[iPDE].u)
-        {
-            RTHCPHYS HCPhysPT = pVM->pgm.s.pInterPD->a[iPDE].u & X86_PDE_PG_MASK;
-            if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]))
-                pPT = pVM->pgm.s.apInterPTs[0];
-            else if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]))
-                pPT = pVM->pgm.s.apInterPTs[1];
-            else
-            {
-                /** @todo this must be handled with a relocation of the conflicting mapping!
-                 * Which of course cannot be done because we're in the middle of the initialization. bad design! */
-                AssertLogRelMsgFailedReturn(("Conflict between core code and PGMR3Mapping(). uAddress=%RHv\n", uAddress),
-                                            VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
-            }
-        }
-        if (pPT->a[iPTE].u)
-            AssertLogRelMsgFailedReturn(("Conflict iPTE=%#x iPDE=%#x uAddress=%RHv pPT->a[iPTE].u=%RX32\n", iPTE, iPDE, uAddress, pPT->a[iPTE].u),
-                                        VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
-
-        /*
-         * PAE.
-         */
-        const unsigned iPDPE= (uAddress >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-        iPDE = (uAddress >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-        iPTE = (uAddress >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK;
-        Assert(iPDPE < 4);
-        Assert(pVM->pgm.s.apInterPaePDs[iPDPE]);
-        PX86PTPAE pPTPae = pPTPaeDefault;
-        if (pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u)
-        {
-            RTHCPHYS HCPhysPT = pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u & X86_PDE_PAE_PG_MASK;
-            if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]))
-                pPTPae = pVM->pgm.s.apInterPaePTs[0];
-            else if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]))
-                pPTPae = pVM->pgm.s.apInterPaePTs[1];
-            else
-            {
-                /** @todo this must be handled with a relocation of the conflicting mapping!
-                 * Which of course cannot be done because we're in the middle of the initialization. bad design! */
-                AssertLogRelMsgFailedReturn(("Conflict between core code and PGMR3Mapping(). uAddress=%RHv\n", uAddress),
-                                            VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
-            }
-        }
-        if (pPTPae->a[iPTE].u)
-            AssertLogRelMsgFailedReturn(("Conflict iPTE=%#x iPDE=%#x uAddress=%RHv pPTPae->a[iPTE].u=%#RX64\n", iPTE, iPDE, uAddress, pPTPae->a[iPTE].u),
-                                        VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
-
-        /* next */
-        uAddress += PAGE_SIZE;
-        cPages--;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-
-/**
- * Sets up the intermediate page tables for a verified mapping.
- *
- * @param   pVM             VM handle.
- * @param   uAddress        Address of the mapping.
- * @param   HCPhys          The physical address of the page range.
- * @param   cPages          Number of pages.
- * @param   pPTDefault      Pointer to the default page table for this mapping.
- * @param   pPTPaeDefault   Pointer to the default page table for this mapping.
- */
-static void pgmR3MapIntermediateDoOne(PVM pVM, uintptr_t uAddress, RTHCPHYS HCPhys, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault)
-{
-    while (cPages > 0)
-    {
-        /*
-         * 32-Bit.
-         */
-        unsigned iPDE = (uAddress >> X86_PD_SHIFT) & X86_PD_MASK;
-        unsigned iPTE = (uAddress >> X86_PT_SHIFT) & X86_PT_MASK;
-        PX86PT pPT;
-        if (pVM->pgm.s.pInterPD->a[iPDE].u)
-            pPT = (PX86PT)MMPagePhys2Page(pVM, pVM->pgm.s.pInterPD->a[iPDE].u & X86_PDE_PG_MASK);
-        else
-        {
-            pVM->pgm.s.pInterPD->a[iPDE].u = X86_PDE_P | X86_PDE_A | X86_PDE_RW
-                                           | (uint32_t)MMPage2Phys(pVM, pPTDefault);
-            pPT = pPTDefault;
-        }
-        pPT->a[iPTE].u = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D | (uint32_t)HCPhys;
-
-        /*
-         * PAE
-         */
-        const unsigned iPDPE= (uAddress >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
-        iPDE = (uAddress >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
-        iPTE = (uAddress >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK;
-        Assert(iPDPE < 4);
-        Assert(pVM->pgm.s.apInterPaePDs[iPDPE]);
-        PX86PTPAE pPTPae;
-        if (pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u)
-            pPTPae = (PX86PTPAE)MMPagePhys2Page(pVM, pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u & X86_PDE_PAE_PG_MASK);
-        else
-        {
-            pPTPae = pPTPaeDefault;
-            pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u = X86_PDE_P | X86_PDE_A | X86_PDE_RW
-                                                       | MMPage2Phys(pVM, pPTPaeDefault);
-        }
-        pPTPae->a[iPTE].u = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D | HCPhys;
-
-        /* next */
-        cPages--;
-        HCPhys += PAGE_SIZE;
-        uAddress += PAGE_SIZE;
-    }
-}
-
-
-/**
- * Clears all PDEs involved with the mapping in the shadow and intermediate page tables.
- *
- * @param   pVM         The VM handle.
- * @param   pMap        Pointer to the mapping in question.
- * @param   iOldPDE     The index of the 32-bit PDE corresponding to the base of the mapping.
- */
-static void pgmR3MapClearPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iOldPDE)
-{
-    unsigned i     = pMap->cPTs;
-    PVMCPU   pVCpu = VMMGetCpu(pVM);
-    pgmLock(pVM);                           /* to avoid assertions */
-
-    pgmMapClearShadowPDEs(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pMap, iOldPDE, false /*fDeactivateCR3*/);
-
-    iOldPDE += i;
-    while (i-- > 0)
-    {
-        iOldPDE--;
-
-        /*
-         * 32-bit.
-         */
-        pVM->pgm.s.pInterPD->a[iOldPDE].u        = 0;
-
-        /*
-         * PAE.
-         */
-        const unsigned iPD = iOldPDE / 256;         /* iOldPDE * 2 / 512; iOldPDE is in 4 MB pages */
-        unsigned iPDE = iOldPDE * 2 % 512;
-        pVM->pgm.s.apInterPaePDs[iPD]->a[iPDE].u = 0;
-        iPDE++;
-        AssertFatal(iPDE < 512);
-        pVM->pgm.s.apInterPaePDs[iPD]->a[iPDE].u = 0;
-    }
-
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Sets all PDEs involved with the mapping in the shadow and intermediate page tables.
- *
- * @param   pVM         The VM handle.
- * @param   pMap        Pointer to the mapping in question.
- * @param   iNewPDE     The index of the 32-bit PDE corresponding to the base of the mapping.
- */
-static void pgmR3MapSetPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE)
-{
-    PPGM   pPGM  = &pVM->pgm.s;
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    pgmLock(pVM);                           /* to avoid assertions */
-
-    Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s) || PGMGetGuestMode(pVCpu) <= PGMMODE_PAE_NX);
-
-    pgmMapSetShadowPDEs(pVM, pMap, iNewPDE);
-
-    /*
-     * Init the page tables and insert them into the page directories.
-     */
-    unsigned i = pMap->cPTs;
-    iNewPDE += i;
-    while (i-- > 0)
-    {
-        iNewPDE--;
-
-        /*
-         * 32-bit.
-         */
-        X86PDE Pde;
-        /* Default mapping page directory flags are read/write and supervisor; individual page attributes determine the final flags */
-        Pde.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | (uint32_t)pMap->aPTs[i].HCPhysPT;
-        pPGM->pInterPD->a[iNewPDE]        = Pde;
-        /*
-         * PAE.
-         */
-        const unsigned iPD = iNewPDE / 256;
-        unsigned iPDE = iNewPDE * 2 % 512;
-        X86PDEPAE PdePae0;
-        PdePae0.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT0;
-        pPGM->apInterPaePDs[iPD]->a[iPDE] = PdePae0;
-        iPDE++;
-        AssertFatal(iPDE < 512);
-        X86PDEPAE PdePae1;
-        PdePae1.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT1;
-        pPGM->apInterPaePDs[iPD]->a[iPDE] = PdePae1;
-    }
-
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Relocates a mapping to a new address.
- *
- * @param   pVM                 VM handle.
- * @param   pMapping            The mapping to relocate.
- * @param   GCPtrOldMapping     The address of the start of the old mapping.
- *                              NIL_RTGCPTR if not currently mapped.
- * @param   GCPtrNewMapping     The address of the start of the new mapping.
- */
-static void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping)
-{
-    Log(("PGM: Relocating %s from %RGv to %RGv\n", pMapping->pszDesc, GCPtrOldMapping, GCPtrNewMapping));
-    AssertMsg(GCPtrOldMapping == pMapping->GCPtr, ("%RGv vs %RGv\n", GCPtrOldMapping, pMapping->GCPtr));
-    AssertMsg((GCPtrOldMapping >> X86_PD_SHIFT) < X86_PG_ENTRIES, ("%RGv\n", GCPtrOldMapping));
-    AssertMsg((GCPtrNewMapping >> X86_PD_SHIFT) < X86_PG_ENTRIES, ("%RGv\n", GCPtrOldMapping));
-
-    /*
-     * Relocate the page table(s).
-     */
-    if (GCPtrOldMapping != NIL_RTGCPTR)
-        pgmR3MapClearPDEs(pVM, pMapping, GCPtrOldMapping >> X86_PD_SHIFT);
-    pgmR3MapSetPDEs(pVM, pMapping, GCPtrNewMapping >> X86_PD_SHIFT);
-
-    /*
-     * Update and resort the mapping list.
-     */
-
-    /* Find previous mapping for pMapping, put result into pPrevMap. */
-    PPGMMAPPING pPrevMap = NULL;
-    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
-    while (pCur && pCur != pMapping)
-    {
-        /* next */
-        pPrevMap = pCur;
-        pCur = pCur->pNextR3;
-    }
-    Assert(pCur);
-
-    /* Find mapping which >= than pMapping. */
-    RTGCPTR     GCPtrNew = GCPtrNewMapping;
-    PPGMMAPPING pPrev = NULL;
-    pCur = pVM->pgm.s.pMappingsR3;
-    while (pCur && pCur->GCPtr < GCPtrNew)
-    {
-        /* next */
-        pPrev = pCur;
-        pCur = pCur->pNextR3;
-    }
-
-    if (pCur != pMapping && pPrev != pMapping)
-    {
-        /*
-         * Unlink.
-         */
-        if (pPrevMap)
-        {
-            pPrevMap->pNextR3 = pMapping->pNextR3;
-            pPrevMap->pNextRC = pMapping->pNextRC;
-            pPrevMap->pNextR0 = pMapping->pNextR0;
-        }
-        else
-        {
-            pVM->pgm.s.pMappingsR3 = pMapping->pNextR3;
-            pVM->pgm.s.pMappingsRC = pMapping->pNextRC;
-            pVM->pgm.s.pMappingsR0 = pMapping->pNextR0;
-        }
-
-        /*
-         * Link
-         */
-        pMapping->pNextR3 = pCur;
-        if (pPrev)
-        {
-            pMapping->pNextRC = pPrev->pNextRC;
-            pMapping->pNextR0 = pPrev->pNextR0;
-            pPrev->pNextR3 = pMapping;
-            pPrev->pNextRC = MMHyperR3ToRC(pVM, pMapping);
-            pPrev->pNextR0 = MMHyperR3ToR0(pVM, pMapping);
-        }
-        else
-        {
-            pMapping->pNextRC = pVM->pgm.s.pMappingsRC;
-            pMapping->pNextR0 = pVM->pgm.s.pMappingsR0;
-            pVM->pgm.s.pMappingsR3 = pMapping;
-            pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pMapping);
-            pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pMapping);
-        }
-    }
-
-    /*
-     * Update the entry.
-     */
-    pMapping->GCPtr = GCPtrNew;
-    pMapping->GCPtrLast = GCPtrNew + pMapping->cb - 1;
-
-    /*
-     * Callback to execute the relocation.
-     */
-    pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_RELOCATE, pMapping->pvUser);
-}
-
-
-/**
- * Checks if a new mapping address wasn't previously used and caused a clash with guest mappings.
- *
- * @returns VBox status code.
- * @param   pMapping            The mapping which conflicts.
- * @param   GCPtr               New mapping address to try
- */
-bool pgmR3MapIsKnownConflictAddress(PPGMMAPPING pMapping, RTGCPTR GCPtr)
-{
-    for (unsigned i = 0; i < RT_ELEMENTS(pMapping->aGCPtrConflicts); i++)
-    {
-        if (GCPtr == pMapping->aGCPtrConflicts[i])
-            return true;
-    }
-    return false;
-}
-
-
-/**
- * Resolves a conflict between a page table based GC mapping and
- * the Guest OS page tables. (32 bits version)
- *
- * @returns VBox status code.
- * @param   pVM                 VM Handle.
- * @param   pMapping            The mapping which conflicts.
- * @param   pPDSrc              The page directory of the guest OS.
- * @param   GCPtrOldMapping     The address of the start of the current mapping.
- */
-int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping)
-{
-    STAM_REL_COUNTER_INC(&pVM->pgm.s.cRelocations);
-    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
-
-    /* Raw mode only which implies one VCPU. */
-    Assert(pVM->cCpus == 1);
-
-    pMapping->aGCPtrConflicts[pMapping->cConflicts & (PGMMAPPING_CONFLICT_MAX-1)] = GCPtrOldMapping;
-    pMapping->cConflicts++;
-
-    /*
-     * Scan for free page directory entries.
-     *
-     * Note that we do not support mappings at the very end of the
-     * address space since that will break our GCPtrEnd assumptions.
-     */
-    const unsigned  cPTs = pMapping->cPTs;
-    unsigned        iPDNew = RT_ELEMENTS(pPDSrc->a) - cPTs; /* (+ 1 - 1) */
-    while (iPDNew-- > 0)
-    {
-        if (pPDSrc->a[iPDNew].n.u1Present)
-            continue;
-
-        if (pgmR3MapIsKnownConflictAddress(pMapping, iPDNew << X86_PD_SHIFT))
-            continue;
-
-        if (cPTs > 1)
-        {
-            bool fOk = true;
-            for (unsigned i = 1; fOk && i < cPTs; i++)
-                if (pPDSrc->a[iPDNew + i].n.u1Present)
-                    fOk = false;
-            if (!fOk)
-                continue;
-        }
-
-        /*
-         * Check that it's not conflicting with an intermediate page table mapping.
-         */
-        bool        fOk = true;
-        unsigned    i   = cPTs;
-        while (fOk && i-- > 0)
-            fOk = !pVM->pgm.s.pInterPD->a[iPDNew + i].n.u1Present;
-        if (!fOk)
-            continue;
-        /** @todo AMD64 should check the PAE directories and skip the 32bit stuff. */
-
-        /*
-         * Ask for the mapping.
-         */
-        RTGCPTR GCPtrNewMapping = (RTGCPTR32)iPDNew << X86_PD_SHIFT;
-
-        if (pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_SUGGEST, pMapping->pvUser))
-        {
-            pgmR3MapRelocate(pVM, pMapping, GCPtrOldMapping, GCPtrNewMapping);
-            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
-            return VINF_SUCCESS;
-        }
-    }
-
-    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
-    AssertMsgFailed(("Failed to relocate page table mapping '%s' from %#x! (cPTs=%d)\n", pMapping->pszDesc, GCPtrOldMapping, cPTs));
-    return VERR_PGM_NO_HYPERVISOR_ADDRESS;
-}
-
-
-/**
- * Resolves a conflict between a page table based GC mapping and
- * the Guest OS page tables. (PAE bits version)
- *
- * @returns VBox status code.
- * @param   pVM                 VM Handle.
- * @param   pMapping            The mapping which conflicts.
- * @param   GCPtrOldMapping     The address of the start of the current mapping.
- */
-int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping)
-{
-    STAM_REL_COUNTER_INC(&pVM->pgm.s.cRelocations);
-    STAM_PROFILE_START(&pVM->pgm.s.StatR3ResolveConflict, a);
-
-    /* Raw mode only which implies one VCPU. */
-    Assert(pVM->cCpus == 1);
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    pMapping->aGCPtrConflicts[pMapping->cConflicts & (PGMMAPPING_CONFLICT_MAX-1)] = GCPtrOldMapping;
-    pMapping->cConflicts++;
-
-    for (int iPDPTE = X86_PG_PAE_PDPE_ENTRIES - 1; iPDPTE >= 0; iPDPTE--)
-    {
-        unsigned  iPDSrc;
-        PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, (RTGCPTR32)iPDPTE << X86_PDPT_SHIFT, &iPDSrc, NULL);
-
-        /*
-         * Scan for free page directory entries.
-         *
-         * Note that we do not support mappings at the very end of the
-         * address space since that will break our GCPtrEnd assumptions.
-         * Nor do we support mappings crossing page directories.
-         */
-        const unsigned  cPTs = pMapping->cb >> X86_PD_PAE_SHIFT;
-        unsigned        iPDNew = RT_ELEMENTS(pPDSrc->a) - cPTs; /* (+ 1 - 1) */
-
-        while (iPDNew-- > 0)
-        {
-            /* Ugly assumption that mappings start on a 4 MB boundary. */
-            if (iPDNew & 1)
-                continue;
-
-            if (pgmR3MapIsKnownConflictAddress(pMapping, ((RTGCPTR32)iPDPTE << X86_PDPT_SHIFT) + (iPDNew << X86_PD_PAE_SHIFT)))
-                continue;
-
-            if (pPDSrc)
-            {
-                if (pPDSrc->a[iPDNew].n.u1Present)
-                    continue;
-                if (cPTs > 1)
-                {
-                    bool fOk = true;
-                    for (unsigned i = 1; fOk && i < cPTs; i++)
-                        if (pPDSrc->a[iPDNew + i].n.u1Present)
-                            fOk = false;
-                    if (!fOk)
-                        continue;
-                }
-            }
-            /*
-             * Check that it's not conflicting with an intermediate page table mapping.
-             */
-            bool        fOk = true;
-            unsigned    i   = cPTs;
-            while (fOk && i-- > 0)
-                fOk = !pVM->pgm.s.apInterPaePDs[iPDPTE]->a[iPDNew + i].n.u1Present;
-            if (!fOk)
-                continue;
-
-            /*
-             * Ask for the mapping.
-             */
-            RTGCPTR GCPtrNewMapping = ((RTGCPTR32)iPDPTE << X86_PDPT_SHIFT) + ((RTGCPTR32)iPDNew << X86_PD_PAE_SHIFT);
-
-            if (pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_SUGGEST, pMapping->pvUser))
-            {
-                pgmR3MapRelocate(pVM, pMapping, GCPtrOldMapping, GCPtrNewMapping);
-                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
-                return VINF_SUCCESS;
-            }
-        }
-    }
-    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
-    AssertMsgFailed(("Failed to relocate page table mapping '%s' from %#x! (cPTs=%d)\n", pMapping->pszDesc, GCPtrOldMapping, pMapping->cb >> X86_PD_PAE_SHIFT));
-    return VERR_PGM_NO_HYPERVISOR_ADDRESS;
-}
-
-
-/**
- * Read memory from the guest mappings.
- *
- * This will use the page tables associated with the mappings to
- * read the memory. This means that not all kind of memory is readable
- * since we don't necessarily know how to convert that physical address
- * to a HC virtual one.
- *
- * @returns VBox status.
- * @param   pVM         VM handle.
- * @param   pvDst       The destination address (HC of course).
- * @param   GCPtrSrc    The source address (GC virtual address).
- * @param   cb          Number of bytes to read.
- *
- * @remarks The is indirectly for DBGF only.
- * @todo    Consider renaming it to indicate it's special usage, or just
- *          reimplement it in MMR3HyperReadGCVirt.
- */
-VMMR3DECL(int) PGMR3MapRead(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
-{
-    /*
-     * Simplicity over speed... Chop the request up into chunks
-     * which don't cross pages.
-     */
-    if (cb + (GCPtrSrc & PAGE_OFFSET_MASK) > PAGE_SIZE)
-    {
-        for (;;)
-        {
-            size_t cbRead = RT_MIN(cb, PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK));
-            int rc = PGMR3MapRead(pVM, pvDst, GCPtrSrc, cbRead);
-            if (RT_FAILURE(rc))
-                return rc;
-            cb -= cbRead;
-            if (!cb)
-                break;
-            pvDst = (char *)pvDst + cbRead;
-            GCPtrSrc += cbRead;
-        }
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Find the mapping.
-     */
-    PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings);
-    while (pCur)
-    {
-        RTGCPTR off = GCPtrSrc - pCur->GCPtr;
-        if (off < pCur->cb)
-        {
-            if (off + cb > pCur->cb)
-            {
-                AssertMsgFailed(("Invalid page range %RGv LB%#x. mapping '%s' %RGv to %RGv\n",
-                                 GCPtrSrc, cb, pCur->pszDesc, pCur->GCPtr, pCur->GCPtrLast));
-                return VERR_INVALID_PARAMETER;
-            }
-
-            unsigned iPT  = off >> X86_PD_SHIFT;
-            unsigned iPTE = (off >> PAGE_SHIFT) & X86_PT_MASK;
-            while (cb > 0 && iPTE < RT_ELEMENTS(CTXALLSUFF(pCur->aPTs[iPT].pPT)->a))
-            {
-                PCPGMSHWPTEPAE pPte = &pCur->aPTs[iPT].CTXALLSUFF(paPaePTs)[iPTE / 512].a[iPTE % 512];
-                if (!PGMSHWPTEPAE_IS_P(*pPte))
-                    return VERR_PAGE_NOT_PRESENT;
-                RTHCPHYS HCPhys = PGMSHWPTEPAE_GET_HCPHYS(*pPte);
-
-                /*
-                 * Get the virtual page from the physical one.
-                 */
-                void *pvPage;
-                int rc = MMR3HCPhys2HCVirt(pVM, HCPhys, &pvPage);
-                if (RT_FAILURE(rc))
-                    return rc;
-
-                memcpy(pvDst, (char *)pvPage + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
-                return VINF_SUCCESS;
-            }
-        }
-
-        /* next */
-        pCur = CTXALLSUFF(pCur->pNext);
-    }
-
-    return VERR_INVALID_POINTER;
-}
-
-
-/**
- * Info callback for 'pgmhandlers'.
- *
- * @param   pHlp        The output helpers.
- * @param   pszArgs     The arguments. phys or virt.
- */
-DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    if (pVM->pgm.s.fMappingsDisabled)
-        pHlp->pfnPrintf(pHlp, "\nThe mappings are DISABLED.\n");
-    else if (pVM->pgm.s.fMappingsFixed)
-        pHlp->pfnPrintf(pHlp, "\nThe mappings are FIXED: %RGv-%RGv\n",
-                        pVM->pgm.s.GCPtrMappingFixed, pVM->pgm.s.GCPtrMappingFixed + pVM->pgm.s.cbMappingFixed - 1);
-    else if (pVM->pgm.s.fMappingsFixedRestored)
-        pHlp->pfnPrintf(pHlp, "\nThe mappings are FLOATING-RESTORED-FIXED: %RGv-%RGv\n",
-                        pVM->pgm.s.GCPtrMappingFixed, pVM->pgm.s.GCPtrMappingFixed + pVM->pgm.s.cbMappingFixed - 1);
-    else
-        pHlp->pfnPrintf(pHlp, "\nThe mappings are FLOATING.\n");
-
-    PPGMMAPPING pCur;
-    for (pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
-    {
-        pHlp->pfnPrintf(pHlp, "%RGv - %RGv  %s\n", pCur->GCPtr, pCur->GCPtrLast, pCur->pszDesc);
-        if (pCur->cConflicts > 0)
-        {
-            pHlp->pfnPrintf(pHlp, "  %u conflict%s: ", pCur->cConflicts, pCur->cConflicts == 1 ? "" : "s");
-            uint32_t cLeft = RT_MIN(pCur->cConflicts, RT_ELEMENTS(pCur->aGCPtrConflicts));
-            uint32_t i     = pCur->cConflicts;
-            while (cLeft-- > 0)
-            {
-                i = (i - 1) & (PGMMAPPING_CONFLICT_MAX - 1);
-                pHlp->pfnPrintf(pHlp, cLeft ? "%RGv, " : "%RGv\n", pCur->aGCPtrConflicts[i]);
-            }
-        }
-    }
-}
-
Index: unk/src/VBox/VMM/PGMPhys.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGMPhys.cpp	(revision 35332)
+++ 	(revision )
@@ -1,4324 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager and Monitor, Physical Memory Addressing.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM_PHYS
-#include <VBox/pgm.h>
-#include <VBox/iom.h>
-#include <VBox/mm.h>
-#include <VBox/stam.h>
-#include <VBox/rem.h>
-#include <VBox/pdmdev.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include "PGMInline.h"
-#include <VBox/sup.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-#include <iprt/asm.h>
-#include <iprt/thread.h>
-#include <iprt/string.h>
-#include <iprt/system.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The number of pages to free in one batch. */
-#define PGMPHYS_FREE_PAGE_BATCH_SIZE    128
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
-
-
-/*
- * PGMR3PhysReadU8-64
- * PGMR3PhysWriteU8-64
- */
-#define PGMPHYSFN_READNAME  PGMR3PhysReadU8
-#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
-#define PGMPHYS_DATASIZE    1
-#define PGMPHYS_DATATYPE    uint8_t
-#include "PGMPhysRWTmpl.h"
-
-#define PGMPHYSFN_READNAME  PGMR3PhysReadU16
-#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
-#define PGMPHYS_DATASIZE    2
-#define PGMPHYS_DATATYPE    uint16_t
-#include "PGMPhysRWTmpl.h"
-
-#define PGMPHYSFN_READNAME  PGMR3PhysReadU32
-#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
-#define PGMPHYS_DATASIZE    4
-#define PGMPHYS_DATATYPE    uint32_t
-#include "PGMPhysRWTmpl.h"
-
-#define PGMPHYSFN_READNAME  PGMR3PhysReadU64
-#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
-#define PGMPHYS_DATASIZE    8
-#define PGMPHYS_DATATYPE    uint64_t
-#include "PGMPhysRWTmpl.h"
-
-
-/**
- * EMT worker for PGMR3PhysReadExternal.
- */
-static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
-{
-    PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Read from physical memory, external users.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS.
- *
- * @param   pVM             VM Handle.
- * @param   GCPhys          Physical address to read from.
- * @param   pvBuf           Where to read into.
- * @param   cbRead          How many bytes to read.
- *
- * @thread  Any but EMTs.
- */
-VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
-{
-    VM_ASSERT_OTHER_THREAD(pVM);
-
-    AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
-    LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
-
-    pgmLock(pVM);
-
-    /*
-     * Copy loop on ram ranges.
-     */
-    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-    for (;;)
-    {
-        /* Find range. */
-        while (pRam && GCPhys > pRam->GCPhysLast)
-            pRam = pRam->CTX_SUFF(pNext);
-        /* Inside range or not? */
-        if (pRam && GCPhys >= pRam->GCPhys)
-        {
-            /*
-             * Must work our way thru this page by page.
-             */
-            RTGCPHYS off = GCPhys - pRam->GCPhys;
-            while (off < pRam->cb)
-            {
-                unsigned iPage = off >> PAGE_SHIFT;
-                PPGMPAGE pPage = &pRam->aPages[iPage];
-
-                /*
-                 * If the page has an ALL access handler, we'll have to
-                 * delegate the job to EMT.
-                 */
-                if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
-                {
-                    pgmUnlock(pVM);
-
-                    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
-                                           pVM, &GCPhys, pvBuf, cbRead);
-                }
-                Assert(!PGM_PAGE_IS_MMIO(pPage));
-
-                /*
-                 * Simple stuff, go ahead.
-                 */
-                size_t   cb    = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
-                if (cb > cbRead)
-                    cb = cbRead;
-                const void *pvSrc;
-                int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
-                if (RT_SUCCESS(rc))
-                    memcpy(pvBuf, pvSrc, cb);
-                else
-                {
-                    AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
-                                           pRam->GCPhys + off, pPage, rc));
-                    memset(pvBuf, 0xff, cb);
-                }
-
-                /* next page */
-                if (cb >= cbRead)
-                {
-                    pgmUnlock(pVM);
-                    return VINF_SUCCESS;
-                }
-                cbRead -= cb;
-                off    += cb;
-                GCPhys += cb;
-                pvBuf   = (char *)pvBuf + cb;
-            } /* walk pages in ram range. */
-        }
-        else
-        {
-            LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
-
-            /*
-             * Unassigned address space.
-             */
-            if (!pRam)
-                break;
-            size_t cb = pRam->GCPhys - GCPhys;
-            if (cb >= cbRead)
-            {
-                memset(pvBuf, 0xff, cbRead);
-                break;
-            }
-            memset(pvBuf, 0xff, cb);
-
-            cbRead -= cb;
-            pvBuf   = (char *)pvBuf + cb;
-            GCPhys += cb;
-        }
-    } /* Ram range walk */
-
-    pgmUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * EMT worker for PGMR3PhysWriteExternal.
- */
-static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
-{
-    /** @todo VERR_EM_NO_MEMORY */
-    PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Write to physical memory, external users.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS.
- * @retval  VERR_EM_NO_MEMORY.
- *
- * @param   pVM             VM Handle.
- * @param   GCPhys          Physical address to write to.
- * @param   pvBuf           What to write.
- * @param   cbWrite         How many bytes to write.
- * @param   pszWho          Who is writing.  For tracking down who is writing
- *                          after we've saved the state.
- *
- * @thread  Any but EMTs.
- */
-VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, const char *pszWho)
-{
-    VM_ASSERT_OTHER_THREAD(pVM);
-
-    AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
-              ("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
-               GCPhys, cbWrite, pszWho));
-    AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
-    LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
-
-    pgmLock(pVM);
-
-    /*
-     * Copy loop on ram ranges, stop when we hit something difficult.
-     */
-    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-    for (;;)
-    {
-        /* Find range. */
-        while (pRam && GCPhys > pRam->GCPhysLast)
-            pRam = pRam->CTX_SUFF(pNext);
-        /* Inside range or not? */
-        if (pRam && GCPhys >= pRam->GCPhys)
-        {
-            /*
-             * Must work our way thru this page by page.
-             */
-            RTGCPTR off = GCPhys - pRam->GCPhys;
-            while (off < pRam->cb)
-            {
-                RTGCPTR     iPage = off >> PAGE_SHIFT;
-                PPGMPAGE    pPage = &pRam->aPages[iPage];
-
-                /*
-                 * Is the page problematic, we have to do the work on the EMT.
-                 *
-                 * Allocating writable pages and access handlers are
-                 * problematic, write monitored pages are simple and can be
-                 * dealt with here.
-                 */
-                if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
-                    ||  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
-                {
-                    if (    PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
-                        && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
-                        pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
-                    else
-                    {
-                        pgmUnlock(pVM);
-
-                        return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
-                                               pVM, &GCPhys, pvBuf, cbWrite);
-                    }
-                }
-                Assert(!PGM_PAGE_IS_MMIO(pPage));
-
-                /*
-                 * Simple stuff, go ahead.
-                 */
-                size_t      cb    = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
-                if (cb > cbWrite)
-                    cb = cbWrite;
-                void *pvDst;
-                int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
-                if (RT_SUCCESS(rc))
-                    memcpy(pvDst, pvBuf, cb);
-                else
-                    AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
-                                           pRam->GCPhys + off, pPage, rc));
-
-                /* next page */
-                if (cb >= cbWrite)
-                {
-                    pgmUnlock(pVM);
-                    return VINF_SUCCESS;
-                }
-
-                cbWrite -= cb;
-                off     += cb;
-                GCPhys  += cb;
-                pvBuf    = (const char *)pvBuf + cb;
-            } /* walk pages in ram range */
-        }
-        else
-        {
-            /*
-             * Unassigned address space, skip it.
-             */
-            if (!pRam)
-                break;
-            size_t cb = pRam->GCPhys - GCPhys;
-            if (cb >= cbWrite)
-                break;
-            cbWrite -= cb;
-            pvBuf   = (const char *)pvBuf + cb;
-            GCPhys += cb;
-        }
-    } /* Ram range walk */
-
-    pgmUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
- *
- * @returns see PGMR3PhysGCPhys2CCPtrExternal
- * @param   pVM         The VM handle.
- * @param   pGCPhys     Pointer to the guest physical address.
- * @param   ppv         Where to store the mapping address.
- * @param   pLock       Where to store the lock.
- */
-static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
-{
-    /*
-     * Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
-     * an access handler after it succeeds.
-     */
-    int rc = pgmLock(pVM);
-    AssertRCReturn(rc, rc);
-
-    rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
-    if (RT_SUCCESS(rc))
-    {
-        PPGMPAGEMAPTLBE pTlbe;
-        int rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, *pGCPhys, &pTlbe);
-        AssertFatalRC(rc2);
-        PPGMPAGE pPage = pTlbe->pPage;
-        if (PGM_PAGE_IS_MMIO(pPage))
-        {
-            PGMPhysReleasePageMappingLock(pVM, pLock);
-            rc = VERR_PGM_PHYS_PAGE_RESERVED;
-        }
-        else if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
-#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
-                 ||  pgmPoolIsDirtyPage(pVM, *pGCPhys)
-#endif
-                )
-        {
-            /* We *must* flush any corresponding pgm pool page here, otherwise we'll
-             * not be informed about writes and keep bogus gst->shw mappings around.
-             */
-            pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
-            Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
-            /** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
-             *        active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
-        }
-    }
-
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Requests the mapping of a guest page into ring-3, external threads.
- *
- * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
- * release it.
- *
- * This API will assume your intention is to write to the page, and will
- * therefore replace shared and zero pages. If you do not intend to modify the
- * page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
- *          backing or if the page has any active access handlers. The caller
- *          must fall back on using PGMR3PhysWriteExternal.
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
- *
- * @param   pVM         The VM handle.
- * @param   GCPhys      The guest physical address of the page that should be mapped.
- * @param   ppv         Where to store the address corresponding to GCPhys.
- * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
- *
- * @remark  Avoid calling this API from within critical sections (other than the
- *          PGM one) because of the deadlock risk when we have to delegating the
- *          task to an EMT.
- * @thread  Any.
- */
-VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
-{
-    AssertPtr(ppv);
-    AssertPtr(pLock);
-
-    Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
-
-    int rc = pgmLock(pVM);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Query the Physical TLB entry for the page (may fail).
-     */
-    PPGMPAGEMAPTLBE pTlbe;
-    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
-    if (RT_SUCCESS(rc))
-    {
-        PPGMPAGE pPage = pTlbe->pPage;
-        if (PGM_PAGE_IS_MMIO(pPage))
-            rc = VERR_PGM_PHYS_PAGE_RESERVED;
-        else
-        {
-            /*
-             * If the page is shared, the zero page, or being write monitored
-             * it must be converted to an page that's writable if possible.
-             * We can only deal with write monitored pages here, the rest have
-             * to be on an EMT.
-             */
-            if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
-                ||  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
-#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
-                ||  pgmPoolIsDirtyPage(pVM, GCPhys)
-#endif
-               )
-            {
-                if (    PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
-                    &&  !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
-#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
-                    &&  !pgmPoolIsDirtyPage(pVM, GCPhys)
-#endif
-                   )
-                    pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
-                else
-                {
-                    pgmUnlock(pVM);
-
-                    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
-                                           pVM, &GCPhys, ppv, pLock);
-                }
-            }
-
-            /*
-             * Now, just perform the locking and calculate the return address.
-             */
-            PPGMPAGEMAP pMap = pTlbe->pMap;
-            if (pMap)
-                pMap->cRefs++;
-
-            unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
-            if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
-            {
-                if (cLocks == 0)
-                    pVM->pgm.s.cWriteLockedPages++;
-                PGM_PAGE_INC_WRITE_LOCKS(pPage);
-            }
-            else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
-            {
-                PGM_PAGE_INC_WRITE_LOCKS(pPage);
-                AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
-                if (pMap)
-                    pMap->cRefs++; /* Extra ref to prevent it from going away. */
-            }
-
-            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
-            pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
-            pLock->pvMap = pMap;
-        }
-    }
-
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Requests the mapping of a guest page into ring-3, external threads.
- *
- * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
- * release it.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
- *          backing or if the page as an active ALL access handler. The caller
- *          must fall back on using PGMPhysRead.
- * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
- *
- * @param   pVM         The VM handle.
- * @param   GCPhys      The guest physical address of the page that should be mapped.
- * @param   ppv         Where to store the address corresponding to GCPhys.
- * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
- *
- * @remark  Avoid calling this API from within critical sections (other than
- *          the PGM one) because of the deadlock risk.
- * @thread  Any.
- */
-VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
-{
-    int rc = pgmLock(pVM);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Query the Physical TLB entry for the page (may fail).
-     */
-    PPGMPAGEMAPTLBE pTlbe;
-    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
-    if (RT_SUCCESS(rc))
-    {
-        PPGMPAGE pPage = pTlbe->pPage;
-#if 1
-        /* MMIO pages doesn't have any readable backing. */
-        if (PGM_PAGE_IS_MMIO(pPage))
-            rc = VERR_PGM_PHYS_PAGE_RESERVED;
-#else
-        if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
-            rc = VERR_PGM_PHYS_PAGE_RESERVED;
-#endif
-        else
-        {
-            /*
-             * Now, just perform the locking and calculate the return address.
-             */
-            PPGMPAGEMAP pMap = pTlbe->pMap;
-            if (pMap)
-                pMap->cRefs++;
-
-            unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
-            if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
-            {
-                if (cLocks == 0)
-                    pVM->pgm.s.cReadLockedPages++;
-                PGM_PAGE_INC_READ_LOCKS(pPage);
-            }
-            else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
-            {
-                PGM_PAGE_INC_READ_LOCKS(pPage);
-                AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
-                if (pMap)
-                    pMap->cRefs++; /* Extra ref to prevent it from going away. */
-            }
-
-            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
-            pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
-            pLock->pvMap = pMap;
-        }
-    }
-
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
- *
- * Called when anything was relocated.
- *
- * @param   pVM         Pointer to the shared VM structure.
- */
-void pgmR3PhysRelinkRamRanges(PVM pVM)
-{
-    PPGMRAMRANGE pCur;
-
-#ifdef VBOX_STRICT
-    for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
-    {
-        Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
-        Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
-        Assert((pCur->GCPhys     & PAGE_OFFSET_MASK) == 0);
-        Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
-        Assert((pCur->cb         & PAGE_OFFSET_MASK) == 0);
-        Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
-        for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesR3; pCur2; pCur2 = pCur2->pNextR3)
-            Assert(   pCur2 == pCur
-                   || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
-    }
-#endif
-
-    pCur = pVM->pgm.s.pRamRangesR3;
-    if (pCur)
-    {
-        pVM->pgm.s.pRamRangesR0 = pCur->pSelfR0;
-        pVM->pgm.s.pRamRangesRC = pCur->pSelfRC;
-
-        for (; pCur->pNextR3; pCur = pCur->pNextR3)
-        {
-            pCur->pNextR0 = pCur->pNextR3->pSelfR0;
-            pCur->pNextRC = pCur->pNextR3->pSelfRC;
-        }
-
-        Assert(pCur->pNextR0 == NIL_RTR0PTR);
-        Assert(pCur->pNextRC == NIL_RTRCPTR);
-    }
-    else
-    {
-        Assert(pVM->pgm.s.pRamRangesR0 == NIL_RTR0PTR);
-        Assert(pVM->pgm.s.pRamRangesRC == NIL_RTRCPTR);
-    }
-    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
-}
-
-
-/**
- * Links a new RAM range into the list.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pNew        Pointer to the new list entry.
- * @param   pPrev       Pointer to the previous list entry. If NULL, insert as head.
- */
-static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
-{
-    AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
-    Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
-    Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
-
-    pgmLock(pVM);
-
-    PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesR3;
-    pNew->pNextR3 = pRam;
-    pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
-    pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
-
-    if (pPrev)
-    {
-        pPrev->pNextR3 = pNew;
-        pPrev->pNextR0 = pNew->pSelfR0;
-        pPrev->pNextRC = pNew->pSelfRC;
-    }
-    else
-    {
-        pVM->pgm.s.pRamRangesR3 = pNew;
-        pVM->pgm.s.pRamRangesR0 = pNew->pSelfR0;
-        pVM->pgm.s.pRamRangesRC = pNew->pSelfRC;
-    }
-    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Unlink an existing RAM range from the list.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pRam        Pointer to the new list entry.
- * @param   pPrev       Pointer to the previous list entry. If NULL, insert as head.
- */
-static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
-{
-    Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesR3 == pRam);
-    Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
-    Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
-
-    pgmLock(pVM);
-
-    PPGMRAMRANGE pNext = pRam->pNextR3;
-    if (pPrev)
-    {
-        pPrev->pNextR3 = pNext;
-        pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
-        pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
-    }
-    else
-    {
-        Assert(pVM->pgm.s.pRamRangesR3 == pRam);
-        pVM->pgm.s.pRamRangesR3 = pNext;
-        pVM->pgm.s.pRamRangesR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
-        pVM->pgm.s.pRamRangesRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
-    }
-    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Unlink an existing RAM range from the list.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pRam        Pointer to the new list entry.
- */
-static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
-{
-    pgmLock(pVM);
-
-    /* find prev. */
-    PPGMRAMRANGE pPrev = NULL;
-    PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3;
-    while (pCur != pRam)
-    {
-        pPrev = pCur;
-        pCur = pCur->pNextR3;
-    }
-    AssertFatal(pCur);
-
-    pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Frees a range of pages, replacing them with ZERO pages of the specified type.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pRam        The RAM range in which the pages resides.
- * @param   GCPhys      The address of the first page.
- * @param   GCPhysLast  The address of the last page.
- * @param   uType       The page type to replace then with.
- */
-static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
-{
-    Assert(PGMIsLockOwner(pVM));
-    uint32_t            cPendingPages = 0;
-    PGMMFREEPAGESREQ    pReq;
-    int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
-    AssertLogRelRCReturn(rc, rc);
-
-    /* Iterate the pages. */
-    PPGMPAGE pPageDst   = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-    uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
-    while (cPagesLeft-- > 0)
-    {
-        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
-        AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
-
-        PGM_PAGE_SET_TYPE(pPageDst, uType);
-
-        GCPhys += PAGE_SIZE;
-        pPageDst++;
-    }
-
-    if (cPendingPages)
-    {
-        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
-        AssertLogRelRCReturn(rc, rc);
-    }
-    GMMR3FreePagesCleanup(pReq);
-
-    return rc;
-}
-
-#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
-/**
- * Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
- *
- * This is only called on one of the EMTs while the other ones are waiting for
- * it to complete this function.
- *
- * @returns VINF_SUCCESS (VBox strict status code).
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
- * @param   pvUser      User parameter
- */
-static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    uintptr_t          *paUser          = (uintptr_t *)pvUser;
-    bool                fInflate        = !!paUser[0];
-    unsigned            cPages          = paUser[1];
-    RTGCPHYS           *paPhysPage      = (RTGCPHYS *)paUser[2];
-    uint32_t            cPendingPages   = 0;
-    PGMMFREEPAGESREQ    pReq;
-    int                 rc;
-
-    Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
-    pgmLock(pVM);
-
-    if (fInflate)
-    {
-        /* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
-        pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
-
-        /* Replace pages with ZERO pages. */
-        rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
-        if (RT_FAILURE(rc))
-        {
-            pgmUnlock(pVM);
-            AssertLogRelRC(rc);
-            return rc;
-        }
-
-        /* Iterate the pages. */
-        for (unsigned i = 0; i < cPages; i++)
-        {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
-            if (    pPage == NULL
-                ||  pPage->uTypeY != PGMPAGETYPE_RAM)
-            {
-                Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], (pPage) ? pPage->uTypeY : 0));
-                break;
-            }
-
-            LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
-
-            /* Flush the shadow PT if this page was previously used as a guest page table. */
-            pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
-
-            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
-            if (RT_FAILURE(rc))
-            {
-                pgmUnlock(pVM);
-                AssertLogRelRC(rc);
-                return rc;
-            }
-            Assert(PGM_PAGE_IS_ZERO(pPage));
-            PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_BALLOONED);
-        }
-
-        if (cPendingPages)
-        {
-            rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
-            if (RT_FAILURE(rc))
-            {
-                pgmUnlock(pVM);
-                AssertLogRelRC(rc);
-                return rc;
-            }
-        }
-        GMMR3FreePagesCleanup(pReq);
-    }
-    else
-    {
-        /* Iterate the pages. */
-        for (unsigned i = 0; i < cPages; i++)
-        {
-            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
-            AssertBreak(pPage && pPage->uTypeY == PGMPAGETYPE_RAM);
-
-            LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
-
-            Assert(PGM_PAGE_IS_BALLOONED(pPage));
-
-            /* Change back to zero page. */
-            PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
-        }
-
-        /* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
-    }
-
-    /* Notify GMM about the balloon change. */
-    rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
-    if (RT_SUCCESS(rc))
-    {
-        if (!fInflate)
-        {
-            Assert(pVM->pgm.s.cBalloonedPages >= cPages);
-            pVM->pgm.s.cBalloonedPages -= cPages;
-        }
-        else
-            pVM->pgm.s.cBalloonedPages += cPages;
-    }
-
-    pgmUnlock(pVM);
-
-    /* Flush the recompiler's TLB as well. */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
-
-    AssertLogRelRC(rc);
-    return rc;
-}
-
-/**
- * Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   fInflate    Inflate or deflate memory balloon
- * @param   cPages      Number of pages to free
- * @param   paPhysPage  Array of guest physical addresses
- */
-static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
-{
-    uintptr_t paUser[3];
-
-    paUser[0] = fInflate;
-    paUser[1] = cPages;
-    paUser[2] = (uintptr_t)paPhysPage;
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
-    AssertRC(rc);
-
-    /* Made a copy in PGMR3PhysFreeRamPages; free it here. */
-    RTMemFree(paPhysPage);
-}
-#endif
-
-/**
- * Inflate or deflate a memory balloon
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   fInflate    Inflate or deflate memory balloon
- * @param   cPages      Number of pages to free
- * @param   paPhysPage  Array of guest physical addresses
- */
-VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
-{
-    /* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
-#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
-    int rc;
-
-    /* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
-    AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
-
-    /* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
-     * In the SMP case we post a request packet to postpone the job.
-     */
-    if (pVM->cCpus > 1)
-    {
-        unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
-        RTGCPHYS *paPhysPageCopy = (RTGCPHYS *)RTMemAlloc(cbPhysPage);
-        AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
-
-        memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
-
-        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
-        AssertRC(rc);
-    }
-    else
-    {
-        uintptr_t paUser[3];
-
-        paUser[0] = fInflate;
-        paUser[1] = cPages;
-        paUser[2] = (uintptr_t)paPhysPage;
-        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
-        AssertRC(rc);
-    }
-    return rc;
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif
-}
-
-/**
- * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all physical RAM
- *
- * This is only called on one of the EMTs while the other ones are waiting for
- * it to complete this function.
- *
- * @returns VINF_SUCCESS (VBox strict status code).
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
- * @param   pvUser      User parameter
- */
-static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-
-    pgmLock(pVM);
-#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
-    pgmPoolResetDirtyPages(pVM);
-#endif
-
-    /** @todo pointless to write protect the physical page pointed to by RSP. */
-
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-         pRam;
-         pRam = pRam->CTX_SUFF(pNext))
-    {
-        unsigned cPages = pRam->cb >> PAGE_SHIFT;
-        for (unsigned iPage = 0; iPage < cPages; iPage++)
-        {
-            PPGMPAGE    pPage = &pRam->aPages[iPage];
-            PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
-
-            if (    RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
-                ||  enmPageType == PGMPAGETYPE_MMIO2)
-            {
-                /*
-                    * A RAM page.
-                    */
-                switch (PGM_PAGE_GET_STATE(pPage))
-                {
-                case PGM_PAGE_STATE_ALLOCATED:
-                    /** @todo Optimize this: Don't always re-enable write
-                        * monitoring if the page is known to be very busy. */
-                    if (PGM_PAGE_IS_WRITTEN_TO(pPage))
-                    {
-                        PGM_PAGE_CLEAR_WRITTEN_TO(pPage);
-                        /* Remember this dirty page for the next (memory) sync. */
-                        PGM_PAGE_SET_FT_DIRTY(pPage);
-                    }
-
-                    PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_WRITE_MONITORED);
-                    pVM->pgm.s.cMonitoredPages++;
-                    break;
-
-                case PGM_PAGE_STATE_SHARED:
-                    AssertFailed();
-                    break;
-
-                case PGM_PAGE_STATE_WRITE_MONITORED:    /* nothing to change. */
-                default:
-                    break;
-                }
-            }
-        }
-    }
-    pgmR3PoolWriteProtectPages(pVM);
-    PGM_INVL_ALL_VCPU_TLBS(pVM);
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-        CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
-
-    pgmUnlock(pVM);
-    return rc;
-}
-
-/**
- * Protect all physical RAM to monitor writes
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
-{
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
-    AssertRC(rc);
-    return rc;
-}
-
-/**
- * Enumerate all dirty FT pages
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pfnEnum     Enumerate callback handler
- * @param   pvUser      Enumerate callback handler parameter
- */
-VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-
-    pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
-         pRam;
-         pRam = pRam->CTX_SUFF(pNext))
-    {
-        unsigned cPages = pRam->cb >> PAGE_SHIFT;
-        for (unsigned iPage = 0; iPage < cPages; iPage++)
-        {
-            PPGMPAGE    pPage = &pRam->aPages[iPage];
-            PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
-
-            if (    RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
-                ||  enmPageType == PGMPAGETYPE_MMIO2)
-            {
-                /*
-                    * A RAM page.
-                    */
-                switch (PGM_PAGE_GET_STATE(pPage))
-                {
-                case PGM_PAGE_STATE_ALLOCATED:
-                case PGM_PAGE_STATE_WRITE_MONITORED:
-                    if (    !PGM_PAGE_IS_WRITTEN_TO(pPage)  /* not very recently updated? */
-                        &&  PGM_PAGE_IS_FT_DIRTY(pPage))
-                    {
-                        unsigned       cbPageRange = PAGE_SIZE;
-                        unsigned       iPageClean  = iPage + 1;
-                        RTGCPHYS       GCPhysPage  = pRam->GCPhys + iPage * PAGE_SIZE;
-                        uint8_t       *pu8Page     = NULL;
-                        PGMPAGEMAPLOCK Lock;
-
-                        /* Find the next clean page, so we can merge adjacent dirty pages. */
-                        for (; iPageClean < cPages; iPageClean++)
-                        {
-                            PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
-                            if (    RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
-                                ||  PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
-                                ||  PGM_PAGE_IS_WRITTEN_TO(pPageNext)
-                                ||  !PGM_PAGE_IS_FT_DIRTY(pPageNext)
-                                /* Crossing a chunk boundary? */
-                                ||  (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
-                                )
-                                break;
-
-                            cbPageRange += PAGE_SIZE;
-                        }
-
-                        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
-                        if (RT_SUCCESS(rc))
-                        {
-                            /** @todo this is risky; the range might be changed, but little choice as the sync costs a lot of time */
-                            pgmUnlock(pVM);
-                            pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
-                            pgmLock(pVM);
-                            PGMPhysReleasePageMappingLock(pVM, &Lock);
-                        }
-
-                        for (iPage; iPage < iPageClean; iPage++)
-                            PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
-
-                        iPage = iPageClean - 1;
-                    }
-                    break;
-                }
-            }
-        }
-    }
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Gets the number of ram ranges.
- *
- * @returns Number of ram ranges.  Returns UINT32_MAX if @a pVM is invalid.
- * @param   pVM             The VM handle.
- */
-VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
-
-    pgmLock(pVM);
-    uint32_t cRamRanges = 0;
-    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRanges); pCur; pCur = pCur->CTX_SUFF(pNext))
-        cRamRanges++;
-    pgmUnlock(pVM);
-    return cRamRanges;
-}
-
-
-/**
- * Get information about a range.
- *
- * @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
- * @param   pVM             The VM handle
- * @param   iRange          The ordinal of the range.
- * @param   pGCPhysStart    Where to return the start of the range. Optional.
- * @param   pGCPhysLast     Where to return the address of the last byte in the
- *                          range. Optional.
- * @param   pfIsMmio        Where to indicate that this is a pure MMIO range.
- *                          Optional.
- */
-VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
-                                 const char **ppszDesc, bool *pfIsMmio)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    pgmLock(pVM);
-    uint32_t iCurRange = 0;
-    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRanges); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
-        if (iCurRange == iRange)
-        {
-            if (pGCPhysStart)
-                *pGCPhysStart = pCur->GCPhys;
-            if (pGCPhysLast)
-                *pGCPhysLast = pCur->GCPhysLast;
-            if (pfIsMmio)
-                *pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
-
-            pgmUnlock(pVM);
-            return VINF_SUCCESS;
-        }
-    pgmUnlock(pVM);
-    return VERR_OUT_OF_RANGE;
-}
-
-
-/**
- * Query the amount of free memory inside VMMR0
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   puTotalAllocSize    Pointer to total allocated  memory inside VMMR0 (in bytes)
- * @param   puTotalFreeSize     Pointer to total free (allocated but not used yet) memory inside VMMR0 (in bytes)
- * @param   puTotalBalloonSize  Pointer to total ballooned memory inside VMMR0 (in bytes)
- * @param   puTotalSharedSize   Pointer to total shared memory inside VMMR0 (in bytes)
- */
-VMMR3DECL(int) PGMR3QueryVMMMemoryStats(PVM pVM, uint64_t *puTotalAllocSize, uint64_t *puTotalFreeSize, uint64_t *puTotalBalloonSize, uint64_t *puTotalSharedSize)
-{
-    int rc;
-
-    uint64_t cAllocPages = 0, cFreePages = 0, cBalloonPages = 0, cSharedPages = 0;
-    rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
-    AssertRCReturn(rc, rc);
-
-    if (puTotalAllocSize)
-        *puTotalAllocSize = cAllocPages * _4K;
-
-    if (puTotalFreeSize)
-        *puTotalFreeSize = cFreePages * _4K;
-
-    if (puTotalBalloonSize)
-        *puTotalBalloonSize = cBalloonPages * _4K;
-
-    if (puTotalSharedSize)
-        *puTotalSharedSize = cSharedPages * _4K;
-
-    Log(("PGMR3QueryVMMMemoryStats: all=%x free=%x ballooned=%x shared=%x\n", cAllocPages, cFreePages, cBalloonPages, cSharedPages));
-    return VINF_SUCCESS;
-}
-
-/**
- * Query memory stats for the VM
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   puTotalAllocSize    Pointer to total allocated  memory inside the VM (in bytes)
- * @param   puTotalFreeSize     Pointer to total free (allocated but not used yet) memory inside the VM (in bytes)
- * @param   puTotalBalloonSize  Pointer to total ballooned memory inside the VM (in bytes)
- * @param   puTotalSharedSize   Pointer to total shared memory inside the VM (in bytes)
- */
-VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t *pulTotalMem, uint64_t *pulPrivateMem, uint64_t *puTotalSharedMem, uint64_t *puTotalZeroMem)
-{
-    if (pulTotalMem)
-        *pulTotalMem = (uint64_t)pVM->pgm.s.cAllPages * _4K;
-
-    if (pulPrivateMem)
-        *pulPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages * _4K;
-
-    if (puTotalSharedMem)
-        *puTotalSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages * _4K;
-
-    if (puTotalZeroMem)
-        *puTotalZeroMem = (uint64_t)pVM->pgm.s.cZeroPages * _4K;
-
-    Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
-    return VINF_SUCCESS;
-}
-
-/**
- * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
- *
- * @param   pVM             The VM handle.
- * @param   pNew            The new RAM range.
- * @param   GCPhys          The address of the RAM range.
- * @param   GCPhysLast      The last address of the RAM range.
- * @param   RCPtrNew        The RC address if the range is floating. NIL_RTRCPTR
- *                          if in HMA.
- * @param   R0PtrNew        Ditto for R0.
- * @param   pszDesc         The description.
- * @param   pPrev           The previous RAM range (for linking).
- */
-static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
-                                         RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
-{
-    /*
-     * Initialize the range.
-     */
-    pNew->pSelfR0       = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
-    pNew->pSelfRC       = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
-    pNew->GCPhys        = GCPhys;
-    pNew->GCPhysLast    = GCPhysLast;
-    pNew->cb            = GCPhysLast - GCPhys + 1;
-    pNew->pszDesc       = pszDesc;
-    pNew->fFlags        = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
-    pNew->pvR3          = NULL;
-    pNew->paLSPages     = NULL;
-
-    uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
-    RTGCPHYS iPage = cPages;
-    while (iPage-- > 0)
-        PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
-
-    /* Update the page count stats. */
-    pVM->pgm.s.cZeroPages += cPages;
-    pVM->pgm.s.cAllPages  += cPages;
-
-    /*
-     * Link it.
-     */
-    pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
-}
-
-
-/**
- * Relocate a floating RAM range.
- *
- * @copydoc FNPGMRELOCATE.
- */
-static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
-{
-    PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
-    Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
-    Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
-
-    switch (enmMode)
-    {
-        case PGMRELOCATECALL_SUGGEST:
-            return true;
-        case PGMRELOCATECALL_RELOCATE:
-        {
-            /* Update myself and then relink all the ranges. */
-            pgmLock(pVM);
-            pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
-            pgmR3PhysRelinkRamRanges(pVM);
-            pgmUnlock(pVM);
-            return true;
-        }
-
-        default:
-            AssertFailedReturn(false);
-    }
-}
-
-
-/**
- * PGMR3PhysRegisterRam worker that registers a high chunk.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   GCPhys          The address of the RAM.
- * @param   cRamPages       The number of RAM pages to register.
- * @param   cbChunk         The size of the PGMRAMRANGE guest mapping.
- * @param   iChunk          The chunk number.
- * @param   pszDesc         The RAM range description.
- * @param   ppPrev          Previous RAM range pointer. In/Out.
- */
-static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
-                                         uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
-                                         PPGMRAMRANGE *ppPrev)
-{
-    const char *pszDescChunk = iChunk == 0
-                             ? pszDesc
-                             : MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
-    AssertReturn(pszDescChunk, VERR_NO_MEMORY);
-
-    /*
-     * Allocate memory for the new chunk.
-     */
-    size_t const cChunkPages  = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
-    PSUPPAGE     paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
-    AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
-    RTR0PTR      R0PtrChunk   = NIL_RTR0PTR;
-    void        *pvChunk      = NULL;
-    int rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk,
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-                              VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
-#else
-                              NULL,
-#endif
-                              paChunkPages);
-    if (RT_SUCCESS(rc))
-    {
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-        if (!VMMIsHwVirtExtForced(pVM))
-            R0PtrChunk = NIL_RTR0PTR;
-#else
-        R0PtrChunk = (uintptr_t)pvChunk;
-#endif
-        memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
-
-        PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
-
-        /*
-         * Create a mapping and map the pages into it.
-         * We push these in below the HMA.
-         */
-        RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
-        rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /*fFlags*/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
-        if (RT_SUCCESS(rc))
-        {
-            pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
-
-            RTGCPTR const   GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
-            RTGCPTR         GCPtrPage  = GCPtrChunk;
-            for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
-                rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Ok, init and link the range.
-                 */
-                pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
-                                             (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
-                *ppPrev = pNew;
-            }
-        }
-
-        if (RT_FAILURE(rc))
-            SUPR3PageFreeEx(pvChunk, cChunkPages);
-    }
-
-    RTMemTmpFree(paChunkPages);
-    return rc;
-}
-
-
-/**
- * Sets up a range RAM.
- *
- * This will check for conflicting registrations, make a resource
- * reservation for the memory (with GMM), and setup the per-page
- * tracking structures (PGMPAGE).
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   GCPhys          The physical address of the RAM.
- * @param   cb              The size of the RAM.
- * @param   pszDesc         The description - not copied, so, don't free or change it.
- */
-VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
-{
-   /*
-     * Validate input.
-     */
-    Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
-    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
-    AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
-    AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
-    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
-    AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-
-    pgmLock(pVM);
-
-    /*
-     * Find range location and check for conflicts.
-     * (We don't lock here because the locking by EMT is only required on update.)
-     */
-    PPGMRAMRANGE    pPrev = NULL;
-    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesR3;
-    while (pRam && GCPhysLast >= pRam->GCPhys)
-    {
-        if (    GCPhysLast >= pRam->GCPhys
-            &&  GCPhys     <= pRam->GCPhysLast)
-            AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
-                                         GCPhys, GCPhysLast, pszDesc,
-                                         pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
-                                        VERR_PGM_RAM_CONFLICT);
-
-        /* next */
-        pPrev = pRam;
-        pRam = pRam->pNextR3;
-    }
-
-    /*
-     * Register it with GMM (the API bitches).
-     */
-    const RTGCPHYS cPages = cb >> PAGE_SHIFT;
-    int rc = MMR3IncreaseBaseReservation(pVM, cPages);
-    if (RT_FAILURE(rc))
-    {
-        pgmUnlock(pVM);
-        return rc;
-    }
-
-    if (    GCPhys >= _4G
-        &&  cPages > 256)
-    {
-        /*
-         * The PGMRAMRANGE structures for the high memory can get very big.
-         * In order to avoid SUPR3PageAllocEx allocation failures due to the
-         * allocation size limit there and also to avoid being unable to find
-         * guest mapping space for them, we split this memory up into 4MB in
-         * (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
-         * mode.
-         *
-         * The first and last page of each mapping are guard pages and marked
-         * not-present. So, we've got 4186112 and 16769024 bytes available for
-         * the PGMRAMRANGE structure.
-         *
-         * Note! The sizes used here will influence the saved state.
-         */
-        uint32_t cbChunk;
-        uint32_t cPagesPerChunk;
-        if (VMMIsHwVirtExtForced(pVM))
-        {
-            cbChunk = 16U*_1M;
-            cPagesPerChunk = 1048048; /* max ~1048059 */
-            AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U*_1M - PAGE_SIZE * 2);
-        }
-        else
-        {
-            cbChunk = 4U*_1M;
-            cPagesPerChunk = 261616; /* max ~261627 */
-            AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616  <  4U*_1M - PAGE_SIZE * 2);
-        }
-        AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
-
-        RTGCPHYS cPagesLeft  = cPages;
-        RTGCPHYS GCPhysChunk = GCPhys;
-        uint32_t iChunk      = 0;
-        while (cPagesLeft > 0)
-        {
-            uint32_t cPagesInChunk = cPagesLeft;
-            if (cPagesInChunk > cPagesPerChunk)
-                cPagesInChunk = cPagesPerChunk;
-
-            rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
-            AssertRCReturn(rc, rc);
-
-            /* advance */
-            GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
-            cPagesLeft  -= cPagesInChunk;
-            iChunk++;
-        }
-    }
-    else
-    {
-        /*
-         * Allocate, initialize and link the new RAM range.
-         */
-        const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
-        PPGMRAMRANGE pNew;
-        rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
-        AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
-
-        pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
-    }
-    PGMPhysInvalidatePageMapTLB(pVM);
-    pgmUnlock(pVM);
-
-    /*
-     * Notify REM.
-     */
-    REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
- *
- * We do this late in the init process so that all the ROM and MMIO ranges have
- * been registered already and we don't go wasting memory on them.
- *
- * @returns VBox status code.
- *
- * @param   pVM     Pointer to the shared VM structure.
- */
-int pgmR3PhysRamPreAllocate(PVM pVM)
-{
-    Assert(pVM->pgm.s.fRamPreAlloc);
-    Log(("pgmR3PhysRamPreAllocate: enter\n"));
-
-    /*
-     * Walk the RAM ranges and allocate all RAM pages, halt at
-     * the first allocation error.
-     */
-    uint64_t cPages = 0;
-    uint64_t NanoTS = RTTimeNanoTS();
-    pgmLock(pVM);
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
-    {
-        PPGMPAGE    pPage  = &pRam->aPages[0];
-        RTGCPHYS    GCPhys = pRam->GCPhys;
-        uint32_t    cLeft  = pRam->cb >> PAGE_SHIFT;
-        while (cLeft-- > 0)
-        {
-            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
-            {
-                switch (PGM_PAGE_GET_STATE(pPage))
-                {
-                    case PGM_PAGE_STATE_ZERO:
-                    {
-                        int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
-                        if (RT_FAILURE(rc))
-                        {
-                            LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
-                            pgmUnlock(pVM);
-                            return rc;
-                        }
-                        cPages++;
-                        break;
-                    }
-
-                    case PGM_PAGE_STATE_BALLOONED:
-                    case PGM_PAGE_STATE_ALLOCATED:
-                    case PGM_PAGE_STATE_WRITE_MONITORED:
-                    case PGM_PAGE_STATE_SHARED:
-                        /* nothing to do here. */
-                        break;
-                }
-            }
-
-            /* next */
-            pPage++;
-            GCPhys += PAGE_SIZE;
-        }
-    }
-    pgmUnlock(pVM);
-    NanoTS = RTTimeNanoTS() - NanoTS;
-
-    LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
-    Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Resets (zeros) the RAM.
- *
- * ASSUMES that the caller owns the PGM lock.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- */
-int pgmR3PhysRamReset(PVM pVM)
-{
-    Assert(PGMIsLockOwner(pVM));
-
-    /* Reset the memory balloon. */
-    int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
-    AssertRC(rc);
-
-#ifdef VBOX_WITH_PAGE_SHARING
-    /* Clear all registered shared modules. */
-    rc = GMMR3ResetSharedModules(pVM);
-    AssertRC(rc);
-#endif
-    /* Reset counters. */
-    pVM->pgm.s.cReusedSharedPages = 0;
-    pVM->pgm.s.cBalloonedPages    = 0;
-
-    /*
-     * We batch up pages that should be freed instead of calling GMM for
-     * each and every one of them.
-     */
-    uint32_t            cPendingPages = 0;
-    PGMMFREEPAGESREQ    pReq;
-    rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
-    AssertLogRelRCReturn(rc, rc);
-
-    /*
-     * Walk the ram ranges.
-     */
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
-    {
-        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
-        AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
-
-        if (!pVM->pgm.s.fRamPreAlloc)
-        {
-            /* Replace all RAM pages by ZERO pages. */
-            while (iPage-- > 0)
-            {
-                PPGMPAGE pPage = &pRam->aPages[iPage];
-                switch (PGM_PAGE_GET_TYPE(pPage))
-                {
-                    case PGMPAGETYPE_RAM:
-                        /* Do not replace pages part of a 2 MB continuous range with zero pages, but zero them instead. */
-                        if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
-                        {
-                            void *pvPage;
-                            rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
-                            AssertLogRelRCReturn(rc, rc);
-                            ASMMemZeroPage(pvPage);
-                        }
-                        else
-                        if (PGM_PAGE_IS_BALLOONED(pPage))
-                        {
-                            /* Turn into a zero page; the balloon status is lost when the VM reboots. */
-                            PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
-                        }
-                        else
-                        if (!PGM_PAGE_IS_ZERO(pPage))
-                        {
-                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
-                            AssertLogRelRCReturn(rc, rc);
-                        }
-                        break;
-
-                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
-                        pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
-                                                           true /*fDoAccounting*/);
-                        break;
-
-                    case PGMPAGETYPE_MMIO2:
-                    case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
-                    case PGMPAGETYPE_ROM:
-                    case PGMPAGETYPE_MMIO:
-                        break;
-                    default:
-                        AssertFailed();
-                }
-            } /* for each page */
-        }
-        else
-        {
-            /* Zero the memory. */
-            while (iPage-- > 0)
-            {
-                PPGMPAGE pPage = &pRam->aPages[iPage];
-                switch (PGM_PAGE_GET_TYPE(pPage))
-                {
-                    case PGMPAGETYPE_RAM:
-                        switch (PGM_PAGE_GET_STATE(pPage))
-                        {
-                            case PGM_PAGE_STATE_ZERO:
-                                break;
-
-                            case PGM_PAGE_STATE_BALLOONED:
-                                /* Turn into a zero page; the balloon status is lost when the VM reboots. */
-                                PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
-                                break;
-
-                            case PGM_PAGE_STATE_SHARED:
-                            case PGM_PAGE_STATE_WRITE_MONITORED:
-                                rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
-                                AssertLogRelRCReturn(rc, rc);
-                                /* no break */
-
-                            case PGM_PAGE_STATE_ALLOCATED:
-                            {
-                                void *pvPage;
-                                rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
-                                AssertLogRelRCReturn(rc, rc);
-                                ASMMemZeroPage(pvPage);
-                                break;
-                            }
-                        }
-                        break;
-
-                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
-                        pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
-                                                           true /*fDoAccounting*/);
-                        break;
-
-                    case PGMPAGETYPE_MMIO2:
-                    case PGMPAGETYPE_ROM_SHADOW:
-                    case PGMPAGETYPE_ROM:
-                    case PGMPAGETYPE_MMIO:
-                        break;
-                    default:
-                        AssertFailed();
-
-                }
-            } /* for each page */
-        }
-
-    }
-
-    /*
-     * Finish off any pages pending freeing.
-     */
-    if (cPendingPages)
-    {
-        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
-        AssertLogRelRCReturn(rc, rc);
-    }
-    GMMR3FreePagesCleanup(pReq);
-
-    return VINF_SUCCESS;
-}
-
-/**
- * Frees all RAM during VM termination
- *
- * ASSUMES that the caller owns the PGM lock.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- */
-int pgmR3PhysRamTerm(PVM pVM)
-{
-    Assert(PGMIsLockOwner(pVM));
-
-    /* Reset the memory balloon. */
-    int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
-    AssertRC(rc);
-
-#ifdef VBOX_WITH_PAGE_SHARING
-    /* Clear all registered shared modules. */
-    rc = GMMR3ResetSharedModules(pVM);
-    AssertRC(rc);
-#endif
-
-    /*
-     * We batch up pages that should be freed instead of calling GMM for
-     * each and every one of them.
-     */
-    uint32_t            cPendingPages = 0;
-    PGMMFREEPAGESREQ    pReq;
-    rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
-    AssertLogRelRCReturn(rc, rc);
-
-    /*
-     * Walk the ram ranges.
-     */
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
-    {
-        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
-        AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
-
-        /* Replace all RAM pages by ZERO pages. */
-        while (iPage-- > 0)
-        {
-            PPGMPAGE pPage = &pRam->aPages[iPage];
-            switch (PGM_PAGE_GET_TYPE(pPage))
-            {
-                case PGMPAGETYPE_RAM:
-                    /* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
-                    if (PGM_PAGE_IS_SHARED(pPage))
-                    {
-                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
-                        AssertLogRelRCReturn(rc, rc);
-                    }
-                    break;
-
-                case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
-                case PGMPAGETYPE_MMIO2:
-                case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
-                case PGMPAGETYPE_ROM:
-                case PGMPAGETYPE_MMIO:
-                    break;
-                default:
-                    AssertFailed();
-            }
-        } /* for each page */
-    }
-
-    /*
-     * Finish off any pages pending freeing.
-     */
-    if (cPendingPages)
-    {
-        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
-        AssertLogRelRCReturn(rc, rc);
-    }
-    GMMR3FreePagesCleanup(pReq);
-    return VINF_SUCCESS;
-}
-
-/**
- * This is the interface IOM is using to register an MMIO region.
- *
- * It will check for conflicts and ensure that a RAM range structure
- * is present before calling the PGMR3HandlerPhysicalRegister API to
- * register the callbacks.
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   GCPhys          The start of the MMIO region.
- * @param   cb              The size of the MMIO region.
- * @param   pfnHandlerR3    The address of the ring-3 handler. (IOMR3MMIOHandler)
- * @param   pvUserR3        The user argument for R3.
- * @param   pfnHandlerR0    The address of the ring-0 handler. (IOMMMIOHandler)
- * @param   pvUserR0        The user argument for R0.
- * @param   pfnHandlerRC    The address of the RC handler. (IOMMMIOHandler)
- * @param   pvUserRC        The user argument for RC.
- * @param   pszDesc         The description of the MMIO region.
- */
-VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
-                                     R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
-                                     R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
-                                     RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
-                                     R3PTRTYPE(const char *) pszDesc)
-{
-    /*
-     * Assert on some assumption.
-     */
-    VM_ASSERT_EMT(pVM);
-    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
-    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
-
-    /*
-     * Make sure there's a RAM range structure for the region.
-     */
-    int rc;
-    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
-    bool fRamExists = false;
-    PPGMRAMRANGE pRamPrev = NULL;
-    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
-    while (pRam && GCPhysLast >= pRam->GCPhys)
-    {
-        if (    GCPhysLast >= pRam->GCPhys
-            &&  GCPhys     <= pRam->GCPhysLast)
-        {
-            /* Simplification: all within the same range. */
-            AssertLogRelMsgReturn(   GCPhys     >= pRam->GCPhys
-                                  && GCPhysLast <= pRam->GCPhysLast,
-                                  ("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
-                                   GCPhys, GCPhysLast, pszDesc,
-                                   pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
-                                  VERR_PGM_RAM_CONFLICT);
-
-            /* Check that it's all RAM or MMIO pages. */
-            PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-            uint32_t cLeft = cb >> PAGE_SHIFT;
-            while (cLeft-- > 0)
-            {
-                AssertLogRelMsgReturn(   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
-                                      || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
-                                      ("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
-                                       GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
-                                      VERR_PGM_RAM_CONFLICT);
-                pPage++;
-            }
-
-            /* Looks good. */
-            fRamExists = true;
-            break;
-        }
-
-        /* next */
-        pRamPrev = pRam;
-        pRam = pRam->pNextR3;
-    }
-    PPGMRAMRANGE pNew;
-    if (fRamExists)
-    {
-        pNew = NULL;
-
-        /*
-         * Make all the pages in the range MMIO/ZERO pages, freeing any
-         * RAM pages currently mapped here. This might not be 100% correct
-         * for PCI memory, but we're doing the same thing for MMIO2 pages.
-         */
-        rc = pgmLock(pVM);
-        if (RT_SUCCESS(rc))
-        {
-            rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
-            pgmUnlock(pVM);
-        }
-        AssertRCReturn(rc, rc);
-
-        /* Force a PGM pool flush as guest ram references have been changed. */
-        /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
-        PVMCPU pVCpu = VMMGetCpu(pVM);
-        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-    }
-    else
-    {
-        pgmLock(pVM);
-
-        /*
-         * No RAM range, insert an ad hoc one.
-         *
-         * Note that we don't have to tell REM about this range because
-         * PGMHandlerPhysicalRegisterEx will do that for us.
-         */
-        Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
-
-        const uint32_t cPages = cb >> PAGE_SHIFT;
-        const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
-        rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
-        AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
-
-        /* Initialize the range. */
-        pNew->pSelfR0       = MMHyperCCToR0(pVM, pNew);
-        pNew->pSelfRC       = MMHyperCCToRC(pVM, pNew);
-        pNew->GCPhys        = GCPhys;
-        pNew->GCPhysLast    = GCPhysLast;
-        pNew->cb            = cb;
-        pNew->pszDesc       = pszDesc;
-        pNew->fFlags        = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
-        pNew->pvR3          = NULL;
-        pNew->paLSPages     = NULL;
-
-        uint32_t iPage = cPages;
-        while (iPage-- > 0)
-            PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
-        Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
-
-        /* update the page count stats. */
-        pVM->pgm.s.cPureMmioPages += cPages;
-        pVM->pgm.s.cAllPages      += cPages;
-
-        /* link it */
-        pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
-
-        pgmUnlock(pVM);
-    }
-
-    /*
-     * Register the access handler.
-     */
-    rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
-                                      pfnHandlerR3, pvUserR3,
-                                      pfnHandlerR0, pvUserR0,
-                                      pfnHandlerRC, pvUserRC, pszDesc);
-    if (    RT_FAILURE(rc)
-        &&  !fRamExists)
-    {
-        pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
-        pVM->pgm.s.cAllPages      -= cb >> PAGE_SHIFT;
-
-        /* remove the ad hoc range. */
-        pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
-        pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
-        MMHyperFree(pVM, pRam);
-    }
-    PGMPhysInvalidatePageMapTLB(pVM);
-
-    return rc;
-}
-
-
-/**
- * This is the interface IOM is using to register an MMIO region.
- *
- * It will take care of calling PGMHandlerPhysicalDeregister and clean up
- * any ad hoc PGMRAMRANGE left behind.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   GCPhys          The start of the MMIO region.
- * @param   cb              The size of the MMIO region.
- */
-VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
-{
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * First deregister the handler, then check if we should remove the ram range.
-     */
-    int rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
-    if (RT_SUCCESS(rc))
-    {
-        RTGCPHYS        GCPhysLast  = GCPhys + (cb - 1);
-        PPGMRAMRANGE    pRamPrev    = NULL;
-        PPGMRAMRANGE    pRam        = pVM->pgm.s.pRamRangesR3;
-        while (pRam && GCPhysLast >= pRam->GCPhys)
-        {
-            /** @todo We're being a bit too careful here. rewrite. */
-            if (    GCPhysLast == pRam->GCPhysLast
-                &&  GCPhys     == pRam->GCPhys)
-            {
-                Assert(pRam->cb == cb);
-
-                /*
-                 * See if all the pages are dead MMIO pages.
-                 */
-                uint32_t const  cPages   = cb >> PAGE_SHIFT;
-                bool            fAllMMIO = true;
-                uint32_t        iPage    = 0;
-                uint32_t        cLeft    = cPages;
-                while (cLeft-- > 0)
-                {
-                    PPGMPAGE    pPage    = &pRam->aPages[iPage];
-                    if (    PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
-                        /*|| not-out-of-action later */)
-                    {
-                        fAllMMIO = false;
-                        Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
-                        AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
-                        break;
-                    }
-                    Assert(PGM_PAGE_IS_ZERO(pPage));
-                    pPage++;
-                }
-                if (fAllMMIO)
-                {
-                    /*
-                     * Ad-hoc range, unlink and free it.
-                     */
-                    Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
-                         GCPhys, GCPhysLast, pRam->pszDesc));
-
-                    pVM->pgm.s.cAllPages      -= cPages;
-                    pVM->pgm.s.cPureMmioPages -= cPages;
-
-                    pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
-                    pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
-                    MMHyperFree(pVM, pRam);
-                    break;
-                }
-            }
-
-            /*
-             * Range match? It will all be within one range (see PGMAllHandler.cpp).
-             */
-            if (    GCPhysLast >= pRam->GCPhys
-                &&  GCPhys     <= pRam->GCPhysLast)
-            {
-                Assert(GCPhys     >= pRam->GCPhys);
-                Assert(GCPhysLast <= pRam->GCPhysLast);
-
-                /*
-                 * Turn the pages back into RAM pages.
-                 */
-                uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
-                uint32_t cLeft = cb >> PAGE_SHIFT;
-                while (cLeft--)
-                {
-                    PPGMPAGE pPage = &pRam->aPages[iPage];
-                    AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
-                    AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
-                    if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
-                        PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_RAM);
-                }
-                break;
-            }
-
-            /* next */
-            pRamPrev = pRam;
-            pRam = pRam->pNextR3;
-        }
-    }
-
-    /* Force a PGM pool flush as guest ram references have been changed. */
-    /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-
-    PGMPhysInvalidatePageMapTLB(pVM);
-    return rc;
-}
-
-
-/**
- * Locate a MMIO2 range.
- *
- * @returns Pointer to the MMIO2 range.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The device instance owning the region.
- * @param   iRegion         The region.
- */
-DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
-{
-    /*
-     * Search the list.
-     */
-    for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
-        if (   pCur->pDevInsR3 == pDevIns
-            && pCur->iRegion == iRegion)
-            return pCur;
-    return NULL;
-}
-
-
-/**
- * Allocate and register an MMIO2 region.
- *
- * As mentioned elsewhere, MMIO2 is just RAM spelled differently.  It's RAM
- * associated with a device. It is also non-shared memory with a permanent
- * ring-3 mapping and page backing (presently).
- *
- * A MMIO2 range may overlap with base memory if a lot of RAM is configured for
- * the VM, in which case we'll drop the base memory pages.  Presently we will
- * make no attempt to preserve anything that happens to be present in the base
- * memory that is replaced, this is of course incorrectly but it's too much
- * effort.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
- *          memory.
- * @retval  VERR_ALREADY_EXISTS if the region already exists.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The device instance owning the region.
- * @param   iRegion         The region number.  If the MMIO2 memory is a PCI
- *                          I/O region this number has to be the number of that
- *                          region. Otherwise it can be any number safe
- *                          UINT8_MAX.
- * @param   cb              The size of the region.  Must be page aligned.
- * @param   fFlags          Reserved for future use, must be zero.
- * @param   ppv             Where to store the pointer to the ring-3 mapping of
- *                          the memory.
- * @param   pszDesc         The description.
- */
-VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void **ppv, const char *pszDesc)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
-    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
-    AssertPtrReturn(ppv, VERR_INVALID_POINTER);
-    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
-    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
-    AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
-    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(cb, VERR_INVALID_PARAMETER);
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
-
-    const uint32_t cPages = cb >> PAGE_SHIFT;
-    AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
-    AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
-
-    /*
-     * For the 2nd+ instance, mangle the description string so it's unique.
-     */
-    if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
-    {
-        pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
-        if (!pszDesc)
-            return VERR_NO_MEMORY;
-    }
-
-    /*
-     * Try reserve and allocate the backing memory first as this is what is
-     * most likely to fail.
-     */
-    int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
-    if (RT_SUCCESS(rc))
-    {
-        void *pvPages;
-        PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
-        if (RT_SUCCESS(rc))
-            rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pvPages, NULL /*pR0Ptr*/, paPages);
-        if (RT_SUCCESS(rc))
-        {
-            memset(pvPages, 0, cPages * PAGE_SIZE);
-
-            /*
-             * Create the MMIO2 range record for it.
-             */
-            const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
-            PPGMMMIO2RANGE pNew;
-            rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
-            AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
-            if (RT_SUCCESS(rc))
-            {
-                pNew->pDevInsR3             = pDevIns;
-                pNew->pvR3                  = pvPages;
-                //pNew->pNext               = NULL;
-                //pNew->fMapped             = false;
-                //pNew->fOverlapping        = false;
-                pNew->iRegion               = iRegion;
-                pNew->idSavedState          = UINT8_MAX;
-                pNew->RamRange.pSelfR0      = MMHyperCCToR0(pVM, &pNew->RamRange);
-                pNew->RamRange.pSelfRC      = MMHyperCCToRC(pVM, &pNew->RamRange);
-                pNew->RamRange.GCPhys       = NIL_RTGCPHYS;
-                pNew->RamRange.GCPhysLast   = NIL_RTGCPHYS;
-                pNew->RamRange.pszDesc      = pszDesc;
-                pNew->RamRange.cb           = cb;
-                pNew->RamRange.fFlags       = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
-                pNew->RamRange.pvR3         = pvPages;
-                //pNew->RamRange.paLSPages    = NULL;
-
-                uint32_t iPage = cPages;
-                while (iPage-- > 0)
-                {
-                    PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
-                                  paPages[iPage].Phys, NIL_GMM_PAGEID,
-                                  PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
-                }
-
-                /* update page count stats */
-                pVM->pgm.s.cAllPages     += cPages;
-                pVM->pgm.s.cPrivatePages += cPages;
-
-                /*
-                 * Link it into the list.
-                 * Since there is no particular order, just push it.
-                 */
-                pgmLock(pVM);
-                pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
-                pVM->pgm.s.pMmio2RangesR3 = pNew;
-                pgmUnlock(pVM);
-
-                *ppv = pvPages;
-                RTMemTmpFree(paPages);
-                PGMPhysInvalidatePageMapTLB(pVM);
-                return VINF_SUCCESS;
-            }
-
-            SUPR3PageFreeEx(pvPages, cPages);
-        }
-        RTMemTmpFree(paPages);
-        MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
-    }
-    if (pDevIns->iInstance > 0)
-        MMR3HeapFree((void *)pszDesc);
-    return rc;
-}
-
-
-/**
- * Deregisters and frees an MMIO2 region.
- *
- * Any physical (and virtual) access handlers registered for the region must
- * be deregistered before calling this function.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The device instance owning the region.
- * @param   iRegion         The region. If it's UINT32_MAX it'll be a wildcard match.
- */
-VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
-    AssertReturn(iRegion <= UINT8_MAX || iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
-
-    pgmLock(pVM);
-    int rc = VINF_SUCCESS;
-    unsigned cFound = 0;
-    PPGMMMIO2RANGE pPrev = NULL;
-    PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
-    while (pCur)
-    {
-        if (    pCur->pDevInsR3 == pDevIns
-            &&  (   iRegion == UINT32_MAX
-                 || pCur->iRegion == iRegion))
-        {
-            cFound++;
-
-            /*
-             * Unmap it if it's mapped.
-             */
-            if (pCur->fMapped)
-            {
-                int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
-                AssertRC(rc2);
-                if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                    rc = rc2;
-            }
-
-            /*
-             * Unlink it
-             */
-            PPGMMMIO2RANGE pNext = pCur->pNextR3;
-            if (pPrev)
-                pPrev->pNextR3 = pNext;
-            else
-                pVM->pgm.s.pMmio2RangesR3 = pNext;
-            pCur->pNextR3 = NULL;
-
-            /*
-             * Free the memory.
-             */
-            int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
-            AssertRC(rc2);
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-
-            uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
-            rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
-            AssertRC(rc2);
-            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
-                rc = rc2;
-
-            /* we're leaking hyper memory here if done at runtime. */
-#ifdef VBOX_STRICT
-            VMSTATE const enmState = VMR3GetState(pVM);
-            AssertMsg(   enmState == VMSTATE_POWERING_OFF
-                      || enmState == VMSTATE_POWERING_OFF_LS
-                      || enmState == VMSTATE_OFF
-                      || enmState == VMSTATE_OFF_LS
-                      || enmState == VMSTATE_DESTROYING
-                      || enmState == VMSTATE_TERMINATED
-                      || enmState == VMSTATE_CREATING
-                      , ("%s\n", VMR3GetStateName(enmState)));
-#endif
-            /*rc = MMHyperFree(pVM, pCur);
-            AssertRCReturn(rc, rc); - not safe, see the alloc call. */
-
-
-            /* update page count stats */
-            pVM->pgm.s.cAllPages     -= cPages;
-            pVM->pgm.s.cPrivatePages -= cPages;
-
-            /* next */
-            pCur = pNext;
-        }
-        else
-        {
-            pPrev = pCur;
-            pCur = pCur->pNextR3;
-        }
-    }
-    PGMPhysInvalidatePageMapTLB(pVM);
-    pgmUnlock(pVM);
-    return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
-}
-
-
-/**
- * Maps a MMIO2 region.
- *
- * This is done when a guest / the bios / state loading changes the
- * PCI config. The replacing of base memory has the same restrictions
- * as during registration, of course.
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The
- */
-VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
-{
-    /*
-     * Validate input
-     */
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
-    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
-    AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
-    AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
-    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-
-    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
-    AssertReturn(pCur, VERR_NOT_FOUND);
-    AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
-    Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
-    Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
-
-    const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
-    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
-
-    /*
-     * Find our location in the ram range list, checking for
-     * restriction we don't bother implementing yet (partially overlapping).
-     */
-    bool fRamExists = false;
-    PPGMRAMRANGE pRamPrev = NULL;
-    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
-    while (pRam && GCPhysLast >= pRam->GCPhys)
-    {
-        if (    GCPhys     <= pRam->GCPhysLast
-            &&  GCPhysLast >= pRam->GCPhys)
-        {
-            /* completely within? */
-            AssertLogRelMsgReturn(   GCPhys     >= pRam->GCPhys
-                                  && GCPhysLast <= pRam->GCPhysLast,
-                                  ("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
-                                   GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
-                                   pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
-                                  VERR_PGM_RAM_CONFLICT);
-            fRamExists = true;
-            break;
-        }
-
-        /* next */
-        pRamPrev = pRam;
-        pRam = pRam->pNextR3;
-    }
-    if (fRamExists)
-    {
-        PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
-        while (cPagesLeft-- > 0)
-        {
-            AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
-                                  ("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
-                                   GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
-                                  VERR_PGM_RAM_CONFLICT);
-            pPage++;
-        }
-    }
-    Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
-         GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
-
-    /*
-     * Make the changes.
-     */
-    pgmLock(pVM);
-
-    pCur->RamRange.GCPhys = GCPhys;
-    pCur->RamRange.GCPhysLast = GCPhysLast;
-    pCur->fMapped = true;
-    pCur->fOverlapping = fRamExists;
-
-    if (fRamExists)
-    {
-/** @todo use pgmR3PhysFreePageRange here. */
-        uint32_t            cPendingPages = 0;
-        PGMMFREEPAGESREQ    pReq;
-        int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
-        AssertLogRelRCReturn(rc, rc);
-
-        /* replace the pages, freeing all present RAM pages. */
-        PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
-        PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
-        while (cPagesLeft-- > 0)
-        {
-            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
-            AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
-
-            RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
-            PGM_PAGE_SET_HCPHYS(pPageDst, HCPhys);
-            PGM_PAGE_SET_TYPE(pPageDst, PGMPAGETYPE_MMIO2);
-            PGM_PAGE_SET_STATE(pPageDst, PGM_PAGE_STATE_ALLOCATED);
-            PGM_PAGE_SET_PDE_TYPE(pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
-            PGM_PAGE_SET_PTE_INDEX(pPageDst, 0);
-            PGM_PAGE_SET_TRACKING(pPageDst, 0);
-
-            pVM->pgm.s.cZeroPages--;
-            GCPhys += PAGE_SIZE;
-            pPageSrc++;
-            pPageDst++;
-        }
-
-        /* Flush physical page map TLB. */
-        PGMPhysInvalidatePageMapTLB(pVM);
-
-        if (cPendingPages)
-        {
-            rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
-            AssertLogRelRCReturn(rc, rc);
-        }
-        GMMR3FreePagesCleanup(pReq);
-
-        /* Force a PGM pool flush as guest ram references have been changed. */
-        /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
-        PVMCPU pVCpu = VMMGetCpu(pVM);
-        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-
-        pgmUnlock(pVM);
-    }
-    else
-    {
-        RTGCPHYS cb = pCur->RamRange.cb;
-
-        /* Clear the tracking data of pages we're going to reactivate. */
-        PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
-        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
-        while (cPagesLeft-- > 0)
-        {
-            PGM_PAGE_SET_TRACKING(pPageSrc, 0);
-            PGM_PAGE_SET_PTE_INDEX(pPageSrc, 0);
-            pPageSrc++;
-        }
-
-        /* link in the ram range */
-        pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
-        pgmUnlock(pVM);
-
-        REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
-    }
-
-    PGMPhysInvalidatePageMapTLB(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Unmaps a MMIO2 region.
- *
- * This is done when a guest / the bios / state loading changes the
- * PCI config. The replacing of base memory has the same restrictions
- * as during registration, of course.
- */
-VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
-{
-    /*
-     * Validate input
-     */
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
-    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
-    AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
-    AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
-    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-
-    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
-    AssertReturn(pCur, VERR_NOT_FOUND);
-    AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
-    AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
-    Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
-
-    Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
-         pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
-
-    /*
-     * Unmap it.
-     */
-    pgmLock(pVM);
-
-    RTGCPHYS    GCPhysRangeREM;
-    RTGCPHYS    cbRangeREM;
-    bool        fInformREM;
-    if (pCur->fOverlapping)
-    {
-        /* Restore the RAM pages we've replaced. */
-        PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
-        while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
-            pRam = pRam->pNextR3;
-
-        PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
-        while (cPagesLeft-- > 0)
-        {
-            PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
-            pVM->pgm.s.cZeroPages++;
-            pPageDst++;
-        }
-
-        /* Flush physical page map TLB. */
-        PGMPhysInvalidatePageMapTLB(pVM);
-
-        GCPhysRangeREM = NIL_RTGCPHYS;  /* shuts up gcc */
-        cbRangeREM     = RTGCPHYS_MAX;  /* ditto */
-        fInformREM     = false;
-    }
-    else
-    {
-        GCPhysRangeREM = pCur->RamRange.GCPhys;
-        cbRangeREM     = pCur->RamRange.cb;
-        fInformREM     = true;
-
-        pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
-    }
-
-    pCur->RamRange.GCPhys = NIL_RTGCPHYS;
-    pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
-    pCur->fOverlapping = false;
-    pCur->fMapped = false;
-
-    /* Force a PGM pool flush as guest ram references have been changed. */
-    /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-
-    PGMPhysInvalidatePageMapTLB(pVM);
-    pgmUnlock(pVM);
-
-    if (fInformREM)
-        REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Checks if the given address is an MMIO2 base address or not.
- *
- * @returns true/false accordingly.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The owner of the memory, optional.
- * @param   GCPhys          The address to check.
- */
-VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
-{
-    /*
-     * Validate input
-     */
-    VM_ASSERT_EMT_RETURN(pVM, false);
-    AssertPtrReturn(pDevIns, false);
-    AssertReturn(GCPhys != NIL_RTGCPHYS, false);
-    AssertReturn(GCPhys != 0, false);
-    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
-
-    /*
-     * Search the list.
-     */
-    pgmLock(pVM);
-    for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
-        if (pCur->RamRange.GCPhys == GCPhys)
-        {
-            Assert(pCur->fMapped);
-            pgmUnlock(pVM);
-            return true;
-        }
-    pgmUnlock(pVM);
-    return false;
-}
-
-
-/**
- * Gets the HC physical address of a page in the MMIO2 region.
- *
- * This is API is intended for MMHyper and shouldn't be called
- * by anyone else...
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pDevIns         The owner of the memory, optional.
- * @param   iRegion         The region.
- * @param   off             The page expressed an offset into the MMIO2 region.
- * @param   pHCPhys         Where to store the result.
- */
-VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
-{
-    /*
-     * Validate input
-     */
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
-    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
-
-    pgmLock(pVM);
-    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
-    AssertReturn(pCur, VERR_NOT_FOUND);
-    AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
-
-    PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
-    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
-    pgmUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Maps a portion of an MMIO2 region into kernel space (host).
- *
- * The kernel mapping will become invalid when the MMIO2 memory is deregistered
- * or the VM is terminated.
- *
- * @return VBox status code.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   pDevIns     The device owning the MMIO2 memory.
- * @param   iRegion     The region.
- * @param   off         The offset into the region. Must be page aligned.
- * @param   cb          The number of bytes to map. Must be page aligned.
- * @param   pszDesc     Mapping description.
- * @param   pR0Ptr      Where to store the R0 address.
- */
-VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
-                                       const char *pszDesc, PRTR0PTR pR0Ptr)
-{
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
-    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
-    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
-
-    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
-    AssertReturn(pCur, VERR_NOT_FOUND);
-    AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
-    AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
-    AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
-
-    /*
-     * Pass the request on to the support library/driver.
-     */
-    int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
-
-    return rc;
-}
-
-
-/**
- * Registers a ROM image.
- *
- * Shadowed ROM images requires double the amount of backing memory, so,
- * don't use that unless you have to. Shadowing of ROM images is process
- * where we can select where the reads go and where the writes go. On real
- * hardware the chipset provides means to configure this. We provide
- * PGMR3PhysProtectROM() for this purpose.
- *
- * A read-only copy of the ROM image will always be kept around while we
- * will allocate RAM pages for the changes on demand (unless all memory
- * is configured to be preallocated).
- *
- * @returns VBox status.
- * @param   pVM                 VM Handle.
- * @param   pDevIns             The device instance owning the ROM.
- * @param   GCPhys              First physical address in the range.
- *                              Must be page aligned!
- * @param   cb                  The size of the range (in bytes).
- *                              Must be page aligned!
- * @param   pvBinary            Pointer to the binary data backing the ROM image.
- * @param   cbBinary            The size of the binary data pvBinary points to.
- *                              This must be less or equal to @a cb.
- * @param   fFlags              Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
- *                              and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
- * @param   pszDesc             Pointer to description string. This must not be freed.
- *
- * @remark  There is no way to remove the rom, automatically on device cleanup or
- *          manually from the device yet. This isn't difficult in any way, it's
- *          just not something we expect to be necessary for a while.
- */
-VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
-                                    const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
-{
-    Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
-         pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
-    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
-    AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
-    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
-    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
-    AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED | PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
-    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
-
-    const uint32_t cPages = cb >> PAGE_SHIFT;
-
-    /*
-     * Find the ROM location in the ROM list first.
-     */
-    PPGMROMRANGE    pRomPrev = NULL;
-    PPGMROMRANGE    pRom = pVM->pgm.s.pRomRangesR3;
-    while (pRom && GCPhysLast >= pRom->GCPhys)
-    {
-        if (    GCPhys     <= pRom->GCPhysLast
-            &&  GCPhysLast >= pRom->GCPhys)
-            AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
-                                         GCPhys, GCPhysLast, pszDesc,
-                                         pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
-                                        VERR_PGM_RAM_CONFLICT);
-        /* next */
-        pRomPrev = pRom;
-        pRom = pRom->pNextR3;
-    }
-
-    /*
-     * Find the RAM location and check for conflicts.
-     *
-     * Conflict detection is a bit different than for RAM
-     * registration since a ROM can be located within a RAM
-     * range. So, what we have to check for is other memory
-     * types (other than RAM that is) and that we don't span
-     * more than one RAM range (layz).
-     */
-    bool            fRamExists = false;
-    PPGMRAMRANGE    pRamPrev = NULL;
-    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesR3;
-    while (pRam && GCPhysLast >= pRam->GCPhys)
-    {
-        if (    GCPhys     <= pRam->GCPhysLast
-            &&  GCPhysLast >= pRam->GCPhys)
-        {
-            /* completely within? */
-            AssertLogRelMsgReturn(   GCPhys     >= pRam->GCPhys
-                                  && GCPhysLast <= pRam->GCPhysLast,
-                                  ("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
-                                   GCPhys, GCPhysLast, pszDesc,
-                                   pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
-                                  VERR_PGM_RAM_CONFLICT);
-            fRamExists = true;
-            break;
-        }
-
-        /* next */
-        pRamPrev = pRam;
-        pRam = pRam->pNextR3;
-    }
-    if (fRamExists)
-    {
-        PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-        uint32_t cPagesLeft = cPages;
-        while (cPagesLeft-- > 0)
-        {
-            AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
-                                  ("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
-                                   pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
-                                   pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
-            Assert(PGM_PAGE_IS_ZERO(pPage));
-            pPage++;
-        }
-    }
-
-    /*
-     * Update the base memory reservation if necessary.
-     */
-    uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
-    if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
-        cExtraBaseCost += cPages;
-    if (cExtraBaseCost)
-    {
-        int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Allocate memory for the virgin copy of the RAM.
-     */
-    PGMMALLOCATEPAGESREQ pReq;
-    int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
-    AssertRCReturn(rc, rc);
-
-    for (uint32_t iPage = 0; iPage < cPages; iPage++)
-    {
-        pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
-        pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
-        pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
-    }
-
-    pgmLock(pVM);
-    rc = GMMR3AllocatePagesPerform(pVM, pReq);
-    pgmUnlock(pVM);
-    if (RT_FAILURE(rc))
-    {
-        GMMR3AllocatePagesCleanup(pReq);
-        return rc;
-    }
-
-    /*
-     * Allocate the new ROM range and RAM range (if necessary).
-     */
-    PPGMROMRANGE pRomNew;
-    rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
-    if (RT_SUCCESS(rc))
-    {
-        PPGMRAMRANGE pRamNew = NULL;
-        if (!fRamExists)
-            rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
-        if (RT_SUCCESS(rc))
-        {
-            pgmLock(pVM);
-
-            /*
-             * Initialize and insert the RAM range (if required).
-             */
-            PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
-            if (!fRamExists)
-            {
-                pRamNew->pSelfR0       = MMHyperCCToR0(pVM, pRamNew);
-                pRamNew->pSelfRC       = MMHyperCCToRC(pVM, pRamNew);
-                pRamNew->GCPhys        = GCPhys;
-                pRamNew->GCPhysLast    = GCPhysLast;
-                pRamNew->cb            = cb;
-                pRamNew->pszDesc       = pszDesc;
-                pRamNew->fFlags        = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
-                pRamNew->pvR3          = NULL;
-                pRamNew->paLSPages     = NULL;
-
-                PPGMPAGE pPage = &pRamNew->aPages[0];
-                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
-                {
-                    PGM_PAGE_INIT(pPage,
-                                  pReq->aPages[iPage].HCPhysGCPhys,
-                                  pReq->aPages[iPage].idPage,
-                                  PGMPAGETYPE_ROM,
-                                  PGM_PAGE_STATE_ALLOCATED);
-
-                    pRomPage->Virgin = *pPage;
-                }
-
-                pVM->pgm.s.cAllPages += cPages;
-                pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
-            }
-            else
-            {
-                PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
-                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
-                {
-                    PGM_PAGE_SET_TYPE(pPage,   PGMPAGETYPE_ROM);
-                    PGM_PAGE_SET_HCPHYS(pPage, pReq->aPages[iPage].HCPhysGCPhys);
-                    PGM_PAGE_SET_STATE(pPage,  PGM_PAGE_STATE_ALLOCATED);
-                    PGM_PAGE_SET_PAGEID(pPage, pReq->aPages[iPage].idPage);
-                    PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
-                    PGM_PAGE_SET_PTE_INDEX(pPage, 0);
-                    PGM_PAGE_SET_TRACKING(pPage, 0);
-
-                    pRomPage->Virgin = *pPage;
-                }
-
-                pRamNew = pRam;
-
-                pVM->pgm.s.cZeroPages -= cPages;
-            }
-            pVM->pgm.s.cPrivatePages += cPages;
-
-            /* Flush physical page map TLB. */
-            PGMPhysInvalidatePageMapTLB(pVM);
-
-            pgmUnlock(pVM);
-
-
-            /*
-             * !HACK ALERT!  REM + (Shadowed) ROM ==> mess.
-             *
-             * If it's shadowed we'll register the handler after the ROM notification
-             * so we get the access handler callbacks that we should. If it isn't
-             * shadowed we'll do it the other way around to make REM use the built-in
-             * ROM behavior and not the handler behavior (which is to route all access
-             * to PGM atm).
-             */
-            if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
-            {
-                REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
-                rc = PGMR3HandlerPhysicalRegister(pVM,
-                                                  fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
-                                                  ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
-                                                  : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
-                                                  GCPhys, GCPhysLast,
-                                                  pgmR3PhysRomWriteHandler, pRomNew,
-                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
-                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
-            }
-            else
-            {
-                rc = PGMR3HandlerPhysicalRegister(pVM,
-                                                  fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
-                                                  ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
-                                                  : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
-                                                  GCPhys, GCPhysLast,
-                                                  pgmR3PhysRomWriteHandler, pRomNew,
-                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
-                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
-                REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
-            }
-            if (RT_SUCCESS(rc))
-            {
-                pgmLock(pVM);
-
-                /*
-                 * Copy the image over to the virgin pages.
-                 * This must be done after linking in the RAM range.
-                 */
-                size_t          cbBinaryLeft = cbBinary;
-                PPGMPAGE        pRamPage     = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
-                for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
-                {
-                    void *pvDstPage;
-                    rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
-                    if (RT_FAILURE(rc))
-                    {
-                        VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
-                        break;
-                    }
-                    if (cbBinaryLeft >= PAGE_SIZE)
-                    {
-                        memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
-                        cbBinaryLeft -= PAGE_SIZE;
-                    }
-                    else
-                    {
-                        ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
-                        if (cbBinaryLeft > 0)
-                        {
-                            memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
-                            cbBinaryLeft = 0;
-                        }
-                    }
-                }
-                if (RT_SUCCESS(rc))
-                {
-                    /*
-                     * Initialize the ROM range.
-                     * Note that the Virgin member of the pages has already been initialized above.
-                     */
-                    pRomNew->GCPhys     = GCPhys;
-                    pRomNew->GCPhysLast = GCPhysLast;
-                    pRomNew->cb         = cb;
-                    pRomNew->fFlags     = fFlags;
-                    pRomNew->idSavedState = UINT8_MAX;
-                    pRomNew->cbOriginal = cbBinary;
-#ifdef VBOX_STRICT
-                    pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
-                                        ? pvBinary : RTMemDup(pvBinary, cbBinary);
-#else
-                    pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
-#endif
-                    pRomNew->pszDesc    = pszDesc;
-
-                    for (unsigned iPage = 0; iPage < cPages; iPage++)
-                    {
-                        PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
-                        pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
-                        PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
-                    }
-
-                    /* update the page count stats for the shadow pages. */
-                    if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
-                    {
-                        pVM->pgm.s.cZeroPages += cPages;
-                        pVM->pgm.s.cAllPages  += cPages;
-                    }
-
-                    /*
-                     * Insert the ROM range, tell REM and return successfully.
-                     */
-                    pRomNew->pNextR3 = pRom;
-                    pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
-                    pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
-
-                    if (pRomPrev)
-                    {
-                        pRomPrev->pNextR3 = pRomNew;
-                        pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
-                        pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
-                    }
-                    else
-                    {
-                        pVM->pgm.s.pRomRangesR3 = pRomNew;
-                        pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
-                        pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
-                    }
-
-                    PGMPhysInvalidatePageMapTLB(pVM);
-                    GMMR3AllocatePagesCleanup(pReq);
-                    pgmUnlock(pVM);
-                    return VINF_SUCCESS;
-                }
-
-                /* bail out */
-
-                pgmUnlock(pVM);
-                int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
-                AssertRC(rc2);
-                pgmLock(pVM);
-            }
-
-            if (!fRamExists)
-            {
-                pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
-                MMHyperFree(pVM, pRamNew);
-            }
-        }
-        MMHyperFree(pVM, pRomNew);
-    }
-
-    /** @todo Purge the mapping cache or something... */
-    GMMR3FreeAllocatedPages(pVM, pReq);
-    GMMR3AllocatePagesCleanup(pReq);
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * \#PF Handler callback for ROM write accesses.
- *
- * @returns VINF_SUCCESS if the handler have carried out the operation.
- * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
- * @param   pVM             VM Handle.
- * @param   GCPhys          The physical address the guest is writing to.
- * @param   pvPhys          The HC mapping of that address.
- * @param   pvBuf           What the guest is reading/writing.
- * @param   cbBuf           How much it's reading/writing.
- * @param   enmAccessType   The access type.
- * @param   pvUser          User argument.
- */
-static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
-                                                  PGMACCESSTYPE enmAccessType, void *pvUser)
-{
-    PPGMROMRANGE    pRom     = (PPGMROMRANGE)pvUser;
-    const uint32_t  iPage    = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
-    Assert(iPage < (pRom->cb >> PAGE_SHIFT));
-    PPGMROMPAGE     pRomPage = &pRom->aPages[iPage];
-    Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
-
-    if (enmAccessType == PGMACCESSTYPE_READ)
-    {
-        switch (pRomPage->enmProt)
-        {
-            /*
-             * Take the default action.
-             */
-            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
-            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
-            case PGMROMPROT_READ_ROM_WRITE_RAM:
-            case PGMROMPROT_READ_RAM_WRITE_RAM:
-                return VINF_PGM_HANDLER_DO_DEFAULT;
-
-            default:
-                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
-                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
-                                      VERR_INTERNAL_ERROR);
-        }
-    }
-    else
-    {
-        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
-        switch (pRomPage->enmProt)
-        {
-            /*
-             * Ignore writes.
-             */
-            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
-            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
-                return VINF_SUCCESS;
-
-            /*
-             * Write to the RAM page.
-             */
-            case PGMROMPROT_READ_ROM_WRITE_RAM:
-            case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
-            {
-                /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
-                Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
-
-                /*
-                 * Take the lock, do lazy allocation, map the page and copy the data.
-                 *
-                 * Note that we have to bypass the mapping TLB since it works on
-                 * guest physical addresses and entering the shadow page would
-                 * kind of screw things up...
-                 */
-                int rc = pgmLock(pVM);
-                AssertRC(rc);
-
-                PPGMPAGE pShadowPage = &pRomPage->Shadow;
-                if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
-                {
-                    pShadowPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
-                    AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
-                }
-
-                void *pvDstPage;
-                rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
-                if (RT_SUCCESS(rc))
-                {
-                    memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
-                    pRomPage->LiveSave.fWrittenTo = true;
-                }
-
-                pgmUnlock(pVM);
-                return rc;
-            }
-
-            default:
-                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
-                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
-                                      VERR_INTERNAL_ERROR);
-        }
-    }
-}
-
-
-/**
- * Called by PGMR3Reset to reset the shadow, switch to the virgin,
- * and verify that the virgin part is untouched.
- *
- * This is done after the normal memory has been cleared.
- *
- * ASSUMES that the caller owns the PGM lock.
- *
- * @param   pVM         The VM handle.
- */
-int pgmR3PhysRomReset(PVM pVM)
-{
-    Assert(PGMIsLockOwner(pVM));
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-    {
-        const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
-
-        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
-        {
-            /*
-             * Reset the physical handler.
-             */
-            int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
-            AssertRCReturn(rc, rc);
-
-            /*
-             * What we do with the shadow pages depends on the memory
-             * preallocation option. If not enabled, we'll just throw
-             * out all the dirty pages and replace them by the zero page.
-             */
-            if (!pVM->pgm.s.fRamPreAlloc)
-            {
-                /* Free the dirty pages. */
-                uint32_t            cPendingPages = 0;
-                PGMMFREEPAGESREQ    pReq;
-                rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
-                AssertRCReturn(rc, rc);
-
-                for (uint32_t iPage = 0; iPage < cPages; iPage++)
-                    if (    !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
-                        &&  !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
-                    {
-                        Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
-                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
-                                             pRom->GCPhys + (iPage << PAGE_SHIFT));
-                        AssertLogRelRCReturn(rc, rc);
-                    }
-
-                if (cPendingPages)
-                {
-                    rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
-                    AssertLogRelRCReturn(rc, rc);
-                }
-                GMMR3FreePagesCleanup(pReq);
-            }
-            else
-            {
-                /* clear all the shadow pages. */
-                for (uint32_t iPage = 0; iPage < cPages; iPage++)
-                {
-                    if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
-                        continue;
-                    Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
-                    void *pvDstPage;
-                    const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
-                    rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
-                    if (RT_FAILURE(rc))
-                        break;
-                    ASMMemZeroPage(pvDstPage);
-                }
-                AssertRCReturn(rc, rc);
-            }
-        }
-
-#ifdef VBOX_STRICT
-        /*
-         * Verify that the virgin page is unchanged if possible.
-         */
-        if (pRom->pvOriginal)
-        {
-            size_t         cbSrcLeft = pRom->cbOriginal;
-            uint8_t const *pbSrcPage = (uint8_t const *)pRom->pvOriginal;
-            for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
-            {
-                const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
-                void const *pvDstPage;
-                int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
-                if (RT_FAILURE(rc))
-                    break;
-
-                if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
-                    LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
-                            GCPhys, pRom->pszDesc));
-                cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
-            }
-        }
-#endif
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Called by PGMR3Term to free resources.
- *
- * ASSUMES that the caller owns the PGM lock.
- *
- * @param   pVM         The VM handle.
- */
-void pgmR3PhysRomTerm(PVM pVM)
-{
-#ifdef RT_STRICT
-    /*
-     * Free the heap copy of the original bits.
-     */
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-    {
-        if (   pRom->pvOriginal
-            && !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
-        {
-            RTMemFree((void *)pRom->pvOriginal);
-            pRom->pvOriginal = NULL;
-        }
-    }
-#endif
-}
-
-
-/**
- * Change the shadowing of a range of ROM pages.
- *
- * This is intended for implementing chipset specific memory registers
- * and will not be very strict about the input. It will silently ignore
- * any pages that are not the part of a shadowed ROM.
- *
- * @returns VBox status code.
- * @retval  VINF_PGM_SYNC_CR3
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   GCPhys      Where to start. Page aligned.
- * @param   cb          How much to change. Page aligned.
- * @param   enmProt     The new ROM protection.
- */
-VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
-{
-    /*
-     * Check input
-     */
-    if (!cb)
-        return VINF_SUCCESS;
-    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
-    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
-    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
-    AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
-
-    /*
-     * Process the request.
-     */
-    pgmLock(pVM);
-    int  rc = VINF_SUCCESS;
-    bool fFlushTLB = false;
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-    {
-        if (    GCPhys     <= pRom->GCPhysLast
-            &&  GCPhysLast >= pRom->GCPhys
-            &&  (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
-        {
-            /*
-             * Iterate the relevant pages and make necessary the changes.
-             */
-            bool fChanges = false;
-            uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
-                                  ? pRom->cb >> PAGE_SHIFT
-                                  : (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
-            for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
-                 iPage < cPages;
-                 iPage++)
-            {
-                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
-                if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
-                {
-                    fChanges = true;
-
-                    /* flush references to the page. */
-                    PPGMPAGE pRamPage = pgmPhysGetPage(&pVM->pgm.s, pRom->GCPhys + (iPage << PAGE_SHIFT));
-                    int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
-                                                       true /*fFlushPTEs*/, &fFlushTLB);
-                    if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
-                        rc = rc2;
-
-                    PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
-                    PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
-
-                    *pOld = *pRamPage;
-                    *pRamPage = *pNew;
-                    /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
-                }
-                pRomPage->enmProt = enmProt;
-            }
-
-            /*
-             * Reset the access handler if we made changes, no need
-             * to optimize this.
-             */
-            if (fChanges)
-            {
-                int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
-                if (RT_FAILURE(rc2))
-                {
-                    pgmUnlock(pVM);
-                    AssertRC(rc);
-                    return rc2;
-                }
-            }
-
-            /* Advance - cb isn't updated. */
-            GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
-        }
-    }
-    pgmUnlock(pVM);
-    if (fFlushTLB)
-        PGM_INVL_ALL_VCPU_TLBS(pVM);
-
-    return rc;
-}
-
-
-/**
- * Sets the Address Gate 20 state.
- *
- * @param   pVCpu       The VCPU to operate on.
- * @param   fEnable     True if the gate should be enabled.
- *                      False if the gate should be disabled.
- */
-VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
-{
-    LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
-    if (pVCpu->pgm.s.fA20Enabled != fEnable)
-    {
-        pVCpu->pgm.s.fA20Enabled = fEnable;
-        pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
-        REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
-        /** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
-    }
-}
-
-#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
-/**
- * Tree enumeration callback for dealing with age rollover.
- * It will perform a simple compression of the current age.
- */
-static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
-{
-    Assert(PGMIsLockOwner((PVM)pvUser));
-    /* Age compression - ASSUMES iNow == 4. */
-    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
-    if (pChunk->iAge >= UINT32_C(0xffffff00))
-        pChunk->iAge = 3;
-    else if (pChunk->iAge >= UINT32_C(0xfffff000))
-        pChunk->iAge = 2;
-    else if (pChunk->iAge)
-        pChunk->iAge = 1;
-    else /* iAge = 0 */
-        pChunk->iAge = 4;
-    return 0;
-}
-
-
-/**
- * Tree enumeration callback that updates the chunks that have
- * been used since the last
- */
-static DECLCALLBACK(int) pgmR3PhysChunkAgeingCallback(PAVLU32NODECORE pNode, void *pvUser)
-{
-    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
-    if (!pChunk->iAge)
-    {
-        PVM pVM = (PVM)pvUser;
-        pChunk->iAge = pVM->pgm.s.ChunkR3Map.iNow;
-    }
-    return 0;
-}
-
-
-/**
- * Performs ageing of the ring-3 chunk mappings.
- *
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(void) PGMR3PhysChunkAgeing(PVM pVM)
-{
-    pgmLock(pVM);
-    pVM->pgm.s.ChunkR3Map.AgeingCountdown = RT_MIN(pVM->pgm.s.ChunkR3Map.cMax / 4, 1024);
-    pVM->pgm.s.ChunkR3Map.iNow++;
-    if (pVM->pgm.s.ChunkR3Map.iNow == 0)
-    {
-        pVM->pgm.s.ChunkR3Map.iNow = 4;
-        RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingRolloverCallback, pVM);
-    }
-    else
-        RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingCallback, pVM);
-    pgmUnlock(pVM);
-}
-
-
-/**
- * The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
- */
-typedef struct PGMR3PHYSCHUNKUNMAPCB
-{
-    PVM                 pVM;            /**< The VM handle. */
-    PPGMCHUNKR3MAP      pChunk;         /**< The chunk to unmap. */
-    uint32_t            iLastAge;       /**< Highest age found so far. */
-} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
-
-
-/**
- * Callback used to find the mapping that's been unused for
- * the longest time.
- */
-static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
-{
-    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
-    PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
-
-    if (    pChunk->iAge
-        &&  !pChunk->cRefs
-        &&  pArg->iLastAge < pChunk->iAge)
-    {
-        /*
-         * Check that it's not in any of the TLBs.
-         */
-        PVM pVM = pArg->pVM;
-        for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
-            if (pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk == pChunk)
-            {
-                pChunk = NULL;
-                break;
-            }
-        if (pChunk)
-            for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
-                if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
-                {
-                    pChunk = NULL;
-                    break;
-                }
-        if (pChunk)
-        {
-            pArg->pChunk = pChunk;
-            pArg->iLastAge = pChunk->iAge;
-        }
-    }
-    return 0;
-}
-
-
-/**
- * Finds a good candidate for unmapping when the ring-3 mapping cache is full.
- *
- * The candidate will not be part of any TLBs, so no need to flush
- * anything afterwards.
- *
- * @returns Chunk id.
- * @param   pVM         The VM handle.
- */
-static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
-{
-    Assert(PGMIsLockOwner(pVM));
-
-    /*
-     * Do tree ageing first?
-     */
-    if (pVM->pgm.s.ChunkR3Map.AgeingCountdown-- == 0)
-    {
-        STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
-        PGMR3PhysChunkAgeing(pVM);
-        STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
-    }
-
-    /*
-     * Enumerate the age tree starting with the left most node.
-     */
-    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
-    PGMR3PHYSCHUNKUNMAPCB Args;
-    Args.pVM      = pVM;
-    Args.pChunk   = NULL;
-    Args.iLastAge = 0;
-    RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
-    Assert(Args.pChunk);
-    if (Args.pChunk)
-    {
-        STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
-        return Args.pChunk->Core.Key;
-    }
-
-    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
-    return INT32_MAX;
-}
-
-/**
- * Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
- *
- * This is only called on one of the EMTs while the other ones are waiting for
- * it to complete this function.
- *
- * @returns VINF_SUCCESS (VBox strict status code).
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
- * @param   pvUser      User pointer. Unused
- *
- */
-DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    int rc = VINF_SUCCESS;
-    pgmLock(pVM);
-
-    if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
-    {
-        /* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
-        /* todo: also not really efficient to unmap a chunk that contains PD or PT pages. */
-        pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
-
-        /*
-         * Request the ring-0 part to unmap a chunk to make space in the mapping cache.
-         */
-        GMMMAPUNMAPCHUNKREQ Req;
-        Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-        Req.Hdr.cbReq = sizeof(Req);
-        Req.pvR3 = NULL;
-        Req.idChunkMap = NIL_GMM_CHUNKID;
-        Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
-
-        if (Req.idChunkUnmap != INT32_MAX)
-        {
-            STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
-            rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
-            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
-            if (RT_SUCCESS(rc))
-            {
-                /* remove the unmapped one. */
-                PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
-                AssertRelease(pUnmappedChunk);
-                pUnmappedChunk->pv = NULL;
-                pUnmappedChunk->Core.Key = UINT32_MAX;
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-                MMR3HeapFree(pUnmappedChunk);
-#else
-                MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
-#endif
-                pVM->pgm.s.ChunkR3Map.c--;
-                pVM->pgm.s.cUnmappedChunks++;
-
-                /* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses) */
-                /* todo: we should not flush chunks which include cr3 mappings. */
-                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-                {
-                    PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
-
-                    pPGM->pGst32BitPdR3    = NULL;
-                    pPGM->pGstPaePdptR3    = NULL;
-                    pPGM->pGstAmd64Pml4R3  = NULL;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-                    pPGM->pGst32BitPdR0    = NIL_RTR0PTR;
-                    pPGM->pGstPaePdptR0    = NIL_RTR0PTR;
-                    pPGM->pGstAmd64Pml4R0  = NIL_RTR0PTR;
-#endif
-                    for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
-                    {
-                        pPGM->apGstPaePDsR3[i]             = NULL;
-#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
-                        pPGM->apGstPaePDsR0[i]             = NIL_RTR0PTR;
-#endif
-                    }
-
-                    /* Flush REM TLBs. */
-                    CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
-                }
-
-                /* Flush REM translation blocks. */
-                REMFlushTBs(pVM);
-            }
-        }
-    }
-    pgmUnlock(pVM);
-    return rc;
-}
-
-/**
- * Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-void pgmR3PhysUnmapChunk(PVM pVM)
-{
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
-    AssertRC(rc);
-}
-#endif /* PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST */
-
-/**
- * Maps the given chunk into the ring-3 mapping cache.
- *
- * This will call ring-0.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   idChunk     The chunk in question.
- * @param   ppChunk     Where to store the chunk tracking structure.
- *
- * @remarks Called from within the PGM critical section.
- * @remarks Can be called from any thread!
- */
-int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
-{
-    int rc;
-
-    Assert(PGMIsLockOwner(pVM));
-    /*
-     * Allocate a new tracking structure first.
-     */
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
-#else
-    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
-#endif
-    AssertReturn(pChunk, VERR_NO_MEMORY);
-    pChunk->Core.Key = idChunk;
-
-    /*
-     * Request the ring-0 part to map the chunk in question.
-     */
-    GMMMAPUNMAPCHUNKREQ Req;
-    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-    Req.Hdr.cbReq = sizeof(Req);
-    Req.pvR3 = NULL;
-    Req.idChunkMap = idChunk;
-    Req.idChunkUnmap = NIL_GMM_CHUNKID;
-
-    /* Must be callable from any thread, so can't use VMMR3CallR0. */
-    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
-    rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
-    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Update the tree.
-         */
-        /* insert the new one. */
-        AssertPtr(Req.pvR3);
-        pChunk->pv = Req.pvR3;
-        bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
-        AssertRelease(fRc);
-        pVM->pgm.s.ChunkR3Map.c++;
-        pVM->pgm.s.cMappedChunks++;
-
-        /* If we're running out of virtual address space, then we should unmap another chunk. */
-        if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
-        {
-#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
-            /* Postpone the unmap operation (which requires a rendezvous operation) as we own the PGM lock here. */
-            rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
-            AssertRC(rc);
-#else
-            AssertFatalFailed();  /* can't happen */
-#endif
-        }
-    }
-    else
-    {
-        AssertRC(rc);
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-        MMR3HeapFree(pChunk);
-#else
-        MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
-#endif
-        pChunk = NULL;
-    }
-
-    *ppChunk = pChunk;
-    return rc;
-}
-
-
-/**
- * For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
- *
- * @returns see pgmR3PhysChunkMap.
- * @param   pVM         The VM handle.
- * @param   idChunk     The chunk to map.
- */
-VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
-{
-    PPGMCHUNKR3MAP pChunk;
-    int rc;
-
-    pgmLock(pVM);
-    rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Invalidates the TLB for the ring-3 mapping cache.
- *
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
-{
-    pgmLock(pVM);
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
-    {
-        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
-        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
-    }
-    /* The page map TLB references chunks, so invalidate that one too. */
-    PGMPhysInvalidatePageMapTLB(pVM);
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_PAGE to allocate a large (2MB) page
- * for use with a nested paging PDE.
- *
- * @returns The following VBox status codes.
- * @retval  VINF_SUCCESS on success.
- * @retval  VINF_EM_NO_MEMORY if we're out of memory.
- *
- * @param   pVM         The VM handle.
- * @param   GCPhys      GC physical start address of the 2 MB range
- */
-VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
-{
-#ifdef PGM_WITH_LARGE_PAGES
-    uint64_t u64TimeStamp1, u64TimeStamp2;
-
-    pgmLock(pVM);
-
-    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
-    u64TimeStamp1 = RTTimeMilliTS();
-    int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
-    u64TimeStamp2 = RTTimeMilliTS();
-    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
-    if (RT_SUCCESS(rc))
-    {
-        Assert(pVM->pgm.s.cLargeHandyPages == 1);
-
-        uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
-        RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
-
-        void *pv;
-
-        /* Map the large page into our address space.
-         *
-         * Note: assuming that within the 2 MB range:
-         * - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
-         * - user space mapping is continuous as well
-         * - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
-         */
-        rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
-        AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
-
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Clear the pages.
-             */
-            STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
-            for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
-            {
-                ASMMemZeroPage(pv);
-
-                PPGMPAGE pPage;
-                rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
-                AssertRC(rc);
-
-                Assert(PGM_PAGE_IS_ZERO(pPage));
-                STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
-                pVM->pgm.s.cZeroPages--;
-
-                /*
-                 * Do the PGMPAGE modifications.
-                 */
-                pVM->pgm.s.cPrivatePages++;
-                PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
-                PGM_PAGE_SET_PAGEID(pPage, idPage);
-                PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
-                PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PDE);
-                PGM_PAGE_SET_PTE_INDEX(pPage, 0);
-                PGM_PAGE_SET_TRACKING(pPage, 0);
-
-                /* Somewhat dirty assumption that page ids are increasing. */
-                idPage++;
-
-                HCPhys += PAGE_SIZE;
-                GCPhys += PAGE_SIZE;
-
-                pv = (void *)((uintptr_t)pv + PAGE_SIZE);
-
-                Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
-            }
-            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
-
-            /* Flush all TLBs. */
-            PGM_INVL_ALL_VCPU_TLBS(pVM);
-            PGMPhysInvalidatePageMapTLB(pVM);
-        }
-        pVM->pgm.s.cLargeHandyPages = 0;
-    }
-
-    if (RT_SUCCESS(rc))
-    {
-        static uint32_t cTimeOut = 0;
-        uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
-
-        if (u64TimeStampDelta > 100)
-        {
-            STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
-            if (    ++cTimeOut > 10
-                ||  u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
-            {
-                /* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
-                 * E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
-                 */
-                LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
-                PGMSetLargePageUsage(pVM, false);
-            }
-        }
-        else
-        if (cTimeOut > 0)
-            cTimeOut--;
-    }
-
-    pgmUnlock(pVM);
-    return rc;
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif /* PGM_WITH_LARGE_PAGES */
-}
-
-
-/**
- * Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
- *
- * This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
- * signal and clear the out of memory condition. When contracted, this API is
- * used to try clear the condition when the user wants to resume.
- *
- * @returns The following VBox status codes.
- * @retval  VINF_SUCCESS on success. FFs cleared.
- * @retval  VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
- *          this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
- *
- * @param   pVM         The VM handle.
- *
- * @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
- *          in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
- *          pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
- *          handler.
- */
-VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
-{
-    pgmLock(pVM);
-
-    /*
-     * Allocate more pages, noting down the index of the first new page.
-     */
-    uint32_t iClear = pVM->pgm.s.cHandyPages;
-    AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_INTERNAL_ERROR);
-    Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
-    int rcAlloc = VINF_SUCCESS;
-    int rcSeed  = VINF_SUCCESS;
-    int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
-    while (rc == VERR_GMM_SEED_ME)
-    {
-        void *pvChunk;
-        rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
-        if (RT_SUCCESS(rc))
-        {
-            rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
-            if (RT_FAILURE(rc))
-                SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
-        }
-        if (RT_SUCCESS(rc))
-            rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
-    }
-
-    /* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
-    if (    rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
-        &&  pVM->pgm.s.cHandyPages > 0)
-    {
-        /* Still handy pages left, so don't panic. */
-        rc = VINF_SUCCESS;
-    }
-
-    if (RT_SUCCESS(rc))
-    {
-        AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        Assert(pVM->pgm.s.cHandyPages > 0);
-        VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
-        VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
-
-        /*
-         * Clear the pages.
-         */
-        while (iClear < pVM->pgm.s.cHandyPages)
-        {
-            PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
-            void *pv;
-            rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
-            AssertLogRelMsgBreak(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc));
-            ASMMemZeroPage(pv);
-            iClear++;
-            Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
-        }
-    }
-    else
-    {
-        uint64_t cAllocPages, cMaxPages, cBalloonPages;
-
-        /*
-         * We should never get here unless there is a genuine shortage of
-         * memory (or some internal error). Flag the error so the VM can be
-         * suspended ASAP and the user informed. If we're totally out of
-         * handy pages we will return failure.
-         */
-        /* Report the failure. */
-        LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
-                "     cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
-                rc, rcAlloc, rcSeed,
-                pVM->pgm.s.cHandyPages,
-                pVM->pgm.s.cAllPages,
-                pVM->pgm.s.cPrivatePages,
-                pVM->pgm.s.cSharedPages,
-                pVM->pgm.s.cZeroPages));
-
-        if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
-        {
-            LogRel(("GMM: Statistics:\n"
-                    "     Allocated pages: %RX64\n"
-                    "     Maximum   pages: %RX64\n"
-                    "     Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
-        }
-
-        if (    rc != VERR_NO_MEMORY
-            &&  rc != VERR_LOCK_FAILED)
-        {
-            for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
-            {
-                LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
-                        i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
-                        pVM->pgm.s.aHandyPages[i].idSharedPage));
-                uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
-                if (idPage != NIL_GMM_PAGEID)
-                {
-                    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
-                         pRam;
-                         pRam = pRam->pNextR3)
-                    {
-                        uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
-                        for (uint32_t iPage = 0; iPage < cPages; iPage++)
-                            if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
-                                LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
-                                        pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
-                    }
-                }
-            }
-        }
-
-        /* Set the FFs and adjust rc. */
-        VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
-        VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
-        if (    rc == VERR_NO_MEMORY
-            ||  rc == VERR_LOCK_FAILED)
-            rc = VINF_EM_NO_MEMORY;
-    }
-
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Frees the specified RAM page and replaces it with the ZERO page.
- *
- * This is used by ballooning, remapping MMIO2, RAM reset and state loading.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pReq            Pointer to the request.
- * @param   pcPendingPages  Where the number of pages waiting to be freed are
- *                          kept.  This will normally be incremented.
- * @param   pPage           Pointer to the page structure.
- * @param   GCPhys          The guest physical address of the page, if applicable.
- *
- * @remarks The caller must own the PGM lock.
- */
-int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
-{
-    /*
-     * Assert sanity.
-     */
-    Assert(PGMIsLockOwner(pVM));
-    if (RT_UNLIKELY(    PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
-                    &&  PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
-    {
-        AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
-        return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
-    }
-    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
-
-    if (    PGM_PAGE_IS_ZERO(pPage)
-        ||  PGM_PAGE_IS_BALLOONED(pPage))
-        return VINF_SUCCESS;
-
-    const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
-    Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
-    if (RT_UNLIKELY(    idPage == NIL_GMM_PAGEID
-                    ||  idPage > GMM_PAGEID_LAST
-                    ||  PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
-    {
-        AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
-        return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
-    }
-
-    /* update page count stats. */
-    if (PGM_PAGE_IS_SHARED(pPage))
-        pVM->pgm.s.cSharedPages--;
-    else
-        pVM->pgm.s.cPrivatePages--;
-    pVM->pgm.s.cZeroPages++;
-
-    /* Deal with write monitored pages. */
-    if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
-    {
-        PGM_PAGE_SET_WRITTEN_TO(pPage);
-        pVM->pgm.s.cWrittenToPages++;
-    }
-
-    /*
-     * pPage = ZERO page.
-     */
-    PGM_PAGE_SET_HCPHYS(pPage, pVM->pgm.s.HCPhysZeroPg);
-    PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
-    PGM_PAGE_SET_PAGEID(pPage, NIL_GMM_PAGEID);
-    PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
-    PGM_PAGE_SET_PTE_INDEX(pPage, 0);
-    PGM_PAGE_SET_TRACKING(pPage, 0);
-
-    /* Flush physical page map TLB entry. */
-    PGMPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
-
-    /*
-     * Make sure it's not in the handy page array.
-     */
-    for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
-    {
-        if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
-        {
-            pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
-            break;
-        }
-        if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
-        {
-            pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
-            break;
-        }
-    }
-
-    /*
-     * Push it onto the page array.
-     */
-    uint32_t iPage = *pcPendingPages;
-    Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
-    *pcPendingPages += 1;
-
-    pReq->aPages[iPage].idPage = idPage;
-
-    if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
-        return VINF_SUCCESS;
-
-    /*
-     * Flush the pages.
-     */
-    int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
-    if (RT_SUCCESS(rc))
-    {
-        GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
-        *pcPendingPages = 0;
-    }
-    return rc;
-}
-
-
-/**
- * Converts a GC physical address to a HC ring-3 pointer, with some
- * additional checks.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success.
- * @retval  VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
- *          access handler of some kind.
- * @retval  VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
- *          accesses or is odd in any way.
- * @retval  VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
- *
- * @param   pVM         The VM handle.
- * @param   GCPhys      The GC physical address to convert.
- * @param   fWritable   Whether write access is required.
- * @param   ppv         Where to store the pointer corresponding to GCPhys on
- *                      success.
- */
-VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
-{
-    pgmLock(pVM);
-
-    PPGMRAMRANGE pRam;
-    PPGMPAGE pPage;
-    int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
-    if (RT_SUCCESS(rc))
-    {
-        if (PGM_PAGE_IS_BALLOONED(pPage))
-            rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
-        else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
-            rc = VINF_SUCCESS;
-        else
-        {
-            if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
-                rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
-            else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
-            {
-                /** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
-                 *        in -norawr0 mode. */
-                if (fWritable)
-                    rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
-            }
-            else
-            {
-                /* Temporarily disabled physical handler(s), since the recompiler
-                   doesn't get notified when it's reset we'll have to pretend it's
-                   operating normally. */
-                if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
-                    rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
-                else
-                    rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
-            }
-        }
-        if (RT_SUCCESS(rc))
-        {
-            int rc2;
-
-            /* Make sure what we return is writable. */
-            if (fWritable)
-                switch (PGM_PAGE_GET_STATE(pPage))
-                {
-                    case PGM_PAGE_STATE_ALLOCATED:
-                        break;
-                    case PGM_PAGE_STATE_BALLOONED:
-                        AssertFailed();
-                        break;
-                    case PGM_PAGE_STATE_ZERO:
-                    case PGM_PAGE_STATE_SHARED:
-                        if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
-                            break;
-                    case PGM_PAGE_STATE_WRITE_MONITORED:
-                        rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
-                        AssertLogRelRCReturn(rc2, rc2);
-                        break;
-                }
-
-            /* Get a ring-3 mapping of the address. */
-            PPGMPAGER3MAPTLBE pTlbe;
-            rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
-            AssertLogRelRCReturn(rc2, rc2);
-            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
-            /** @todo mapping/locking hell; this isn't horribly efficient since
-             *        pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
-
-            Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
-        }
-        else
-            Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
-
-        /* else: handler catching all access, no pointer returned. */
-    }
-    else
-        rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
-
-    pgmUnlock(pVM);
-    return rc;
-}
-
Index: unk/src/VBox/VMM/PGMPhysRWTmpl.h
===================================================================
--- /trunk/src/VBox/VMM/PGMPhysRWTmpl.h	(revision 35332)
+++ 	(revision )
@@ -1,57 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager and Monitor, Physical Memory Access Template.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/**
- * Read physical memory. (one byte/word/dword)
- *
- * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
- * want to ignore those.
- *
- * @param   pVM             VM Handle.
- * @param   GCPhys          Physical address start reading from.
- */
-VMMDECL(PGMPHYS_DATATYPE) PGMPHYSFN_READNAME(PVM pVM, RTGCPHYS GCPhys)
-{
-    Assert(VM_IS_EMT(pVM));
-    PGMPHYS_DATATYPE val;
-    PGMPhysRead(pVM, GCPhys, &val, sizeof(val));
-    return val;
-}
-
-
-/**
- * Write to physical memory. (one byte/word/dword)
- *
- * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
- * want to ignore those.
- *
- * @param   pVM             VM Handle.
- * @param   GCPhys          Physical address to write to.
- * @param   val             What to write.
- */
-VMMDECL(void) PGMPHYSFN_WRITENAME(PVM pVM, RTGCPHYS GCPhys, PGMPHYS_DATATYPE val)
-{
-    Assert(VM_IS_EMT(pVM));
-    PGMPhysWrite(pVM, GCPhys, &val, sizeof(val));
-}
-
-#undef PGMPHYSFN_READNAME
-#undef PGMPHYSFN_WRITENAME
-#undef PGMPHYS_DATATYPE
-#undef PGMPHYS_DATASIZE
-
Index: unk/src/VBox/VMM/PGMPool.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGMPool.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1071 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM Shadow Page Pool.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_pgm_pool       PGM Shadow Page Pool
- *
- * Motivations:
- *      -# Relationship between shadow page tables and physical guest pages. This
- *         should allow us to skip most of the global flushes now following access
- *         handler changes. The main expense is flushing shadow pages.
- *      -# Limit the pool size if necessary (default is kind of limitless).
- *      -# Allocate shadow pages from RC. We use to only do this in SyncCR3.
- *      -# Required for 64-bit guests.
- *      -# Combining the PD cache and page pool in order to simplify caching.
- *
- *
- * @section sec_pgm_pool_outline    Design Outline
- *
- * The shadow page pool tracks pages used for shadowing paging structures (i.e.
- * page tables, page directory, page directory pointer table and page map
- * level-4). Each page in the pool has an unique identifier. This identifier is
- * used to link a guest physical page to a shadow PT. The identifier is a
- * non-zero value and has a relativly low max value - say 14 bits. This makes it
- * possible to fit it into the upper bits of the of the aHCPhys entries in the
- * ram range.
- *
- * By restricting host physical memory to the first 48 bits (which is the
- * announced physical memory range of the K8L chip (scheduled for 2008)), we
- * can safely use the upper 16 bits for shadow page ID and reference counting.
- *
- * Update: The 48 bit assumption will be lifted with the new physical memory
- * management (PGMPAGE), so we won't have any trouble when someone stuffs 2TB
- * into a box in some years.
- *
- * Now, it's possible for a page to be aliased, i.e. mapped by more than one PT
- * or PD. This is solved by creating a list of physical cross reference extents
- * when ever this happens. Each node in the list (extent) is can contain 3 page
- * pool indexes. The list it self is chained using indexes into the paPhysExt
- * array.
- *
- *
- * @section sec_pgm_pool_life       Life Cycle of a Shadow Page
- *
- * -# The SyncPT function requests a page from the pool.
- *    The request includes the kind of page it is (PT/PD, PAE/legacy), the
- *    address of the page it's shadowing, and more.
- * -# The pool responds to the request by allocating a new page.
- *    When the cache is enabled, it will first check if it's in the cache.
- *    Should the pool be exhausted, one of two things can be done:
- *      -# Flush the whole pool and current CR3.
- *      -# Use the cache to find a page which can be flushed (~age).
- * -# The SyncPT function will sync one or more pages and insert it into the
- *    shadow PD.
- * -# The SyncPage function may sync more pages on a later \#PFs.
- * -# The page is freed / flushed in SyncCR3 (perhaps) and some other cases.
- *    When caching is enabled, the page isn't flush but remains in the cache.
- *
- *
- * @section sec_pgm_pool_impl       Monitoring
- *
- * We always monitor PAGE_SIZE chunks of memory. When we've got multiple shadow
- * pages for the same PAGE_SIZE of guest memory (PAE and mixed PD/PT) the pages
- * sharing the monitor get linked using the iMonitoredNext/Prev. The head page
- * is the pvUser to the access handlers.
- *
- *
- * @section sec_pgm_pool_impl       Implementation
- *
- * The pool will take pages from the MM page pool. The tracking data
- * (attributes, bitmaps and so on) are allocated from the hypervisor heap. The
- * pool content can be accessed both by using the page id and the physical
- * address (HC). The former is managed by means of an array, the latter by an
- * offset based AVL tree.
- *
- * Flushing of a pool page means that we iterate the content (we know what kind
- * it is) and updates the link information in the ram range.
- *
- * ...
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM_POOL
-#include <VBox/pgm.h>
-#include <VBox/mm.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include "PGMInline.h"
-
-#include <VBox/log.h>
-#include <VBox/err.h>
-#include <iprt/asm.h>
-#include <iprt/string.h>
-#include <VBox/dbg.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
-#ifdef VBOX_WITH_DEBUGGER
-static DECLCALLBACK(int)  pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-#endif
-
-#ifdef VBOX_WITH_DEBUGGER
-/** Command descriptors. */
-static const DBGCCMD    g_aCmds[] =
-{
-    /* pszCmd,  cArgsMin, cArgsMax, paArgDesc,                cArgDescs,    pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
-    { "pgmpoolcheck",  0, 0,        NULL,                     0,            NULL,               0,          pgmR3PoolCmdCheck,  "",                 "Check the pgm pool pages." },
-};
-#endif
-
-/**
- * Initializes the pool
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-int pgmR3PoolInit(PVM pVM)
-{
-    AssertCompile(NIL_PGMPOOL_IDX == 0);
-    /* pPage->cLocked is an unsigned byte. */
-    AssertCompile(VMM_MAX_CPU_COUNT <= 255);
-
-    /*
-     * Query Pool config.
-     */
-    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM/Pool");
-
-    /* Default pgm pool size equals 1024 pages. */
-    uint16_t cMaxPages = 4*_1M >> PAGE_SHIFT;
-
-#if HC_ARCH_BITS == 64
-    uint64_t cbRam = 0;
-    CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
-
-    /* We should increase the pgm pool size for guests with more than 2 GB of ram */
-    if (cbRam >= UINT64_C(2) * _1G)
-    {
-        /* In the nested paging case we require 2 + 513 * (cbRam/1GB) pages to
-         * store all page table descriptors.
-         */
-        uint64_t u64MaxPages = cbRam  / (_1G / UINT64_C(512));
-        if (u64MaxPages > PGMPOOL_IDX_LAST)
-            cMaxPages = PGMPOOL_IDX_LAST;
-        else
-            cMaxPages = (uint16_t)u64MaxPages;
-    }
-#endif
-
-    /** @cfgm{/PGM/Pool/MaxPages, uint16_t, #pages, 16, 0x3fff, 1024}
-     * The max size of the shadow page pool in pages. The pool will grow dynamically
-     * up to this limit.
-     */
-    int rc = CFGMR3QueryU16Def(pCfg, "MaxPages", &cMaxPages, cMaxPages);
-    AssertLogRelRCReturn(rc, rc);
-    AssertLogRelMsgReturn(cMaxPages <= PGMPOOL_IDX_LAST && cMaxPages >= RT_ALIGN(PGMPOOL_IDX_FIRST, 16),
-                          ("cMaxPages=%u (%#x)\n", cMaxPages, cMaxPages), VERR_INVALID_PARAMETER);
-    cMaxPages = RT_ALIGN(cMaxPages, 16);
-
-    /** todo:
-     * We need to be much more careful with our allocation strategy here.
-     * For nested paging we don't need pool user info nor extents at all, but we can't check for nested paging here (too early during init to get a confirmation it can be used)
-     * The default for large memory configs is a bit large for shadow paging, so I've restricted the extent maximum to 2k (2k * 16 = 32k of hyper heap)
-     *
-     * Also when large page support is enabled, we typically don't need so much,
-     * although that depends on the availability of 2 MB chunks on the host.
-     */
-
-    /** @cfgm{/PGM/Pool/MaxUsers, uint16_t, #users, MaxUsers, 32K, MaxPages*2}
-     * The max number of shadow page user tracking records. Each shadow page has
-     * zero of other shadow pages (or CR3s) that references it, or uses it if you
-     * like. The structures describing these relationships are allocated from a
-     * fixed sized pool. This configuration variable defines the pool size.
-     */
-    uint16_t cMaxUsers;
-    rc = CFGMR3QueryU16Def(pCfg, "MaxUsers", &cMaxUsers, cMaxPages * 2);
-    AssertLogRelRCReturn(rc, rc);
-    AssertLogRelMsgReturn(cMaxUsers >= cMaxPages && cMaxPages <= _32K,
-                          ("cMaxUsers=%u (%#x)\n", cMaxUsers, cMaxUsers), VERR_INVALID_PARAMETER);
-
-    /** @cfgm{/PGM/Pool/MaxPhysExts, uint16_t, #extents, 16, MaxPages * 2, MAX(MaxPages*2,0x3fff)}
-     * The max number of extents for tracking aliased guest pages.
-     */
-    uint16_t cMaxPhysExts;
-    rc = CFGMR3QueryU16Def(pCfg, "MaxPhysExts", &cMaxPhysExts, RT_MAX(cMaxPages * 2, 2048 /* 2k max as this eat too much hyper heap */));
-    AssertLogRelRCReturn(rc, rc);
-    AssertLogRelMsgReturn(cMaxPhysExts >= 16 && cMaxPages <= PGMPOOL_IDX_LAST,
-                          ("cMaxPhysExts=%u (%#x)\n", cMaxPhysExts, cMaxPhysExts), VERR_INVALID_PARAMETER);
-
-    /** @cfgm{/PGM/Pool/ChacheEnabled, bool, true}
-     * Enables or disabling caching of shadow pages. Caching means that we will try
-     * reuse shadow pages instead of recreating them everything SyncCR3, SyncPT or
-     * SyncPage requests one. When reusing a shadow page, we can save time
-     * reconstructing it and it's children.
-     */
-    bool fCacheEnabled;
-    rc = CFGMR3QueryBoolDef(pCfg, "CacheEnabled", &fCacheEnabled, true);
-    AssertLogRelRCReturn(rc, rc);
-
-    LogRel(("pgmR3PoolInit: cMaxPages=%#RX16 cMaxUsers=%#RX16 cMaxPhysExts=%#RX16 fCacheEnable=%RTbool\n",
-             cMaxPages, cMaxUsers, cMaxPhysExts, fCacheEnabled));
-
-    /*
-     * Allocate the data structures.
-     */
-    uint32_t cb = RT_OFFSETOF(PGMPOOL, aPages[cMaxPages]);
-    cb += cMaxUsers * sizeof(PGMPOOLUSER);
-    cb += cMaxPhysExts * sizeof(PGMPOOLPHYSEXT);
-    PPGMPOOL pPool;
-    rc = MMR3HyperAllocOnceNoRel(pVM, cb, 0, MM_TAG_PGM_POOL, (void **)&pPool);
-    if (RT_FAILURE(rc))
-        return rc;
-    pVM->pgm.s.pPoolR3 = pPool;
-    pVM->pgm.s.pPoolR0 = MMHyperR3ToR0(pVM, pPool);
-    pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pPool);
-
-    /*
-     * Initialize it.
-     */
-    pPool->pVMR3     = pVM;
-    pPool->pVMR0     = pVM->pVMR0;
-    pPool->pVMRC     = pVM->pVMRC;
-    pPool->cMaxPages = cMaxPages;
-    pPool->cCurPages = PGMPOOL_IDX_FIRST;
-    pPool->iUserFreeHead = 0;
-    pPool->cMaxUsers = cMaxUsers;
-    PPGMPOOLUSER paUsers = (PPGMPOOLUSER)&pPool->aPages[pPool->cMaxPages];
-    pPool->paUsersR3 = paUsers;
-    pPool->paUsersR0 = MMHyperR3ToR0(pVM, paUsers);
-    pPool->paUsersRC = MMHyperR3ToRC(pVM, paUsers);
-    for (unsigned i = 0; i < cMaxUsers; i++)
-    {
-        paUsers[i].iNext = i + 1;
-        paUsers[i].iUser = NIL_PGMPOOL_IDX;
-        paUsers[i].iUserTable = 0xfffffffe;
-    }
-    paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
-    pPool->iPhysExtFreeHead = 0;
-    pPool->cMaxPhysExts = cMaxPhysExts;
-    PPGMPOOLPHYSEXT paPhysExts = (PPGMPOOLPHYSEXT)&paUsers[cMaxUsers];
-    pPool->paPhysExtsR3 = paPhysExts;
-    pPool->paPhysExtsR0 = MMHyperR3ToR0(pVM, paPhysExts);
-    pPool->paPhysExtsRC = MMHyperR3ToRC(pVM, paPhysExts);
-    for (unsigned i = 0; i < cMaxPhysExts; i++)
-    {
-        paPhysExts[i].iNext = i + 1;
-        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
-        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
-        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
-        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
-        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
-        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
-    }
-    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
-    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
-        pPool->aiHash[i] = NIL_PGMPOOL_IDX;
-    pPool->iAgeHead = NIL_PGMPOOL_IDX;
-    pPool->iAgeTail = NIL_PGMPOOL_IDX;
-    pPool->fCacheEnabled = fCacheEnabled;
-    pPool->pfnAccessHandlerR3 = pgmR3PoolAccessHandler;
-    pPool->pszAccessHandler = "Guest Paging Access Handler";
-    pPool->HCPhysTree = 0;
-
-    /* The NIL entry. */
-    Assert(NIL_PGMPOOL_IDX == 0);
-    pPool->aPages[NIL_PGMPOOL_IDX].enmKind = PGMPOOLKIND_INVALID;
-
-    /* The Shadow 32-bit PD. (32 bits guest paging) */
-    pPool->aPages[PGMPOOL_IDX_PD].Core.Key  = NIL_RTHCPHYS;
-    pPool->aPages[PGMPOOL_IDX_PD].GCPhys    = NIL_RTGCPHYS;
-    pPool->aPages[PGMPOOL_IDX_PD].pvPageR3  = 0;
-    pPool->aPages[PGMPOOL_IDX_PD].enmKind   = PGMPOOLKIND_32BIT_PD;
-    pPool->aPages[PGMPOOL_IDX_PD].idx       = PGMPOOL_IDX_PD;
-
-    /* The Shadow PDPT. */
-    pPool->aPages[PGMPOOL_IDX_PDPT].Core.Key  = NIL_RTHCPHYS;
-    pPool->aPages[PGMPOOL_IDX_PDPT].GCPhys    = NIL_RTGCPHYS;
-    pPool->aPages[PGMPOOL_IDX_PDPT].pvPageR3  = 0;
-    pPool->aPages[PGMPOOL_IDX_PDPT].enmKind   = PGMPOOLKIND_PAE_PDPT;
-    pPool->aPages[PGMPOOL_IDX_PDPT].idx       = PGMPOOL_IDX_PDPT;
-
-    /* The Shadow AMD64 CR3. */
-    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].Core.Key  = NIL_RTHCPHYS;
-    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].GCPhys    = NIL_RTGCPHYS;
-    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].pvPageR3  = 0;
-    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].enmKind   = PGMPOOLKIND_64BIT_PML4;
-    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].idx       = PGMPOOL_IDX_AMD64_CR3;
-
-    /* The Nested Paging CR3. */
-    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].Core.Key  = NIL_RTHCPHYS;
-    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].GCPhys    = NIL_RTGCPHYS;
-    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].pvPageR3  = 0;
-    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].enmKind   = PGMPOOLKIND_ROOT_NESTED;
-    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].idx       = PGMPOOL_IDX_NESTED_ROOT;
-
-    /*
-     * Set common stuff.
-     */
-    for (unsigned iPage = 1; iPage < PGMPOOL_IDX_FIRST; iPage++)
-    {
-        pPool->aPages[iPage].iNext          = NIL_PGMPOOL_IDX;
-        pPool->aPages[iPage].iUserHead      = NIL_PGMPOOL_USER_INDEX;
-        pPool->aPages[iPage].iModifiedNext  = NIL_PGMPOOL_IDX;
-        pPool->aPages[iPage].iModifiedPrev  = NIL_PGMPOOL_IDX;
-        pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
-        pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
-        pPool->aPages[iPage].iAgeNext       = NIL_PGMPOOL_IDX;
-        pPool->aPages[iPage].iAgePrev       = NIL_PGMPOOL_IDX;
-        Assert(pPool->aPages[iPage].idx == iPage);
-        Assert(pPool->aPages[iPage].GCPhys == NIL_RTGCPHYS);
-        Assert(!pPool->aPages[iPage].fSeenNonGlobal);
-        Assert(!pPool->aPages[iPage].fMonitored);
-        Assert(!pPool->aPages[iPage].fCached);
-        Assert(!pPool->aPages[iPage].fZeroed);
-        Assert(!pPool->aPages[iPage].fReusedFlushPending);
-    }
-
-#ifdef VBOX_WITH_STATISTICS
-    /*
-     * Register statistics.
-     */
-    STAM_REG(pVM, &pPool->cCurPages,                    STAMTYPE_U16,       "/PGM/Pool/cCurPages",      STAMUNIT_PAGES,             "Current pool size.");
-    STAM_REG(pVM, &pPool->cMaxPages,                    STAMTYPE_U16,       "/PGM/Pool/cMaxPages",      STAMUNIT_PAGES,             "Max pool size.");
-    STAM_REG(pVM, &pPool->cUsedPages,                   STAMTYPE_U16,       "/PGM/Pool/cUsedPages",     STAMUNIT_PAGES,             "The number of pages currently in use.");
-    STAM_REG(pVM, &pPool->cUsedPagesHigh,               STAMTYPE_U16_RESET, "/PGM/Pool/cUsedPagesHigh", STAMUNIT_PAGES,             "The high watermark for cUsedPages.");
-    STAM_REG(pVM, &pPool->StatAlloc,                  STAMTYPE_PROFILE_ADV, "/PGM/Pool/Alloc",          STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolAlloc.");
-    STAM_REG(pVM, &pPool->StatClearAll,                 STAMTYPE_PROFILE,   "/PGM/Pool/ClearAll",       STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmR3PoolClearAll.");
-    STAM_REG(pVM, &pPool->StatR3Reset,                  STAMTYPE_PROFILE,   "/PGM/Pool/R3Reset",        STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmR3PoolReset.");
-    STAM_REG(pVM, &pPool->StatFlushPage,                STAMTYPE_PROFILE,   "/PGM/Pool/FlushPage",      STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolFlushPage.");
-    STAM_REG(pVM, &pPool->StatFree,                     STAMTYPE_PROFILE,   "/PGM/Pool/Free",           STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolFree.");
-    STAM_REG(pVM, &pPool->StatForceFlushPage,           STAMTYPE_COUNTER,   "/PGM/Pool/FlushForce",     STAMUNIT_OCCURENCES,        "Counting explicit flushes by PGMPoolFlushPage().");
-    STAM_REG(pVM, &pPool->StatForceFlushDirtyPage,      STAMTYPE_COUNTER,   "/PGM/Pool/FlushForceDirty",     STAMUNIT_OCCURENCES,   "Counting explicit flushes of dirty pages by PGMPoolFlushPage().");
-    STAM_REG(pVM, &pPool->StatForceFlushReused,         STAMTYPE_COUNTER,   "/PGM/Pool/FlushReused",    STAMUNIT_OCCURENCES,        "Counting flushes for reused pages.");
-    STAM_REG(pVM, &pPool->StatZeroPage,                 STAMTYPE_PROFILE,   "/PGM/Pool/ZeroPage",       STAMUNIT_TICKS_PER_CALL,    "Profiling time spent zeroing pages. Overlaps with Alloc.");
-    STAM_REG(pVM, &pPool->cMaxUsers,                    STAMTYPE_U16,       "/PGM/Pool/Track/cMaxUsers",            STAMUNIT_COUNT,      "Max user tracking records.");
-    STAM_REG(pVM, &pPool->cPresent,                     STAMTYPE_U32,       "/PGM/Pool/Track/cPresent",             STAMUNIT_COUNT,      "Number of present page table entries.");
-    STAM_REG(pVM, &pPool->StatTrackDeref,               STAMTYPE_PROFILE,   "/PGM/Pool/Track/Deref",                STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackDeref.");
-    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPT,       STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPT",        STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPT.");
-    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTs,      STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPTs",       STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTs.");
-    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTsSlow,  STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPTsSlow",   STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTsSlow.");
-    STAM_REG(pVM, &pPool->StatTrackFlushEntry,          STAMTYPE_COUNTER,   "/PGM/Pool/Track/Entry/Flush",          STAMUNIT_COUNT,          "Nr of flushed entries.");
-    STAM_REG(pVM, &pPool->StatTrackFlushEntryKeep,      STAMTYPE_COUNTER,   "/PGM/Pool/Track/Entry/Update",         STAMUNIT_COUNT,          "Nr of updated entries.");
-    STAM_REG(pVM, &pPool->StatTrackFreeUpOneUser,       STAMTYPE_COUNTER,   "/PGM/Pool/Track/FreeUpOneUser",        STAMUNIT_TICKS_PER_CALL, "The number of times we were out of user tracking records.");
-    STAM_REG(pVM, &pPool->StatTrackDerefGCPhys,         STAMTYPE_PROFILE,   "/PGM/Pool/Track/DrefGCPhys",           STAMUNIT_TICKS_PER_CALL, "Profiling deref activity related tracking GC physical pages.");
-    STAM_REG(pVM, &pPool->StatTrackLinearRamSearches,   STAMTYPE_COUNTER,   "/PGM/Pool/Track/LinearRamSearches",    STAMUNIT_OCCURENCES, "The number of times we had to do linear ram searches.");
-    STAM_REG(pVM, &pPool->StamTrackPhysExtAllocFailures,STAMTYPE_COUNTER,   "/PGM/Pool/Track/PhysExtAllocFailures", STAMUNIT_OCCURENCES, "The number of failing pgmPoolTrackPhysExtAlloc calls.");
-    STAM_REG(pVM, &pPool->StatMonitorRZ,                STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorRZEmulateInstr,    STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFlushPage,       STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFlushReinit,     STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFlushModOverflow,STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFork,            STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
-    STAM_REG(pVM, &pPool->StatMonitorRZHandled,         STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access we've handled (except REP STOSD).");
-    STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch1,  STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/IntrFailPatch1",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction.");
-    STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch2,  STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/IntrFailPatch2",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction during flushing.");
-    STAM_REG(pVM, &pPool->StatMonitorRZRepPrefix,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
-    STAM_REG(pVM, &pPool->StatMonitorRZRepStosd,        STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPT,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PT",        STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPD,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PD",        STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPDPT,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PDPT",      STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
-    STAM_REG(pVM, &pPool->StatMonitorRZFaultPML4,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PML4",      STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
-    STAM_REG(pVM, &pPool->StatMonitorR3,                STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3",                 STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorR3EmulateInstr,    STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FlushPage,       STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the R3 access handler.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FlushReinit,     STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FlushModOverflow,STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
-    STAM_REG(pVM, &pPool->StatMonitorR3Fork,            STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
-    STAM_REG(pVM, &pPool->StatMonitorR3Handled,         STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access we've handled (except REP STOSD).");
-    STAM_REG(pVM, &pPool->StatMonitorR3RepPrefix,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
-    STAM_REG(pVM, &pPool->StatMonitorR3RepStosd,        STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FaultPT,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PT",        STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FaultPD,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PD",        STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FaultPDPT,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PDPT",      STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
-    STAM_REG(pVM, &pPool->StatMonitorR3FaultPML4,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PML4",      STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
-    STAM_REG(pVM, &pPool->StatMonitorR3Async,           STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Async",           STAMUNIT_OCCURENCES,     "Times we're called in an async thread and need to flush.");
-    STAM_REG(pVM, &pPool->cModifiedPages,               STAMTYPE_U16,       "/PGM/Pool/Monitor/cModifiedPages",     STAMUNIT_PAGES,          "The current cModifiedPages value.");
-    STAM_REG(pVM, &pPool->cModifiedPagesHigh,           STAMTYPE_U16_RESET, "/PGM/Pool/Monitor/cModifiedPagesHigh", STAMUNIT_PAGES,          "The high watermark for cModifiedPages.");
-    STAM_REG(pVM, &pPool->StatResetDirtyPages,          STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/Resets",       STAMUNIT_OCCURENCES,     "Times we've called pgmPoolResetDirtyPages (and there were dirty page).");
-    STAM_REG(pVM, &pPool->StatDirtyPage,                STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/Pages",        STAMUNIT_OCCURENCES,     "Times we've called pgmPoolAddDirtyPage.");
-    STAM_REG(pVM, &pPool->StatDirtyPageDupFlush,        STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/FlushDup",     STAMUNIT_OCCURENCES,     "Times we've had to flush duplicates for dirty page management.");
-    STAM_REG(pVM, &pPool->StatDirtyPageOverFlowFlush,   STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/FlushOverflow",STAMUNIT_OCCURENCES,     "Times we've had to flush because of overflow.");
-    STAM_REG(pVM, &pPool->StatCacheHits,                STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Hits",                 STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls satisfied by the cache.");
-    STAM_REG(pVM, &pPool->StatCacheMisses,              STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Misses",               STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls not statisfied by the cache.");
-    STAM_REG(pVM, &pPool->StatCacheKindMismatches,      STAMTYPE_COUNTER,   "/PGM/Pool/Cache/KindMismatches",       STAMUNIT_OCCURENCES, "The number of shadow page kind mismatches. (Better be low, preferably 0!)");
-    STAM_REG(pVM, &pPool->StatCacheFreeUpOne,           STAMTYPE_COUNTER,   "/PGM/Pool/Cache/FreeUpOne",            STAMUNIT_OCCURENCES, "The number of times the cache was asked to free up a page.");
-    STAM_REG(pVM, &pPool->StatCacheCacheable,           STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Cacheable",            STAMUNIT_OCCURENCES, "The number of cacheable allocations.");
-    STAM_REG(pVM, &pPool->StatCacheUncacheable,         STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Uncacheable",          STAMUNIT_OCCURENCES, "The number of uncacheable allocations.");
-#endif /* VBOX_WITH_STATISTICS */
-
-#ifdef VBOX_WITH_DEBUGGER
-    /*
-     * Debugger commands.
-     */
-    static bool s_fRegisteredCmds = false;
-    if (!s_fRegisteredCmds)
-    {
-        rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
-        if (RT_SUCCESS(rc))
-            s_fRegisteredCmds = true;
-    }
-#endif
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Relocate the page pool data.
- *
- * @param   pVM     The VM handle.
- */
-void pgmR3PoolRelocate(PVM pVM)
-{
-    pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3);
-    pVM->pgm.s.pPoolR3->pVMRC = pVM->pVMRC;
-    pVM->pgm.s.pPoolR3->paUsersRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paUsersR3);
-    pVM->pgm.s.pPoolR3->paPhysExtsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paPhysExtsR3);
-    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerRC);
-    AssertReleaseRC(rc);
-    /* init order hack. */
-    if (!pVM->pgm.s.pPoolR3->pfnAccessHandlerR0)
-    {
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerR0);
-        AssertReleaseRC(rc);
-    }
-}
-
-
-/**
- * Grows the shadow page pool.
- *
- * I.e. adds more pages to it, assuming that hasn't reached cMaxPages yet.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-VMMR3DECL(int) PGMR3PoolGrow(PVM pVM)
-{
-    PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
-    AssertReturn(pPool->cCurPages < pPool->cMaxPages, VERR_INTERNAL_ERROR);
-
-    pgmLock(pVM);
-
-    /*
-     * How much to grow it by?
-     */
-    uint32_t cPages = pPool->cMaxPages - pPool->cCurPages;
-    cPages = RT_MIN(PGMPOOL_CFG_MAX_GROW, cPages);
-    LogFlow(("PGMR3PoolGrow: Growing the pool by %d (%#x) pages.\n", cPages, cPages));
-
-    for (unsigned i = pPool->cCurPages; cPages-- > 0; i++)
-    {
-        PPGMPOOLPAGE pPage = &pPool->aPages[i];
-
-        /* Allocate all pages in low (below 4 GB) memory as 32 bits guests need a page table root in low memory. */
-        pPage->pvPageR3 = MMR3PageAllocLow(pVM);
-        if (!pPage->pvPageR3)
-        {
-            Log(("We're out of memory!! i=%d\n", i));
-            pgmUnlock(pVM);
-            return i ? VINF_SUCCESS : VERR_NO_PAGE_MEMORY;
-        }
-        pPage->Core.Key  = MMPage2Phys(pVM, pPage->pvPageR3);
-        AssertFatal(pPage->Core.Key < _4G);
-        pPage->GCPhys    = NIL_RTGCPHYS;
-        pPage->enmKind   = PGMPOOLKIND_FREE;
-        pPage->idx       = pPage - &pPool->aPages[0];
-        LogFlow(("PGMR3PoolGrow: insert page #%#x - %RHp\n", pPage->idx, pPage->Core.Key));
-        pPage->iNext     = pPool->iFreeHead;
-        pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
-        pPage->iModifiedNext  = NIL_PGMPOOL_IDX;
-        pPage->iModifiedPrev  = NIL_PGMPOOL_IDX;
-        pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
-        pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
-        pPage->iAgeNext  = NIL_PGMPOOL_IDX;
-        pPage->iAgePrev  = NIL_PGMPOOL_IDX;
-        /* commit it */
-        bool fRc = RTAvloHCPhysInsert(&pPool->HCPhysTree, &pPage->Core); Assert(fRc); NOREF(fRc);
-        pPool->iFreeHead = i;
-        pPool->cCurPages = i + 1;
-    }
-
-    pgmUnlock(pVM);
-    Assert(pPool->cCurPages <= pPool->cMaxPages);
-    return VINF_SUCCESS;
-}
-
-
-
-/**
- * Worker used by pgmR3PoolAccessHandler when it's invoked by an async thread.
- *
- * @param   pPool   The pool.
- * @param   pPage   The page.
- */
-static DECLCALLBACK(void) pgmR3PoolFlushReusedPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
-{
-    /* for the present this should be safe enough I think... */
-    pgmLock(pPool->pVMR3);
-    if (    pPage->fReusedFlushPending
-        &&  pPage->enmKind != PGMPOOLKIND_FREE)
-        pgmPoolFlushPage(pPool, pPage);
-    pgmUnlock(pPool->pVMR3);
-}
-
-
-/**
- * \#PF Handler callback for PT write accesses.
- *
- * The handler can not raise any faults, it's mainly for monitoring write access
- * to certain pages.
- *
- * @returns VINF_SUCCESS if the handler has carried out the operation.
- * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
- * @param   pVM             VM Handle.
- * @param   GCPhys          The physical address the guest is writing to.
- * @param   pvPhys          The HC mapping of that address.
- * @param   pvBuf           What the guest is reading/writing.
- * @param   cbBuf           How much it's reading/writing.
- * @param   enmAccessType   The access type.
- * @param   pvUser          User argument.
- */
-static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
-{
-    STAM_PROFILE_START(&pVM->pgm.s.pPoolR3->StatMonitorR3, a);
-    PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
-    PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
-    LogFlow(("pgmR3PoolAccessHandler: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
-             GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
-
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    /*
-     * We don't have to be very sophisticated about this since there are relativly few calls here.
-     * However, we must try our best to detect any non-cpu accesses (disk / networking).
-     *
-     * Just to make life more interesting, we'll have to deal with the async threads too.
-     * We cannot flush a page if we're in an async thread because of REM notifications.
-     */
-    pgmLock(pVM);
-    if (PHYS_PAGE_ADDRESS(GCPhys) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
-    {
-        /* Pool page changed while we were waiting for the lock; ignore. */
-        Log(("CPU%d: pgmR3PoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
-        pgmUnlock(pVM);
-        return VINF_PGM_HANDLER_DO_DEFAULT;
-    }
-
-    Assert(pPage->enmKind != PGMPOOLKIND_FREE);
-
-    /* @todo this code doesn't make any sense. remove the if (!pVCpu) block */
-    if (!pVCpu) /** @todo This shouldn't happen any longer, all access handlers will be called on an EMT. All ring-3 handlers, except MMIO, already own the PGM lock. @bugref{3170} */
-    {
-        Log(("pgmR3PoolAccessHandler: async thread, requesting EMT to flush the page: %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
-             pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
-        STAM_COUNTER_INC(&pPool->StatMonitorR3Async);
-        if (!pPage->fReusedFlushPending)
-        {
-            pgmUnlock(pVM);
-            int rc = VMR3ReqCallVoidNoWait(pPool->pVMR3, VMCPUID_ANY, (PFNRT)pgmR3PoolFlushReusedPage, 2, pPool, pPage);
-            AssertRCReturn(rc, rc);
-            pgmLock(pVM);
-            pPage->fReusedFlushPending = true;
-            pPage->cModifications += 0x1000;
-        }
-
-        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
-        /** @todo r=bird: making unsafe assumption about not crossing entries here! */
-        while (cbBuf > 4)
-        {
-            cbBuf -= 4;
-            pvPhys = (uint8_t *)pvPhys + 4;
-            GCPhys += 4;
-            pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
-        }
-        STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
-    }
-    else if (    (   pPage->cModifications < 96 /* it's cheaper here. */
-                  || pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
-                  )
-             &&  cbBuf <= 4)
-    {
-        /* Clear the shadow entry. */
-        if (!pPage->cModifications++)
-            pgmPoolMonitorModifiedInsert(pPool, pPage);
-        /** @todo r=bird: making unsafe assumption about not crossing entries here! */
-        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
-        STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
-    }
-    else
-    {
-        pgmPoolMonitorChainFlush(pPool, pPage); /* ASSUME that VERR_PGM_POOL_CLEARED can be ignored here and that FFs will deal with it in due time. */
-        STAM_PROFILE_STOP_EX(&pPool->StatMonitorR3, &pPool->StatMonitorR3FlushPage, a);
-    }
-    pgmUnlock(pVM);
-    return VINF_PGM_HANDLER_DO_DEFAULT;
-}
-
-
-/**
- * Rendezvous callback used by pgmR3PoolClearAll that clears all shadow pages
- * and all modification counters.
- *
- * This is only called on one of the EMTs while the other ones are waiting for
- * it to complete this function.
- *
- * @returns VINF_SUCCESS (VBox strict status code).
- * @param   pVM     The VM handle.
- * @param   pVCpu   The VMCPU for the EMT we're being called on. Unused.
- * @param   fpvFlushRemTlb  When not NULL, we'll flush the REM TLB as well.
- *                          (This is the pvUser, so it has to be void *.)
- *
- */
-DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl)
-{
-    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
-    STAM_PROFILE_START(&pPool->StatClearAll, c);
-
-    pgmLock(pVM);
-    Log(("pgmR3PoolClearAllRendezvous: cUsedPages=%d fpvFlushRemTbl=%RTbool\n", pPool->cUsedPages, !!fpvFlushRemTbl));
-
-    /*
-     * Iterate all the pages until we've encountered all that are in use.
-     * This is a simple but not quite optimal solution.
-     */
-    unsigned cModifiedPages = 0; NOREF(cModifiedPages);
-    unsigned cLeft = pPool->cUsedPages;
-    unsigned iPage = pPool->cCurPages;
-    while (--iPage >= PGMPOOL_IDX_FIRST)
-    {
-        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
-        if (pPage->GCPhys != NIL_RTGCPHYS)
-        {
-            switch (pPage->enmKind)
-            {
-                /*
-                 * We only care about shadow page tables that reference physical memory
-                 */
-#ifdef PGM_WITH_LARGE_PAGES
-                case PGMPOOLKIND_EPT_PD_FOR_PHYS: /* Large pages reference 2 MB of physical memory, so we must clear them. */
-                    if (pPage->cPresent)
-                    {
-                        PX86PDPAE pShwPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
-                        for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
-                        {
-                            if (    pShwPD->a[i].n.u1Present
-                                &&  pShwPD->a[i].b.u1Size)
-                            {
-                                Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
-                                pShwPD->a[i].u = 0;
-                                Assert(pPage->cPresent);
-                                pPage->cPresent--;
-                            }
-                        }
-                        if (pPage->cPresent == 0)
-                            pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
-                    }
-                    goto default_case;
-
-                case PGMPOOLKIND_PAE_PD_PHYS:   /* Large pages reference 2 MB of physical memory, so we must clear them. */
-                    if (pPage->cPresent)
-                    {
-                        PEPTPD pShwPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
-                        for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
-                        {
-                            Assert((pShwPD->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
-                            if (    pShwPD->a[i].n.u1Present
-                                &&  pShwPD->a[i].b.u1Size)
-                            {
-                                Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
-                                pShwPD->a[i].u = 0;
-                                Assert(pPage->cPresent);
-                                pPage->cPresent--;
-                            }
-                        }
-                        if (pPage->cPresent == 0)
-                            pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
-                    }
-                    goto default_case;
-#endif /* PGM_WITH_LARGE_PAGES */
-
-                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
-                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
-                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
-                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
-                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
-                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
-                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
-                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
-                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
-                {
-                    if (pPage->cPresent)
-                    {
-                        void *pvShw = PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
-                        STAM_PROFILE_START(&pPool->StatZeroPage, z);
-#if 0
-                        /* Useful check for leaking references; *very* expensive though. */
-                        switch (pPage->enmKind)
-                        {
-                            case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
-                            case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
-                            case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
-                            case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
-                            case PGMPOOLKIND_PAE_PT_FOR_PHYS:
-                            {
-                                bool fFoundFirst = false;
-                                PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)pvShw;
-                                for (unsigned ptIndex = 0; ptIndex < RT_ELEMENTS(pPT->a); ptIndex++)
-                                {
-                                    if (pPT->a[ptIndex].u)
-                                    {
-                                        if (!fFoundFirst)
-                                        {
-                                            AssertFatalMsg(pPage->iFirstPresent <= ptIndex, ("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
-                                            if (pPage->iFirstPresent != ptIndex)
-                                                Log(("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
-                                            fFoundFirst = true;
-                                        }
-                                        if (PGMSHWPTEPAE_IS_P(pPT->a[ptIndex]))
-                                        {
-                                            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pPT->a[ptIndex]), NIL_RTGCPHYS);
-                                            if (pPage->iFirstPresent == ptIndex)
-                                                pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
-                                        }
-                                    }
-                                }
-                                AssertFatalMsg(pPage->cPresent == 0, ("cPresent = %d pPage = %RGv\n", pPage->cPresent, pPage->GCPhys));
-                                break;
-                            }
-                            default:
-                                break;
-                        }
-#endif
-                        ASMMemZeroPage(pvShw);
-                        STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
-                        pPage->cPresent = 0;
-                        pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
-                    }
-                }
-                /* fall thru */
-
-#ifdef PGM_WITH_LARGE_PAGES
-                default_case:
-#endif
-                default:
-                    Assert(!pPage->cModifications || ++cModifiedPages);
-                    Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
-                    Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
-                    pPage->iModifiedNext = NIL_PGMPOOL_IDX;
-                    pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
-                    pPage->cModifications = 0;
-                    break;
-
-            }
-            if (!--cLeft)
-                break;
-        }
-    }
-
-    /* swipe the special pages too. */
-    for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
-    {
-        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
-        if (pPage->GCPhys != NIL_RTGCPHYS)
-        {
-            Assert(!pPage->cModifications || ++cModifiedPages);
-            Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
-            Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
-            pPage->iModifiedNext = NIL_PGMPOOL_IDX;
-            pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
-            pPage->cModifications = 0;
-        }
-    }
-
-#ifndef DEBUG_michael
-    AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
-#endif
-    pPool->iModifiedHead = NIL_PGMPOOL_IDX;
-    pPool->cModifiedPages = 0;
-
-    /*
-     * Clear all the GCPhys links and rebuild the phys ext free list.
-     */
-    for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
-         pRam;
-         pRam = pRam->CTX_SUFF(pNext))
-    {
-        iPage = pRam->cb >> PAGE_SHIFT;
-        while (iPage-- > 0)
-            PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
-    }
-
-    pPool->iPhysExtFreeHead = 0;
-    PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
-    const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
-    for (unsigned i = 0; i < cMaxPhysExts; i++)
-    {
-        paPhysExts[i].iNext = i + 1;
-        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
-        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
-        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
-        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
-        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
-        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
-    }
-    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
-
-
-#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
-    /* Reset all dirty pages to reactivate the page monitoring. */
-    /* Note: we must do this *after* clearing all page references and shadow page tables as there might be stale references to
-     *       recently removed MMIO ranges around that might otherwise end up asserting in pgmPoolTracDerefGCPhysHint
-     */
-    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
-    {
-        PPGMPOOLPAGE pPage;
-        unsigned     idxPage;
-
-        if (pPool->aDirtyPages[i].uIdx == NIL_PGMPOOL_IDX)
-            continue;
-
-        idxPage = pPool->aDirtyPages[i].uIdx;
-        AssertRelease(idxPage != NIL_PGMPOOL_IDX);
-        pPage = &pPool->aPages[idxPage];
-        Assert(pPage->idx == idxPage);
-        Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
-
-        AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, i));
-
-        Log(("Reactivate dirty page %RGp\n", pPage->GCPhys));
-
-        /* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
-        int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
-        Assert(rc == VINF_SUCCESS);
-        pPage->fDirty = false;
-
-        pPool->aDirtyPages[i].uIdx = NIL_PGMPOOL_IDX;
-    }
-
-    /* Clear all dirty pages. */
-    pPool->idxFreeDirtyPage = 0;
-    pPool->cDirtyPages      = 0;
-#endif
-
-    /* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-        pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
-
-    /* Flush job finished. */
-    VM_FF_CLEAR(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
-    pPool->cPresent = 0;
-    pgmUnlock(pVM);
-
-    PGM_INVL_ALL_VCPU_TLBS(pVM);
-
-    if (fpvFlushRemTbl)
-        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-            CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
-
-    STAM_PROFILE_STOP(&pPool->StatClearAll, c);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Clears the shadow page pool.
- *
- * @param   pVM             The VM handle.
- * @param   fFlushRemTlb    When set, the REM TLB is scheduled for flushing as
- *                          well.
- */
-void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb)
-{
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PoolClearAllRendezvous, &fFlushRemTlb);
-    AssertRC(rc);
-}
-
-/**
- * Protect all pgm pool page table entries to monitor writes
- *
- * @param   pVM         The VM handle.
- *
- * Remark: assumes the caller will flush all TLBs (!!)
- */
-void pgmR3PoolWriteProtectPages(PVM pVM)
-{
-    Assert(PGMIsLockOwner(pVM));
-    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
-    unsigned cLeft = pPool->cUsedPages;
-    unsigned iPage = pPool->cCurPages;
-    while (--iPage >= PGMPOOL_IDX_FIRST)
-    {
-        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
-        if (    pPage->GCPhys != NIL_RTGCPHYS
-            &&  pPage->cPresent)
-        {
-            union
-            {
-                void           *pv;
-                PX86PT          pPT;
-                PPGMSHWPTPAE    pPTPae;
-                PEPTPT          pPTEpt;
-            } uShw;
-            uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
-
-            switch (pPage->enmKind)
-            {
-                /*
-                 * We only care about shadow page tables.
-                 */
-                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
-                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
-                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
-                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPT->a); iShw++)
-                    {
-                        if (uShw.pPT->a[iShw].n.u1Present)
-                            uShw.pPT->a[iShw].n.u1Write = 0;
-                    }
-                    break;
-
-                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
-                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
-                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
-                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
-                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
-                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTPae->a); iShw++)
-                    {
-                        if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
-                            PGMSHWPTEPAE_SET_RO(uShw.pPTPae->a[iShw]);
-                    }
-                    break;
-
-                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
-                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTEpt->a); iShw++)
-                    {
-                        if (uShw.pPTEpt->a[iShw].n.u1Present)
-                            uShw.pPTEpt->a[iShw].n.u1Write = 0;
-                    }
-                    break;
-
-                default:
-                    break;
-            }
-            if (!--cLeft)
-                break;
-        }
-    }
-}
-
-#ifdef VBOX_WITH_DEBUGGER
-/**
- * The '.pgmpoolcheck' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
-
-    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
-
-    for (unsigned i = 0; i < pPool->cCurPages; i++)
-    {
-        PPGMPOOLPAGE pPage = &pPool->aPages[i];
-        bool fFirstMsg = true;
-
-        /* Todo: cover other paging modes too. */
-        if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
-        {
-            PPGMSHWPTPAE pShwPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
-            {
-                PX86PTPAE pGstPT;
-                PGMPAGEMAPLOCK LockPage;
-                int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, pPage->GCPhys, (const void **)&pGstPT, &LockPage);     AssertReleaseRC(rc);
-
-                /* Check if any PTEs are out of sync. */
-                for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
-                {
-                    if (PGMSHWPTEPAE_IS_P(pShwPT->a[j]))
-                    {
-                        RTHCPHYS HCPhys = NIL_RTHCPHYS;
-                        rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[j].u & X86_PTE_PAE_PG_MASK, &HCPhys);
-                        if (   rc != VINF_SUCCESS
-                            || PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[j]) != HCPhys)
-                        {
-                            if (fFirstMsg)
-                            {
-                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
-                                fFirstMsg = false;
-                            }
-                            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch HCPhys: rc=%Rrc idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
-                        }
-                        else if (   PGMSHWPTEPAE_IS_RW(pShwPT->a[j])
-                                 && !pGstPT->a[j].n.u1Write)
-                        {
-                            if (fFirstMsg)
-                            {
-                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
-                                fFirstMsg = false;
-                            }
-                            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch r/w gst/shw: idx=%d guest %RX64 shw=%RX64 vs %RHp\n", j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
-                        }
-                    }
-                }
-                PGMPhysReleasePageMappingLock(pVM, &LockPage);
-            }
-
-            /* Make sure this page table can't be written to from any shadow mapping. */
-            RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
-            int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pPage->GCPhys, &HCPhysPT);
-            AssertMsgRC(rc, ("PGMPhysGCPhys2HCPhys failed with rc=%d for %RGp\n", rc, pPage->GCPhys));
-            if (rc == VINF_SUCCESS)
-            {
-                for (unsigned j = 0; j < pPool->cCurPages; j++)
-                {
-                    PPGMPOOLPAGE pTempPage = &pPool->aPages[j];
-
-                    if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
-                    {
-                        PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pTempPage);
-
-                        for (unsigned k = 0; k < RT_ELEMENTS(pShwPT->a); k++)
-                        {
-                            if (    PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[k])
-# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
-                                &&  !pPage->fDirty
-# endif
-                                &&  PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[k]) == HCPhysPT)
-                            {
-                                if (fFirstMsg)
-                                {
-                                    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
-                                    fFirstMsg = false;
-                                }
-                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch: r/w: GCPhys=%RGp idx=%d shw %RX64 %RX64\n", pTempPage->GCPhys, k, PGMSHWPTEPAE_GET_LOG(pShwPT->a[k]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[k]));
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-    return VINF_SUCCESS;
-}
-#endif /* VBOX_WITH_DEBUGGER */
Index: unk/src/VBox/VMM/PGMSavedState.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGMSavedState.cpp	(revision 35332)
+++ 	(revision )
@@ -1,3200 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager and Monitor, The Saved State Part.
- */
-
-/*
- * Copyright (C) 2006-2009 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM
-#include <VBox/pgm.h>
-#include <VBox/stam.h>
-#include <VBox/ssm.h>
-#include <VBox/pdmdrv.h>
-#include <VBox/pdmdev.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include "PGMInline.h"
-
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/ftm.h>
-
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/crc.h>
-#include <iprt/mem.h>
-#include <iprt/sha.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** Saved state data unit version.
- * @todo remove the guest mappings from the saved state at next version change! */
-#define PGM_SAVED_STATE_VERSION                 12
-/** Saved state before the balloon change. */
-#define PGM_SAVED_STATE_VERSION_PRE_BALLOON     11
-/** Saved state data unit version used during 3.1 development, misses the RAM
- *  config. */
-#define PGM_SAVED_STATE_VERSION_NO_RAM_CFG      10
-/** Saved state data unit version for 3.0 (pre teleportation). */
-#define PGM_SAVED_STATE_VERSION_3_0_0           9
-/** Saved state data unit version for 2.2.2 and later. */
-#define PGM_SAVED_STATE_VERSION_2_2_2           8
-/** Saved state data unit version for 2.2.0. */
-#define PGM_SAVED_STATE_VERSION_RR_DESC         7
-/** Saved state data unit version. */
-#define PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE   6
-
-
-/** @name Sparse state record types
- * @{  */
-/** Zero page. No data. */
-#define PGM_STATE_REC_RAM_ZERO          UINT8_C(0x00)
-/** Raw page. */
-#define PGM_STATE_REC_RAM_RAW           UINT8_C(0x01)
-/** Raw MMIO2 page. */
-#define PGM_STATE_REC_MMIO2_RAW         UINT8_C(0x02)
-/** Zero MMIO2 page. */
-#define PGM_STATE_REC_MMIO2_ZERO        UINT8_C(0x03)
-/** Virgin ROM page. Followed by protection (8-bit) and the raw bits. */
-#define PGM_STATE_REC_ROM_VIRGIN        UINT8_C(0x04)
-/** Raw shadowed ROM page. The protection (8-bit) precedes the raw bits. */
-#define PGM_STATE_REC_ROM_SHW_RAW       UINT8_C(0x05)
-/** Zero shadowed ROM page. The protection (8-bit) is the only payload. */
-#define PGM_STATE_REC_ROM_SHW_ZERO      UINT8_C(0x06)
-/** ROM protection (8-bit). */
-#define PGM_STATE_REC_ROM_PROT          UINT8_C(0x07)
-/** The last record type. */
-#define PGM_STATE_REC_LAST              PGM_STATE_REC_ROM_PROT
-/** End marker. */
-#define PGM_STATE_REC_END               UINT8_C(0xff)
-/** Flag indicating that the data is preceded by the page address.
- *  For RAW pages this is a RTGCPHYS.  For MMIO2 and ROM pages this is a 8-bit
- *  range ID and a 32-bit page index.
- */
-#define PGM_STATE_REC_FLAG_ADDR         UINT8_C(0x80)
-/** @} */
-
-/** The CRC-32 for a zero page. */
-#define PGM_STATE_CRC32_ZERO_PAGE       UINT32_C(0xc71c0011)
-/** The CRC-32 for a zero half page. */
-#define PGM_STATE_CRC32_ZERO_HALF_PAGE  UINT32_C(0xf1e8ba9e)
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/** For loading old saved states. (pre-smp) */
-typedef struct
-{
-    /** If set no conflict checks are required.  (boolean) */
-    bool                            fMappingsFixed;
-    /** Size of fixed mapping */
-    uint32_t                        cbMappingFixed;
-    /** Base address (GC) of fixed mapping */
-    RTGCPTR                         GCPtrMappingFixed;
-    /** A20 gate mask.
-     * Our current approach to A20 emulation is to let REM do it and don't bother
-     * anywhere else. The interesting guests will be operating with it enabled anyway.
-     * But should the need arise, we'll subject physical addresses to this mask. */
-    RTGCPHYS                        GCPhysA20Mask;
-    /** A20 gate state - boolean! */
-    bool                            fA20Enabled;
-    /** The guest paging mode. */
-    PGMMODE                         enmGuestMode;
-} PGMOLD;
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-/** PGM fields to save/load. */
-
-static const SSMFIELD s_aPGMFields[] =
-{
-    SSMFIELD_ENTRY(         PGM, fMappingsFixed),
-    SSMFIELD_ENTRY_GCPTR(   PGM, GCPtrMappingFixed),
-    SSMFIELD_ENTRY(         PGM, cbMappingFixed),
-    SSMFIELD_ENTRY(         PGM, cBalloonedPages),
-    SSMFIELD_ENTRY_TERM()
-};
-
-static const SSMFIELD s_aPGMFieldsPreBalloon[] =
-{
-    SSMFIELD_ENTRY(         PGM, fMappingsFixed),
-    SSMFIELD_ENTRY_GCPTR(   PGM, GCPtrMappingFixed),
-    SSMFIELD_ENTRY(         PGM, cbMappingFixed),
-    SSMFIELD_ENTRY_TERM()
-};
-
-static const SSMFIELD s_aPGMCpuFields[] =
-{
-    SSMFIELD_ENTRY(         PGMCPU, fA20Enabled),
-    SSMFIELD_ENTRY_GCPHYS(  PGMCPU, GCPhysA20Mask),
-    SSMFIELD_ENTRY(         PGMCPU, enmGuestMode),
-    SSMFIELD_ENTRY_TERM()
-};
-
-static const SSMFIELD s_aPGMFields_Old[] =
-{
-    SSMFIELD_ENTRY(         PGMOLD, fMappingsFixed),
-    SSMFIELD_ENTRY_GCPTR(   PGMOLD, GCPtrMappingFixed),
-    SSMFIELD_ENTRY(         PGMOLD, cbMappingFixed),
-    SSMFIELD_ENTRY(         PGMOLD, fA20Enabled),
-    SSMFIELD_ENTRY_GCPHYS(  PGMOLD, GCPhysA20Mask),
-    SSMFIELD_ENTRY(         PGMOLD, enmGuestMode),
-    SSMFIELD_ENTRY_TERM()
-};
-
-
-/**
- * Find the ROM tracking structure for the given page.
- *
- * @returns Pointer to the ROM page structure. NULL if the caller didn't check
- *          that it's a ROM page.
- * @param   pVM         The VM handle.
- * @param   GCPhys      The address of the ROM page.
- */
-static PPGMROMPAGE pgmR3GetRomPage(PVM pVM, RTGCPHYS GCPhys) /** @todo change this to take a hint. */
-{
-    for (PPGMROMRANGE pRomRange = pVM->pgm.s.CTX_SUFF(pRomRanges);
-         pRomRange;
-         pRomRange = pRomRange->CTX_SUFF(pNext))
-    {
-        RTGCPHYS off = GCPhys - pRomRange->GCPhys;
-        if (GCPhys - pRomRange->GCPhys < pRomRange->cb)
-            return &pRomRange->aPages[off >> PAGE_SHIFT];
-    }
-    return NULL;
-}
-
-
-/**
- * Prepares the ROM pages for a live save.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- */
-static int pgmR3PrepRomPages(PVM pVM)
-{
-    /*
-     * Initialize the live save tracking in the ROM page descriptors.
-     */
-    pgmLock(pVM);
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-    {
-        PPGMRAMRANGE    pRamHint = NULL;;
-        uint32_t const  cPages   = pRom->cb >> PAGE_SHIFT;
-
-        for (uint32_t iPage = 0; iPage < cPages; iPage++)
-        {
-            pRom->aPages[iPage].LiveSave.u8Prot           = (uint8_t)PGMROMPROT_INVALID;
-            pRom->aPages[iPage].LiveSave.fWrittenTo       = false;
-            pRom->aPages[iPage].LiveSave.fDirty           = true;
-            pRom->aPages[iPage].LiveSave.fDirtiedRecently = true;
-            if (!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
-            {
-                if (PGMROMPROT_IS_ROM(pRom->aPages[iPage].enmProt))
-                    pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow) && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow);
-                else
-                {
-                    RTGCPHYS GCPhys = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-                    PPGMPAGE pPage;
-                    int rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pPage, &pRamHint);
-                    AssertLogRelMsgRC(rc, ("%Rrc GCPhys=%RGp\n", rc, GCPhys));
-                    if (RT_SUCCESS(rc))
-                        pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(pPage) && !PGM_PAGE_IS_BALLOONED(pPage);
-                    else
-                        pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow) && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow);
-                }
-            }
-        }
-
-        pVM->pgm.s.LiveSave.Rom.cDirtyPages += cPages;
-        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
-            pVM->pgm.s.LiveSave.Rom.cDirtyPages += cPages;
-    }
-    pgmUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Assigns IDs to the ROM ranges and saves them.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                Saved state handle.
- */
-static int pgmR3SaveRomRanges(PVM pVM, PSSMHANDLE pSSM)
-{
-    pgmLock(pVM);
-    uint8_t id = 1;
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3, id++)
-    {
-        pRom->idSavedState = id;
-        SSMR3PutU8(pSSM, id);
-        SSMR3PutStrZ(pSSM, "");         /* device name */
-        SSMR3PutU32(pSSM, 0);           /* device instance */
-        SSMR3PutU8(pSSM, 0);            /* region */
-        SSMR3PutStrZ(pSSM, pRom->pszDesc);
-        SSMR3PutGCPhys(pSSM, pRom->GCPhys);
-        int rc = SSMR3PutGCPhys(pSSM, pRom->cb);
-        if (RT_FAILURE(rc))
-            break;
-    }
-    pgmUnlock(pVM);
-    return SSMR3PutU8(pSSM, UINT8_MAX);
-}
-
-
-/**
- * Loads the ROM range ID assignments.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int pgmR3LoadRomRanges(PVM pVM, PSSMHANDLE pSSM)
-{
-    Assert(PGMIsLockOwner(pVM));
-
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-        pRom->idSavedState = UINT8_MAX;
-
-    for (;;)
-    {
-        /*
-         * Read the data.
-         */
-        uint8_t id;
-        int rc = SSMR3GetU8(pSSM, &id);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (id == UINT8_MAX)
-        {
-            for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-                AssertLogRelMsg(pRom->idSavedState != UINT8_MAX,
-                                ("The \"%s\" ROM was not found in the saved state. Probably due to some misconfiguration\n",
-                                 pRom->pszDesc));
-            return VINF_SUCCESS;        /* the end */
-        }
-        AssertLogRelReturn(id != 0, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-        char szDevName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
-        rc = SSMR3GetStrZ(pSSM, szDevName, sizeof(szDevName));
-        AssertLogRelRCReturn(rc, rc);
-
-        uint32_t    uInstance;
-        SSMR3GetU32(pSSM, &uInstance);
-        uint8_t     iRegion;
-        SSMR3GetU8(pSSM, &iRegion);
-
-        char szDesc[64];
-        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
-        AssertLogRelRCReturn(rc, rc);
-
-        RTGCPHYS GCPhys;
-        SSMR3GetGCPhys(pSSM, &GCPhys);
-        RTGCPHYS cb;
-        rc = SSMR3GetGCPhys(pSSM, &cb);
-        if (RT_FAILURE(rc))
-            return rc;
-        AssertLogRelMsgReturn(!(GCPhys & PAGE_OFFSET_MASK), ("GCPhys=%RGp %s\n", GCPhys, szDesc), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-        AssertLogRelMsgReturn(!(cb & PAGE_OFFSET_MASK),     ("cb=%RGp %s\n", cb, szDesc),         VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-        /*
-         * Locate a matching ROM range.
-         */
-        AssertLogRelMsgReturn(   uInstance == 0
-                              && iRegion == 0
-                              && szDevName[0] == '\0',
-                              ("GCPhys=%RGp %s\n", GCPhys, szDesc),
-                              VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-        PPGMROMRANGE pRom;
-        for (pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-        {
-            if (    pRom->idSavedState == UINT8_MAX
-                &&  !strcmp(pRom->pszDesc, szDesc))
-            {
-                pRom->idSavedState = id;
-                break;
-            }
-        }
-        if (!pRom)
-            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("ROM at %RGp by the name '%s' was not found"), GCPhys, szDesc);
-    } /* forever */
-}
-
-
-/**
- * Scan ROM pages.
- *
- * @param   pVM                 The VM handle.
- */
-static void pgmR3ScanRomPages(PVM pVM)
-{
-    /*
-     * The shadow ROMs.
-     */
-    pgmLock(pVM);
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-    {
-        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
-        {
-            uint32_t const cPages = pRom->cb >> PAGE_SHIFT;
-            for (uint32_t iPage = 0; iPage < cPages; iPage++)
-            {
-                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
-                if (pRomPage->LiveSave.fWrittenTo)
-                {
-                    pRomPage->LiveSave.fWrittenTo = false;
-                    if (!pRomPage->LiveSave.fDirty)
-                    {
-                        pRomPage->LiveSave.fDirty = true;
-                        pVM->pgm.s.LiveSave.Rom.cReadyPages--;
-                        pVM->pgm.s.LiveSave.Rom.cDirtyPages++;
-                    }
-                    pRomPage->LiveSave.fDirtiedRecently = true;
-                }
-                else
-                    pRomPage->LiveSave.fDirtiedRecently = false;
-            }
-        }
-    }
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Takes care of the virgin ROM pages in the first pass.
- *
- * This is an attempt at simplifying the handling of ROM pages a little bit.
- * This ASSUMES that no new ROM ranges will be added and that they won't be
- * relinked in any way.
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- * @param   fLiveSave           Whether we're in a live save or not.
- */
-static int pgmR3SaveRomVirginPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave)
-{
-    if (FTMIsDeltaLoadSaveActive(pVM))
-        return VINF_SUCCESS;    /* nothing to do as nothing has changed here */
-
-    pgmLock(pVM);
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-    {
-        uint32_t const cPages = pRom->cb >> PAGE_SHIFT;
-        for (uint32_t iPage = 0; iPage < cPages; iPage++)
-        {
-            RTGCPHYS   GCPhys  = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-            PGMROMPROT enmProt = pRom->aPages[iPage].enmProt;
-
-            /* Get the virgin page descriptor. */
-            PPGMPAGE pPage;
-            if (PGMROMPROT_IS_ROM(enmProt))
-                pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
-            else
-                pPage = &pRom->aPages[iPage].Virgin;
-
-            /* Get the page bits. (Cannot use pgmPhysGCPhys2CCPtrInternalReadOnly here!) */
-            int rc = VINF_SUCCESS;
-            char abPage[PAGE_SIZE];
-            if (    !PGM_PAGE_IS_ZERO(pPage)
-                &&  !PGM_PAGE_IS_BALLOONED(pPage))
-            {
-                void const *pvPage;
-                rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvPage);
-                if (RT_SUCCESS(rc))
-                    memcpy(abPage, pvPage, PAGE_SIZE);
-            }
-            else
-                ASMMemZeroPage(abPage);
-            pgmUnlock(pVM);
-            AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
-
-            /* Save it. */
-            if (iPage > 0)
-                SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_VIRGIN);
-            else
-            {
-                SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_VIRGIN | PGM_STATE_REC_FLAG_ADDR);
-                SSMR3PutU8(pSSM, pRom->idSavedState);
-                SSMR3PutU32(pSSM, iPage);
-            }
-            SSMR3PutU8(pSSM, (uint8_t)enmProt);
-            rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
-            if (RT_FAILURE(rc))
-                return rc;
-
-            /* Update state. */
-            pgmLock(pVM);
-            pRom->aPages[iPage].LiveSave.u8Prot = (uint8_t)enmProt;
-            if (fLiveSave)
-            {
-                pVM->pgm.s.LiveSave.Rom.cDirtyPages--;
-                pVM->pgm.s.LiveSave.Rom.cReadyPages++;
-                pVM->pgm.s.LiveSave.cSavedPages++;
-            }
-        }
-    }
-    pgmUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Saves dirty pages in the shadowed ROM ranges.
- *
- * Used by pgmR3LiveExecPart2 and pgmR3SaveExecMemory.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- * @param   fLiveSave           Whether it's a live save or not.
- * @param   fFinalPass          Whether this is the final pass or not.
- */
-static int pgmR3SaveShadowedRomPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, bool fFinalPass)
-{
-    if (FTMIsDeltaLoadSaveActive(pVM))
-        return VINF_SUCCESS;    /* nothing to do as we deal with those pages separately */
-
-    /*
-     * The Shadowed ROMs.
-     *
-     * ASSUMES that the ROM ranges are fixed.
-     * ASSUMES that all the ROM ranges are mapped.
-     */
-    pgmLock(pVM);
-    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-    {
-        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
-        {
-            uint32_t const cPages    = pRom->cb >> PAGE_SHIFT;
-            uint32_t       iPrevPage = cPages;
-            for (uint32_t iPage = 0; iPage < cPages; iPage++)
-            {
-                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
-                if (    !fLiveSave
-                    ||  (   pRomPage->LiveSave.fDirty
-                         && (   (   !pRomPage->LiveSave.fDirtiedRecently
-                                 && !pRomPage->LiveSave.fWrittenTo)
-                             || fFinalPass
-                             )
-                         )
-                    )
-                {
-                    uint8_t     abPage[PAGE_SIZE];
-                    PGMROMPROT  enmProt = pRomPage->enmProt;
-                    RTGCPHYS    GCPhys  = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-                    PPGMPAGE    pPage   = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow : pgmPhysGetPage(&pVM->pgm.s, GCPhys);
-                    bool        fZero   = PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_BALLOONED(pPage);
-                    int         rc      = VINF_SUCCESS;
-                    if (!fZero)
-                    {
-                        void const *pvPage;
-                        rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvPage);
-                        if (RT_SUCCESS(rc))
-                            memcpy(abPage, pvPage, PAGE_SIZE);
-                    }
-                    if (fLiveSave && RT_SUCCESS(rc))
-                    {
-                        pRomPage->LiveSave.u8Prot = (uint8_t)enmProt;
-                        pRomPage->LiveSave.fDirty = false;
-                        pVM->pgm.s.LiveSave.Rom.cReadyPages++;
-                        pVM->pgm.s.LiveSave.Rom.cDirtyPages--;
-                        pVM->pgm.s.LiveSave.cSavedPages++;
-                    }
-                    pgmUnlock(pVM);
-                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
-
-                    if (iPage - 1U == iPrevPage && iPage > 0)
-                        SSMR3PutU8(pSSM, (fZero ? PGM_STATE_REC_ROM_SHW_ZERO : PGM_STATE_REC_ROM_SHW_RAW));
-                    else
-                    {
-                        SSMR3PutU8(pSSM, (fZero ? PGM_STATE_REC_ROM_SHW_ZERO : PGM_STATE_REC_ROM_SHW_RAW) | PGM_STATE_REC_FLAG_ADDR);
-                        SSMR3PutU8(pSSM, pRom->idSavedState);
-                        SSMR3PutU32(pSSM, iPage);
-                    }
-                    rc = SSMR3PutU8(pSSM, (uint8_t)enmProt);
-                    if (!fZero)
-                        rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
-                    if (RT_FAILURE(rc))
-                        return rc;
-
-                    pgmLock(pVM);
-                    iPrevPage = iPage;
-                }
-                /*
-                 * In the final pass, make sure the protection is in sync.
-                 */
-                else if (   fFinalPass
-                         && pRomPage->LiveSave.u8Prot != pRomPage->enmProt)
-                {
-                    PGMROMPROT enmProt = pRomPage->enmProt;
-                    pRomPage->LiveSave.u8Prot = (uint8_t)enmProt;
-                    pgmUnlock(pVM);
-
-                    if (iPage - 1U == iPrevPage && iPage > 0)
-                        SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_PROT);
-                    else
-                    {
-                        SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_PROT | PGM_STATE_REC_FLAG_ADDR);
-                        SSMR3PutU8(pSSM, pRom->idSavedState);
-                        SSMR3PutU32(pSSM, iPage);
-                    }
-                    int rc = SSMR3PutU8(pSSM, (uint8_t)enmProt);
-                    if (RT_FAILURE(rc))
-                        return rc;
-
-                    pgmLock(pVM);
-                    iPrevPage = iPage;
-                }
-            }
-        }
-    }
-    pgmUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Cleans up ROM pages after a live save.
- *
- * @param   pVM                 The VM handle.
- */
-static void pgmR3DoneRomPages(PVM pVM)
-{
-    NOREF(pVM);
-}
-
-
-/**
- * Prepares the MMIO2 pages for a live save.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- */
-static int pgmR3PrepMmio2Pages(PVM pVM)
-{
-    /*
-     * Initialize the live save tracking in the MMIO2 ranges.
-     * ASSUME nothing changes here.
-     */
-    pgmLock(pVM);
-    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
-    {
-        uint32_t const  cPages = pMmio2->RamRange.cb >> PAGE_SHIFT;
-        pgmUnlock(pVM);
-
-        PPGMLIVESAVEMMIO2PAGE paLSPages = (PPGMLIVESAVEMMIO2PAGE)MMR3HeapAllocZ(pVM, MM_TAG_PGM, sizeof(PGMLIVESAVEMMIO2PAGE) * cPages);
-        if (!paLSPages)
-            return VERR_NO_MEMORY;
-        for (uint32_t iPage = 0; iPage < cPages; iPage++)
-        {
-            /* Initialize it as a dirty zero page. */
-            paLSPages[iPage].fDirty          = true;
-            paLSPages[iPage].cUnchangedScans = 0;
-            paLSPages[iPage].fZero           = true;
-            paLSPages[iPage].u32CrcH1        = PGM_STATE_CRC32_ZERO_HALF_PAGE;
-            paLSPages[iPage].u32CrcH2        = PGM_STATE_CRC32_ZERO_HALF_PAGE;
-        }
-
-        pgmLock(pVM);
-        pMmio2->paLSPages = paLSPages;
-        pVM->pgm.s.LiveSave.Mmio2.cDirtyPages += cPages;
-    }
-    pgmUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Assigns IDs to the MMIO2 ranges and saves them.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                Saved state handle.
- */
-static int pgmR3SaveMmio2Ranges(PVM pVM, PSSMHANDLE pSSM)
-{
-    pgmLock(pVM);
-    uint8_t id = 1;
-    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3, id++)
-    {
-        pMmio2->idSavedState = id;
-        SSMR3PutU8(pSSM, id);
-        SSMR3PutStrZ(pSSM, pMmio2->pDevInsR3->pReg->szName);
-        SSMR3PutU32(pSSM, pMmio2->pDevInsR3->iInstance);
-        SSMR3PutU8(pSSM, pMmio2->iRegion);
-        SSMR3PutStrZ(pSSM, pMmio2->RamRange.pszDesc);
-        int rc = SSMR3PutGCPhys(pSSM, pMmio2->RamRange.cb);
-        if (RT_FAILURE(rc))
-            break;
-    }
-    pgmUnlock(pVM);
-    return SSMR3PutU8(pSSM, UINT8_MAX);
-}
-
-
-/**
- * Loads the MMIO2 range ID assignments.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int pgmR3LoadMmio2Ranges(PVM pVM, PSSMHANDLE pSSM)
-{
-    Assert(PGMIsLockOwner(pVM));
-
-    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
-        pMmio2->idSavedState = UINT8_MAX;
-
-    for (;;)
-    {
-        /*
-         * Read the data.
-         */
-        uint8_t id;
-        int rc = SSMR3GetU8(pSSM, &id);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (id == UINT8_MAX)
-        {
-            for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
-                AssertLogRelMsg(pMmio2->idSavedState != UINT8_MAX, ("%s\n", pMmio2->RamRange.pszDesc));
-            return VINF_SUCCESS;        /* the end */
-        }
-        AssertLogRelReturn(id != 0, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-        char szDevName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
-        rc = SSMR3GetStrZ(pSSM, szDevName, sizeof(szDevName));
-        AssertLogRelRCReturn(rc, rc);
-
-        uint32_t    uInstance;
-        SSMR3GetU32(pSSM, &uInstance);
-        uint8_t     iRegion;
-        SSMR3GetU8(pSSM, &iRegion);
-
-        char szDesc[64];
-        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
-        AssertLogRelRCReturn(rc, rc);
-
-        RTGCPHYS cb;
-        rc = SSMR3GetGCPhys(pSSM, &cb);
-        AssertLogRelMsgReturn(!(cb & PAGE_OFFSET_MASK), ("cb=%RGp %s\n", cb, szDesc), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-        /*
-         * Locate a matching MMIO2 range.
-         */
-        PPGMMMIO2RANGE pMmio2;
-        for (pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
-        {
-            if (    pMmio2->idSavedState == UINT8_MAX
-                &&  pMmio2->iRegion == iRegion
-                &&  pMmio2->pDevInsR3->iInstance == uInstance
-                &&  !strcmp(pMmio2->pDevInsR3->pReg->szName, szDevName))
-            {
-                pMmio2->idSavedState = id;
-                break;
-            }
-        }
-        if (!pMmio2)
-            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Failed to locate a MMIO2 range called '%s' owned by %s/%u, region %d"),
-                                    szDesc, szDevName, uInstance, iRegion);
-
-        /*
-         * Validate the configuration, the size of the MMIO2 region should be
-         * the same.
-         */
-        if (cb != pMmio2->RamRange.cb)
-        {
-            LogRel(("PGM: MMIO2 region \"%s\" size mismatch: saved=%RGp config=%RGp\n",
-                    pMmio2->RamRange.pszDesc, cb, pMmio2->RamRange.cb));
-            if (cb > pMmio2->RamRange.cb) /* bad idea? */
-                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("MMIO2 region \"%s\" size mismatch: saved=%RGp config=%RGp"),
-                                        pMmio2->RamRange.pszDesc, cb, pMmio2->RamRange.cb);
-        }
-    } /* forever */
-}
-
-
-/**
- * Scans one MMIO2 page.
- *
- * @returns True if changed, false if unchanged.
- *
- * @param   pVM                 The VM handle
- * @param   pbPage              The page bits.
- * @param   pLSPage             The live save tracking structure for the page.
- *
- */
-DECLINLINE(bool) pgmR3ScanMmio2Page(PVM pVM, uint8_t const *pbPage, PPGMLIVESAVEMMIO2PAGE pLSPage)
-{
-    /*
-     * Special handling of zero pages.
-     */
-    bool const fZero = pLSPage->fZero;
-    if (fZero)
-    {
-        if (ASMMemIsZeroPage(pbPage))
-        {
-            /* Not modified. */
-            if (pLSPage->fDirty)
-                pLSPage->cUnchangedScans++;
-            return false;
-        }
-
-        pLSPage->fZero    = false;
-        pLSPage->u32CrcH1 = RTCrc32(pbPage, PAGE_SIZE / 2);
-    }
-    else
-    {
-        /*
-         * CRC the first half, if it doesn't match the page is dirty and
-         * we won't check the 2nd half (we'll do that next time).
-         */
-        uint32_t u32CrcH1 = RTCrc32(pbPage, PAGE_SIZE / 2);
-        if (u32CrcH1 == pLSPage->u32CrcH1)
-        {
-            uint32_t u32CrcH2 = RTCrc32(pbPage + PAGE_SIZE / 2, PAGE_SIZE / 2);
-            if (u32CrcH2 == pLSPage->u32CrcH2)
-            {
-                /* Probably not modified. */
-                if (pLSPage->fDirty)
-                    pLSPage->cUnchangedScans++;
-                return false;
-            }
-
-            pLSPage->u32CrcH2 = u32CrcH2;
-        }
-        else
-        {
-            pLSPage->u32CrcH1 = u32CrcH1;
-            if (    u32CrcH1 == PGM_STATE_CRC32_ZERO_HALF_PAGE
-                &&  ASMMemIsZeroPage(pbPage))
-            {
-                pLSPage->u32CrcH2 = PGM_STATE_CRC32_ZERO_HALF_PAGE;
-                pLSPage->fZero    = true;
-            }
-        }
-    }
-
-    /* dirty page path */
-    pLSPage->cUnchangedScans = 0;
-    if (!pLSPage->fDirty)
-    {
-        pLSPage->fDirty = true;
-        pVM->pgm.s.LiveSave.Mmio2.cReadyPages--;
-        pVM->pgm.s.LiveSave.Mmio2.cDirtyPages++;
-        if (fZero)
-            pVM->pgm.s.LiveSave.Mmio2.cZeroPages--;
-    }
-    return true;
-}
-
-
-/**
- * Scan for MMIO2 page modifications.
- *
- * @param   pVM                 The VM handle.
- * @param   uPass               The pass number.
- */
-static void pgmR3ScanMmio2Pages(PVM pVM, uint32_t uPass)
-{
-    /*
-     * Since this is a bit expensive we lower the scan rate after a little while.
-     */
-    if (    (    (uPass & 3) != 0
-             &&  uPass > 10)
-        ||  uPass == SSM_PASS_FINAL)
-        return;
-
-    pgmLock(pVM);                       /* paranoia */
-    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
-    {
-        PPGMLIVESAVEMMIO2PAGE paLSPages = pMmio2->paLSPages;
-        uint32_t              cPages    = pMmio2->RamRange.cb >> PAGE_SHIFT;
-        pgmUnlock(pVM);
-
-        for (uint32_t iPage = 0; iPage < cPages; iPage++)
-        {
-            uint8_t const *pbPage = (uint8_t const *)pMmio2->pvR3 + iPage * PAGE_SIZE;
-            pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]);
-        }
-
-        pgmLock(pVM);
-    }
-    pgmUnlock(pVM);
-
-}
-
-
-/**
- * Save quiescent MMIO2 pages.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- * @param   fLiveSave           Whether it's a live save or not.
- * @param   uPass               The pass number.
- */
-static int pgmR3SaveMmio2Pages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, uint32_t uPass)
-{
-    /** @todo implement live saving of MMIO2 pages. (Need some way of telling the
-     *        device that we wish to know about changes.) */
-
-    int rc = VINF_SUCCESS;
-    if (uPass == SSM_PASS_FINAL)
-    {
-        /*
-         * The mop up round.
-         */
-        pgmLock(pVM);
-        for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3;
-             pMmio2 && RT_SUCCESS(rc);
-             pMmio2 = pMmio2->pNextR3)
-        {
-            PPGMLIVESAVEMMIO2PAGE paLSPages = pMmio2->paLSPages;
-            uint8_t const        *pbPage    = (uint8_t const *)pMmio2->RamRange.pvR3;
-            uint32_t              cPages    = pMmio2->RamRange.cb >> PAGE_SHIFT;
-            uint32_t              iPageLast = cPages;
-            for (uint32_t iPage = 0; iPage < cPages; iPage++, pbPage += PAGE_SIZE)
-            {
-                uint8_t u8Type;
-                if (!fLiveSave)
-                    u8Type = ASMMemIsZeroPage(pbPage) ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
-                else
-                {
-                    /* Try figure if it's a clean page, compare the SHA-1 to be really sure. */
-                    if (   !paLSPages[iPage].fDirty
-                        && !pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]))
-                    {
-                        if (paLSPages[iPage].fZero)
-                            continue;
-
-                        uint8_t abSha1Hash[RTSHA1_HASH_SIZE];
-                        RTSha1(pbPage, PAGE_SIZE, abSha1Hash);
-                        if (!memcmp(abSha1Hash, paLSPages[iPage].abSha1Saved, sizeof(abSha1Hash)))
-                            continue;
-                    }
-                    u8Type = paLSPages[iPage].fZero ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
-                    pVM->pgm.s.LiveSave.cSavedPages++;
-                }
-
-                if (iPage != 0 && iPage == iPageLast + 1)
-                    rc = SSMR3PutU8(pSSM, u8Type);
-                else
-                {
-                    SSMR3PutU8(pSSM, u8Type | PGM_STATE_REC_FLAG_ADDR);
-                    SSMR3PutU8(pSSM, pMmio2->idSavedState);
-                    rc = SSMR3PutU32(pSSM, iPage);
-                }
-                if (u8Type == PGM_STATE_REC_MMIO2_RAW)
-                    rc = SSMR3PutMem(pSSM, pbPage, PAGE_SIZE);
-                if (RT_FAILURE(rc))
-                    break;
-                iPageLast = iPage;
-            }
-        }
-        pgmUnlock(pVM);
-    }
-    /*
-     * Reduce the rate after a little while since the current MMIO2 approach is
-     * a bit expensive.
-     * We position it two passes after the scan pass to avoid saving busy pages.
-     */
-    else if (   uPass <= 10
-             || (uPass & 3) == 2)
-    {
-        pgmLock(pVM);
-        for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3;
-             pMmio2 && RT_SUCCESS(rc);
-             pMmio2 = pMmio2->pNextR3)
-        {
-            PPGMLIVESAVEMMIO2PAGE paLSPages = pMmio2->paLSPages;
-            uint8_t const        *pbPage    = (uint8_t const *)pMmio2->RamRange.pvR3;
-            uint32_t              cPages    = pMmio2->RamRange.cb >> PAGE_SHIFT;
-            uint32_t              iPageLast = cPages;
-            pgmUnlock(pVM);
-
-            for (uint32_t iPage = 0; iPage < cPages; iPage++, pbPage += PAGE_SIZE)
-            {
-                /* Skip clean pages and pages which hasn't quiesced. */
-                if (!paLSPages[iPage].fDirty)
-                    continue;
-                if (paLSPages[iPage].cUnchangedScans < 3)
-                    continue;
-                if (pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]))
-                    continue;
-
-                /* Save it. */
-                bool const fZero = paLSPages[iPage].fZero;
-                uint8_t abPage[PAGE_SIZE];
-                if (!fZero)
-                {
-                    memcpy(abPage, pbPage, PAGE_SIZE);
-                    RTSha1(abPage, PAGE_SIZE, paLSPages[iPage].abSha1Saved);
-                }
-
-                uint8_t u8Type = paLSPages[iPage].fZero ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
-                if (iPage != 0 && iPage == iPageLast + 1)
-                    rc = SSMR3PutU8(pSSM, u8Type);
-                else
-                {
-                    SSMR3PutU8(pSSM, u8Type | PGM_STATE_REC_FLAG_ADDR);
-                    SSMR3PutU8(pSSM, pMmio2->idSavedState);
-                    rc = SSMR3PutU32(pSSM, iPage);
-                }
-                if (u8Type == PGM_STATE_REC_MMIO2_RAW)
-                    rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
-                if (RT_FAILURE(rc))
-                    break;
-
-                /* Housekeeping. */
-                paLSPages[iPage].fDirty = false;
-                pVM->pgm.s.LiveSave.Mmio2.cDirtyPages--;
-                pVM->pgm.s.LiveSave.Mmio2.cReadyPages++;
-                if (u8Type == PGM_STATE_REC_MMIO2_ZERO)
-                    pVM->pgm.s.LiveSave.Mmio2.cZeroPages++;
-                pVM->pgm.s.LiveSave.cSavedPages++;
-                iPageLast = iPage;
-            }
-
-            pgmLock(pVM);
-        }
-        pgmUnlock(pVM);
-    }
-
-    return rc;
-}
-
-
-/**
- * Cleans up MMIO2 pages after a live save.
- *
- * @param   pVM                 The VM handle.
- */
-static void pgmR3DoneMmio2Pages(PVM pVM)
-{
-    /*
-     * Free the tracking structures for the MMIO2 pages.
-     * We do the freeing outside the lock in case the VM is running.
-     */
-    pgmLock(pVM);
-    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
-    {
-        void *pvMmio2ToFree = pMmio2->paLSPages;
-        if (pvMmio2ToFree)
-        {
-            pMmio2->paLSPages = NULL;
-            pgmUnlock(pVM);
-            MMR3HeapFree(pvMmio2ToFree);
-            pgmLock(pVM);
-        }
-    }
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Prepares the RAM pages for a live save.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- */
-static int pgmR3PrepRamPages(PVM pVM)
-{
-
-    /*
-     * Try allocating tracking structures for the ram ranges.
-     *
-     * To avoid lock contention, we leave the lock every time we're allocating
-     * a new array.  This means we'll have to ditch the allocation and start
-     * all over again if the RAM range list changes in-between.
-     *
-     * Note! pgmR3SaveDone will always be called and it is therefore responsible
-     *       for cleaning up.
-     */
-    PPGMRAMRANGE pCur;
-    pgmLock(pVM);
-    do
-    {
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
-        {
-            if (   !pCur->paLSPages
-                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
-            {
-                uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
-                uint32_t const  cPages = pCur->cb >> PAGE_SHIFT;
-                pgmUnlock(pVM);
-                PPGMLIVESAVERAMPAGE paLSPages = (PPGMLIVESAVERAMPAGE)MMR3HeapAllocZ(pVM, MM_TAG_PGM, cPages * sizeof(PGMLIVESAVERAMPAGE));
-                if (!paLSPages)
-                    return VERR_NO_MEMORY;
-                pgmLock(pVM);
-                if (pVM->pgm.s.idRamRangesGen != idRamRangesGen)
-                {
-                    pgmUnlock(pVM);
-                    MMR3HeapFree(paLSPages);
-                    pgmLock(pVM);
-                    break;              /* try again */
-                }
-                pCur->paLSPages = paLSPages;
-
-                /*
-                 * Initialize the array.
-                 */
-                uint32_t iPage = cPages;
-                while (iPage-- > 0)
-                {
-                    /** @todo yield critsect! (after moving this away from EMT0) */
-                    PCPGMPAGE pPage = &pCur->aPages[iPage];
-                    paLSPages[iPage].cDirtied               = 0;
-                    paLSPages[iPage].fDirty                 = 1; /* everything is dirty at this time */
-                    paLSPages[iPage].fWriteMonitored        = 0;
-                    paLSPages[iPage].fWriteMonitoredJustNow = 0;
-                    paLSPages[iPage].u2Reserved             = 0;
-                    switch (PGM_PAGE_GET_TYPE(pPage))
-                    {
-                        case PGMPAGETYPE_RAM:
-                            if (    PGM_PAGE_IS_ZERO(pPage)
-                                ||  PGM_PAGE_IS_BALLOONED(pPage))
-                            {
-                                paLSPages[iPage].fZero   = 1;
-                                paLSPages[iPage].fShared = 0;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                paLSPages[iPage].u32Crc  = PGM_STATE_CRC32_ZERO_PAGE;
-#endif
-                            }
-                            else if (PGM_PAGE_IS_SHARED(pPage))
-                            {
-                                paLSPages[iPage].fZero   = 0;
-                                paLSPages[iPage].fShared = 1;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                paLSPages[iPage].u32Crc  = UINT32_MAX;
-#endif
-                            }
-                            else
-                            {
-                                paLSPages[iPage].fZero   = 0;
-                                paLSPages[iPage].fShared = 0;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                paLSPages[iPage].u32Crc  = UINT32_MAX;
-#endif
-                            }
-                            paLSPages[iPage].fIgnore     = 0;
-                            pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
-                            break;
-
-                        case PGMPAGETYPE_ROM_SHADOW:
-                        case PGMPAGETYPE_ROM:
-                        {
-                            paLSPages[iPage].fZero   = 0;
-                            paLSPages[iPage].fShared = 0;
-                            paLSPages[iPage].fDirty  = 0;
-                            paLSPages[iPage].fIgnore = 1;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                            paLSPages[iPage].u32Crc  = UINT32_MAX;
-#endif
-                            pVM->pgm.s.LiveSave.cIgnoredPages++;
-                            break;
-                        }
-
-                        default:
-                            AssertMsgFailed(("%R[pgmpage]", pPage));
-                        case PGMPAGETYPE_MMIO2:
-                        case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
-                            paLSPages[iPage].fZero   = 0;
-                            paLSPages[iPage].fShared = 0;
-                            paLSPages[iPage].fDirty  = 0;
-                            paLSPages[iPage].fIgnore = 1;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                            paLSPages[iPage].u32Crc  = UINT32_MAX;
-#endif
-                            pVM->pgm.s.LiveSave.cIgnoredPages++;
-                            break;
-
-                        case PGMPAGETYPE_MMIO:
-                            paLSPages[iPage].fZero   = 0;
-                            paLSPages[iPage].fShared = 0;
-                            paLSPages[iPage].fDirty  = 0;
-                            paLSPages[iPage].fIgnore = 1;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                            paLSPages[iPage].u32Crc  = UINT32_MAX;
-#endif
-                            pVM->pgm.s.LiveSave.cIgnoredPages++;
-                            break;
-                    }
-                }
-            }
-        }
-    } while (pCur);
-    pgmUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Saves the RAM configuration.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int pgmR3SaveRamConfig(PVM pVM, PSSMHANDLE pSSM)
-{
-    uint32_t cbRamHole = 0;
-    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
-    AssertRCReturn(rc, rc);
-
-    uint64_t cbRam     = 0;
-    rc = CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
-    AssertRCReturn(rc, rc);
-
-    SSMR3PutU32(pSSM, cbRamHole);
-    return SSMR3PutU64(pSSM, cbRam);
-}
-
-
-/**
- * Loads and verifies the RAM configuration.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int pgmR3LoadRamConfig(PVM pVM, PSSMHANDLE pSSM)
-{
-    uint32_t cbRamHoleCfg = 0;
-    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHoleCfg, MM_RAM_HOLE_SIZE_DEFAULT);
-    AssertRCReturn(rc, rc);
-
-    uint64_t cbRamCfg     = 0;
-    rc = CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRamCfg, 0);
-    AssertRCReturn(rc, rc);
-
-    uint32_t cbRamHoleSaved;
-    SSMR3GetU32(pSSM, &cbRamHoleSaved);
-
-    uint64_t cbRamSaved;
-    rc = SSMR3GetU64(pSSM, &cbRamSaved);
-    AssertRCReturn(rc, rc);
-
-    if (   cbRamHoleCfg != cbRamHoleSaved
-        || cbRamCfg     != cbRamSaved)
-        return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Ram config mismatch: saved=%RX64/%RX32 config=%RX64/%RX32 (RAM/Hole)"),
-                                cbRamSaved, cbRamHoleSaved, cbRamCfg, cbRamHoleCfg);
-    return VINF_SUCCESS;
-}
-
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-
-/**
- * Calculates the CRC-32 for a RAM page and updates the live save page tracking
- * info with it.
- *
- * @param   pVM                 The VM handle.
- * @param   pCur                The current RAM range.
- * @param   paLSPages           The current array of live save page tracking
- *                              structures.
- * @param   iPage               The page index.
- */
-static void pgmR3StateCalcCrc32ForRamPage(PVM pVM, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage)
-{
-    RTGCPHYS    GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-    void const *pvPage;
-    int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, &pCur->aPages[iPage], GCPhys, &pvPage);
-    if (RT_SUCCESS(rc))
-        paLSPages[iPage].u32Crc = RTCrc32(pvPage, PAGE_SIZE);
-    else
-        paLSPages[iPage].u32Crc = UINT32_MAX; /* Invalid */
-}
-
-
-/**
- * Verifies the CRC-32 for a page given it's raw bits.
- *
- * @param   pvPage              The page bits.
- * @param   pCur                The current RAM range.
- * @param   paLSPages           The current array of live save page tracking
- *                              structures.
- * @param   iPage               The page index.
- */
-static void pgmR3StateVerifyCrc32ForPage(void const *pvPage, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage, const  char *pszWhere)
-{
-    if (paLSPages[iPage].u32Crc != UINT32_MAX)
-    {
-        uint32_t u32Crc = RTCrc32(pvPage, PAGE_SIZE);
-        Assert(   (   !PGM_PAGE_IS_ZERO(&pCur->aPages[iPage])
-                   && !PGM_PAGE_IS_BALLOONED(&pCur->aPages[iPage]))
-               || u32Crc == PGM_STATE_CRC32_ZERO_PAGE);
-        AssertMsg(paLSPages[iPage].u32Crc == u32Crc,
-                  ("%08x != %08x for %RGp %R[pgmpage] %s\n", paLSPages[iPage].u32Crc, u32Crc,
-                   pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pCur->aPages[iPage], pszWhere));
-    }
-}
-
-
-/**
- * Verifies the CRC-32 for a RAM page.
- *
- * @param   pVM                 The VM handle.
- * @param   pCur                The current RAM range.
- * @param   paLSPages           The current array of live save page tracking
- *                              structures.
- * @param   iPage               The page index.
- */
-static void pgmR3StateVerifyCrc32ForRamPage(PVM pVM, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage, const char *pszWhere)
-{
-    if (paLSPages[iPage].u32Crc != UINT32_MAX)
-    {
-        RTGCPHYS    GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-        void const *pvPage;
-        int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, &pCur->aPages[iPage], GCPhys, &pvPage);
-        if (RT_SUCCESS(rc))
-            pgmR3StateVerifyCrc32ForPage(pvPage, pCur, paLSPages, iPage, pszWhere);
-    }
-}
-
-#endif /* PGMLIVESAVERAMPAGE_WITH_CRC32 */
-
-/**
- * Scan for RAM page modifications and reprotect them.
- *
- * @param   pVM                 The VM handle.
- * @param   fFinalPass          Whether this is the final pass or not.
- */
-static void pgmR3ScanRamPages(PVM pVM, bool fFinalPass)
-{
-    /*
-     * The RAM.
-     */
-    RTGCPHYS GCPhysCur = 0;
-    PPGMRAMRANGE pCur;
-    pgmLock(pVM);
-    do
-    {
-        uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
-        {
-            if (    pCur->GCPhysLast > GCPhysCur
-                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
-            {
-                PPGMLIVESAVERAMPAGE paLSPages = pCur->paLSPages;
-                uint32_t         cPages    = pCur->cb >> PAGE_SHIFT;
-                uint32_t         iPage     = GCPhysCur <= pCur->GCPhys ? 0 : (GCPhysCur - pCur->GCPhys) >> PAGE_SHIFT;
-                GCPhysCur = 0;
-                for (; iPage < cPages; iPage++)
-                {
-                    /* Do yield first. */
-                    if (   !fFinalPass
-#ifndef PGMLIVESAVERAMPAGE_WITH_CRC32
-                        && (iPage & 0x7ff) == 0x100
-#endif
-                        && PDMR3CritSectYield(&pVM->pgm.s.CritSect)
-                        && pVM->pgm.s.idRamRangesGen != idRamRangesGen)
-                    {
-                        GCPhysCur = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-                        break; /* restart */
-                    }
-
-                    /* Skip already ignored pages. */
-                    if (paLSPages[iPage].fIgnore)
-                        continue;
-
-                    if (RT_LIKELY(PGM_PAGE_GET_TYPE(&pCur->aPages[iPage]) == PGMPAGETYPE_RAM))
-                    {
-                        /*
-                         * A RAM page.
-                         */
-                        switch (PGM_PAGE_GET_STATE(&pCur->aPages[iPage]))
-                        {
-                            case PGM_PAGE_STATE_ALLOCATED:
-                                /** @todo Optimize this: Don't always re-enable write
-                                 * monitoring if the page is known to be very busy. */
-                                if (PGM_PAGE_IS_WRITTEN_TO(&pCur->aPages[iPage]))
-                                {
-                                    Assert(paLSPages[iPage].fWriteMonitored);
-                                    PGM_PAGE_CLEAR_WRITTEN_TO(&pCur->aPages[iPage]);
-                                    Assert(pVM->pgm.s.cWrittenToPages > 0);
-                                    pVM->pgm.s.cWrittenToPages--;
-                                }
-                                else
-                                {
-                                    Assert(!paLSPages[iPage].fWriteMonitored);
-                                    pVM->pgm.s.LiveSave.Ram.cMonitoredPages++;
-                                }
-
-                                if (!paLSPages[iPage].fDirty)
-                                {
-                                    pVM->pgm.s.LiveSave.Ram.cReadyPages--;
-                                    if (paLSPages[iPage].fZero)
-                                        pVM->pgm.s.LiveSave.Ram.cZeroPages--;
-                                    pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
-                                    if (++paLSPages[iPage].cDirtied > PGMLIVSAVEPAGE_MAX_DIRTIED)
-                                        paLSPages[iPage].cDirtied = PGMLIVSAVEPAGE_MAX_DIRTIED;
-                                }
-
-                                PGM_PAGE_SET_STATE(&pCur->aPages[iPage], PGM_PAGE_STATE_WRITE_MONITORED);
-                                pVM->pgm.s.cMonitoredPages++;
-                                paLSPages[iPage].fWriteMonitored        = 1;
-                                paLSPages[iPage].fWriteMonitoredJustNow = 1;
-                                paLSPages[iPage].fDirty                 = 1;
-                                paLSPages[iPage].fZero                  = 0;
-                                paLSPages[iPage].fShared                = 0;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                paLSPages[iPage].u32Crc                 = UINT32_MAX; /* invalid */
-#endif
-                                break;
-
-                            case PGM_PAGE_STATE_WRITE_MONITORED:
-                                Assert(paLSPages[iPage].fWriteMonitored);
-                                if (PGM_PAGE_GET_WRITE_LOCKS(&pCur->aPages[iPage]) == 0)
-                                {
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                    if (paLSPages[iPage].fWriteMonitoredJustNow)
-                                        pgmR3StateCalcCrc32ForRamPage(pVM, pCur, paLSPages, iPage);
-                                    else
-                                        pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "scan");
-#endif
-                                    paLSPages[iPage].fWriteMonitoredJustNow = 0;
-                                }
-                                else
-                                {
-                                    paLSPages[iPage].fWriteMonitoredJustNow = 1;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                    paLSPages[iPage].u32Crc                 = UINT32_MAX; /* invalid */
-#endif
-                                    if (!paLSPages[iPage].fDirty)
-                                    {
-                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
-                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
-                                        if (++paLSPages[iPage].cDirtied > PGMLIVSAVEPAGE_MAX_DIRTIED)
-                                            paLSPages[iPage].cDirtied = PGMLIVSAVEPAGE_MAX_DIRTIED;
-                                    }
-                                }
-                                break;
-
-                            case PGM_PAGE_STATE_ZERO:
-                                if (!paLSPages[iPage].fZero)
-                                {
-                                    if (!paLSPages[iPage].fDirty)
-                                    {
-                                        paLSPages[iPage].fDirty = 1;
-                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
-                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
-                                    }
-                                    paLSPages[iPage].fZero = 1;
-                                    paLSPages[iPage].fShared = 0;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                    paLSPages[iPage].u32Crc = PGM_STATE_CRC32_ZERO_PAGE;
-#endif
-                                }
-                                break;
-
-                            case PGM_PAGE_STATE_BALLOONED:
-                                if (!paLSPages[iPage].fZero)
-                                {
-                                    if (!paLSPages[iPage].fDirty)
-                                    {
-                                        paLSPages[iPage].fDirty = 1;
-                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
-                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
-                                    }
-                                    paLSPages[iPage].fZero = 1;
-                                    paLSPages[iPage].fShared = 0;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                    paLSPages[iPage].u32Crc = PGM_STATE_CRC32_ZERO_PAGE;
-#endif
-                                }
-                                break;
-
-                            case PGM_PAGE_STATE_SHARED:
-                                if (!paLSPages[iPage].fShared)
-                                {
-                                    if (!paLSPages[iPage].fDirty)
-                                    {
-                                        paLSPages[iPage].fDirty = 1;
-                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
-                                        if (paLSPages[iPage].fZero)
-                                            pVM->pgm.s.LiveSave.Ram.cZeroPages--;
-                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
-                                    }
-                                    paLSPages[iPage].fZero = 0;
-                                    paLSPages[iPage].fShared = 1;
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                                    pgmR3StateCalcCrc32ForRamPage(pVM, pCur, paLSPages, iPage);
-#endif
-                                }
-                                break;
-                        }
-                    }
-                    else
-                    {
-                        /*
-                         * All other types => Ignore the page.
-                         */
-                        Assert(!paLSPages[iPage].fIgnore); /* skipped before switch */
-                        paLSPages[iPage].fIgnore = 1;
-                        if (paLSPages[iPage].fWriteMonitored)
-                        {
-                            /** @todo this doesn't hold water when we start monitoring MMIO2 and ROM shadow
-                             *        pages! */
-                            if (RT_UNLIKELY(PGM_PAGE_GET_STATE(&pCur->aPages[iPage]) == PGM_PAGE_STATE_WRITE_MONITORED))
-                            {
-                                AssertMsgFailed(("%R[pgmpage]", &pCur->aPages[iPage])); /* shouldn't happen. */
-                                PGM_PAGE_SET_STATE(&pCur->aPages[iPage], PGM_PAGE_STATE_ALLOCATED);
-                                Assert(pVM->pgm.s.cMonitoredPages > 0);
-                                pVM->pgm.s.cMonitoredPages--;
-                            }
-                            if (PGM_PAGE_IS_WRITTEN_TO(&pCur->aPages[iPage]))
-                            {
-                                PGM_PAGE_CLEAR_WRITTEN_TO(&pCur->aPages[iPage]);
-                                Assert(pVM->pgm.s.cWrittenToPages > 0);
-                                pVM->pgm.s.cWrittenToPages--;
-                            }
-                            pVM->pgm.s.LiveSave.Ram.cMonitoredPages--;
-                        }
-
-                        /** @todo the counting doesn't quite work out here. fix later? */
-                        if (paLSPages[iPage].fDirty)
-                            pVM->pgm.s.LiveSave.Ram.cDirtyPages--;
-                        else
-                        {
-                            pVM->pgm.s.LiveSave.Ram.cReadyPages--;
-                            if (paLSPages[iPage].fZero)
-                                pVM->pgm.s.LiveSave.Ram.cZeroPages--;
-                        }
-                        pVM->pgm.s.LiveSave.cIgnoredPages++;
-                    }
-                } /* for each page in range */
-
-                if (GCPhysCur != 0)
-                    break; /* Yield + ramrange change */
-                GCPhysCur = pCur->GCPhysLast;
-            }
-        } /* for each range */
-    } while (pCur);
-    pgmUnlock(pVM);
-}
-
-
-/**
- * Save quiescent RAM pages.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- * @param   fLiveSave           Whether it's a live save or not.
- * @param   uPass               The pass number.
- */
-static int pgmR3SaveRamPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, uint32_t uPass)
-{
-    /*
-     * The RAM.
-     */
-    RTGCPHYS GCPhysLast = NIL_RTGCPHYS;
-    RTGCPHYS GCPhysCur = 0;
-    PPGMRAMRANGE pCur;
-    bool fFTMDeltaSaveActive = FTMIsDeltaLoadSaveActive(pVM);
-
-    pgmLock(pVM);
-    do
-    {
-        uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
-        {
-            if (   pCur->GCPhysLast > GCPhysCur
-                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
-            {
-                PPGMLIVESAVERAMPAGE paLSPages = pCur->paLSPages;
-                uint32_t         cPages    = pCur->cb >> PAGE_SHIFT;
-                uint32_t         iPage     = GCPhysCur <= pCur->GCPhys ? 0 : (GCPhysCur - pCur->GCPhys) >> PAGE_SHIFT;
-                GCPhysCur = 0;
-                for (; iPage < cPages; iPage++)
-                {
-                    /* Do yield first. */
-                    if (   uPass != SSM_PASS_FINAL
-                        && (iPage & 0x7ff) == 0x100
-                        && PDMR3CritSectYield(&pVM->pgm.s.CritSect)
-                        && pVM->pgm.s.idRamRangesGen != idRamRangesGen)
-                    {
-                        GCPhysCur = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-                        break; /* restart */
-                    }
-
-                    PPGMPAGE pCurPage = &pCur->aPages[iPage];
-
-                    /*
-                     * Only save pages that haven't changed since last scan and are dirty.
-                     */
-                    if (    uPass != SSM_PASS_FINAL
-                        &&  paLSPages)
-                    {
-                        if (!paLSPages[iPage].fDirty)
-                            continue;
-                        if (paLSPages[iPage].fWriteMonitoredJustNow)
-                            continue;
-                        if (paLSPages[iPage].fIgnore)
-                            continue;
-                        if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM) /* in case of recent remappings */
-                            continue;
-                        if (    PGM_PAGE_GET_STATE(pCurPage)
-                            !=  (  paLSPages[iPage].fZero
-                                 ? PGM_PAGE_STATE_ZERO
-                                 : paLSPages[iPage].fShared
-                                 ? PGM_PAGE_STATE_SHARED
-                                 : PGM_PAGE_STATE_WRITE_MONITORED))
-                            continue;
-                        if (PGM_PAGE_GET_WRITE_LOCKS(&pCur->aPages[iPage]) > 0)
-                            continue;
-                    }
-                    else
-                    {
-                        if (   paLSPages
-                            && !paLSPages[iPage].fDirty
-                            && !paLSPages[iPage].fIgnore)
-                        {
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                            if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM)
-                                pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "save#1");
-#endif
-                            continue;
-                        }
-                        if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM)
-                            continue;
-                    }
-
-                    /*
-                     * Do the saving outside the PGM critsect since SSM may block on I/O.
-                     */
-                    int         rc;
-                    RTGCPHYS    GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-                    bool        fZero  = PGM_PAGE_IS_ZERO(pCurPage) || PGM_PAGE_IS_BALLOONED(pCurPage);
-                    bool        fSkipped = false;
-
-                    if (!fZero)
-                    {
-                        /*
-                         * Copy the page and then save it outside the lock (since any
-                         * SSM call may block).
-                         */
-                        uint8_t     abPage[PAGE_SIZE];
-                        void const *pvPage;
-                        rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pCurPage, GCPhys, &pvPage);
-                        if (RT_SUCCESS(rc))
-                        {
-                            memcpy(abPage, pvPage, PAGE_SIZE);
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                            if (paLSPages)
-                                pgmR3StateVerifyCrc32ForPage(abPage, pCur, paLSPages, iPage, "save#3");
-#endif
-                        }
-                        pgmUnlock(pVM);
-                        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
-
-                        /* Try save some memory when restoring. */
-                        if (!ASMMemIsZeroPage(pvPage))
-                        {
-                            if (fFTMDeltaSaveActive)
-                            {
-                                if (    PGM_PAGE_IS_WRITTEN_TO(pCurPage)
-                                    ||  PGM_PAGE_IS_FT_DIRTY(pCurPage))
-                                {
-                                    if (GCPhys == GCPhysLast + PAGE_SIZE)
-                                        SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW);
-                                    else
-                                    {
-                                        SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW | PGM_STATE_REC_FLAG_ADDR);
-                                        SSMR3PutGCPhys(pSSM, GCPhys);
-                                    }
-                                    rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
-                                    PGM_PAGE_CLEAR_WRITTEN_TO(pCurPage);
-                                    PGM_PAGE_CLEAR_FT_DIRTY(pCurPage);
-                                }
-                                /* else nothing changed, so skip it. */
-                                else
-                                    fSkipped = true;
-                            }
-                            else
-                            {
-                                if (GCPhys == GCPhysLast + PAGE_SIZE)
-                                    SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW);
-                                else
-                                {
-                                    SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW | PGM_STATE_REC_FLAG_ADDR);
-                                    SSMR3PutGCPhys(pSSM, GCPhys);
-                                }
-                                rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
-                            }
-                        }
-                        else
-                        {
-                            if (GCPhys == GCPhysLast + PAGE_SIZE)
-                                rc = SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO);
-                            else
-                            {
-                                SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO | PGM_STATE_REC_FLAG_ADDR);
-                                rc = SSMR3PutGCPhys(pSSM, GCPhys);
-                            }
-                        }
-                    }
-                    else
-                    {
-                        /*
-                         * Dirty zero page.
-                         */
-#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
-                        if (paLSPages)
-                            pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "save#2");
-#endif
-                        pgmUnlock(pVM);
-
-                        if (GCPhys == GCPhysLast + PAGE_SIZE)
-                            rc = SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO);
-                        else
-                        {
-                            SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO | PGM_STATE_REC_FLAG_ADDR);
-                            rc = SSMR3PutGCPhys(pSSM, GCPhys);
-                        }
-                    }
-                    if (RT_FAILURE(rc))
-                        return rc;
-
-                    pgmLock(pVM);
-                    if (!fSkipped)
-                        GCPhysLast = GCPhys;
-                    if (paLSPages)
-                    {
-                        paLSPages[iPage].fDirty = 0;
-                        pVM->pgm.s.LiveSave.Ram.cReadyPages++;
-                        if (fZero)
-                            pVM->pgm.s.LiveSave.Ram.cZeroPages++;
-                        pVM->pgm.s.LiveSave.Ram.cDirtyPages--;
-                        pVM->pgm.s.LiveSave.cSavedPages++;
-                    }
-                    if (idRamRangesGen != pVM->pgm.s.idRamRangesGen)
-                    {
-                        GCPhysCur = GCPhys | PAGE_OFFSET_MASK;
-                        break; /* restart */
-                    }
-
-                } /* for each page in range */
-
-                if (GCPhysCur != 0)
-                    break; /* Yield + ramrange change */
-                GCPhysCur = pCur->GCPhysLast;
-            }
-        } /* for each range */
-    } while (pCur);
-
-    pgmUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Cleans up RAM pages after a live save.
- *
- * @param   pVM                 The VM handle.
- */
-static void pgmR3DoneRamPages(PVM pVM)
-{
-    /*
-     * Free the tracking arrays and disable write monitoring.
-     *
-     * Play nice with the PGM lock in case we're called while the VM is still
-     * running.  This means we have to delay the freeing since we wish to use
-     * paLSPages as an indicator of which RAM ranges which we need to scan for
-     * write monitored pages.
-     */
-    void *pvToFree = NULL;
-    PPGMRAMRANGE pCur;
-    uint32_t cMonitoredPages = 0;
-    pgmLock(pVM);
-    do
-    {
-        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
-        {
-            if (pCur->paLSPages)
-            {
-                if (pvToFree)
-                {
-                    uint32_t idRamRangesGen = pVM->pgm.s.idRamRangesGen;
-                    pgmUnlock(pVM);
-                    MMR3HeapFree(pvToFree);
-                    pvToFree = NULL;
-                    pgmLock(pVM);
-                    if (idRamRangesGen != pVM->pgm.s.idRamRangesGen)
-                        break;          /* start over again. */
-                }
-
-                pvToFree = pCur->paLSPages;
-                pCur->paLSPages = NULL;
-
-                uint32_t iPage = pCur->cb >> PAGE_SHIFT;
-                while (iPage--)
-                {
-                    PPGMPAGE pPage = &pCur->aPages[iPage];
-                    PGM_PAGE_CLEAR_WRITTEN_TO(pPage);
-                    if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
-                    {
-                        PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
-                        cMonitoredPages++;
-                    }
-                }
-            }
-        }
-    } while (pCur);
-
-    Assert(pVM->pgm.s.cMonitoredPages >= cMonitoredPages);
-    if (pVM->pgm.s.cMonitoredPages < cMonitoredPages)
-        pVM->pgm.s.cMonitoredPages = 0;
-    else
-        pVM->pgm.s.cMonitoredPages -= cMonitoredPages;
-
-    pgmUnlock(pVM);
-
-    MMR3HeapFree(pvToFree);
-    pvToFree = NULL;
-}
-
-
-/**
- * Execute a live save pass.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle.
- * @param   pSSM        The SSM handle.
- */
-static DECLCALLBACK(int) pgmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    int rc;
-
-    /*
-     * Save the MMIO2 and ROM range IDs in pass 0.
-     */
-    if (uPass == 0)
-    {
-        rc = pgmR3SaveRamConfig(pVM, pSSM);
-        if (RT_FAILURE(rc))
-            return rc;
-        rc = pgmR3SaveRomRanges(pVM, pSSM);
-        if (RT_FAILURE(rc))
-            return rc;
-        rc = pgmR3SaveMmio2Ranges(pVM, pSSM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-    /*
-     * Reset the page-per-second estimate to avoid inflation by the initial
-     * load of zero pages.  pgmR3LiveVote ASSUMES this is done at pass 7.
-     */
-    else if (uPass == 7)
-    {
-        pVM->pgm.s.LiveSave.cSavedPages  = 0;
-        pVM->pgm.s.LiveSave.uSaveStartNS = RTTimeNanoTS();
-    }
-
-    /*
-     * Do the scanning.
-     */
-    pgmR3ScanRomPages(pVM);
-    pgmR3ScanMmio2Pages(pVM, uPass);
-    pgmR3ScanRamPages(pVM, false /*fFinalPass*/);
-    pgmR3PoolClearAll(pVM, true /*fFlushRemTlb*/); /** @todo this could perhaps be optimized a bit. */
-
-    /*
-     * Save the pages.
-     */
-    if (uPass == 0)
-        rc = pgmR3SaveRomVirginPages(  pVM, pSSM, true /*fLiveSave*/);
-    else
-        rc = VINF_SUCCESS;
-    if (RT_SUCCESS(rc))
-        rc = pgmR3SaveShadowedRomPages(pVM, pSSM, true /*fLiveSave*/, false /*fFinalPass*/);
-    if (RT_SUCCESS(rc))
-        rc = pgmR3SaveMmio2Pages(      pVM, pSSM, true /*fLiveSave*/, uPass);
-    if (RT_SUCCESS(rc))
-        rc = pgmR3SaveRamPages(        pVM, pSSM, true /*fLiveSave*/, uPass);
-    SSMR3PutU8(pSSM, PGM_STATE_REC_END);    /* (Ignore the rc, SSM takes care of it.) */
-
-    return rc;
-}
-
-
-/**
- * Votes on whether the live save phase is done or not.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle.
- * @param   pSSM        The SSM handle.
- * @param   uPass       The data pass.
- */
-static DECLCALLBACK(int)  pgmR3LiveVote(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    /*
-     * Update and calculate parameters used in the decision making.
-     */
-    const uint32_t cHistoryEntries = RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory);
-
-    /* update history. */
-    pgmLock(pVM);
-    uint32_t const cWrittenToPages = pVM->pgm.s.cWrittenToPages;
-    pgmUnlock(pVM);
-    uint32_t const cDirtyNow = pVM->pgm.s.LiveSave.Rom.cDirtyPages
-                             + pVM->pgm.s.LiveSave.Mmio2.cDirtyPages
-                             + pVM->pgm.s.LiveSave.Ram.cDirtyPages
-                             + cWrittenToPages;
-    uint32_t i = pVM->pgm.s.LiveSave.iDirtyPagesHistory;
-    pVM->pgm.s.LiveSave.acDirtyPagesHistory[i] = cDirtyNow;
-    pVM->pgm.s.LiveSave.iDirtyPagesHistory = (i + 1) % cHistoryEntries;
-
-    /* calc shortterm average (4 passes). */
-    AssertCompile(RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory) > 4);
-    uint64_t cTotal = pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
-    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 1) % cHistoryEntries];
-    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 2) % cHistoryEntries];
-    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 3) % cHistoryEntries];
-    uint32_t const cDirtyPagesShort = cTotal / 4;
-    pVM->pgm.s.LiveSave.cDirtyPagesShort = cDirtyPagesShort;
-
-    /* calc longterm average. */
-    cTotal = 0;
-    if (uPass < cHistoryEntries)
-        for (i = 0; i < cHistoryEntries && i <= uPass; i++)
-              cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
-    else
-        for (i = 0; i < cHistoryEntries; i++)
-            cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
-    uint32_t const cDirtyPagesLong = cTotal / cHistoryEntries;
-    pVM->pgm.s.LiveSave.cDirtyPagesLong = cDirtyPagesLong;
-
-    /* estimate the speed */
-    uint64_t cNsElapsed = RTTimeNanoTS() - pVM->pgm.s.LiveSave.uSaveStartNS;
-    uint32_t cPagesPerSecond = (uint32_t)(   pVM->pgm.s.LiveSave.cSavedPages
-                                          / ((long double)cNsElapsed / 1000000000.0) );
-    pVM->pgm.s.LiveSave.cPagesPerSecond = cPagesPerSecond;
-
-    /*
-     * Try make a decision.
-     */
-    if (    cDirtyPagesShort <= cDirtyPagesLong
-        &&  (   cDirtyNow    <= cDirtyPagesShort
-             || cDirtyNow - cDirtyPagesShort < RT_MIN(cDirtyPagesShort / 8, 16)
-            )
-       )
-    {
-        if (uPass > 10)
-        {
-            uint32_t cMsLeftShort   = (uint32_t)(cDirtyPagesShort / (long double)cPagesPerSecond * 1000.0);
-            uint32_t cMsLeftLong    = (uint32_t)(cDirtyPagesLong  / (long double)cPagesPerSecond * 1000.0);
-            uint32_t cMsMaxDowntime = SSMR3HandleMaxDowntime(pSSM);
-            if (cMsMaxDowntime < 32)
-                cMsMaxDowntime = 32;
-            if (   (   cMsLeftLong  <= cMsMaxDowntime
-                    && cMsLeftShort <  cMsMaxDowntime)
-                || cMsLeftShort < cMsMaxDowntime / 2
-               )
-            {
-                Log(("pgmR3LiveVote: VINF_SUCCESS - pass=%d cDirtyPagesShort=%u|%ums cDirtyPagesLong=%u|%ums cMsMaxDowntime=%u\n",
-                     uPass, cDirtyPagesShort, cMsLeftShort, cDirtyPagesLong, cMsLeftLong, cMsMaxDowntime));
-                return VINF_SUCCESS;
-            }
-        }
-        else
-        {
-            if (   (   cDirtyPagesShort <= 128
-                    && cDirtyPagesLong  <= 1024)
-                || cDirtyPagesLong <= 256
-               )
-            {
-                Log(("pgmR3LiveVote: VINF_SUCCESS - pass=%d cDirtyPagesShort=%u cDirtyPagesLong=%u\n", uPass, cDirtyPagesShort, cDirtyPagesLong));
-                return VINF_SUCCESS;
-            }
-        }
-    }
-
-    /*
-     * Come up with a completion percentage.  Currently this is a simple
-     * dirty page (long term) vs. total pages ratio + some pass trickery.
-     */
-    unsigned uPctDirty = (unsigned)(  (long double)cDirtyPagesLong
-                                    / (pVM->pgm.s.cAllPages - pVM->pgm.s.LiveSave.cIgnoredPages - pVM->pgm.s.cZeroPages) );
-    if (uPctDirty <= 100)
-        SSMR3HandleReportLivePercent(pSSM, RT_MIN(100 - uPctDirty, uPass * 2));
-    else
-        AssertMsgFailed(("uPctDirty=%u cDirtyPagesLong=%#x cAllPages=%#x cIgnoredPages=%#x cZeroPages=%#x\n",
-                         uPctDirty, cDirtyPagesLong, pVM->pgm.s.cAllPages, pVM->pgm.s.LiveSave.cIgnoredPages, pVM->pgm.s.cZeroPages));
-
-    return VINF_SSM_VOTE_FOR_ANOTHER_PASS;
-}
-
-
-/**
- * Prepare for a live save operation.
- *
- * This will attempt to allocate and initialize the tracking structures.  It
- * will also prepare for write monitoring of pages and initialize PGM::LiveSave.
- * pgmR3SaveDone will do the cleanups.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle.
- * @param   pSSM        The SSM handle.
- */
-static DECLCALLBACK(int) pgmR3LivePrep(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Indicate that we will be using the write monitoring.
-     */
-    pgmLock(pVM);
-    /** @todo find a way of mediating this when more users are added. */
-    if (pVM->pgm.s.fPhysWriteMonitoringEngaged)
-    {
-        pgmUnlock(pVM);
-        AssertLogRelFailedReturn(VERR_INTERNAL_ERROR_2);
-    }
-    pVM->pgm.s.fPhysWriteMonitoringEngaged = true;
-    pgmUnlock(pVM);
-
-    /*
-     * Initialize the statistics.
-     */
-    pVM->pgm.s.LiveSave.Rom.cReadyPages   = 0;
-    pVM->pgm.s.LiveSave.Rom.cDirtyPages   = 0;
-    pVM->pgm.s.LiveSave.Mmio2.cReadyPages = 0;
-    pVM->pgm.s.LiveSave.Mmio2.cDirtyPages = 0;
-    pVM->pgm.s.LiveSave.Ram.cReadyPages   = 0;
-    pVM->pgm.s.LiveSave.Ram.cDirtyPages   = 0;
-    pVM->pgm.s.LiveSave.cIgnoredPages     = 0;
-    pVM->pgm.s.LiveSave.fActive           = true;
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory); i++)
-        pVM->pgm.s.LiveSave.acDirtyPagesHistory[i] = UINT32_MAX / 2;
-    pVM->pgm.s.LiveSave.iDirtyPagesHistory = 0;
-    pVM->pgm.s.LiveSave.cSavedPages       = 0;
-    pVM->pgm.s.LiveSave.uSaveStartNS      = RTTimeNanoTS();
-    pVM->pgm.s.LiveSave.cPagesPerSecond   = 8192;
-
-    /*
-     * Per page type.
-     */
-    int rc = pgmR3PrepRomPages(pVM);
-    if (RT_SUCCESS(rc))
-        rc = pgmR3PrepMmio2Pages(pVM);
-    if (RT_SUCCESS(rc))
-        rc = pgmR3PrepRamPages(pVM);
-    return rc;
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) pgmR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
-{
-    int         rc;
-    unsigned    i;
-    PPGM        pPGM = &pVM->pgm.s;
-
-    /*
-     * Lock PGM and set the no-more-writes indicator.
-     */
-    pgmLock(pVM);
-    pVM->pgm.s.fNoMorePhysWrites = true;
-
-    /*
-     * Save basic data (required / unaffected by relocation).
-     */
-    bool const fMappingsFixed  = pVM->pgm.s.fMappingsFixed;
-    pVM->pgm.s.fMappingsFixed |= pVM->pgm.s.fMappingsFixedRestored;
-    SSMR3PutStruct(pSSM, pPGM, &s_aPGMFields[0]);
-    pVM->pgm.s.fMappingsFixed  = fMappingsFixed;
-
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-        SSMR3PutStruct(pSSM, &pVM->aCpus[idCpu].pgm.s, &s_aPGMCpuFields[0]);
-
-    /*
-     * The guest mappings.
-     */
-    i = 0;
-    for (PPGMMAPPING pMapping = pPGM->pMappingsR3; pMapping; pMapping = pMapping->pNextR3, i++)
-    {
-        SSMR3PutU32(      pSSM, i);
-        SSMR3PutStrZ(     pSSM, pMapping->pszDesc); /* This is the best unique id we have... */
-        SSMR3PutGCPtr(    pSSM, pMapping->GCPtr);
-        SSMR3PutGCUIntPtr(pSSM, pMapping->cPTs);
-    }
-    rc = SSMR3PutU32(pSSM, ~0); /* terminator. */
-
-    /*
-     * Save the (remainder of the) memory.
-     */
-    if (RT_SUCCESS(rc))
-    {
-        if (pVM->pgm.s.LiveSave.fActive)
-        {
-            pgmR3ScanRomPages(pVM);
-            pgmR3ScanMmio2Pages(pVM, SSM_PASS_FINAL);
-            pgmR3ScanRamPages(pVM, true /*fFinalPass*/);
-
-            rc = pgmR3SaveShadowedRomPages(    pVM, pSSM, true /*fLiveSave*/, true /*fFinalPass*/);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveMmio2Pages(      pVM, pSSM, true /*fLiveSave*/, SSM_PASS_FINAL);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveRamPages(        pVM, pSSM, true /*fLiveSave*/, SSM_PASS_FINAL);
-        }
-        else
-        {
-            rc = pgmR3SaveRamConfig(pVM, pSSM);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveRomRanges(pVM, pSSM);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveMmio2Ranges(pVM, pSSM);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveRomVirginPages(  pVM, pSSM, false /*fLiveSave*/);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveShadowedRomPages(pVM, pSSM, false /*fLiveSave*/, true /*fFinalPass*/);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveMmio2Pages(      pVM, pSSM, false /*fLiveSave*/, SSM_PASS_FINAL);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3SaveRamPages(        pVM, pSSM, false /*fLiveSave*/, SSM_PASS_FINAL);
-        }
-        SSMR3PutU8(pSSM, PGM_STATE_REC_END);    /* (Ignore the rc, SSM takes of it.) */
-    }
-
-    pgmUnlock(pVM);
-    return rc;
-}
-
-
-/**
- * Cleans up after an save state operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) pgmR3SaveDone(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Do per page type cleanups first.
-     */
-    if (pVM->pgm.s.LiveSave.fActive)
-    {
-        pgmR3DoneRomPages(pVM);
-        pgmR3DoneMmio2Pages(pVM);
-        pgmR3DoneRamPages(pVM);
-    }
-
-    /*
-     * Clear the live save indicator and disengage write monitoring.
-     */
-    pgmLock(pVM);
-    pVM->pgm.s.LiveSave.fActive = false;
-    /** @todo this is blindly assuming that we're the only user of write
-     *        monitoring. Fix this when more users are added. */
-    pVM->pgm.s.fPhysWriteMonitoringEngaged = false;
-    pgmUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Prepare state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) pgmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Call the reset function to make sure all the memory is cleared.
-     */
-    PGMR3Reset(pVM);
-    pVM->pgm.s.LiveSave.fActive = false;
-    NOREF(pSSM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Load an ignored page.
- *
- * @returns VBox status code.
- * @param   pSSM            The saved state handle.
- */
-static int pgmR3LoadPageToDevNullOld(PSSMHANDLE pSSM)
-{
-    uint8_t abPage[PAGE_SIZE];
-    return SSMR3GetMem(pSSM, &abPage[0], sizeof(abPage));
-}
-
-
-/**
- * Loads a page without any bits in the saved state, i.e. making sure it's
- * really zero.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   uType           The page type or PGMPAGETYPE_INVALID (old saved
- *                          state).
- * @param   pPage           The guest page tracking structure.
- * @param   GCPhys          The page address.
- * @param   pRam            The ram range (logging).
- */
-static int pgmR3LoadPageZeroOld(PVM pVM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
-{
-    if (    PGM_PAGE_GET_TYPE(pPage) != uType
-        &&  uType != PGMPAGETYPE_INVALID)
-        return VERR_SSM_UNEXPECTED_DATA;
-
-    /* I think this should be sufficient. */
-    if (    !PGM_PAGE_IS_ZERO(pPage)
-        &&  !PGM_PAGE_IS_BALLOONED(pPage))
-        return VERR_SSM_UNEXPECTED_DATA;
-
-    NOREF(pVM);
-    NOREF(GCPhys);
-    NOREF(pRam);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Loads a page from the saved state.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pSSM            The SSM handle.
- * @param   uType           The page type or PGMPAGETYEP_INVALID (old saved
- *                          state).
- * @param   pPage           The guest page tracking structure.
- * @param   GCPhys          The page address.
- * @param   pRam            The ram range (logging).
- */
-static int pgmR3LoadPageBitsOld(PVM pVM, PSSMHANDLE pSSM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
-{
-    /*
-     * Match up the type, dealing with MMIO2 aliases (dropped).
-     */
-    AssertLogRelMsgReturn(   PGM_PAGE_GET_TYPE(pPage) == uType
-                          || uType == PGMPAGETYPE_INVALID,
-                          ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc),
-                          VERR_SSM_UNEXPECTED_DATA);
-
-    /*
-     * Load the page.
-     */
-    void *pvPage;
-    int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvPage);
-    if (RT_SUCCESS(rc))
-        rc = SSMR3GetMem(pSSM, pvPage, PAGE_SIZE);
-
-    return rc;
-}
-
-
-/**
- * Loads a page (counter part to pgmR3SavePage).
- *
- * @returns VBox status code, fully bitched errors.
- * @param   pVM             The VM handle.
- * @param   pSSM            The SSM handle.
- * @param   uType           The page type.
- * @param   pPage           The page.
- * @param   GCPhys          The page address.
- * @param   pRam            The RAM range (for error messages).
- */
-static int pgmR3LoadPageOld(PVM pVM, PSSMHANDLE pSSM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
-{
-    uint8_t uState;
-    int rc = SSMR3GetU8(pSSM, &uState);
-    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s rc=%Rrc\n", pPage, GCPhys, pRam->pszDesc, rc), rc);
-    if (uState == 0 /* zero */)
-        rc = pgmR3LoadPageZeroOld(pVM, uType, pPage, GCPhys, pRam);
-    else if (uState == 1)
-        rc = pgmR3LoadPageBitsOld(pVM, pSSM, uType, pPage, GCPhys, pRam);
-    else
-        rc = VERR_INTERNAL_ERROR;
-    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] uState=%d uType=%d GCPhys=%RGp %s rc=%Rrc\n",
-                                 pPage, uState, uType, GCPhys, pRam->pszDesc, rc),
-                            rc);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Loads a shadowed ROM page.
- *
- * @returns VBox status code, errors are fully bitched.
- * @param   pVM             The VM handle.
- * @param   pSSM            The saved state handle.
- * @param   pPage           The page.
- * @param   GCPhys          The page address.
- * @param   pRam            The RAM range (for error messages).
- */
-static int pgmR3LoadShadowedRomPageOld(PVM pVM, PSSMHANDLE pSSM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
-{
-    /*
-     * Load and set the protection first, then load the two pages, the first
-     * one is the active the other is the passive.
-     */
-    PPGMROMPAGE pRomPage = pgmR3GetRomPage(pVM, GCPhys);
-    AssertLogRelMsgReturn(pRomPage, ("GCPhys=%RGp %s\n", GCPhys, pRam->pszDesc), VERR_INTERNAL_ERROR);
-
-    uint8_t uProt;
-    int rc = SSMR3GetU8(pSSM, &uProt);
-    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc), rc);
-    PGMROMPROT  enmProt = (PGMROMPROT)uProt;
-    AssertLogRelMsgReturn(    enmProt >= PGMROMPROT_INVALID
-                          &&  enmProt <  PGMROMPROT_END,
-                          ("enmProt=%d pPage=%R[pgmpage] GCPhys=%#x %s\n", enmProt, pPage, GCPhys, pRam->pszDesc),
-                          VERR_SSM_UNEXPECTED_DATA);
-
-    if (pRomPage->enmProt != enmProt)
-    {
-        rc = PGMR3PhysRomProtect(pVM, GCPhys, PAGE_SIZE, enmProt);
-        AssertLogRelRCReturn(rc, rc);
-        AssertLogRelReturn(pRomPage->enmProt == enmProt, VERR_INTERNAL_ERROR);
-    }
-
-    PPGMPAGE pPageActive  = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Virgin      : &pRomPage->Shadow;
-    PPGMPAGE pPagePassive = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow      : &pRomPage->Virgin;
-    uint8_t  u8ActiveType = PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM        : PGMPAGETYPE_ROM_SHADOW;
-    uint8_t  u8PassiveType= PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM_SHADOW : PGMPAGETYPE_ROM;
-
-    /** @todo this isn't entirely correct as long as pgmPhysGCPhys2CCPtrInternal is
-     *        used down the line (will the 2nd page will be written to the first
-     *        one because of a false TLB hit since the TLB is using GCPhys and
-     *        doesn't check the HCPhys of the desired page). */
-    rc = pgmR3LoadPageOld(pVM, pSSM, u8ActiveType, pPage, GCPhys, pRam);
-    if (RT_SUCCESS(rc))
-    {
-        *pPageActive = *pPage;
-        rc = pgmR3LoadPageOld(pVM, pSSM, u8PassiveType, pPagePassive, GCPhys, pRam);
-    }
-    return rc;
-}
-
-/**
- * Ram range flags and bits for older versions of the saved state.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM handle
- * @param   pSSM        The SSM handle.
- * @param   uVersion    The saved state version.
- */
-static int pgmR3LoadMemoryOld(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
-{
-    PPGM pPGM = &pVM->pgm.s;
-
-    /*
-     * Ram range flags and bits.
-     */
-    uint32_t i = 0;
-    for (PPGMRAMRANGE pRam = pPGM->pRamRangesR3; ; pRam = pRam->pNextR3, i++)
-    {
-        /* Check the sequence number / separator. */
-        uint32_t u32Sep;
-        int rc = SSMR3GetU32(pSSM, &u32Sep);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (u32Sep == ~0U)
-            break;
-        if (u32Sep != i)
-        {
-            AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
-            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-        }
-        AssertLogRelReturn(pRam, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-        /* Get the range details. */
-        RTGCPHYS GCPhys;
-        SSMR3GetGCPhys(pSSM, &GCPhys);
-        RTGCPHYS GCPhysLast;
-        SSMR3GetGCPhys(pSSM, &GCPhysLast);
-        RTGCPHYS cb;
-        SSMR3GetGCPhys(pSSM, &cb);
-        uint8_t     fHaveBits;
-        rc = SSMR3GetU8(pSSM, &fHaveBits);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (fHaveBits & ~1)
-        {
-            AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
-            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-        }
-        size_t  cchDesc = 0;
-        char    szDesc[256];
-        szDesc[0] = '\0';
-        if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
-        {
-            rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
-            if (RT_FAILURE(rc))
-                return rc;
-            /* Since we've modified the description strings in r45878, only compare
-               them if the saved state is more recent. */
-            if (uVersion != PGM_SAVED_STATE_VERSION_RR_DESC)
-                cchDesc = strlen(szDesc);
-        }
-
-        /*
-         * Match it up with the current range.
-         *
-         * Note there is a hack for dealing with the high BIOS mapping
-         * in the old saved state format, this means we might not have
-         * a 1:1 match on success.
-         */
-        if (    (   GCPhys     != pRam->GCPhys
-                 || GCPhysLast != pRam->GCPhysLast
-                 || cb         != pRam->cb
-                 ||  (   cchDesc
-                      && strcmp(szDesc, pRam->pszDesc)) )
-                /* Hack for PDMDevHlpPhysReserve(pDevIns, 0xfff80000, 0x80000, "High ROM Region"); */
-            &&  (   uVersion != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE
-                 || GCPhys     != UINT32_C(0xfff80000)
-                 || GCPhysLast != UINT32_C(0xffffffff)
-                 || pRam->GCPhysLast != GCPhysLast
-                 || pRam->GCPhys     <  GCPhys
-                 || !fHaveBits)
-           )
-        {
-            LogRel(("Ram range: %RGp-%RGp %RGp bytes %s %s\n"
-                    "State    : %RGp-%RGp %RGp bytes %s %s\n",
-                    pRam->GCPhys, pRam->GCPhysLast, pRam->cb, pRam->pvR3 ? "bits" : "nobits", pRam->pszDesc,
-                    GCPhys, GCPhysLast, cb, fHaveBits ? "bits" : "nobits", szDesc));
-            /*
-             * If we're loading a state for debugging purpose, don't make a fuss if
-             * the MMIO and ROM stuff isn't 100% right, just skip the mismatches.
-             */
-            if (    SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT
-                ||  GCPhys < 8 * _1M)
-                return SSMR3SetCfgError(pSSM, RT_SRC_POS,
-                                        N_("RAM range mismatch; saved={%RGp-%RGp %RGp bytes %s %s} config={%RGp-%RGp %RGp bytes %s %s}"),
-                                        GCPhys, GCPhysLast, cb, fHaveBits ? "bits" : "nobits", szDesc,
-                                        pRam->GCPhys, pRam->GCPhysLast, pRam->cb, pRam->pvR3 ? "bits" : "nobits", pRam->pszDesc);
-
-            AssertMsgFailed(("debug skipping not implemented, sorry\n"));
-            continue;
-        }
-
-        uint32_t cPages = (GCPhysLast - GCPhys + 1) >> PAGE_SHIFT;
-        if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
-        {
-            /*
-             * Load the pages one by one.
-             */
-            for (uint32_t iPage = 0; iPage < cPages; iPage++)
-            {
-                RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
-                PPGMPAGE        pPage      = &pRam->aPages[iPage];
-                uint8_t         uType;
-                rc = SSMR3GetU8(pSSM, &uType);
-                AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] iPage=%#x GCPhysPage=%#x %s\n", pPage, iPage, GCPhysPage, pRam->pszDesc), rc);
-                if (uType == PGMPAGETYPE_ROM_SHADOW)
-                    rc = pgmR3LoadShadowedRomPageOld(pVM, pSSM, pPage, GCPhysPage, pRam);
-                else
-                    rc = pgmR3LoadPageOld(pVM, pSSM, uType, pPage, GCPhysPage, pRam);
-                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
-            }
-        }
-        else
-        {
-            /*
-             * Old format.
-             */
-
-            /* Of the page flags, pick up MMIO2 and ROM/RESERVED for the !fHaveBits case.
-               The rest is generally irrelevant and wrong since the stuff have to match registrations. */
-            uint32_t fFlags = 0;
-            for (uint32_t iPage = 0; iPage < cPages; iPage++)
-            {
-                uint16_t u16Flags;
-                rc = SSMR3GetU16(pSSM, &u16Flags);
-                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
-                fFlags |= u16Flags;
-            }
-
-            /* Load the bits */
-            if (    !fHaveBits
-                &&  GCPhysLast < UINT32_C(0xe0000000))
-            {
-                /*
-                 * Dynamic chunks.
-                 */
-                const uint32_t cPagesInChunk = (1*1024*1024) >> PAGE_SHIFT;
-                AssertLogRelMsgReturn(cPages % cPagesInChunk == 0,
-                                      ("cPages=%#x cPagesInChunk=%#x\n", cPages, cPagesInChunk, pRam->GCPhys, pRam->pszDesc),
-                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-                for (uint32_t iPage = 0; iPage < cPages; /* incremented by inner loop */ )
-                {
-                    uint8_t fPresent;
-                    rc = SSMR3GetU8(pSSM, &fPresent);
-                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
-                    AssertLogRelMsgReturn(fPresent == (uint8_t)true || fPresent == (uint8_t)false,
-                                          ("fPresent=%#x iPage=%#x GCPhys=%#x %s\n", fPresent, iPage, pRam->GCPhys, pRam->pszDesc),
-                                          VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-                    for (uint32_t iChunkPage = 0; iChunkPage < cPagesInChunk; iChunkPage++, iPage++)
-                    {
-                        RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
-                        PPGMPAGE        pPage      = &pRam->aPages[iPage];
-                        if (fPresent)
-                        {
-                            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
-                                rc = pgmR3LoadPageToDevNullOld(pSSM);
-                            else
-                                rc = pgmR3LoadPageBitsOld(pVM, pSSM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
-                        }
-                        else
-                            rc = pgmR3LoadPageZeroOld(pVM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
-                        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
-                    }
-                }
-            }
-            else if (pRam->pvR3)
-            {
-                /*
-                 * MMIO2.
-                 */
-                AssertLogRelMsgReturn((fFlags & 0x0f) == RT_BIT(3) /*MM_RAM_FLAGS_MMIO2*/,
-                                      ("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
-                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-                AssertLogRelMsgReturn(pRam->pvR3,
-                                      ("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc),
-                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-                rc = SSMR3GetMem(pSSM, pRam->pvR3, pRam->cb);
-                AssertLogRelMsgRCReturn(rc, ("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc), rc);
-            }
-            else if (GCPhysLast < UINT32_C(0xfff80000))
-            {
-                /*
-                 * PCI MMIO, no pages saved.
-                 */
-            }
-            else
-            {
-                /*
-                 * Load the 0xfff80000..0xffffffff BIOS range.
-                 * It starts with X reserved pages that we have to skip over since
-                 * the RAMRANGE create by the new code won't include those.
-                 */
-                AssertLogRelMsgReturn(   !(fFlags & RT_BIT(3) /*MM_RAM_FLAGS_MMIO2*/)
-                                      && (fFlags  & RT_BIT(0) /*MM_RAM_FLAGS_RESERVED*/),
-                                      ("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
-                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-                AssertLogRelMsgReturn(GCPhys == UINT32_C(0xfff80000),
-                                      ("GCPhys=%RGp pRamRange{GCPhys=%#x %s}\n", GCPhys, pRam->GCPhys, pRam->pszDesc),
-                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-                /* Skip wasted reserved pages before the ROM. */
-                while (GCPhys < pRam->GCPhys)
-                {
-                    rc = pgmR3LoadPageToDevNullOld(pSSM);
-                    GCPhys += PAGE_SIZE;
-                }
-
-                /* Load the bios pages. */
-                cPages = pRam->cb >> PAGE_SHIFT;
-                for (uint32_t iPage = 0; iPage < cPages; iPage++)
-                {
-                    RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
-                    PPGMPAGE        pPage      = &pRam->aPages[iPage];
-
-                    AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM,
-                                          ("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, GCPhys),
-                                          VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-                    rc = pgmR3LoadPageBitsOld(pVM, pSSM, PGMPAGETYPE_ROM, pPage, GCPhysPage, pRam);
-                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
-                }
-            }
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for pgmR3Load and pgmR3LoadLocked.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- * @param   uVersion            The saved state version.
- *
- * @todo    This needs splitting up if more record types or code twists are
- *          added...
- */
-static int pgmR3LoadMemory(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    /*
-     * Process page records until we hit the terminator.
-     */
-    RTGCPHYS        GCPhys   = NIL_RTGCPHYS;
-    PPGMRAMRANGE    pRamHint = NULL;
-    uint8_t         id       = UINT8_MAX;
-    uint32_t        iPage    = UINT32_MAX - 10;
-    PPGMROMRANGE    pRom     = NULL;
-    PPGMMMIO2RANGE  pMmio2   = NULL;
-
-    /*
-     * We batch up pages that should be freed instead of calling GMM for
-     * each and every one of them.  Note that we'll lose the pages in most
-     * failure paths - this should probably be addressed one day.
-     */
-    uint32_t            cPendingPages = 0;
-    PGMMFREEPAGESREQ    pReq;
-    int rc = GMMR3FreePagesPrepare(pVM, &pReq, 128 /* batch size */, GMMACCOUNT_BASE);
-    AssertLogRelRCReturn(rc, rc);
-
-    for (;;)
-    {
-        /*
-         * Get the record type and flags.
-         */
-        uint8_t u8;
-        rc = SSMR3GetU8(pSSM, &u8);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (u8 == PGM_STATE_REC_END)
-        {
-            /*
-             * Finish off any pages pending freeing.
-             */
-            if (cPendingPages)
-            {
-                rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
-                AssertLogRelRCReturn(rc, rc);
-            }
-            GMMR3FreePagesCleanup(pReq);
-            return VINF_SUCCESS;
-        }
-        AssertLogRelMsgReturn((u8 & ~PGM_STATE_REC_FLAG_ADDR) <= PGM_STATE_REC_LAST, ("%#x\n", u8), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-        switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
-        {
-            /*
-             * RAM page.
-             */
-            case PGM_STATE_REC_RAM_ZERO:
-            case PGM_STATE_REC_RAM_RAW:
-            {
-                /*
-                 * Get the address and resolve it into a page descriptor.
-                 */
-                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
-                    GCPhys += PAGE_SIZE;
-                else
-                {
-                    rc = SSMR3GetGCPhys(pSSM, &GCPhys);
-                    if (RT_FAILURE(rc))
-                        return rc;
-                }
-                AssertLogRelMsgReturn(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-                PPGMPAGE pPage;
-                rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pPage, &pRamHint);
-                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
-
-                /*
-                 * Take action according to the record type.
-                 */
-                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
-                {
-                    case PGM_STATE_REC_RAM_ZERO:
-                    {
-                        if (    PGM_PAGE_IS_ZERO(pPage)
-                            ||  PGM_PAGE_IS_BALLOONED(pPage))
-                            break;
-                        AssertLogRelMsgReturn(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED, ("GCPhys=%RGp %R[pgmpage]\n", GCPhys, pPage), VERR_INTERNAL_ERROR_5);
-
-                        /* If this is a ROM page, we must clear it and not try
-                           free it... */
-                        if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM
-                            || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM_SHADOW)
-                        {
-                            void *pvDstPage;
-                            rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDstPage);
-                            AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp %R[pgmpage] rc=%Rrc\n", GCPhys, pPage, rc), rc);
-                            ASMMemZeroPage(pvDstPage);
-                        }
-                        /* Free it only if it's not part of a previously
-                           allocated large page (no need to clear the page). */
-                        else if (PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE)
-                        {
-                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys);
-                            AssertRCReturn(rc, rc);
-                        }
-                        break;
-                    }
-
-                    case PGM_STATE_REC_RAM_RAW:
-                    {
-                        void *pvDstPage;
-                        rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDstPage);
-                        AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp %R[pgmpage] rc=%Rrc\n", GCPhys, pPage, rc), rc);
-                        rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
-                        if (RT_FAILURE(rc))
-                            return rc;
-                        break;
-                    }
-
-                    default:
-                        AssertMsgFailedReturn(("%#x\n", u8), VERR_INTERNAL_ERROR);
-                }
-                id = UINT8_MAX;
-                break;
-            }
-
-            /*
-             * MMIO2 page.
-             */
-            case PGM_STATE_REC_MMIO2_RAW:
-            case PGM_STATE_REC_MMIO2_ZERO:
-            {
-                /*
-                 * Get the ID + page number and resolved that into a MMIO2 page.
-                 */
-                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
-                    iPage++;
-                else
-                {
-                    SSMR3GetU8(pSSM, &id);
-                    rc = SSMR3GetU32(pSSM, &iPage);
-                    if (RT_FAILURE(rc))
-                        return rc;
-                }
-                if (    !pMmio2
-                    ||  pMmio2->idSavedState != id)
-                {
-                    for (pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
-                        if (pMmio2->idSavedState == id)
-                            break;
-                    AssertLogRelMsgReturn(pMmio2, ("id=%#u iPage=%#x\n", id, iPage), VERR_INTERNAL_ERROR);
-                }
-                AssertLogRelMsgReturn(iPage < (pMmio2->RamRange.cb >> PAGE_SHIFT), ("iPage=%#x cb=%RGp %s\n", iPage, pMmio2->RamRange.cb, pMmio2->RamRange.pszDesc), VERR_INTERNAL_ERROR);
-                void *pvDstPage = (uint8_t *)pMmio2->RamRange.pvR3 + ((size_t)iPage << PAGE_SHIFT);
-
-                /*
-                 * Load the page bits.
-                 */
-                if ((u8 & ~PGM_STATE_REC_FLAG_ADDR) == PGM_STATE_REC_MMIO2_ZERO)
-                    ASMMemZeroPage(pvDstPage);
-                else
-                {
-                    rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
-                    if (RT_FAILURE(rc))
-                        return rc;
-                }
-                GCPhys = NIL_RTGCPHYS;
-                break;
-            }
-
-            /*
-             * ROM pages.
-             */
-            case PGM_STATE_REC_ROM_VIRGIN:
-            case PGM_STATE_REC_ROM_SHW_RAW:
-            case PGM_STATE_REC_ROM_SHW_ZERO:
-            case PGM_STATE_REC_ROM_PROT:
-            {
-                /*
-                 * Get the ID + page number and resolved that into a ROM page descriptor.
-                 */
-                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
-                    iPage++;
-                else
-                {
-                    SSMR3GetU8(pSSM, &id);
-                    rc = SSMR3GetU32(pSSM, &iPage);
-                    if (RT_FAILURE(rc))
-                        return rc;
-                }
-                if (    !pRom
-                    ||  pRom->idSavedState != id)
-                {
-                    for (pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
-                        if (pRom->idSavedState == id)
-                            break;
-                    AssertLogRelMsgReturn(pRom, ("id=%#u iPage=%#x\n", id, iPage), VERR_INTERNAL_ERROR);
-                }
-                AssertLogRelMsgReturn(iPage < (pRom->cb >> PAGE_SHIFT), ("iPage=%#x cb=%RGp %s\n", iPage, pRom->cb, pRom->pszDesc), VERR_INTERNAL_ERROR);
-                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
-                GCPhys = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
-
-                /*
-                 * Get and set the protection.
-                 */
-                uint8_t u8Prot;
-                rc = SSMR3GetU8(pSSM, &u8Prot);
-                if (RT_FAILURE(rc))
-                    return rc;
-                PGMROMPROT enmProt = (PGMROMPROT)u8Prot;
-                AssertLogRelMsgReturn(enmProt > PGMROMPROT_INVALID && enmProt < PGMROMPROT_END, ("GCPhys=%RGp enmProt=%d\n", GCPhys, enmProt), VERR_INTERNAL_ERROR);
-
-                if (enmProt != pRomPage->enmProt)
-                {
-                    if (RT_UNLIKELY(!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)))
-                        return SSMR3SetCfgError(pSSM, RT_SRC_POS,
-                                                N_("Protection change of unshadowed ROM page: GCPhys=%RGp enmProt=%d %s"),
-                                                GCPhys, enmProt, pRom->pszDesc);
-                    rc = PGMR3PhysRomProtect(pVM, GCPhys, PAGE_SIZE, enmProt);
-                    AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp rc=%Rrc\n", GCPhys, rc), rc);
-                    AssertLogRelReturn(pRomPage->enmProt == enmProt, VERR_INTERNAL_ERROR);
-                }
-                if ((u8 & ~PGM_STATE_REC_FLAG_ADDR) == PGM_STATE_REC_ROM_PROT)
-                    break; /* done */
-
-                /*
-                 * Get the right page descriptor.
-                 */
-                PPGMPAGE pRealPage;
-                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
-                {
-                    case PGM_STATE_REC_ROM_VIRGIN:
-                        if (!PGMROMPROT_IS_ROM(enmProt))
-                            pRealPage = &pRomPage->Virgin;
-                        else
-                            pRealPage = NULL;
-                        break;
-
-                    case PGM_STATE_REC_ROM_SHW_RAW:
-                    case PGM_STATE_REC_ROM_SHW_ZERO:
-                        if (RT_UNLIKELY(!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)))
-                            return SSMR3SetCfgError(pSSM, RT_SRC_POS,
-                                                    N_("Shadowed / non-shadowed page type mismatch: GCPhys=%RGp enmProt=%d %s"),
-                                                    GCPhys, enmProt, pRom->pszDesc);
-                        if (PGMROMPROT_IS_ROM(enmProt))
-                            pRealPage = &pRomPage->Shadow;
-                        else
-                            pRealPage = NULL;
-                        break;
-
-                    default: AssertLogRelFailedReturn(VERR_INTERNAL_ERROR); /* shut up gcc */
-                }
-                if (!pRealPage)
-                {
-                    rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pRealPage, &pRamHint);
-                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
-                }
-
-                /*
-                 * Make it writable and map it (if necessary).
-                 */
-                void *pvDstPage = NULL;
-                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
-                {
-                    case PGM_STATE_REC_ROM_SHW_ZERO:
-                        if (    PGM_PAGE_IS_ZERO(pRealPage)
-                            ||  PGM_PAGE_IS_BALLOONED(pRealPage))
-                            break;
-                        /** @todo implement zero page replacing. */
-                        /* fall thru */
-                    case PGM_STATE_REC_ROM_VIRGIN:
-                    case PGM_STATE_REC_ROM_SHW_RAW:
-                    {
-                        rc = pgmPhysPageMakeWritableAndMap(pVM, pRealPage, GCPhys, &pvDstPage);
-                        AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp rc=%Rrc\n", GCPhys, rc), rc);
-                        break;
-                    }
-                }
-
-                /*
-                 * Load the bits.
-                 */
-                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
-                {
-                    case PGM_STATE_REC_ROM_SHW_ZERO:
-                        if (pvDstPage)
-                            ASMMemZeroPage(pvDstPage);
-                        break;
-
-                    case PGM_STATE_REC_ROM_VIRGIN:
-                    case PGM_STATE_REC_ROM_SHW_RAW:
-                        rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
-                        if (RT_FAILURE(rc))
-                            return rc;
-                        break;
-                }
-                GCPhys = NIL_RTGCPHYS;
-                break;
-            }
-
-            /*
-             * Unknown type.
-             */
-            default:
-                AssertLogRelMsgFailedReturn(("%#x\n", u8), VERR_INTERNAL_ERROR);
-        }
-    } /* forever */
-}
-
-
-/**
- * Worker for pgmR3Load.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- * @param   uVersion            The saved state version.
- */
-static int pgmR3LoadFinalLocked(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
-{
-    PPGM        pPGM = &pVM->pgm.s;
-    int         rc;
-    uint32_t    u32Sep;
-
-    /*
-     * Load basic data (required / unaffected by relocation).
-     */
-    if (uVersion >= PGM_SAVED_STATE_VERSION_3_0_0)
-    {
-        if (uVersion > PGM_SAVED_STATE_VERSION_PRE_BALLOON)
-            rc = SSMR3GetStruct(pSSM, pPGM, &s_aPGMFields[0]);
-        else
-            rc = SSMR3GetStruct(pSSM, pPGM, &s_aPGMFieldsPreBalloon[0]);
-
-        AssertLogRelRCReturn(rc, rc);
-
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            rc = SSMR3GetStruct(pSSM, &pVM->aCpus[i].pgm.s, &s_aPGMCpuFields[0]);
-            AssertLogRelRCReturn(rc, rc);
-        }
-    }
-    else if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
-    {
-        AssertRelease(pVM->cCpus == 1);
-
-        PGMOLD pgmOld;
-        rc = SSMR3GetStruct(pSSM, &pgmOld, &s_aPGMFields_Old[0]);
-        AssertLogRelRCReturn(rc, rc);
-
-        pPGM->fMappingsFixed    = pgmOld.fMappingsFixed;
-        pPGM->GCPtrMappingFixed = pgmOld.GCPtrMappingFixed;
-        pPGM->cbMappingFixed    = pgmOld.cbMappingFixed;
-
-        pVM->aCpus[0].pgm.s.fA20Enabled   = pgmOld.fA20Enabled;
-        pVM->aCpus[0].pgm.s.GCPhysA20Mask = pgmOld.GCPhysA20Mask;
-        pVM->aCpus[0].pgm.s.enmGuestMode  = pgmOld.enmGuestMode;
-    }
-    else
-    {
-        AssertRelease(pVM->cCpus == 1);
-
-        SSMR3GetBool(pSSM,      &pPGM->fMappingsFixed);
-        SSMR3GetGCPtr(pSSM,     &pPGM->GCPtrMappingFixed);
-        SSMR3GetU32(pSSM,       &pPGM->cbMappingFixed);
-
-        uint32_t cbRamSizeIgnored;
-        rc = SSMR3GetU32(pSSM,  &cbRamSizeIgnored);
-        if (RT_FAILURE(rc))
-            return rc;
-        SSMR3GetGCPhys(pSSM,    &pVM->aCpus[0].pgm.s.GCPhysA20Mask);
-
-        uint32_t u32 = 0;
-        SSMR3GetUInt(pSSM,      &u32);
-        pVM->aCpus[0].pgm.s.fA20Enabled = !!u32;
-        SSMR3GetUInt(pSSM,      &pVM->aCpus[0].pgm.s.fSyncFlags);
-        RTUINT uGuestMode;
-        SSMR3GetUInt(pSSM,      &uGuestMode);
-        pVM->aCpus[0].pgm.s.enmGuestMode = (PGMMODE)uGuestMode;
-
-        /* check separator. */
-        SSMR3GetU32(pSSM, &u32Sep);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (u32Sep != (uint32_t)~0)
-        {
-            AssertMsgFailed(("u32Sep=%#x (first)\n", u32Sep));
-            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-        }
-    }
-
-    /*
-     * The guest mappings - skipped now, see re-fixation in the caller.
-     */
-    uint32_t i = 0;
-    for (;; i++)
-    {
-        rc = SSMR3GetU32(pSSM, &u32Sep);        /* sequence number */
-        if (RT_FAILURE(rc))
-            return rc;
-        if (u32Sep == ~0U)
-            break;
-        AssertMsgReturn(u32Sep == i, ("u32Sep=%#x i=%#x\n", u32Sep, i), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
-
-        char szDesc[256];
-        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
-        if (RT_FAILURE(rc))
-            return rc;
-        RTGCPTR GCPtrIgnore;
-        SSMR3GetGCPtr(pSSM, &GCPtrIgnore);      /* GCPtr */
-        rc = SSMR3GetGCPtr(pSSM, &GCPtrIgnore); /* cPTs  */
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Load the RAM contents.
-     */
-    if (uVersion > PGM_SAVED_STATE_VERSION_3_0_0)
-    {
-        if (!pVM->pgm.s.LiveSave.fActive)
-        {
-            if (uVersion > PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
-            {
-                rc = pgmR3LoadRamConfig(pVM, pSSM);
-                if (RT_FAILURE(rc))
-                    return rc;
-            }
-            rc = pgmR3LoadRomRanges(pVM, pSSM);
-            if (RT_FAILURE(rc))
-                return rc;
-            rc = pgmR3LoadMmio2Ranges(pVM, pSSM);
-            if (RT_FAILURE(rc))
-                return rc;
-        }
-
-        rc = pgmR3LoadMemory(pVM, pSSM, SSM_PASS_FINAL);
-    }
-    else
-        rc = pgmR3LoadMemoryOld(pVM, pSSM, uVersion);
-
-    /* Refresh balloon accounting. */
-    if (pVM->pgm.s.cBalloonedPages)
-    {
-        rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_INFLATE, pVM->pgm.s.cBalloonedPages);
-        AssertRCReturn(rc, rc);
-    }
-    return rc;
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) pgmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    int     rc;
-    PPGM    pPGM = &pVM->pgm.s;
-
-    /*
-     * Validate version.
-     */
-    if (   (   uPass != SSM_PASS_FINAL
-            && uVersion != PGM_SAVED_STATE_VERSION
-            && uVersion != PGM_SAVED_STATE_VERSION_PRE_BALLOON
-            && uVersion != PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
-        || (   uVersion != PGM_SAVED_STATE_VERSION
-            && uVersion != PGM_SAVED_STATE_VERSION_PRE_BALLOON
-            && uVersion != PGM_SAVED_STATE_VERSION_NO_RAM_CFG
-            && uVersion != PGM_SAVED_STATE_VERSION_3_0_0
-            && uVersion != PGM_SAVED_STATE_VERSION_2_2_2
-            && uVersion != PGM_SAVED_STATE_VERSION_RR_DESC
-            && uVersion != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE)
-       )
-    {
-        AssertMsgFailed(("pgmR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, PGM_SAVED_STATE_VERSION));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    /*
-     * Do the loading while owning the lock because a bunch of the functions
-     * we're using requires this.
-     */
-    if (uPass != SSM_PASS_FINAL)
-    {
-        pgmLock(pVM);
-        if (uPass != 0)
-            rc = pgmR3LoadMemory(pVM, pSSM, uPass);
-        else
-        {
-            pVM->pgm.s.LiveSave.fActive = true;
-            if (uVersion > PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
-                rc = pgmR3LoadRamConfig(pVM, pSSM);
-            else
-                rc = VINF_SUCCESS;
-            if (RT_SUCCESS(rc))
-                rc = pgmR3LoadRomRanges(pVM, pSSM);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3LoadMmio2Ranges(pVM, pSSM);
-            if (RT_SUCCESS(rc))
-                rc = pgmR3LoadMemory(pVM, pSSM, uPass);
-        }
-        pgmUnlock(pVM);
-    }
-    else
-    {
-        pgmLock(pVM);
-        rc = pgmR3LoadFinalLocked(pVM, pSSM, uVersion);
-        pVM->pgm.s.LiveSave.fActive = false;
-        pgmUnlock(pVM);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * We require a full resync now.
-             */
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-            {
-                PVMCPU pVCpu = &pVM->aCpus[i];
-                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
-                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-                pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
-            }
-
-            pgmR3HandlerPhysicalUpdateAll(pVM);
-
-            /*
-             * Change the paging mode and restore PGMCPU::GCPhysCR3.
-             * (The latter requires the CPUM state to be restored already.)
-             */
-            if (CPUMR3IsStateRestorePending(pVM))
-                return SSMR3SetLoadError(pSSM, VERR_WRONG_ORDER, RT_SRC_POS,
-                                         N_("PGM was unexpectedly restored before CPUM"));
-
-            for (VMCPUID i = 0; i < pVM->cCpus; i++)
-            {
-                PVMCPU pVCpu = &pVM->aCpus[i];
-
-                rc = PGMR3ChangeMode(pVM, pVCpu, pVCpu->pgm.s.enmGuestMode);
-                AssertLogRelRCReturn(rc, rc);
-
-                /* Update pVM->pgm.s.GCPhysCR3. */
-                Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS || FTMIsDeltaLoadSaveActive(pVM));
-                RTGCPHYS GCPhysCR3 = CPUMGetGuestCR3(pVCpu);
-                if (    pVCpu->pgm.s.enmGuestMode == PGMMODE_PAE
-                    ||  pVCpu->pgm.s.enmGuestMode == PGMMODE_PAE_NX
-                    ||  pVCpu->pgm.s.enmGuestMode == PGMMODE_AMD64
-                    ||  pVCpu->pgm.s.enmGuestMode == PGMMODE_AMD64_NX)
-                    GCPhysCR3 = (GCPhysCR3 & X86_CR3_PAE_PAGE_MASK);
-                else
-                    GCPhysCR3 = (GCPhysCR3 & X86_CR3_PAGE_MASK);
-                pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
-
-                /* Update the PSE, NX flags and validity masks. */
-                pVCpu->pgm.s.fGst32BitPageSizeExtension = CPUMIsGuestPageSizeExtEnabled(pVCpu);
-                PGMNotifyNxeChanged(pVCpu, CPUMIsGuestNXEnabled(pVCpu));
-            }
-
-            /*
-             * Try re-fixate the guest mappings.
-             */
-            pVM->pgm.s.fMappingsFixedRestored = false;
-            if (   pVM->pgm.s.fMappingsFixed
-                && pgmMapAreMappingsEnabled(&pVM->pgm.s))
-            {
-                RTGCPTR     GCPtrFixed    = pVM->pgm.s.GCPtrMappingFixed;
-                uint32_t    cbFixed       = pVM->pgm.s.cbMappingFixed;
-                pVM->pgm.s.fMappingsFixed = false;
-
-                uint32_t    cbRequired;
-                int rc2 = PGMR3MappingsSize(pVM, &cbRequired); AssertRC(rc2);
-                if (   RT_SUCCESS(rc2)
-                    && cbRequired > cbFixed)
-                    rc2 = VERR_OUT_OF_RANGE;
-                if (RT_SUCCESS(rc2))
-                    rc2 = pgmR3MappingsFixInternal(pVM, GCPtrFixed, cbFixed);
-                if (RT_FAILURE(rc2))
-                {
-                    LogRel(("PGM: Unable to re-fixate the guest mappings at %RGv-%RGv: rc=%Rrc (cbRequired=%#x)\n",
-                            GCPtrFixed, GCPtrFixed + cbFixed, rc2, cbRequired));
-                    pVM->pgm.s.fMappingsFixed         = false;
-                    pVM->pgm.s.fMappingsFixedRestored = true;
-                    pVM->pgm.s.GCPtrMappingFixed      = GCPtrFixed;
-                    pVM->pgm.s.cbMappingFixed         = cbFixed;
-                }
-            }
-            else
-            {
-                /* We used to set fixed + disabled while we only use disabled now,
-                   so wipe the state to avoid any confusion. */
-                pVM->pgm.s.fMappingsFixed    = false;
-                pVM->pgm.s.GCPtrMappingFixed = NIL_RTGCPTR;
-                pVM->pgm.s.cbMappingFixed    = 0;
-            }
-
-            /*
-             * If we have floating mappings, do a CR3 sync now to make sure the HMA
-             * doesn't conflict with guest code / data and thereby cause trouble
-             * when restoring other components like PATM.
-             */
-            if (pgmMapAreMappingsFloating(&pVM->pgm.s))
-            {
-                PVMCPU pVCpu = &pVM->aCpus[0];
-                rc = PGMSyncCR3(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR3(pVCpu),  CPUMGetGuestCR4(pVCpu), true);
-                if (RT_FAILURE(rc))
-                    return SSMR3SetLoadError(pSSM, VERR_WRONG_ORDER, RT_SRC_POS,
-                                             N_("PGMSyncCR3 failed unexpectedly with rc=%Rrc"), rc);
-
-                /* Make sure to re-sync before executing code. */
-                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
-                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
-                pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
-            }
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * Registers the saved state callbacks with SSM.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to VM structure.
- * @param   cbRam   The RAM size.
- */
-int pgmR3InitSavedState(PVM pVM, uint64_t cbRam)
-{
-    return SSMR3RegisterInternal(pVM, "pgm", 1, PGM_SAVED_STATE_VERSION, (size_t)cbRam + sizeof(PGM),
-                                 pgmR3LivePrep, pgmR3LiveExec, pgmR3LiveVote,
-                                 NULL,          pgmR3SaveExec, pgmR3SaveDone,
-                                 pgmR3LoadPrep, pgmR3Load,     NULL);
-}
-
Index: unk/src/VBox/VMM/PGMSharedPage.cpp
===================================================================
--- /trunk/src/VBox/VMM/PGMSharedPage.cpp	(revision 35332)
+++ 	(revision )
@@ -1,418 +1,0 @@
-/* $Id$ */
-/** @file
- * PGM - Page Manager and Monitor, Shared page handling
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_PGM_SHARED
-#include <VBox/pgm.h>
-#include <VBox/stam.h>
-#include "PGMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/sup.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/mem.h>
-#include <iprt/string.h>
-
-#include "PGMInline.h"
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
-/** Keep a copy of all registered shared modules for the .pgmcheckduppages debugger command. */
-static PGMMREGISTERSHAREDMODULEREQ g_apSharedModules[512] = {0};
-static unsigned g_cSharedModules = 0;
-#endif
-
-/**
- * Registers a new shared module for the VM
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle
- * @param   enmGuestOS          Guest OS type
- * @param   pszModuleName       Module name
- * @param   pszVersion          Module version
- * @param   GCBaseAddr          Module base address
- * @param   cbModule            Module size
- * @param   cRegions            Number of shared region descriptors
- * @param   pRegions            Shared region(s)
- */
-VMMR3DECL(int) PGMR3SharedModuleRegister(PVM pVM, VBOXOSFAMILY enmGuestOS, char *pszModuleName, char *pszVersion, RTGCPTR GCBaseAddr, uint32_t cbModule,
-                                         unsigned cRegions, VMMDEVSHAREDREGIONDESC *pRegions)
-{
-#ifdef VBOX_WITH_PAGE_SHARING
-    PGMMREGISTERSHAREDMODULEREQ pReq;
-
-    Log(("PGMR3SharedModuleRegister family=%d name=%s version=%s base=%RGv size=%x cRegions=%d\n", enmGuestOS, pszModuleName, pszVersion, GCBaseAddr, cbModule, cRegions));
-
-    /* Sanity check. */
-    AssertReturn(cRegions < VMMDEVSHAREDREGIONDESC_MAX, VERR_INVALID_PARAMETER);
-
-    pReq = (PGMMREGISTERSHAREDMODULEREQ)RTMemAllocZ(RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]));
-    AssertReturn(pReq, VERR_NO_MEMORY);
-
-    pReq->enmGuestOS    = enmGuestOS;
-    pReq->GCBaseAddr    = GCBaseAddr;
-    pReq->cbModule      = cbModule;
-    pReq->cRegions      = cRegions;
-    for (unsigned i = 0; i < cRegions; i++)
-        pReq->aRegions[i] = pRegions[i];
-
-    if (    RTStrCopy(pReq->szName, sizeof(pReq->szName), pszModuleName) != VINF_SUCCESS
-        ||  RTStrCopy(pReq->szVersion, sizeof(pReq->szVersion), pszVersion) != VINF_SUCCESS)
-    {
-        RTMemFree(pReq);
-        return VERR_BUFFER_OVERFLOW;
-    }
-
-    int rc = GMMR3RegisterSharedModule(pVM, pReq);
-# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
-    if (rc == VINF_SUCCESS)
-    {
-        PGMMREGISTERSHAREDMODULEREQ *ppSharedModule = NULL;
-
-        if (g_cSharedModules < RT_ELEMENTS(g_apSharedModules))
-        {
-            for (unsigned i = 0; i < RT_ELEMENTS(g_apSharedModules); i++)
-            {
-                if (g_apSharedModules[i] == NULL)
-                {
-                    ppSharedModule = &g_apSharedModules[i];
-                    break;
-                }
-            }
-            Assert(ppSharedModule);
-
-            if (ppSharedModule)
-            {
-                *ppSharedModule = (PGMMREGISTERSHAREDMODULEREQ)RTMemAllocZ(RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]));
-                memcpy(*ppSharedModule, pReq, RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]));
-                g_cSharedModules++;
-            }
-        }
-    }
-# endif
-
-    RTMemFree(pReq);
-    Assert(rc == VINF_SUCCESS || rc == VINF_PGM_SHARED_MODULE_COLLISION || rc == VINF_PGM_SHARED_MODULE_ALREADY_REGISTERED);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    return VINF_SUCCESS;
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif
-}
-
-/**
- * Unregisters a shared module for the VM
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle
- * @param   pszModuleName       Module name
- * @param   pszVersion          Module version
- * @param   GCBaseAddr          Module base address
- * @param   cbModule            Module size
- */
-VMMR3DECL(int) PGMR3SharedModuleUnregister(PVM pVM, char *pszModuleName, char *pszVersion, RTGCPTR GCBaseAddr, uint32_t cbModule)
-{
-#ifdef VBOX_WITH_PAGE_SHARING
-    PGMMUNREGISTERSHAREDMODULEREQ pReq;
-
-    Log(("PGMR3SharedModuleUnregister name=%s version=%s base=%RGv size=%x\n", pszModuleName, pszVersion, GCBaseAddr, cbModule));
-
-    pReq = (PGMMUNREGISTERSHAREDMODULEREQ)RTMemAllocZ(sizeof(*pReq));
-    AssertReturn(pReq, VERR_NO_MEMORY);
-
-    pReq->GCBaseAddr    = GCBaseAddr;
-    pReq->cbModule      = cbModule;
-
-    if (    RTStrCopy(pReq->szName, sizeof(pReq->szName), pszModuleName) != VINF_SUCCESS
-        ||  RTStrCopy(pReq->szVersion, sizeof(pReq->szVersion), pszVersion) != VINF_SUCCESS)
-    {
-        RTMemFree(pReq);
-        return VERR_BUFFER_OVERFLOW;
-    }
-    int rc = GMMR3UnregisterSharedModule(pVM, pReq);
-    RTMemFree(pReq);
-
-# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
-    for (unsigned i = 0; i < g_cSharedModules; i++)
-    {
-        if (    g_apSharedModules[i]
-            &&  !strcmp(g_apSharedModules[i]->szName, pszModuleName)
-            &&  !strcmp(g_apSharedModules[i]->szVersion, pszVersion))
-        {
-            RTMemFree(g_apSharedModules[i]);
-            g_apSharedModules[i] = NULL;
-            g_cSharedModules--;
-            break;
-        }
-    }
-# endif
-    return rc;
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif
-}
-
-#ifdef VBOX_WITH_PAGE_SHARING
-/**
- * Rendezvous callback that will be called once.
- *
- * @returns VBox strict status code.
- * @param   pVM                 VM handle.
- * @param   pVCpu               The VMCPU handle for the calling EMT.
- * @param   pvUser              Not used;
- */
-static DECLCALLBACK(VBOXSTRICTRC) pgmR3SharedModuleRegRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    VMCPUID idCpu = *(VMCPUID *)pvUser;
-
-    /* Execute on the VCPU that issued the original request to make sure we're in the right cr3 context. */
-    if (pVCpu->idCpu != idCpu)
-    {
-        Assert(pVM->cCpus > 1);
-        return VINF_SUCCESS;
-    }
-
-    /* Flush all pending handy page operations before changing any shared page assignments. */
-    int rc = PGMR3PhysAllocateHandyPages(pVM);
-    AssertRC(rc);
-
-    /* Lock it here as we can't deal with busy locks in this ring-0 path. */
-    pgmLock(pVM);
-    rc = GMMR3CheckSharedModules(pVM);
-    pgmUnlock(pVM);
-    AssertLogRelRC(rc);
-    return rc;
-}
-
-/**
- * Shared module check helper (called on the way out).
- *
- * @param   pVM         The VM handle.
- * @param   VMCPUID     VCPU id
- */
-static DECLCALLBACK(void) pgmR3CheckSharedModulesHelper(PVM pVM, VMCPUID idCpu)
-{
-    /* We must stall other VCPUs as we'd otherwise have to send IPI flush commands for every single change we make. */
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, pgmR3SharedModuleRegRendezvous, &idCpu);
-    Assert(rc == VINF_SUCCESS);
-}
-#endif
-
-/**
- * Check all registered modules for changes.
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle
- */
-VMMR3DECL(int) PGMR3SharedModuleCheckAll(PVM pVM)
-{
-#ifdef VBOX_WITH_PAGE_SHARING
-    /* Queue the actual registration as we are under the IOM lock right now. Perform this operation on the way out. */
-    return VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3CheckSharedModulesHelper, 2, pVM, VMMGetCpuId(pVM));
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif
-}
-
-/**
- * Query the state of a page in a shared module
- *
- * @returns VBox status code.
- * @param   pVM                 VM handle
- * @param   GCPtrPage           Page address
- * @param   pfShared            Shared status (out)
- * @param   puPageFlags         Page flags (out)
- */
-VMMR3DECL(int) PGMR3SharedModuleGetPageState(PVM pVM, RTGCPTR GCPtrPage, bool *pfShared, uint64_t *puPageFlags)
-{
-#if defined(VBOX_WITH_PAGE_SHARING) && defined(DEBUG)
-    /* Debug only API for the page fusion testcase. */
-    RTGCPHYS GCPhys;
-    uint64_t fFlags;
-
-    pgmLock(pVM);
-
-    int rc = PGMGstGetPage(VMMGetCpu(pVM), GCPtrPage, &fFlags, &GCPhys);
-    switch (rc)
-    {
-    case VINF_SUCCESS:
-    {
-        PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
-        if (pPage)
-        {
-            *pfShared    = PGM_PAGE_IS_SHARED(pPage);
-            *puPageFlags = fFlags;
-        }
-        else
-            rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
-        break;
-    }
-    case VERR_PAGE_NOT_PRESENT:
-    case VERR_PAGE_TABLE_NOT_PRESENT:
-    case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
-    case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
-        *pfShared = false;
-        *puPageFlags = 0;
-        rc = VINF_SUCCESS;
-        break;
-
-    default:
-        break;
-    }
-
-    pgmUnlock(pVM);
-    return rc;
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif
-}
-
-
-#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
-/**
- * The '.pgmcheckduppages' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-DECLCALLBACK(int)  pgmR3CmdCheckDuplicatePages(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    unsigned cBallooned = 0;
-    unsigned cShared = 0;
-    unsigned cZero = 0;
-    unsigned cUnique = 0;
-    unsigned cDuplicate = 0;
-    unsigned cAllocZero = 0;
-    unsigned cPages = 0;
-
-    pgmLock(pVM);
-
-    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
-    {
-        PPGMPAGE    pPage  = &pRam->aPages[0];
-        RTGCPHYS    GCPhys = pRam->GCPhys;
-        uint32_t    cLeft  = pRam->cb >> PAGE_SHIFT;
-        while (cLeft-- > 0)
-        {
-            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
-            {
-                switch (PGM_PAGE_GET_STATE(pPage))
-                {
-                    case PGM_PAGE_STATE_ZERO:
-                        cZero++;
-                        break;
-
-                    case PGM_PAGE_STATE_BALLOONED:
-                        cBallooned++;
-                        break;
-
-                    case PGM_PAGE_STATE_SHARED:
-                        cShared++;
-                        break;
-
-                    case PGM_PAGE_STATE_ALLOCATED:
-                    case PGM_PAGE_STATE_WRITE_MONITORED:
-                    {
-                        const void *pvPage;
-                        /* Check if the page was allocated, but completely zero. */
-                        int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvPage);
-                        if (    rc == VINF_SUCCESS
-                            &&  ASMMemIsZeroPage(pvPage))
-                        {
-                            cAllocZero++;
-                        }
-                        else
-                        if (GMMR3IsDuplicatePage(pVM, PGM_PAGE_GET_PAGEID(pPage)))
-                            cDuplicate++;
-                        else
-                            cUnique++;
-
-                        break;
-                    }
-
-                    default:
-                        AssertFailed();
-                        break;
-                }
-            }
-
-            /* next */
-            pPage++;
-            GCPhys += PAGE_SIZE;
-            cPages++;
-            /* Give some feedback for every processed megabyte. */
-            if ((cPages & 0x7f) == 0)
-                pCmdHlp->pfnPrintf(pCmdHlp, NULL, ".");
-        }
-    }
-    pgmUnlock(pVM);
-
-    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "\nNumber of zero pages      %08x (%d MB)\n", cZero, cZero / 256);
-    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of alloczero pages %08x (%d MB)\n", cAllocZero, cAllocZero / 256);
-    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of ballooned pages %08x (%d MB)\n", cBallooned, cBallooned / 256);
-    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of shared pages    %08x (%d MB)\n", cShared, cShared / 256);
-    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of unique pages    %08x (%d MB)\n", cUnique, cUnique / 256);
-    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of duplicate pages %08x (%d MB)\n", cDuplicate, cDuplicate / 256);
-    return VINF_SUCCESS;
-}
-
-/**
- * The '.pgmsharedmodules' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-DECLCALLBACK(int)  pgmR3CmdShowSharedModules(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    unsigned i = 0;
-
-    pgmLock(pVM);
-    do
-    {
-        if (g_apSharedModules[i])
-        {
-            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Shared module %s (%s):\n", g_apSharedModules[i]->szName, g_apSharedModules[i]->szVersion);
-            for (unsigned j = 0; j < g_apSharedModules[i]->cRegions; j++)
-                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "--- Region %d: base %RGv size %x\n", j, g_apSharedModules[i]->aRegions[j].GCRegionAddr, g_apSharedModules[i]->aRegions[j].cbRegion);
-        }
-        i++;
-    }
-    while (i < RT_ELEMENTS(g_apSharedModules));
-    pgmUnlock(pVM);
-
-    return VINF_SUCCESS;
-}
-
-#endif /* VBOX_STRICT && HC_ARCH_BITS == 64*/
Index: unk/src/VBox/VMM/PGMShw.h
===================================================================
--- /trunk/src/VBox/VMM/PGMShw.h	(revision 35332)
+++ 	(revision )
@@ -1,269 +1,0 @@
-/* $Id$ */
-/** @file
- * VBox - Page Manager / Monitor, Shadow Paging Template.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-#undef SHWPT
-#undef PSHWPT
-#undef SHWPTE
-#undef PSHWPTE
-#undef SHWPD
-#undef PSHWPD
-#undef SHWPDE
-#undef PSHWPDE
-#undef SHW_PDE_PG_MASK
-#undef SHW_PD_SHIFT
-#undef SHW_PD_MASK
-#undef SHW_PTE_PG_MASK
-#undef SHW_PT_SHIFT
-#undef SHW_PT_MASK
-#undef SHW_TOTAL_PD_ENTRIES
-#undef SHW_PDPT_SHIFT
-#undef SHW_PDPT_MASK
-#undef SHW_PDPE_PG_MASK
-#undef SHW_POOL_ROOT_IDX
-
-#if PGM_SHW_TYPE == PGM_TYPE_32BIT
-# define SHWPT                  X86PT
-# define PSHWPT                 PX86PT
-# define SHWPTE                 X86PTE
-# define PSHWPTE                PX86PTE
-# define SHWPD                  X86PD
-# define PSHWPD                 PX86PD
-# define SHWPDE                 X86PDE
-# define PSHWPDE                PX86PDE
-# define SHW_PDE_PG_MASK        X86_PDE_PG_MASK
-# define SHW_PD_SHIFT           X86_PD_SHIFT
-# define SHW_PD_MASK            X86_PD_MASK
-# define SHW_TOTAL_PD_ENTRIES   X86_PG_ENTRIES
-# define SHW_PTE_PG_MASK        X86_PTE_PG_MASK
-# define SHW_PT_SHIFT           X86_PT_SHIFT
-# define SHW_PT_MASK            X86_PT_MASK
-# define SHW_POOL_ROOT_IDX      PGMPOOL_IDX_PD
-
-#elif PGM_SHW_TYPE == PGM_TYPE_EPT
-# define SHWPT                  EPTPT
-# define PSHWPT                 PEPTPT
-# define SHWPTE                 EPTPTE
-# define PSHWPTE                PEPTPTE
-# define SHWPD                  EPTPD
-# define PSHWPD                 PEPTPD
-# define SHWPDE                 EPTPDE
-# define PSHWPDE                PEPTPDE
-# define SHW_PDE_PG_MASK        EPT_PDE_PG_MASK
-# define SHW_PD_SHIFT           EPT_PD_SHIFT
-# define SHW_PD_MASK            EPT_PD_MASK
-# define SHW_PTE_PG_MASK        EPT_PTE_PG_MASK
-# define SHW_PT_SHIFT           EPT_PT_SHIFT
-# define SHW_PT_MASK            EPT_PT_MASK
-# define SHW_PDPT_SHIFT         EPT_PDPT_SHIFT
-# define SHW_PDPT_MASK          EPT_PDPT_MASK
-# define SHW_PDPE_PG_MASK       EPT_PDPE_PG_MASK
-# define SHW_TOTAL_PD_ENTRIES   (EPT_PG_AMD64_ENTRIES*EPT_PG_AMD64_PDPE_ENTRIES)
-# define SHW_POOL_ROOT_IDX      PGMPOOL_IDX_NESTED_ROOT      /* do not use! exception is real mode & protected mode without paging. */
-
-#else
-# define SHWPT                  PGMSHWPTPAE
-# define PSHWPT                 PPGMSHWPTPAE
-# define SHWPTE                 PGMSHWPTEPAE
-# define PSHWPTE                PPGMSHWPTEPAE
-# define SHWPD                  X86PDPAE
-# define PSHWPD                 PX86PDPAE
-# define SHWPDE                 X86PDEPAE
-# define PSHWPDE                PX86PDEPAE
-# define SHW_PDE_PG_MASK        X86_PDE_PAE_PG_MASK
-# define SHW_PD_SHIFT           X86_PD_PAE_SHIFT
-# define SHW_PD_MASK            X86_PD_PAE_MASK
-# define SHW_PTE_PG_MASK        X86_PTE_PAE_PG_MASK
-# define SHW_PT_SHIFT           X86_PT_PAE_SHIFT
-# define SHW_PT_MASK            X86_PT_PAE_MASK
-
-# if PGM_SHW_TYPE == PGM_TYPE_AMD64
-#  define SHW_PDPT_SHIFT        X86_PDPT_SHIFT
-#  define SHW_PDPT_MASK         X86_PDPT_MASK_AMD64
-#  define SHW_PDPE_PG_MASK      X86_PDPE_PG_MASK
-#  define SHW_TOTAL_PD_ENTRIES  (X86_PG_AMD64_ENTRIES*X86_PG_AMD64_PDPE_ENTRIES)
-#  define SHW_POOL_ROOT_IDX     PGMPOOL_IDX_AMD64_CR3
-
-# else /* 32 bits PAE mode */
-#  define SHW_PDPT_SHIFT        X86_PDPT_SHIFT
-#  define SHW_PDPT_MASK         X86_PDPT_MASK_PAE
-#  define SHW_PDPE_PG_MASK      X86_PDPE_PG_MASK
-#  define SHW_TOTAL_PD_ENTRIES  (X86_PG_PAE_ENTRIES*X86_PG_PAE_PDPE_ENTRIES)
-#  define SHW_POOL_ROOT_IDX     PGMPOOL_IDX_PDPT
-# endif
-#endif
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-RT_C_DECLS_BEGIN
-/* r3 */
-PGM_SHW_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0);
-PGM_SHW_DECL(int, Enter)(PVMCPU pVCpu, bool fIs64BitsPagingMode);
-PGM_SHW_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta);
-PGM_SHW_DECL(int, Exit)(PVMCPU pVCpu);
-
-/* all */
-PGM_SHW_DECL(int, GetPage)(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys);
-PGM_SHW_DECL(int, ModifyPage)(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags);
-RT_C_DECLS_END
-
-
-/**
- * Initializes the guest bit of the paging mode data.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
- *                          This is used early in the init process to avoid trouble with PDM
- *                          not being initialized yet.
- */
-PGM_SHW_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0)
-{
-    Assert(pModeData->uShwType == PGM_SHW_TYPE || pModeData->uShwType == PGM_TYPE_NESTED);
-
-    /* Ring-3 */
-    pModeData->pfnR3ShwRelocate          = PGM_SHW_NAME(Relocate);
-    pModeData->pfnR3ShwExit              = PGM_SHW_NAME(Exit);
-    pModeData->pfnR3ShwGetPage           = PGM_SHW_NAME(GetPage);
-    pModeData->pfnR3ShwModifyPage        = PGM_SHW_NAME(ModifyPage);
-
-    if (fResolveGCAndR0)
-    {
-        int rc;
-
-#if PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT /* No AMD64 for traditional virtualization, only VT-x and AMD-V. */
-        /* GC */
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_SHW_NAME_RC_STR(GetPage),    &pModeData->pfnRCShwGetPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_RC_STR(GetPage),  rc), rc);
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_SHW_NAME_RC_STR(ModifyPage), &pModeData->pfnRCShwModifyPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_RC_STR(ModifyPage), rc), rc);
-#endif /* Not AMD64 shadow paging. */
-
-        /* Ring-0 */
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_SHW_NAME_R0_STR(GetPage),    &pModeData->pfnR0ShwGetPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_R0_STR(GetPage),  rc), rc);
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_SHW_NAME_R0_STR(ModifyPage), &pModeData->pfnR0ShwModifyPage);
-        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_R0_STR(ModifyPage), rc), rc);
-    }
-    return VINF_SUCCESS;
-}
-
-/**
- * Enters the shadow mode.
- *
- * @returns VBox status code.
- * @param   pVCpu                   The VMCPU to operate on.
- * @param   fIs64BitsPagingMode     New shadow paging mode is for 64 bits? (only relevant for 64 bits guests on a 32 bits AMD-V nested paging host)
- */
-PGM_SHW_DECL(int, Enter)(PVMCPU pVCpu, bool fIs64BitsPagingMode)
-{
-#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
-
-# if PGM_SHW_TYPE == PGM_TYPE_NESTED && HC_ARCH_BITS == 32
-    /* Must distinguish between 32 and 64 bits guest paging modes as we'll use a different shadow paging root/mode in both cases. */
-    RTGCPHYS     GCPhysCR3 = (fIs64BitsPagingMode) ? RT_BIT_64(63) : RT_BIT_64(62);
-# else
-    RTGCPHYS     GCPhysCR3 = RT_BIT_64(63);
-# endif
-    PPGMPOOLPAGE pNewShwPageCR3;
-    PVM          pVM       = pVCpu->pVMR3;
-    PPGMPOOL     pPool     = pVM->pgm.s.CTX_SUFF(pPool);
-
-    Assert(HWACCMIsNestedPagingActive(pVM) == pVM->pgm.s.fNestedPaging);
-    Assert(pVM->pgm.s.fNestedPaging);
-    Assert(!pVCpu->pgm.s.pShwPageCR3R3);
-
-    pgmLock(pVM);
-
-    int rc = pgmPoolAllocEx(pVM, GCPhysCR3, PGMPOOLKIND_ROOT_NESTED, PGMPOOLACCESS_DONTCARE, PGMPOOL_IDX_NESTED_ROOT,
-                            GCPhysCR3 >> PAGE_SHIFT, true /*fLockPage*/, &pNewShwPageCR3);
-    AssertFatalRC(rc);
-
-    pVCpu->pgm.s.iShwUser      = PGMPOOL_IDX_NESTED_ROOT;
-    pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
-    pVCpu->pgm.s.pShwPageCR3R3 = pNewShwPageCR3;
-
-    pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.pShwPageCR3R3);
-    pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.pShwPageCR3R3);
-
-    pgmUnlock(pVM);
-
-    Log(("Enter nested shadow paging mode: root %RHv phys %RHp\n", pVCpu->pgm.s.pShwPageCR3R3, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->Core.Key));
-#endif
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Relocate any GC pointers related to shadow mode paging.
- *
- * @returns VBox status code.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   offDelta    The relocation offset.
- */
-PGM_SHW_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta)
-{
-    pVCpu->pgm.s.pShwPageCR3RC += offDelta;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Exits the shadow mode.
- *
- * @returns VBox status code.
- * @param   pVCpu       The VMCPU to operate on.
- */
-PGM_SHW_DECL(int, Exit)(PVMCPU pVCpu)
-{
-    PVM pVM = pVCpu->pVMR3;
-
-    if (    (   pVCpu->pgm.s.enmShadowMode == PGMMODE_NESTED
-             || pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT)
-        &&  pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
-    {
-        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
-
-        Assert(pVCpu->pgm.s.iShwUser == PGMPOOL_IDX_NESTED_ROOT);
-
-        pgmLock(pVM);
-
-        /* Do *not* unlock this page as we have two of them floating around in the 32-bit host & 64-bit guest case.
-         * We currently assert when you try to free one of them; don't bother to really allow this.
-         *
-         * Note that this is two nested paging root pages max. This isn't a leak. They are reused.
-         */
-        /* pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); */
-
-        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
-        pVCpu->pgm.s.pShwPageCR3R3 = 0;
-        pVCpu->pgm.s.pShwPageCR3R0 = 0;
-        pVCpu->pgm.s.pShwPageCR3RC = 0;
-        pVCpu->pgm.s.iShwUser      = 0;
-        pVCpu->pgm.s.iShwUserTable = 0;
-
-        pgmUnlock(pVM);
-
-        Log(("Leave nested shadow paging mode\n"));
-    }
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/REMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/REMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,286 +1,0 @@
-/* $Id$ */
-/** @file
- * REM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___REMInternal_h
-#define ___REMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/cpum.h>
-#include <VBox/stam.h>
-#include <VBox/pgm.h>
-#include <VBox/pdmcritsect.h>
-#ifdef REM_INCLUDE_CPU_H
-# include "target-i386/cpu.h"
-#endif
-
-
-
-/** @defgroup grp_rem_int   Internals
- * @ingroup grp_rem
- * @internal
- * @{
- */
-
-/** The saved state version number. */
-#define REM_SAVED_STATE_VERSION_VER1_6   6
-#define REM_SAVED_STATE_VERSION          7
-
-
-/** @def REM_MONITOR_CODE_PAGES
- * Enable to monitor code pages that have been translated by the recompiler. */
-/** Currently broken and interferes with CSAM monitoring (see #2784) */
-////#define REM_MONITOR_CODE_PAGES
-#ifdef DOXYGEN_RUNNING
-# define REM_MONITOR_CODE_PAGES
-#endif
-
-typedef enum REMHANDLERNOTIFICATIONKIND
-{
-    /** The usual invalid 0 entry. */
-    REMHANDLERNOTIFICATIONKIND_INVALID = 0,
-    /** REMR3NotifyHandlerPhysicalRegister. */
-    REMHANDLERNOTIFICATIONKIND_PHYSICAL_REGISTER,
-    /** REMR3NotifyHandlerPhysicalDeregister. */
-    REMHANDLERNOTIFICATIONKIND_PHYSICAL_DEREGISTER,
-    /** REMR3NotifyHandlerPhysicalModify. */
-    REMHANDLERNOTIFICATIONKIND_PHYSICAL_MODIFY,
-    /** The usual 32-bit hack. */
-    REMHANDLERNOTIFICATIONKIND_32BIT_HACK = 0x7fffffff
-} REMHANDLERNOTIFICATIONKIND;
-
-
-/**
- * A recorded handler notification.
- */
-typedef struct REMHANDLERNOTIFICATION
-{
-    /** The notification kind. */
-    REMHANDLERNOTIFICATIONKIND  enmKind;
-    uint32_t                    padding;
-    /** Type specific data. */
-    union
-    {
-        struct
-        {
-            RTGCPHYS            GCPhys;
-            RTGCPHYS            cb;
-            PGMPHYSHANDLERTYPE  enmType;
-            bool                fHasHCHandler;
-        } PhysicalRegister;
-
-        struct
-        {
-            RTGCPHYS            GCPhys;
-            RTGCPHYS            cb;
-            PGMPHYSHANDLERTYPE  enmType;
-            bool                fHasHCHandler;
-            bool                fRestoreAsRAM;
-        } PhysicalDeregister;
-
-        struct
-        {
-            RTGCPHYS            GCPhysOld;
-            RTGCPHYS            GCPhysNew;
-            RTGCPHYS            cb;
-            PGMPHYSHANDLERTYPE  enmType;
-            bool                fHasHCHandler;
-            bool                fRestoreAsRAM;
-        } PhysicalModify;
-        uint64_t                padding[5];
-    } u;
-    uint32_t                    idxSelf;
-    uint32_t volatile           idxNext;
-} REMHANDLERNOTIFICATION;
-/** Pointer to a handler notification record. */
-typedef REMHANDLERNOTIFICATION *PREMHANDLERNOTIFICATION;
-
-/**
- * Converts a REM pointer into a VM pointer.
- * @returns Pointer to the VM structure the REM is part of.
- * @param   pREM    Pointer to REM instance data.
- */
-#define REM2VM(pREM)  ( (PVM)((char*)pREM - pREM->offVM) )
-
-
-/**
- * REM Data (part of VM)
- */
-typedef struct REM
-{
-    /** Offset to the VM structure. */
-    RTINT                   offVM;
-    /** Alignment padding. */
-    RTUINT                  uPadding0;
-
-    /** Cached pointer of the register context of the current VCPU. */
-    R3PTRTYPE(PCPUMCTX)     pCtx;
-
-    /** In REM mode.
-     * I.e. the correct CPU state and some other bits are with REM. */
-    bool volatile           fInREM;
-    /** In REMR3State. */
-    bool                    fInStateSync;
-
-    /** Set when the translation blocks cache need to be flushed. */
-    bool                    fFlushTBs;
-
-    /** Ignore CR3 load notifications from the REM. */
-    bool                    fIgnoreCR3Load;
-    /** Ignore invlpg notifications from the REM. */
-    bool                    fIgnoreInvlPg;
-    /** Ignore CR0, CR4 and EFER load. */
-    bool                    fIgnoreCpuMode;
-    /** Ignore set page. */
-    bool                    fIgnoreSetPage;
-    bool                    bPadding1;
-
-    /** Ignore all that can be ignored. */
-    uint32_t                cIgnoreAll;
-
-    /** Number of times REMR3CanExecuteRaw has been called.
-     * It is used to prevent rescheduling on the first call. */
-    uint32_t                cCanExecuteRaw;
-
-    /** Pending interrupt (~0 -> nothing). */
-    uint32_t                u32PendingInterrupt;
-
-    /** Number of recorded invlpg instructions. */
-    uint32_t volatile       cInvalidatedPages;
-#if HC_ARCH_BITS == 32
-    uint32_t                uPadding2;
-#endif
-    /** Array of recorded invlpg instruction.
-     * These instructions are replayed when entering REM. */
-    RTGCPTR                 aGCPtrInvalidatedPages[48];
-
-    /** Array of recorded handler notifications.
-     * These are replayed when entering REM. */
-    REMHANDLERNOTIFICATION  aHandlerNotifications[64];
-    volatile uint32_t       idxPendingList;
-    volatile uint32_t       idxFreeList;
-
-    /** MMIO memory type.
-     * This is used to register MMIO physical access handlers. */
-    int32_t                 iMMIOMemType;
-    /** Handler memory type.
-     * This is used to register non-MMIO physical access handlers which are executed in HC. */
-    int32_t                 iHandlerMemType;
-
-    /** Pending exception */
-    uint32_t                uPendingException;
-    /** Nr of pending exceptions */
-    uint32_t                cPendingExceptions;
-    /** Pending exception's EIP */
-    RTGCPTR                 uPendingExcptEIP;
-    /** Pending exception's CR2 */
-    RTGCPTR                 uPendingExcptCR2;
-
-    /** The highest known RAM address. */
-    RTGCPHYS                GCPhysLastRam;
-    /** Whether GCPhysLastRam has been fixed (see REMR3Init()). */
-    bool                    fGCPhysLastRamFixed;
-
-    /** Pending rc. */
-    int32_t                 rc;
-
-    /** REM critical section.
-     * This protects cpu_register_physical_memory usage
-     */
-    PDMCRITSECT             CritSectRegister;
-
-    /** Time spent in QEMU. */
-    STAMPROFILEADV          StatsInQEMU;
-    /** Time spent in rawmode.c. */
-    STAMPROFILEADV          StatsInRAWEx;
-    /** Time spent switching state. */
-    STAMPROFILE             StatsState;
-    /** Time spent switching state back. */
-    STAMPROFILE             StatsStateBack;
-
-    /** Padding the CPUX86State structure to 64 byte. */
-    uint32_t                abPadding[HC_ARCH_BITS == 32 ? 4 : 4];
-
-# define REM_ENV_SIZE       0xff00
-
-    /** Recompiler CPU state. */
-#ifdef REM_INCLUDE_CPU_H
-    CPUX86State             Env;
-#else
-    struct FakeEnv
-    {
-        char                achPadding[REM_ENV_SIZE];
-    }                       Env;
-#endif /* !REM_INCLUDE_CPU_H */
-} REM;
-
-/** Pointer to the REM Data. */
-typedef REM *PREM;
-
-
-#ifdef REM_INCLUDE_CPU_H
-bool    remR3CanExecuteRaw(CPUState *env, RTGCPTR eip, unsigned fFlags, int *piException);
-void    remR3CSAMCheckEIP(CPUState *env, RTGCPTR GCPtrCode);
-bool    remR3GetOpcode(CPUState *env, RTGCPTR GCPtrInstr, uint8_t *pu8Byte);
-bool    remR3DisasInstr(CPUState *env, int f32BitCode, char *pszPrefix);
-void    remR3FlushPage(CPUState *env, RTGCPTR GCPtr);
-void    remR3SetPage(CPUState *env, CPUTLBEntry *pRead,  CPUTLBEntry *pWrite, int prot, int is_user);
-void    remR3FlushTLB(CPUState *env, bool fGlobal);
-void    remR3ProtectCode(CPUState *env, RTGCPTR GCPtr);
-void    remR3ChangeCpuMode(CPUState *env);
-void    remR3DmaRun(CPUState *env);
-void    remR3TimersRun(CPUState *env);
-int     remR3NotifyTrap(CPUState *env, uint32_t uTrap, uint32_t uErrorCode, RTGCPTR pvNextEIP);
-void    remR3TrapStat(CPUState *env, uint32_t uTrap);
-void    remR3CpuId(CPUState *env, unsigned uOperator, void *pvEAX, void *pvEBX, void *pvECX, void *pvEDX);
-void    remR3RecordCall(CPUState *env);
-#endif /* REM_INCLUDE_CPU_H */
-void    remR3TrapClear(PVM pVM);
-void    remR3RaiseRC(PVM pVM, int rc);
-void    remR3DumpLnxSyscall(PVMCPU pVCpu);
-void    remR3DumpOBsdSyscall(PVMCPU pVCpu);
-
-
-/** @todo r=bird: clean up the RAWEx stats. */
-/* temporary hacks */
-#define RAWEx_ProfileStart(a, b)    remR3ProfileStart(b)
-#define RAWEx_ProfileStop(a, b)     remR3ProfileStop(b)
-
-
-#ifdef VBOX_WITH_STATISTICS
-
-# define STATS_EMULATE_SINGLE_INSTR         1
-# define STATS_QEMU_COMPILATION             2
-# define STATS_QEMU_RUN_EMULATED_CODE       3
-# define STATS_QEMU_TOTAL                   4
-# define STATS_QEMU_RUN_TIMERS              5
-# define STATS_TLB_LOOKUP                   6
-# define STATS_IRQ_HANDLING                 7
-# define STATS_RAW_CHECK                    8
-
-void remR3ProfileStart(int statcode);
-void remR3ProfileStop(int statcode);
-
-#else  /* !VBOX_WITH_STATISTICS */
-# define remR3ProfileStart(c)   do { } while (0)
-# define remR3ProfileStop(c)    do { } while (0)
-#endif /* !VBOX_WITH_STATISTICS */
-
-/** @} */
-
-#endif
-
Index: unk/src/VBox/VMM/SELM.cpp
===================================================================
--- /trunk/src/VBox/VMM/SELM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2681 +1,0 @@
-/* $Id$ */
-/** @file
- * SELM - The Selector Manager.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_selm   SELM - The Selector Manager
- *
- * SELM takes care of GDT, LDT and TSS shadowing in raw-mode, and the injection
- * of a few hyper selector for the raw-mode context.  In the hardware assisted
- * virtualization mode its only task is to decode entries in the guest GDT or
- * LDT once in a while.
- *
- * @see grp_selm
- *
- *
- * @section seg_selm_shadowing   Shadowing
- *
- * SELMR3UpdateFromCPUM() and SELMR3SyncTSS() does the bulk synchronization
- * work.  The three structures (GDT, LDT, TSS) are all shadowed wholesale atm.
- * The idea is to do it in a more on-demand fashion when we get time.  There
- * also a whole bunch of issues with the current synchronization of all three
- * tables, see notes and todos in the code.
- *
- * When the guest makes changes to the GDT we will try update the shadow copy
- * without involving SELMR3UpdateFromCPUM(), see selmGCSyncGDTEntry().
- *
- * When the guest make LDT changes we'll trigger a full resync of the LDT
- * (SELMR3UpdateFromCPUM()), which, needless to say, isn't optimal.
- *
- * The TSS shadowing is limited to the fields we need to care about, namely SS0
- * and ESP0.  The Patch Manager makes use of these.  We monitor updates to the
- * guest TSS and will try keep our SS0 and ESP0 copies up to date this way
- * rather than go the SELMR3SyncTSS() route.
- *
- * When in raw-mode SELM also injects a few extra GDT selectors which are used
- * by the raw-mode (hyper) context.  These start their life at the high end of
- * the table and will be relocated when the guest tries to make use of them...
- * Well, that was that idea at least, only the code isn't quite there yet which
- * is why we have trouble with guests which actually have a full sized GDT.
- *
- * So, the summary of the current GDT, LDT and TSS shadowing is that there is a
- * lot of relatively simple and enjoyable work to be done, see @bugref{3267}.
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_SELM
-#include <VBox/selm.h>
-#include <VBox/cpum.h>
-#include <VBox/stam.h>
-#include <VBox/mm.h>
-#include <VBox/ssm.h>
-#include <VBox/pgm.h>
-#include <VBox/trpm.h>
-#include <VBox/dbgf.h>
-#include "SELMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-
-#include <iprt/assert.h>
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-#include <iprt/string.h>
-
-
-/**
- * Enable or disable tracking of Guest's GDT/LDT/TSS.
- * @{
- */
-#define SELM_TRACK_GUEST_GDT_CHANGES
-#define SELM_TRACK_GUEST_LDT_CHANGES
-#define SELM_TRACK_GUEST_TSS_CHANGES
-/** @} */
-
-/**
- * Enable or disable tracking of Shadow GDT/LDT/TSS.
- * @{
- */
-#define SELM_TRACK_SHADOW_GDT_CHANGES
-#define SELM_TRACK_SHADOW_LDT_CHANGES
-#define SELM_TRACK_SHADOW_TSS_CHANGES
-/** @} */
-
-
-/** SELM saved state version. */
-#define SELM_SAVED_STATE_VERSION    5
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int)  selmR3Save(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int)  selmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static DECLCALLBACK(int)  selmR3LoadDone(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int)  selmR3GuestGDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
-static DECLCALLBACK(int)  selmR3GuestLDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
-static DECLCALLBACK(int)  selmR3GuestTSSWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
-static DECLCALLBACK(void) selmR3InfoGdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) selmR3InfoGdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) selmR3InfoLdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void) selmR3InfoLdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-//static DECLCALLBACK(void) selmR3InfoTss(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-//static DECLCALLBACK(void) selmR3InfoTssGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-
-
-
-/**
- * Initializes the SELM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) SELMR3Init(PVM pVM)
-{
-    LogFlow(("SELMR3Init\n"));
-
-    /*
-     * Assert alignment and sizes.
-     * (The TSS block requires contiguous back.)
-     */
-    AssertCompile(sizeof(pVM->selm.s) <= sizeof(pVM->selm.padding));    AssertRelease(sizeof(pVM->selm.s) <= sizeof(pVM->selm.padding));
-    AssertCompileMemberAlignment(VM, selm.s, 32);                       AssertRelease(!(RT_OFFSETOF(VM, selm.s) & 31));
-#if 0 /* doesn't work */
-    AssertCompile((RT_OFFSETOF(VM, selm.s.Tss)       & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.Tss));
-    AssertCompile((RT_OFFSETOF(VM, selm.s.TssTrap08) & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.TssTrap08));
-#endif
-    AssertRelease((RT_OFFSETOF(VM, selm.s.Tss)       & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.Tss));
-    AssertRelease((RT_OFFSETOF(VM, selm.s.TssTrap08) & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.TssTrap08));
-    AssertRelease(sizeof(pVM->selm.s.Tss.IntRedirBitmap) == 0x20);
-
-    /*
-     * Init the structure.
-     */
-    pVM->selm.s.offVM                                = RT_OFFSETOF(VM, selm);
-    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]         = (SELM_GDT_ELEMENTS - 0x1) << 3;
-    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]         = (SELM_GDT_ELEMENTS - 0x2) << 3;
-    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64]       = (SELM_GDT_ELEMENTS - 0x3) << 3;
-    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]        = (SELM_GDT_ELEMENTS - 0x4) << 3;
-    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = (SELM_GDT_ELEMENTS - 0x5) << 3;
-
-    /*
-     * Allocate GDT table.
-     */
-    int rc = MMR3HyperAllocOnceNoRel(pVM, sizeof(pVM->selm.s.paGdtR3[0]) * SELM_GDT_ELEMENTS,
-                                     PAGE_SIZE, MM_TAG_SELM, (void **)&pVM->selm.s.paGdtR3);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Allocate LDT area.
-     */
-    rc = MMR3HyperAllocOnceNoRel(pVM, _64K + PAGE_SIZE, PAGE_SIZE, MM_TAG_SELM, &pVM->selm.s.pvLdtR3);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Init Guest's and Shadow GDT, LDT, TSS changes control variables.
-     */
-    pVM->selm.s.cbEffGuestGdtLimit = 0;
-    pVM->selm.s.GuestGdtr.pGdt     = RTRCPTR_MAX;
-    pVM->selm.s.GCPtrGuestLdt      = RTRCPTR_MAX;
-    pVM->selm.s.GCPtrGuestTss      = RTRCPTR_MAX;
-
-    pVM->selm.s.paGdtRC            = NIL_RTRCPTR; /* Must be set in SELMR3Relocate because of monitoring. */
-    pVM->selm.s.pvLdtRC            = RTRCPTR_MAX;
-    pVM->selm.s.pvMonShwTssRC      = RTRCPTR_MAX;
-    pVM->selm.s.GCSelTss           = RTSEL_MAX;
-
-    pVM->selm.s.fDisableMonitoring = false;
-    pVM->selm.s.fSyncTSSRing0Stack = false;
-
-    /* The I/O bitmap starts right after the virtual interrupt redirection bitmap. Outside the TSS on purpose; the CPU will not check it
-     * for I/O operations. */
-    pVM->selm.s.Tss.offIoBitmap = sizeof(VBOXTSS);
-    /* bit set to 1 means no redirection */
-    memset(pVM->selm.s.Tss.IntRedirBitmap, 0xff, sizeof(pVM->selm.s.Tss.IntRedirBitmap));
-
-    /*
-     * Register the saved state data unit.
-     */
-    rc = SSMR3RegisterInternal(pVM, "selm", 1, SELM_SAVED_STATE_VERSION, sizeof(SELM),
-                               NULL, NULL, NULL,
-                               NULL, selmR3Save, NULL,
-                               NULL, selmR3Load, selmR3LoadDone);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Statistics.
-     */
-    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestGDTHandled,     STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/GDTInt",  STAMUNIT_OCCURENCES,     "The number of handled writes to the Guest GDT.");
-    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestGDTUnhandled,   STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/GDTEmu",  STAMUNIT_OCCURENCES,     "The number of unhandled writes to the Guest GDT.");
-    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestLDT,            STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/LDT",     STAMUNIT_OCCURENCES,     "The number of writes to the Guest LDT was detected.");
-    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSHandled,     STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSInt",  STAMUNIT_OCCURENCES,     "The number of handled writes to the Guest TSS.");
-    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSRedir,       STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSRedir",STAMUNIT_OCCURENCES,     "The number of handled redir bitmap writes to the Guest TSS.");
-    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSHandledChanged,STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSIntChg", STAMUNIT_OCCURENCES, "The number of handled writes to the Guest TSS where the R0 stack changed.");
-    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSUnhandled,   STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSEmu",  STAMUNIT_OCCURENCES,     "The number of unhandled writes to the Guest TSS.");
-    STAM_REG(pVM, &pVM->selm.s.StatTSSSync,                    STAMTYPE_PROFILE, "/PROF/SELM/TSSSync",           STAMUNIT_TICKS_PER_CALL, "Profiling of the SELMR3SyncTSS() body.");
-    STAM_REG(pVM, &pVM->selm.s.StatUpdateFromCPUM,             STAMTYPE_PROFILE, "/PROF/SELM/UpdateFromCPUM",    STAMUNIT_TICKS_PER_CALL, "Profiling of the SELMR3UpdateFromCPUM() body.");
-
-    STAM_REG(pVM, &pVM->selm.s.StatHyperSelsChanged,           STAMTYPE_COUNTER, "/SELM/HyperSels/Changed",      STAMUNIT_OCCURENCES,     "The number of times we had to relocate our hypervisor selectors.");
-    STAM_REG(pVM, &pVM->selm.s.StatScanForHyperSels,           STAMTYPE_COUNTER, "/SELM/HyperSels/Scan",         STAMUNIT_OCCURENCES,     "The number of times we had find free hypervisor selectors.");
-
-    /*
-     * Default action when entering raw mode for the first time
-     */
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-
-    /*
-     * Register info handlers.
-     */
-    DBGFR3InfoRegisterInternal(pVM, "gdt",      "Displays the shadow GDT. No arguments.",   &selmR3InfoGdt);
-    DBGFR3InfoRegisterInternal(pVM, "gdtguest", "Displays the guest GDT. No arguments.",    &selmR3InfoGdtGuest);
-    DBGFR3InfoRegisterInternal(pVM, "ldt",      "Displays the shadow LDT. No arguments.",   &selmR3InfoLdt);
-    DBGFR3InfoRegisterInternal(pVM, "ldtguest", "Displays the guest LDT. No arguments.",    &selmR3InfoLdtGuest);
-    //DBGFR3InfoRegisterInternal(pVM, "tss",      "Displays the shadow TSS. No arguments.",   &selmR3InfoTss);
-    //DBGFR3InfoRegisterInternal(pVM, "tssguest", "Displays the guest TSS. No arguments.",    &selmR3InfoTssGuest);
-
-    return rc;
-}
-
-
-/**
- * Finalizes HMA page attributes.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-VMMR3DECL(int) SELMR3InitFinalize(PVM pVM)
-{
-    /** @cfgm{/DoubleFault,bool,false}
-     * Enables catching of double faults in the raw-mode context VMM code.  This can
-     * be used when the triple faults or hangs occur and one suspect an unhandled
-     * double fault.  This is not enabled by default because it means making the
-     * hyper selectors writeable for all supervisor code, including the guest's.
-     * The double fault is a task switch and thus requires write access to the GDT
-     * of the TSS (to set it busy), to the old TSS (to store state), and to the Trap
-     * 8 TSS for the back link.
-     */
-    bool f;
-#if defined(DEBUG_bird)
-    int rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "DoubleFault", &f, true);
-#else
-    int rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "DoubleFault", &f, false);
-#endif
-    AssertLogRelRCReturn(rc, rc);
-    if (f)
-    {
-        PX86DESC paGdt = pVM->selm.s.paGdtR3;
-        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> 3]), sizeof(paGdt[0]),
-                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
-        AssertRC(rc);
-        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] >> 3]), sizeof(paGdt[0]),
-                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
-        AssertRC(rc);
-        rc = PGMMapSetPage(pVM, VM_RC_ADDR(pVM, &pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]), sizeof(pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]),
-                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
-        AssertRC(rc);
-        rc = PGMMapSetPage(pVM, VM_RC_ADDR(pVM, &pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08]), sizeof(pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08]),
-                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
-        AssertRC(rc);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Setup the hypervisor GDT selectors in our shadow table
- *
- * @param   pVM     The VM handle.
- */
-static void selmR3SetupHyperGDTSelectors(PVM pVM)
-{
-    PX86DESC paGdt = pVM->selm.s.paGdtR3;
-
-    /*
-     * Set up global code and data descriptors for use in the guest context.
-     * Both are wide open (base 0, limit 4GB)
-     */
-    PX86DESC   pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] >> 3];
-    pDesc->Gen.u16LimitLow      = 0xffff;
-    pDesc->Gen.u4LimitHigh      = 0xf;
-    pDesc->Gen.u16BaseLow       = 0;
-    pDesc->Gen.u8BaseHigh1      = 0;
-    pDesc->Gen.u8BaseHigh2      = 0;
-    pDesc->Gen.u4Type           = X86_SEL_TYPE_ER_ACC;
-    pDesc->Gen.u1DescType       = 1; /* not system, but code/data */
-    pDesc->Gen.u2Dpl            = 0; /* supervisor */
-    pDesc->Gen.u1Present        = 1;
-    pDesc->Gen.u1Available      = 0;
-    pDesc->Gen.u1Long           = 0;
-    pDesc->Gen.u1DefBig         = 1; /* def 32 bit */
-    pDesc->Gen.u1Granularity    = 1; /* 4KB limit */
-
-    /* data */
-    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] >> 3];
-    pDesc->Gen.u16LimitLow      = 0xffff;
-    pDesc->Gen.u4LimitHigh      = 0xf;
-    pDesc->Gen.u16BaseLow       = 0;
-    pDesc->Gen.u8BaseHigh1      = 0;
-    pDesc->Gen.u8BaseHigh2      = 0;
-    pDesc->Gen.u4Type           = X86_SEL_TYPE_RW_ACC;
-    pDesc->Gen.u1DescType       = 1; /* not system, but code/data */
-    pDesc->Gen.u2Dpl            = 0; /* supervisor */
-    pDesc->Gen.u1Present        = 1;
-    pDesc->Gen.u1Available      = 0;
-    pDesc->Gen.u1Long           = 0;
-    pDesc->Gen.u1DefBig         = 1; /* big */
-    pDesc->Gen.u1Granularity    = 1; /* 4KB limit */
-
-    /* 64-bit mode code (& data?) */
-    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] >> 3];
-    pDesc->Gen.u16LimitLow      = 0xffff;
-    pDesc->Gen.u4LimitHigh      = 0xf;
-    pDesc->Gen.u16BaseLow       = 0;
-    pDesc->Gen.u8BaseHigh1      = 0;
-    pDesc->Gen.u8BaseHigh2      = 0;
-    pDesc->Gen.u4Type           = X86_SEL_TYPE_ER_ACC;
-    pDesc->Gen.u1DescType       = 1; /* not system, but code/data */
-    pDesc->Gen.u2Dpl            = 0; /* supervisor */
-    pDesc->Gen.u1Present        = 1;
-    pDesc->Gen.u1Available      = 0;
-    pDesc->Gen.u1Long           = 1; /* The Long (L) attribute bit. */
-    pDesc->Gen.u1DefBig         = 0; /* With L=1 this must be 0. */
-    pDesc->Gen.u1Granularity    = 1; /* 4KB limit */
-
-    /*
-     * TSS descriptor
-     */
-    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] >> 3];
-    RTRCPTR RCPtrTSS = VM_RC_ADDR(pVM, &pVM->selm.s.Tss);
-    pDesc->Gen.u16BaseLow       = RT_LOWORD(RCPtrTSS);
-    pDesc->Gen.u8BaseHigh1      = RT_BYTE3(RCPtrTSS);
-    pDesc->Gen.u8BaseHigh2      = RT_BYTE4(RCPtrTSS);
-    pDesc->Gen.u16LimitLow      = sizeof(VBOXTSS) - 1;
-    pDesc->Gen.u4LimitHigh      = 0;
-    pDesc->Gen.u4Type           = X86_SEL_TYPE_SYS_386_TSS_AVAIL;
-    pDesc->Gen.u1DescType       = 0; /* system */
-    pDesc->Gen.u2Dpl            = 0; /* supervisor */
-    pDesc->Gen.u1Present        = 1;
-    pDesc->Gen.u1Available      = 0;
-    pDesc->Gen.u1Long           = 0;
-    pDesc->Gen.u1DefBig         = 0;
-    pDesc->Gen.u1Granularity    = 0; /* byte limit */
-
-    /*
-     * TSS descriptor for trap 08
-     */
-    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> 3];
-    pDesc->Gen.u16LimitLow      = sizeof(VBOXTSS) - 1;
-    pDesc->Gen.u4LimitHigh      = 0;
-    RCPtrTSS = VM_RC_ADDR(pVM, &pVM->selm.s.TssTrap08);
-    pDesc->Gen.u16BaseLow       = RT_LOWORD(RCPtrTSS);
-    pDesc->Gen.u8BaseHigh1      = RT_BYTE3(RCPtrTSS);
-    pDesc->Gen.u8BaseHigh2      = RT_BYTE4(RCPtrTSS);
-    pDesc->Gen.u4Type           = X86_SEL_TYPE_SYS_386_TSS_AVAIL;
-    pDesc->Gen.u1DescType       = 0; /* system */
-    pDesc->Gen.u2Dpl            = 0; /* supervisor */
-    pDesc->Gen.u1Present        = 1;
-    pDesc->Gen.u1Available      = 0;
-    pDesc->Gen.u1Long           = 0;
-    pDesc->Gen.u1DefBig         = 0;
-    pDesc->Gen.u1Granularity    = 0; /* byte limit */
-}
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM     The VM.
- */
-VMMR3DECL(void) SELMR3Relocate(PVM pVM)
-{
-    PX86DESC paGdt = pVM->selm.s.paGdtR3;
-    LogFlow(("SELMR3Relocate\n"));
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        /*
-         * Update GDTR and selector.
-         */
-        CPUMSetHyperGDTR(pVCpu, MMHyperR3ToRC(pVM, paGdt), SELM_GDT_ELEMENTS * sizeof(paGdt[0]) - 1);
-
-        /** @todo selector relocations should be a separate operation? */
-        CPUMSetHyperCS(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]);
-        CPUMSetHyperDS(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]);
-        CPUMSetHyperES(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]);
-        CPUMSetHyperSS(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]);
-        CPUMSetHyperTR(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]);
-    }
-
-    selmR3SetupHyperGDTSelectors(pVM);
-
-/** @todo SELM must be called when any of the CR3s changes during a cpu mode change. */
-/** @todo PGM knows the proper CR3 values these days, not CPUM. */
-    /*
-     * Update the TSSes.
-     */
-    /* Only applies to raw mode which supports only 1 VCPU */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    /* Current TSS */
-    pVM->selm.s.Tss.cr3     = PGMGetHyperCR3(pVCpu);
-    pVM->selm.s.Tss.ss0     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
-    pVM->selm.s.Tss.esp0    = VMMGetStackRC(pVCpu);
-    pVM->selm.s.Tss.cs      = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS];
-    pVM->selm.s.Tss.ds      = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
-    pVM->selm.s.Tss.es      = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
-    pVM->selm.s.Tss.offIoBitmap = sizeof(VBOXTSS);
-
-    /* trap 08 */
-    pVM->selm.s.TssTrap08.cr3    = PGMGetInterRCCR3(pVM, pVCpu);                   /* this should give use better survival chances. */
-    pVM->selm.s.TssTrap08.ss0    = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
-    pVM->selm.s.TssTrap08.ss     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
-    pVM->selm.s.TssTrap08.esp0   = VMMGetStackRC(pVCpu) - PAGE_SIZE / 2;  /* upper half can be analysed this way. */
-    pVM->selm.s.TssTrap08.esp    = pVM->selm.s.TssTrap08.esp0;
-    pVM->selm.s.TssTrap08.ebp    = pVM->selm.s.TssTrap08.esp0;
-    pVM->selm.s.TssTrap08.cs     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS];
-    pVM->selm.s.TssTrap08.ds     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
-    pVM->selm.s.TssTrap08.es     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
-    pVM->selm.s.TssTrap08.fs     = 0;
-    pVM->selm.s.TssTrap08.gs     = 0;
-    pVM->selm.s.TssTrap08.selLdt = 0;
-    pVM->selm.s.TssTrap08.eflags = 0x2;    /* all cleared */
-    pVM->selm.s.TssTrap08.ecx    = VM_RC_ADDR(pVM, &pVM->selm.s.Tss);       /* setup ecx to normal Hypervisor TSS address. */
-    pVM->selm.s.TssTrap08.edi    = pVM->selm.s.TssTrap08.ecx;
-    pVM->selm.s.TssTrap08.eax    = pVM->selm.s.TssTrap08.ecx;
-    pVM->selm.s.TssTrap08.edx    = VM_RC_ADDR(pVM, pVM);                    /* setup edx VM address. */
-    pVM->selm.s.TssTrap08.edi    = pVM->selm.s.TssTrap08.edx;
-    pVM->selm.s.TssTrap08.ebx    = pVM->selm.s.TssTrap08.edx;
-    pVM->selm.s.TssTrap08.offIoBitmap = sizeof(VBOXTSS);
-    /* TRPM will be updating the eip */
-
-    if (    !pVM->selm.s.fDisableMonitoring
-        &&  !VMMIsHwVirtExtForced(pVM))
-    {
-        /*
-         * Update shadow GDT/LDT/TSS write access handlers.
-         */
-        int rc;
-#ifdef SELM_TRACK_SHADOW_GDT_CHANGES
-        if (pVM->selm.s.paGdtRC != NIL_RTRCPTR)
-        {
-            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.paGdtRC);
-            AssertRC(rc);
-        }
-        pVM->selm.s.paGdtRC = MMHyperR3ToRC(pVM, paGdt);
-        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->selm.s.paGdtRC,
-                                         pVM->selm.s.paGdtRC + SELM_GDT_ELEMENTS * sizeof(paGdt[0]) - 1,
-                                         0, 0, "selmRCShadowGDTWriteHandler", 0, "Shadow GDT write access handler");
-        AssertRC(rc);
-#endif
-#ifdef SELM_TRACK_SHADOW_TSS_CHANGES
-        if (pVM->selm.s.pvMonShwTssRC != RTRCPTR_MAX)
-        {
-            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvMonShwTssRC);
-            AssertRC(rc);
-        }
-        pVM->selm.s.pvMonShwTssRC = VM_RC_ADDR(pVM, &pVM->selm.s.Tss);
-        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->selm.s.pvMonShwTssRC,
-                                         pVM->selm.s.pvMonShwTssRC + sizeof(pVM->selm.s.Tss) - 1,
-                                         0, 0, "selmRCShadowTSSWriteHandler", 0, "Shadow TSS write access handler");
-        AssertRC(rc);
-#endif
-
-        /*
-         * Update the GC LDT region handler and address.
-         */
-#ifdef SELM_TRACK_SHADOW_LDT_CHANGES
-        if (pVM->selm.s.pvLdtRC != RTRCPTR_MAX)
-        {
-            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvLdtRC);
-            AssertRC(rc);
-        }
-#endif
-        pVM->selm.s.pvLdtRC = MMHyperR3ToRC(pVM, pVM->selm.s.pvLdtR3);
-#ifdef SELM_TRACK_SHADOW_LDT_CHANGES
-        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->selm.s.pvLdtRC,
-                                         pVM->selm.s.pvLdtRC + _64K + PAGE_SIZE - 1,
-                                         0, 0, "selmRCShadowLDTWriteHandler", 0, "Shadow LDT write access handler");
-        AssertRC(rc);
-#endif
-    }
-}
-
-
-/**
- * Terminates the SELM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) SELMR3Term(PVM pVM)
-{
-    return 0;
-}
-
-
-/**
- * The VM is being reset.
- *
- * For the SELM component this means that any GDT/LDT/TSS monitors
- * needs to be removed.
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) SELMR3Reset(PVM pVM)
-{
-    LogFlow(("SELMR3Reset:\n"));
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Uninstall guest GDT/LDT/TSS write access handlers.
-     */
-    int rc;
-#ifdef SELM_TRACK_GUEST_GDT_CHANGES
-    if (pVM->selm.s.GuestGdtr.pGdt != RTRCPTR_MAX && pVM->selm.s.fGDTRangeRegistered)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GuestGdtr.pGdt);
-        AssertRC(rc);
-        pVM->selm.s.GuestGdtr.pGdt = RTRCPTR_MAX;
-        pVM->selm.s.GuestGdtr.cbGdt = 0;
-    }
-    pVM->selm.s.fGDTRangeRegistered = false;
-#endif
-#ifdef SELM_TRACK_GUEST_LDT_CHANGES
-    if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
-        AssertRC(rc);
-        pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
-    }
-#endif
-#ifdef SELM_TRACK_GUEST_TSS_CHANGES
-    if (pVM->selm.s.GCPtrGuestTss != RTRCPTR_MAX)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestTss);
-        AssertRC(rc);
-        pVM->selm.s.GCPtrGuestTss = RTRCPTR_MAX;
-        pVM->selm.s.GCSelTss      = RTSEL_MAX;
-    }
-#endif
-
-    /*
-     * Re-initialize other members.
-     */
-    pVM->selm.s.cbLdtLimit = 0;
-    pVM->selm.s.offLdtHyper = 0;
-    pVM->selm.s.cbMonitoredGuestTss = 0;
-
-    pVM->selm.s.fSyncTSSRing0Stack = false;
-
-    /*
-     * Default action when entering raw mode for the first time
-     */
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-}
-
-/**
- * Disable GDT/LDT/TSS monitoring and syncing
- *
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(void) SELMR3DisableMonitoring(PVM pVM)
-{
-    /*
-     * Uninstall guest GDT/LDT/TSS write access handlers.
-     */
-    int rc;
-#ifdef SELM_TRACK_GUEST_GDT_CHANGES
-    if (pVM->selm.s.GuestGdtr.pGdt != RTRCPTR_MAX && pVM->selm.s.fGDTRangeRegistered)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GuestGdtr.pGdt);
-        AssertRC(rc);
-        pVM->selm.s.GuestGdtr.pGdt = RTRCPTR_MAX;
-        pVM->selm.s.GuestGdtr.cbGdt = 0;
-    }
-    pVM->selm.s.fGDTRangeRegistered = false;
-#endif
-#ifdef SELM_TRACK_GUEST_LDT_CHANGES
-    if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
-        AssertRC(rc);
-        pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
-    }
-#endif
-#ifdef SELM_TRACK_GUEST_TSS_CHANGES
-    if (pVM->selm.s.GCPtrGuestTss != RTRCPTR_MAX)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestTss);
-        AssertRC(rc);
-        pVM->selm.s.GCPtrGuestTss = RTRCPTR_MAX;
-        pVM->selm.s.GCSelTss      = RTSEL_MAX;
-    }
-#endif
-
-    /*
-     * Unregister shadow GDT/LDT/TSS write access handlers.
-     */
-#ifdef SELM_TRACK_SHADOW_GDT_CHANGES
-    if (pVM->selm.s.paGdtRC != NIL_RTRCPTR)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.paGdtRC);
-        AssertRC(rc);
-        pVM->selm.s.paGdtRC = NIL_RTRCPTR;
-    }
-#endif
-#ifdef SELM_TRACK_SHADOW_TSS_CHANGES
-    if (pVM->selm.s.pvMonShwTssRC != RTRCPTR_MAX)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvMonShwTssRC);
-        AssertRC(rc);
-        pVM->selm.s.pvMonShwTssRC = RTRCPTR_MAX;
-    }
-#endif
-#ifdef SELM_TRACK_SHADOW_LDT_CHANGES
-    if (pVM->selm.s.pvLdtRC != RTRCPTR_MAX)
-    {
-        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvLdtRC);
-        AssertRC(rc);
-        pVM->selm.s.pvLdtRC = RTRCPTR_MAX;
-    }
-#endif
-
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-
-    pVM->selm.s.fDisableMonitoring = true;
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) selmR3Save(PVM pVM, PSSMHANDLE pSSM)
-{
-    LogFlow(("selmR3Save:\n"));
-
-    /*
-     * Save the basic bits - fortunately all the other things can be resynced on load.
-     */
-    PSELM pSelm = &pVM->selm.s;
-
-    SSMR3PutBool(pSSM, pSelm->fDisableMonitoring);
-    SSMR3PutBool(pSSM, pSelm->fSyncTSSRing0Stack);
-    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_CS]);
-    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_DS]);
-    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_CS64]);
-    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_CS64]); /* reserved for DS64. */
-    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_TSS]);
-    return SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_TSS_TRAP08]);
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) selmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    LogFlow(("selmR3Load:\n"));
-    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
-
-    /*
-     * Validate version.
-     */
-    if (uVersion != SELM_SAVED_STATE_VERSION)
-    {
-        AssertMsgFailed(("selmR3Load: Invalid version uVersion=%d!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    /*
-     * Do a reset.
-     */
-    SELMR3Reset(pVM);
-
-    /* Get the monitoring flag. */
-    SSMR3GetBool(pSSM, &pVM->selm.s.fDisableMonitoring);
-
-    /* Get the TSS state flag. */
-    SSMR3GetBool(pSSM, &pVM->selm.s.fSyncTSSRing0Stack);
-
-    /*
-     * Get the selectors.
-     */
-    RTSEL SelCS;
-    SSMR3GetSel(pSSM, &SelCS);
-    RTSEL SelDS;
-    SSMR3GetSel(pSSM, &SelDS);
-    RTSEL SelCS64;
-    SSMR3GetSel(pSSM, &SelCS64);
-    RTSEL SelDS64;
-    SSMR3GetSel(pSSM, &SelDS64);
-    RTSEL SelTSS;
-    SSMR3GetSel(pSSM, &SelTSS);
-    RTSEL SelTSSTrap08;
-    SSMR3GetSel(pSSM, &SelTSSTrap08);
-
-    /* Copy the selectors; they will be checked during relocation. */
-    PSELM pSelm = &pVM->selm.s;
-    pSelm->aHyperSel[SELM_HYPER_SEL_CS]         = SelCS;
-    pSelm->aHyperSel[SELM_HYPER_SEL_DS]         = SelDS;
-    pSelm->aHyperSel[SELM_HYPER_SEL_CS64]       = SelCS64;
-    pSelm->aHyperSel[SELM_HYPER_SEL_TSS]        = SelTSS;
-    pSelm->aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = SelTSSTrap08;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Sync the GDT, LDT and TSS after loading the state.
- *
- * Just to play save, we set the FFs to force syncing before
- * executing GC code.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) selmR3LoadDone(PVM pVM, PSSMHANDLE pSSM)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    LogFlow(("selmR3LoadDone:\n"));
-
-    /*
-     * Don't do anything if it's a load failure.
-     */
-    int rc = SSMR3HandleGetStatus(pSSM);
-    if (RT_FAILURE(rc))
-        return VINF_SUCCESS;
-
-    /*
-     * Do the syncing if we're in protected mode.
-     */
-    if (PGMGetGuestMode(pVCpu) != PGMMODE_REAL)
-    {
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-        SELMR3UpdateFromCPUM(pVM, pVCpu);
-    }
-
-    /*
-     * Flag everything for resync on next raw mode entry.
-     */
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Updates the Guest GDT & LDT virtualization based on current CPU state.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VMCPU to operate on.
- */
-VMMR3DECL(int) SELMR3UpdateFromCPUM(PVM pVM, PVMCPU pVCpu)
-{
-    int    rc    = VINF_SUCCESS;
-
-    if (pVM->selm.s.fDisableMonitoring)
-    {
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-
-        return VINF_SUCCESS;
-    }
-
-    STAM_PROFILE_START(&pVM->selm.s.StatUpdateFromCPUM, a);
-
-    /*
-     * GDT sync
-     */
-    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_GDT))
-    {
-        /*
-         * Always assume the best
-         */
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
-
-        /* If the GDT was changed, then make sure the LDT is checked too */
-        /** @todo only do this if the actual ldtr selector was changed; this is a bit excessive */
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-        /* Same goes for the TSS selector */
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-
-        /*
-         * Get the GDTR and check if there is anything to do (there usually is).
-         */
-        VBOXGDTR    GDTR;
-        CPUMGetGuestGDTR(pVCpu, &GDTR);
-        if (GDTR.cbGdt < sizeof(X86DESC))
-        {
-            Log(("No GDT entries...\n"));
-            STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-            return VINF_SUCCESS;
-        }
-
-        /*
-         * Read the Guest GDT.
-         * ASSUMES that the entire GDT is in memory.
-         */
-        RTUINT      cbEffLimit = GDTR.cbGdt;
-        PX86DESC    pGDTE = &pVM->selm.s.paGdtR3[1];
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, pGDTE, GDTR.pGdt + sizeof(X86DESC), cbEffLimit + 1 - sizeof(X86DESC));
-        if (RT_FAILURE(rc))
-        {
-            /*
-             * Read it page by page.
-             *
-             * Keep track of the last valid page and delay memsets and
-             * adjust cbEffLimit to reflect the effective size. The latter
-             * is something we do in the belief that the guest will probably
-             * never actually commit the last page, thus allowing us to keep
-             * our selectors in the high end of the GDT.
-             */
-            RTUINT  cbLeft = cbEffLimit + 1 - sizeof(X86DESC);
-            RTGCPTR GCPtrSrc = (RTGCPTR)GDTR.pGdt + sizeof(X86DESC);
-            uint8_t *pu8Dst = (uint8_t *)&pVM->selm.s.paGdtR3[1];
-            uint8_t *pu8DstInvalid = pu8Dst;
-
-            while (cbLeft)
-            {
-                RTUINT cb = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
-                cb = RT_MIN(cb, cbLeft);
-                rc = PGMPhysSimpleReadGCPtr(pVCpu, pu8Dst, GCPtrSrc, cb);
-                if (RT_SUCCESS(rc))
-                {
-                    if (pu8DstInvalid != pu8Dst)
-                        memset(pu8DstInvalid, 0, pu8Dst - pu8DstInvalid);
-                    GCPtrSrc += cb;
-                    pu8Dst += cb;
-                    pu8DstInvalid = pu8Dst;
-                }
-                else if (   rc == VERR_PAGE_NOT_PRESENT
-                         || rc == VERR_PAGE_TABLE_NOT_PRESENT)
-                {
-                    GCPtrSrc += cb;
-                    pu8Dst += cb;
-                }
-                else
-                {
-                    AssertReleaseMsgFailed(("Couldn't read GDT at %016RX64, rc=%Rrc!\n", GDTR.pGdt, rc));
-                    STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-                    return VERR_NOT_IMPLEMENTED;
-                }
-                cbLeft -= cb;
-            }
-
-            /* any invalid pages at the end? */
-            if (pu8DstInvalid != pu8Dst)
-            {
-                cbEffLimit = pu8DstInvalid - (uint8_t *)pVM->selm.s.paGdtR3 - 1;
-                /* If any GDTEs was invalidated, zero them. */
-                if (cbEffLimit < pVM->selm.s.cbEffGuestGdtLimit)
-                    memset(pu8DstInvalid + cbEffLimit + 1, 0, pVM->selm.s.cbEffGuestGdtLimit - cbEffLimit);
-            }
-
-            /* keep track of the effective limit. */
-            if (cbEffLimit != pVM->selm.s.cbEffGuestGdtLimit)
-            {
-                Log(("SELMR3UpdateFromCPUM: cbEffGuestGdtLimit=%#x -> %#x (actual %#x)\n",
-                     pVM->selm.s.cbEffGuestGdtLimit, cbEffLimit, GDTR.cbGdt));
-                pVM->selm.s.cbEffGuestGdtLimit = cbEffLimit;
-            }
-        }
-
-        /*
-         * Check if the Guest GDT intrudes on our GDT entries.
-         */
-        /** @todo we should try to minimize relocations by making sure our current selectors can be reused. */
-        RTSEL aHyperSel[SELM_HYPER_SEL_MAX];
-        if (cbEffLimit >= SELM_HYPER_DEFAULT_BASE)
-        {
-            PX86DESC    pGDTEStart = pVM->selm.s.paGdtR3;
-            PX86DESC    pGDTECur   = (PX86DESC)((char *)pGDTEStart + GDTR.cbGdt + 1 - sizeof(X86DESC));
-            int         iGDT       = 0;
-
-            Log(("Internal SELM GDT conflict: use non-present entries\n"));
-            STAM_COUNTER_INC(&pVM->selm.s.StatScanForHyperSels);
-            while (pGDTECur > pGDTEStart)
-            {
-                /* We can reuse non-present entries */
-                if (!pGDTECur->Gen.u1Present)
-                {
-                    aHyperSel[iGDT] = ((uintptr_t)pGDTECur - (uintptr_t)pVM->selm.s.paGdtR3) / sizeof(X86DESC);
-                    aHyperSel[iGDT] = aHyperSel[iGDT] << X86_SEL_SHIFT;
-                    Log(("SELM: Found unused GDT %04X\n", aHyperSel[iGDT]));
-                    iGDT++;
-                    if (iGDT >= SELM_HYPER_SEL_MAX)
-                        break;
-                }
-
-                pGDTECur--;
-            }
-            if (iGDT != SELM_HYPER_SEL_MAX)
-            {
-                AssertReleaseMsgFailed(("Internal SELM GDT conflict.\n"));
-                STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-                return VERR_NOT_IMPLEMENTED;
-            }
-        }
-        else
-        {
-            aHyperSel[SELM_HYPER_SEL_CS]         = SELM_HYPER_DEFAULT_SEL_CS;
-            aHyperSel[SELM_HYPER_SEL_DS]         = SELM_HYPER_DEFAULT_SEL_DS;
-            aHyperSel[SELM_HYPER_SEL_CS64]       = SELM_HYPER_DEFAULT_SEL_CS64;
-            aHyperSel[SELM_HYPER_SEL_TSS]        = SELM_HYPER_DEFAULT_SEL_TSS;
-            aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = SELM_HYPER_DEFAULT_SEL_TSS_TRAP08;
-        }
-
-        /*
-         * Work thru the copied GDT entries adjusting them for correct virtualization.
-         */
-        PX86DESC pGDTEEnd = (PX86DESC)((char *)pGDTE + cbEffLimit + 1 - sizeof(X86DESC));
-        while (pGDTE < pGDTEEnd)
-        {
-            if (pGDTE->Gen.u1Present)
-            {
-                /*
-                 * Code and data selectors are generally 1:1, with the
-                 * 'little' adjustment we do for DPL 0 selectors.
-                 */
-                if (pGDTE->Gen.u1DescType)
-                {
-                    /*
-                     * Hack for A-bit against Trap E on read-only GDT.
-                     */
-                    /** @todo Fix this by loading ds and cs before turning off WP. */
-                    pGDTE->Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
-
-                    /*
-                     * All DPL 0 code and data segments are squeezed into DPL 1.
-                     *
-                     * We're skipping conforming segments here because those
-                     * cannot give us any trouble.
-                     */
-                    if (    pGDTE->Gen.u2Dpl == 0
-                        &&      (pGDTE->Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
-                            !=  (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF) )
-                        pGDTE->Gen.u2Dpl = 1;
-                }
-                else
-                {
-                    /*
-                     * System type selectors are marked not present.
-                     * Recompiler or special handling is required for these.
-                     */
-                    /** @todo what about interrupt gates and rawr0? */
-                    pGDTE->Gen.u1Present = 0;
-                }
-            }
-
-            /* Next GDT entry. */
-            pGDTE++;
-        }
-
-        /*
-         * Check if our hypervisor selectors were changed.
-         */
-        if (    aHyperSel[SELM_HYPER_SEL_CS]         != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]
-            ||  aHyperSel[SELM_HYPER_SEL_DS]         != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]
-            ||  aHyperSel[SELM_HYPER_SEL_CS64]       != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64]
-            ||  aHyperSel[SELM_HYPER_SEL_TSS]        != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]
-            ||  aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08])
-        {
-            /* Reinitialize our hypervisor GDTs */
-            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]         = aHyperSel[SELM_HYPER_SEL_CS];
-            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]         = aHyperSel[SELM_HYPER_SEL_DS];
-            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64]       = aHyperSel[SELM_HYPER_SEL_CS64];
-            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]        = aHyperSel[SELM_HYPER_SEL_TSS];
-            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = aHyperSel[SELM_HYPER_SEL_TSS_TRAP08];
-
-            STAM_COUNTER_INC(&pVM->selm.s.StatHyperSelsChanged);
-
-            /*
-             * Do the relocation callbacks to let everyone update their hyper selector dependencies.
-             * (SELMR3Relocate will call selmR3SetupHyperGDTSelectors() for us.)
-             */
-            VMR3Relocate(pVM, 0);
-        }
-        else if (cbEffLimit >= SELM_HYPER_DEFAULT_BASE)
-            /* We overwrote all entries above, so we have to save them again. */
-            selmR3SetupHyperGDTSelectors(pVM);
-
-        /*
-         * Adjust the cached GDT limit.
-         * Any GDT entries which have been removed must be cleared.
-         */
-        if (pVM->selm.s.GuestGdtr.cbGdt != GDTR.cbGdt)
-        {
-            if (pVM->selm.s.GuestGdtr.cbGdt > GDTR.cbGdt)
-                memset(pGDTE, 0, pVM->selm.s.GuestGdtr.cbGdt - GDTR.cbGdt);
-#ifndef SELM_TRACK_GUEST_GDT_CHANGES
-            pVM->selm.s.GuestGdtr.cbGdt = GDTR.cbGdt;
-#endif
-        }
-
-#ifdef SELM_TRACK_GUEST_GDT_CHANGES
-        /*
-         * Check if Guest's GDTR is changed.
-         */
-        if (    GDTR.pGdt != pVM->selm.s.GuestGdtr.pGdt
-            ||  GDTR.cbGdt != pVM->selm.s.GuestGdtr.cbGdt)
-        {
-            Log(("SELMR3UpdateFromCPUM: Guest's GDT is changed to pGdt=%016RX64 cbGdt=%08X\n", GDTR.pGdt, GDTR.cbGdt));
-
-            /*
-             * [Re]Register write virtual handler for guest's GDT.
-             */
-            if (pVM->selm.s.GuestGdtr.pGdt != RTRCPTR_MAX && pVM->selm.s.fGDTRangeRegistered)
-            {
-                rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GuestGdtr.pGdt);
-                AssertRC(rc);
-            }
-
-            rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, GDTR.pGdt, GDTR.pGdt + GDTR.cbGdt /* already inclusive */,
-                                             0, selmR3GuestGDTWriteHandler, "selmRCGuestGDTWriteHandler", 0, "Guest GDT write access handler");
-            if (RT_FAILURE(rc))
-                return rc;
-
-            /* Update saved Guest GDTR. */
-            pVM->selm.s.GuestGdtr = GDTR;
-            pVM->selm.s.fGDTRangeRegistered = true;
-        }
-#endif
-    }
-
-    /*
-     * TSS sync
-     */
-    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_TSS))
-    {
-        SELMR3SyncTSS(pVM, pVCpu);
-    }
-
-    /*
-     * LDT sync
-     */
-    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_LDT))
-    {
-        /*
-         * Always assume the best
-         */
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
-
-        /*
-         * LDT handling is done similarly to the GDT handling with a shadow
-         * array. However, since the LDT is expected to be swappable (at least
-         * some ancient OSes makes it swappable) it must be floating and
-         * synced on a per-page basis.
-         *
-         * Eventually we will change this to be fully on demand. Meaning that
-         * we will only sync pages containing LDT selectors actually used and
-         * let the #PF handler lazily sync pages as they are used.
-         * (This applies to GDT too, when we start making OS/2 fast.)
-         */
-
-        /*
-         * First, determine the current LDT selector.
-         */
-        RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
-        if ((SelLdt & X86_SEL_MASK) == 0)
-        {
-            /* ldtr = 0 - update hyper LDTR and deregister any active handler. */
-            CPUMSetHyperLDTR(pVCpu, 0);
-#ifdef SELM_TRACK_GUEST_LDT_CHANGES
-            if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
-            {
-                rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
-                AssertRC(rc);
-                pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
-            }
-#endif
-            STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-            return VINF_SUCCESS;
-        }
-
-        /*
-         * Get the LDT selector.
-         */
-        PX86DESC    pDesc = &pVM->selm.s.paGdtR3[SelLdt >> X86_SEL_SHIFT];
-        RTGCPTR     GCPtrLdt = X86DESC_BASE(*pDesc);
-        unsigned    cbLdt = X86DESC_LIMIT(*pDesc);
-        if (pDesc->Gen.u1Granularity)
-            cbLdt = (cbLdt << PAGE_SHIFT) | PAGE_OFFSET_MASK;
-
-        /*
-         * Validate it.
-         */
-        if (    !cbLdt
-            ||  SelLdt >= pVM->selm.s.GuestGdtr.cbGdt
-            ||  pDesc->Gen.u1DescType
-            ||  pDesc->Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
-        {
-            AssertMsg(!cbLdt, ("Invalid LDT %04x!\n", SelLdt));
-
-            /* cbLdt > 0:
-             * This is quite impossible, so we do as most people do when faced with
-             * the impossible, we simply ignore it.
-             */
-            CPUMSetHyperLDTR(pVCpu, 0);
-#ifdef SELM_TRACK_GUEST_LDT_CHANGES
-            if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
-            {
-                rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
-                AssertRC(rc);
-                pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
-            }
-#endif
-            STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-            return VINF_SUCCESS;
-        }
-        /** @todo check what intel does about odd limits. */
-        AssertMsg(RT_ALIGN(cbLdt + 1, sizeof(X86DESC)) == cbLdt + 1 && cbLdt <= 0xffff, ("cbLdt=%d\n", cbLdt));
-
-        /*
-         * Use the cached guest ldt address if the descriptor has already been modified (see below)
-         * (this is necessary due to redundant LDT updates; see todo above at GDT sync)
-         */
-        if (MMHyperIsInsideArea(pVM, GCPtrLdt))
-            GCPtrLdt = pVM->selm.s.GCPtrGuestLdt;   /* use the old one */
-
-
-#ifdef SELM_TRACK_GUEST_LDT_CHANGES
-        /** @todo Handle only present LDT segments. */
-    //    if (pDesc->Gen.u1Present)
-        {
-            /*
-             * Check if Guest's LDT address/limit is changed.
-             */
-            if (    GCPtrLdt != pVM->selm.s.GCPtrGuestLdt
-                ||  cbLdt != pVM->selm.s.cbLdtLimit)
-            {
-                Log(("SELMR3UpdateFromCPUM: Guest LDT changed to from %RGv:%04x to %RGv:%04x. (GDTR=%016RX64:%04x)\n",
-                     pVM->selm.s.GCPtrGuestLdt, pVM->selm.s.cbLdtLimit, GCPtrLdt, cbLdt, pVM->selm.s.GuestGdtr.pGdt, pVM->selm.s.GuestGdtr.cbGdt));
-
-                /*
-                 * [Re]Register write virtual handler for guest's GDT.
-                 * In the event of LDT overlapping something, don't install it just assume it's being updated.
-                 */
-                if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
-                {
-                    rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
-                    AssertRC(rc);
-                }
-#ifdef DEBUG
-                if (pDesc->Gen.u1Present)
-                    Log(("LDT selector marked not present!!\n"));
-#endif
-                rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, GCPtrLdt, GCPtrLdt + cbLdt /* already inclusive */,
-                                                 0, selmR3GuestLDTWriteHandler, "selmRCGuestLDTWriteHandler", 0, "Guest LDT write access handler");
-                if (rc == VERR_PGM_HANDLER_VIRTUAL_CONFLICT)
-                {
-                    /** @todo investigate the various cases where conflicts happen and try avoid them by enh. the instruction emulation. */
-                    pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
-                    Log(("WARNING: Guest LDT (%RGv:%04x) conflicted with existing access range!! Assumes LDT is begin updated. (GDTR=%016RX64:%04x)\n",
-                         GCPtrLdt, cbLdt, pVM->selm.s.GuestGdtr.pGdt, pVM->selm.s.GuestGdtr.cbGdt));
-                }
-                else if (RT_SUCCESS(rc))
-                    pVM->selm.s.GCPtrGuestLdt = GCPtrLdt;
-                else
-                {
-                    CPUMSetHyperLDTR(pVCpu, 0);
-                    STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-                    return rc;
-                }
-
-                pVM->selm.s.cbLdtLimit = cbLdt;
-            }
-        }
-#else
-        pVM->selm.s.cbLdtLimit = cbLdt;
-#endif
-
-        /*
-         * Calc Shadow LDT base.
-         */
-        unsigned    off;
-        pVM->selm.s.offLdtHyper = off = (GCPtrLdt & PAGE_OFFSET_MASK);
-        RTGCPTR     GCPtrShadowLDT  = (RTGCPTR)((RTGCUINTPTR)pVM->selm.s.pvLdtRC + off);
-        PX86DESC    pShadowLDT      = (PX86DESC)((uintptr_t)pVM->selm.s.pvLdtR3 + off);
-
-        /*
-         * Enable the LDT selector in the shadow GDT.
-         */
-        pDesc->Gen.u1Present    = 1;
-        pDesc->Gen.u16BaseLow   = RT_LOWORD(GCPtrShadowLDT);
-        pDesc->Gen.u8BaseHigh1  = RT_BYTE3(GCPtrShadowLDT);
-        pDesc->Gen.u8BaseHigh2  = RT_BYTE4(GCPtrShadowLDT);
-        pDesc->Gen.u1Available  = 0;
-        pDesc->Gen.u1Long       = 0;
-        if (cbLdt > 0xffff)
-        {
-            cbLdt = 0xffff;
-            pDesc->Gen.u4LimitHigh  = 0;
-            pDesc->Gen.u16LimitLow  = pDesc->Gen.u1Granularity ? 0xf : 0xffff;
-        }
-
-        /*
-         * Set Hyper LDTR and notify TRPM.
-         */
-        CPUMSetHyperLDTR(pVCpu, SelLdt);
-
-        /*
-         * Loop synchronising the LDT page by page.
-         */
-        /** @todo investigate how intel handle various operations on half present cross page entries. */
-        off = GCPtrLdt & (sizeof(X86DESC) - 1);
-        AssertMsg(!off, ("LDT is not aligned on entry size! GCPtrLdt=%08x\n", GCPtrLdt));
-
-        /* Note: Do not skip the first selector; unlike the GDT, a zero LDT selector is perfectly valid. */
-        unsigned    cbLeft = cbLdt + 1;
-        PX86DESC   pLDTE = pShadowLDT;
-        while (cbLeft)
-        {
-            /*
-             * Read a chunk.
-             */
-            unsigned  cbChunk = PAGE_SIZE - ((RTGCUINTPTR)GCPtrLdt & PAGE_OFFSET_MASK);
-            if (cbChunk > cbLeft)
-                cbChunk = cbLeft;
-            rc = PGMPhysSimpleReadGCPtr(pVCpu, pShadowLDT, GCPtrLdt, cbChunk);
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Mark page
-                 */
-                rc = PGMMapSetPage(pVM, GCPtrShadowLDT & PAGE_BASE_GC_MASK, PAGE_SIZE, X86_PTE_P | X86_PTE_A | X86_PTE_D);
-                AssertRC(rc);
-
-                /*
-                 * Loop thru the available LDT entries.
-                 * Figure out where to start and end and the potential cross pageness of
-                 * things adds a little complexity. pLDTE is updated there and not in the
-                 * 'next' part of the loop. The pLDTEEnd is inclusive.
-                 */
-                PX86DESC pLDTEEnd = (PX86DESC)((uintptr_t)pShadowLDT + cbChunk) - 1;
-                if (pLDTE + 1 < pShadowLDT)
-                    pLDTE = (PX86DESC)((uintptr_t)pShadowLDT + off);
-                while (pLDTE <= pLDTEEnd)
-                {
-                    if (pLDTE->Gen.u1Present)
-                    {
-                        /*
-                         * Code and data selectors are generally 1:1, with the
-                         * 'little' adjustment we do for DPL 0 selectors.
-                         */
-                        if (pLDTE->Gen.u1DescType)
-                        {
-                            /*
-                             * Hack for A-bit against Trap E on read-only GDT.
-                             */
-                            /** @todo Fix this by loading ds and cs before turning off WP. */
-                            if (!(pLDTE->Gen.u4Type & X86_SEL_TYPE_ACCESSED))
-                                pLDTE->Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
-
-                            /*
-                             * All DPL 0 code and data segments are squeezed into DPL 1.
-                             *
-                             * We're skipping conforming segments here because those
-                             * cannot give us any trouble.
-                             */
-                            if (    pLDTE->Gen.u2Dpl == 0
-                                &&      (pLDTE->Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
-                                    !=  (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF) )
-                                pLDTE->Gen.u2Dpl = 1;
-                        }
-                        else
-                        {
-                            /*
-                             * System type selectors are marked not present.
-                             * Recompiler or special handling is required for these.
-                             */
-                            /** @todo what about interrupt gates and rawr0? */
-                            pLDTE->Gen.u1Present = 0;
-                        }
-                    }
-
-                    /* Next LDT entry. */
-                    pLDTE++;
-                }
-            }
-            else
-            {
-                AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("rc=%Rrc\n", rc));
-                rc = PGMMapSetPage(pVM, GCPtrShadowLDT & PAGE_BASE_GC_MASK, PAGE_SIZE, 0);
-                AssertRC(rc);
-            }
-
-            /*
-             * Advance to the next page.
-             */
-            cbLeft          -= cbChunk;
-            GCPtrShadowLDT  += cbChunk;
-            pShadowLDT       = (PX86DESC)((char *)pShadowLDT + cbChunk);
-            GCPtrLdt        += cbChunk;
-        }
-    }
-
-    STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * \#PF Handler callback for virtual access handler ranges.
- *
- * Important to realize that a physical page in a range can have aliases, and
- * for ALL and WRITE handlers these will also trigger.
- *
- * @returns VINF_SUCCESS if the handler have carried out the operation.
- * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
- * @param   pVM             VM Handle.
- * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
- * @param   pvPtr           The HC mapping of that address.
- * @param   pvBuf           What the guest is reading/writing.
- * @param   cbBuf           How much it's reading/writing.
- * @param   enmAccessType   The access type.
- * @param   pvUser          User argument.
- */
-static DECLCALLBACK(int) selmR3GuestGDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
-{
-    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
-    Log(("selmR3GuestGDTWriteHandler: write to %RGv size %d\n", GCPtr, cbBuf));
-
-    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_SELM_SYNC_GDT);
-    return VINF_PGM_HANDLER_DO_DEFAULT;
-}
-
-
-/**
- * \#PF Handler callback for virtual access handler ranges.
- *
- * Important to realize that a physical page in a range can have aliases, and
- * for ALL and WRITE handlers these will also trigger.
- *
- * @returns VINF_SUCCESS if the handler have carried out the operation.
- * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
- * @param   pVM             VM Handle.
- * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
- * @param   pvPtr           The HC mapping of that address.
- * @param   pvBuf           What the guest is reading/writing.
- * @param   cbBuf           How much it's reading/writing.
- * @param   enmAccessType   The access type.
- * @param   pvUser          User argument.
- */
-static DECLCALLBACK(int) selmR3GuestLDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
-{
-    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
-    Log(("selmR3GuestLDTWriteHandler: write to %RGv size %d\n", GCPtr, cbBuf));
-    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_SELM_SYNC_LDT);
-    return VINF_PGM_HANDLER_DO_DEFAULT;
-}
-
-
-/**
- * \#PF Handler callback for virtual access handler ranges.
- *
- * Important to realize that a physical page in a range can have aliases, and
- * for ALL and WRITE handlers these will also trigger.
- *
- * @returns VINF_SUCCESS if the handler have carried out the operation.
- * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
- * @param   pVM             VM Handle.
- * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
- * @param   pvPtr           The HC mapping of that address.
- * @param   pvBuf           What the guest is reading/writing.
- * @param   cbBuf           How much it's reading/writing.
- * @param   enmAccessType   The access type.
- * @param   pvUser          User argument.
- */
-static DECLCALLBACK(int) selmR3GuestTSSWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
-{
-    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
-    Log(("selmR3GuestTSSWriteHandler: write %.*Rhxs to %RGv size %d\n", RT_MIN(8, cbBuf), pvBuf, GCPtr, cbBuf));
-
-    /** @todo This can be optimized by checking for the ESP0 offset and tracking TR
-     *        reloads in REM (setting VM_FF_SELM_SYNC_TSS if TR is reloaded). We
-     *        should probably also deregister the virtual handler if TR.base/size
-     *        changes while we're in REM. */
-
-    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_SELM_SYNC_TSS);
-
-    return VINF_PGM_HANDLER_DO_DEFAULT;
-}
-
-
-/**
- * Synchronize the shadowed fields in the TSS.
- *
- * At present we're shadowing the ring-0 stack selector & pointer, and the
- * interrupt redirection bitmap (if present). We take the lazy approach wrt to
- * REM and this function is called both if REM made any changes to the TSS or
- * loaded TR.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VMCPU to operate on.
- */
-VMMR3DECL(int) SELMR3SyncTSS(PVM pVM, PVMCPU pVCpu)
-{
-    int    rc;
-
-    if (pVM->selm.s.fDisableMonitoring)
-    {
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-        return VINF_SUCCESS;
-    }
-
-    STAM_PROFILE_START(&pVM->selm.s.StatTSSSync, a);
-    Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_TSS));
-
-    /*
-     * Get TR and extract and store the basic info.
-     *
-     * Note! The TSS limit is not checked by the LTR code, so we
-     *       have to be a bit careful with it. We make sure cbTss
-     *       won't be zero if TR is valid and if it's NULL we'll
-     *       make sure cbTss is 0.
-     */
-    CPUMSELREGHID   trHid;
-    RTSEL           SelTss   = CPUMGetGuestTR(pVCpu, &trHid);
-    RTGCPTR         GCPtrTss = trHid.u64Base;
-    uint32_t        cbTss    = trHid.u32Limit;
-    Assert(     (SelTss & X86_SEL_MASK)
-           ||   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
-           ||   (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */));
-    if (SelTss & X86_SEL_MASK)
-    {
-        Assert(!(SelTss & X86_SEL_LDT));
-        Assert(trHid.Attr.n.u1DescType == 0);
-        Assert(   trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY
-               || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY);
-        if (!++cbTss)
-            cbTss = UINT32_MAX;
-    }
-    else
-    {
-        Assert(   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
-               || (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */));
-        cbTss = 0; /* the reset case. */
-    }
-    pVM->selm.s.cbGuestTss     = cbTss;
-    pVM->selm.s.fGuestTss32Bit = trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
-                              || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY;
-
-    /*
-     * Figure out the size of what need to monitor.
-     */
-    /* We're not interested in any 16-bit TSSes. */
-    uint32_t cbMonitoredTss = cbTss;
-    if (    trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL
-        &&  trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
-        cbMonitoredTss = 0;
-
-    pVM->selm.s.offGuestIoBitmap = 0;
-    bool fNoRing1Stack = true;
-    if (cbMonitoredTss)
-    {
-        /*
-         * 32-bit TSS. What we're really keen on is the SS0 and ESP0 fields.
-         * If VME is enabled we also want to keep an eye on the interrupt
-         * redirection bitmap.
-         */
-        VBOXTSS Tss;
-        uint32_t cr4 = CPUMGetGuestCR4(pVCpu);
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, &Tss, GCPtrTss, RT_OFFSETOF(VBOXTSS, IntRedirBitmap));
-        if (    !(cr4 & X86_CR4_VME)
-            ||   (  RT_SUCCESS(rc)
-                 && Tss.offIoBitmap < sizeof(VBOXTSS) /* too small */
-                 && Tss.offIoBitmap > cbTss)          /* beyond the end */ /** @todo not sure how the partial case is handled; probably not allowed. */
-           )
-            /* No interrupt redirection bitmap, just ESP0 and SS0. */
-            cbMonitoredTss = RT_UOFFSETOF(VBOXTSS, padding_ss0);
-        else if (RT_SUCCESS(rc))
-        {
-            /*
-             * Everything up to and including the interrupt redirection bitmap. Unfortunately
-             * this can be quite a large chunk. We use to skip it earlier and just hope it
-             * was kind of static...
-             *
-             * Update the virtual interrupt redirection bitmap while we're here.
-             * (It is located in the 32 bytes before TR:offIoBitmap.)
-             */
-            cbMonitoredTss = Tss.offIoBitmap;
-            pVM->selm.s.offGuestIoBitmap = Tss.offIoBitmap;
-
-            uint32_t offRedirBitmap = Tss.offIoBitmap - sizeof(Tss.IntRedirBitmap);
-            rc = PGMPhysSimpleReadGCPtr(pVCpu, &pVM->selm.s.Tss.IntRedirBitmap,
-                                        GCPtrTss + offRedirBitmap, sizeof(Tss.IntRedirBitmap));
-            AssertRC(rc);
-            /** @todo memset the bitmap on failure? */
-            Log2(("Redirection bitmap:\n"));
-            Log2(("%.*Rhxd\n", sizeof(Tss.IntRedirBitmap), &pVM->selm.s.Tss.IntRedirBitmap));
-        }
-        else
-        {
-            cbMonitoredTss = RT_OFFSETOF(VBOXTSS, IntRedirBitmap);
-            pVM->selm.s.offGuestIoBitmap = 0;
-            /** @todo memset the bitmap? */
-        }
-
-        /*
-         * Update the ring 0 stack selector and base address.
-         */
-        if (RT_SUCCESS(rc))
-        {
-#ifdef LOG_ENABLED
-            if (LogIsEnabled())
-            {
-                uint32_t ssr0, espr0;
-                SELMGetRing1Stack(pVM, &ssr0, &espr0);
-                if ((ssr0 & ~1) != Tss.ss0 || espr0 != Tss.esp0)
-                {
-                    RTGCPHYS GCPhys = NIL_RTGCPHYS;
-                    rc = PGMGstGetPage(pVCpu, GCPtrTss, NULL, &GCPhys); AssertRC(rc);
-                    Log(("SELMR3SyncTSS: Updating TSS ring 0 stack to %04X:%08X from %04X:%08X; TSS Phys=%RGp)\n",
-                         Tss.ss0, Tss.esp0, (ssr0 & ~1), espr0,  GCPhys));
-                    AssertMsg(ssr0 != Tss.ss0,
-                              ("ring-1 leak into TSS.SS0! %04X:%08X from %04X:%08X; TSS Phys=%RGp)\n",
-                               Tss.ss0, Tss.esp0, (ssr0 & ~1), espr0,  GCPhys));
-                }
-                Log(("offIoBitmap=%#x\n", Tss.offIoBitmap));
-            }
-#endif /* LOG_ENABLED */
-            AssertMsg(!(Tss.ss0 & 3), ("ring-1 leak into TSS.SS0? %04X:%08X\n", Tss.ss0, Tss.esp0));
-
-            /* Update our TSS structure for the guest's ring 1 stack */
-            selmSetRing1Stack(pVM, Tss.ss0 | 1, Tss.esp0);
-            pVM->selm.s.fSyncTSSRing0Stack = fNoRing1Stack = false;
-        }
-    }
-
-    /*
-     * Flush the ring-1 stack and the direct syscall dispatching if we
-     * cannot obtain SS0:ESP0.
-     */
-    if (fNoRing1Stack)
-    {
-        selmSetRing1Stack(pVM, 0 /* invalid SS */, 0);
-        pVM->selm.s.fSyncTSSRing0Stack = cbMonitoredTss != 0;
-
-        /** @todo handle these dependencies better! */
-        TRPMR3SetGuestTrapHandler(pVM, 0x2E, TRPM_INVALID_HANDLER);
-        TRPMR3SetGuestTrapHandler(pVM, 0x80, TRPM_INVALID_HANDLER);
-    }
-
-    /*
-     * Check for monitor changes and apply them.
-     */
-    if (    GCPtrTss        != pVM->selm.s.GCPtrGuestTss
-        ||  cbMonitoredTss  != pVM->selm.s.cbMonitoredGuestTss)
-    {
-        Log(("SELMR3SyncTSS: Guest's TSS is changed to pTss=%RGv cbMonitoredTss=%08X cbGuestTss=%#08x\n",
-             GCPtrTss, cbMonitoredTss, pVM->selm.s.cbGuestTss));
-
-        /* Release the old range first. */
-        if (pVM->selm.s.GCPtrGuestTss != RTRCPTR_MAX)
-        {
-            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestTss);
-            AssertRC(rc);
-        }
-
-        /* Register the write handler if TS != 0. */
-        if (cbMonitoredTss != 0)
-        {
-            rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, GCPtrTss, GCPtrTss + cbMonitoredTss - 1,
-                                             0, selmR3GuestTSSWriteHandler,
-                                             "selmRCGuestTSSWriteHandler", 0, "Guest TSS write access handler");
-            if (RT_FAILURE(rc))
-            {
-                STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
-                return rc;
-            }
-
-            /* Update saved Guest TSS info. */
-            pVM->selm.s.GCPtrGuestTss       = GCPtrTss;
-            pVM->selm.s.cbMonitoredGuestTss = cbMonitoredTss;
-            pVM->selm.s.GCSelTss            = SelTss;
-        }
-        else
-        {
-            pVM->selm.s.GCPtrGuestTss       = RTRCPTR_MAX;
-            pVM->selm.s.cbMonitoredGuestTss = 0;
-            pVM->selm.s.GCSelTss            = 0;
-        }
-    }
-
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-
-    STAM_PROFILE_STOP(&pVM->selm.s.StatTSSSync, a);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Compares the Guest GDT and LDT with the shadow tables.
- * This is a VBOX_STRICT only function.
- *
- * @returns VBox status code.
- * @param   pVM         The VM Handle.
- */
-VMMR3DECL(int) SELMR3DebugCheck(PVM pVM)
-{
-#ifdef VBOX_STRICT
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    /*
-     * Get GDTR and check for conflict.
-     */
-    VBOXGDTR  GDTR;
-    CPUMGetGuestGDTR(pVCpu, &GDTR);
-    if (GDTR.cbGdt == 0)
-        return VINF_SUCCESS;
-
-    if (GDTR.cbGdt >= (unsigned)(pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> X86_SEL_SHIFT))
-        Log(("SELMR3DebugCheck: guest GDT size forced us to look for unused selectors.\n"));
-
-    if (GDTR.cbGdt != pVM->selm.s.GuestGdtr.cbGdt)
-        Log(("SELMR3DebugCheck: limits have changed! new=%d old=%d\n", GDTR.cbGdt, pVM->selm.s.GuestGdtr.cbGdt));
-
-    /*
-     * Loop thru the GDT checking each entry.
-     */
-    RTGCPTR     GCPtrGDTEGuest = GDTR.pGdt;
-    PX86DESC    pGDTE = pVM->selm.s.paGdtR3;
-    PX86DESC    pGDTEEnd = (PX86DESC)((uintptr_t)pGDTE + GDTR.cbGdt);
-    while (pGDTE < pGDTEEnd)
-    {
-        X86DESC    GDTEGuest;
-        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &GDTEGuest, GCPtrGDTEGuest, sizeof(GDTEGuest));
-        if (RT_SUCCESS(rc))
-        {
-            if (pGDTE->Gen.u1DescType || pGDTE->Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
-            {
-                if (   pGDTE->Gen.u16LimitLow != GDTEGuest.Gen.u16LimitLow
-                    || pGDTE->Gen.u4LimitHigh != GDTEGuest.Gen.u4LimitHigh
-                    || pGDTE->Gen.u16BaseLow  != GDTEGuest.Gen.u16BaseLow
-                    || pGDTE->Gen.u8BaseHigh1 != GDTEGuest.Gen.u8BaseHigh1
-                    || pGDTE->Gen.u8BaseHigh2 != GDTEGuest.Gen.u8BaseHigh2
-                    || pGDTE->Gen.u1DefBig    != GDTEGuest.Gen.u1DefBig
-                    || pGDTE->Gen.u1DescType  != GDTEGuest.Gen.u1DescType)
-                {
-                    unsigned iGDT = pGDTE - pVM->selm.s.paGdtR3;
-                    SELMR3DumpDescriptor(*pGDTE, iGDT << 3, "SELMR3DebugCheck: GDT mismatch, shadow");
-                    SELMR3DumpDescriptor(GDTEGuest, iGDT << 3, "SELMR3DebugCheck: GDT mismatch,  guest");
-                }
-            }
-        }
-
-        /* Advance to the next descriptor. */
-        GCPtrGDTEGuest += sizeof(X86DESC);
-        pGDTE++;
-    }
-
-
-    /*
-     * LDT?
-     */
-    RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
-    if ((SelLdt & X86_SEL_MASK) == 0)
-        return VINF_SUCCESS;
-    if (SelLdt > GDTR.cbGdt)
-    {
-        Log(("SELMR3DebugCheck: ldt is out of bound SelLdt=%#x\n", SelLdt));
-        return VERR_INTERNAL_ERROR;
-    }
-    X86DESC    LDTDesc;
-    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &LDTDesc, GDTR.pGdt + (SelLdt & X86_SEL_MASK), sizeof(LDTDesc));
-    if (RT_FAILURE(rc))
-    {
-        Log(("SELMR3DebugCheck: Failed to read LDT descriptor. rc=%d\n", rc));
-        return rc;
-    }
-    RTGCPTR     GCPtrLDTEGuest = X86DESC_BASE(LDTDesc);
-    unsigned    cbLdt = X86DESC_LIMIT(LDTDesc);
-    if (LDTDesc.Gen.u1Granularity)
-        cbLdt = (cbLdt << PAGE_SHIFT) | PAGE_OFFSET_MASK;
-
-    /*
-     * Validate it.
-     */
-    if (!cbLdt)
-        return VINF_SUCCESS;
-    /** @todo check what intel does about odd limits. */
-    AssertMsg(RT_ALIGN(cbLdt + 1, sizeof(X86DESC)) == cbLdt + 1 && cbLdt <= 0xffff, ("cbLdt=%d\n", cbLdt));
-    if (    LDTDesc.Gen.u1DescType
-        ||  LDTDesc.Gen.u4Type != X86_SEL_TYPE_SYS_LDT
-        ||  SelLdt >= pVM->selm.s.GuestGdtr.cbGdt)
-    {
-        Log(("SELmR3DebugCheck: Invalid LDT %04x!\n", SelLdt));
-        return VERR_INTERNAL_ERROR;
-    }
-
-    /*
-     * Loop thru the LDT checking each entry.
-     */
-    unsigned    off = (GCPtrLDTEGuest & PAGE_OFFSET_MASK);
-    PX86DESC    pLDTE = (PX86DESC)((uintptr_t)pVM->selm.s.pvLdtR3 + off);
-    PX86DESC    pLDTEEnd = (PX86DESC)((uintptr_t)pGDTE + cbLdt);
-    while (pLDTE < pLDTEEnd)
-    {
-        X86DESC    LDTEGuest;
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, &LDTEGuest, GCPtrLDTEGuest, sizeof(LDTEGuest));
-        if (RT_SUCCESS(rc))
-        {
-            if (   pLDTE->Gen.u16LimitLow != LDTEGuest.Gen.u16LimitLow
-                || pLDTE->Gen.u4LimitHigh != LDTEGuest.Gen.u4LimitHigh
-                || pLDTE->Gen.u16BaseLow  != LDTEGuest.Gen.u16BaseLow
-                || pLDTE->Gen.u8BaseHigh1 != LDTEGuest.Gen.u8BaseHigh1
-                || pLDTE->Gen.u8BaseHigh2 != LDTEGuest.Gen.u8BaseHigh2
-                || pLDTE->Gen.u1DefBig    != LDTEGuest.Gen.u1DefBig
-                || pLDTE->Gen.u1DescType  != LDTEGuest.Gen.u1DescType)
-            {
-                unsigned iLDT = pLDTE - (PX86DESC)((uintptr_t)pVM->selm.s.pvLdtR3 + off);
-                SELMR3DumpDescriptor(*pLDTE, iLDT << 3, "SELMR3DebugCheck: LDT mismatch, shadow");
-                SELMR3DumpDescriptor(LDTEGuest, iLDT << 3, "SELMR3DebugCheck: LDT mismatch,  guest");
-            }
-        }
-
-        /* Advance to the next descriptor. */
-        GCPtrLDTEGuest += sizeof(X86DESC);
-        pLDTE++;
-    }
-
-#else  /* !VBOX_STRICT */
-    NOREF(pVM);
-#endif /* !VBOX_STRICT */
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Validates the RawR0 TSS values against the one in the Guest TSS.
- *
- * @returns true if it matches.
- * @returns false and assertions on mismatch..
- * @param   pVM     VM Handle.
- */
-VMMR3DECL(bool) SELMR3CheckTSS(PVM pVM)
-{
-#ifdef VBOX_STRICT
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_TSS))
-        return true;
-
-    /*
-     * Get TR and extract the basic info.
-     */
-    CPUMSELREGHID   trHid;
-    RTSEL           SelTss   = CPUMGetGuestTR(pVCpu, &trHid);
-    RTGCPTR         GCPtrTss = trHid.u64Base;
-    uint32_t        cbTss    = trHid.u32Limit;
-    Assert(     (SelTss & X86_SEL_MASK)
-           ||   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
-           ||   (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */));
-    if (SelTss & X86_SEL_MASK)
-    {
-        AssertReturn(!(SelTss & X86_SEL_LDT), false);
-        AssertReturn(trHid.Attr.n.u1DescType == 0, false);
-        AssertReturn(   trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY
-                     || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY,
-                     false);
-        if (!++cbTss)
-            cbTss = UINT32_MAX;
-    }
-    else
-    {
-        AssertReturn(   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
-                     || (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */),
-                     false);
-        cbTss = 0; /* the reset case. */
-    }
-    AssertMsgReturn(pVM->selm.s.cbGuestTss == cbTss, ("%#x %#x\n", pVM->selm.s.cbGuestTss, cbTss), false);
-    AssertMsgReturn(pVM->selm.s.fGuestTss32Bit == (   trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
-                                                   || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY),
-                    ("%RTbool u4Type=%d\n", pVM->selm.s.fGuestTss32Bit, trHid.Attr.n.u4Type),
-                    false);
-    AssertMsgReturn(    pVM->selm.s.GCSelTss == SelTss
-                    ||  (!pVM->selm.s.GCSelTss && !(SelTss & X86_SEL_LDT)),
-                    ("%#x %#x\n", pVM->selm.s.GCSelTss, SelTss),
-                    false);
-    AssertMsgReturn(    pVM->selm.s.GCPtrGuestTss == GCPtrTss
-                    ||  (pVM->selm.s.GCPtrGuestTss == RTRCPTR_MAX && !GCPtrTss),
-                    ("%#RGv %#RGv\n", pVM->selm.s.GCPtrGuestTss, GCPtrTss),
-                    false);
-
-
-    /*
-     * Figure out the size of what need to monitor.
-     */
-    bool fNoRing1Stack = true;
-    /* We're not interested in any 16-bit TSSes. */
-    uint32_t cbMonitoredTss = cbTss;
-    if (    trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL
-        &&  trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
-        cbMonitoredTss = 0;
-    if (cbMonitoredTss)
-    {
-        VBOXTSS Tss;
-        uint32_t cr4 = CPUMGetGuestCR4(pVCpu);
-        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Tss, GCPtrTss, RT_OFFSETOF(VBOXTSS, IntRedirBitmap));
-        AssertReturn(   rc == VINF_SUCCESS
-                        /* Happens early in XP boot during page table switching. */
-                     || (   (rc == VERR_PAGE_TABLE_NOT_PRESENT || rc == VERR_PAGE_NOT_PRESENT)
-                         && !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF)),
-                     false);
-        if (    !(cr4 & X86_CR4_VME)
-            ||   (  RT_SUCCESS(rc)
-                 && Tss.offIoBitmap < sizeof(VBOXTSS) /* too small */
-                 && Tss.offIoBitmap > cbTss)
-           )
-            cbMonitoredTss = RT_UOFFSETOF(VBOXTSS, padding_ss0);
-        else if (RT_SUCCESS(rc))
-        {
-            cbMonitoredTss = Tss.offIoBitmap;
-            AssertMsgReturn(pVM->selm.s.offGuestIoBitmap == Tss.offIoBitmap,
-                            ("#x %#x\n", pVM->selm.s.offGuestIoBitmap, Tss.offIoBitmap),
-                            false);
-
-            /* check the bitmap */
-            uint32_t offRedirBitmap = Tss.offIoBitmap - sizeof(Tss.IntRedirBitmap);
-            rc = PGMPhysSimpleReadGCPtr(pVCpu, &Tss.IntRedirBitmap,
-                                        GCPtrTss + offRedirBitmap, sizeof(Tss.IntRedirBitmap));
-            AssertRCReturn(rc, false);
-            AssertMsgReturn(!memcmp(&Tss.IntRedirBitmap[0], &pVM->selm.s.Tss.IntRedirBitmap[0], sizeof(Tss.IntRedirBitmap)),
-                            ("offIoBitmap=%#x cbTss=%#x\n"
-                             " Guest: %.32Rhxs\n"
-                             "Shadow: %.32Rhxs\n",
-                             Tss.offIoBitmap, cbTss,
-                             &Tss.IntRedirBitmap[0],
-                             &pVM->selm.s.Tss.IntRedirBitmap[0]),
-                            false);
-        }
-        else
-            cbMonitoredTss = RT_OFFSETOF(VBOXTSS, IntRedirBitmap);
-
-        /*
-         * Check SS0 and ESP0.
-         */
-        if (    !pVM->selm.s.fSyncTSSRing0Stack
-            &&  RT_SUCCESS(rc))
-        {
-            if (    Tss.esp0 != pVM->selm.s.Tss.esp1
-                ||  Tss.ss0  != (pVM->selm.s.Tss.ss1 & ~1))
-            {
-                RTGCPHYS GCPhys;
-                rc = PGMGstGetPage(pVCpu, GCPtrTss, NULL, &GCPhys); AssertRC(rc);
-                AssertMsgFailed(("TSS out of sync!! (%04X:%08X vs %04X:%08X (guest)) Tss=%RGv Phys=%RGp\n",
-                                 (pVM->selm.s.Tss.ss1 & ~1), pVM->selm.s.Tss.esp1,
-                                 Tss.ss1, Tss.esp1, GCPtrTss, GCPhys));
-                return false;
-            }
-        }
-        AssertMsgReturn(pVM->selm.s.cbMonitoredGuestTss == cbMonitoredTss, ("%#x %#x\n", pVM->selm.s.cbMonitoredGuestTss, cbMonitoredTss), false);
-    }
-    else
-    {
-        AssertMsgReturn(pVM->selm.s.Tss.ss1 == 0 && pVM->selm.s.Tss.esp1 == 0, ("%04x:%08x\n", pVM->selm.s.Tss.ss1, pVM->selm.s.Tss.esp1), false);
-        AssertReturn(!pVM->selm.s.fSyncTSSRing0Stack, false);
-        AssertMsgReturn(pVM->selm.s.cbMonitoredGuestTss == cbMonitoredTss, ("%#x %#x\n", pVM->selm.s.cbMonitoredGuestTss, cbMonitoredTss), false);
-    }
-
-
-
-    return true;
-
-#else  /* !VBOX_STRICT */
-    NOREF(pVM);
-    return true;
-#endif /* !VBOX_STRICT */
-}
-
-
-/**
- * Returns flat address and limit of LDT by LDT selector from guest GDTR.
- *
- * Fully validate selector.
- *
- * @returns VBox status.
- * @param   pVM       VM Handle.
- * @param   SelLdt    LDT selector.
- * @param   ppvLdt    Where to store the flat address of LDT.
- * @param   pcbLimit  Where to store LDT limit.
- */
-VMMDECL(int) SELMGetLDTFromSel(PVM pVM, RTSEL SelLdt, PRTGCPTR ppvLdt, unsigned *pcbLimit)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    /* Get guest GDTR. */
-    VBOXGDTR GDTR;
-    CPUMGetGuestGDTR(pVCpu, &GDTR);
-
-    /* Check selector TI and GDT limit. */
-    if (   (SelLdt & X86_SEL_LDT)
-        || SelLdt > GDTR.cbGdt)
-        return VERR_INVALID_SELECTOR;
-
-    /* Read descriptor from GC. */
-    X86DESC Desc;
-    int rc = PGMPhysSimpleReadGCPtr(pVCpu, (void *)&Desc, (RTGCPTR)(GDTR.pGdt + (SelLdt & X86_SEL_MASK)), sizeof(Desc));
-    if (RT_FAILURE(rc))
-    {
-        /* fatal */
-        Log(("Can't read LDT descriptor for selector=%04X\n", SelLdt));
-        return VERR_SELECTOR_NOT_PRESENT;
-    }
-
-    /* Check if LDT descriptor is not present. */
-    if (Desc.Gen.u1Present == 0)
-        return VERR_SELECTOR_NOT_PRESENT;
-
-    /* Check LDT descriptor type. */
-    if (    Desc.Gen.u1DescType == 1
-        ||  Desc.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
-        return VERR_INVALID_SELECTOR;
-
-    /* LDT descriptor is ok. */
-    if (ppvLdt)
-    {
-        *ppvLdt = (RTGCPTR)X86DESC_BASE(Desc);
-        *pcbLimit = X86DESC_LIMIT(Desc);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets information about a 64-bit selector, SELMR3GetSelectorInfo helper.
- *
- * See SELMR3GetSelectorInfo for details.
- *
- * @returns VBox status code, see SELMR3GetSelectorInfo for details.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   Sel         The selector to get info about.
- * @param   pSelInfo    Where to store the information.
- */
-static int selmR3GetSelectorInfo64(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
-{
-    /*
-     * Read it from the guest descriptor table.
-     */
-    X86DESC64   Desc;
-    VBOXGDTR    Gdtr;
-    RTGCPTR     GCPtrDesc;
-    CPUMGetGuestGDTR(pVCpu, &Gdtr);
-    if (!(Sel & X86_SEL_LDT))
-    {
-        /* GDT */
-        if ((unsigned)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
-            return VERR_INVALID_SELECTOR;
-        GCPtrDesc = Gdtr.pGdt + (Sel & X86_SEL_MASK);
-    }
-    else
-    {
-        /*
-         * LDT - must locate the LDT first.
-         */
-        RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
-        if (    (unsigned)(SelLdt & X86_SEL_MASK) < sizeof(X86DESC) /* the first selector is invalid, right? */ /** @todo r=bird: No, I don't think so */
-            ||  (unsigned)(SelLdt & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
-            return VERR_INVALID_SELECTOR;
-        GCPtrDesc = Gdtr.pGdt + (SelLdt & X86_SEL_MASK);
-        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /* validate the LDT descriptor. */
-        if (Desc.Gen.u1Present == 0)
-            return VERR_SELECTOR_NOT_PRESENT;
-        if (    Desc.Gen.u1DescType == 1
-            ||  Desc.Gen.u4Type != AMD64_SEL_TYPE_SYS_LDT)
-            return VERR_INVALID_SELECTOR;
-
-        uint32_t cbLimit = X86DESC_LIMIT(Desc);
-        if (Desc.Gen.u1Granularity)
-            cbLimit = (cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
-        if ((uint32_t)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > cbLimit)
-            return VERR_INVALID_SELECTOR;
-
-        /* calc the descriptor location. */
-        GCPtrDesc = X86DESC64_BASE(Desc);
-        GCPtrDesc += (Sel & X86_SEL_MASK);
-    }
-
-    /* read the descriptor. */
-    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
-    if (RT_FAILURE(rc))
-    {
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(X86DESC));
-        if (RT_FAILURE(rc))
-            return rc;
-        Desc.au64[1] = 0;
-    }
-
-    /*
-     * Extract the base and limit
-     * (We ignore the present bit here, which is probably a bit silly...)
-     */
-    pSelInfo->Sel     = Sel;
-    pSelInfo->fFlags  = DBGFSELINFO_FLAGS_LONG_MODE;
-    pSelInfo->u.Raw64 = Desc;
-    if (Desc.Gen.u1DescType)
-    {
-        /*
-         * 64-bit code selectors are wide open, it's not possible to detect
-         * 64-bit data or stack selectors without also dragging in assumptions
-         * about current CS (i.e. that's we're executing in 64-bit mode).  So,
-         * the selinfo user needs to deal with this in the context the info is
-         * used unfortunately.
-         */
-        if (    Desc.Gen.u1Long
-            &&  !Desc.Gen.u1DefBig
-            &&  (Desc.Gen.u4Type & X86_SEL_TYPE_CODE))
-        {
-            /* Note! We ignore the segment limit hacks that was added by AMD. */
-            pSelInfo->GCPtrBase = 0;
-            pSelInfo->cbLimit   = ~(RTGCUINTPTR)0;
-        }
-        else
-        {
-            pSelInfo->cbLimit = X86DESC_LIMIT(Desc);
-            if (Desc.Gen.u1Granularity)
-                pSelInfo->cbLimit = (pSelInfo->cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
-            pSelInfo->GCPtrBase = X86DESC_BASE(Desc);
-        }
-        pSelInfo->SelGate = 0;
-    }
-    else if (   Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_LDT
-             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TSS_AVAIL
-             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY)
-    {
-        /* Note. LDT descriptors are weird in long mode, we ignore the footnote
-           in the AMD manual here as a simplification. */
-        pSelInfo->GCPtrBase = X86DESC64_BASE(Desc);
-        pSelInfo->cbLimit = X86DESC_LIMIT(Desc);
-        if (Desc.Gen.u1Granularity)
-            pSelInfo->cbLimit = (pSelInfo->cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
-        pSelInfo->SelGate = 0;
-    }
-    else if (   Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE
-             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TRAP_GATE
-             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_INT_GATE)
-    {
-        pSelInfo->cbLimit   = X86DESC64_BASE(Desc);
-        pSelInfo->GCPtrBase = Desc.Gate.u16OffsetLow
-                            | ((uint32_t)Desc.Gate.u16OffsetHigh << 16)
-                            | ((uint64_t)Desc.Gate.u32OffsetTop << 32);
-        pSelInfo->SelGate   = Desc.Gate.u16Sel;
-        pSelInfo->fFlags   |= DBGFSELINFO_FLAGS_GATE;
-    }
-    else
-    {
-        pSelInfo->cbLimit   = 0;
-        pSelInfo->GCPtrBase = 0;
-        pSelInfo->SelGate   = 0;
-        pSelInfo->fFlags   |= DBGFSELINFO_FLAGS_INVALID;
-    }
-    if (!Desc.Gen.u1Present)
-        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_NOT_PRESENT;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for selmR3GetSelectorInfo32 and SELMR3GetShadowSelectorInfo that
- * interprets a legacy descriptor table entry and fills in the selector info
- * structure from it.
- *
- * @param  pSelInfo     Where to store the selector info. Only the fFlags and
- *                      Sel members have been initialized.
- * @param  pDesc        The legacy descriptor to parse.
- */
-DECLINLINE(void) selmR3SelInfoFromDesc32(PDBGFSELINFO pSelInfo, PCX86DESC pDesc)
-{
-    pSelInfo->u.Raw64.au64[1] = 0;
-    pSelInfo->u.Raw = *pDesc;
-    if (    pDesc->Gen.u1DescType
-        ||  !(pDesc->Gen.u4Type & 4))
-    {
-        pSelInfo->cbLimit = X86DESC_LIMIT(*pDesc);
-        if (pDesc->Gen.u1Granularity)
-            pSelInfo->cbLimit = (pSelInfo->cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
-        pSelInfo->GCPtrBase = X86DESC_BASE(*pDesc);
-        pSelInfo->SelGate = 0;
-    }
-    else if (pDesc->Gen.u4Type != X86_SEL_TYPE_SYS_UNDEFINED4)
-    {
-        pSelInfo->cbLimit = 0;
-        if (pDesc->Gen.u4Type == X86_SEL_TYPE_SYS_TASK_GATE)
-            pSelInfo->GCPtrBase = 0;
-        else
-            pSelInfo->GCPtrBase = pDesc->Gate.u16OffsetLow
-                                | (uint32_t)pDesc->Gate.u16OffsetHigh << 16;
-        pSelInfo->SelGate = pDesc->Gate.u16Sel;
-        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_GATE;
-    }
-    else
-    {
-        pSelInfo->cbLimit = 0;
-        pSelInfo->GCPtrBase = 0;
-        pSelInfo->SelGate = 0;
-        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_INVALID;
-    }
-    if (!pDesc->Gen.u1Present)
-        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_NOT_PRESENT;
-}
-
-
-/**
- * Gets information about a 64-bit selector, SELMR3GetSelectorInfo helper.
- *
- * See SELMR3GetSelectorInfo for details.
- *
- * @returns VBox status code, see SELMR3GetSelectorInfo for details.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   Sel         The selector to get info about.
- * @param   pSelInfo    Where to store the information.
- */
-static int selmR3GetSelectorInfo32(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
-{
-    /*
-     * Read the descriptor entry
-     */
-    pSelInfo->fFlags = 0;
-    X86DESC Desc;
-    if (    !(Sel & X86_SEL_LDT)
-        && (    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] == (Sel & X86_SEL_MASK)
-            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] == (Sel & X86_SEL_MASK)
-            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] == (Sel & X86_SEL_MASK)
-            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] == (Sel & X86_SEL_MASK)
-            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] == (Sel & X86_SEL_MASK))
-       )
-    {
-        /*
-         * Hypervisor descriptor.
-         */
-        pSelInfo->fFlags = DBGFSELINFO_FLAGS_HYPER;
-        if (CPUMIsGuestInProtectedMode(pVCpu))
-            pSelInfo->fFlags |= DBGFSELINFO_FLAGS_PROT_MODE;
-        else
-            pSelInfo->fFlags |= DBGFSELINFO_FLAGS_REAL_MODE;
-
-        Desc = pVM->selm.s.paGdtR3[Sel >> X86_SEL_SHIFT];
-    }
-    else if (CPUMIsGuestInProtectedMode(pVCpu))
-    {
-        /*
-         * Read it from the guest descriptor table.
-         */
-        pSelInfo->fFlags = DBGFSELINFO_FLAGS_PROT_MODE;
-
-        VBOXGDTR    Gdtr;
-        RTGCPTR     GCPtrDesc;
-        CPUMGetGuestGDTR(pVCpu, &Gdtr);
-        if (!(Sel & X86_SEL_LDT))
-        {
-            /* GDT */
-            if ((unsigned)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
-                return VERR_INVALID_SELECTOR;
-            GCPtrDesc = Gdtr.pGdt + (Sel & X86_SEL_MASK);
-        }
-        else
-        {
-            /*
-             * LDT - must locate the LDT first...
-             */
-            RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
-            if (    (unsigned)(SelLdt & X86_SEL_MASK) < sizeof(X86DESC) /* the first selector is invalid, right? */ /** @todo r=bird: No, I don't think so */
-                ||  (unsigned)(SelLdt & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
-                return VERR_INVALID_SELECTOR;
-            GCPtrDesc = Gdtr.pGdt + (SelLdt & X86_SEL_MASK);
-            int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
-            if (RT_FAILURE(rc))
-                return rc;
-
-            /* validate the LDT descriptor. */
-            if (Desc.Gen.u1Present == 0)
-                return VERR_SELECTOR_NOT_PRESENT;
-            if (    Desc.Gen.u1DescType == 1
-                ||  Desc.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
-                return VERR_INVALID_SELECTOR;
-
-            unsigned cbLimit = X86DESC_LIMIT(Desc);
-            if (Desc.Gen.u1Granularity)
-                cbLimit = (cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
-            if ((unsigned)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > cbLimit)
-                return VERR_INVALID_SELECTOR;
-
-            /* calc the descriptor location. */
-            GCPtrDesc = X86DESC_BASE(Desc);
-            GCPtrDesc += (Sel & X86_SEL_MASK);
-        }
-
-        /* read the descriptor. */
-        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-    else
-    {
-        /*
-         * We're in real mode.
-         */
-        pSelInfo->Sel       = Sel;
-        pSelInfo->GCPtrBase = Sel << 4;
-        pSelInfo->cbLimit   = 0xffff;
-        pSelInfo->fFlags    = DBGFSELINFO_FLAGS_REAL_MODE;
-        pSelInfo->u.Raw64.au64[0] = 0;
-        pSelInfo->u.Raw64.au64[1] = 0;
-        pSelInfo->SelGate   = 0;
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Extract the base and limit or sel:offset for gates.
-     */
-    pSelInfo->Sel = Sel;
-    selmR3SelInfoFromDesc32(pSelInfo, &Desc);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets information about a selector.
- *
- * Intended for the debugger mostly and will prefer the guest descriptor tables
- * over the shadow ones.
- *
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
- *          descriptor table.
- * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
- *          is not returned if the selector itself isn't present, you have to
- *          check that for yourself (see DBGFSELINFO::fFlags).
- * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
- *          pagetable or page backing the selector table wasn't present.
- * @returns Other VBox status code on other errors.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       The virtual CPU handle.
- * @param   Sel         The selector to get info about.
- * @param   pSelInfo    Where to store the information.
- */
-VMMR3DECL(int) SELMR3GetSelectorInfo(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
-{
-    AssertPtr(pSelInfo);
-    if (CPUMIsGuestInLongMode(pVCpu))
-        return selmR3GetSelectorInfo64(pVM, pVCpu, Sel, pSelInfo);
-    return selmR3GetSelectorInfo32(pVM, pVCpu, Sel, pSelInfo);
-}
-
-
-/**
- * Gets information about a selector from the shadow tables.
- *
- * This is intended to be faster than the SELMR3GetSelectorInfo() method, but
- * requires that the caller ensures that the shadow tables are up to date.
- *
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
- *          descriptor table.
- * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
- *          is not returned if the selector itself isn't present, you have to
- *          check that for yourself (see DBGFSELINFO::fFlags).
- * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
- *          pagetable or page backing the selector table wasn't present.
- * @returns Other VBox status code on other errors.
- *
- * @param   pVM         VM handle.
- * @param   Sel         The selector to get info about.
- * @param   pSelInfo    Where to store the information.
- *
- * @remarks Don't use this when in hardware assisted virtualization mode.
- */
-VMMR3DECL(int) SELMR3GetShadowSelectorInfo(PVM pVM, RTSEL Sel, PDBGFSELINFO pSelInfo)
-{
-    Assert(pSelInfo);
-
-    /*
-     * Read the descriptor entry
-     */
-    X86DESC Desc;
-    if (!(Sel & X86_SEL_LDT))
-    {
-        /*
-         * Global descriptor.
-         */
-        Desc = pVM->selm.s.paGdtR3[Sel >> X86_SEL_SHIFT];
-        pSelInfo->fFlags =    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] == (Sel & X86_SEL_MASK)
-                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] == (Sel & X86_SEL_MASK)
-                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] == (Sel & X86_SEL_MASK)
-                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] == (Sel & X86_SEL_MASK)
-                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] == (Sel & X86_SEL_MASK)
-                         ? DBGFSELINFO_FLAGS_HYPER
-                         : 0;
-        /** @todo check that the GDT offset is valid. */
-    }
-    else
-    {
-        /*
-         * Local Descriptor.
-         */
-        PX86DESC paLDT = (PX86DESC)((char *)pVM->selm.s.pvLdtR3 + pVM->selm.s.offLdtHyper);
-        Desc = paLDT[Sel >> X86_SEL_SHIFT];
-        /** @todo check if the LDT page is actually available. */
-        /** @todo check that the LDT offset is valid. */
-        pSelInfo->fFlags = 0;
-    }
-    if (CPUMIsGuestInProtectedMode(VMMGetCpu0(pVM)))
-        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_PROT_MODE;
-    else
-        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_REAL_MODE;
-
-    /*
-     * Extract the base and limit or sel:offset for gates.
-     */
-    pSelInfo->Sel = Sel;
-    selmR3SelInfoFromDesc32(pSelInfo, &Desc);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Formats a descriptor.
- *
- * @param   Desc        Descriptor to format.
- * @param   Sel         Selector number.
- * @param   pszOutput   Output buffer.
- * @param   cchOutput   Size of output buffer.
- */
-static void selmR3FormatDescriptor(X86DESC Desc, RTSEL Sel, char *pszOutput, size_t cchOutput)
-{
-    /*
-     * Make variable description string.
-     */
-    static struct
-    {
-        unsigned    cch;
-        const char *psz;
-    } const aTypes[32] =
-    {
-#define STRENTRY(str) { sizeof(str) - 1, str }
-        /* system */
-        STRENTRY("Reserved0 "),                  /* 0x00 */
-        STRENTRY("TSS16Avail "),                 /* 0x01 */
-        STRENTRY("LDT "),                        /* 0x02 */
-        STRENTRY("TSS16Busy "),                  /* 0x03 */
-        STRENTRY("Call16 "),                     /* 0x04 */
-        STRENTRY("Task "),                       /* 0x05 */
-        STRENTRY("Int16 "),                      /* 0x06 */
-        STRENTRY("Trap16 "),                     /* 0x07 */
-        STRENTRY("Reserved8 "),                  /* 0x08 */
-        STRENTRY("TSS32Avail "),                 /* 0x09 */
-        STRENTRY("ReservedA "),                  /* 0x0a */
-        STRENTRY("TSS32Busy "),                  /* 0x0b */
-        STRENTRY("Call32 "),                     /* 0x0c */
-        STRENTRY("ReservedD "),                  /* 0x0d */
-        STRENTRY("Int32 "),                      /* 0x0e */
-        STRENTRY("Trap32 "),                     /* 0x0f */
-        /* non system */
-        STRENTRY("DataRO "),                     /* 0x10 */
-        STRENTRY("DataRO Accessed "),            /* 0x11 */
-        STRENTRY("DataRW "),                     /* 0x12 */
-        STRENTRY("DataRW Accessed "),            /* 0x13 */
-        STRENTRY("DataDownRO "),                 /* 0x14 */
-        STRENTRY("DataDownRO Accessed "),        /* 0x15 */
-        STRENTRY("DataDownRW "),                 /* 0x16 */
-        STRENTRY("DataDownRW Accessed "),        /* 0x17 */
-        STRENTRY("CodeEO "),                     /* 0x18 */
-        STRENTRY("CodeEO Accessed "),            /* 0x19 */
-        STRENTRY("CodeER "),                     /* 0x1a */
-        STRENTRY("CodeER Accessed "),            /* 0x1b */
-        STRENTRY("CodeConfEO "),                 /* 0x1c */
-        STRENTRY("CodeConfEO Accessed "),        /* 0x1d */
-        STRENTRY("CodeConfER "),                 /* 0x1e */
-        STRENTRY("CodeConfER Accessed ")         /* 0x1f */
-#undef SYSENTRY
-    };
-#define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
-    char        szMsg[128];
-    char       *psz = &szMsg[0];
-    unsigned    i = Desc.Gen.u1DescType << 4 | Desc.Gen.u4Type;
-    memcpy(psz, aTypes[i].psz, aTypes[i].cch);
-    psz += aTypes[i].cch;
-
-    if (Desc.Gen.u1Present)
-        ADD_STR(psz, "Present ");
-    else
-        ADD_STR(psz, "Not-Present ");
-    if (Desc.Gen.u1Granularity)
-        ADD_STR(psz, "Page ");
-    if (Desc.Gen.u1DefBig)
-        ADD_STR(psz, "32-bit ");
-    else
-        ADD_STR(psz, "16-bit ");
-#undef ADD_STR
-    *psz = '\0';
-
-    /*
-     * Limit and Base and format the output.
-     */
-    uint32_t    u32Limit = X86DESC_LIMIT(Desc);
-    if (Desc.Gen.u1Granularity)
-        u32Limit = u32Limit << PAGE_SHIFT | PAGE_OFFSET_MASK;
-    uint32_t    u32Base =  X86DESC_BASE(Desc);
-
-    RTStrPrintf(pszOutput, cchOutput, "%04x - %08x %08x - base=%08x limit=%08x dpl=%d %s",
-                Sel, Desc.au32[0], Desc.au32[1], u32Base, u32Limit, Desc.Gen.u2Dpl, szMsg);
-}
-
-
-/**
- * Dumps a descriptor.
- *
- * @param   Desc    Descriptor to dump.
- * @param   Sel     Selector number.
- * @param   pszMsg  Message to prepend the log entry with.
- */
-VMMR3DECL(void) SELMR3DumpDescriptor(X86DESC  Desc, RTSEL Sel, const char *pszMsg)
-{
-    char szOutput[128];
-    selmR3FormatDescriptor(Desc, Sel, &szOutput[0], sizeof(szOutput));
-    Log(("%s: %s\n", pszMsg, szOutput));
-    NOREF(szOutput[0]);
-}
-
-
-/**
- * Display the shadow gdt.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) selmR3InfoGdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    pHlp->pfnPrintf(pHlp, "Shadow GDT (GCAddr=%RRv):\n", MMHyperR3ToRC(pVM, pVM->selm.s.paGdtR3));
-    for (unsigned iGDT = 0; iGDT < SELM_GDT_ELEMENTS; iGDT++)
-    {
-        if (pVM->selm.s.paGdtR3[iGDT].Gen.u1Present)
-        {
-            char szOutput[128];
-            selmR3FormatDescriptor(pVM->selm.s.paGdtR3[iGDT], iGDT << X86_SEL_SHIFT, &szOutput[0], sizeof(szOutput));
-            const char *psz = "";
-            if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] >> X86_SEL_SHIFT))
-                psz = " HyperCS";
-            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] >> X86_SEL_SHIFT))
-                psz = " HyperDS";
-            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] >> X86_SEL_SHIFT))
-                psz = " HyperCS64";
-            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] >> X86_SEL_SHIFT))
-                psz = " HyperTSS";
-            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> X86_SEL_SHIFT))
-                psz = " HyperTSSTrap08";
-            pHlp->pfnPrintf(pHlp, "%s%s\n", szOutput, psz);
-        }
-    }
-}
-
-
-/**
- * Display the guest gdt.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) selmR3InfoGdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    /** @todo SMP support! */
-    PVMCPU      pVCpu = &pVM->aCpus[0];
-
-    VBOXGDTR    GDTR;
-    CPUMGetGuestGDTR(pVCpu, &GDTR);
-    RTGCPTR     GCPtrGDT = GDTR.pGdt;
-    unsigned    cGDTs = ((unsigned)GDTR.cbGdt + 1) / sizeof(X86DESC);
-
-    pHlp->pfnPrintf(pHlp, "Guest GDT (GCAddr=%RGv limit=%x):\n", GCPtrGDT, GDTR.cbGdt);
-    for (unsigned iGDT = 0; iGDT < cGDTs; iGDT++, GCPtrGDT += sizeof(X86DESC))
-    {
-        X86DESC GDTE;
-        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &GDTE, GCPtrGDT, sizeof(GDTE));
-        if (RT_SUCCESS(rc))
-        {
-            if (GDTE.Gen.u1Present)
-            {
-                char szOutput[128];
-                selmR3FormatDescriptor(GDTE, iGDT << X86_SEL_SHIFT, &szOutput[0], sizeof(szOutput));
-                pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
-            }
-        }
-        else if (rc == VERR_PAGE_NOT_PRESENT)
-        {
-            if ((GCPtrGDT & PAGE_OFFSET_MASK) + sizeof(X86DESC) - 1 < sizeof(X86DESC))
-                pHlp->pfnPrintf(pHlp, "%04x - page not present (GCAddr=%RGv)\n", iGDT << X86_SEL_SHIFT, GCPtrGDT);
-        }
-        else
-            pHlp->pfnPrintf(pHlp, "%04x - read error rc=%Rrc GCAddr=%RGv\n", iGDT << X86_SEL_SHIFT, rc, GCPtrGDT);
-    }
-}
-
-
-/**
- * Display the shadow ldt.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) selmR3InfoLdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    unsigned    cLDTs = ((unsigned)pVM->selm.s.cbLdtLimit + 1) >> X86_SEL_SHIFT;
-    PX86DESC    paLDT = (PX86DESC)((char *)pVM->selm.s.pvLdtR3 + pVM->selm.s.offLdtHyper);
-    pHlp->pfnPrintf(pHlp, "Shadow LDT (GCAddr=%RRv limit=%#x):\n", pVM->selm.s.pvLdtRC + pVM->selm.s.offLdtHyper, pVM->selm.s.cbLdtLimit);
-    for (unsigned iLDT = 0; iLDT < cLDTs; iLDT++)
-    {
-        if (paLDT[iLDT].Gen.u1Present)
-        {
-            char szOutput[128];
-            selmR3FormatDescriptor(paLDT[iLDT], (iLDT << X86_SEL_SHIFT) | X86_SEL_LDT, &szOutput[0], sizeof(szOutput));
-            pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
-        }
-    }
-}
-
-
-/**
- * Display the guest ldt.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) selmR3InfoLdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    /** @todo SMP support! */
-    PVMCPU      pVCpu = &pVM->aCpus[0];
-
-    RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
-    if (!(SelLdt & X86_SEL_MASK))
-    {
-        pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x): Null-Selector\n", SelLdt);
-        return;
-    }
-
-    RTGCPTR     GCPtrLdt;
-    unsigned    cbLdt;
-    int rc = SELMGetLDTFromSel(pVM, SelLdt, &GCPtrLdt, &cbLdt);
-    if (RT_FAILURE(rc))
-    {
-        pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x): rc=%Rrc\n", SelLdt, rc);
-        return;
-    }
-
-    pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x GCAddr=%RGv limit=%x):\n", SelLdt, GCPtrLdt, cbLdt);
-    unsigned    cLdts  = (cbLdt + 1) >> X86_SEL_SHIFT;
-    for (unsigned iLdt = 0; iLdt < cLdts; iLdt++, GCPtrLdt += sizeof(X86DESC))
-    {
-        X86DESC LdtE;
-        rc = PGMPhysSimpleReadGCPtr(pVCpu, &LdtE, GCPtrLdt, sizeof(LdtE));
-        if (RT_SUCCESS(rc))
-        {
-            if (LdtE.Gen.u1Present)
-            {
-                char szOutput[128];
-                selmR3FormatDescriptor(LdtE, (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, &szOutput[0], sizeof(szOutput));
-                pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
-            }
-        }
-        else if (rc == VERR_PAGE_NOT_PRESENT)
-        {
-            if ((GCPtrLdt & PAGE_OFFSET_MASK) + sizeof(X86DESC) - 1 < sizeof(X86DESC))
-                pHlp->pfnPrintf(pHlp, "%04x - page not present (GCAddr=%RGv)\n", (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, GCPtrLdt);
-        }
-        else
-            pHlp->pfnPrintf(pHlp, "%04x - read error rc=%Rrc GCAddr=%RGv\n", (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, rc, GCPtrLdt);
-    }
-}
-
-
-/**
- * Dumps the hypervisor GDT
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) SELMR3DumpHyperGDT(PVM pVM)
-{
-    DBGFR3Info(pVM, "gdt", NULL, NULL);
-}
-
-
-/**
- * Dumps the hypervisor LDT
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) SELMR3DumpHyperLDT(PVM pVM)
-{
-    DBGFR3Info(pVM, "ldt", NULL, NULL);
-}
-
-
-/**
- * Dumps the guest GDT
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) SELMR3DumpGuestGDT(PVM pVM)
-{
-    DBGFR3Info(pVM, "gdtguest", NULL, NULL);
-}
-
-
-/**
- * Dumps the guest LDT
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) SELMR3DumpGuestLDT(PVM pVM)
-{
-    DBGFR3Info(pVM, "ldtguest", NULL, NULL);
-}
-
Index: unk/src/VBox/VMM/SELMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/SELMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,193 +1,0 @@
-/* $Id$ */
-/** @file
- * SELM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___SELMInternal_h
-#define ___SELMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/stam.h>
-#include <VBox/cpum.h>
-
-
-
-/** @defgroup grp_selm_int   Internals
- * @ingroup grp_selm
- * @internal
- * @{
- */
-
-/** The number of GDTS allocated for our GDT. (full size) */
-#define SELM_GDT_ELEMENTS                   8192
-
-/** aHyperSel index to retrieve hypervisor selectors */
-/** The Flat CS selector used by the VMM inside the GC. */
-#define SELM_HYPER_SEL_CS                   0
-/** The Flat DS selector used by the VMM inside the GC. */
-#define SELM_HYPER_SEL_DS                   1
-/** The 64-bit mode CS selector used by the VMM inside the GC. */
-#define SELM_HYPER_SEL_CS64                 2
-/** The TSS selector used by the VMM inside the GC. */
-#define SELM_HYPER_SEL_TSS                  3
-/** The TSS selector for taking trap 08 (\#DF). */
-#define SELM_HYPER_SEL_TSS_TRAP08           4
-/** Number of GDTs we need for internal use */
-#define SELM_HYPER_SEL_MAX                  (SELM_HYPER_SEL_TSS_TRAP08 + 1)
-
-
-/** Default GDT selectors we use for the hypervisor. */
-#define SELM_HYPER_DEFAULT_SEL_CS           ((SELM_GDT_ELEMENTS - 0x1) << 3)
-#define SELM_HYPER_DEFAULT_SEL_DS           ((SELM_GDT_ELEMENTS - 0x2) << 3)
-#define SELM_HYPER_DEFAULT_SEL_CS64         ((SELM_GDT_ELEMENTS - 0x3) << 3)
-#define SELM_HYPER_DEFAULT_SEL_TSS          ((SELM_GDT_ELEMENTS - 0x4) << 3)
-#define SELM_HYPER_DEFAULT_SEL_TSS_TRAP08   ((SELM_GDT_ELEMENTS - 0x5) << 3)
-/** The lowest value default we use. */
-#define SELM_HYPER_DEFAULT_BASE             SELM_HYPER_DEFAULT_SEL_TSS_TRAP08
-
-/**
- * Converts a SELM pointer into a VM pointer.
- * @returns Pointer to the VM structure the SELM is part of.
- * @param   pSELM   Pointer to SELM instance data.
- */
-#define SELM2VM(pSELM)  ( (PVM)((char *)pSELM - pSELM->offVM) )
-
-
-
-/**
- * SELM Data (part of VM)
- */
-typedef struct SELM
-{
-    /** Offset to the VM structure.
-     * See SELM2VM(). */
-    RTINT                   offVM;
-
-    /** Flat CS, DS, 64 bit mode CS, TSS & trap 8 TSS. */
-    RTSEL                   aHyperSel[SELM_HYPER_SEL_MAX];
-
-    /** Pointer to the GCs - R3 Ptr.
-     * This size is governed by SELM_GDT_ELEMENTS. */
-    R3PTRTYPE(PX86DESC)     paGdtR3;
-    /** Pointer to the GCs - RC Ptr.
-     * This is not initialized until the first relocation because it's used to
-     * check if the shadow GDT virtual handler requires deregistration. */
-    RCPTRTYPE(PX86DESC)     paGdtRC;
-    /** Current (last) Guest's GDTR.
-     * The pGdt member is set to RTRCPTR_MAX if we're not monitoring the guest GDT. */
-    VBOXGDTR                GuestGdtr;
-    /** The current (last) effective Guest GDT size. */
-    RTUINT                  cbEffGuestGdtLimit;
-
-    uint32_t                padding0;
-
-    /** R3 pointer to the LDT shadow area in HMA. */
-    R3PTRTYPE(void *)       pvLdtR3;
-    /** RC pointer to the LDT shadow area in HMA. */
-    RCPTRTYPE(void *)       pvLdtRC;
-#if GC_ARCH_BITS == 64
-    RTRCPTR                 padding1;
-#endif
-    /** The address of the guest LDT.
-     * RTRCPTR_MAX if not monitored. */
-    RTGCPTR                 GCPtrGuestLdt;
-    /** Current LDT limit, both Guest and Shadow. */
-    RTUINT                  cbLdtLimit;
-    /** Current LDT offset relative to pvLdtR3/pvLdtRC. */
-    RTUINT                  offLdtHyper;
-#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
-    uint32_t                padding2[2];
-#endif
-    /** TSS. (This is 16 byte aligned!)
-      * @todo I/O bitmap & interrupt redirection table? */
-    VBOXTSS                 Tss;
-
-    /** TSS for trap 08 (\#DF). */
-    VBOXTSS                 TssTrap08;
-
-    /** Monitored shadow TSS address. */
-    RCPTRTYPE(void *)       pvMonShwTssRC;
-#if GC_ARCH_BITS == 64
-    RTRCPTR                 padding3;
-#endif
-    /** GC Pointer to the current Guest's TSS.
-     * RTRCPTR_MAX if not monitored. */
-    RTGCPTR                 GCPtrGuestTss;
-    /** The size of the guest TSS. */
-    RTUINT                  cbGuestTss;
-    /** Set if it's a 32-bit TSS. */
-    bool                    fGuestTss32Bit;
-    /** The size of the Guest's TSS part we're monitoring. */
-    RTUINT                  cbMonitoredGuestTss;
-    /** The guest TSS selector at last sync (part of monitoring).
-     * Contains RTSEL_MAX if not set. */
-    RTSEL                   GCSelTss;
-    /** The last known offset of the I/O bitmap.
-     * This is only used if we monitor the bitmap. */
-    uint16_t                offGuestIoBitmap;
-
-    /** Indicates that the Guest GDT access handler have been registered. */
-    bool                    fGDTRangeRegistered;
-
-    /** Indicates whether LDT/GDT/TSS monitoring and syncing is disabled. */
-    bool                    fDisableMonitoring;
-
-    /** Indicates whether the TSS stack selector & base address need to be refreshed.  */
-    bool                    fSyncTSSRing0Stack;
-    bool                    fPadding2[1+2];
-
-    /** SELMR3UpdateFromCPUM() profiling. */
-    STAMPROFILE             StatUpdateFromCPUM;
-    /** SELMR3SyncTSS() profiling. */
-    STAMPROFILE             StatTSSSync;
-
-    /** GC: The number of handled writes to the Guest's GDT. */
-    STAMCOUNTER             StatRCWriteGuestGDTHandled;
-    /** GC: The number of unhandled write to the Guest's GDT. */
-    STAMCOUNTER             StatRCWriteGuestGDTUnhandled;
-    /** GC: The number of times writes to Guest's LDT was detected. */
-    STAMCOUNTER             StatRCWriteGuestLDT;
-    /** GC: The number of handled writes to the Guest's TSS. */
-    STAMCOUNTER             StatRCWriteGuestTSSHandled;
-    /** GC: The number of handled writes to the Guest's TSS where we detected a change. */
-    STAMCOUNTER             StatRCWriteGuestTSSHandledChanged;
-    /** GC: The number of handled redir writes to the Guest's TSS where we detected a change. */
-    STAMCOUNTER             StatRCWriteGuestTSSRedir;
-    /** GC: The number of unhandled writes to the Guest's TSS. */
-    STAMCOUNTER             StatRCWriteGuestTSSUnhandled;
-    /** The number of times we had to relocate our hypervisor selectors. */
-    STAMCOUNTER             StatHyperSelsChanged;
-    /** The number of times we had find free hypervisor selectors. */
-    STAMCOUNTER             StatScanForHyperSels;
-} SELM, *PSELM;
-
-RT_C_DECLS_BEGIN
-
-VMMRCDECL(int) selmRCGuestGDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-VMMRCDECL(int) selmRCGuestLDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-VMMRCDECL(int) selmRCGuestTSSWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-
-VMMRCDECL(int) selmRCShadowGDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-VMMRCDECL(int) selmRCShadowLDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-VMMRCDECL(int) selmRCShadowTSSWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-
-void           selmSetRing1Stack(PVM pVM, uint32_t ss, RTGCPTR32 esp);
-
-RT_C_DECLS_END
-
-/** @} */
-
-#endif
Index: unk/src/VBox/VMM/SSM.cpp
===================================================================
--- /trunk/src/VBox/VMM/SSM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,9101 +1,0 @@
-/* $Id$ */
-/** @file
- * SSM - Saved State Manager.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page pg_ssm        SSM - The Saved State Manager
- *
- * The Saved State Manager (SSM) implements facilities for saving and loading a
- * VM state in a structural manner using callbacks for named data units.
- *
- * At init time each of the VMM components, Devices, Drivers and one or two
- * other things will register data units which they need to save and restore.
- * Each unit have a unique name (ascii), instance number, and a set of callbacks
- * associated with it.  The name will be used to identify the unit during
- * restore.  The callbacks are for the two operations, save and restore.  There
- * are three callbacks for each of the two - a prepare, a execute and a complete
- * - giving each component ample opportunity to perform actions both before and
- * afterwards.
- *
- * The SSM provides a number of APIs for encoding and decoding the data: @see
- * grp_ssm
- *
- *
- *
- * @section sec_ssm_live_snapshots  Live Snapshots
- *
- * The live snapshots feature (LS) is similar to teleportation (TP) and was a
- * natural first step when implementing TP.  The main differences between LS and
- * TP are that after a live snapshot we will have a saved state file, disk image
- * snapshots, and the VM will still be running.
- *
- * Compared to normal saved stated and snapshots, the difference is in that the
- * VM is running while we do most of the saving.  Prior to LS, there was only
- * one round of callbacks during saving and the VM was paused during it.  With
- * LS there are 1 or more passes while the VM is still running and a final one
- * after it has been paused.  The runtime passes are executed on a dedicated
- * thread running at at the same priority as the EMTs so that the saving doesn't
- * starve or lose in scheduling questions (note: not implemented yet). The final
- * pass is done on EMT(0).
- *
- * There are a couple of common reasons why LS and TP will fail:
- *   - Memory configuration changed (PCI memory mappings).
- *   - Takes too long (TP) / Too much output (LS).
- *
- *
- * The live saving sequence is something like this:
- *
- *      -# SSMR3LiveSave is called on EMT0.  It returns a saved state
- *         handle.
- *      -# SSMR3LiveDoStep1 is called on a non-EMT.  This will save the major
- *         parts of the state while the VM may still be running.
- *      -# The VM is suspended.
- *      -# SSMR3LiveDoStep2 is called on EMT0 to save the remainder of the state
- *         in the normal way.
- *      -# The client does any necessary reconfiguration of harddisks and
- *         similar.
- *      -# SSMR3LiveDone is called on EMT0 to close the handle.
- *      -# The VM is resumed or powered off and destroyed.
- *
- *
- * @section sec_ssm_teleportation   Teleportation
- *
- * As mentioned in the previous section, the main differences between this and
- * live snapshots are in where the saved state is written and what state the
- * local VM is in afterwards - at least from the VMM point of view.  The
- * necessary administrative work - establishing the connection to the remote
- * machine, cloning the VM config on it and doing lowlevel saved state data
- * transfer - is taken care of by layer above the VMM (i.e.  Main).
- *
- * The SSM data format was made streamable for the purpose of teleportation
- * (v1.2 was the last non-streamable version).
- *
- *
- * @section sec_ssm_format          Saved State Format
- *
- * The stream format starts with a header (SSMFILEHDR) that indicates the
- * version and such things, it is followed by zero or more saved state units
- * (name + instance + pass), and the stream concludes with a footer
- * (SSMFILEFTR) that contains unit counts and optionally a checksum for the
- * entire file.  (In version 1.2 and earlier, the checksum was in the header and
- * there was no footer.  This meant that the header was updated after the entire
- * file was written.)
- *
- * The saved state units each starts with a variable sized header
- * (SSMFILEUNITHDRV2) that contains the name, instance and pass.  The data
- * follows the header and is encoded as records with a 2-8 byte record header
- * indicating the type, flags and size.  The first byte in the record header
- * indicates the type and flags:
- *
- *   - bits 0..3: Record type:
- *       - type 0: Invalid.
- *       - type 1: Terminator with CRC-32 and unit size.
- *       - type 2: Raw data record.
- *       - type 3: Raw data compressed by LZF. The data is prefixed by a 8-bit
- *                 field containing the length of the uncompressed data given in
- *                 1KB units.
- *       - type 4: Zero data. The record header is followed by a 8-bit field
- *                 counting the length of the zero data given in 1KB units.
- *       - type 5: Named data - length prefixed name followed by the data. This
- *                 type is not implemented yet as we're missing the API part, so
- *                 the type assignment is tentative.
- *       - types 6 thru 15 are current undefined.
- *   - bit 4: Important (set), can be skipped (clear).
- *   - bit 5: Undefined flag, must be zero.
- *   - bit 6: Undefined flag, must be zero.
- *   - bit 7: "magic" bit, always set.
- *
- * Record header byte 2 (optionally thru 7) is the size of the following data
- * encoded in UTF-8 style.  To make buffering simpler and more efficient during
- * the save operation, the strict checks enforcing optimal encoding has been
- * relaxed for the 2 and 3 byte encodings.
- *
- * (In version 1.2 and earlier the unit data was compressed and not record
- * based. The unit header contained the compressed size of the data, i.e. it
- * needed updating after the data was written.)
- *
- *
- * @section sec_ssm_future          Future Changes
- *
- * There are plans to extend SSM to make it easier to be both backwards and
- * (somewhat) forwards compatible.  One of the new features will be being able
- * to classify units and data items as unimportant (added to the format in
- * v2.0).  Another suggested feature is naming data items (also added to the
- * format in v2.0), perhaps by extending the SSMR3PutStruct API.  Both features
- * will require API changes, the naming may possibly require both buffering of
- * the stream as well as some helper managing them.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_SSM
-#include <VBox/ssm.h>
-#include <VBox/dbgf.h>
-#include <VBox/mm.h>
-#include "SSMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <VBox/version.h>
-
-#include <iprt/asm.h>
-#include <iprt/assert.h>
-#include <iprt/crc.h>
-#include <iprt/file.h>
-#include <iprt/mem.h>
-#include <iprt/param.h>
-#include <iprt/thread.h>
-#include <iprt/semaphore.h>
-#include <iprt/string.h>
-#include <iprt/uuid.h>
-#include <iprt/zip.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The max length of a unit name. */
-#define SSM_MAX_NAME_SIZE                       48
-
-/** Saved state file magic base string. */
-#define SSMFILEHDR_MAGIC_BASE                   "\177VirtualBox SavedState "
-/** Saved state file magic indicating version 1.x. */
-#define SSMFILEHDR_MAGIC_V1_X                   "\177VirtualBox SavedState V1."
-/** Saved state file v1.1 magic. */
-#define SSMFILEHDR_MAGIC_V1_1                   "\177VirtualBox SavedState V1.1\n"
-/** Saved state file v1.2 magic. */
-#define SSMFILEHDR_MAGIC_V1_2                   "\177VirtualBox SavedState V1.2\n\0\0\0"
-/** Saved state file v2.0 magic. */
-#define SSMFILEHDR_MAGIC_V2_0                   "\177VirtualBox SavedState V2.0\n\0\0\0"
-
-/** @name SSMFILEHDR::fFlags
- * @{ */
-/** The stream is checksummed up to the footer using CRC-32. */
-#define SSMFILEHDR_FLAGS_STREAM_CRC32           RT_BIT_32(0)
-/** Indicates that the file was produced by a live save. */
-#define SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE       RT_BIT_32(1)
-/** @} */
-
-/** The directory magic. */
-#define SSMFILEDIR_MAGIC                        "\nDir\n\0\0"
-
-/** Saved state file v2.0 magic. */
-#define SSMFILEFTR_MAGIC                        "\nFooter"
-
-/** Data unit magic. */
-#define SSMFILEUNITHDR_MAGIC                    "\nUnit\n\0"
-/** Data end marker magic. */
-#define SSMFILEUNITHDR_END                      "\nTheEnd"
-
-
-/** @name Record Types (data unit)
- * @{ */
-/** The record type mask. */
-#define SSM_REC_TYPE_MASK                       UINT8_C(0x0f)
-/** Invalid record. */
-#define SSM_REC_TYPE_INVALID                    0
-/** Normal termination record, see SSMRECTERM. */
-#define SSM_REC_TYPE_TERM                       1
-/** Raw data. The data follows the size field without further ado. */
-#define SSM_REC_TYPE_RAW                        2
-/** Raw data compressed by LZF.
- * The record header is followed by a 8-bit field containing the size of the
- * uncompressed data in 1KB units.  The compressed data is after it. */
-#define SSM_REC_TYPE_RAW_LZF                    3
-/** Raw zero data.
- * The record header is followed by a 8-bit field containing the size of the
- * zero data in 1KB units. */
-#define SSM_REC_TYPE_RAW_ZERO                   4
-/** Named data items.
- * A length prefix zero terminated string (i.e. max 255) followed by the data.  */
-#define SSM_REC_TYPE_NAMED                      5
-/** Macro for validating the record type.
- * This can be used with the flags+type byte, no need to mask out the type first. */
-#define SSM_REC_TYPE_IS_VALID(u8Type)           (   ((u8Type) & SSM_REC_TYPE_MASK) >  SSM_REC_TYPE_INVALID \
-                                                 && ((u8Type) & SSM_REC_TYPE_MASK) <= SSM_REC_TYPE_NAMED )
-/** @} */
-
-/** The flag mask. */
-#define SSM_REC_FLAGS_MASK                      UINT8_C(0xf0)
-/** The record is important if this flag is set, if clear it can be omitted. */
-#define SSM_REC_FLAGS_IMPORTANT                 UINT8_C(0x10)
-/** This flag is always set. */
-#define SSM_REC_FLAGS_FIXED                     UINT8_C(0x80)
-/** Macro for validating the flags.
- * No need to mask the flags out of the flags+type byte before invoking this macro.  */
-#define SSM_REC_FLAGS_ARE_VALID(fFlags)         ( ((fFlags) & UINT8_C(0xe0)) == UINT8_C(0x80) )
-
-/** Macro for validating the type and flags byte in a data record. */
-#define SSM_REC_ARE_TYPE_AND_FLAGS_VALID(u8)    ( SSM_REC_FLAGS_ARE_VALID(u8) && SSM_REC_TYPE_IS_VALID(u8) )
-
-/** @name SSMRECTERM::fFlags
- * @{ */
-/** There is a CRC-32 value for the stream. */
-#define SSMRECTERM_FLAGS_CRC32                  UINT16_C(0x0001)
-/** @} */
-
-/** Start structure magic. (Isaac Asimov) */
-#define SSMR3STRUCT_BEGIN                       UINT32_C(0x19200102)
-/** End structure magic. (Isaac Asimov) */
-#define SSMR3STRUCT_END                         UINT32_C(0x19920406)
-
-
-/** Number of bytes to log in Log2 and Log4 statements. */
-#define SSM_LOG_BYTES                           16
-
-/** SSMHANDLE::fCancelled value indicating that the operation has been
- *  cancelled. */
-#define SSMHANDLE_CANCELLED                     UINT32_C(0xdeadbeef)
-/** SSMHANDLE::fCancelled value indicating no cancellation. */
-#define SSMHANDLE_OK                            UINT32_C(0x77777777)
-
-
-/** Macro for checking the u32CRC field of a structure.
- * The Msg can assume there are  u32ActualCRC and u32CRC in the context. */
-#define SSM_CHECK_CRC32_RET(p, cb, Msg) \
-    do \
-    { \
-        uint32_t u32CRC = (p)->u32CRC; \
-        (p)->u32CRC = 0; \
-        uint32_t u32ActualCRC = RTCrc32((p), (cb)); \
-        (p)->u32CRC = u32CRC; \
-        AssertLogRelMsgReturn(u32ActualCRC == u32CRC, Msg, VERR_SSM_INTEGRITY_CRC); \
-    } while (0)
-
-/** The number of bytes to compress is one block.
- * Must be a multiple of 1KB.  */
-#define SSM_ZIP_BLOCK_SIZE                      _4K
-AssertCompile(SSM_ZIP_BLOCK_SIZE / _1K * _1K == SSM_ZIP_BLOCK_SIZE);
-
-
-/**
- * Asserts that the handle is writable and returns with VERR_SSM_INVALID_STATE
- * if it isn't.
- */
-#define SSM_ASSERT_WRITEABLE_RET(pSSM) \
-    AssertMsgReturn(   pSSM->enmOp == SSMSTATE_SAVE_EXEC \
-                    || pSSM->enmOp == SSMSTATE_LIVE_EXEC,\
-                    ("Invalid state %d\n", pSSM->enmOp), VERR_SSM_INVALID_STATE);
-
-/**
- * Asserts that the handle is readable and returns with VERR_SSM_INVALID_STATE
- * if it isn't.
- */
-#define SSM_ASSERT_READABLE_RET(pSSM) \
-    AssertMsgReturn(   pSSM->enmOp == SSMSTATE_LOAD_EXEC \
-                    || pSSM->enmOp == SSMSTATE_OPEN_READ,\
-                    ("Invalid state %d\n", pSSM->enmOp), VERR_SSM_INVALID_STATE);
-
-/** Checks for cancellation and returns if pending.
- * Sets SSMHANDLE::rc to VERR_SSM_CANCELLED (if it still indicates success) and
- * then returns SSMHANDLE::rc. (Debug logging only.) */
-#define SSM_CHECK_CANCELLED_RET(pSSM) \
-    do \
-    { \
-        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED)) \
-        { \
-            LogFlow(("%Rfn: Cancelled -> VERR_SSM_CANCELLED\n", __PRETTY_FUNCTION__)); \
-            if (RT_SUCCESS((pSSM)->rc)) \
-                (pSSM)->rc = VERR_SSM_CANCELLED; \
-            return (pSSM)->rc; \
-        } \
-    } while (0)
-
-/**
- * Asserts that the handle is somewhat valid.  No returns as this is just a
- * simple safeguard for catching bad API calls. */
-#define SSM_ASSERT_VALID_HANDLE(pSSM) \
-    do \
-    { \
-        AssertPtr(pSSM); \
-        Assert(pSSM->enmOp > SSMSTATE_INVALID && pSSM->enmOp < SSMSTATE_END); \
-    } while (0)
-
-
-/** @def SSM_HOST_IS_MSC_32
- * Set to 1 if the host is 32-bit MSC, otherwise set to 0.
- *   */
-#if defined(_MSC_VER) && HC_ARCH_BITS == 32
-# define SSM_HOST_IS_MSC_32     1
-#else
-# define SSM_HOST_IS_MSC_32     0
-#endif
-
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/** SSM state. */
-typedef enum SSMSTATE
-{
-    SSMSTATE_INVALID = 0,
-    SSMSTATE_LIVE_PREP,
-    SSMSTATE_LIVE_STEP1,
-    SSMSTATE_LIVE_EXEC,
-    SSMSTATE_LIVE_VOTE,
-    SSMSTATE_LIVE_STEP2,
-    SSMSTATE_SAVE_PREP,
-    SSMSTATE_SAVE_EXEC,
-    SSMSTATE_SAVE_DONE,
-    SSMSTATE_LOAD_PREP,
-    SSMSTATE_LOAD_EXEC,
-    SSMSTATE_LOAD_DONE,
-    SSMSTATE_OPEN_READ,
-    SSMSTATE_END
-} SSMSTATE;
-
-
-/** Pointer to a SSM stream buffer. */
-typedef struct SSMSTRMBUF *PSSMSTRMBUF;
-/**
- * A SSM stream buffer.
- */
-typedef struct SSMSTRMBUF
-{
-    /** The buffer data. */
-    uint8_t                 abData[_64K];
-
-    /** The stream position of this buffer. */
-    uint64_t                offStream;
-    /** The amount of buffered data. */
-    uint32_t                cb;
-    /** End of stream indicator (for read streams only). */
-    bool                    fEndOfStream;
-    /** The nano timestamp set by ssmR3StrmGetFreeBuf. */
-    uint64_t                NanoTS;
-    /** Pointer to the next buffer in the chain. */
-    PSSMSTRMBUF volatile    pNext;
-} SSMSTRMBUF;
-
-/**
- * SSM stream.
- *
- * This is a typical producer / consumer setup with a dedicated I/O thread and
- * fixed number of buffers for read ahead and write back.
- */
-typedef struct SSMSTRM
-{
-    /** The stream method table. */
-    PCSSMSTRMOPS            pOps;
-    /** The user argument for the stream methods.
-     * For file based streams, this is the file handle and not a pointer. */
-    void                   *pvUser;
-
-    /** Write (set) or read (clear) stream. */
-    bool                    fWrite;
-    /** Termination indicator. */
-    bool volatile           fTerminating;
-    /** Indicates whether it is necessary to seek before the next buffer is
-     *  read from the stream.  This is used to avoid a seek in ssmR3StrmPeekAt. */
-    bool                    fNeedSeek;
-    /** Stream error status. */
-    int32_t volatile        rc;
-    /** The handle of the I/O thread. This is set to nil when not active. */
-    RTTHREAD                hIoThread;
-    /** Where to seek to. */
-    uint64_t                offNeedSeekTo;
-
-    /** The head of the consumer queue.
-     * For save the consumer is the I/O thread.  For load the I/O thread is the
-     * producer. */
-    PSSMSTRMBUF volatile    pHead;
-    /** Chain of free buffers.
-     * The consumer/producer roles are the inverse of pHead.  */
-    PSSMSTRMBUF volatile    pFree;
-    /** Event that's signalled when pHead is updated. */
-    RTSEMEVENT              hEvtHead;
-    /** Event that's signalled when pFree is updated. */
-    RTSEMEVENT              hEvtFree;
-
-    /** List of pending buffers that has been dequeued from pHead and reversed. */
-    PSSMSTRMBUF             pPending;
-    /** Pointer to the current buffer. */
-    PSSMSTRMBUF             pCur;
-    /** The stream offset of the current buffer. */
-    uint64_t                offCurStream;
-    /** The current buffer offset. */
-    uint32_t                off;
-    /** Whether we're checksumming reads/writes. */
-    bool                    fChecksummed;
-    /** The stream CRC if fChecksummed is set. */
-    uint32_t                u32StreamCRC;
-    /** How far into the buffer u32StreamCRC is up-to-date.
-     * This may lag behind off as it's desirable to checksum as large blocks as
-     * possible.  */
-    uint32_t                offStreamCRC;
-} SSMSTRM;
-/** Pointer to a SSM stream. */
-typedef SSMSTRM *PSSMSTRM;
-
-
-/**
- * Handle structure.
- */
-typedef struct SSMHANDLE
-{
-    /** Stream/buffer manager. */
-    SSMSTRM                 Strm;
-
-    /** The VM handle. */
-    PVM                     pVM;
-    /** The current operation. */
-    SSMSTATE                enmOp;
-    /** What to do after save completes. (move the enum) */
-    SSMAFTER                enmAfter;
-    /** Flag indicating that the operation has been cancelled. */
-    uint32_t volatile       fCancelled;
-    /** The current rc of the save operation. */
-    int32_t                 rc;
-    /** Number of compressed bytes left in the current data unit (V1). */
-    uint64_t                cbUnitLeftV1;
-    /** The current uncompressed offset into the data unit. */
-    uint64_t                offUnit;
-    /** Indicates that this is a live save or restore operation. */
-    bool                    fLiveSave;
-
-    /** Pointer to the progress callback function. */
-    PFNVMPROGRESS           pfnProgress;
-    /** User specified argument to the callback function. */
-    void                   *pvUser;
-    /** Next completion percentage. (corresponds to offEstProgress) */
-    unsigned                uPercent;
-    /** The position of the next progress callback in the estimated file. */
-    uint64_t                offEstProgress;
-    /** The estimated total byte count.
-     * (Only valid after the prep.) */
-    uint64_t                cbEstTotal;
-    /** Current position in the estimated file. */
-    uint64_t                offEst;
-    /** End of current unit in the estimated file. */
-    uint64_t                offEstUnitEnd;
-    /** The amount of % we reserve for the 'live' stage */
-    unsigned                uPercentLive;
-    /** The amount of % we reserve for the 'prepare' phase */
-    unsigned                uPercentPrepare;
-    /** The amount of % we reserve for the 'done' stage */
-    unsigned                uPercentDone;
-    /** The lowest value reported via SSMR3HandleReportLivePercent during one
-     * vote run. */
-    unsigned                uReportedLivePercent;
-    /** The filename, NULL if remote stream. */
-    const char             *pszFilename;
-
-    union
-    {
-        /** Write data. */
-        struct
-        {
-            /** Offset into the databuffer. */
-            uint32_t        offDataBuffer;
-            /** Space for the record header.  */
-            uint8_t         abRecHdr[1+7];
-            /** Data buffer. */
-            uint8_t         abDataBuffer[4096];
-            /** The maximum downtime given as milliseconds. */
-            uint32_t        cMsMaxDowntime;
-        } Write;
-
-        /** Read data. */
-        struct
-        {
-            /** V1: The decompressor of the current data unit. */
-            PRTZIPDECOMP    pZipDecompV1;
-            /** The major format version number. */
-            uint32_t        uFmtVerMajor;
-            /** The minor format version number. */
-            uint32_t        uFmtVerMinor;
-
-            /** V2: Unread bytes in the current record. */
-            uint32_t        cbRecLeft;
-            /** V2: Bytes in the data buffer. */
-            uint32_t        cbDataBuffer;
-            /** V2: Current buffer position. */
-            uint32_t        offDataBuffer;
-            /** V2: End of data indicator. */
-            bool            fEndOfData;
-            /** V2: The type and flags byte fo the current record. */
-            uint8_t         u8TypeAndFlags;
-
-            /** @name Context info for SSMR3SetLoadError.
-             * @{  */
-            /** Pointer to the header for the current unit. */
-            PSSMUNIT        pCurUnit;
-            /** The version of the current unit if in the load exec stage. */
-            uint32_t        uCurUnitVer;
-            /** The pass number of the current unit if in the load exec stage. */
-            uint32_t        uCurUnitPass;
-            /** Whether SSMR3SetLoadError[V] has been called.
-             * @note Using ASMAtomicXchgBool because I'm very lazy. */
-            bool volatile   fHaveSetError;
-            /** @} */
-
-            /** RTGCPHYS size in bytes. (Only applicable when loading/reading.) */
-            unsigned        cbGCPhys;
-            /** RTGCPTR size in bytes. (Only applicable when loading/reading.) */
-            unsigned        cbGCPtr;
-            /** Whether cbGCPtr is fixed or settable. */
-            bool            fFixedGCPtrSize;
-
-            /** 32-bit MSC saved this? */
-            bool            fIsHostMsc32;
-            /** "Host OS" dot "architecture", picked up from recent SSM data units. */
-            char            szHostOSAndArch[32];
-
-            /** @name Header info (set by ssmR3ValidateFile)
-             * @{ */
-            /** The size of the file header. */
-            uint32_t        cbFileHdr;
-            /** The major version number. */
-            uint16_t        u16VerMajor;
-            /** The minor version number. */
-            uint16_t        u16VerMinor;
-            /** The build number. */
-            uint32_t        u32VerBuild;
-            /** The SVN revision. */
-            uint32_t        u32SvnRev;
-            /** 32 or 64 depending on the host. */
-            uint8_t         cHostBits;
-            /** Whether the stream is checksummed (SSMFILEHDR_FLAGS_STREAM_CRC32). */
-            bool            fStreamCrc32;
-            /** The CRC of the loaded file. */
-            uint32_t        u32LoadCRC;
-            /** The size of the load file. */
-            uint64_t        cbLoadFile;
-            /** @} */
-
-            /** V2: Data buffer.
-             * @remarks Be extremely careful when changing the size of this buffer! */
-            uint8_t         abDataBuffer[4096];
-
-            /** V2: Decompression buffer for when we cannot use the stream buffer. */
-            uint8_t         abComprBuffer[4096];
-        } Read;
-    } u;
-} SSMHANDLE;
-
-
-/**
- * Header of the saved state file.
- *
- * Added in r5xxxx on 2009-07-2?, VirtualBox v3.0.51.
- */
-typedef struct SSMFILEHDR
-{
-    /** Magic string which identifies this file as a version of VBox saved state
-     *  file format (SSMFILEHDR_MAGIC_V2_0). */
-    char            szMagic[32];
-    /** The major version number. */
-    uint16_t        u16VerMajor;
-    /** The minor version number. */
-    uint16_t        u16VerMinor;
-    /** The build number. */
-    uint32_t        u32VerBuild;
-    /** The SVN revision. */
-    uint32_t        u32SvnRev;
-    /** 32 or 64 depending on the host. */
-    uint8_t         cHostBits;
-    /** The size of RTGCPHYS. */
-    uint8_t         cbGCPhys;
-    /** The size of RTGCPTR. */
-    uint8_t         cbGCPtr;
-    /** Reserved header space - must be zero. */
-    uint8_t         u8Reserved;
-    /** The number of units that (may) have stored data in the file. */
-    uint32_t        cUnits;
-    /** Flags, see SSMFILEHDR_FLAGS_XXX.  */
-    uint32_t        fFlags;
-    /** The maximum size of decompressed data. */
-    uint32_t        cbMaxDecompr;
-    /** The checksum of this header.
-     * This field is set to zero when calculating the checksum. */
-    uint32_t        u32CRC;
-} SSMFILEHDR;
-AssertCompileSize(SSMFILEHDR, 64);
-AssertCompileMemberOffset(SSMFILEHDR, u32CRC, 60);
-AssertCompileMemberSize(SSMFILEHDR, szMagic, sizeof(SSMFILEHDR_MAGIC_V2_0));
-/** Pointer to a saved state file header. */
-typedef SSMFILEHDR *PSSMFILEHDR;
-/** Pointer to a const saved state file header. */
-typedef SSMFILEHDR const *PCSSMFILEHDR;
-
-
-/**
- * Header of the saved state file.
- *
- * Added in r40980 on 2008-12-15, VirtualBox v2.0.51.
- *
- * @remarks This is a superset of SSMFILEHDRV11.
- */
-typedef struct SSMFILEHDRV12
-{
-    /** Magic string which identifies this file as a version of VBox saved state
-     *  file format (SSMFILEHDR_MAGIC_V1_2). */
-    char            achMagic[32];
-    /** The size of this file. Used to check
-     * whether the save completed and that things are fine otherwise. */
-    uint64_t        cbFile;
-    /** File checksum. The actual calculation skips past the u32CRC field. */
-    uint32_t        u32CRC;
-    /** Padding. */
-    uint32_t        u32Reserved;
-    /** The machine UUID. (Ignored if NIL.) */
-    RTUUID          MachineUuid;
-
-    /** The major version number. */
-    uint16_t        u16VerMajor;
-    /** The minor version number. */
-    uint16_t        u16VerMinor;
-    /** The build number. */
-    uint32_t        u32VerBuild;
-    /** The SVN revision. */
-    uint32_t        u32SvnRev;
-
-    /** 32 or 64 depending on the host. */
-    uint8_t         cHostBits;
-    /** The size of RTGCPHYS. */
-    uint8_t         cbGCPhys;
-    /** The size of RTGCPTR. */
-    uint8_t         cbGCPtr;
-    /** Padding. */
-    uint8_t         au8Reserved;
-} SSMFILEHDRV12;
-AssertCompileSize(SSMFILEHDRV12, 64+16);
-AssertCompileMemberOffset(SSMFILEHDRV12, u32CRC, 40);
-AssertCompileMemberSize(SSMFILEHDRV12, achMagic, sizeof(SSMFILEHDR_MAGIC_V1_2));
-/** Pointer to a saved state file header. */
-typedef SSMFILEHDRV12 *PSSMFILEHDRV12;
-
-
-/**
- * Header of the saved state file, version 1.1.
- *
- * Added in r23677 on 2007-08-17, VirtualBox v1.4.1.
- */
-typedef struct SSMFILEHDRV11
-{
-    /** Magic string which identifies this file as a version of VBox saved state
-     *  file format (SSMFILEHDR_MAGIC_V1_1). */
-    char            achMagic[32];
-    /** The size of this file. Used to check
-     * whether the save completed and that things are fine otherwise. */
-    uint64_t        cbFile;
-    /** File checksum. The actual calculation skips past the u32CRC field. */
-    uint32_t        u32CRC;
-    /** Padding. */
-    uint32_t        u32Reserved;
-    /** The machine UUID. (Ignored if NIL.) */
-    RTUUID          MachineUuid;
-} SSMFILEHDRV11;
-AssertCompileSize(SSMFILEHDRV11, 64);
-AssertCompileMemberOffset(SSMFILEHDRV11, u32CRC, 40);
-/** Pointer to a saved state file header. */
-typedef SSMFILEHDRV11 *PSSMFILEHDRV11;
-
-
-/**
- * Data unit header.
- */
-typedef struct SSMFILEUNITHDRV2
-{
-    /** Magic (SSMFILEUNITHDR_MAGIC or SSMFILEUNITHDR_END). */
-    char            szMagic[8];
-    /** The offset in the saved state stream of the start of this unit.
-     * This is mainly intended for sanity checking. */
-    uint64_t        offStream;
-    /** The CRC-in-progress value this unit starts at. */
-    uint32_t        u32CurStreamCRC;
-    /** The checksum of this structure, including the whole name.
-     * Calculated with this field set to zero.  */
-    uint32_t        u32CRC;
-    /** Data version. */
-    uint32_t        u32Version;
-    /** Instance number. */
-    uint32_t        u32Instance;
-    /** Data pass number. */
-    uint32_t        u32Pass;
-    /** Flags reserved for future extensions. Must be zero. */
-    uint32_t        fFlags;
-    /** Size of the data unit name including the terminator. (bytes) */
-    uint32_t        cbName;
-    /** Data unit name, variable size. */
-    char            szName[SSM_MAX_NAME_SIZE];
-} SSMFILEUNITHDRV2;
-AssertCompileMemberOffset(SSMFILEUNITHDRV2, szName, 44);
-AssertCompileMemberSize(SSMFILEUNITHDRV2, szMagic, sizeof(SSMFILEUNITHDR_MAGIC));
-AssertCompileMemberSize(SSMFILEUNITHDRV2, szMagic, sizeof(SSMFILEUNITHDR_END));
-/** Pointer to SSMFILEUNITHDRV2.  */
-typedef SSMFILEUNITHDRV2 *PSSMFILEUNITHDRV2;
-
-
-/**
- * Data unit header.
- *
- * This is used by v1.0, v1.1 and v1.2 of the format.
- */
-typedef struct SSMFILEUNITHDRV1
-{
-    /** Magic (SSMFILEUNITHDR_MAGIC or SSMFILEUNITHDR_END). */
-    char            achMagic[8];
-    /** Number of bytes in this data unit including the header. */
-    uint64_t        cbUnit;
-    /** Data version. */
-    uint32_t        u32Version;
-    /** Instance number. */
-    uint32_t        u32Instance;
-    /** Size of the data unit name including the terminator. (bytes) */
-    uint32_t        cchName;
-    /** Data unit name. */
-    char            szName[1];
-} SSMFILEUNITHDRV1;
-/** Pointer to SSMFILEUNITHDR.  */
-typedef SSMFILEUNITHDRV1 *PSSMFILEUNITHDRV1;
-
-
-/**
- * Termination data record.
- */
-typedef struct SSMRECTERM
-{
-    uint8_t         u8TypeAndFlags;
-    /** The record size (sizeof(SSMRECTERM) - 2). */
-    uint8_t         cbRec;
-    /** Flags, see SSMRECTERM_FLAGS_CRC32. */
-    uint16_t        fFlags;
-    /** The checksum of the stream up to fFlags (exclusive). */
-    uint32_t        u32StreamCRC;
-    /** The length of this data unit in bytes (including this record). */
-    uint64_t        cbUnit;
-} SSMRECTERM;
-AssertCompileSize(SSMRECTERM, 16);
-AssertCompileMemberAlignment(SSMRECTERM, cbUnit, 8);
-/** Pointer to a termination record. */
-typedef SSMRECTERM *PSSMRECTERM;
-/** Pointer to a const termination record. */
-typedef SSMRECTERM const *PCSSMRECTERM;
-
-
-/**
- * Directory entry.
- */
-typedef struct SSMFILEDIRENTRY
-{
-    /** The offset of the data unit. */
-    uint64_t        off;
-    /** The instance number. */
-    uint32_t        u32Instance;
-    /** The CRC-32 of the name excluding the terminator. (lazy bird) */
-    uint32_t        u32NameCRC;
-} SSMFILEDIRENTRY;
-AssertCompileSize(SSMFILEDIRENTRY, 16);
-/** Pointer to a directory entry. */
-typedef SSMFILEDIRENTRY *PSSMFILEDIRENTRY;
-/** Pointer to a const directory entry. */
-typedef SSMFILEDIRENTRY const *PCSSMFILEDIRENTRY;
-
-/**
- * Directory for the data units from the final pass.
- *
- * This is used to speed up SSMR3Seek (it would have to decompress and parse the
- * whole stream otherwise).
- */
-typedef struct SSMFILEDIR
-{
-    /** Magic string (SSMFILEDIR_MAGIC). */
-    char            szMagic[8];
-    /** The CRC-32 for the whole directory.
-     * Calculated with this field set to zero. */
-    uint32_t        u32CRC;
-    /** The number of directory entries. */
-    uint32_t        cEntries;
-    /** The directory entries (variable size). */
-    SSMFILEDIRENTRY aEntries[1];
-} SSMFILEDIR;
-AssertCompileSize(SSMFILEDIR, 32);
-/** Pointer to a directory. */
-typedef SSMFILEDIR *PSSMFILEDIR;
-/** Pointer to a const directory. */
-typedef SSMFILEDIR *PSSMFILEDIR;
-
-
-/**
- * Footer structure
- */
-typedef struct SSMFILEFTR
-{
-    /** Magic string (SSMFILEFTR_MAGIC). */
-    char            szMagic[8];
-    /** The offset of this record in the stream. */
-    uint64_t        offStream;
-    /** The CRC for the stream.
-     * This is set to zero if SSMFILEHDR_FLAGS_STREAM_CRC32 is clear. */
-    uint32_t        u32StreamCRC;
-    /** Number directory entries. */
-    uint32_t        cDirEntries;
-    /** Reserved footer space - must be zero. */
-    uint32_t        u32Reserved;
-    /** The CRC-32 for this structure.
-     * Calculated with this field set to zero. */
-    uint32_t        u32CRC;
-} SSMFILEFTR;
-AssertCompileSize(SSMFILEFTR, 32);
-/** Pointer to a footer. */
-typedef SSMFILEFTR *PSSMFILEFTR;
-/** Pointer to a const footer. */
-typedef SSMFILEFTR const *PCSSMFILEFTR;
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-/** Zeros used by the struct putter.
- * This must be at least 8 bytes or the code breaks. */
-static uint8_t const    g_abZero[_1K] = {0};
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-#ifndef SSM_STANDALONE
-static int                  ssmR3LazyInit(PVM pVM);
-static DECLCALLBACK(int)    ssmR3SelfLiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
-static DECLCALLBACK(int)    ssmR3SelfSaveExec(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int)    ssmR3SelfLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static DECLCALLBACK(int)    ssmR3LiveControlLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static int                  ssmR3Register(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess, const char *pszBefore, PSSMUNIT *ppUnit);
-static int                  ssmR3LiveControlEmit(PSSMHANDLE pSSM, long double lrdPct, uint32_t uPass);
-#endif
-
-static int                  ssmR3StrmWriteBuffers(PSSMSTRM pStrm);
-static int                  ssmR3StrmReadMore(PSSMSTRM pStrm);
-
-static int                  ssmR3DataFlushBuffer(PSSMHANDLE pSSM);
-static int                  ssmR3DataReadRecHdrV2(PSSMHANDLE pSSM);
-
-
-#ifndef SSM_STANDALONE
-
-/**
- * Cleans up resources allocated by SSM on VM termination.
- *
- * @param   pVM                 The VM handle.
- */
-VMMR3_INT_DECL(void) SSMR3Term(PVM pVM)
-{
-    if (pVM->ssm.s.fInitialized)
-    {
-        pVM->ssm.s.fInitialized = false;
-        RTCritSectDelete(&pVM->ssm.s.CancelCritSect);
-    }
-}
-
-
-/**
- * Performs lazy initialization of the SSM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM.
- */
-static int ssmR3LazyInit(PVM pVM)
-{
-    /*
-     * Register a saved state unit which we use to put the VirtualBox version,
-     * revision and similar stuff in.
-     */
-    pVM->ssm.s.fInitialized = true;
-    int rc = SSMR3RegisterInternal(pVM, "SSM", 0 /*uInstance*/, 1 /*uVersion*/, 64 /*cbGuess*/,
-                                   NULL /*pfnLivePrep*/, ssmR3SelfLiveExec, NULL /*pfnLiveVote*/,
-                                   NULL /*pfnSavePrep*/, ssmR3SelfSaveExec, NULL /*pfnSaveDone*/,
-                                   NULL /*pfnSavePrep*/, ssmR3SelfLoadExec, NULL /*pfnSaveDone*/);
-    if (RT_SUCCESS(rc))
-        rc = SSMR3RegisterInternal(pVM, "SSMLiveControl", 0 /*uInstance*/, 1 /*uVersion*/, 1 /*cbGuess*/,
-                                   NULL /*pfnLivePrep*/, NULL /*pfnLiveExec*/,     NULL /*pfnLiveVote*/,
-                                   NULL /*pfnSavePrep*/, NULL /*pfnSaveExec*/,     NULL /*pfnSaveDone*/,
-                                   NULL /*pfnSavePrep*/, ssmR3LiveControlLoadExec, NULL /*pfnSaveDone*/);
-
-    /*
-     * Initialize the cancellation critsect now.
-     */
-    if (RT_SUCCESS(rc))
-        rc = RTCritSectInit(&pVM->ssm.s.CancelCritSect);
-    if (RT_SUCCESS(rc))
-    {
-        STAM_REL_REG_USED(pVM, &pVM->ssm.s.uPass, STAMTYPE_U32, "/SSM/uPass", STAMUNIT_COUNT, "Current pass");
-    }
-
-    pVM->ssm.s.fInitialized = RT_SUCCESS(rc);
-    return rc;
-}
-
-
-/**
- * Do ssmR3SelfSaveExec in pass 0.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pSSM            The SSM handle.
- * @param   uPass           The data pass number.
- */
-static DECLCALLBACK(int) ssmR3SelfLiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    if (uPass == 0)
-    {
-        int rc = ssmR3SelfSaveExec(pVM, pSSM);
-        if (RT_SUCCESS(rc))
-            rc = VINF_SSM_DONT_CALL_AGAIN;
-        return rc;
-    }
-    AssertFailed();
-    return VERR_INTERNAL_ERROR_3;
-}
-
-
-/**
- * For saving usful things without having to go thru the tedious process of
- * adding it to the header.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pSSM            The SSM handle.
- */
-static DECLCALLBACK(int) ssmR3SelfSaveExec(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * String table containing pairs of variable and value string.
-     * Terminated by two empty strings.
-     */
-    SSMR3PutStrZ(pSSM, "Build Type");
-    SSMR3PutStrZ(pSSM, KBUILD_TYPE);
-    SSMR3PutStrZ(pSSM, "Host OS");
-    SSMR3PutStrZ(pSSM, KBUILD_TARGET "." KBUILD_TARGET_ARCH);
-#ifdef VBOX_OSE
-    SSMR3PutStrZ(pSSM, "OSE");
-    SSMR3PutStrZ(pSSM, "true");
-#endif
-
-    /* terminator */
-    SSMR3PutStrZ(pSSM, "");
-    return SSMR3PutStrZ(pSSM, "");
-}
-
-
-/**
- * For load the version + revision and stuff.
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pSSM            The SSM handle.
- * @param   uVersion        The version (1).
- * @param   uPass           The pass.
- */
-static DECLCALLBACK(int) ssmR3SelfLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    AssertLogRelMsgReturn(uVersion == 1, ("%d", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
-
-    /*
-     * The first and last passes contains a {name, value} string table that is
-     * terminated by two emptry strings.  It contains useful informal build
-     * info and can be very handy when something goes wrong after restore.
-     */
-    if (    uPass == 0
-        ||  uPass == SSM_PASS_FINAL)
-    {
-        for (unsigned i = 0; ; i++)
-        {
-            char szVar[128];
-            char szValue[1024];
-            int rc = SSMR3GetStrZ(pSSM, szVar, sizeof(szVar));
-            AssertRCReturn(rc, rc);
-            rc = SSMR3GetStrZ(pSSM, szValue, sizeof(szValue));
-            AssertRCReturn(rc, rc);
-            if (!szVar[0] && !szValue[0])
-                break;
-            if (i == 0)
-                LogRel(("SSM: Saved state info:\n"));
-            LogRel(("SSM:   %s: %s\n", szVar, szValue));
-
-            /*
-             * Detect 32-bit MSC for handling SSMFIELD_ENTRY_PAD_MSC32_AUTO.
-             * Save the Host OS for SSMR3HandleHostOSAndArch
-             */
-            if (!strcmp(szVar, "Host OS"))
-            {
-                bool fIsHostMsc32 = !strcmp(szValue, "win.x86");
-                if (fIsHostMsc32 != pSSM->u.Read.fIsHostMsc32)
-                {
-                    LogRel(("SSM: (fIsHostMsc32 %RTbool => %RTbool)\n", pSSM->u.Read.fIsHostMsc32, fIsHostMsc32));
-                    pSSM->u.Read.fIsHostMsc32 = fIsHostMsc32;
-                }
-
-                size_t cchValue = strlen(szValue);
-                size_t cchCopy = RT_MIN(cchValue, sizeof(pSSM->u.Read.szHostOSAndArch) - 1);
-                Assert(cchValue == cchCopy);
-                memcpy(pSSM->u.Read.szHostOSAndArch, szValue, cchCopy);
-                pSSM->u.Read.szHostOSAndArch[cchCopy] = '\0';
-            }
-        }
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Load exec callback for the special live save state unit that tracks the
- * progress of a live save.
- *
- * This is saved by ssmR3LiveControlEmit().
- *
- * @returns VBox status code.
- * @param   pVM             Pointer to the shared VM structure.
- * @param   pSSM            The SSM handle.
- * @param   uVersion        The version (1).
- * @param   uPass           The pass.
- */
-static DECLCALLBACK(int) ssmR3LiveControlLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    AssertLogRelMsgReturn(uVersion == 1, ("%d", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
-
-    uint16_t uPartsPerTenThousand;
-    int rc = SSMR3GetU16(pSSM, &uPartsPerTenThousand);
-    if (RT_SUCCESS(rc))
-    {
-        /* Scale it down to fit in our exec range. */
-        unsigned uPct = (unsigned)(  (long double)uPartsPerTenThousand / 100
-                                   * (100 - pSSM->uPercentPrepare - pSSM->uPercentDone) / 100)
-                      + pSSM->uPercentPrepare;
-        if (uPct != pSSM->uPercent)
-        {
-            AssertMsg(uPct < 100, ("uPct=%d uPartsPerTenThousand=%d uPercentPrepare=%d uPercentDone=%d\n", uPct, uPartsPerTenThousand, pSSM->uPercentPrepare, pSSM->uPercentDone));
-            pSSM->uPercent = uPct;
-            if (pSSM->pfnProgress)
-                pSSM->pfnProgress(pVM, RT_MIN(uPct, 100 - pSSM->uPercentDone), pSSM->pvUser);
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Internal registration worker.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pszName         Data unit name.
- * @param   uInstance       The instance id.
- * @param   uVersion        The data unit version.
- * @param   cbGuess         The guessed data unit size.
- * @param   pszBefore       Name of data unit to be placed in front of.
- *                          Optional.
- * @param   ppUnit          Where to store the inserted unit node.
- *                          Caller must fill in the missing details.
- */
-static int ssmR3Register(PVM pVM, const char *pszName, uint32_t uInstance,
-                         uint32_t uVersion, size_t cbGuess, const char *pszBefore, PSSMUNIT *ppUnit)
-{
-    /*
-     * Validate input.
-     */
-    AssertPtr(pszName);
-    AssertReturn(*pszName, VERR_INVALID_PARAMETER);
-    size_t cchName = strlen(pszName);
-    AssertMsgReturn(cchName < SSM_MAX_NAME_SIZE, ("%zu >= %u: %s\n", cchName, SSM_MAX_NAME_SIZE, pszName), VERR_OUT_OF_RANGE);
-
-    AssertReturn(!pszBefore || *pszBefore, VERR_INVALID_PARAMETER);
-    size_t cchBefore = pszBefore ? strlen(pszBefore) : 0;
-    AssertMsgReturn(cchBefore < SSM_MAX_NAME_SIZE, ("%zu >= %u: %s\n", cchBefore, SSM_MAX_NAME_SIZE, pszBefore), VERR_OUT_OF_RANGE);
-
-    /*
-     * Lazy init.
-     */
-    if (!pVM->ssm.s.fInitialized)
-    {
-        int rc = ssmR3LazyInit(pVM);
-        AssertRCReturn(rc, rc);
-    }
-
-    /*
-     * Walk to the end of the list checking for duplicates as we go.
-     */
-    PSSMUNIT    pUnitBeforePrev = NULL;
-    PSSMUNIT    pUnitBefore     = NULL;
-    PSSMUNIT    pUnitPrev       = NULL;
-    PSSMUNIT    pUnit           = pVM->ssm.s.pHead;
-    while (pUnit)
-    {
-        if (    pUnit->u32Instance == uInstance
-            &&  pUnit->cchName == cchName
-            &&  !memcmp(pUnit->szName, pszName, cchName))
-        {
-            AssertMsgFailed(("Duplicate registration %s\n", pszName));
-            return VERR_SSM_UNIT_EXISTS;
-        }
-        if (    pUnit->cchName == cchBefore
-            &&  !pUnitBefore
-            &&  !memcmp(pUnit->szName, pszBefore, cchBefore))
-        {
-            pUnitBeforePrev = pUnitPrev;
-            pUnitBefore     = pUnit;
-        }
-
-        /* next */
-        pUnitPrev = pUnit;
-        pUnit = pUnit->pNext;
-    }
-
-    /*
-     * Allocate new node.
-     */
-    pUnit = (PSSMUNIT)MMR3HeapAllocZ(pVM, MM_TAG_SSM, RT_OFFSETOF(SSMUNIT, szName[cchName + 1]));
-    if (!pUnit)
-        return VERR_NO_MEMORY;
-
-    /*
-     * Fill in (some) data. (Stuff is zero'd.)
-     */
-    pUnit->u32Version   = uVersion;
-    pUnit->u32Instance  = uInstance;
-    pUnit->cbGuess      = cbGuess;
-    pUnit->cchName      = cchName;
-    memcpy(pUnit->szName, pszName, cchName);
-
-    /*
-     * Insert
-     */
-    if (pUnitBefore)
-    {
-        pUnit->pNext = pUnitBefore;
-        if (pUnitBeforePrev)
-            pUnitBeforePrev->pNext = pUnit;
-        else
-            pVM->ssm.s.pHead       = pUnit;
-    }
-    else if (pUnitPrev)
-        pUnitPrev->pNext = pUnit;
-    else
-        pVM->ssm.s.pHead = pUnit;
-    pVM->ssm.s.cUnits++;
-
-    *ppUnit = pUnit;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Register a PDM Devices data unit.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   pDevIns         Device instance.
- * @param   pszName         Data unit name.
- * @param   uInstance       The instance identifier of the data unit.
- *                          This must together with the name be unique.
- * @param   uVersion        Data layout version number.
- * @param   cbGuess         The approximate amount of data in the unit.
- *                          Only for progress indicators.
- * @param   pszBefore       Name of data unit which we should be put in front
- *                          of. Optional (NULL).
- *
- * @param   pfnLivePrep     Prepare live save callback, optional.
- * @param   pfnLiveExec     Execute live save callback, optional.
- * @param   pfnLiveVote     Vote live save callback, optional.
- *
- * @param   pfnSavePrep     Prepare save callback, optional.
- * @param   pfnSaveExec     Execute save callback, optional.
- * @param   pfnSaveDone     Done save callback, optional.
- *
- * @param   pfnLoadPrep     Prepare load callback, optional.
- * @param   pfnLoadExec     Execute load callback, optional.
- * @param   pfnLoadDone     Done load callback, optional.
- */
-VMMR3DECL(int) SSMR3RegisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess, const char *pszBefore,
-    PFNSSMDEVLIVEPREP pfnLivePrep, PFNSSMDEVLIVEEXEC pfnLiveExec, PFNSSMDEVLIVEVOTE pfnLiveVote,
-    PFNSSMDEVSAVEPREP pfnSavePrep, PFNSSMDEVSAVEEXEC pfnSaveExec, PFNSSMDEVSAVEDONE pfnSaveDone,
-    PFNSSMDEVLOADPREP pfnLoadPrep, PFNSSMDEVLOADEXEC pfnLoadExec, PFNSSMDEVLOADDONE pfnLoadDone)
-{
-    PSSMUNIT pUnit;
-    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, pszBefore, &pUnit);
-    if (RT_SUCCESS(rc))
-    {
-        pUnit->enmType = SSMUNITTYPE_DEV;
-        pUnit->u.Dev.pfnLivePrep = pfnLivePrep;
-        pUnit->u.Dev.pfnLiveExec = pfnLiveExec;
-        pUnit->u.Dev.pfnLiveVote = pfnLiveVote;
-        pUnit->u.Dev.pfnSavePrep = pfnSavePrep;
-        pUnit->u.Dev.pfnSaveExec = pfnSaveExec;
-        pUnit->u.Dev.pfnSaveDone = pfnSaveDone;
-        pUnit->u.Dev.pfnLoadPrep = pfnLoadPrep;
-        pUnit->u.Dev.pfnLoadExec = pfnLoadExec;
-        pUnit->u.Dev.pfnLoadDone = pfnLoadDone;
-        pUnit->u.Dev.pDevIns = pDevIns;
-    }
-    return rc;
-}
-
-
-/**
- * Register a PDM driver data unit.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   pDrvIns         Driver instance.
- * @param   pszName         Data unit name.
- * @param   uInstance       The instance identifier of the data unit.
- *                          This must together with the name be unique.
- * @param   uVersion        Data layout version number.
- * @param   cbGuess         The approximate amount of data in the unit.
- *                          Only for progress indicators.
- *
- * @param   pfnLivePrep     Prepare live save callback, optional.
- * @param   pfnLiveExec     Execute live save callback, optional.
- * @param   pfnLiveVote     Vote live save callback, optional.
- *
- * @param   pfnSavePrep     Prepare save callback, optional.
- * @param   pfnSaveExec     Execute save callback, optional.
- * @param   pfnSaveDone     Done save callback, optional.
- *
- * @param   pfnLoadPrep     Prepare load callback, optional.
- * @param   pfnLoadExec     Execute load callback, optional.
- * @param   pfnLoadDone     Done load callback, optional.
- */
-VMMR3DECL(int) SSMR3RegisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
-    PFNSSMDRVLIVEPREP pfnLivePrep, PFNSSMDRVLIVEEXEC pfnLiveExec, PFNSSMDRVLIVEVOTE pfnLiveVote,
-    PFNSSMDRVSAVEPREP pfnSavePrep, PFNSSMDRVSAVEEXEC pfnSaveExec, PFNSSMDRVSAVEDONE pfnSaveDone,
-    PFNSSMDRVLOADPREP pfnLoadPrep, PFNSSMDRVLOADEXEC pfnLoadExec, PFNSSMDRVLOADDONE pfnLoadDone)
-{
-    PSSMUNIT pUnit;
-    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
-    if (RT_SUCCESS(rc))
-    {
-        pUnit->enmType = SSMUNITTYPE_DRV;
-        pUnit->u.Drv.pfnSavePrep = pfnSavePrep;
-        pUnit->u.Drv.pfnSaveExec = pfnSaveExec;
-        pUnit->u.Drv.pfnSaveDone = pfnSaveDone;
-        pUnit->u.Drv.pfnLoadPrep = pfnLoadPrep;
-        pUnit->u.Drv.pfnLoadExec = pfnLoadExec;
-        pUnit->u.Drv.pfnLoadDone = pfnLoadDone;
-        pUnit->u.Drv.pDrvIns = pDrvIns;
-    }
-    return rc;
-}
-
-
-/**
- * Register a internal data unit.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   pszName         Data unit name.
- * @param   uInstance       The instance identifier of the data unit.
- *                          This must together with the name be unique.
- * @param   uVersion        Data layout version number.
- * @param   cbGuess         The approximate amount of data in the unit.
- *                          Only for progress indicators.
- *
- * @param   pfnLivePrep     Prepare live save callback, optional.
- * @param   pfnLiveExec     Execute live save callback, optional.
- * @param   pfnLiveVote     Vote live save callback, optional.
- *
- * @param   pfnSavePrep     Prepare save callback, optional.
- * @param   pfnSaveExec     Execute save callback, optional.
- * @param   pfnSaveDone     Done save callback, optional.
- *
- * @param   pfnLoadPrep     Prepare load callback, optional.
- * @param   pfnLoadExec     Execute load callback, optional.
- * @param   pfnLoadDone     Done load callback, optional.
- */
-VMMR3DECL(int) SSMR3RegisterInternal(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
-    PFNSSMINTLIVEPREP pfnLivePrep, PFNSSMINTLIVEEXEC pfnLiveExec, PFNSSMINTLIVEVOTE pfnLiveVote,
-    PFNSSMINTSAVEPREP pfnSavePrep, PFNSSMINTSAVEEXEC pfnSaveExec, PFNSSMINTSAVEDONE pfnSaveDone,
-    PFNSSMINTLOADPREP pfnLoadPrep, PFNSSMINTLOADEXEC pfnLoadExec, PFNSSMINTLOADDONE pfnLoadDone)
-{
-    PSSMUNIT pUnit;
-    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
-    if (RT_SUCCESS(rc))
-    {
-        pUnit->enmType = SSMUNITTYPE_INTERNAL;
-        pUnit->u.Internal.pfnLivePrep = pfnLivePrep;
-        pUnit->u.Internal.pfnLiveExec = pfnLiveExec;
-        pUnit->u.Internal.pfnLiveVote = pfnLiveVote;
-        pUnit->u.Internal.pfnSavePrep = pfnSavePrep;
-        pUnit->u.Internal.pfnSaveExec = pfnSaveExec;
-        pUnit->u.Internal.pfnSaveDone = pfnSaveDone;
-        pUnit->u.Internal.pfnLoadPrep = pfnLoadPrep;
-        pUnit->u.Internal.pfnLoadExec = pfnLoadExec;
-        pUnit->u.Internal.pfnLoadDone = pfnLoadDone;
-    }
-    return rc;
-}
-
-
-/**
- * Register an external data unit.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   pszName         Data unit name.
- * @param   uInstance       The instance identifier of the data unit.
- *                          This must together with the name be unique.
- * @param   uVersion        Data layout version number.
- * @param   cbGuess         The approximate amount of data in the unit.
- *                          Only for progress indicators.
- *
- * @param   pfnLivePrep     Prepare live save callback, optional.
- * @param   pfnLiveExec     Execute live save callback, optional.
- * @param   pfnLiveVote     Vote live save callback, optional.
- *
- * @param   pfnSavePrep     Prepare save callback, optional.
- * @param   pfnSaveExec     Execute save callback, optional.
- * @param   pfnSaveDone     Done save callback, optional.
- *
- * @param   pfnLoadPrep     Prepare load callback, optional.
- * @param   pfnLoadExec     Execute load callback, optional.
- * @param   pfnLoadDone     Done load callback, optional.
- * @param   pvUser          User argument.
- */
-VMMR3DECL(int) SSMR3RegisterExternal(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
-    PFNSSMEXTLIVEPREP pfnLivePrep, PFNSSMEXTLIVEEXEC pfnLiveExec, PFNSSMEXTLIVEVOTE pfnLiveVote,
-    PFNSSMEXTSAVEPREP pfnSavePrep, PFNSSMEXTSAVEEXEC pfnSaveExec, PFNSSMEXTSAVEDONE pfnSaveDone,
-    PFNSSMEXTLOADPREP pfnLoadPrep, PFNSSMEXTLOADEXEC pfnLoadExec, PFNSSMEXTLOADDONE pfnLoadDone, void *pvUser)
-{
-    PSSMUNIT pUnit;
-    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
-    if (RT_SUCCESS(rc))
-    {
-        pUnit->enmType = SSMUNITTYPE_EXTERNAL;
-        pUnit->u.External.pfnLivePrep = pfnLivePrep;
-        pUnit->u.External.pfnLiveExec = pfnLiveExec;
-        pUnit->u.External.pfnLiveVote = pfnLiveVote;
-        pUnit->u.External.pfnSavePrep = pfnSavePrep;
-        pUnit->u.External.pfnSaveExec = pfnSaveExec;
-        pUnit->u.External.pfnSaveDone = pfnSaveDone;
-        pUnit->u.External.pfnLoadPrep = pfnLoadPrep;
-        pUnit->u.External.pfnLoadExec = pfnLoadExec;
-        pUnit->u.External.pfnLoadDone = pfnLoadDone;
-        pUnit->u.External.pvUser      = pvUser;
-    }
-    return rc;
-}
-
-
-/**
- * Deregister one or more PDM Device data units.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   pDevIns         Device instance.
- * @param   pszName         Data unit name.
- *                          Use NULL to deregister all data units for that device instance.
- * @param   uInstance       The instance identifier of the data unit.
- *                          This must together with the name be unique.
- * @remark  Only for dynamic data units and dynamic unloaded modules.
- */
-VMMR3_INT_DECL(int) SSMR3DeregisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName, uint32_t uInstance)
-{
-    /*
-     * Validate input.
-     */
-    if (!pDevIns)
-    {
-        AssertMsgFailed(("pDevIns is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Search the list.
-     */
-    size_t      cchName = pszName ? strlen(pszName) : 0;
-    int         rc = pszName ? VERR_SSM_UNIT_NOT_FOUND : VINF_SUCCESS;
-    PSSMUNIT    pUnitPrev = NULL;
-    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
-    while (pUnit)
-    {
-        if (    pUnit->enmType == SSMUNITTYPE_DEV
-            &&  (   !pszName
-                 || (   pUnit->cchName == cchName
-                     && !memcmp(pUnit->szName, pszName, cchName)))
-            &&  pUnit->u32Instance == uInstance
-            )
-        {
-            if (pUnit->u.Dev.pDevIns == pDevIns)
-            {
-                /*
-                 * Unlink it, advance pointer, and free the node.
-                 */
-                PSSMUNIT pFree = pUnit;
-                pUnit = pUnit->pNext;
-                if (pUnitPrev)
-                    pUnitPrev->pNext = pUnit;
-                else
-                    pVM->ssm.s.pHead = pUnit;
-                pVM->ssm.s.cUnits--;
-                Log(("SSM: Removed data unit '%s' (pdm dev).\n", pFree->szName));
-                MMR3HeapFree(pFree);
-
-                if (pszName)
-                    return VINF_SUCCESS;
-                rc = VINF_SUCCESS;
-                continue;
-            }
-            else if (pszName)
-            {
-                AssertMsgFailed(("Caller is not owner! Owner=%p Caller=%p %s\n",
-                                 pUnit->u.Dev.pDevIns, pDevIns, pszName));
-                return VERR_SSM_UNIT_NOT_OWNER;
-            }
-        }
-
-        /* next */
-        pUnitPrev = pUnit;
-        pUnit = pUnit->pNext;
-    }
-
-    return rc;
-}
-
-
-/**
- * Deregister one ore more PDM Driver data units.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pDrvIns         Driver instance.
- * @param   pszName         Data unit name.
- *                          Use NULL to deregister all data units for that driver instance.
- * @param   uInstance       The instance identifier of the data unit.
- *                          This must together with the name be unique. Ignored if pszName is NULL.
- * @remark  Only for dynamic data units and dynamic unloaded modules.
- */
-VMMR3_INT_DECL(int) SSMR3DeregisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName, uint32_t uInstance)
-{
-    /*
-     * Validate input.
-     */
-    if (!pDrvIns)
-    {
-        AssertMsgFailed(("pDrvIns is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Search the list.
-     */
-    size_t      cchName = pszName ? strlen(pszName) : 0;
-    int         rc = pszName ? VERR_SSM_UNIT_NOT_FOUND : VINF_SUCCESS;
-    PSSMUNIT    pUnitPrev = NULL;
-    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
-    while (pUnit)
-    {
-        if (    pUnit->enmType == SSMUNITTYPE_DRV
-            &&  (   !pszName
-                 || (   pUnit->cchName == cchName
-                     && !memcmp(pUnit->szName, pszName, cchName)
-                     && pUnit->u32Instance == uInstance))
-            )
-        {
-            if (pUnit->u.Drv.pDrvIns == pDrvIns)
-            {
-                /*
-                 * Unlink it, advance pointer, and free the node.
-                 */
-                PSSMUNIT pFree = pUnit;
-                pUnit = pUnit->pNext;
-                if (pUnitPrev)
-                    pUnitPrev->pNext = pUnit;
-                else
-                    pVM->ssm.s.pHead = pUnit;
-                pVM->ssm.s.cUnits--;
-                Log(("SSM: Removed data unit '%s' (pdm drv).\n", pFree->szName));
-                MMR3HeapFree(pFree);
-
-                if (pszName)
-                    return VINF_SUCCESS;
-                rc = VINF_SUCCESS;
-                continue;
-            }
-
-            AssertMsgReturn(!pszName,
-                            ("Caller is not owner! Owner=%p Caller=%p %s\n", pUnit->u.Drv.pDrvIns, pDrvIns, pszName),
-                            VERR_SSM_UNIT_NOT_OWNER);
-        }
-
-        /* next */
-        pUnitPrev = pUnit;
-        pUnit = pUnit->pNext;
-    }
-
-    return rc;
-}
-
-
-/**
- * Deregister a data unit.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   enmType         Unit type
- * @param   pszName         Data unit name.
- * @remark  Only for dynamic data units.
- */
-static int ssmR3DeregisterByNameAndType(PVM pVM, const char *pszName, SSMUNITTYPE enmType)
-{
-    /*
-     * Validate input.
-     */
-    if (!pszName)
-    {
-        AssertMsgFailed(("pszName is NULL!\n"));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /*
-     * Search the list.
-     */
-    size_t      cchName = strlen(pszName);
-    int         rc = VERR_SSM_UNIT_NOT_FOUND;
-    PSSMUNIT    pUnitPrev = NULL;
-    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
-    while (pUnit)
-    {
-        if (    pUnit->enmType == enmType
-            &&  pUnit->cchName == cchName
-            &&  !memcmp(pUnit->szName, pszName, cchName))
-        {
-            /*
-             * Unlink it, advance pointer, and free the node.
-             */
-            PSSMUNIT pFree = pUnit;
-            pUnit = pUnit->pNext;
-            if (pUnitPrev)
-                pUnitPrev->pNext = pUnit;
-            else
-                pVM->ssm.s.pHead = pUnit;
-            pVM->ssm.s.cUnits--;
-            Log(("SSM: Removed data unit '%s' (type=%d).\n", pFree->szName, enmType));
-            MMR3HeapFree(pFree);
-            return VINF_SUCCESS;
-        }
-
-        /* next */
-        pUnitPrev = pUnit;
-        pUnit = pUnit->pNext;
-    }
-
-    return rc;
-}
-
-
-/**
- * Deregister an internal data unit.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszName         Data unit name.
- * @remark  Only for dynamic data units.
- */
-VMMR3DECL(int) SSMR3DeregisterInternal(PVM pVM, const char *pszName)
-{
-    return ssmR3DeregisterByNameAndType(pVM, pszName, SSMUNITTYPE_INTERNAL);
-}
-
-
-/**
- * Deregister an external data unit.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszName         Data unit name.
- * @remark  Only for dynamic data units.
- */
-VMMR3DECL(int) SSMR3DeregisterExternal(PVM pVM, const char *pszName)
-{
-    return ssmR3DeregisterByNameAndType(pVM, pszName, SSMUNITTYPE_EXTERNAL);
-}
-
-#endif /* !SSM_STANDALONE */
-
-
-/**
- * Initializes the stream after/before opening the file/whatever.
- *
- * @returns VINF_SUCCESS or VERR_NO_MEMORY.
- * @param   pStrm           The stream handle.
- * @param   fChecksummed    Whether the stream is to be checksummed while
- *                          written/read.
- * @param   cBuffers        The number of buffers.
- */
-static int ssmR3StrmInitInternal(PSSMSTRM pStrm, bool fChecksummed, uint32_t cBuffers)
-{
-    Assert(cBuffers > 0);
-
-    /*
-     * Init the common data members.
-     */
-    pStrm->fTerminating = false;
-    pStrm->fNeedSeek    = false;
-    pStrm->rc           = VINF_SUCCESS;
-    pStrm->hIoThread    = NIL_RTTHREAD;
-    pStrm->offNeedSeekTo= UINT64_MAX;
-
-    pStrm->pHead        = NULL;
-    pStrm->pFree        = NULL;
-    pStrm->hEvtHead     = NIL_RTSEMEVENT;
-    pStrm->hEvtFree     = NIL_RTSEMEVENT;
-
-    pStrm->pPending     = NULL;
-    pStrm->pCur         = NULL;
-    pStrm->offCurStream = 0;
-    pStrm->off          = 0;
-    pStrm->fChecksummed = fChecksummed;
-    pStrm->u32StreamCRC = fChecksummed ? RTCrc32Start() : 0;
-    pStrm->offStreamCRC = 0;
-
-    /*
-     * Allocate the buffers.  Page align them in case that makes the kernel
-     * and/or cpu happier in some way.
-     */
-    int rc = VINF_SUCCESS;
-    for (uint32_t i = 0; i < cBuffers; i++)
-    {
-        PSSMSTRMBUF pBuf = (PSSMSTRMBUF)RTMemPageAllocZ(sizeof(*pBuf));
-        if (!pBuf)
-        {
-            if (i > 2)
-            {
-                LogRel(("ssmR3StrmAllocBuffer: WARNING: Could only get %d stream buffers.\n", i));
-                break;
-            }
-            LogRel(("ssmR3StrmAllocBuffer: Failed to allocate stream buffers. (i=%d)\n", i));
-            return VERR_NO_MEMORY;
-        }
-
-        /* link it */
-        pBuf->pNext = pStrm->pFree;
-        pStrm->pFree = pBuf;
-    }
-
-    /*
-     * Create the event semaphores.
-     */
-    rc = RTSemEventCreate(&pStrm->hEvtHead);
-    if (RT_FAILURE(rc))
-        return rc;
-    rc = RTSemEventCreate(&pStrm->hEvtFree);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Destroys a list of buffers.
- *
- * @param   pHead            Pointer to the head.
- */
-static void ssmR3StrmDestroyBufList(PSSMSTRMBUF pHead)
-{
-    while (pHead)
-    {
-        PSSMSTRMBUF pCur = pHead;
-        pHead = pCur->pNext;
-        pCur->pNext = NULL;
-        RTMemPageFree(pCur, sizeof(*pCur));
-    }
-}
-
-
-/**
- * Cleans up a stream after ssmR3StrmInitInternal has been called (regardless of
- * it succeeded or not).
- *
- * @param   pStrm           The stream handle.
- */
-static void ssmR3StrmDelete(PSSMSTRM pStrm)
-{
-    RTMemPageFree(pStrm->pCur, sizeof(*pStrm->pCur));
-    pStrm->pCur = NULL;
-    ssmR3StrmDestroyBufList(pStrm->pHead);
-    pStrm->pHead = NULL;
-    ssmR3StrmDestroyBufList(pStrm->pPending);
-    pStrm->pPending = NULL;
-    ssmR3StrmDestroyBufList(pStrm->pFree);
-    pStrm->pFree = NULL;
-
-    RTSemEventDestroy(pStrm->hEvtHead);
-    pStrm->hEvtHead = NIL_RTSEMEVENT;
-
-    RTSemEventDestroy(pStrm->hEvtFree);
-    pStrm->hEvtFree = NIL_RTSEMEVENT;
-}
-
-
-/**
- * Initializes a stream that uses a method table.
- *
- * @returns VBox status code.
- * @param   pStrm           The stream manager structure.
- * @param   pStreamOps      The stream method table.
- * @param   pvUser          The user argument for the stream methods.
- * @param   fWrite          Whether to open for writing or reading.
- * @param   fChecksummed    Whether the stream is to be checksummed while
- *                          written/read.
- * @param   cBuffers        The number of buffers.
- */
-static int ssmR3StrmInit(PSSMSTRM pStrm, PCSSMSTRMOPS pStreamOps, void *pvUser, bool fWrite, bool fChecksummed, uint32_t cBuffers)
-{
-    int rc = ssmR3StrmInitInternal(pStrm, fChecksummed, cBuffers);
-    if (RT_SUCCESS(rc))
-    {
-        pStrm->pOps   = pStreamOps;
-        pStrm->pvUser = pvUser;
-        pStrm->fWrite = fWrite;
-        return VINF_SUCCESS;
-    }
-
-    ssmR3StrmDelete(pStrm);
-    pStrm->rc = rc;
-    return rc;
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnWrite
- */
-static DECLCALLBACK(int) ssmR3FileWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
-{
-    Assert(RTFileTell((RTFILE)(uintptr_t)pvUser) == offStream); NOREF(offStream);
-    return RTFileWriteAt((RTFILE)(uintptr_t)pvUser, offStream, pvBuf, cbToWrite, NULL); /** @todo use RTFileWrite */
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnRead
- */
-static DECLCALLBACK(int) ssmR3FileRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
-{
-    Assert(RTFileTell((RTFILE)(uintptr_t)pvUser) == offStream); NOREF(offStream);
-    return RTFileRead((RTFILE)(uintptr_t)pvUser, pvBuf, cbToRead, pcbRead);
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnSeek
- */
-static DECLCALLBACK(int) ssmR3FileSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
-{
-    return RTFileSeek((RTFILE)(uintptr_t)pvUser, offSeek, uMethod, poffActual);
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnTell
- */
-static DECLCALLBACK(uint64_t) ssmR3FileTell(void *pvUser)
-{
-    return RTFileTell((RTFILE)(uintptr_t)pvUser);
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnSize
- */
-static DECLCALLBACK(int) ssmR3FileSize(void *pvUser, uint64_t *pcb)
-{
-    return RTFileGetSize((RTFILE)(uintptr_t)pvUser, pcb);
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnIsOk
- */
-static DECLCALLBACK(int) ssmR3FileIsOk(void *pvUser)
-{
-    /*
-     * Check that there is still some space left on the disk.
-     */
-    RTFOFF cbFree;
-    int rc = RTFileQueryFsSizes((RTFILE)(uintptr_t)pvUser, NULL, &cbFree, NULL, NULL);
-#define SSM_MIN_DISK_FREE    ((RTFOFF)( 10 * _1M ))
-    if (RT_SUCCESS(rc))
-    {
-        if (cbFree < SSM_MIN_DISK_FREE)
-        {
-            LogRel(("SSM: Giving up: Low on disk space. (cbFree=%RTfoff, SSM_MIN_DISK_FREE=%RTfoff).\n",
-                    cbFree, SSM_MIN_DISK_FREE));
-            rc = VERR_SSM_LOW_ON_DISK_SPACE;
-        }
-    }
-    else if (rc == VERR_NOT_SUPPORTED)
-        rc = VINF_SUCCESS;
-    else
-        AssertLogRelRC(rc);
-    return rc;
-}
-
-
-/**
- * @copydoc SSMSTRMOPS::pfnClose
- */
-static DECLCALLBACK(int) ssmR3FileClose(void *pvUser, bool fCancelled)
-{
-    NOREF(fCancelled);
-    return RTFileClose((RTFILE)(uintptr_t)pvUser);
-}
-
-
-/**
- * Method table for a file based stream.
- */
-static SSMSTRMOPS const g_ssmR3FileOps =
-{
-    SSMSTRMOPS_VERSION,
-    ssmR3FileWrite,
-    ssmR3FileRead,
-    ssmR3FileSeek,
-    ssmR3FileTell,
-    ssmR3FileSize,
-    ssmR3FileIsOk,
-    ssmR3FileClose,
-    SSMSTRMOPS_VERSION
-};
-
-
-/**
- * Opens a file stream.
- *
- * @returns VBox status code.
- * @param   pStrm           The stream manager structure.
- * @param   pszFilename     The file to open or create.
- * @param   fWrite          Whether to open for writing or reading.
- * @param   fChecksummed    Whether the stream is to be checksummed while
- *                          written/read.
- * @param   cBuffers        The number of buffers.
- */
-static int ssmR3StrmOpenFile(PSSMSTRM pStrm, const char *pszFilename, bool fWrite, bool fChecksummed, uint32_t cBuffers)
-{
-    int rc = ssmR3StrmInitInternal(pStrm, fChecksummed, cBuffers);
-    if (RT_SUCCESS(rc))
-    {
-        uint32_t fFlags = fWrite
-                        ? RTFILE_O_READWRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE
-                        : RTFILE_O_READ      | RTFILE_O_OPEN           | RTFILE_O_DENY_WRITE;
-        RTFILE hFile;
-        rc = RTFileOpen(&hFile, pszFilename, fFlags);
-        if (RT_SUCCESS(rc))
-        {
-            pStrm->pOps   = &g_ssmR3FileOps;
-            pStrm->pvUser = (void *)(uintptr_t)hFile;
-            pStrm->fWrite = fWrite;
-            return VINF_SUCCESS;
-        }
-    }
-
-    ssmR3StrmDelete(pStrm);
-    pStrm->rc = rc;
-    return rc;
-}
-
-
-/**
- * Raise an error condition on the stream.
- *
- * @returns true if we raised the error condition, false if the stream already
- *          had an error condition set.
- *
- * @param   pStrm           The stream handle.
- * @param   rc              The VBox error status code.
- *
- * @thread  Any.
- */
-DECLINLINE(bool) ssmR3StrmSetError(PSSMSTRM pStrm, int rc)
-{
-    Assert(RT_FAILURE_NP(rc));
-    return ASMAtomicCmpXchgS32(&pStrm->rc, rc, VINF_SUCCESS);
-}
-
-
-/**
- * Puts a buffer into the free list.
- *
- * @param   pStrm           The stream handle.
- * @param   pBuf            The buffer.
- *
- * @thread  The consumer.
- */
-static void ssmR3StrmPutFreeBuf(PSSMSTRM pStrm, PSSMSTRMBUF pBuf)
-{
-    for (;;)
-    {
-        PSSMSTRMBUF pCurFreeHead = ASMAtomicUoReadPtrT(&pStrm->pFree, PSSMSTRMBUF);
-        ASMAtomicUoWritePtr(&pBuf->pNext, pCurFreeHead);
-        if (ASMAtomicCmpXchgPtr(&pStrm->pFree, pBuf, pCurFreeHead))
-        {
-            int rc = RTSemEventSignal(pStrm->hEvtFree);
-            AssertRC(rc);
-            return;
-        }
-    }
-}
-
-
-/**
- * Gets a free buffer, waits for one if necessary.
- *
- * @returns Pointer to the buffer on success. NULL if we're terminating.
- * @param   pStrm           The stream handle.
- *
- * @thread  The producer.
- */
-static PSSMSTRMBUF ssmR3StrmGetFreeBuf(PSSMSTRM pStrm)
-{
-    for (;;)
-    {
-        PSSMSTRMBUF pMine = ASMAtomicUoReadPtrT(&pStrm->pFree, PSSMSTRMBUF);
-        if (!pMine)
-        {
-            if (pStrm->fTerminating)
-                return NULL;
-            if (RT_FAILURE(pStrm->rc))
-                return NULL;
-            if (    pStrm->fWrite
-                &&  pStrm->hIoThread == NIL_RTTHREAD)
-            {
-                int rc = ssmR3StrmWriteBuffers(pStrm);
-                if (RT_FAILURE(rc))
-                    return NULL;
-            }
-            int rc = RTSemEventWaitNoResume(pStrm->hEvtFree, 30000);
-            if (    rc == VERR_SEM_DESTROYED
-                ||  pStrm->fTerminating)
-                return NULL;
-            continue;
-        }
-
-        if (ASMAtomicCmpXchgPtr(&pStrm->pFree, pMine->pNext, pMine))
-        {
-            pMine->offStream    = UINT64_MAX;
-            pMine->cb           = 0;
-            pMine->pNext        = NULL;
-            pMine->fEndOfStream = false;
-            pMine->NanoTS       = RTTimeNanoTS();
-            return pMine;
-        }
-    }
-}
-
-
-/**
- * Puts a buffer onto the queue.
- *
- * @param   pBuf        The buffer.
- *
- * @thread  The producer.
- */
-static void ssmR3StrmPutBuf(PSSMSTRM pStrm, PSSMSTRMBUF pBuf)
-{
-    for (;;)
-    {
-        PSSMSTRMBUF pCurHead = ASMAtomicUoReadPtrT(&pStrm->pHead, PSSMSTRMBUF);
-        ASMAtomicUoWritePtr(&pBuf->pNext, pCurHead);
-        if (ASMAtomicCmpXchgPtr(&pStrm->pHead, pBuf, pCurHead))
-        {
-            int rc = RTSemEventSignal(pStrm->hEvtHead);
-            AssertRC(rc);
-            return;
-        }
-    }
-}
-
-
-/**
- * Reverses the list.
- *
- * @returns The head of the reversed list.
- * @param   pHead           The head of the list to reverse.
- */
-static PSSMSTRMBUF ssmR3StrmReverseList(PSSMSTRMBUF pHead)
-{
-    PSSMSTRMBUF pRevHead = NULL;
-    while (pHead)
-    {
-        PSSMSTRMBUF pCur = pHead;
-        pHead = pCur->pNext;
-        pCur->pNext = pRevHead;
-        pRevHead = pCur;
-    }
-    return pRevHead;
-}
-
-
-/**
- * Gets one buffer from the queue, will wait for one to become ready if
- * necessary.
- *
- * @returns Pointer to the buffer on success. NULL if we're terminating.
- * @param   pBuf        The buffer.
- *
- * @thread  The consumer.
- */
-static PSSMSTRMBUF ssmR3StrmGetBuf(PSSMSTRM pStrm)
-{
-    for (;;)
-    {
-        PSSMSTRMBUF pMine = pStrm->pPending;
-        if (pMine)
-        {
-            pStrm->pPending = pMine->pNext;
-            pMine->pNext = NULL;
-            return pMine;
-        }
-
-        pMine = ASMAtomicXchgPtrT(&pStrm->pHead, NULL, PSSMSTRMBUF);
-        if (pMine)
-            pStrm->pPending = ssmR3StrmReverseList(pMine);
-        else
-        {
-            if (pStrm->fTerminating)
-                return NULL;
-            if (RT_FAILURE(pStrm->rc))
-                return NULL;
-            if (    !pStrm->fWrite
-                &&  pStrm->hIoThread == NIL_RTTHREAD)
-            {
-                int rc = ssmR3StrmReadMore(pStrm);
-                if (RT_FAILURE(rc))
-                    return NULL;
-                continue;
-            }
-
-            int rc = RTSemEventWaitNoResume(pStrm->hEvtHead, 30000);
-            if (    rc == VERR_SEM_DESTROYED
-                ||  pStrm->fTerminating)
-                return NULL;
-        }
-    }
-}
-
-
-/**
- * Flushes the current buffer (both write and read streams).
- *
- * @param   pStrm           The stream handle.
- */
-static void ssmR3StrmFlushCurBuf(PSSMSTRM pStrm)
-{
-    if (pStrm->pCur)
-    {
-        PSSMSTRMBUF pBuf = pStrm->pCur;
-        pStrm->pCur = NULL;
-
-        if (pStrm->fWrite)
-        {
-            uint32_t cb     = pStrm->off;
-            pBuf->cb        = cb;
-            pBuf->offStream = pStrm->offCurStream;
-            if (    pStrm->fChecksummed
-                &&  pStrm->offStreamCRC < cb)
-                pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC,
-                                                     &pBuf->abData[pStrm->offStreamCRC],
-                                                     cb - pStrm->offStreamCRC);
-            pStrm->offCurStream += cb;
-            pStrm->off           = 0;
-            pStrm->offStreamCRC  = 0;
-
-            ssmR3StrmPutBuf(pStrm, pBuf);
-        }
-        else
-        {
-            uint32_t cb = pBuf->cb;
-            if (    pStrm->fChecksummed
-                &&  pStrm->offStreamCRC < cb)
-                pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC,
-                                                     &pBuf->abData[pStrm->offStreamCRC],
-                                                     cb - pStrm->offStreamCRC);
-            pStrm->offCurStream += cb;
-            pStrm->off           = 0;
-            pStrm->offStreamCRC  = 0;
-
-            ssmR3StrmPutFreeBuf(pStrm, pBuf);
-        }
-    }
-}
-
-
-/**
- * Flush buffered data.
- *
- * @returns VBox status code. Returns VINF_EOF if we encounter a buffer with the
- *          fEndOfStream indicator set.
- * @param   pStrm           The stream handle.
- *
- * @thread  The producer thread.
- */
-static int ssmR3StrmWriteBuffers(PSSMSTRM pStrm)
-{
-    Assert(pStrm->fWrite);
-
-    /*
-     * Just return if the stream has a pending error condition.
-     */
-    int rc = pStrm->rc;
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Grab the pending list and write it out.
-     */
-    PSSMSTRMBUF pHead = ASMAtomicXchgPtrT(&pStrm->pHead, NULL, PSSMSTRMBUF);
-    if (!pHead)
-        return VINF_SUCCESS;
-    pHead = ssmR3StrmReverseList(pHead);
-
-    while (pHead)
-    {
-        /* pop */
-        PSSMSTRMBUF pCur = pHead;
-        pHead = pCur->pNext;
-
-        /* flush */
-        rc = pStrm->pOps->pfnIsOk(pStrm->pvUser);
-        if (RT_SUCCESS(rc))
-            rc = pStrm->pOps->pfnWrite(pStrm->pvUser, pCur->offStream, &pCur->abData[0], pCur->cb);
-        if (    RT_FAILURE(rc)
-            &&  ssmR3StrmSetError(pStrm, rc))
-            LogRel(("ssmR3StrmWriteBuffers: Write failed with rc=%Rrc at offStream=%#llx\n", rc, pCur->offStream));
-
-        /* free */
-        bool fEndOfStream = pCur->fEndOfStream;
-        ssmR3StrmPutFreeBuf(pStrm, pCur);
-        if (fEndOfStream)
-        {
-            Assert(!pHead);
-            return VINF_EOF;
-        }
-    }
-
-    return pStrm->rc;
-}
-
-
-/**
- * Closes the stream after first flushing any pending write.
- *
- * @returns VBox status code.
- * @param   pStrm           The stream handle.
- * @param   fCancelled      Indicates whether the operation was cancelled or
- *                          not.
- */
-static int ssmR3StrmClose(PSSMSTRM pStrm, bool fCancelled)
-{
-    /*
-     * Flush, terminate the I/O thread, and close the stream.
-     */
-    if (pStrm->fWrite)
-    {
-        ssmR3StrmFlushCurBuf(pStrm);
-        if (pStrm->hIoThread == NIL_RTTHREAD)
-            ssmR3StrmWriteBuffers(pStrm);
-    }
-
-    if (pStrm->hIoThread != NIL_RTTHREAD)
-        ASMAtomicWriteBool(&pStrm->fTerminating, true);
-
-    int rc;
-    if (pStrm->fWrite)
-    {
-        if (pStrm->hIoThread != NIL_RTTHREAD)
-        {
-            int rc2 = RTSemEventSignal(pStrm->hEvtHead);
-            AssertLogRelRC(rc2);
-            int rc3 = RTThreadWait(pStrm->hIoThread, RT_INDEFINITE_WAIT, NULL);
-            AssertLogRelRC(rc3);
-            pStrm->hIoThread = NIL_RTTHREAD;
-        }
-
-        rc = pStrm->pOps->pfnClose(pStrm->pvUser, fCancelled);
-        if (RT_FAILURE(rc))
-            ssmR3StrmSetError(pStrm, rc);
-    }
-    else
-    {
-        rc = pStrm->pOps->pfnClose(pStrm->pvUser, fCancelled);
-        if (RT_FAILURE(rc))
-            ssmR3StrmSetError(pStrm, rc);
-
-        if (pStrm->hIoThread != NIL_RTTHREAD)
-        {
-            int rc2 = RTSemEventSignal(pStrm->hEvtFree);
-            AssertLogRelRC(rc2);
-            int rc3 = RTThreadWait(pStrm->hIoThread, RT_INDEFINITE_WAIT, NULL);
-            AssertLogRelRC(rc3);
-            pStrm->hIoThread = NIL_RTTHREAD;
-        }
-    }
-
-    pStrm->pOps   = NULL;
-    pStrm->pvUser = NULL;
-
-    rc = pStrm->rc;
-    ssmR3StrmDelete(pStrm);
-
-    return rc;
-}
-
-
-/**
- * Stream output routine.
- *
- * @returns VBox status code.
- * @param   pStrm       The stream handle.
- * @param   pvBuf       What to write.
- * @param   cbToWrite   How much to write.
- *
- * @thread  The producer in a write stream (never the I/O thread).
- */
-static int ssmR3StrmWrite(PSSMSTRM pStrm, const void *pvBuf, size_t cbToWrite)
-{
-    AssertReturn(cbToWrite > 0, VINF_SUCCESS);
-    Assert(pStrm->fWrite);
-
-    /*
-     * Squeeze as much as possible into the current buffer.
-     */
-    PSSMSTRMBUF pBuf = pStrm->pCur;
-    if (RT_LIKELY(pBuf))
-    {
-        uint32_t cbLeft = RT_SIZEOFMEMB(SSMSTRMBUF, abData) - pStrm->off;
-        if (RT_LIKELY(cbLeft >= cbToWrite))
-        {
-            memcpy(&pBuf->abData[pStrm->off], pvBuf, cbToWrite);
-            pStrm->off += (uint32_t)cbToWrite;
-            return VINF_SUCCESS;
-        }
-
-        if (cbLeft > 0)
-        {
-            memcpy(&pBuf->abData[pStrm->off], pvBuf, cbLeft);
-            pStrm->off += cbLeft;
-            cbToWrite  -= cbLeft;
-            pvBuf       = (uint8_t const *)pvBuf + cbLeft;
-        }
-        Assert(pStrm->off == RT_SIZEOFMEMB(SSMSTRMBUF, abData));
-    }
-
-    /*
-     * Need one or more new buffers.
-     */
-    do
-    {
-        /*
-         * Flush the current buffer and replace it with a new one.
-         */
-        ssmR3StrmFlushCurBuf(pStrm);
-        pBuf = ssmR3StrmGetFreeBuf(pStrm);
-        if (!pBuf)
-            break;
-        pStrm->pCur = pBuf;
-        Assert(pStrm->off == 0);
-
-        /*
-         * Copy data to the buffer.
-         */
-        uint32_t cbCopy = RT_SIZEOFMEMB(SSMSTRMBUF, abData);
-        if (cbCopy > cbToWrite)
-            cbCopy = (uint32_t)cbToWrite;
-        memcpy(&pBuf->abData[0], pvBuf, cbCopy);
-        pStrm->off  = cbCopy;
-        cbToWrite  -= cbCopy;
-        pvBuf       = (uint8_t const *)pvBuf + cbCopy;
-    } while (cbToWrite > 0);
-
-    return pStrm->rc;
-}
-
-
-/**
- * Reserves space in the current buffer so the caller can write directly to the
- * buffer instead of doing double buffering.
- *
- * @returns VBox status code
- * @param   pStrm       The stream handle.
- * @param   cb          The amount of buffer space to reserve.
- * @param   ppb         Where to return the pointer.
- */
-static int ssmR3StrmReserveWriteBufferSpace(PSSMSTRM pStrm, size_t cb, uint8_t **ppb)
-{
-    Assert(pStrm->fWrite);
-    Assert(RT_SIZEOFMEMB(SSMSTRMBUF, abData) / 4 >= cb);
-
-    /*
-     * Check if there is room in the current buffer, it not flush it.
-     */
-    PSSMSTRMBUF pBuf = pStrm->pCur;
-    if (pBuf)
-    {
-        uint32_t cbLeft = RT_SIZEOFMEMB(SSMSTRMBUF, abData) - pStrm->off;
-        if (cbLeft >= cb)
-        {
-            *ppb = &pBuf->abData[pStrm->off];
-            return VINF_SUCCESS;
-        }
-
-        ssmR3StrmFlushCurBuf(pStrm);
-    }
-
-    /*
-     * Get a fresh buffer and return a pointer into it.
-     */
-    pBuf = ssmR3StrmGetFreeBuf(pStrm);
-    if (pBuf)
-    {
-        pStrm->pCur = pBuf;
-        Assert(pStrm->off == 0);
-        *ppb = &pBuf->abData[0];
-    }
-    else
-        *ppb = NULL; /* make gcc happy. */
-    return pStrm->rc;
-}
-
-
-/**
- * Commits buffer space reserved by ssmR3StrmReserveWriteBufferSpace.
- *
- * @returns VBox status code.
- * @param   pStrm       The stream handle.
- * @param   cb          The amount of buffer space to commit.  This can be less
- *                      that what was reserved initially.
- */
-static int ssmR3StrmCommitWriteBufferSpace(PSSMSTRM pStrm, size_t cb)
-{
-    Assert(pStrm->pCur);
-    Assert(pStrm->off + cb <= RT_SIZEOFMEMB(SSMSTRMBUF, abData));
-    pStrm->off += (uint32_t)cb;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Marks the end of the stream.
- *
- * This will cause the I/O thread to quit waiting for more buffers.
- *
- * @returns VBox status code.
- * @param   pStrm       The stream handle.
- */
-static int ssmR3StrmSetEnd(PSSMSTRM pStrm)
-{
-    Assert(pStrm->fWrite);
-    PSSMSTRMBUF pBuf = pStrm->pCur;
-    if (RT_UNLIKELY(!pStrm->pCur))
-    {
-        pBuf = ssmR3StrmGetFreeBuf(pStrm);
-        if (!pBuf)
-            return pStrm->rc;
-        pStrm->pCur = pBuf;
-        Assert(pStrm->off == 0);
-    }
-    pBuf->fEndOfStream = true;
-    ssmR3StrmFlushCurBuf(pStrm);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Read more from the stream.
- *
- * @returns VBox status code. VERR_EOF gets translated into VINF_EOF.
- * @param   pStrm       The stream handle.
- *
- * @thread  The I/O thread when we got one, otherwise the stream user.
- */
-static int ssmR3StrmReadMore(PSSMSTRM pStrm)
-{
-    int rc;
-    Log6(("ssmR3StrmReadMore:\n"));
-
-    /*
-     * Undo seek done by ssmR3StrmPeekAt.
-     */
-    if (pStrm->fNeedSeek)
-    {
-        rc = pStrm->pOps->pfnSeek(pStrm->pvUser, pStrm->offNeedSeekTo, RTFILE_SEEK_BEGIN, NULL);
-        if (RT_FAILURE(rc))
-        {
-            if (ssmR3StrmSetError(pStrm, rc))
-                LogRel(("ssmR3StrmReadMore: RTFileSeek(,%#llx,) failed with rc=%Rrc\n", pStrm->offNeedSeekTo, rc));
-            return rc;
-        }
-        pStrm->fNeedSeek     = false;
-        pStrm->offNeedSeekTo = UINT64_MAX;
-    }
-
-    /*
-     * Get a free buffer and try fill it up.
-     */
-    PSSMSTRMBUF pBuf = ssmR3StrmGetFreeBuf(pStrm);
-    if (!pBuf)
-        return pStrm->rc;
-
-    pBuf->offStream = pStrm->pOps->pfnTell(pStrm->pvUser);
-    size_t cbRead   = sizeof(pBuf->abData);
-    rc = pStrm->pOps->pfnRead(pStrm->pvUser, pBuf->offStream, &pBuf->abData[0], cbRead, &cbRead);
-    if (    RT_SUCCESS(rc)
-        &&  cbRead > 0)
-    {
-        pBuf->cb           = (uint32_t)cbRead;
-        pBuf->fEndOfStream = false;
-        Log6(("ssmR3StrmReadMore: %#010llx %#x\n", pBuf->offStream, pBuf->cb));
-        ssmR3StrmPutBuf(pStrm, pBuf);
-    }
-    else if (   (   RT_SUCCESS_NP(rc)
-                 && cbRead == 0)
-             || rc == VERR_EOF)
-    {
-        pBuf->cb           = 0;
-        pBuf->fEndOfStream = true;
-        Log6(("ssmR3StrmReadMore: %#010llx 0 EOF!\n", pBuf->offStream));
-        ssmR3StrmPutBuf(pStrm, pBuf);
-        rc = VINF_EOF;
-    }
-    else
-    {
-        Log6(("ssmR3StrmReadMore: %#010llx rc=%Rrc!\n", pBuf->offStream, rc));
-        if (ssmR3StrmSetError(pStrm, rc))
-            LogRel(("ssmR3StrmReadMore: RTFileRead(,,%#x,) -> %Rrc at offset %#llx\n",
-                    sizeof(pBuf->abData), rc, pBuf->offStream));
-        ssmR3StrmPutFreeBuf(pStrm, pBuf);
-    }
-    return rc;
-}
-
-
-/**
- * Stream input routine.
- *
- * @returns VBox status code.
- * @param   pStrm       The stream handle.
- * @param   pvBuf       Where to put what we read.
- * @param   cbToRead    How much to read.
- */
-static int ssmR3StrmRead(PSSMSTRM pStrm, void *pvBuf, size_t cbToRead)
-{
-    AssertReturn(cbToRead > 0, VINF_SUCCESS);
-    Assert(!pStrm->fWrite);
-
-    /*
-     * Read from the current buffer if we got one.
-     */
-    PSSMSTRMBUF pBuf = pStrm->pCur;
-    if (RT_LIKELY(pBuf))
-    {
-        Assert(pStrm->off <= pBuf->cb);
-        uint32_t cbLeft = pBuf->cb - pStrm->off;
-        if (cbLeft >= cbToRead)
-        {
-            memcpy(pvBuf, &pBuf->abData[pStrm->off], cbToRead);
-            pStrm->off += (uint32_t)cbToRead;
-            Assert(pStrm->off <= pBuf->cb);
-            return VINF_SUCCESS;
-        }
-        if (cbLeft)
-        {
-            memcpy(pvBuf, &pBuf->abData[pStrm->off], cbLeft);
-            pStrm->off += cbLeft;
-            cbToRead   -= cbLeft;
-            pvBuf       = (uint8_t *)pvBuf + cbLeft;
-        }
-        else if (pBuf->fEndOfStream)
-            return VERR_EOF;
-        Assert(pStrm->off == pBuf->cb);
-    }
-
-    /*
-     * Get more buffers from the stream.
-     */
-    int rc = VINF_SUCCESS;
-    do
-    {
-        /*
-         * Check for EOF first - never flush the EOF buffer.
-         */
-        if (    pBuf
-            &&  pBuf->fEndOfStream)
-            return VERR_EOF;
-
-        /*
-         * Flush the current buffer and get the next one.
-         */
-        ssmR3StrmFlushCurBuf(pStrm);
-        pBuf = ssmR3StrmGetBuf(pStrm);
-        if (!pBuf)
-        {
-            rc = pStrm->rc;
-            break;
-        }
-        pStrm->pCur = pBuf;
-        Assert(pStrm->off == 0);
-        Assert(pStrm->offCurStream == pBuf->offStream);
-        if (!pBuf->cb)
-        {
-            Assert(pBuf->fEndOfStream);
-            return VERR_EOF;
-        }
-
-        /*
-         * Read data from the buffer.
-         */
-        uint32_t cbCopy = pBuf->cb;
-        if (cbCopy > cbToRead)
-            cbCopy = (uint32_t)cbToRead;
-        memcpy(pvBuf, &pBuf->abData[0], cbCopy);
-        pStrm->off  = cbCopy;
-        cbToRead   -= cbCopy;
-        pvBuf       = (uint8_t *)pvBuf + cbCopy;
-        Assert(!pStrm->pCur || pStrm->off <= pStrm->pCur->cb);
-    } while (cbToRead > 0);
-
-    return rc;
-}
-
-
-/**
- * Reads data from the stream but instead of copying it to some output buffer
- * the caller gets a pointer to into the current stream buffer.
- *
- * The returned pointer becomes invalid after the next stream operation!
- *
- * @returns Pointer to the read data residing in the stream buffer.  NULL is
- *          returned if the request amount of data isn't available in the
- *          buffer.  The caller must fall back on ssmR3StrmRead when this
- *          happens.
- *
- * @param   pStrm       The stream handle.
- * @param   cbToRead    The number of bytes to tread.
- */
-static uint8_t const *ssmR3StrmReadDirect(PSSMSTRM pStrm, size_t cbToRead)
-{
-    AssertReturn(cbToRead > 0, VINF_SUCCESS);
-    Assert(!pStrm->fWrite);
-
-    /*
-     * Too lazy to fetch more data for the odd case that we're
-     * exactly at the boundary between two buffers.
-     */
-    PSSMSTRMBUF pBuf = pStrm->pCur;
-    if (RT_LIKELY(pBuf))
-    {
-        Assert(pStrm->off <= pBuf->cb);
-        uint32_t cbLeft = pBuf->cb - pStrm->off;
-        if (cbLeft >= cbToRead)
-        {
-            uint8_t const *pb = &pBuf->abData[pStrm->off];
-            pStrm->off += (uint32_t)cbToRead;
-            Assert(pStrm->off <= pBuf->cb);
-            return pb;
-        }
-    }
-    return NULL;
-}
-
-
-/**
- * Check that the stream is OK and flush data that is getting old
- *
- * The checking is mainly for testing for cancellation and out of space
- * conditions.
- *
- * @returns VBox status code.
- * @param   pStrm       The stream handle.
- */
-static int ssmR3StrmCheckAndFlush(PSSMSTRM pStrm)
-{
-    int rc = pStrm->pOps->pfnIsOk(pStrm->pvUser);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    if (    pStrm->fWrite
-        &&  pStrm->hIoThread != NIL_RTTHREAD
-        &&  !pStrm->pHead /* the worker is probably idle */
-        &&  pStrm->pCur
-        &&  RTTimeNanoTS() - pStrm->pCur->NanoTS > 500*1000*1000 /* 0.5s */
-       )
-        ssmR3StrmFlushCurBuf(pStrm);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Tell current stream position.
- *
- * @returns stream position.
- * @param   pStrm       The stream handle.
- */
-static uint64_t ssmR3StrmTell(PSSMSTRM pStrm)
-{
-    return pStrm->offCurStream + pStrm->off;
-}
-
-
-/**
- * Gets the intermediate stream CRC up to the current position.
- *
- * @returns CRC.
- * @param   pStrm       The stream handle.
- */
-static uint32_t ssmR3StrmCurCRC(PSSMSTRM pStrm)
-{
-    if (!pStrm->fChecksummed)
-        return 0;
-    if (pStrm->offStreamCRC < pStrm->off)
-    {
-        PSSMSTRMBUF pBuf = pStrm->pCur; Assert(pBuf);
-        pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC, &pBuf->abData[pStrm->offStreamCRC], pStrm->off - pStrm->offStreamCRC);
-        pStrm->offStreamCRC = pStrm->off;
-    }
-    else
-        Assert(pStrm->offStreamCRC == pStrm->off);
-    return pStrm->u32StreamCRC;
-}
-
-
-/**
- * Gets the final stream CRC up to the current position.
- *
- * @returns CRC.
- * @param   pStrm       The stream handle.
- */
-static uint32_t ssmR3StrmFinalCRC(PSSMSTRM pStrm)
-{
-    if (!pStrm->fChecksummed)
-        return 0;
-    return RTCrc32Finish(ssmR3StrmCurCRC(pStrm));
-}
-
-
-/**
- * Disables checksumming of the stream.
- *
- * @param   pStrm       The stream handle.
- */
-static void ssmR3StrmDisableChecksumming(PSSMSTRM pStrm)
-{
-    pStrm->fChecksummed = false;
-}
-
-
-/**
- * Used by SSMR3Seek to position the stream at the new unit.
- *
- * @returns VBox status code.
- * @param   pStrm       The strem handle.
- * @param   off         The seek offset.
- * @param   uMethod     The seek method.
- * @param   u32CurCRC   The current CRC at the seek position.
- */
-static int ssmR3StrmSeek(PSSMSTRM pStrm, int64_t off, uint32_t uMethod, uint32_t u32CurCRC)
-{
-    AssertReturn(!pStrm->fWrite, VERR_NOT_SUPPORTED);
-    AssertReturn(pStrm->hIoThread == NIL_RTTHREAD, VERR_WRONG_ORDER);
-
-    uint64_t offStream;
-    int rc = pStrm->pOps->pfnSeek(pStrm->pvUser, off, uMethod, &offStream);
-    if (RT_SUCCESS(rc))
-    {
-        pStrm->fNeedSeek    = false;
-        pStrm->offNeedSeekTo= UINT64_MAX;
-        pStrm->offCurStream = offStream;
-        pStrm->off          = 0;
-        pStrm->offStreamCRC = 0;
-        if (pStrm->fChecksummed)
-            pStrm->u32StreamCRC = u32CurCRC;
-        if (pStrm->pCur)
-        {
-            ssmR3StrmPutFreeBuf(pStrm, pStrm->pCur);
-            pStrm->pCur = NULL;
-        }
-    }
-    return rc;
-}
-
-
-/**
- * Skip some bytes in the stream.
- *
- * This is only used if someone didn't read all of their data in the V1 format,
- * so don't bother making this very efficient yet.
- *
- * @returns VBox status code.
- * @param   pStrm       The stream handle.
- * @param   offDst      The destination offset.
- */
-static int ssmR3StrmSkipTo(PSSMSTRM pStrm, uint64_t offDst)
-{
-    /* dead simple - lazy bird! */
-    for (;;)
-    {
-        uint64_t offCur = ssmR3StrmTell(pStrm);
-        AssertReturn(offCur <= offDst, VERR_INTERNAL_ERROR_4);
-        if (offCur == offDst)
-            return VINF_SUCCESS;
-
-        uint8_t abBuf[4096];
-        size_t cbToRead = RT_MIN(sizeof(abBuf), offDst - offCur);
-        int rc = ssmR3StrmRead(pStrm, abBuf, cbToRead);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-}
-
-
-/**
- * Get the size of the file.
- *
- * This does not work for non-file streams!
- *
- * @returns The file size, or UINT64_MAX if not a file stream.
- * @param   pStrm       The stream handle.
- */
-static uint64_t ssmR3StrmGetSize(PSSMSTRM pStrm)
-{
-    uint64_t cbFile;
-    int rc = pStrm->pOps->pfnSize(pStrm->pvUser, &cbFile);
-    AssertLogRelRCReturn(rc, UINT64_MAX);
-    return cbFile;
-}
-
-
-/***
- * Tests if the stream is a file stream or not.
- *
- * @returns true / false.
- * @param   pStrm       The stream handle.
- */
-static bool ssmR3StrmIsFile(PSSMSTRM pStrm)
-{
-    return pStrm->pOps == &g_ssmR3FileOps;
-}
-
-
-/**
- * Peeks at data in a file stream without buffering anything (or upsetting
- * the buffering for that matter).
- *
- * @returns VBox status code.
- * @param   pStrm       The stream handle
- * @param   off         The offset to start peeking at. Use a negative offset to
- *                      peek at something relative to the end of the file.
- * @param   pvBuf       Output buffer.
- * @param   cbToRead    How much to read.
- * @param   poff        Where to optionally store the position.  Useful when
- *                      using a negative off.
- *
- * @remarks Failures occurring while peeking will not be raised on the stream.
- */
-static int ssmR3StrmPeekAt(PSSMSTRM pStrm, RTFOFF off, void *pvBuf, size_t cbToRead, uint64_t *poff)
-{
-    AssertReturn(!pStrm->fWrite, VERR_NOT_SUPPORTED);
-    AssertReturn(pStrm->hIoThread == NIL_RTTHREAD, VERR_WRONG_ORDER);
-
-    if (!pStrm->fNeedSeek)
-    {
-        pStrm->fNeedSeek     = true;
-        pStrm->offNeedSeekTo = pStrm->offCurStream + (pStrm->pCur ? pStrm->pCur->cb : 0);
-    }
-    uint64_t offActual;
-    int rc = pStrm->pOps->pfnSeek(pStrm->pvUser, off, off >= 0 ? RTFILE_SEEK_BEGIN : RTFILE_SEEK_END, &offActual);
-    if (RT_SUCCESS(rc))
-    {
-        if (poff)
-            *poff = offActual;
-        rc = pStrm->pOps->pfnRead(pStrm->pvUser, offActual, pvBuf, cbToRead, NULL);
-    }
-
-    return rc;
-}
-
-
-/**
- * The I/O thread.
- *
- * @returns VINF_SUCCESS (ignored).
- * @param   hSelf       The thread handle.
- * @param   pvStrm      The stream handle.
- */
-static DECLCALLBACK(int) ssmR3StrmIoThread(RTTHREAD hSelf, void *pvStrm)
-{
-    PSSMSTRM pStrm = (PSSMSTRM)pvStrm;
-    ASMAtomicWriteHandle(&pStrm->hIoThread, hSelf); /* paranoia */
-
-    Log(("ssmR3StrmIoThread: starts working\n"));
-    if (pStrm->fWrite)
-    {
-        /*
-         * Write until error or terminated.
-         */
-        for (;;)
-        {
-            int rc = ssmR3StrmWriteBuffers(pStrm);
-            if (    RT_FAILURE(rc)
-                ||  rc == VINF_EOF)
-            {
-                Log(("ssmR3StrmIoThread: quitting writing with rc=%Rrc.\n", rc));
-                break;
-            }
-            if (RT_FAILURE(pStrm->rc))
-            {
-                Log(("ssmR3StrmIoThread: quitting writing with stream rc=%Rrc\n", pStrm->rc));
-                break;
-            }
-
-            if (ASMAtomicReadBool(&pStrm->fTerminating))
-            {
-                if (!ASMAtomicReadPtrT(&pStrm->pHead, PSSMSTRMBUF))
-                {
-                    Log(("ssmR3StrmIoThread: quitting writing because of pending termination.\n"));
-                    break;
-                }
-                Log(("ssmR3StrmIoThread: postponing termination because of pending buffers.\n"));
-            }
-            else if (!ASMAtomicReadPtrT(&pStrm->pHead, PSSMSTRMBUF))
-            {
-                rc = RTSemEventWait(pStrm->hEvtHead, RT_INDEFINITE_WAIT);
-                AssertLogRelRC(rc);
-            }
-        }
-
-        if (!ASMAtomicReadBool(&pStrm->fTerminating))
-            RTSemEventSignal(pStrm->hEvtFree);
-    }
-    else
-    {
-        /*
-         * Read until end of file, error or termination.
-         */
-        for (;;)
-        {
-            if (ASMAtomicReadBool(&pStrm->fTerminating))
-            {
-                Log(("ssmR3StrmIoThread: quitting reading because of pending termination.\n"));
-                break;
-            }
-
-            int rc = ssmR3StrmReadMore(pStrm);
-            if (    RT_FAILURE(rc)
-                ||  rc == VINF_EOF)
-            {
-                Log(("ssmR3StrmIoThread: quitting reading with rc=%Rrc\n", rc));
-                break;
-            }
-            if (RT_FAILURE(pStrm->rc))
-            {
-                Log(("ssmR3StrmIoThread: quitting reading with stream rc=%Rrc\n", pStrm->rc));
-                break;
-            }
-        }
-
-        if (!ASMAtomicReadBool(&pStrm->fTerminating))
-            RTSemEventSignal(pStrm->hEvtHead);
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Starts the I/O thread for the specified stream.
- *
- * @param   pStrm       The stream handle.
- */
-static void ssmR3StrmStartIoThread(PSSMSTRM pStrm)
-{
-    Assert(pStrm->hIoThread == NIL_RTTHREAD);
-
-    RTTHREAD hThread;
-    int rc = RTThreadCreate(&hThread, ssmR3StrmIoThread, pStrm, 0, RTTHREADTYPE_IO, RTTHREADFLAGS_WAITABLE, "SSM-IO");
-    AssertRCReturnVoid(rc);
-    ASMAtomicWriteHandle(&pStrm->hIoThread, hThread); /* paranoia */
-}
-
-
-/**
- * Works the progress calculation for non-live saves and restores.
- *
- * @param   pSSM        The SSM handle.
- * @param   cbAdvance   Number of bytes to advance (with in the current unit).
- */
-static void ssmR3ProgressByByte(PSSMHANDLE pSSM, uint64_t cbAdvance)
-{
-    if (!pSSM->fLiveSave)
-    {
-        /* Can't advance it beyond the estimated end of the unit. */
-        uint64_t cbLeft = pSSM->offEstUnitEnd - pSSM->offEst;
-        if (cbAdvance > cbLeft)
-            cbAdvance = cbLeft;
-        pSSM->offEst += cbAdvance;
-
-        /* uPercentPrepare% prepare, xx% exec, uPercentDone% done+crc. This is not
-           quite right for live save, but the non-live stage there is very short. */
-        while (   pSSM->offEst >= pSSM->offEstProgress
-               && pSSM->uPercent <= 100 - pSSM->uPercentDone)
-        {
-            if (pSSM->pfnProgress)
-                pSSM->pfnProgress(pSSM->pVM, pSSM->uPercent, pSSM->pvUser);
-            pSSM->uPercent++;
-            pSSM->offEstProgress = (pSSM->uPercent - pSSM->uPercentPrepare - pSSM->uPercentLive) * pSSM->cbEstTotal
-                                 / (100 - pSSM->uPercentDone - pSSM->uPercentPrepare - pSSM->uPercentLive);
-        }
-    }
-}
-
-
-/**
- * Makes the SSM operation cancellable or not (via SSMR3Cancel).
- *
- * @param   pVM             The VM handle.
- * @param   pSSM            The saved state handle. (SSMHANDLE::rc may be set.)
- * @param   fCancellable    The new state.
- */
-static void ssmR3SetCancellable(PVM pVM, PSSMHANDLE pSSM, bool fCancellable)
-{
-    RTCritSectEnter(&pVM->ssm.s.CancelCritSect);
-    if (fCancellable)
-    {
-        Assert(!pVM->ssm.s.pSSM);
-        pVM->ssm.s.pSSM = pSSM;
-    }
-    else
-    {
-        if (pVM->ssm.s.pSSM == pSSM)
-            pVM->ssm.s.pSSM = NULL;
-
-        uint32_t fCancelled = ASMAtomicUoReadU32(&pSSM->fCancelled);
-        if (    fCancelled == SSMHANDLE_CANCELLED
-            &&  RT_SUCCESS(pSSM->rc))
-            pSSM->rc = VERR_SSM_CANCELLED;
-    }
-
-    RTCritSectLeave(&pVM->ssm.s.CancelCritSect);
-}
-
-
-/**
- * Gets the host bit count of the saved state.
- *
- * Works for on both save and load handles.
- *
- * @returns 32 or 64.
- * @param   pSSM                The saved state handle.
- */
-DECLINLINE(uint32_t) ssmR3GetHostBits(PSSMHANDLE pSSM)
-{
-    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
-    {
-        uint32_t cBits = pSSM->u.Read.cHostBits;
-        if (cBits)
-            return cBits;
-    }
-    return HC_ARCH_BITS;
-}
-
-
-/**
- * Saved state origins on a host using 32-bit MSC?
- *
- * Works for on both save and load handles.
- *
- * @returns true/false.
- * @param   pSSM                The saved state handle.
- */
-DECLINLINE(bool) ssmR3IsHostMsc32(PSSMHANDLE pSSM)
-{
-    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
-        return pSSM->u.Read.fIsHostMsc32;
-    return SSM_HOST_IS_MSC_32;
-}
-
-#ifndef SSM_STANDALONE
-
-/**
- * Finishes a data unit.
- * All buffers and compressor instances are flushed and destroyed.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- */
-static int ssmR3DataWriteFinish(PSSMHANDLE pSSM)
-{
-    //Log2(("ssmR3DataWriteFinish: %#010llx start\n", ssmR3StrmTell(&pSSM->Strm)));
-    int rc = ssmR3DataFlushBuffer(pSSM);
-    if (RT_SUCCESS(rc))
-    {
-        pSSM->offUnit = UINT64_MAX;
-        return VINF_SUCCESS;
-    }
-
-    if (RT_SUCCESS(pSSM->rc))
-        pSSM->rc = rc;
-    Log2(("ssmR3DataWriteFinish: failure rc=%Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Begins writing the data of a data unit.
- *
- * Errors are signalled via pSSM->rc.
- *
- * @param   pSSM            The saved state handle.
- */
-static void ssmR3DataWriteBegin(PSSMHANDLE pSSM)
-{
-    pSSM->offUnit = 0;
-}
-
-
-/**
- * Writes a record to the current data item in the saved state file.
- *
- * @returns VBox status code. Sets pSSM->rc on failure.
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           The bits to write.
- * @param   cbBuf           The number of bytes to write.
- */
-static int ssmR3DataWriteRaw(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
-{
-    Log2(("ssmR3DataWriteRaw: %08llx|%08llx: pvBuf=%p cbBuf=%#x %.*Rhxs%s\n",
-          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pvBuf, cbBuf, RT_MIN(cbBuf, SSM_LOG_BYTES), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
-
-    /*
-     * Check that everything is fine.
-     */
-    if (RT_FAILURE(pSSM->rc))
-        return pSSM->rc;
-
-    /*
-     * Write the data item in 1MB chunks for progress indicator reasons.
-     */
-    while (cbBuf > 0)
-    {
-        size_t cbChunk = RT_MIN(cbBuf, _1M);
-        int rc = ssmR3StrmWrite(&pSSM->Strm, pvBuf, cbChunk);
-        if (RT_FAILURE(rc))
-            return rc;
-        pSSM->offUnit += cbChunk;
-        cbBuf -= cbChunk;
-        pvBuf = (char *)pvBuf + cbChunk;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Writes a record header for the specified amount of data.
- *
- * @returns VBox status code. Sets pSSM->rc on failure.
- * @param   pSSM            The saved state handle
- * @param   cb              The amount of data.
- * @param   u8TypeAndFlags  The record type and flags.
- */
-static int ssmR3DataWriteRecHdr(PSSMHANDLE pSSM, size_t cb, uint8_t u8TypeAndFlags)
-{
-    size_t  cbHdr;
-    uint8_t abHdr[8];
-    abHdr[0] = u8TypeAndFlags;
-    if (cb < 0x80)
-    {
-        cbHdr = 2;
-        abHdr[1] = (uint8_t)cb;
-    }
-    else if (cb < 0x00000800)
-    {
-        cbHdr = 3;
-        abHdr[1] = (uint8_t)(0xc0 | (cb >> 6));
-        abHdr[2] = (uint8_t)(0x80 | (cb & 0x3f));
-    }
-    else if (cb < 0x00010000)
-    {
-        cbHdr = 4;
-        abHdr[1] = (uint8_t)(0xe0 | (cb >> 12));
-        abHdr[2] = (uint8_t)(0x80 | ((cb >> 6) & 0x3f));
-        abHdr[3] = (uint8_t)(0x80 | (cb & 0x3f));
-    }
-    else if (cb < 0x00200000)
-    {
-        cbHdr = 5;
-        abHdr[1] = (uint8_t)(0xf0 |  (cb >> 18));
-        abHdr[2] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
-        abHdr[3] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
-        abHdr[4] = (uint8_t)(0x80 |  (cb        & 0x3f));
-    }
-    else if (cb < 0x04000000)
-    {
-        cbHdr = 6;
-        abHdr[1] = (uint8_t)(0xf8 |  (cb >> 24));
-        abHdr[2] = (uint8_t)(0x80 | ((cb >> 18) & 0x3f));
-        abHdr[3] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
-        abHdr[4] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
-        abHdr[5] = (uint8_t)(0x80 |  (cb        & 0x3f));
-    }
-    else if (cb <= 0x7fffffff)
-    {
-        cbHdr = 7;
-        abHdr[1] = (uint8_t)(0xfc |  (cb >> 30));
-        abHdr[2] = (uint8_t)(0x80 | ((cb >> 24) & 0x3f));
-        abHdr[3] = (uint8_t)(0x80 | ((cb >> 18) & 0x3f));
-        abHdr[4] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
-        abHdr[5] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
-        abHdr[6] = (uint8_t)(0x80 | (cb & 0x3f));
-    }
-    else
-        AssertLogRelMsgFailedReturn(("cb=%#x\n", cb), pSSM->rc = VERR_INTERNAL_ERROR);
-
-    Log3(("ssmR3DataWriteRecHdr: %08llx|%08llx/%08x: Type=%02x fImportant=%RTbool cbHdr=%u\n",
-          ssmR3StrmTell(&pSSM->Strm) + cbHdr, pSSM->offUnit + cbHdr, cb, u8TypeAndFlags & SSM_REC_TYPE_MASK, !!(u8TypeAndFlags & SSM_REC_FLAGS_IMPORTANT), cbHdr));
-
-    return ssmR3DataWriteRaw(pSSM, &abHdr[0], cbHdr);
-}
-
-
-/**
- * Worker that flushes the buffered data.
- *
- * @returns VBox status code. Will set pSSM->rc on error.
- * @param   pSSM            The saved state handle.
- */
-static int ssmR3DataFlushBuffer(PSSMHANDLE pSSM)
-{
-    /*
-     * Check how much there current is in the buffer.
-     */
-    uint32_t cb = pSSM->u.Write.offDataBuffer;
-    if (!cb)
-        return pSSM->rc;
-    pSSM->u.Write.offDataBuffer = 0;
-
-    /*
-     * Write a record header and then the data.
-     * (No need for fancy optimizations here any longer since the stream is
-     * fully buffered.)
-     */
-    int rc = ssmR3DataWriteRecHdr(pSSM, cb, SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW);
-    if (RT_SUCCESS(rc))
-        rc = ssmR3DataWriteRaw(pSSM, pSSM->u.Write.abDataBuffer, cb);
-    ssmR3ProgressByByte(pSSM, cb);
-    return rc;
-}
-
-
-/**
- * ssmR3DataWrite worker that writes big stuff.
- *
- * @returns VBox status code
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           The bits to write.
- * @param   cbBuf           The number of bytes to write.
- */
-static int ssmR3DataWriteBig(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
-{
-    int rc = ssmR3DataFlushBuffer(pSSM);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Split it up into compression blocks.
-         */
-        for (;;)
-        {
-            AssertCompile(SSM_ZIP_BLOCK_SIZE == PAGE_SIZE);
-            if (    cbBuf >= SSM_ZIP_BLOCK_SIZE
-                && (    ((uintptr_t)pvBuf & 0xf)
-                    ||  !ASMMemIsZeroPage(pvBuf))
-               )
-            {
-                /*
-                 * Compress it.
-                 */
-                AssertCompile(1 + 3 + 1 + SSM_ZIP_BLOCK_SIZE < 0x00010000);
-                uint8_t *pb;
-                rc = ssmR3StrmReserveWriteBufferSpace(&pSSM->Strm, 1 + 3 + 1 + SSM_ZIP_BLOCK_SIZE, &pb);
-                if (RT_FAILURE(rc))
-                    break;
-                size_t cbRec = SSM_ZIP_BLOCK_SIZE - (SSM_ZIP_BLOCK_SIZE / 16);
-                rc = RTZipBlockCompress(RTZIPTYPE_LZF, RTZIPLEVEL_FAST, 0 /*fFlags*/,
-                                        pvBuf, SSM_ZIP_BLOCK_SIZE,
-                                        pb + 1 + 3 + 1, cbRec, &cbRec);
-                if (RT_SUCCESS(rc))
-                {
-                    pb[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW_LZF;
-                    pb[4] = SSM_ZIP_BLOCK_SIZE / _1K;
-                    cbRec += 1;
-                }
-                else
-                {
-                    pb[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW;
-                    memcpy(&pb[4], pvBuf, SSM_ZIP_BLOCK_SIZE);
-                    cbRec = SSM_ZIP_BLOCK_SIZE;
-                }
-                pb[1] = (uint8_t)(0xe0 | ( cbRec >> 12));
-                pb[2] = (uint8_t)(0x80 | ((cbRec >>  6) & 0x3f));
-                pb[3] = (uint8_t)(0x80 | ( cbRec        & 0x3f));
-                cbRec += 1 + 3;
-                rc = ssmR3StrmCommitWriteBufferSpace(&pSSM->Strm, cbRec);
-                if (RT_FAILURE(rc))
-                    break;
-
-                pSSM->offUnit += cbRec;
-                ssmR3ProgressByByte(pSSM, SSM_ZIP_BLOCK_SIZE);
-
-                /* advance */
-                if (cbBuf == SSM_ZIP_BLOCK_SIZE)
-                    return VINF_SUCCESS;
-                cbBuf -= SSM_ZIP_BLOCK_SIZE;
-                pvBuf = (uint8_t const*)pvBuf + SSM_ZIP_BLOCK_SIZE;
-            }
-            else if (cbBuf >= SSM_ZIP_BLOCK_SIZE)
-            {
-                /*
-                 * Zero block.
-                 */
-                uint8_t abRec[3];
-                abRec[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW_ZERO;
-                abRec[1] = 1;
-                abRec[2] = SSM_ZIP_BLOCK_SIZE / _1K;
-                Log3(("ssmR3DataWriteBig: %08llx|%08llx/%08x: ZERO\n", ssmR3StrmTell(&pSSM->Strm) + 2, pSSM->offUnit + 2, 1));
-                rc = ssmR3DataWriteRaw(pSSM, &abRec[0], sizeof(abRec));
-                if (RT_FAILURE(rc))
-                    break;
-
-                /* advance */
-                ssmR3ProgressByByte(pSSM, SSM_ZIP_BLOCK_SIZE);
-                if (cbBuf == SSM_ZIP_BLOCK_SIZE)
-                    return VINF_SUCCESS;
-                cbBuf -= SSM_ZIP_BLOCK_SIZE;
-                pvBuf = (uint8_t const*)pvBuf + SSM_ZIP_BLOCK_SIZE;
-            }
-            else
-            {
-                /*
-                 * Less than one block left, store it the simple way.
-                 */
-                rc = ssmR3DataWriteRecHdr(pSSM, cbBuf, SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW);
-                if (RT_SUCCESS(rc))
-                    rc = ssmR3DataWriteRaw(pSSM, pvBuf, cbBuf);
-                ssmR3ProgressByByte(pSSM, cbBuf);
-                break;
-            }
-        }
-    }
-    return rc;
-}
-
-
-/**
- * ssmR3DataWrite worker that is called when there isn't enough room in the
- * buffer for the current chunk of data.
- *
- * This will first flush the buffer and then add the new bits to it.
- *
- * @returns VBox status code
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           The bits to write.
- * @param   cbBuf           The number of bytes to write.
- */
-static int ssmR3DataWriteFlushAndBuffer(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
-{
-    int rc = ssmR3DataFlushBuffer(pSSM);
-    if (RT_SUCCESS(rc))
-    {
-        memcpy(&pSSM->u.Write.abDataBuffer[0], pvBuf, cbBuf);
-        pSSM->u.Write.offDataBuffer = (uint32_t)cbBuf;
-    }
-    return rc;
-}
-
-
-/**
- * Writes data to the current data unit.
- *
- * This is an inlined wrapper that optimizes the small writes that so many of
- * the APIs make.
- *
- * @returns VBox status code
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           The bits to write.
- * @param   cbBuf           The number of bytes to write.
- */
-DECLINLINE(int) ssmR3DataWrite(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
-{
-    if (cbBuf > sizeof(pSSM->u.Write.abDataBuffer) / 8)
-        return ssmR3DataWriteBig(pSSM, pvBuf, cbBuf);
-    if (!cbBuf)
-        return VINF_SUCCESS;
-
-    uint32_t off = pSSM->u.Write.offDataBuffer;
-    if (RT_UNLIKELY(cbBuf + off > sizeof(pSSM->u.Write.abDataBuffer)))
-        return ssmR3DataWriteFlushAndBuffer(pSSM, pvBuf, cbBuf);
-
-    memcpy(&pSSM->u.Write.abDataBuffer[off], pvBuf, cbBuf);
-    pSSM->u.Write.offDataBuffer = off + (uint32_t)cbBuf;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Puts a structure.
- *
- * @returns VBox status code.
- * @param   pSSM            The saved state handle.
- * @param   pvStruct        The structure address.
- * @param   paFields        The array of structure fields descriptions.
- *                          The array must be terminated by a SSMFIELD_ENTRY_TERM().
- */
-VMMR3DECL(int) SSMR3PutStruct(PSSMHANDLE pSSM, const void *pvStruct, PCSSMFIELD paFields)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    AssertPtr(pvStruct);
-    AssertPtr(paFields);
-
-    /* begin marker. */
-    int rc = SSMR3PutU32(pSSM, SSMR3STRUCT_BEGIN);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /* put the fields */
-    for (PCSSMFIELD pCur = paFields;
-         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
-         pCur++)
-    {
-        uint8_t const *pbField = (uint8_t const *)pvStruct + pCur->off;
-        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
-        {
-            case SSMFIELDTRANS_NO_TRANSFORMATION:
-                rc = ssmR3DataWrite(pSSM, pbField, pCur->cb);
-                break;
-
-            case SSMFIELDTRANS_GCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3PutGCPtr(pSSM, *(PRTGCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_GCPHYS:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3PutGCPhys(pSSM, *(PRTGCPHYS)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3PutRCPtr(pSSM, *(PRTRCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR_ARRAY:
-            {
-                uint32_t const cEntries = pCur->cb / sizeof(RTRCPTR);
-                AssertMsgReturn(pCur->cb == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = VINF_SUCCESS;
-                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
-                    rc = SSMR3PutRCPtr(pSSM, ((PRTRCPTR)pbField)[i]);
-                break;
-            }
-
-            default:
-                AssertMsgFailedReturn(("%#x\n", pCur->pfnGetPutOrTransformer), VERR_SSM_FIELD_COMPLEX);
-        }
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /* end marker */
-    return SSMR3PutU32(pSSM, SSMR3STRUCT_END);
-}
-
-
-/**
- * SSMR3PutStructEx helper that puts a HCPTR that is used as a NULL indicator.
- *
- * @returns VBox status code.
- *
- * @param   pSSM            The saved state handle.
- * @param   pv              The value to put.
- * @param   fFlags          SSMSTRUCT_FLAGS_XXX.
- */
-DECLINLINE(int) ssmR3PutHCPtrNI(PSSMHANDLE pSSM, void *pv, uint32_t fFlags)
-{
-    int rc;
-    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-        rc = ssmR3DataWrite(pSSM, &pv, sizeof(void *));
-    else
-        rc = SSMR3PutBool(pSSM, pv != NULL);
-    return rc;
-}
-
-
-/**
- * SSMR3PutStructEx helper that puts an arbitrary number of zeros.
- *
- * @returns VBox status code.
- * @param   pSSM            The saved state handle.
- * @param   cbToFill        The number of zeros to stuff into the state.
- */
-static int ssmR3PutZeros(PSSMHANDLE pSSM, uint32_t cbToFill)
-{
-    while (cbToFill > 0)
-    {
-        uint32_t cb = RT_MIN(sizeof(g_abZero), cbToFill);
-        int rc = ssmR3DataWrite(pSSM, g_abZero, cb);
-        if (RT_FAILURE(rc))
-            return rc;
-        cbToFill -= cb;
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Puts a structure, extended API.
- *
- * @returns VBox status code.
- * @param   pSSM            The saved state handle.
- * @param   pvStruct        The structure address.
- * @param   cbStruct        The size of the struct (use for validation only).
- * @param   fFlags          Combination of SSMSTRUCT_FLAGS_XXX defines.
- * @param   paFields        The array of structure fields descriptions. The
- *                          array must be terminated by a SSMFIELD_ENTRY_TERM().
- * @param   pvUser          User argument for any callbacks that paFields might
- *                          contain.
- */
-VMMR3DECL(int) SSMR3PutStructEx(PSSMHANDLE pSSM, const void *pvStruct, size_t cbStruct,
-                                uint32_t fFlags, PCSSMFIELD paFields, void *pvUser)
-{
-    int rc;
-
-    /*
-     * Validation.
-     */
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    AssertMsgReturn(!(fFlags & ~SSMSTRUCT_FLAGS_VALID_MASK), ("%#x\n", fFlags), VERR_INVALID_PARAMETER);
-    AssertPtr(pvStruct);
-    AssertPtr(paFields);
-
-
-    /*
-     * Begin marker.
-     */
-    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
-    {
-        rc = SSMR3PutU32(pSSM, SSMR3STRUCT_BEGIN);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Put the fields
-     */
-    uint32_t    off          = 0;
-    for (PCSSMFIELD pCur = paFields;
-         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
-         pCur++)
-    {
-        uint32_t const offField =    (!SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer) || pCur->off != UINT32_MAX / 2)
-                                  &&  !SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
-                                ? pCur->off
-                                : off;
-        uint32_t const cbField  = SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
-                                ? 0
-                                : SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer)
-                                ? RT_HIWORD(pCur->cb)
-                                : pCur->cb;
-        AssertMsgReturn(   cbField            <= cbStruct
-                        && offField + cbField <= cbStruct
-                        && offField + cbField >= offField,
-                        ("off=%#x cb=%#x cbStruct=%#x (%s)\n", cbField, offField, cbStruct, pCur->pszName),
-                        VERR_SSM_FIELD_OUT_OF_BOUNDS);
-        AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
-                        || off == offField,
-                        ("off=%#x offField=%#x (%s)\n", off, offField, pCur->pszName),
-                        VERR_SSM_FIELD_NOT_CONSECUTIVE);
-
-        rc = VINF_SUCCESS;
-        uint8_t const *pbField = (uint8_t const *)pvStruct + offField;
-        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
-        {
-            case SSMFIELDTRANS_NO_TRANSFORMATION:
-                rc = ssmR3DataWrite(pSSM, pbField, cbField);
-                break;
-
-            case SSMFIELDTRANS_GCPHYS:
-                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3PutGCPhys(pSSM, *(PRTGCPHYS)pbField);
-                break;
-
-            case SSMFIELDTRANS_GCPTR:
-                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3PutGCPtr(pSSM, *(PRTGCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR:
-                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3PutRCPtr(pSSM, *(PRTRCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR_ARRAY:
-            {
-                uint32_t const cEntries = cbField / sizeof(RTRCPTR);
-                AssertMsgReturn(cbField == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
-                    rc = SSMR3PutRCPtr(pSSM, ((PRTRCPTR)pbField)[i]);
-                break;
-            }
-
-            case SSMFIELDTRANS_HCPTR_NI:
-                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3PutHCPtrNI(pSSM, *(void * const *)pbField, fFlags);
-                break;
-
-            case SSMFIELDTRANS_HCPTR_NI_ARRAY:
-            {
-                uint32_t const cEntries = cbField / sizeof(void *);
-                AssertMsgReturn(cbField == cEntries * sizeof(void *) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
-                    rc = ssmR3PutHCPtrNI(pSSM, ((void * const *)pbField)[i], fFlags);
-                break;
-            }
-
-            case SSMFIELDTRANS_HCPTR_HACK_U32:
-                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                AssertMsgReturn(*(uintptr_t *)pbField <= UINT32_MAX, ("%p (%s)\n", *(uintptr_t *)pbField, pCur->pszName), VERR_SSM_FIELD_INVALID_VALUE);
-                rc = ssmR3DataWrite(pSSM, pbField, sizeof(uint32_t));
-                if ((fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE) && sizeof(void *) != sizeof(uint32_t))
-                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(uint32_t));
-                break;
-
-
-            case SSMFIELDTRANS_IGNORE:
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = ssmR3PutZeros(pSSM, cbField);
-                break;
-
-            case SSMFIELDTRANS_IGN_GCPHYS:
-                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPHYS));
-                break;
-
-            case SSMFIELDTRANS_IGN_GCPTR:
-                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPTR));
-                break;
-
-            case SSMFIELDTRANS_IGN_RCPTR:
-                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTRCPTR));
-                break;
-
-            case SSMFIELDTRANS_IGN_HCPTR:
-                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(void *));
-                break;
-
-
-            case SSMFIELDTRANS_OLD:
-                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3PutZeros(pSSM, pCur->cb);
-                break;
-
-            case SSMFIELDTRANS_OLD_GCPHYS:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPHYS));
-                break;
-
-            case SSMFIELDTRANS_OLD_GCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPTR));
-                break;
-
-            case SSMFIELDTRANS_OLD_RCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTRCPTR));
-                break;
-
-            case SSMFIELDTRANS_OLD_HCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(void *) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(void *));
-                break;
-
-            case SSMFIELDTRANS_OLD_PAD_HC:
-                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3PutZeros(pSSM, HC_ARCH_BITS == 64 ? RT_HIWORD(pCur->cb) : RT_LOWORD(pCur->cb));
-                break;
-
-            case SSMFIELDTRANS_OLD_PAD_MSC32:
-                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (SSM_HOST_IS_MSC_32)
-                    rc = ssmR3PutZeros(pSSM, pCur->cb);
-                break;
-
-
-            case SSMFIELDTRANS_PAD_HC:
-            case SSMFIELDTRANS_PAD_HC32:
-            case SSMFIELDTRANS_PAD_HC64:
-            case SSMFIELDTRANS_PAD_HC_AUTO:
-            case SSMFIELDTRANS_PAD_MSC32_AUTO:
-            {
-                uint32_t cb32    = RT_BYTE1(pCur->cb);
-                uint32_t cb64    = RT_BYTE2(pCur->cb);
-                uint32_t cbCtx   =    HC_ARCH_BITS == 64
-                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
-                                       && !SSM_HOST_IS_MSC_32)
-                                 ? cb64 : cb32;
-                uint32_t cbSaved =    ssmR3GetHostBits(pSSM) == 64
-                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
-                                       && !ssmR3IsHostMsc32(pSSM))
-                                 ? cb64 : cb32;
-                AssertMsgReturn(    cbField == cbCtx
-                                &&  (   (   pCur->off == UINT32_MAX / 2
-                                         && (   cbField == 0
-                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_HC_AUTO
-                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
-                                            )
-                                        )
-                                     || (pCur->off != UINT32_MAX / 2 && cbField != 0)
-                                     )
-                                , ("cbField=%#x cb32=%#x cb64=%#x HC_ARCH_BITS=%u cbCtx=%#x cbSaved=%#x off=%#x\n",
-                                   cbField, cb32, cb64, HC_ARCH_BITS, cbCtx, cbSaved, pCur->off),
-                                VERR_SSM_FIELD_INVALID_PADDING_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = ssmR3PutZeros(pSSM, cbSaved);
-                break;
-            }
-
-            default:
-                AssertPtrReturn(pCur->pfnGetPutOrTransformer, VERR_SSM_FIELD_INVALID_CALLBACK);
-                rc = pCur->pfnGetPutOrTransformer(pSSM, pCur, (void *)pvStruct, fFlags, false /*fGetOrPut*/, pvUser);
-                break;
-        }
-        if (RT_FAILURE(rc))
-            return rc;
-
-        off = offField + cbField;
-    }
-    AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
-                    || off == cbStruct,
-                    ("off=%#x cbStruct=%#x\n", off, cbStruct),
-                    VERR_SSM_FIELD_NOT_CONSECUTIVE);
-
-    /*
-     * End marker
-     */
-    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
-    {
-        rc = SSMR3PutU32(pSSM, SSMR3STRUCT_END);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Saves a boolean item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   fBool           Item to save.
- */
-VMMR3DECL(int) SSMR3PutBool(PSSMHANDLE pSSM, bool fBool)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    uint8_t u8 = fBool; /* enforce 1 byte size */
-    return ssmR3DataWrite(pSSM, &u8, sizeof(u8));
-}
-
-
-/**
- * Saves a 8-bit unsigned integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u8              Item to save.
- */
-VMMR3DECL(int) SSMR3PutU8(PSSMHANDLE pSSM, uint8_t u8)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u8, sizeof(u8));
-}
-
-
-/**
- * Saves a 8-bit signed integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   i8              Item to save.
- */
-VMMR3DECL(int) SSMR3PutS8(PSSMHANDLE pSSM, int8_t i8)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &i8, sizeof(i8));
-}
-
-
-/**
- * Saves a 16-bit unsigned integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u16             Item to save.
- */
-VMMR3DECL(int) SSMR3PutU16(PSSMHANDLE pSSM, uint16_t u16)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u16, sizeof(u16));
-}
-
-
-/**
- * Saves a 16-bit signed integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   i16             Item to save.
- */
-VMMR3DECL(int) SSMR3PutS16(PSSMHANDLE pSSM, int16_t i16)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &i16, sizeof(i16));
-}
-
-
-/**
- * Saves a 32-bit unsigned integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u32             Item to save.
- */
-VMMR3DECL(int) SSMR3PutU32(PSSMHANDLE pSSM, uint32_t u32)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u32, sizeof(u32));
-}
-
-
-/**
- * Saves a 32-bit signed integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   i32             Item to save.
- */
-VMMR3DECL(int) SSMR3PutS32(PSSMHANDLE pSSM, int32_t i32)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &i32, sizeof(i32));
-}
-
-
-/**
- * Saves a 64-bit unsigned integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u64             Item to save.
- */
-VMMR3DECL(int) SSMR3PutU64(PSSMHANDLE pSSM, uint64_t u64)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u64, sizeof(u64));
-}
-
-
-/**
- * Saves a 64-bit signed integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   i64             Item to save.
- */
-VMMR3DECL(int) SSMR3PutS64(PSSMHANDLE pSSM, int64_t i64)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &i64, sizeof(i64));
-}
-
-
-/**
- * Saves a 128-bit unsigned integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u128            Item to save.
- */
-VMMR3DECL(int) SSMR3PutU128(PSSMHANDLE pSSM, uint128_t u128)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u128, sizeof(u128));
-}
-
-
-/**
- * Saves a 128-bit signed integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   i128            Item to save.
- */
-VMMR3DECL(int) SSMR3PutS128(PSSMHANDLE pSSM, int128_t i128)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &i128, sizeof(i128));
-}
-
-
-/**
- * Saves a VBox unsigned integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u               Item to save.
- */
-VMMR3DECL(int) SSMR3PutUInt(PSSMHANDLE pSSM, RTUINT u)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u, sizeof(u));
-}
-
-
-/**
- * Saves a VBox signed integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   i               Item to save.
- */
-VMMR3DECL(int) SSMR3PutSInt(PSSMHANDLE pSSM, RTINT i)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &i, sizeof(i));
-}
-
-
-/**
- * Saves a GC natural unsigned integer item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u               Item to save.
- *
- * @deprecated Silly type, don't use it.
- */
-VMMR3DECL(int) SSMR3PutGCUInt(PSSMHANDLE pSSM, RTGCUINT u)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u, sizeof(u));
-}
-
-
-/**
- * Saves a GC unsigned integer register item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   u               Item to save.
- */
-VMMR3DECL(int) SSMR3PutGCUIntReg(PSSMHANDLE pSSM, RTGCUINTREG u)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &u, sizeof(u));
-}
-
-
-/**
- * Saves a 32 bits GC physical address item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   GCPhys          The item to save
- */
-VMMR3DECL(int) SSMR3PutGCPhys32(PSSMHANDLE pSSM, RTGCPHYS32 GCPhys)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
-}
-
-
-/**
- * Saves a 64 bits GC physical address item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   GCPhys          The item to save
- */
-VMMR3DECL(int) SSMR3PutGCPhys64(PSSMHANDLE pSSM, RTGCPHYS64 GCPhys)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
-}
-
-
-/**
- * Saves a GC physical address item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   GCPhys          The item to save
- */
-VMMR3DECL(int) SSMR3PutGCPhys(PSSMHANDLE pSSM, RTGCPHYS GCPhys)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
-}
-
-
-/**
- * Saves a GC virtual address item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   GCPtr           The item to save.
- */
-VMMR3DECL(int) SSMR3PutGCPtr(PSSMHANDLE pSSM, RTGCPTR GCPtr)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &GCPtr, sizeof(GCPtr));
-}
-
-
-/**
- * Saves an RC virtual address item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   RCPtr           The item to save.
- */
-VMMR3DECL(int) SSMR3PutRCPtr(PSSMHANDLE pSSM, RTRCPTR RCPtr)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &RCPtr, sizeof(RCPtr));
-}
-
-
-/**
- * Saves a GC virtual address (represented as an unsigned integer) item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   GCPtr           The item to save.
- */
-VMMR3DECL(int) SSMR3PutGCUIntPtr(PSSMHANDLE pSSM, RTGCUINTPTR GCPtr)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &GCPtr, sizeof(GCPtr));
-}
-
-
-/**
- * Saves a I/O port address item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   IOPort          The item to save.
- */
-VMMR3DECL(int) SSMR3PutIOPort(PSSMHANDLE pSSM, RTIOPORT IOPort)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &IOPort, sizeof(IOPort));
-}
-
-
-/**
- * Saves a selector item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   Sel             The item to save.
- */
-VMMR3DECL(int) SSMR3PutSel(PSSMHANDLE pSSM, RTSEL Sel)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, &Sel, sizeof(Sel));
-}
-
-
-/**
- * Saves a memory item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pv              Item to save.
- * @param   cb              Size of the item.
- */
-VMMR3DECL(int) SSMR3PutMem(PSSMHANDLE pSSM, const void *pv, size_t cb)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataWrite(pSSM, pv, cb);
-}
-
-
-/**
- * Saves a zero terminated string item to the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   psz             Item to save.
- */
-VMMR3DECL(int) SSMR3PutStrZ(PSSMHANDLE pSSM, const char *psz)
-{
-    SSM_ASSERT_WRITEABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-
-    size_t cch = strlen(psz);
-    if (cch > _1M)
-    {
-        AssertMsgFailed(("a %zu byte long string, what's this!?!\n", cch));
-        return VERR_TOO_MUCH_DATA;
-    }
-    uint32_t u32 = (uint32_t)cch;
-    int rc = ssmR3DataWrite(pSSM, &u32, sizeof(u32));
-    if (rc)
-        return rc;
-    return ssmR3DataWrite(pSSM, psz, cch);
-}
-
-
-/**
- * Emits a SSMLiveControl unit with a new progress report.
- *
- * @returns VBox status code.
- * @param   pSSM                    The saved state handle.
- * @param   lrdPct                  The progress of the the live save.
- * @param   uPass                   The current pass.
- */
-static int ssmR3LiveControlEmit(PSSMHANDLE pSSM, long double lrdPct, uint32_t uPass)
-{
-    AssertMsg(lrdPct <= 100.0, ("%u\n", lrdPct * 100));
-
-    /*
-     * Make sure we're in one of the two EXEC states or we may fail.
-     */
-    SSMSTATE enmSavedState = pSSM->enmOp;
-    if (enmSavedState == SSMSTATE_LIVE_VOTE)
-        pSSM->enmOp = SSMSTATE_LIVE_EXEC;
-    else if (enmSavedState == SSMSTATE_SAVE_DONE)
-        pSSM->enmOp = SSMSTATE_SAVE_EXEC;
-
-    /*
-     * Write the unit header.
-     */
-    SSMFILEUNITHDRV2 UnitHdr;
-    memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
-    UnitHdr.offStream       = ssmR3StrmTell(&pSSM->Strm);
-    UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
-    UnitHdr.u32CRC          = 0;
-    UnitHdr.u32Version      = 1;
-    UnitHdr.u32Instance     = 0;
-    UnitHdr.u32Pass         = uPass;
-    UnitHdr.fFlags          = 0;
-    UnitHdr.cbName          = sizeof("SSMLiveControl");
-    memcpy(&UnitHdr.szName[0], "SSMLiveControl", UnitHdr.cbName);
-    UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
-    Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
-         UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
-    int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Write the payload.
-         */
-        ssmR3DataWriteBegin(pSSM);
-
-        uint16_t u16PartsPerTenThousand = (uint16_t)(lrdPct * (100 - pSSM->uPercentDone));
-        AssertMsg(u16PartsPerTenThousand <= 10000, ("%u\n", u16PartsPerTenThousand));
-        ssmR3DataWrite(pSSM, &u16PartsPerTenThousand, sizeof(u16PartsPerTenThousand));
-
-        rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Write the termination record and flush the compression stream.
-             */
-            SSMRECTERM TermRec;
-            TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
-            TermRec.cbRec            = sizeof(TermRec) - 2;
-            if (pSSM->Strm.fChecksummed)
-            {
-                TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
-                TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
-            }
-            else
-            {
-                TermRec.fFlags       = 0;
-                TermRec.u32StreamCRC = 0;
-            }
-            TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
-            rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
-            if (RT_SUCCESS(rc))
-                rc = ssmR3DataWriteFinish(pSSM);
-            if (RT_SUCCESS(rc))
-            {
-                pSSM->enmOp = enmSavedState;
-                return rc;
-            }
-        }
-    }
-
-    LogRel(("SSM: Failed to write live control unit. rc=%Rrc\n", rc));
-    if (RT_SUCCESS_NP(pSSM->rc))
-        pSSM->rc = rc;
-    pSSM->enmOp = enmSavedState;
-    return rc;
-}
-
-
-/**
- * Do the pfnSaveDone run.
- *
- * @returns VBox status code (pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3SaveDoDoneRun(PVM pVM, PSSMHANDLE pSSM)
-{
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * Do the done run.
-     */
-    pSSM->enmOp = SSMSTATE_SAVE_DONE;
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-    {
-        if (    pUnit->u.Common.pfnSaveDone
-            &&  (   pUnit->fCalled
-                 || (!pUnit->u.Common.pfnSavePrep && !pUnit->u.Common.pfnSaveExec)))
-        {
-            int rcOld = pSSM->rc;
-            int rc;
-            switch (pUnit->enmType)
-            {
-                case SSMUNITTYPE_DEV:
-                    rc = pUnit->u.Dev.pfnSaveDone(pUnit->u.Dev.pDevIns, pSSM);
-                    break;
-                case SSMUNITTYPE_DRV:
-                    rc = pUnit->u.Drv.pfnSaveDone(pUnit->u.Drv.pDrvIns, pSSM);
-                    break;
-                case SSMUNITTYPE_INTERNAL:
-                    rc = pUnit->u.Internal.pfnSaveDone(pVM, pSSM);
-                    break;
-                case SSMUNITTYPE_EXTERNAL:
-                    rc = pUnit->u.External.pfnSaveDone(pSSM, pUnit->u.External.pvUser);
-                    break;
-                default:
-                    rc = VERR_INTERNAL_ERROR;
-                    break;
-            }
-            if (RT_SUCCESS(rc) && pSSM->rc != rcOld)
-                rc = pSSM->rc;
-            if (RT_FAILURE(rc))
-            {
-                LogRel(("SSM: Done save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
-                if (RT_SUCCESS_NP(pSSM->rc))
-                    pSSM->rc = rc;
-            }
-        }
-    }
-    return pSSM->rc;
-}
-
-
-/**
- * Worker for SSMR3LiveDone and SSMR3Save that closes the handle and deletes the
- * saved state file on failure.
- *
- * @returns VBox status code (pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3SaveDoClose(PVM pVM, PSSMHANDLE pSSM)
-{
-    VM_ASSERT_EMT0(pVM);
-    pVM->ssm.s.uPass = 0;
-
-    /*
-     * Make it non-cancellable, close the stream and delete the file on failure.
-     */
-    ssmR3SetCancellable(pVM, pSSM, false);
-    int rc = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
-    if (RT_SUCCESS(rc))
-        rc = pSSM->rc;
-    if (RT_SUCCESS(rc))
-    {
-        Assert(pSSM->enmOp == SSMSTATE_SAVE_DONE);
-        if (pSSM->pfnProgress)
-            pSSM->pfnProgress(pVM, 100, pSSM->pvUser);
-        LogRel(("SSM: Successfully saved the VM state to '%s'\n",
-                pSSM->pszFilename ? pSSM->pszFilename : "<remote-machine>"));
-    }
-    else
-    {
-        if (pSSM->pszFilename)
-        {
-            int rc2 = RTFileDelete(pSSM->pszFilename);
-            AssertRC(rc2);
-            if (RT_SUCCESS(rc2))
-                LogRel(("SSM: Failed to save the VM state to '%s' (file deleted): %Rrc\n",
-                        pSSM->pszFilename, rc));
-            else
-                LogRel(("SSM: Failed to save the VM state to '%s' (file deletion failed, rc2=%Rrc): %Rrc\n",
-                        pSSM->pszFilename, rc2, rc));
-        }
-        else
-            LogRel(("SSM: Failed to save the VM state.\n"));
-
-        Assert(pSSM->enmOp <= SSMSTATE_SAVE_DONE);
-        if (pSSM->enmOp != SSMSTATE_SAVE_DONE)
-            ssmR3SaveDoDoneRun(pVM, pSSM);
-    }
-
-    /*
-     * Trash the handle before freeing it.
-     */
-    ASMAtomicWriteU32(&pSSM->fCancelled, 0);
-    pSSM->pVM = NULL;
-    pSSM->enmAfter = SSMAFTER_INVALID;
-    pSSM->enmOp = SSMSTATE_INVALID;
-    RTMemFree(pSSM);
-
-    return rc;
-}
-
-
-/**
- * Closes the SSM handle.
- *
- * This must always be called on a handled returned by SSMR3LiveSave.
- *
- * @returns VBox status.
- *
- * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
- *
- * @thread  EMT(0).
- */
-VMMR3_INT_DECL(int) SSMR3LiveDone(PSSMHANDLE pSSM)
-{
-    LogFlow(("SSMR3LiveDone: pSSM=%p\n", pSSM));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
-    PVM pVM = pSSM->pVM;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    VM_ASSERT_EMT0(pVM);
-    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
-                    || pSSM->enmAfter == SSMAFTER_CONTINUE
-                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
-                    ("%d\n", pSSM->enmAfter),
-                    VERR_INVALID_PARAMETER);
-    AssertMsgReturn(    pSSM->enmOp >= SSMSTATE_LIVE_PREP
-                    &&  pSSM->enmOp <= SSMSTATE_SAVE_DONE,
-                    ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
-
-    /*
-     * Join paths with SSMR3Save again.
-     */
-    return ssmR3SaveDoClose(pVM, pSSM);
-}
-
-
-/**
- * Writes the directory.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- * @param   pcEntries           Where to return the number of directory entries.
- */
-static int ssmR3WriteDirectory(PVM pVM, PSSMHANDLE pSSM, uint32_t *pcEntries)
-{
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * Grab some temporary memory for the dictionary.
-     */
-    size_t      cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[pVM->ssm.s.cUnits]);
-    PSSMFILEDIR pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
-    if (!pDir)
-    {
-        LogRel(("ssmR3WriteDirectory: failed to allocate %zu bytes!\n", cbDir));
-        return VERR_NO_TMP_MEMORY;
-    }
-
-    /*
-     * Initialize it.
-     */
-    memcpy(pDir->szMagic, SSMFILEDIR_MAGIC, sizeof(pDir->szMagic));
-    pDir->u32CRC   = 0;
-    pDir->cEntries = 0;
-
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-        if (pUnit->offStream != RTFOFF_MIN)
-        {
-            PSSMFILEDIRENTRY pEntry = &pDir->aEntries[pDir->cEntries++];
-            Assert(pDir->cEntries <= pVM->ssm.s.cUnits);
-            Assert(pUnit->offStream >= (RTFOFF)sizeof(SSMFILEHDR));
-            pEntry->off         = pUnit->offStream;
-            pEntry->u32Instance = pUnit->u32Instance;
-            pEntry->u32NameCRC  = RTCrc32(pUnit->szName, pUnit->cchName);
-        }
-
-    /*
-     * Calculate the actual size and CRC-32, then write the directory
-     * out to the stream.
-     */
-    *pcEntries = pDir->cEntries;
-    cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[pDir->cEntries]);
-    pDir->u32CRC = RTCrc32(pDir, cbDir);
-    int rc = ssmR3StrmWrite(&pSSM->Strm, pDir, cbDir);
-    RTMemTmpFree(pDir);
-    return rc;
-}
-
-
-/**
- * Finalize the saved state stream, i.e. add the end unit, directory
- * and footer.
- *
- * @returns VBox status code (pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3SaveDoFinalization(PVM pVM, PSSMHANDLE pSSM)
-{
-    VM_ASSERT_EMT0(pVM);
-    Assert(RT_SUCCESS(pSSM->rc));
-
-    /*
-     * Write the end unit.
-     */
-    SSMFILEUNITHDRV2 UnitHdr;
-    memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END, sizeof(UnitHdr.szMagic));
-    UnitHdr.offStream       = ssmR3StrmTell(&pSSM->Strm);
-    UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
-    UnitHdr.u32CRC          = 0;
-    UnitHdr.u32Version      = 0;
-    UnitHdr.u32Instance     = 0;
-    UnitHdr.u32Pass         = SSM_PASS_FINAL;
-    UnitHdr.fFlags          = 0;
-    UnitHdr.cbName          = 0;
-    UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[0]));
-    Log(("SSM: Unit at %#9llx: END UNIT\n", UnitHdr.offStream));
-    int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[0]));
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("SSM: Failed writing the end unit: %Rrc\n", rc));
-        return pSSM->rc = rc;
-    }
-
-    /*
-     * Write the directory for the final units and then the footer.
-     */
-    SSMFILEFTR Footer;
-    rc = ssmR3WriteDirectory(pVM, pSSM, &Footer.cDirEntries);
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("SSM: Failed writing the directory: %Rrc\n", rc));
-        return pSSM->rc = rc;
-    }
-
-    memcpy(Footer.szMagic, SSMFILEFTR_MAGIC, sizeof(Footer.szMagic));
-    Footer.offStream    = ssmR3StrmTell(&pSSM->Strm);
-    Footer.u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
-    Footer.u32Reserved  = 0;
-    Footer.u32CRC       = 0;
-    Footer.u32CRC       = RTCrc32(&Footer, sizeof(Footer));
-    Log(("SSM: Footer at %#9llx: \n", Footer.offStream));
-    rc = ssmR3StrmWrite(&pSSM->Strm, &Footer, sizeof(Footer));
-    if (RT_SUCCESS(rc))
-        rc = ssmR3StrmSetEnd(&pSSM->Strm);
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("SSM: Failed writing the footer: %Rrc\n", rc));
-        return pSSM->rc = rc;
-    }
-
-    LogRel(("SSM: Footer at %#llx (%lld), %u directory entries.\n",
-            Footer.offStream, Footer.offStream, Footer.cDirEntries));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Works the progress calculation during the exec part of a live save.
- *
- * @param   pSSM        The SSM handle.
- * @param   iUnit       The current unit number.
- */
-static void ssmR3ProgressByUnit(PSSMHANDLE pSSM, uint32_t iUnit)
-{
-    if (pSSM->fLiveSave)
-    {
-        unsigned    uPctExec = iUnit * 100 / pSSM->pVM->ssm.s.cUnits;
-        unsigned    cPctExec = 100 - pSSM->uPercentDone - pSSM->uPercentPrepare - pSSM->uPercentLive;
-        long double lrdPct   = (long double)uPctExec * cPctExec / 100 + pSSM->uPercentPrepare + pSSM->uPercentLive;
-        unsigned    uPct     = (unsigned)lrdPct;
-        if (uPct != pSSM->uPercent)
-        {
-            ssmR3LiveControlEmit(pSSM, lrdPct, SSM_PASS_FINAL);
-            pSSM->uPercent =  uPct;
-            pSSM->pfnProgress(pSSM->pVM, uPct, pSSM->pvUser);
-        }
-    }
-}
-
-
-/**
- * Do the pfnSaveExec run.
- *
- * @returns VBox status code (pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3SaveDoExecRun(PVM pVM, PSSMHANDLE pSSM)
-{
-    VM_ASSERT_EMT0(pVM);
-    AssertRC(pSSM->rc);
-    pSSM->rc = VINF_SUCCESS;
-    pSSM->enmOp = SSMSTATE_SAVE_EXEC;
-    unsigned iUnit = 0;
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext, iUnit++)
-    {
-        /*
-         * Not all unit have a callback. Skip those which don't and
-         * make sure to keep the progress indicator up to date.
-         */
-        ssmR3ProgressByUnit(pSSM, iUnit);
-        pSSM->offEstUnitEnd += pUnit->cbGuess;
-        if (!pUnit->u.Common.pfnSaveExec)
-        {
-            pUnit->fCalled = true;
-            if (pUnit->cbGuess)
-                ssmR3ProgressByByte(pSSM, pSSM->offEstUnitEnd - pSSM->offEst);
-            continue;
-        }
-        pUnit->offStream = ssmR3StrmTell(&pSSM->Strm);
-
-        /*
-         * Check for cancellation.
-         */
-        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
-        {
-            LogRel(("SSM: Cancelled!\n"));
-            AssertRC(pSSM->rc);
-            return pSSM->rc = VERR_SSM_CANCELLED;
-        }
-
-        /*
-         * Write data unit header
-         */
-        SSMFILEUNITHDRV2 UnitHdr;
-        memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
-        UnitHdr.offStream       = pUnit->offStream;
-        UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
-        UnitHdr.u32CRC          = 0;
-        UnitHdr.u32Version      = pUnit->u32Version;
-        UnitHdr.u32Instance     = pUnit->u32Instance;
-        UnitHdr.u32Pass         = SSM_PASS_FINAL;
-        UnitHdr.fFlags          = 0;
-        UnitHdr.cbName          = (uint32_t)pUnit->cchName + 1;
-        memcpy(&UnitHdr.szName[0], &pUnit->szName[0], UnitHdr.cbName);
-        UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
-        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
-             UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
-        int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("SSM: Failed to write unit header. rc=%Rrc\n", rc));
-            return pSSM->rc = rc;
-        }
-
-        /*
-         * Call the execute handler.
-         */
-        ssmR3DataWriteBegin(pSSM);
-        switch (pUnit->enmType)
-        {
-            case SSMUNITTYPE_DEV:
-                rc = pUnit->u.Dev.pfnSaveExec(pUnit->u.Dev.pDevIns, pSSM);
-                break;
-            case SSMUNITTYPE_DRV:
-                rc = pUnit->u.Drv.pfnSaveExec(pUnit->u.Drv.pDrvIns, pSSM);
-                break;
-            case SSMUNITTYPE_INTERNAL:
-                rc = pUnit->u.Internal.pfnSaveExec(pVM, pSSM);
-                break;
-            case SSMUNITTYPE_EXTERNAL:
-                pUnit->u.External.pfnSaveExec(pSSM, pUnit->u.External.pvUser);
-                rc = pSSM->rc;
-                break;
-            default:
-                rc = VERR_INTERNAL_ERROR;
-                break;
-        }
-        pUnit->fCalled = true;
-        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
-            pSSM->rc = rc;
-        else
-            rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("SSM: Execute save failed with rc=%Rrc for data unit '%s'/#%u.\n", rc, pUnit->szName, pUnit->u32Instance));
-            return rc;
-        }
-
-        /*
-         * Write the termination record and flush the compression stream.
-         */
-        SSMRECTERM TermRec;
-        TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
-        TermRec.cbRec            = sizeof(TermRec) - 2;
-        if (pSSM->Strm.fChecksummed)
-        {
-            TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
-            TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
-        }
-        else
-        {
-            TermRec.fFlags       = 0;
-            TermRec.u32StreamCRC = 0;
-        }
-        TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
-        rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
-        if (RT_SUCCESS(rc))
-            rc = ssmR3DataWriteFinish(pSSM);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("SSM: Failed terminating unit: %Rrc\n", rc));
-            return pSSM->rc = rc;
-        }
-
-        /*
-         * Advance the progress indicator to the end of the current unit.
-         */
-        ssmR3ProgressByByte(pSSM, pSSM->offEstUnitEnd - pSSM->offEst);
-    } /* for each unit */
-    ssmR3ProgressByUnit(pSSM, pVM->ssm.s.cUnits);
-
-    /* (progress should be pending 99% now) */
-    AssertMsg(   pSSM->uPercent == 101 - pSSM->uPercentDone
-              || pSSM->uPercent == 100 - pSSM->uPercentDone,
-              ("%d\n", pSSM->uPercent));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Do the pfnSavePrep run.
- *
- * @returns VBox status code (pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3SaveDoPrepRun(PVM pVM, PSSMHANDLE pSSM)
-{
-    VM_ASSERT_EMT0(pVM);
-    Assert(RT_SUCCESS(pSSM->rc));
-    pSSM->enmOp = SSMSTATE_SAVE_PREP;
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-    {
-        if (pUnit->u.Common.pfnSavePrep)
-        {
-            int rc;
-            switch (pUnit->enmType)
-            {
-                case SSMUNITTYPE_DEV:
-                    rc = pUnit->u.Dev.pfnSavePrep(pUnit->u.Dev.pDevIns, pSSM);
-                    break;
-                case SSMUNITTYPE_DRV:
-                    rc = pUnit->u.Drv.pfnSavePrep(pUnit->u.Drv.pDrvIns, pSSM);
-                    break;
-                case SSMUNITTYPE_INTERNAL:
-                    rc = pUnit->u.Internal.pfnSavePrep(pVM, pSSM);
-                    break;
-                case SSMUNITTYPE_EXTERNAL:
-                    rc = pUnit->u.External.pfnSavePrep(pSSM, pUnit->u.External.pvUser);
-                    break;
-                default:
-                    rc = VERR_INTERNAL_ERROR;
-                    break;
-            }
-            pUnit->fCalled = true;
-            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
-                pSSM->rc = rc;
-            else
-                rc = pSSM->rc;
-            if (RT_FAILURE(rc))
-            {
-                LogRel(("SSM: Prepare save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
-                return rc;
-            }
-        }
-
-        pSSM->cbEstTotal += pUnit->cbGuess;
-    }
-
-    /*
-     * Work the progress indicator if we got one.
-     */
-    if (pSSM->pfnProgress)
-        pSSM->pfnProgress(pVM, pSSM->uPercentPrepare + pSSM->uPercentLive - 1, pSSM->pvUser);
-    pSSM->uPercent = pSSM->uPercentPrepare + pSSM->uPercentLive;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Common worker for SSMR3Save and SSMR3LiveSave.
- *
- * @returns VBox status code (no need to check pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The state handle.
- *
- * @thread  EMT(0)
- */
-static int ssmR3SaveDoCommon(PVM pVM, PSSMHANDLE pSSM)
-{
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * Do the work.
-     */
-    int rc = ssmR3SaveDoPrepRun(pVM, pSSM);
-    if (RT_SUCCESS(rc))
-    {
-        rc = ssmR3SaveDoExecRun(pVM, pSSM);
-        if (RT_SUCCESS(rc))
-            rc = ssmR3SaveDoFinalization(pVM, pSSM);
-    }
-    Assert(pSSM->rc == rc);
-    int rc2 = ssmR3SaveDoDoneRun(pVM, pSSM);
-    if (RT_SUCCESS(rc))
-        rc = rc2;
-
-    return rc;
-}
-
-
-/**
- * Saves the rest of the state on EMT0.
- *
- * @returns VBox status.
- *
- * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
- *
- * @thread  Non-EMT thread. Will involve the EMT at the end of the operation.
- */
-VMMR3_INT_DECL(int) SSMR3LiveDoStep2(PSSMHANDLE pSSM)
-{
-    LogFlow(("SSMR3LiveDoStep2: pSSM=%p\n", pSSM));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
-    PVM pVM = pSSM->pVM;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    VM_ASSERT_EMT0(pVM);
-    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
-                    || pSSM->enmAfter == SSMAFTER_CONTINUE
-                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
-                    ("%d\n", pSSM->enmAfter),
-                    VERR_INVALID_PARAMETER);
-    AssertMsgReturn(pSSM->enmOp == SSMSTATE_LIVE_STEP2, ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
-    AssertRCReturn(pSSM->rc, pSSM->rc);
-
-    /*
-     * Join paths with VMMR3Save.
-     */
-    return ssmR3SaveDoCommon(pVM, pSSM);
-}
-
-
-/**
- * Writes the file header and clear the per-unit data.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The SSM handle.
- */
-static int ssmR3WriteHeaderAndClearPerUnitData(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Write the header.
-     */
-    SSMFILEHDR FileHdr;
-    memcpy(&FileHdr.szMagic, SSMFILEHDR_MAGIC_V2_0, sizeof(FileHdr.szMagic));
-    FileHdr.u16VerMajor  = VBOX_VERSION_MAJOR;
-    FileHdr.u16VerMinor  = VBOX_VERSION_MINOR;
-    FileHdr.u32VerBuild  = VBOX_VERSION_BUILD;
-    FileHdr.u32SvnRev    = VMMGetSvnRev();
-    FileHdr.cHostBits    = HC_ARCH_BITS;
-    FileHdr.cbGCPhys     = sizeof(RTGCPHYS);
-    FileHdr.cbGCPtr      = sizeof(RTGCPTR);
-    FileHdr.u8Reserved   = 0;
-    FileHdr.cUnits       = pVM->ssm.s.cUnits;
-    FileHdr.fFlags       = SSMFILEHDR_FLAGS_STREAM_CRC32;
-    if (pSSM->fLiveSave)
-        FileHdr.fFlags  |= SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE;
-    FileHdr.cbMaxDecompr = RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer);
-    FileHdr.u32CRC       = 0;
-    FileHdr.u32CRC       = RTCrc32(&FileHdr, sizeof(FileHdr));
-    int rc = ssmR3StrmWrite(&pSSM->Strm, &FileHdr, sizeof(FileHdr));
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Clear the per unit flags and offsets.
-     */
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-    {
-        pUnit->fCalled   = false;
-        pUnit->offStream = RTFOFF_MIN;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Creates a new saved state file.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pszFilename         The name of the file.  NULL if pStreamOps is
- *                              used.
- * @param   pStreamOps          The stream methods.  NULL if pszFilename is
- *                              used.
- * @param   pvStreamOpsUser     The user argument to the stream methods.
- * @param   enmAfter            What to do afterwards.
- * @param   pfnProgress         The progress callback.
- * @param   pvProgressUser      The progress callback user argument.
- * @param   ppSSM               Where to return the pointer to the saved state
- *                              handle upon successful return.  Free it using
- *                              RTMemFree after closing the stream.
- */
-static int ssmR3SaveDoCreateFile(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                                 SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, PSSMHANDLE *ppSSM)
-{
-    PSSMHANDLE pSSM = (PSSMHANDLE)RTMemAllocZ(sizeof(*pSSM));
-    if (!pSSM)
-        return VERR_NO_MEMORY;
-
-    pSSM->pVM                       = pVM;
-    pSSM->enmOp                     = SSMSTATE_INVALID;
-    pSSM->enmAfter                  = enmAfter;
-    pSSM->fCancelled                = SSMHANDLE_OK;
-    pSSM->rc                        = VINF_SUCCESS;
-    pSSM->cbUnitLeftV1              = 0;
-    pSSM->offUnit                   = UINT64_MAX;
-    pSSM->fLiveSave                 = false;
-    pSSM->pfnProgress               = pfnProgress;
-    pSSM->pvUser                    = pvProgressUser;
-    pSSM->uPercent                  = 0;
-    pSSM->offEstProgress            = 0;
-    pSSM->cbEstTotal                = 0;
-    pSSM->offEst                    = 0;
-    pSSM->offEstUnitEnd             = 0;
-    pSSM->uPercentLive              = 0;
-    pSSM->uPercentPrepare           = 0;
-    pSSM->uPercentDone              = 0;
-    pSSM->uReportedLivePercent      = 0;
-    pSSM->pszFilename               = pszFilename;
-    pSSM->u.Write.offDataBuffer     = 0;
-    pSSM->u.Write.cMsMaxDowntime    = UINT32_MAX;
-
-    int rc;
-    if (pStreamOps)
-        rc = ssmR3StrmInit(&pSSM->Strm, pStreamOps, pvStreamOpsUser, true /*fWrite*/, true /*fChecksummed*/, 8 /*cBuffers*/);
-    else
-        rc = ssmR3StrmOpenFile(&pSSM->Strm, pszFilename,             true /*fWrite*/, true /*fChecksummed*/, 8 /*cBuffers*/);
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("SSM: Failed to create save state file '%s', rc=%Rrc.\n",  pszFilename, rc));
-        RTMemFree(pSSM);
-        return rc;
-    }
-
-    *ppSSM = pSSM;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Start VM save operation.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   pszFilename     Name of the file to save the state in. NULL if pStreamOps is used.
- * @param   pStreamOps      The stream method table. NULL if pszFilename is
- *                          used.
- * @param   pvStreamOpsUser The user argument to the stream methods.
- * @param   enmAfter        What is planned after a successful save operation.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvUser          User argument for the progress callback.
- *
- * @thread  EMT
- */
-VMMR3DECL(int) SSMR3Save(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                         SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvUser)
-{
-    LogFlow(("SSMR3Save: pszFilename=%p:{%s} enmAfter=%d pfnProgress=%p pvUser=%p\n", pszFilename, pszFilename, enmAfter, pfnProgress, pvUser));
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * Validate input.
-     */
-    AssertMsgReturn(   enmAfter == SSMAFTER_DESTROY
-                    || enmAfter == SSMAFTER_CONTINUE,
-                    ("%d\n", enmAfter),
-                    VERR_INVALID_PARAMETER);
-
-    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
-    if (pStreamOps)
-    {
-        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
-        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
-        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
-    }
-
-    /*
-     * Create the saved state file and handle.
-     *
-     * Note that there might be quite some work to do after executing the saving,
-     * so we reserve 20% for the 'Done' period.
-     */
-    PSSMHANDLE pSSM;
-    int rc = ssmR3SaveDoCreateFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser,
-                                   enmAfter, pfnProgress, pvUser, &pSSM);
-    if (RT_FAILURE(rc))
-        return rc;
-    pSSM->uPercentLive    = 0;
-    pSSM->uPercentPrepare = 20;
-    pSSM->uPercentDone    = 2;
-    pSSM->fLiveSave       = false;
-
-    /*
-     * Write the saved state stream header and join paths with
-     * the other save methods for the rest of the job.
-     */
-    Log(("SSM: Starting state save to file '%s'...\n", pszFilename));
-    ssmR3StrmStartIoThread(&pSSM->Strm);
-    rc = ssmR3WriteHeaderAndClearPerUnitData(pVM, pSSM);
-    if (RT_SUCCESS(rc))
-    {
-        ssmR3SetCancellable(pVM, pSSM, true);
-        ssmR3SaveDoCommon(pVM, pSSM);
-    }
-
-    return ssmR3SaveDoClose(pVM, pSSM);
-}
-
-
-/**
- * Used by PGM to report the completion percentage of the live stage during the
- * vote run.
- *
- * @param   pSSM                The saved state handle.
- * @param   uPercent            The completion percentage.
- */
-VMMR3DECL(void) SSMR3HandleReportLivePercent(PSSMHANDLE pSSM, unsigned uPercent)
-{
-    AssertMsgReturnVoid(pSSM->enmOp == SSMSTATE_LIVE_VOTE, ("%d\n", pSSM->enmOp));
-    AssertReturnVoid(uPercent <= 100);
-    if (uPercent < pSSM->uReportedLivePercent)
-        pSSM->uReportedLivePercent = uPercent;
-}
-
-
-/**
- * Calls pfnLiveVote for all units.
- *
- * @returns VBox status code (no need to check pSSM->rc).
- * @retval  VINF_SUCCESS if we can pass on to step 2.
- * @retval  VINF_SSM_VOTE_FOR_ANOTHER_PASS if we need another pass.
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- * @param   uPass               The current pass.
- */
-static int ssmR3LiveDoVoteRun(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    int rcRet = VINF_SUCCESS;
-    AssertRC(pSSM->rc);
-    pSSM->rc = VINF_SUCCESS;
-    pSSM->enmOp = SSMSTATE_LIVE_VOTE;
-
-    unsigned uPrevPrecent = pSSM->uReportedLivePercent;
-    pSSM->uReportedLivePercent = 101;
-
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-    {
-        if (    pUnit->u.Common.pfnLiveVote
-            &&  !pUnit->fDoneLive)
-        {
-            int rc;
-            switch (pUnit->enmType)
-            {
-                case SSMUNITTYPE_DEV:
-                    rc = pUnit->u.Dev.pfnLiveVote(pUnit->u.Dev.pDevIns, pSSM, uPass);
-                    break;
-                case SSMUNITTYPE_DRV:
-                    rc = pUnit->u.Drv.pfnLiveVote(pUnit->u.Drv.pDrvIns, pSSM, uPass);
-                    break;
-                case SSMUNITTYPE_INTERNAL:
-                    rc = pUnit->u.Internal.pfnLiveVote(pVM, pSSM, uPass);
-                    break;
-                case SSMUNITTYPE_EXTERNAL:
-                    rc = pUnit->u.External.pfnLiveVote(pSSM, pUnit->u.External.pvUser, uPass);
-                    break;
-                default:
-                    rc = VERR_INTERNAL_ERROR;
-                    break;
-            }
-            pUnit->fCalled = true;
-            Assert(pSSM->rc == VINF_SUCCESS);
-            if (rc != VINF_SUCCESS)
-            {
-                if (rc == VINF_SSM_VOTE_FOR_ANOTHER_PASS)
-                {
-                    Log(("ssmR3DoLiveVoteRun: '%s'/#%u -> VINF_SSM_VOTE_FOR_ANOTHER_PASS (pass=%u)\n", pUnit->szName, pUnit->u32Instance, uPass));
-                    rcRet = VINF_SSM_VOTE_FOR_ANOTHER_PASS;
-                }
-                else if (rc == VINF_SSM_VOTE_DONE_DONT_CALL_AGAIN)
-                {
-                    pUnit->fDoneLive = true;
-                    Log(("ssmR3DoLiveVoteRun: '%s'/#%u -> VINF_SSM_VOTE_DONE_DONT_CALL_AGAIN (pass=%u)\n", pUnit->szName, pUnit->u32Instance, uPass));
-                }
-                else
-                {
-                    /*
-                     * rc is usually VERR_SSM_VOTE_FOR_GIVING_UP here, but we allow
-                     * other status codes for better user feed back.  However, no
-                     * other non-error status is allowed.
-                     */
-                    LogRel(("SSM: Error - '%s'/#%u voted %Rrc! (pass=%u)\n", pUnit->szName, pUnit->u32Instance, rc, uPass));
-                    AssertMsgReturn(RT_FAILURE(rc), ("%Rrc; '%s'\n", rc, pUnit->szName), pSSM->rc = VERR_IPE_UNEXPECTED_INFO_STATUS);
-                    return pSSM->rc = rc;
-                }
-            }
-        }
-    }
-    if (rcRet == VINF_SUCCESS)
-    {
-        LogRel(("SSM: Step 1 completed after pass %u.\n", uPass));
-        pSSM->uReportedLivePercent = 100;
-    }
-    else
-    {
-        /*
-         * Work the progress callback.
-         */
-        if (pSSM->uReportedLivePercent > 100)
-            pSSM->uReportedLivePercent = 0;
-        if (   pSSM->uReportedLivePercent != uPrevPrecent
-            && pSSM->pfnProgress
-            && pSSM->uPercentLive)
-        {
-            long double lrdPct = (long double)pSSM->uReportedLivePercent * pSSM->uPercentLive / 100;
-            unsigned    uPct   = (unsigned)lrdPct;
-            if (uPct != pSSM->uPercent)
-            {
-                ssmR3LiveControlEmit(pSSM, lrdPct, uPass);
-                pSSM->uPercent = uPct;
-                pSSM->pfnProgress(pVM, uPct, pSSM->pvUser);
-            }
-        }
-    }
-    return rcRet;
-}
-
-
-/**
- * Calls pfnLiveExec for all units.
- *
- * @returns VBox status code (no need to check pSSM->rc).
- *
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- * @param   uPass               The current pass.
- */
-static int ssmR3LiveDoExecRun(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
-{
-    AssertRC(pSSM->rc);
-    pSSM->rc = VINF_SUCCESS;
-    pSSM->enmOp = SSMSTATE_LIVE_EXEC;
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-    {
-        /*
-         * Skip units without a callback (this is most).
-         */
-        if (   !pUnit->u.Common.pfnLiveExec
-            || pUnit->fDoneLive)
-            continue;
-        pUnit->offStream = ssmR3StrmTell(&pSSM->Strm);
-
-        /*
-         * Check for cancellation.
-         */
-        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
-        {
-            LogRel(("SSM: Cancelled!\n"));
-            AssertRC(pSSM->rc);
-            return pSSM->rc = VERR_SSM_CANCELLED;
-        }
-
-        /*
-         * Write data unit header.
-         */
-        SSMFILEUNITHDRV2 UnitHdr;
-        memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
-        UnitHdr.offStream       = pUnit->offStream;
-        UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
-        UnitHdr.u32CRC          = 0;
-        UnitHdr.u32Version      = pUnit->u32Version;
-        UnitHdr.u32Instance     = pUnit->u32Instance;
-        UnitHdr.u32Pass         = uPass;
-        UnitHdr.fFlags          = 0;
-        UnitHdr.cbName          = (uint32_t)pUnit->cchName + 1;
-        memcpy(&UnitHdr.szName[0], &pUnit->szName[0], UnitHdr.cbName);
-        UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
-        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
-             UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
-        int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("SSM: Failed to write unit header. rc=%Rrc\n", rc));
-            return pSSM->rc = rc;
-        }
-
-        /*
-         * Call the execute handler.
-         */
-        ssmR3DataWriteBegin(pSSM);
-        switch (pUnit->enmType)
-        {
-            case SSMUNITTYPE_DEV:
-                rc = pUnit->u.Dev.pfnLiveExec(pUnit->u.Dev.pDevIns, pSSM, uPass);
-                break;
-            case SSMUNITTYPE_DRV:
-                rc = pUnit->u.Drv.pfnLiveExec(pUnit->u.Drv.pDrvIns, pSSM, uPass);
-                break;
-            case SSMUNITTYPE_INTERNAL:
-                rc = pUnit->u.Internal.pfnLiveExec(pVM, pSSM, uPass);
-                break;
-            case SSMUNITTYPE_EXTERNAL:
-                rc = pUnit->u.External.pfnLiveExec(pSSM, pUnit->u.External.pvUser, uPass);
-                break;
-            default:
-                rc = VERR_INTERNAL_ERROR;
-                break;
-        }
-        pUnit->fCalled = true;
-        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
-            pSSM->rc = rc;
-        else
-        {
-            if (rc == VINF_SSM_DONT_CALL_AGAIN)
-                pUnit->fDoneLive = true;
-            rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
-        }
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("SSM: Execute save failed with rc=%Rrc for data unit '%s'/#%u.\n", rc, pUnit->szName, pUnit->u32Instance));
-            if (RT_SUCCESS(pSSM->rc))
-                pSSM->rc = rc;
-            return rc;
-        }
-
-        /*
-         * Write the termination record and flush the compression stream.
-         */
-        SSMRECTERM TermRec;
-        TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
-        TermRec.cbRec            = sizeof(TermRec) - 2;
-        if (pSSM->Strm.fChecksummed)
-        {
-            TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
-            TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
-        }
-        else
-        {
-            TermRec.fFlags       = 0;
-            TermRec.u32StreamCRC = 0;
-        }
-        TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
-        rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
-        if (RT_SUCCESS(rc))
-            rc = ssmR3DataWriteFinish(pSSM);
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("SSM: Failed terminating unit: %Rrc (pass=%u)\n", rc, uPass));
-            return pSSM->rc = rc;
-        }
-    } /* for each unit */
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Implements the live exec+vote loop.
- *
- * @returns VBox status code (no need to check pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3DoLiveExecVoteLoop(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Calc the max saved state size before we should give up because of insane
-     * amounts of data.
-     */
-#define SSM_MAX_GROWTH_FILE      10000
-#define SSM_MAX_GROWTH_REMOTE   100000
-    uint64_t cbSum = 0;
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-        cbSum += pUnit->cbGuess;
-    uint64_t cbMax = cbSum * (pSSM->pszFilename ? SSM_MAX_GROWTH_FILE : SSM_MAX_GROWTH_REMOTE);
-    AssertLogRelMsgReturn(cbMax > cbSum, ("cbMax=%#RX64, cbSum=%#RX64\n", cbMax, cbSum), pSSM->rc = VERR_OUT_OF_RANGE);
-    if (cbMax < _1G)
-        cbMax = _1G;
-
-    /*
-     * The pass loop.
-     *
-     * The number of iterations is restricted for two reasons, first
-     * to make sure
-     */
-#define SSM_MAX_PASSES  _1M
-    for (uint32_t uPass = 0; uPass < SSM_MAX_PASSES; uPass++)
-    {
-        pVM->ssm.s.uPass = uPass;
-
-        /*
-         * Save state and vote on whether we need more passes or not.
-         */
-        int rc = ssmR3LiveDoExecRun(pVM, pSSM, uPass);
-        if (RT_FAILURE(rc))
-            return rc;
-        rc = ssmR3LiveDoVoteRun(pVM, pSSM, uPass);
-        if (rc == VINF_SUCCESS)
-        {
-            pSSM->enmOp = SSMSTATE_LIVE_STEP2;
-            return VINF_SUCCESS;
-        }
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /*
-         * Check that we're still within sane data amounts.
-         */
-        uint64_t cbSaved = ssmR3StrmTell(&pSSM->Strm);
-        if (cbSaved > cbMax)
-        {
-            LogRel(("SSM: Giving up: Exceeded max state size. (cbSaved=%#RX64, cbMax=%#RX64)\n", cbSaved, cbMax));
-            return pSSM->rc = VERR_SSM_STATE_GREW_TOO_BIG;
-        }
-
-        /*
-         * Check that the stream is still OK.
-         */
-        rc = ssmR3StrmCheckAndFlush(&pSSM->Strm);
-        if (RT_FAILURE(rc))
-            return pSSM->rc = rc;
-    }
-
-    LogRel(("SSM: Giving up: Too many passes! (%u)\n", SSM_MAX_PASSES));
-    return pSSM->rc = VERR_SSM_TOO_MANY_PASSES;
-}
-
-
-/**
- * Calls pfnLivePrep for all units.
- *
- * @returns VBox status code (no need to check pSSM->rc).
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3DoLivePrepRun(PVM pVM, PSSMHANDLE pSSM)
-{
-    /*
-     * Do the prepare run.
-     */
-    pSSM->rc = VINF_SUCCESS;
-    pSSM->enmOp = SSMSTATE_SAVE_PREP;
-    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-    {
-        if (pUnit->u.Common.pfnLivePrep)
-        {
-            int rc;
-            switch (pUnit->enmType)
-            {
-                case SSMUNITTYPE_DEV:
-                    rc = pUnit->u.Dev.pfnLivePrep(pUnit->u.Dev.pDevIns, pSSM);
-                    break;
-                case SSMUNITTYPE_DRV:
-                    rc = pUnit->u.Drv.pfnLivePrep(pUnit->u.Drv.pDrvIns, pSSM);
-                    break;
-                case SSMUNITTYPE_INTERNAL:
-                    rc = pUnit->u.Internal.pfnLivePrep(pVM, pSSM);
-                    break;
-                case SSMUNITTYPE_EXTERNAL:
-                    rc = pUnit->u.External.pfnLivePrep(pSSM, pUnit->u.External.pvUser);
-                    break;
-                default:
-                    rc = VERR_INTERNAL_ERROR;
-                    break;
-            }
-            pUnit->fCalled = true;
-            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
-                pSSM->rc = rc;
-            else
-                rc = pSSM->rc;
-            if (RT_FAILURE(rc))
-            {
-                LogRel(("SSM: Prepare save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
-                return rc;
-            }
-        }
-
-        pSSM->cbEstTotal += pUnit->cbGuess;
-    }
-
-    /*
-     * Work the progress indicator if we got one.
-     */
-    if (pSSM->pfnProgress)
-        pSSM->pfnProgress(pVM, 2, pSSM->pvUser);
-    pSSM->uPercent = 2;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Continue a live state saving operation on the worker thread.
- *
- * @returns VBox status.
- *
- * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
- *
- * @thread  Non-EMT thread. Will involve the EMT at the end of the operation.
- */
-VMMR3_INT_DECL(int) SSMR3LiveDoStep1(PSSMHANDLE pSSM)
-{
-    LogFlow(("SSMR3LiveDoStep1: pSSM=%p\n", pSSM));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
-    PVM pVM = pSSM->pVM;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    VM_ASSERT_OTHER_THREAD(pVM);
-    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
-                    || pSSM->enmAfter == SSMAFTER_CONTINUE
-                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
-                    ("%d\n", pSSM->enmAfter),
-                    VERR_INVALID_PARAMETER);
-    AssertMsgReturn(pSSM->enmOp == SSMSTATE_LIVE_STEP1, ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
-    AssertRCReturn(pSSM->rc, pSSM->rc);
-
-    /*
-     * Do the prep run, then the exec+vote cycle.
-     */
-    int rc = ssmR3DoLivePrepRun(pVM, pSSM);
-    if (RT_SUCCESS(rc))
-        rc = ssmR3DoLiveExecVoteLoop(pVM, pSSM);
-    return rc;
-}
-
-
-/**
- * Start saving the live state.
- *
- * Call SSMR3LiveDoStep1, SSMR3LiveDoStep2 and finally SSMR3LiveDone on success.
- * SSMR3LiveDone should be called even if SSMR3LiveDoStep1 or SSMR3LiveDoStep2
- * fails.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   cMsMaxDowntime  The maximum downtime given as milliseconds.
- * @param   pszFilename     Name of the file to save the state in. This string
- *                          must remain valid until SSMR3LiveDone is called.
- *                          Must be NULL if pStreamOps is used.
- * @param   pStreamOps      The stream method table. NULL if pszFilename is
- *                          used.
- * @param   pvStreamOpsUser The user argument to the stream methods.
- * @param   enmAfter        What is planned after a successful save operation.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvProgressUser  User argument for the progress callback.
- *
- * @thread  EMT0
- */
-VMMR3_INT_DECL(int) SSMR3LiveSave(PVM pVM, uint32_t cMsMaxDowntime,
-                                  const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                                  SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser,
-                                  PSSMHANDLE *ppSSM)
-{
-    LogFlow(("SSMR3LiveSave: cMsMaxDowntime=%u pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p\n",
-             cMsMaxDowntime, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser));
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * Validate input.
-     */
-    AssertMsgReturn(   enmAfter == SSMAFTER_DESTROY
-                    || enmAfter == SSMAFTER_CONTINUE
-                    || enmAfter == SSMAFTER_TELEPORT,
-                    ("%d\n", enmAfter),
-                    VERR_INVALID_PARAMETER);
-    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
-    if (pStreamOps)
-    {
-        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
-        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
-        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
-    }
-
-    /*
-     * Create the saved state file and handle.
-     *
-     * Note that there might be quite some work to do after executing the saving,
-     * so we reserve 20% for the 'Done' period.
-     */
-    PSSMHANDLE pSSM;
-    int rc = ssmR3SaveDoCreateFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser,
-                                   enmAfter, pfnProgress, pvProgressUser, &pSSM);
-    if (RT_FAILURE(rc))
-        return rc;
-    pSSM->uPercentLive           = 93;
-    pSSM->uPercentPrepare        = 2;
-    pSSM->uPercentDone           = 2;
-    pSSM->fLiveSave              = true;
-    pSSM->u.Write.cMsMaxDowntime = cMsMaxDowntime;
-
-    /*
-     * Write the saved state stream header and do the prep run for live saving.
-     */
-    Log(("SSM: Starting state save to file '%s'...\n", pszFilename));
-    ssmR3StrmStartIoThread(&pSSM->Strm);
-    rc = ssmR3WriteHeaderAndClearPerUnitData(pVM, pSSM);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Return and let the requestor thread do the pfnLiveExec/Vote part
-         * via SSMR3SaveFinishLive
-         */
-        pSSM->enmOp = SSMSTATE_LIVE_STEP1;
-        ssmR3SetCancellable(pVM, pSSM, true);
-        *ppSSM = pSSM;
-        return VINF_SUCCESS;
-    }
-    /* bail out. */
-    int rc2 = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
-    RTMemFree(pSSM);
-    rc2 = RTFileDelete(pszFilename);
-    AssertRC(rc2);
-    return rc;
-}
-
-#endif /* !SSM_STANDALONE */
-
-
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-/* ... Loading and reading starts here ... */
-
-
-/**
- * Closes the decompressor of a data unit.
- *
- * @returns pSSM->rc.
- * @param   pSSM            The saved state handle.
- */
-static int ssmR3DataReadFinishV1(PSSMHANDLE pSSM)
-{
-    if (pSSM->u.Read.pZipDecompV1)
-    {
-        int rc = RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
-        AssertRC(rc);
-        pSSM->u.Read.pZipDecompV1 = NULL;
-    }
-    return pSSM->rc;
-}
-
-
-/**
- * Callback for reading compressed data into the input buffer of the
- * decompressor, for saved file format version 1.
- *
- * @returns VBox status code. Set pSSM->rc on error.
- * @param   pvSSM       The SSM handle.
- * @param   pvBuf       Where to store the compressed data.
- * @param   cbBuf       Size of the buffer.
- * @param   pcbRead     Number of bytes actually stored in the buffer.
- */
-static DECLCALLBACK(int) ssmR3ReadInV1(void *pvSSM, void *pvBuf, size_t cbBuf, size_t *pcbRead)
-{
-    PSSMHANDLE  pSSM   = (PSSMHANDLE)pvSSM;
-    size_t      cbRead = cbBuf;
-    if (pSSM->cbUnitLeftV1 < cbBuf)
-        cbRead = (size_t)pSSM->cbUnitLeftV1;
-    if (cbRead)
-    {
-        //Log2(("ssmR3ReadInV1: %#010llx cbBug=%#x cbRead=%#x\n", ssmR3StrmTell(&pSSM->Strm), cbBuf, cbRead));
-        int rc = ssmR3StrmRead(&pSSM->Strm, pvBuf, cbRead);
-        if (RT_SUCCESS(rc))
-        {
-            pSSM->cbUnitLeftV1 -= cbRead;
-            if (pcbRead)
-                *pcbRead = cbRead;
-            ssmR3ProgressByByte(pSSM, cbRead);
-            return VINF_SUCCESS;
-        }
-        return pSSM->rc = rc;
-    }
-
-    if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
-        AssertMsgFailed(("SSM: attempted reading more than the unit!\n"));
-    return pSSM->rc = VERR_SSM_LOADED_TOO_MUCH;
-}
-
-
-/**
- * Internal read worker for reading data from a version 1 unit.
- *
- * @returns VBox status code, pSSM->rc is set on error.
- *
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           Where to store the read data.
- * @param   cbBuf           Number of bytes to read.
- */
-static int ssmR3DataReadV1(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
-{
-    /*
-     * Open the decompressor on the first read.
-     */
-    if (!pSSM->u.Read.pZipDecompV1)
-    {
-        pSSM->rc = RTZipDecompCreate(&pSSM->u.Read.pZipDecompV1, pSSM, ssmR3ReadInV1);
-        if (RT_FAILURE(pSSM->rc))
-            return pSSM->rc;
-    }
-
-    /*
-     * Do the requested read.
-     */
-    int rc = pSSM->rc = RTZipDecompress(pSSM->u.Read.pZipDecompV1, pvBuf, cbBuf, NULL);
-    if (RT_SUCCESS(rc))
-    {
-        Log2(("ssmR3DataRead: pvBuf=%p cbBuf=%#x offUnit=%#llx %.*Rhxs%s\n", pvBuf, cbBuf, pSSM->offUnit, RT_MIN(cbBuf, SSM_LOG_BYTES), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
-        pSSM->offUnit += cbBuf;
-        return VINF_SUCCESS;
-    }
-    AssertMsgFailed(("rc=%Rrc cbBuf=%#x\n", rc, cbBuf));
-    return rc;
-}
-
-
-/**
- * Creates the decompressor for the data unit.
- *
- * pSSM->rc will be set on error.
- *
- * @param   pSSM            The saved state handle.
- */
-static void ssmR3DataReadBeginV2(PSSMHANDLE pSSM)
-{
-    Assert(!pSSM->u.Read.cbDataBuffer || pSSM->u.Read.cbDataBuffer == pSSM->u.Read.offDataBuffer);
-    Assert(!pSSM->u.Read.cbRecLeft);
-
-    pSSM->offUnit = 0;
-    pSSM->u.Read.cbRecLeft      = 0;
-    pSSM->u.Read.cbDataBuffer   = 0;
-    pSSM->u.Read.offDataBuffer  = 0;
-    pSSM->u.Read.fEndOfData     = false;
-    pSSM->u.Read.u8TypeAndFlags = 0;
-}
-
-
-/**
- * Checks for the termination record and closes the decompressor.
- *
- * pSSM->rc will be set on error.
- *
- * @returns pSSM->rc.
- * @param   pSSM            The saved state handle.
- */
-static int ssmR3DataReadFinishV2(PSSMHANDLE pSSM)
-{
-    /*
-     * If we haven't encountered the end of the record, it must be the next one.
-     */
-    int rc = pSSM->rc;
-    if (    !pSSM->u.Read.fEndOfData
-        &&  RT_SUCCESS(rc))
-    {
-        rc = ssmR3DataReadRecHdrV2(pSSM);
-        if (    RT_SUCCESS(rc)
-            &&  !pSSM->u.Read.fEndOfData)
-        {
-            rc = VERR_SSM_LOADED_TOO_LITTLE;
-            AssertFailed();
-        }
-        pSSM->rc = rc;
-    }
-    return rc;
-}
-
-
-/**
- * Read reader that keep works the progress indicator and unit offset.
- *
- * @returns VBox status code. Does NOT set pSSM->rc.
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           Where to put the bits
- * @param   cbBuf           How many bytes to read.
- */
-DECLINLINE(int) ssmR3DataReadV2Raw(PSSMHANDLE pSSM, void *pvBuf, size_t cbToRead)
-{
-    int rc = ssmR3StrmRead(&pSSM->Strm, pvBuf, cbToRead);
-    if (RT_SUCCESS(rc))
-    {
-        pSSM->offUnit += cbToRead;
-        ssmR3ProgressByByte(pSSM, cbToRead);
-        return VINF_SUCCESS;
-    }
-
-    if (rc == VERR_SSM_CANCELLED)
-        return rc;
-
-    if (pSSM->enmAfter != SSMAFTER_DEBUG_IT && rc == VERR_EOF)
-        AssertMsgFailedReturn(("SSM: attempted reading more than the unit! rc=%Rrc\n", rc), VERR_SSM_LOADED_TOO_MUCH);
-    return VERR_SSM_STREAM_ERROR;
-}
-
-
-/**
- * Reads and checks the LZF "header".
- *
- * @returns VBox status code. Sets pSSM->rc on error.
- * @param   pSSM            The saved state handle..
- * @param   pcbDecompr      Where to store the size of the decompressed data.
- */
-DECLINLINE(int) ssmR3DataReadV2RawLzfHdr(PSSMHANDLE pSSM, uint32_t *pcbDecompr)
-{
-    *pcbDecompr = 0; /* shuts up gcc. */
-    AssertLogRelMsgReturn(   pSSM->u.Read.cbRecLeft > 1
-                          && pSSM->u.Read.cbRecLeft <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abComprBuffer) + 2,
-                          ("%#x\n", pSSM->u.Read.cbRecLeft),
-                          VERR_SSM_INTEGRITY_DECOMPRESSION);
-
-    uint8_t cKB;
-    int rc = ssmR3DataReadV2Raw(pSSM, &cKB, 1);
-    if (RT_FAILURE(rc))
-        return pSSM->rc = rc;
-    pSSM->u.Read.cbRecLeft -= sizeof(cKB);
-
-    uint32_t cbDecompr = (uint32_t)cKB * _1K;
-    AssertLogRelMsgReturn(   cbDecompr >= pSSM->u.Read.cbRecLeft
-                          && cbDecompr <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer),
-                          ("%#x\n", cbDecompr),
-                          VERR_SSM_INTEGRITY_DECOMPRESSION);
-
-    *pcbDecompr = cbDecompr;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Reads an LZF block from the stream and decompresses into the specified
- * buffer.
- *
- * @returns VBox status code. Sets pSSM->rc on error.
- * @param   SSM             The saved state handle.
- * @param   pvDst           Pointer to the output buffer.
- * @param   cbDecompr       The size of the decompressed data.
- */
-static int ssmR3DataReadV2RawLzf(PSSMHANDLE pSSM, void *pvDst, size_t cbDecompr)
-{
-    int         rc;
-    uint32_t    cbCompr    = pSSM->u.Read.cbRecLeft;
-    pSSM->u.Read.cbRecLeft = 0;
-
-    /*
-     * Try use the stream buffer directly to avoid copying things around.
-     */
-    uint8_t const *pb = ssmR3StrmReadDirect(&pSSM->Strm, cbCompr);
-    if (pb)
-    {
-        pSSM->offUnit += cbCompr;
-        ssmR3ProgressByByte(pSSM, cbCompr);
-    }
-    else
-    {
-        rc = ssmR3DataReadV2Raw(pSSM, &pSSM->u.Read.abComprBuffer[0], cbCompr);
-        if (RT_FAILURE(rc))
-            return pSSM->rc = rc;
-        pb = &pSSM->u.Read.abComprBuffer[0];
-    }
-
-    /*
-     * Decompress it.
-     */
-    size_t cbDstActual;
-    rc = RTZipBlockDecompress(RTZIPTYPE_LZF, 0 /*fFlags*/,
-                              pb, cbCompr, NULL /*pcbSrcActual*/,
-                              pvDst, cbDecompr, &cbDstActual);
-    if (RT_SUCCESS(rc))
-    {
-        AssertLogRelMsgReturn(cbDstActual == cbDecompr, ("%#x %#x\n", cbDstActual, cbDecompr), VERR_SSM_INTEGRITY_DECOMPRESSION);
-        return VINF_SUCCESS;
-    }
-
-    AssertLogRelMsgFailed(("cbCompr=%#x cbDecompr=%#x rc=%Rrc\n", cbCompr, cbDecompr, rc));
-    return pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION;
-}
-
-
-/**
- * Reads and checks the raw zero "header".
- *
- * @returns VBox status code. Sets pSSM->rc on error.
- * @param   pSSM            The saved state handle..
- * @param   pcbDecompr      Where to store the size of the zero data.
- */
-DECLINLINE(int) ssmR3DataReadV2RawZeroHdr(PSSMHANDLE pSSM, uint32_t *pcbZero)
-{
-    *pcbZero = 0; /* shuts up gcc. */
-    AssertLogRelMsgReturn(pSSM->u.Read.cbRecLeft == 1, ("%#x\n", pSSM->u.Read.cbRecLeft), VERR_SSM_INTEGRITY_DECOMPRESSION);
-
-    uint8_t cKB;
-    int rc = ssmR3DataReadV2Raw(pSSM, &cKB, 1);
-    if (RT_FAILURE(rc))
-        return pSSM->rc = rc;
-    pSSM->u.Read.cbRecLeft = 0;
-
-    uint32_t cbZero = (uint32_t)cKB * _1K;
-    AssertLogRelMsgReturn(cbZero <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer),
-                          ("%#x\n", cbZero), VERR_SSM_INTEGRITY_DECOMPRESSION);
-
-    *pcbZero = cbZero;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for reading the record header.
- *
- * It sets pSSM->u.Read.cbRecLeft, pSSM->u.Read.u8TypeAndFlags and
- * pSSM->u.Read.fEndOfData.  When a termination record is encounter, it will be
- * read in full and validated, the fEndOfData indicator is set, and VINF_SUCCESS
- * is returned.
- *
- * @returns VBox status code. Does not set pSSM->rc.
- * @param   pSSM            The saved state handle.
- */
-static int ssmR3DataReadRecHdrV2(PSSMHANDLE pSSM)
-{
-    AssertLogRelReturn(!pSSM->u.Read.fEndOfData, VERR_SSM_LOADED_TOO_MUCH);
-
-    /*
-     * Read the two mandatory bytes.
-     */
-    uint8_t  abHdr[8];
-    int rc = ssmR3DataReadV2Raw(pSSM, abHdr, 2);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Validate the first byte and check for the termination records.
-     */
-    pSSM->u.Read.u8TypeAndFlags = abHdr[0];
-    AssertLogRelMsgReturn(SSM_REC_ARE_TYPE_AND_FLAGS_VALID(abHdr[0]), ("%#x %#x\n", abHdr[0], abHdr[1]), VERR_SSM_INTEGRITY_REC_HDR);
-    if ((abHdr[0] & SSM_REC_TYPE_MASK) == SSM_REC_TYPE_TERM)
-    {
-        pSSM->u.Read.cbRecLeft = 0;
-        pSSM->u.Read.fEndOfData = true;
-        AssertLogRelMsgReturn(abHdr[1] == sizeof(SSMRECTERM) - 2, ("%#x\n", abHdr[1]), VERR_SSM_INTEGRITY_REC_TERM);
-        AssertLogRelMsgReturn(abHdr[0] & SSM_REC_FLAGS_IMPORTANT, ("%#x\n", abHdr[0]), VERR_SSM_INTEGRITY_REC_TERM);
-
-        /* get the rest */
-        uint32_t    u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
-        SSMRECTERM  TermRec;
-        rc = ssmR3DataReadV2Raw(pSSM, (uint8_t *)&TermRec + 2, sizeof(SSMRECTERM) - 2);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /* validate integrity */
-        AssertLogRelMsgReturn(TermRec.cbUnit == pSSM->offUnit,
-                              ("cbUnit=%#llx offUnit=%#llx\n", TermRec.cbUnit, pSSM->offUnit),
-                              VERR_SSM_INTEGRITY_REC_TERM);
-        AssertLogRelMsgReturn(!(TermRec.fFlags & ~SSMRECTERM_FLAGS_CRC32), ("%#x\n", TermRec.fFlags), VERR_SSM_INTEGRITY_REC_TERM);
-        if (!(TermRec.fFlags & SSMRECTERM_FLAGS_CRC32))
-            AssertLogRelMsgReturn(TermRec.u32StreamCRC == 0, ("%#x\n", TermRec.u32StreamCRC), VERR_SSM_INTEGRITY_REC_TERM);
-        else if (pSSM->Strm.fChecksummed)
-            AssertLogRelMsgReturn(TermRec.u32StreamCRC == u32StreamCRC, ("%#x, %#x\n", TermRec.u32StreamCRC, u32StreamCRC),
-                                  VERR_SSM_INTEGRITY_REC_TERM_CRC);
-
-        Log3(("ssmR3DataReadRecHdrV2: %08llx|%08llx: TERM\n", ssmR3StrmTell(&pSSM->Strm) - sizeof(SSMRECTERM), pSSM->offUnit));
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Figure the size. The 2nd byte is encoded in UTF-8 fashion, so this
-     * is can be highly enjoyable.
-     */
-    uint32_t cbHdr = 2;
-    uint32_t cb = abHdr[1];
-    if (!(cb & 0x80))
-        pSSM->u.Read.cbRecLeft = cb;
-    else
-    {
-        /*
-         * Need more data. Figure how much and read it.
-         */
-        if (!(cb & RT_BIT(5)))
-            cb = 2;
-        else if (!(cb & RT_BIT(4)))
-            cb = 3;
-        else if (!(cb & RT_BIT(3)))
-            cb = 4;
-        else if (!(cb & RT_BIT(2)))
-            cb = 5;
-        else if (!(cb & RT_BIT(1)))
-            cb = 6;
-        else
-            AssertLogRelMsgFailedReturn(("Invalid record size byte: %#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
-        cbHdr = cb + 1;
-
-        rc = ssmR3DataReadV2Raw(pSSM, &abHdr[2], cb - 1);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /*
-         * Validate what we've read.
-         */
-        switch (cb)
-        {
-            case 6:
-                AssertLogRelMsgReturn((abHdr[6] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
-            case 5:
-                AssertLogRelMsgReturn((abHdr[5] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
-            case 4:
-                AssertLogRelMsgReturn((abHdr[4] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
-            case 3:
-                AssertLogRelMsgReturn((abHdr[3] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
-            case 2:
-                AssertLogRelMsgReturn((abHdr[2] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
-                break;
-            default:
-                return VERR_INTERNAL_ERROR;
-        }
-
-        /*
-         * Decode it and validate the range.
-         */
-        switch (cb)
-        {
-            case 6:
-                cb =             (abHdr[6] & 0x3f)
-                    | ((uint32_t)(abHdr[5] & 0x3f) << 6)
-                    | ((uint32_t)(abHdr[4] & 0x3f) << 12)
-                    | ((uint32_t)(abHdr[3] & 0x3f) << 18)
-                    | ((uint32_t)(abHdr[2] & 0x3f) << 24)
-                    | ((uint32_t)(abHdr[1] & 0x01) << 30);
-                AssertLogRelMsgReturn(cb >= 0x04000000 && cb <= 0x7fffffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
-                break;
-            case 5:
-                cb =             (abHdr[5] & 0x3f)
-                    | ((uint32_t)(abHdr[4] & 0x3f) << 6)
-                    | ((uint32_t)(abHdr[3] & 0x3f) << 12)
-                    | ((uint32_t)(abHdr[2] & 0x3f) << 18)
-                    | ((uint32_t)(abHdr[1] & 0x03) << 24);
-                AssertLogRelMsgReturn(cb >= 0x00200000 && cb <= 0x03ffffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
-                break;
-            case 4:
-                cb =             (abHdr[4] & 0x3f)
-                    | ((uint32_t)(abHdr[3] & 0x3f) << 6)
-                    | ((uint32_t)(abHdr[2] & 0x3f) << 12)
-                    | ((uint32_t)(abHdr[1] & 0x07) << 18);
-                AssertLogRelMsgReturn(cb >= 0x00010000 && cb <= 0x001fffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
-                break;
-            case 3:
-                cb =             (abHdr[3] & 0x3f)
-                    | ((uint32_t)(abHdr[2] & 0x3f) << 6)
-                    | ((uint32_t)(abHdr[1] & 0x0f) << 12);
-#if 0  /* disabled to optimize buffering */
-                AssertLogRelMsgReturn(cb >= 0x00000800 && cb <= 0x0000ffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
-#endif
-                break;
-            case 2:
-                cb =             (abHdr[2] & 0x3f)
-                    | ((uint32_t)(abHdr[1] & 0x1f) << 6);
-#if 0  /* disabled to optimize buffering */
-                AssertLogRelMsgReturn(cb >= 0x00000080 && cb <= 0x000007ff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
-#endif
-                break;
-            default:
-                return VERR_INTERNAL_ERROR;
-        }
-
-        pSSM->u.Read.cbRecLeft = cb;
-    }
-
-    Log3(("ssmR3DataReadRecHdrV2: %08llx|%08llx/%08x: Type=%02x fImportant=%RTbool cbHdr=%u\n",
-          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft,
-          pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK,
-          !!(pSSM->u.Read.u8TypeAndFlags & SSM_REC_FLAGS_IMPORTANT),
-          cbHdr
-        )); NOREF(cbHdr);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Buffer miss, do an unbuffered read.
- *
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           Where to store the read data.
- * @param   cbBuf           Number of bytes to read.
- */
-static int ssmR3DataReadUnbufferedV2(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
-{
-    void   const *pvBufOrg = pvBuf; NOREF(pvBufOrg);
-    size_t const  cbBufOrg = cbBuf; NOREF(cbBufOrg);
-
-    /*
-     * Copy out what we've got in the buffer.
-     */
-    uint32_t off        = pSSM->u.Read.offDataBuffer;
-    int32_t  cbInBuffer = pSSM->u.Read.cbDataBuffer - off;
-    Log4(("ssmR3DataReadUnbufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n", ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, cbInBuffer, cbBufOrg));
-    if (cbInBuffer > 0)
-    {
-        uint32_t const cbToCopy = (uint32_t)cbInBuffer;
-        Assert(cbBuf > cbToCopy);
-        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbToCopy);
-        pvBuf  = (uint8_t *)pvBuf + cbToCopy;
-        cbBuf -= cbToCopy;
-        pSSM->u.Read.cbDataBuffer  = 0;
-        pSSM->u.Read.offDataBuffer = 0;
-    }
-
-    /*
-     * Read data.
-     */
-    do
-    {
-        /*
-         * Read the next record header if no more data.
-         */
-        if (!pSSM->u.Read.cbRecLeft)
-        {
-            int rc = ssmR3DataReadRecHdrV2(pSSM);
-            if (RT_FAILURE(rc))
-                return pSSM->rc = rc;
-        }
-        AssertLogRelMsgReturn(!pSSM->u.Read.fEndOfData, ("cbBuf=%zu", cbBuf), pSSM->rc = VERR_SSM_LOADED_TOO_MUCH);
-
-        /*
-         * Read data from the current record.
-         */
-        uint32_t cbToRead;
-        switch (pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK)
-        {
-            case SSM_REC_TYPE_RAW:
-            {
-                cbToRead = (uint32_t)RT_MIN(cbBuf, pSSM->u.Read.cbRecLeft);
-                int rc = ssmR3DataReadV2Raw(pSSM, pvBuf, cbToRead);
-                if (RT_FAILURE(rc))
-                    return pSSM->rc = rc;
-                pSSM->u.Read.cbRecLeft -= cbToRead;
-                break;
-            }
-
-            case SSM_REC_TYPE_RAW_LZF:
-            {
-                int rc = ssmR3DataReadV2RawLzfHdr(pSSM, &cbToRead);
-                if (RT_FAILURE(rc))
-                    return rc;
-                if (cbToRead <= cbBuf)
-                {
-                    rc = ssmR3DataReadV2RawLzf(pSSM, pvBuf, cbToRead);
-                    if (RT_FAILURE(rc))
-                        return rc;
-                }
-                else
-                {
-                    /* The output buffer is too small, use the data buffer. */
-                    rc = ssmR3DataReadV2RawLzf(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
-                    if (RT_FAILURE(rc))
-                        return rc;
-                    pSSM->u.Read.cbDataBuffer  = cbToRead;
-                    cbToRead = (uint32_t)cbBuf;
-                    pSSM->u.Read.offDataBuffer = cbToRead;
-                    memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[0], cbToRead);
-                }
-                break;
-            }
-
-            case SSM_REC_TYPE_RAW_ZERO:
-            {
-                int rc = ssmR3DataReadV2RawZeroHdr(pSSM, &cbToRead);
-                if (RT_FAILURE(rc))
-                    return rc;
-                if (cbToRead > cbBuf)
-                {
-                    /* Spill the remainder into the data buffer. */
-                    memset(&pSSM->u.Read.abDataBuffer[0], 0, cbToRead - cbBuf);
-                    pSSM->u.Read.cbDataBuffer  = cbToRead - (uint32_t)cbBuf;
-                    pSSM->u.Read.offDataBuffer = 0;
-                    cbToRead = (uint32_t)cbBuf;
-                }
-                memset(pvBuf, 0, cbToRead);
-                break;
-            }
-
-            default:
-                AssertMsgFailedReturn(("%x\n", pSSM->u.Read.u8TypeAndFlags), VERR_INTERNAL_ERROR_5);
-        }
-
-        cbBuf -= cbToRead;
-        pvBuf = (uint8_t *)pvBuf + cbToRead;
-    } while (cbBuf > 0);
-
-    Log4(("ssmR3DataReadUnBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n",
-          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, 0, cbBufOrg, RT_MIN(SSM_LOG_BYTES, cbBufOrg), pvBufOrg, cbBufOrg > SSM_LOG_BYTES ? "..." : ""));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Buffer miss, do a buffered read.
- *
- * @returns VBox status code. Sets pSSM->rc on error.
- *
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           Where to store the read data.
- * @param   cbBuf           Number of bytes to read.
- */
-static int ssmR3DataReadBufferedV2(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
-{
-    void   const *pvBufOrg = pvBuf; NOREF(pvBufOrg);
-    size_t const  cbBufOrg = cbBuf; NOREF(cbBufOrg);
-
-    /*
-     * Copy out what we've got in the buffer.
-     */
-    uint32_t off        = pSSM->u.Read.offDataBuffer;
-    int32_t  cbInBuffer = pSSM->u.Read.cbDataBuffer - off;
-    Log4(("ssmR3DataReadBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n", ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, cbInBuffer, cbBufOrg));
-    if (cbInBuffer > 0)
-    {
-        uint32_t const cbToCopy = (uint32_t)cbInBuffer;
-        Assert(cbBuf > cbToCopy);
-        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbToCopy);
-        pvBuf  = (uint8_t *)pvBuf + cbToCopy;
-        cbBuf -= cbToCopy;
-        pSSM->u.Read.cbDataBuffer  = 0;
-        pSSM->u.Read.offDataBuffer = 0;
-    }
-
-    /*
-     * Buffer more data.
-     */
-    do
-    {
-        /*
-         * Read the next record header if no more data.
-         */
-        if (!pSSM->u.Read.cbRecLeft)
-        {
-            int rc = ssmR3DataReadRecHdrV2(pSSM);
-            if (RT_FAILURE(rc))
-                return pSSM->rc = rc;
-        }
-        AssertLogRelMsgReturn(!pSSM->u.Read.fEndOfData, ("cbBuf=%zu", cbBuf), pSSM->rc = VERR_SSM_LOADED_TOO_MUCH);
-
-        /*
-         * Read data from the current record.
-         * LATER: optimize by reading directly into the output buffer for some cases.
-         */
-        uint32_t cbToRead;
-        switch (pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK)
-        {
-            case SSM_REC_TYPE_RAW:
-            {
-                cbToRead = RT_MIN(sizeof(pSSM->u.Read.abDataBuffer), pSSM->u.Read.cbRecLeft);
-                int rc = ssmR3DataReadV2Raw(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
-                if (RT_FAILURE(rc))
-                    return pSSM->rc = rc;
-                pSSM->u.Read.cbRecLeft   -= cbToRead;
-                pSSM->u.Read.cbDataBuffer = cbToRead;
-                break;
-            }
-
-            case SSM_REC_TYPE_RAW_LZF:
-            {
-                int rc = ssmR3DataReadV2RawLzfHdr(pSSM, &cbToRead);
-                if (RT_FAILURE(rc))
-                    return rc;
-                rc = ssmR3DataReadV2RawLzf(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
-                if (RT_FAILURE(rc))
-                    return rc;
-                pSSM->u.Read.cbDataBuffer = cbToRead;
-                break;
-            }
-
-            case SSM_REC_TYPE_RAW_ZERO:
-            {
-                int rc = ssmR3DataReadV2RawZeroHdr(pSSM, &cbToRead);
-                if (RT_FAILURE(rc))
-                    return rc;
-                memset(&pSSM->u.Read.abDataBuffer[0], 0, cbToRead);
-                pSSM->u.Read.cbDataBuffer = cbToRead;
-                break;
-            }
-
-            default:
-                AssertMsgFailedReturn(("%x\n", pSSM->u.Read.u8TypeAndFlags), VERR_INTERNAL_ERROR_5);
-        }
-        /*pSSM->u.Read.offDataBuffer = 0;*/
-
-        /*
-         * Copy data from the buffer.
-         */
-        uint32_t cbToCopy = (uint32_t)RT_MIN(cbBuf, cbToRead);
-        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[0], cbToCopy);
-        cbBuf -= cbToCopy;
-        pvBuf = (uint8_t *)pvBuf + cbToCopy;
-        pSSM->u.Read.offDataBuffer = cbToCopy;
-    } while (cbBuf > 0);
-
-    Log4(("ssmR3DataReadBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n",
-          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer,
-          cbBufOrg, RT_MIN(SSM_LOG_BYTES, cbBufOrg), pvBufOrg, cbBufOrg > SSM_LOG_BYTES ? "..." : ""));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Inlined worker that handles format checks and buffered reads.
- *
- * @param   pSSM            The saved state handle.
- * @param   pvBuf           Where to store the read data.
- * @param   cbBuf           Number of bytes to read.
- */
-DECLINLINE(int) ssmR3DataRead(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
-{
-    /*
-     * Fend off previous errors and V1 data units.
-     */
-    if (RT_FAILURE(pSSM->rc))
-        return pSSM->rc;
-    if (RT_UNLIKELY(pSSM->u.Read.uFmtVerMajor == 1))
-        return ssmR3DataReadV1(pSSM, pvBuf, cbBuf);
-
-    /*
-     * Check if the requested data is buffered.
-     */
-    uint32_t off = pSSM->u.Read.offDataBuffer;
-    if (    off + cbBuf > pSSM->u.Read.cbDataBuffer
-        ||  cbBuf > sizeof(pSSM->u.Read.abDataBuffer))
-    {
-        if (cbBuf <= sizeof(pSSM->u.Read.abDataBuffer) / 8)
-            return ssmR3DataReadBufferedV2(pSSM, pvBuf, cbBuf);
-        return ssmR3DataReadUnbufferedV2(pSSM, pvBuf, cbBuf);
-    }
-
-    memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbBuf);
-    pSSM->u.Read.offDataBuffer = off + (uint32_t)cbBuf;
-    Log4((cbBuf
-          ? "ssmR3DataRead: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n"
-          : "ssmR3DataRead: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n",
-          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer,
-          cbBuf, RT_MIN(SSM_LOG_BYTES, cbBuf), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets a structure.
- *
- * @returns VBox status code.
- * @param   pSSM            The saved state handle.
- * @param   pvStruct        The structure address.
- * @param   paFields        The array of structure fields descriptions.
- *                          The array must be terminated by a SSMFIELD_ENTRY_TERM().
- */
-VMMR3DECL(int) SSMR3GetStruct(PSSMHANDLE pSSM, void *pvStruct, PCSSMFIELD paFields)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    AssertPtr(pvStruct);
-    AssertPtr(paFields);
-
-    /* begin marker. */
-    uint32_t u32Magic;
-    int rc = SSMR3GetU32(pSSM, &u32Magic);
-    if (RT_FAILURE(rc))
-        return rc;
-    AssertMsgReturn(u32Magic == SSMR3STRUCT_BEGIN, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
-
-    /* get the fields */
-    for (PCSSMFIELD pCur = paFields;
-         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
-         pCur++)
-    {
-        uint8_t *pbField = (uint8_t *)pvStruct + pCur->off;
-        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
-        {
-            case SSMFIELDTRANS_NO_TRANSFORMATION:
-                rc = ssmR3DataRead(pSSM, pbField, pCur->cb);
-                break;
-
-            case SSMFIELDTRANS_GCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3GetGCPtr(pSSM, (PRTGCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_GCPHYS:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3GetGCPhys(pSSM, (PRTGCPHYS)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3GetRCPtr(pSSM, (PRTRCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR_ARRAY:
-            {
-                uint32_t const cEntries = pCur->cb / sizeof(RTRCPTR);
-                AssertMsgReturn(pCur->cb == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = VINF_SUCCESS;
-                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
-                    rc = SSMR3GetRCPtr(pSSM, &((PRTRCPTR)pbField)[i]);
-                break;
-            }
-
-            default:
-                AssertMsgFailedReturn(("%#x\n", pCur->pfnGetPutOrTransformer), VERR_SSM_FIELD_COMPLEX);
-        }
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /* end marker */
-    rc = SSMR3GetU32(pSSM, &u32Magic);
-    if (RT_FAILURE(rc))
-        return rc;
-    AssertMsgReturn(u32Magic == SSMR3STRUCT_END, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
-    return rc;
-}
-
-
-/**
- * SSMR3GetStructEx helper that gets a HCPTR that is used as a NULL indicator.
- *
- * @returns VBox status code.
- *
- * @param   pSSM            The saved state handle.
- * @param   ppv             Where to return the value (0/1).
- * @param   fFlags          SSMSTRUCT_FLAGS_XXX.
- */
-DECLINLINE(int) ssmR3GetHCPtrNI(PSSMHANDLE pSSM, void **ppv, uint32_t fFlags)
-{
-    int rc;
-    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-    {
-        if (ssmR3GetHostBits(pSSM) == 64)
-        {
-            uint64_t u;
-            rc = ssmR3DataRead(pSSM, &u, sizeof(u));
-            if (RT_SUCCESS(rc))
-                *ppv = (void *)(u ? 1 : 0);
-        }
-        else
-        {
-            uint32_t u;
-            rc = ssmR3DataRead(pSSM, &u, sizeof(u));
-            if (RT_SUCCESS(rc))
-                *ppv = (void *)(u ? 1 : 0);
-        }
-    }
-    else
-    {
-        bool f;
-        rc = SSMR3GetBool(pSSM, &f);
-        if (RT_SUCCESS(rc))
-            *ppv = (void *)(f ? 1 : 0);
-    }
-    return rc;
-}
-
-
-/**
- * Guts a structure, extended API.
- *
- * @returns VBox status code.
- * @param   pSSM            The saved state handle.
- * @param   pvStruct        The structure address.
- * @param   cbStruct        The size of the struct (use for validation only).
- * @param   fFlags          Combination of SSMSTRUCT_FLAGS_XXX defines.
- * @param   paFields        The array of structure fields descriptions. The
- *                          array must be terminated by a SSMFIELD_ENTRY_TERM().
- * @param   pvUser          User argument for any callbacks that paFields might
- *                          contain.
- */
-VMMR3DECL(int) SSMR3GetStructEx(PSSMHANDLE pSSM, void *pvStruct, size_t cbStruct,
-                                uint32_t fFlags, PCSSMFIELD paFields, void *pvUser)
-{
-    int         rc;
-    uint32_t    u32Magic;
-
-    /*
-     * Validation.
-     */
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    AssertMsgReturn(!(fFlags & ~SSMSTRUCT_FLAGS_VALID_MASK), ("%#x\n", fFlags), VERR_INVALID_PARAMETER);
-    AssertPtr(pvStruct);
-    AssertPtr(paFields);
-
-    /*
-     * Begin marker.
-     */
-    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
-    {
-        rc = SSMR3GetU32(pSSM, &u32Magic);
-        if (RT_FAILURE(rc))
-            return rc;
-        AssertMsgReturn(u32Magic == SSMR3STRUCT_BEGIN, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
-    }
-
-    /*
-     * Put the fields
-     */
-    uint32_t off = 0;
-    for (PCSSMFIELD pCur = paFields;
-         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
-         pCur++)
-    {
-        uint32_t const offField =    (!SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer) || pCur->off != UINT32_MAX / 2)
-                                  &&  !SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
-                                ? pCur->off
-                                : off;
-        uint32_t const cbField  = SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
-                                ? 0
-                                : SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer)
-                                ? RT_HIWORD(pCur->cb)
-                                : pCur->cb;
-        AssertMsgReturn(   cbField            <= cbStruct
-                        && offField + cbField <= cbStruct
-                        && offField + cbField >= offField,
-                        ("off=%#x cb=%#x cbStruct=%#x (%s)\n", cbField, offField, cbStruct, pCur->pszName),
-                        VERR_SSM_FIELD_OUT_OF_BOUNDS);
-        AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
-                        || off == offField,
-                        ("off=%#x offField=%#x (%s)\n", off, offField, pCur->pszName),
-                        VERR_SSM_FIELD_NOT_CONSECUTIVE);
-
-        rc = VINF_SUCCESS;
-        uint8_t       *pbField  = (uint8_t *)pvStruct + offField;
-        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
-        {
-            case SSMFIELDTRANS_NO_TRANSFORMATION:
-                rc = ssmR3DataRead(pSSM, pbField, cbField);
-                break;
-
-            case SSMFIELDTRANS_GCPHYS:
-                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3GetGCPhys(pSSM, (PRTGCPHYS)pbField);
-                break;
-
-            case SSMFIELDTRANS_GCPTR:
-                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3GetGCPtr(pSSM, (PRTGCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR:
-                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3GetRCPtr(pSSM, (PRTRCPTR)pbField);
-                break;
-
-            case SSMFIELDTRANS_RCPTR_ARRAY:
-            {
-                uint32_t const cEntries = cbField / sizeof(RTRCPTR);
-                AssertMsgReturn(cbField == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = VINF_SUCCESS;
-                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
-                    rc = SSMR3GetRCPtr(pSSM, &((PRTRCPTR)pbField)[i]);
-                break;
-            }
-
-            case SSMFIELDTRANS_HCPTR_NI:
-                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = ssmR3GetHCPtrNI(pSSM, (void **)pbField, fFlags);
-                break;
-
-            case SSMFIELDTRANS_HCPTR_NI_ARRAY:
-            {
-                uint32_t const cEntries = cbField / sizeof(void *);
-                AssertMsgReturn(cbField == cEntries * sizeof(void *) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = VINF_SUCCESS;
-                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
-                    rc = ssmR3GetHCPtrNI(pSSM, &((void **)pbField)[i], fFlags);
-                break;
-            }
-
-            case SSMFIELDTRANS_HCPTR_HACK_U32:
-                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                *(uintptr_t *)pbField = 0;
-                rc = ssmR3DataRead(pSSM, pbField, sizeof(uint32_t));
-                if ((fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE) && ssmR3GetHostBits(pSSM) == 64)
-                {
-                    uint32_t u32;
-                    rc = ssmR3DataRead(pSSM, &u32, sizeof(uint32_t));
-                    AssertMsgReturn(RT_FAILURE(rc) || u32 == 0 || (fFlags & SSMSTRUCT_FLAGS_SAVED_AS_MEM),
-                                    ("high=%#x low=%#x (%s)\n", u32, *(uint32_t *)pbField, pCur->pszName),
-                                    VERR_SSM_FIELD_INVALID_VALUE);
-                }
-                break;
-
-
-            case SSMFIELDTRANS_IGNORE:
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = SSMR3Skip(pSSM, cbField);
-                break;
-
-            case SSMFIELDTRANS_IGN_GCPHYS:
-                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPhys);
-                break;
-
-            case SSMFIELDTRANS_IGN_GCPTR:
-                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPtr);
-                break;
-
-            case SSMFIELDTRANS_IGN_RCPTR:
-                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = SSMR3Skip(pSSM, sizeof(RTRCPTR));
-                break;
-
-            case SSMFIELDTRANS_IGN_HCPTR:
-                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) / 8);
-                break;
-
-
-            case SSMFIELDTRANS_OLD:
-                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3Skip(pSSM, pCur->cb);
-                break;
-
-            case SSMFIELDTRANS_OLD_GCPHYS:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPhys);
-                break;
-
-            case SSMFIELDTRANS_OLD_GCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPtr);
-                break;
-
-            case SSMFIELDTRANS_OLD_RCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3Skip(pSSM, sizeof(RTRCPTR));
-                break;
-
-            case SSMFIELDTRANS_OLD_HCPTR:
-                AssertMsgReturn(pCur->cb == sizeof(void *) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) / 8);
-                break;
-
-            case SSMFIELDTRANS_OLD_PAD_HC:
-                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) == 64 ? RT_HIWORD(pCur->cb) : RT_LOWORD(pCur->cb));
-                break;
-
-            case SSMFIELDTRANS_OLD_PAD_MSC32:
-                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
-                if (ssmR3IsHostMsc32(pSSM))
-                    rc = SSMR3Skip(pSSM, pCur->cb);
-                break;
-
-
-            case SSMFIELDTRANS_PAD_HC:
-            case SSMFIELDTRANS_PAD_HC32:
-            case SSMFIELDTRANS_PAD_HC64:
-            case SSMFIELDTRANS_PAD_HC_AUTO:
-            case SSMFIELDTRANS_PAD_MSC32_AUTO:
-            {
-                uint32_t cb32    = RT_BYTE1(pCur->cb);
-                uint32_t cb64    = RT_BYTE2(pCur->cb);
-                uint32_t cbCtx   =    HC_ARCH_BITS == 64
-                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
-                                       && !SSM_HOST_IS_MSC_32)
-                                 ? cb64 : cb32;
-                uint32_t cbSaved =    ssmR3GetHostBits(pSSM) == 64
-                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
-                                       && !ssmR3IsHostMsc32(pSSM))
-                                 ? cb64 : cb32;
-                AssertMsgReturn(    cbField == cbCtx
-                                &&  (   (   pCur->off == UINT32_MAX / 2
-                                         && (   cbField == 0
-                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_HC_AUTO
-                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
-                                            )
-                                        )
-                                     || (pCur->off != UINT32_MAX / 2 && cbField != 0)
-                                     )
-                                , ("cbField=%#x cb32=%#x cb64=%#x HC_ARCH_BITS=%u cbCtx=%#x cbSaved=%#x off=%#x\n",
-                                   cbField, cb32, cb64, HC_ARCH_BITS, cbCtx, cbSaved, pCur->off),
-                                VERR_SSM_FIELD_INVALID_PADDING_SIZE);
-                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
-                    rc = SSMR3Skip(pSSM, cbSaved);
-                break;
-            }
-
-            default:
-                AssertPtrReturn(pCur->pfnGetPutOrTransformer, VERR_SSM_FIELD_INVALID_CALLBACK);
-                rc = pCur->pfnGetPutOrTransformer(pSSM, pCur, pvStruct, fFlags, true /*fGetOrPut*/, pvUser);
-                break;
-        }
-        if (RT_FAILURE(rc))
-            return rc;
-
-        off = offField + cbField;
-    }
-    AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
-                    || off == cbStruct,
-                    ("off=%#x cbStruct=%#x\n", off, cbStruct),
-                    VERR_SSM_FIELD_NOT_CONSECUTIVE);
-
-    /*
-     * End marker
-     */
-    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
-    {
-        rc = SSMR3GetU32(pSSM, &u32Magic);
-        if (RT_FAILURE(rc))
-            return rc;
-        AssertMsgReturn(u32Magic == SSMR3STRUCT_END, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Loads a boolean item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pfBool          Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetBool(PSSMHANDLE pSSM, bool *pfBool)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    uint8_t u8; /* see SSMR3PutBool */
-    int rc = ssmR3DataRead(pSSM, &u8, sizeof(u8));
-    if (RT_SUCCESS(rc))
-    {
-        Assert(u8 <= 1);
-        *pfBool = !!u8;
-    }
-    return rc;
-}
-
-
-/**
- * Loads a 8-bit unsigned integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu8             Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetU8(PSSMHANDLE pSSM, uint8_t *pu8)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pu8, sizeof(*pu8));
-}
-
-
-/**
- * Loads a 8-bit signed integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pi8             Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetS8(PSSMHANDLE pSSM, int8_t *pi8)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pi8, sizeof(*pi8));
-}
-
-
-/**
- * Loads a 16-bit unsigned integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu16            Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetU16(PSSMHANDLE pSSM, uint16_t *pu16)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pu16, sizeof(*pu16));
-}
-
-
-/**
- * Loads a 16-bit signed integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pi16            Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetS16(PSSMHANDLE pSSM, int16_t *pi16)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pi16, sizeof(*pi16));
-}
-
-
-/**
- * Loads a 32-bit unsigned integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu32            Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetU32(PSSMHANDLE pSSM, uint32_t *pu32)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pu32, sizeof(*pu32));
-}
-
-
-/**
- * Loads a 32-bit signed integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pi32            Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetS32(PSSMHANDLE pSSM, int32_t *pi32)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pi32, sizeof(*pi32));
-}
-
-
-/**
- * Loads a 64-bit unsigned integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu64            Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetU64(PSSMHANDLE pSSM, uint64_t *pu64)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pu64, sizeof(*pu64));
-}
-
-
-/**
- * Loads a 64-bit signed integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pi64            Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetS64(PSSMHANDLE pSSM, int64_t *pi64)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pi64, sizeof(*pi64));
-}
-
-
-/**
- * Loads a 128-bit unsigned integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu128           Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetU128(PSSMHANDLE pSSM, uint128_t *pu128)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pu128, sizeof(*pu128));
-}
-
-
-/**
- * Loads a 128-bit signed integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pi128           Where to store the item.
- */
-VMMR3DECL(int) SSMR3GetS128(PSSMHANDLE pSSM, int128_t *pi128)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pi128, sizeof(*pi128));
-}
-
-
-/**
- * Loads a VBox unsigned integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu              Where to store the integer.
- */
-VMMR3DECL(int) SSMR3GetUInt(PSSMHANDLE pSSM, PRTUINT pu)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pu, sizeof(*pu));
-}
-
-
-/**
- * Loads a VBox signed integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pi              Where to store the integer.
- */
-VMMR3DECL(int) SSMR3GetSInt(PSSMHANDLE pSSM, PRTINT pi)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pi, sizeof(*pi));
-}
-
-
-/**
- * Loads a GC natural unsigned integer item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu              Where to store the integer.
- *
- * @deprecated Silly type with an incorrect size, don't use it.
- */
-VMMR3DECL(int) SSMR3GetGCUInt(PSSMHANDLE pSSM, PRTGCUINT pu)
-{
-    AssertCompile(sizeof(RTGCPTR) == sizeof(*pu));
-    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pu);
-}
-
-
-/**
- * Loads a GC unsigned integer register item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pu              Where to store the integer.
- */
-VMMR3DECL(int) SSMR3GetGCUIntReg(PSSMHANDLE pSSM, PRTGCUINTREG pu)
-{
-    AssertCompile(sizeof(RTGCPTR) == sizeof(*pu));
-    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pu);
-}
-
-
-/**
- * Loads a 32 bits GC physical address item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pGCPhys         Where to store the GC physical address.
- */
-VMMR3DECL(int) SSMR3GetGCPhys32(PSSMHANDLE pSSM, PRTGCPHYS32 pGCPhys)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
-}
-
-
-/**
- * Loads a 64 bits GC physical address item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pGCPhys         Where to store the GC physical address.
- */
-VMMR3DECL(int) SSMR3GetGCPhys64(PSSMHANDLE pSSM, PRTGCPHYS64 pGCPhys)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
-}
-
-
-/**
- * Loads a GC physical address item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pGCPhys         Where to store the GC physical address.
- */
-VMMR3DECL(int) SSMR3GetGCPhys(PSSMHANDLE pSSM, PRTGCPHYS pGCPhys)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-
-    /*
-     * Default size?
-     */
-    if (RT_LIKELY(sizeof(*pGCPhys) == pSSM->u.Read.cbGCPhys))
-        return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
-
-    /*
-     * Fiddly.
-     */
-    Assert(sizeof(*pGCPhys)      == sizeof(uint64_t) || sizeof(*pGCPhys)      == sizeof(uint32_t));
-    Assert(pSSM->u.Read.cbGCPhys == sizeof(uint64_t) || pSSM->u.Read.cbGCPhys == sizeof(uint32_t));
-    if (pSSM->u.Read.cbGCPhys == sizeof(uint64_t))
-    {
-        /* 64-bit saved, 32-bit load: try truncate it. */
-        uint64_t u64;
-        int rc = ssmR3DataRead(pSSM, &u64, sizeof(uint64_t));
-        if (RT_FAILURE(rc))
-            return rc;
-        if (u64 >= _4G)
-            return VERR_SSM_GCPHYS_OVERFLOW;
-        *pGCPhys = (RTGCPHYS)u64;
-        return rc;
-    }
-
-    /* 32-bit saved, 64-bit load: clear the high part. */
-    *pGCPhys = 0;
-    return ssmR3DataRead(pSSM, pGCPhys, sizeof(uint32_t));
-}
-
-
-/**
- * Loads a GC virtual address item from the current data unit.
- *
- * Only applies to in the 1.1 format:
- *  - SSMR3GetGCPtr
- *  - SSMR3GetGCUIntPtr
- *  - SSMR3GetGCUInt
- *  - SSMR3GetGCUIntReg
- *
- * Put functions are not affected.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   cbGCPtr         Size of RTGCPTR
- *
- * @remarks This interface only works with saved state version 1.1, if the
- *          format isn't 1.1 the call will be ignored.
- */
-VMMR3_INT_DECL(int) SSMR3HandleSetGCPtrSize(PSSMHANDLE pSSM, unsigned cbGCPtr)
-{
-    Assert(cbGCPtr == sizeof(RTGCPTR32) || cbGCPtr == sizeof(RTGCPTR64));
-    if (!pSSM->u.Read.fFixedGCPtrSize)
-    {
-        Log(("SSMR3SetGCPtrSize: %u -> %u bytes\n", pSSM->u.Read.cbGCPtr, cbGCPtr));
-        pSSM->u.Read.cbGCPtr = cbGCPtr;
-        pSSM->u.Read.fFixedGCPtrSize = true;
-    }
-    else if (   pSSM->u.Read.cbGCPtr != cbGCPtr
-             && pSSM->u.Read.uFmtVerMajor == 1
-             && pSSM->u.Read.uFmtVerMinor == 1)
-        AssertMsgFailed(("SSMR3SetGCPtrSize: already fixed at %u bytes; requested %u bytes\n", pSSM->u.Read.cbGCPtr, cbGCPtr));
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Loads a GC virtual address item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pGCPtr          Where to store the GC virtual address.
- */
-VMMR3DECL(int) SSMR3GetGCPtr(PSSMHANDLE pSSM, PRTGCPTR pGCPtr)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-
-    /*
-     * Default size?
-     */
-    if (RT_LIKELY(sizeof(*pGCPtr) == pSSM->u.Read.cbGCPtr))
-        return ssmR3DataRead(pSSM, pGCPtr, sizeof(*pGCPtr));
-
-    /*
-     * Fiddly.
-     */
-    Assert(sizeof(*pGCPtr)      == sizeof(uint64_t) || sizeof(*pGCPtr)      == sizeof(uint32_t));
-    Assert(pSSM->u.Read.cbGCPtr == sizeof(uint64_t) || pSSM->u.Read.cbGCPtr == sizeof(uint32_t));
-    if (pSSM->u.Read.cbGCPtr == sizeof(uint64_t))
-    {
-        /* 64-bit saved, 32-bit load: try truncate it. */
-        uint64_t u64;
-        int rc = ssmR3DataRead(pSSM, &u64, sizeof(uint64_t));
-        if (RT_FAILURE(rc))
-            return rc;
-        if (u64 >= _4G)
-            return VERR_SSM_GCPTR_OVERFLOW;
-        *pGCPtr = (RTGCPTR)u64;
-        return rc;
-    }
-
-    /* 32-bit saved, 64-bit load: clear the high part. */
-    *pGCPtr = 0;
-    return ssmR3DataRead(pSSM, pGCPtr, sizeof(uint32_t));
-}
-
-
-/**
- * Loads a GC virtual address (represented as unsigned integer) item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pGCPtr          Where to store the GC virtual address.
- */
-VMMR3DECL(int) SSMR3GetGCUIntPtr(PSSMHANDLE pSSM, PRTGCUINTPTR pGCPtr)
-{
-    AssertCompile(sizeof(RTGCPTR) == sizeof(*pGCPtr));
-    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pGCPtr);
-}
-
-
-/**
- * Loads an RC virtual address item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pRCPtr          Where to store the RC virtual address.
- */
-VMMR3DECL(int) SSMR3GetRCPtr(PSSMHANDLE pSSM, PRTRCPTR pRCPtr)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pRCPtr, sizeof(*pRCPtr));
-}
-
-
-/**
- * Loads a I/O port address item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pIOPort         Where to store the I/O port address.
- */
-VMMR3DECL(int) SSMR3GetIOPort(PSSMHANDLE pSSM, PRTIOPORT pIOPort)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pIOPort, sizeof(*pIOPort));
-}
-
-
-/**
- * Loads a selector item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pSel            Where to store the selector.
- */
-VMMR3DECL(int) SSMR3GetSel(PSSMHANDLE pSSM, PRTSEL pSel)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pSel, sizeof(*pSel));
-}
-
-
-/**
- * Loads a memory item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   pv              Where to store the item.
- * @param   cb              Size of the item.
- */
-VMMR3DECL(int) SSMR3GetMem(PSSMHANDLE pSSM, void *pv, size_t cb)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    return ssmR3DataRead(pSSM, pv, cb);
-}
-
-
-/**
- * Loads a string item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   psz             Where to store the item.
- * @param   cbMax           Max size of the item (including '\\0').
- */
-VMMR3DECL(int) SSMR3GetStrZ(PSSMHANDLE pSSM, char *psz, size_t cbMax)
-{
-    return SSMR3GetStrZEx(pSSM, psz, cbMax, NULL);
-}
-
-
-/**
- * Loads a string item from the current data unit.
- *
- * @returns VBox status.
- * @param   pSSM            The saved state handle.
- * @param   psz             Where to store the item.
- * @param   cbMax           Max size of the item (including '\\0').
- * @param   pcbStr          The length of the loaded string excluding the '\\0'. (optional)
- */
-VMMR3DECL(int) SSMR3GetStrZEx(PSSMHANDLE pSSM, char *psz, size_t cbMax, size_t *pcbStr)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-
-    /* read size prefix. */
-    uint32_t u32;
-    int rc = SSMR3GetU32(pSSM, &u32);
-    if (RT_SUCCESS(rc))
-    {
-        if (pcbStr)
-            *pcbStr = u32;
-        if (u32 < cbMax)
-        {
-            /* terminate and read string content. */
-            psz[u32] = '\0';
-            return ssmR3DataRead(pSSM, psz, u32);
-        }
-        return VERR_TOO_MUCH_DATA;
-    }
-    return rc;
-}
-
-
-/**
- * Skips a number of bytes in the current data unit.
- *
- * @returns VBox status code.
- * @param   pSSM                The SSM handle.
- * @param   cb                  The number of bytes to skip.
- */
-VMMR3DECL(int) SSMR3Skip(PSSMHANDLE pSSM, size_t cb)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    while (cb > 0)
-    {
-        uint8_t abBuf[8192];
-        size_t  cbCur = RT_MIN(sizeof(abBuf), cb);
-        cb -= cbCur;
-        int rc = ssmR3DataRead(pSSM, abBuf, cbCur);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Skips to the end of the current data unit.
- *
- * Since version 2 of the format, the load exec callback have to explicitly call
- * this API if it wish to be lazy for some reason.  This is because there seldom
- * is a good reason to not read your entire data unit and it was hiding bugs.
- *
- * @returns VBox status code.
- * @param   pSSM                The saved state handle.
- */
-VMMR3DECL(int) SSMR3SkipToEndOfUnit(PSSMHANDLE pSSM)
-{
-    SSM_ASSERT_READABLE_RET(pSSM);
-    SSM_CHECK_CANCELLED_RET(pSSM);
-    if (pSSM->u.Read.uFmtVerMajor >= 2)
-    {
-        /*
-         * Read until we the end of data condition is raised.
-         */
-        pSSM->u.Read.cbDataBuffer  = 0;
-        pSSM->u.Read.offDataBuffer = 0;
-        if (!pSSM->u.Read.fEndOfData)
-        {
-            do
-            {
-                /* read the rest of the current record */
-                while (pSSM->u.Read.cbRecLeft)
-                {
-                    uint8_t  abBuf[8192];
-                    uint32_t cbToRead = RT_MIN(pSSM->u.Read.cbRecLeft, sizeof(abBuf));
-                    int rc = ssmR3DataReadV2Raw(pSSM, abBuf, cbToRead);
-                    if (RT_FAILURE(rc))
-                        return pSSM->rc = rc;
-                    pSSM->u.Read.cbRecLeft -= cbToRead;
-                }
-
-                /* read the next header. */
-                int rc = ssmR3DataReadRecHdrV2(pSSM);
-                if (RT_FAILURE(rc))
-                    return pSSM->rc = rc;
-            } while (!pSSM->u.Read.fEndOfData);
-        }
-    }
-    /* else: Doesn't matter for the version 1 loading. */
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Calculate the checksum of a file portion.
- *
- * @returns VBox status.
- * @param   pStrm       The stream handle
- * @param   off         Where to start checksumming.
- * @param   cb          How much to checksum.
- * @param   pu32CRC     Where to store the calculated checksum.
- */
-static int ssmR3CalcChecksum(PSSMSTRM pStrm, uint64_t off, uint64_t cb, uint32_t *pu32CRC)
-{
-    /*
-     * Allocate a buffer.
-     */
-    const size_t cbBuf = _32K;
-    void *pvBuf = RTMemTmpAlloc(cbBuf);
-    if (!pvBuf)
-        return VERR_NO_TMP_MEMORY;
-
-    /*
-     * Loop reading and calculating CRC32.
-     */
-    int         rc     = VINF_SUCCESS;
-    uint32_t    u32CRC = RTCrc32Start();
-    while (cb > 0)
-    {
-        /* read chunk */
-        size_t cbToRead = cbBuf;
-        if (cb < cbBuf)
-            cbToRead = cb;
-        rc = ssmR3StrmPeekAt(pStrm, off, pvBuf, cbToRead, NULL);
-        if (RT_FAILURE(rc))
-        {
-            AssertMsgFailed(("Failed with rc=%Rrc while calculating crc.\n", rc));
-            RTMemTmpFree(pvBuf);
-            return rc;
-        }
-
-        /* advance */
-        cb  -= cbToRead;
-        off += cbToRead;
-
-        /* calc crc32. */
-        u32CRC = RTCrc32Process(u32CRC, pvBuf, cbToRead);
-    }
-    RTMemTmpFree(pvBuf);
-
-    /* store the calculated crc */
-    u32CRC = RTCrc32Finish(u32CRC);
-    Log(("SSM: u32CRC=0x%08x\n", u32CRC));
-    *pu32CRC = u32CRC;
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Validates a version 2 footer.
- *
- * @returns VBox status code.
- *
- * @param   pFooter             The footer.
- * @param   offFooter           The stream offset of the footer.
- * @param   cDirEntries         The number of directory entries. UINT32_MAX if
- *                              unknown.
- * @param   fStreamCrc32        Whether the stream is checksummed using CRC-32.
- * @param   u32StreamCRC        The stream checksum.
- */
-static int ssmR3ValidateFooter(PSSMFILEFTR pFooter, uint64_t offFooter, uint32_t cDirEntries, bool fStreamCrc32, uint32_t u32StreamCRC)
-{
-    if (memcmp(pFooter->szMagic, SSMFILEFTR_MAGIC, sizeof(pFooter->szMagic)))
-    {
-        LogRel(("SSM: Bad footer magic: %.*Rhxs\n", sizeof(pFooter->szMagic), &pFooter->szMagic[0]));
-        return VERR_SSM_INTEGRITY_FOOTER;
-    }
-    SSM_CHECK_CRC32_RET(pFooter, sizeof(*pFooter), ("Footer CRC mismatch: %08x, correct is %08x\n", u32CRC, u32ActualCRC));
-    if (pFooter->offStream != offFooter)
-    {
-        LogRel(("SSM: SSMFILEFTR::offStream is wrong: %llx, expected %llx\n", pFooter->offStream, offFooter));
-        return VERR_SSM_INTEGRITY_FOOTER;
-    }
-    if (pFooter->u32Reserved)
-    {
-        LogRel(("SSM: Reserved footer field isn't zero: %08x\n", pFooter->u32Reserved));
-        return VERR_SSM_INTEGRITY_FOOTER;
-    }
-    if (cDirEntries != UINT32_MAX)
-        AssertLogRelMsgReturn(pFooter->cDirEntries == cDirEntries,
-                              ("Footer: cDirEntries=%#x, expected %#x\n", pFooter->cDirEntries, cDirEntries),
-                              VERR_SSM_INTEGRITY_FOOTER);
-    else
-        AssertLogRelMsgReturn(pFooter->cDirEntries < _64K,
-                              ("Footer: cDirEntries=%#x\n", pFooter->cDirEntries),
-                              VERR_SSM_INTEGRITY_FOOTER);
-    if (    !fStreamCrc32
-        &&  pFooter->u32StreamCRC)
-    {
-        LogRel(("SSM: u32StreamCRC field isn't zero, but header says stream checksumming is disabled.\n"));
-        return VERR_SSM_INTEGRITY_FOOTER;
-    }
-    if (    fStreamCrc32
-        &&  pFooter->u32StreamCRC != u32StreamCRC)
-    {
-        LogRel(("SSM: Bad stream CRC: %#x, expected %#x.\n", pFooter->u32StreamCRC, u32StreamCRC));
-        return VERR_SSM_INTEGRITY_CRC;
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Validates the header information stored in the handle.
- *
- * @returns VBox status code.
- *
- * @param   pSSM            The handle.
- * @param   fHaveHostBits   Set if the host bits field is valid.
- * @param   fHaveVersion    Set if we have a version.
- */
-static int ssmR3ValidateHeaderInfo(PSSMHANDLE pSSM, bool fHaveHostBits, bool fHaveVersion)
-{
-    Assert(pSSM->u.Read.cbFileHdr < 256 && pSSM->u.Read.cbFileHdr > 32);
-    Assert(pSSM->u.Read.uFmtVerMajor == 1 || pSSM->u.Read.uFmtVerMajor == 2);
-    Assert(pSSM->u.Read.uFmtVerMinor <= 2);
-
-    if (fHaveVersion)
-    {
-        if (    pSSM->u.Read.u16VerMajor == 0
-            ||  pSSM->u.Read.u16VerMajor > 1000
-            ||  pSSM->u.Read.u16VerMinor > 1000
-            ||  pSSM->u.Read.u32VerBuild > _1M
-            ||  pSSM->u.Read.u32SvnRev == 0
-            ||  pSSM->u.Read.u32SvnRev > 10000000 /*100M*/)
-        {
-            LogRel(("SSM: Incorrect version values: %u.%u.%u.r%u\n",
-                    pSSM->u.Read.u16VerMajor, pSSM->u.Read.u16VerMinor, pSSM->u.Read.u32VerBuild, pSSM->u.Read.u32SvnRev));
-            return VERR_SSM_INTEGRITY_VBOX_VERSION;
-        }
-    }
-    else
-        AssertLogRelReturn(   pSSM->u.Read.u16VerMajor == 0
-                           && pSSM->u.Read.u16VerMinor == 0
-                           && pSSM->u.Read.u32VerBuild == 0
-                           && pSSM->u.Read.u32SvnRev   == 0,
-                           VERR_SSM_INTEGRITY_VBOX_VERSION);
-
-    if (fHaveHostBits)
-    {
-        if (    pSSM->u.Read.cHostBits != 32
-            &&  pSSM->u.Read.cHostBits != 64)
-        {
-            LogRel(("SSM: Incorrect cHostBits value: %u\n", pSSM->u.Read.cHostBits));
-            return VERR_SSM_INTEGRITY_HEADER;
-        }
-    }
-    else
-        AssertLogRelReturn(pSSM->u.Read.cHostBits == 0, VERR_SSM_INTEGRITY_HEADER);
-
-    if (    pSSM->u.Read.cbGCPhys != sizeof(uint32_t)
-        &&  pSSM->u.Read.cbGCPhys != sizeof(uint64_t))
-    {
-        LogRel(("SSM: Incorrect cbGCPhys value: %d\n", pSSM->u.Read.cbGCPhys));
-        return VERR_SSM_INTEGRITY_HEADER;
-    }
-    if (    pSSM->u.Read.cbGCPtr != sizeof(uint32_t)
-        &&  pSSM->u.Read.cbGCPtr != sizeof(uint64_t))
-    {
-        LogRel(("SSM: Incorrect cbGCPtr value: %d\n", pSSM->u.Read.cbGCPtr));
-        return VERR_SSM_INTEGRITY_HEADER;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Reads the header, detects the format version and performs integrity
- * validations.
- *
- * @returns VBox status.
- * @param   pSSM                The saved state handle.  A number of field will
- *                              be updated, mostly header related information.
- *                              fLiveSave is also set if appropriate.
- * @param   fChecksumIt         Whether to checksum the file or not.  This will
- *                              be ignored if it the stream isn't a file.
- * @param   fChecksumOnRead     Whether to validate the checksum while reading
- *                              the stream instead of up front. If not possible,
- *                              verify the checksum up front.
- * @param   pHdr                Where to store the file header.
- */
-static int ssmR3HeaderAndValidate(PSSMHANDLE pSSM, bool fChecksumIt, bool fChecksumOnRead)
-{
-    /*
-     * Read and check the header magic.
-     */
-    union
-    {
-        SSMFILEHDR          v2_0;
-        SSMFILEHDRV12       v1_2;
-        SSMFILEHDRV11       v1_1;
-    } uHdr;
-    int rc = ssmR3StrmRead(&pSSM->Strm, &uHdr, sizeof(uHdr.v2_0.szMagic));
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("SSM: Failed to read file magic header. rc=%Rrc\n", rc));
-        return rc;
-    }
-    if (memcmp(uHdr.v2_0.szMagic, SSMFILEHDR_MAGIC_BASE, sizeof(SSMFILEHDR_MAGIC_BASE) - 1))
-    {
-        Log(("SSM: Not a saved state file. magic=%.*s\n", sizeof(uHdr.v2_0.szMagic) - 1, uHdr.v2_0.szMagic));
-        return VERR_SSM_INTEGRITY_MAGIC;
-    }
-
-    /*
-     * Find the header size and read the rest.
-     */
-    static const struct
-    {
-        char        szMagic[sizeof(SSMFILEHDR_MAGIC_V2_0)];
-        uint32_t    cbHdr;
-        unsigned    uFmtVerMajor;
-        unsigned    uFmtVerMinor;
-    }   s_aVers[] =
-    {
-        { SSMFILEHDR_MAGIC_V2_0, sizeof(SSMFILEHDR),    2, 0 },
-        { SSMFILEHDR_MAGIC_V1_2, sizeof(SSMFILEHDRV12), 1, 2 },
-        { SSMFILEHDR_MAGIC_V1_1, sizeof(SSMFILEHDRV11), 1, 1 },
-    };
-    int iVer = RT_ELEMENTS(s_aVers);
-    while (iVer-- > 0)
-        if (!memcmp(uHdr.v2_0.szMagic, s_aVers[iVer].szMagic, sizeof(uHdr.v2_0.szMagic)))
-            break;
-    if (iVer < 0)
-    {
-        Log(("SSM: Unknown file format version. magic=%.*s\n", sizeof(uHdr.v2_0.szMagic) - 1, uHdr.v2_0.szMagic));
-        return VERR_SSM_INTEGRITY_VERSION;
-    }
-    pSSM->u.Read.uFmtVerMajor   = s_aVers[iVer].uFmtVerMajor;
-    pSSM->u.Read.uFmtVerMinor   = s_aVers[iVer].uFmtVerMinor;
-    pSSM->u.Read.cbFileHdr      = s_aVers[iVer].cbHdr;
-
-    rc = ssmR3StrmRead(&pSSM->Strm, (uint8_t *)&uHdr + sizeof(uHdr.v2_0.szMagic), pSSM->u.Read.cbFileHdr - sizeof(uHdr.v2_0.szMagic));
-    if (RT_FAILURE(rc))
-    {
-        LogRel(("SSM: Failed to read the file header. rc=%Rrc\n", rc));
-        return rc;
-    }
-
-    /*
-     * Make version specific adjustments.
-     */
-    if (pSSM->u.Read.uFmtVerMajor >= 2)
-    {
-        /*
-         * Version 2.0 and later.
-         */
-        if (pSSM->u.Read.uFmtVerMinor == 0)
-        {
-            /* validate the header. */
-            SSM_CHECK_CRC32_RET(&uHdr.v2_0, sizeof(uHdr.v2_0), ("Header CRC mismatch: %08x, correct is %08x\n", u32CRC, u32ActualCRC));
-            if (uHdr.v2_0.u8Reserved)
-            {
-                LogRel(("SSM: Reserved header field isn't zero: %02x\n", uHdr.v2_0.u8Reserved));
-                return VERR_SSM_INTEGRITY;
-            }
-            if (uHdr.v2_0.fFlags & ~(SSMFILEHDR_FLAGS_STREAM_CRC32 | SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE))
-            {
-                LogRel(("SSM: Unknown header flags: %08x\n", uHdr.v2_0.fFlags));
-                return VERR_SSM_INTEGRITY;
-            }
-            if (    uHdr.v2_0.cbMaxDecompr > sizeof(pSSM->u.Read.abDataBuffer)
-                ||  uHdr.v2_0.cbMaxDecompr < _1K
-                ||  (uHdr.v2_0.cbMaxDecompr & 0xff) != 0)
-            {
-                LogRel(("SSM: The cbMaxDecompr header field is out of range: %#x\n", uHdr.v2_0.cbMaxDecompr));
-                return VERR_SSM_INTEGRITY;
-            }
-
-            /* set the header info. */
-            pSSM->u.Read.cHostBits      = uHdr.v2_0.cHostBits;
-            pSSM->u.Read.u16VerMajor    = uHdr.v2_0.u16VerMajor;
-            pSSM->u.Read.u16VerMinor    = uHdr.v2_0.u16VerMinor;
-            pSSM->u.Read.u32VerBuild    = uHdr.v2_0.u32VerBuild;
-            pSSM->u.Read.u32SvnRev      = uHdr.v2_0.u32SvnRev;
-            pSSM->u.Read.cbGCPhys       = uHdr.v2_0.cbGCPhys;
-            pSSM->u.Read.cbGCPtr        = uHdr.v2_0.cbGCPtr;
-            pSSM->u.Read.fFixedGCPtrSize= true;
-            pSSM->u.Read.fStreamCrc32   = !!(uHdr.v2_0.fFlags & SSMFILEHDR_FLAGS_STREAM_CRC32);
-            pSSM->fLiveSave             = !!(uHdr.v2_0.fFlags & SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE);
-        }
-        else
-            AssertFailedReturn(VERR_INTERNAL_ERROR);
-        if (!pSSM->u.Read.fStreamCrc32)
-            ssmR3StrmDisableChecksumming(&pSSM->Strm);
-
-        /*
-         * Read and validate the footer if it's a file.
-         */
-        if (ssmR3StrmIsFile(&pSSM->Strm))
-        {
-            SSMFILEFTR  Footer;
-            uint64_t    offFooter;
-            rc = ssmR3StrmPeekAt(&pSSM->Strm, -(RTFOFF)sizeof(SSMFILEFTR), &Footer, sizeof(Footer), &offFooter);
-            AssertLogRelRCReturn(rc, rc);
-
-            rc = ssmR3ValidateFooter(&Footer, offFooter, UINT32_MAX, pSSM->u.Read.fStreamCrc32, Footer.u32StreamCRC);
-            if (RT_FAILURE(rc))
-                return rc;
-
-            pSSM->u.Read.cbLoadFile = offFooter + sizeof(Footer);
-            pSSM->u.Read.u32LoadCRC = Footer.u32StreamCRC;
-        }
-        else
-        {
-            pSSM->u.Read.cbLoadFile = UINT64_MAX;
-            pSSM->u.Read.u32LoadCRC = 0;
-        }
-
-        /*
-         * Validate the header info we've set in the handle.
-         */
-        rc = ssmR3ValidateHeaderInfo(pSSM, true /*fHaveHostBits*/, true /*fHaveVersion*/);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /*
-         * Check the checksum if that's called for and possible.
-         */
-        if (    pSSM->u.Read.fStreamCrc32
-            &&  fChecksumIt
-            &&  !fChecksumOnRead
-            &&  ssmR3StrmIsFile(&pSSM->Strm))
-        {
-            uint32_t u32CRC;
-            rc = ssmR3CalcChecksum(&pSSM->Strm, 0, pSSM->u.Read.cbLoadFile - sizeof(SSMFILEFTR), &u32CRC);
-            if (RT_FAILURE(rc))
-                return rc;
-            if (u32CRC != pSSM->u.Read.u32LoadCRC)
-            {
-                LogRel(("SSM: Invalid CRC! Calculated %#010x, in footer %#010x\n", u32CRC, pSSM->u.Read.u32LoadCRC));
-                return VERR_SSM_INTEGRITY_CRC;
-            }
-        }
-    }
-    else
-    {
-        /*
-         * Version 1.x of the format.
-         */
-        bool        fHaveHostBits = true;
-        bool        fHaveVersion  = false;
-        RTUUID      MachineUuidFromHdr;
-
-        ssmR3StrmDisableChecksumming(&pSSM->Strm);
-        if (pSSM->u.Read.uFmtVerMinor == 1)
-        {
-            pSSM->u.Read.cHostBits      = 0; /* unknown */
-            pSSM->u.Read.u16VerMajor    = 0;
-            pSSM->u.Read.u16VerMinor    = 0;
-            pSSM->u.Read.u32VerBuild    = 0;
-            pSSM->u.Read.u32SvnRev      = 0;
-            pSSM->u.Read.cbLoadFile     = uHdr.v1_1.cbFile;
-            pSSM->u.Read.u32LoadCRC     = uHdr.v1_1.u32CRC;
-            pSSM->u.Read.cbGCPhys       = sizeof(RTGCPHYS);
-            pSSM->u.Read.cbGCPtr        = sizeof(RTGCPTR);
-            pSSM->u.Read.fFixedGCPtrSize = false; /* settable */
-            pSSM->u.Read.fStreamCrc32   = false;
-
-            MachineUuidFromHdr  = uHdr.v1_1.MachineUuid;
-            fHaveHostBits       = false;
-        }
-        else if (pSSM->u.Read.uFmtVerMinor == 2)
-        {
-            pSSM->u.Read.cHostBits      = uHdr.v1_2.cHostBits;
-            pSSM->u.Read.u16VerMajor    = uHdr.v1_2.u16VerMajor;
-            pSSM->u.Read.u16VerMinor    = uHdr.v1_2.u16VerMinor;
-            pSSM->u.Read.u32VerBuild    = uHdr.v1_2.u32VerBuild;
-            pSSM->u.Read.u32SvnRev      = uHdr.v1_2.u32SvnRev;
-            pSSM->u.Read.cbLoadFile     = uHdr.v1_2.cbFile;
-            pSSM->u.Read.u32LoadCRC     = uHdr.v1_2.u32CRC;
-            pSSM->u.Read.cbGCPhys       = uHdr.v1_2.cbGCPhys;
-            pSSM->u.Read.cbGCPtr        = uHdr.v1_2.cbGCPtr;
-            pSSM->u.Read.fFixedGCPtrSize = true;
-            pSSM->u.Read.fStreamCrc32   = false;
-
-            MachineUuidFromHdr  = uHdr.v1_2.MachineUuid;
-            fHaveVersion        = true;
-        }
-        else
-            AssertFailedReturn(VERR_INTERNAL_ERROR);
-
-        /*
-         * The MachineUuid must be NULL (was never used).
-         */
-        if (!RTUuidIsNull(&MachineUuidFromHdr))
-        {
-            LogRel(("SSM: The UUID of the saved state doesn't match the running VM.\n"));
-            return VERR_SMM_INTEGRITY_MACHINE;
-        }
-
-        /*
-         * Verify the file size.
-         */
-        uint64_t cbFile = ssmR3StrmGetSize(&pSSM->Strm);
-        if (cbFile != pSSM->u.Read.cbLoadFile)
-        {
-            LogRel(("SSM: File size mismatch. hdr.cbFile=%lld actual %lld\n", pSSM->u.Read.cbLoadFile, cbFile));
-            return VERR_SSM_INTEGRITY_SIZE;
-        }
-
-        /*
-         * Validate the header info we've set in the handle.
-         */
-        rc = ssmR3ValidateHeaderInfo(pSSM, fHaveHostBits, fHaveVersion);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /*
-         * Verify the checksum if requested.
-         *
-         * Note! The checksum is not actually generated for the whole file,
-         *       this is of course a bug in the v1.x code that we cannot do
-         *       anything about.
-         */
-        if (    fChecksumIt
-            ||  fChecksumOnRead)
-        {
-            uint32_t u32CRC;
-            rc = ssmR3CalcChecksum(&pSSM->Strm,
-                                   RT_OFFSETOF(SSMFILEHDRV11, u32CRC) + sizeof(uHdr.v1_1.u32CRC),
-                                   cbFile - pSSM->u.Read.cbFileHdr,
-                                   &u32CRC);
-            if (RT_FAILURE(rc))
-                return rc;
-            if (u32CRC != pSSM->u.Read.u32LoadCRC)
-            {
-                LogRel(("SSM: Invalid CRC! Calculated %#010x, in header %#010x\n", u32CRC, pSSM->u.Read.u32LoadCRC));
-                return VERR_SSM_INTEGRITY_CRC;
-            }
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Open a saved state for reading.
- *
- * The file will be positioned at the first data unit upon successful return.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   pszFilename         The filename. NULL if pStreamOps is used.
- * @param   pStreamOps          The stream method table. NULL if pszFilename is
- *                              used.
- * @param   pvUser              The user argument to the stream methods.
- * @param   fChecksumIt         Check the checksum for the entire file.
- * @param   fChecksumOnRead     Whether to validate the checksum while reading
- *                              the stream instead of up front. If not possible,
- *                              verify the checksum up front.
- * @param   pSSM                Pointer to the handle structure. This will be
- *                              completely initialized on success.
- * @param   cBuffers            The number of stream buffers.
- */
-static int ssmR3OpenFile(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvUser,
-                         bool fChecksumIt, bool fChecksumOnRead, uint32_t cBuffers, PSSMHANDLE pSSM)
-{
-    /*
-     * Initialize the handle.
-     */
-    pSSM->pVM                   = pVM;
-    pSSM->enmOp                 = SSMSTATE_INVALID;
-    pSSM->enmAfter              = SSMAFTER_INVALID;
-    pSSM->fCancelled            = SSMHANDLE_OK;
-    pSSM->rc                    = VINF_SUCCESS;
-    pSSM->cbUnitLeftV1          = 0;
-    pSSM->offUnit               = UINT64_MAX;
-    pSSM->fLiveSave             = false;
-    pSSM->pfnProgress           = NULL;
-    pSSM->pvUser                = NULL;
-    pSSM->uPercent              = 0;
-    pSSM->offEstProgress        = 0;
-    pSSM->cbEstTotal            = 0;
-    pSSM->offEst                = 0;
-    pSSM->offEstUnitEnd         = 0;
-    pSSM->uPercentLive          = 0;
-    pSSM->uPercentPrepare       = 5;
-    pSSM->uPercentDone          = 2;
-    pSSM->uReportedLivePercent  = 0;
-    pSSM->pszFilename           = pszFilename;
-
-    pSSM->u.Read.pZipDecompV1   = NULL;
-    pSSM->u.Read.uFmtVerMajor   = UINT32_MAX;
-    pSSM->u.Read.uFmtVerMinor   = UINT32_MAX;
-    pSSM->u.Read.cbFileHdr      = UINT32_MAX;
-    pSSM->u.Read.cbGCPhys       = UINT8_MAX;
-    pSSM->u.Read.cbGCPtr        = UINT8_MAX;
-    pSSM->u.Read.fFixedGCPtrSize= false;
-    pSSM->u.Read.fIsHostMsc32   = SSM_HOST_IS_MSC_32;
-    RT_ZERO(pSSM->u.Read.szHostOSAndArch);
-    pSSM->u.Read.u16VerMajor    = UINT16_MAX;
-    pSSM->u.Read.u16VerMinor    = UINT16_MAX;
-    pSSM->u.Read.u32VerBuild    = UINT32_MAX;
-    pSSM->u.Read.u32SvnRev      = UINT32_MAX;
-    pSSM->u.Read.cHostBits      = UINT8_MAX;
-    pSSM->u.Read.cbLoadFile     = UINT64_MAX;
-
-    pSSM->u.Read.cbRecLeft      = 0;
-    pSSM->u.Read.cbDataBuffer   = 0;
-    pSSM->u.Read.offDataBuffer  = 0;
-    pSSM->u.Read.fEndOfData     = 0;
-    pSSM->u.Read.u8TypeAndFlags = 0;
-
-    pSSM->u.Read.pCurUnit       = NULL;
-    pSSM->u.Read.uCurUnitVer    = UINT32_MAX;
-    pSSM->u.Read.uCurUnitPass   = 0;
-    pSSM->u.Read.fHaveSetError  = false;
-
-    /*
-     * Try open and validate the file.
-     */
-    int rc;
-    if (pStreamOps)
-        rc = ssmR3StrmInit(&pSSM->Strm, pStreamOps, pvUser, false /*fWrite*/, fChecksumOnRead, cBuffers);
-    else
-        rc = ssmR3StrmOpenFile(&pSSM->Strm, pszFilename, false /*fWrite*/, fChecksumOnRead, cBuffers);
-    if (RT_SUCCESS(rc))
-    {
-        rc = ssmR3HeaderAndValidate(pSSM, fChecksumIt, fChecksumOnRead);
-        if (RT_SUCCESS(rc))
-            return rc;
-
-        /* failure path */
-        ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
-    }
-    else
-        Log(("SSM: Failed to open save state file '%s', rc=%Rrc.\n",  pszFilename, rc));
-    return rc;
-}
-
-
-/**
- * Verifies the directory.
- *
- * @returns VBox status code.
- *
- * @param   pDir        The full directory.
- * @param   cbDir       The size of the directory.
- * @param   offDir      The directory stream offset.
- * @param   cDirEntries The directory entry count from the footer.
- * @param   cbHdr       The header size.
- * @param   uSvnRev     The SVN revision that saved the state. Bug detection.
- */
-static int ssmR3ValidateDirectory(PSSMFILEDIR pDir, size_t cbDir, uint64_t offDir, uint32_t cDirEntries,
-                                  uint32_t cbHdr, uint32_t uSvnRev)
-{
-    AssertLogRelReturn(!memcmp(pDir->szMagic, SSMFILEDIR_MAGIC, sizeof(pDir->szMagic)), VERR_SSM_INTEGRITY_DIR_MAGIC);
-    SSM_CHECK_CRC32_RET(pDir, cbDir, ("Bad directory CRC: %08x, actual %08x\n", u32CRC, u32ActualCRC));
-    AssertLogRelMsgReturn(pDir->cEntries == cDirEntries,
-                          ("Bad directory entry count: %#x, expected %#x (from the footer)\n", pDir->cEntries, cDirEntries),
-                           VERR_SSM_INTEGRITY_DIR);
-    AssertLogRelReturn(RT_UOFFSETOF(SSMFILEDIR, aEntries[pDir->cEntries]) == cbDir, VERR_SSM_INTEGRITY_DIR);
-
-    for (uint32_t i = 0; i < pDir->cEntries; i++)
-    {
-        AssertLogRelMsgReturn(  (   pDir->aEntries[i].off >= cbHdr
-                                 && pDir->aEntries[i].off <  offDir)
-                              || (   pDir->aEntries[i].off == 0 /* bug in unreleased code */
-                                  && uSvnRev < 53365),
-                              ("off=%#llx cbHdr=%#x offDir=%#llx\n", pDir->aEntries[i].off, cbHdr, offDir),
-                              VERR_SSM_INTEGRITY_DIR);
-    }
-    return VINF_SUCCESS;
-}
-
-#ifndef SSM_STANDALONE
-
-/**
- * Find a data unit by name.
- *
- * @returns Pointer to the unit.
- * @returns NULL if not found.
- *
- * @param   pVM             VM handle.
- * @param   pszName         Data unit name.
- * @param   uInstance       The data unit instance id.
- */
-static PSSMUNIT ssmR3Find(PVM pVM, const char *pszName, uint32_t uInstance)
-{
-    size_t   cchName = strlen(pszName);
-    PSSMUNIT pUnit = pVM->ssm.s.pHead;
-    while (     pUnit
-           &&   (   pUnit->u32Instance != uInstance
-                 || pUnit->cchName != cchName
-                 || memcmp(pUnit->szName, pszName, cchName)))
-        pUnit = pUnit->pNext;
-    return pUnit;
-}
-
-
-/**
- * Executes the loading of a V1.X file.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3LoadExecV1(PVM pVM, PSSMHANDLE pSSM)
-{
-    int     rc;
-    char   *pszName = NULL;
-    size_t  cchName = 0;
-    pSSM->enmOp = SSMSTATE_LOAD_EXEC;
-    for (;;)
-    {
-        /*
-         * Save the current file position and read the data unit header.
-         */
-        uint64_t         offUnit = ssmR3StrmTell(&pSSM->Strm);
-        SSMFILEUNITHDRV1 UnitHdr;
-        rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV1, szName));
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Check the magic and see if it's valid and whether it is a end header or not.
-             */
-            if (memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(SSMFILEUNITHDR_MAGIC)))
-            {
-                if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_END, sizeof(SSMFILEUNITHDR_END)))
-                {
-                    Log(("SSM: EndOfFile: offset %#9llx size %9d\n", offUnit, UnitHdr.cbUnit));
-                    /* Complete the progress bar (pending 99% afterwards). */
-                    ssmR3ProgressByByte(pSSM, pSSM->cbEstTotal - pSSM->offEst);
-                    break;
-                }
-                LogRel(("SSM: Invalid unit magic at offset %#llx (%lld), '%.*s'!\n",
-                        offUnit, offUnit, sizeof(UnitHdr.achMagic) - 1, &UnitHdr.achMagic[0]));
-                rc = VERR_SSM_INTEGRITY_UNIT_MAGIC;
-                break;
-            }
-
-            /*
-             * Read the name.
-             * Adjust the name buffer first.
-             */
-            if (cchName < UnitHdr.cchName)
-            {
-                if (pszName)
-                    RTMemTmpFree(pszName);
-                cchName = RT_ALIGN_Z(UnitHdr.cchName, 64);
-                pszName = (char *)RTMemTmpAlloc(cchName);
-            }
-            if (pszName)
-            {
-                rc = ssmR3StrmRead(&pSSM->Strm, pszName, UnitHdr.cchName);
-                if (RT_SUCCESS(rc))
-                {
-                    if (pszName[UnitHdr.cchName - 1])
-                    {
-                        LogRel(("SSM: Unit name '%.*s' was not properly terminated.\n", UnitHdr.cchName, pszName));
-                        rc = VERR_SSM_INTEGRITY_UNIT;
-                        break;
-                    }
-                    Log(("SSM: Data unit: offset %#9llx size %9lld '%s'\n", offUnit, UnitHdr.cbUnit, pszName));
-
-                    /*
-                     * Find the data unit in our internal table.
-                     */
-                    PSSMUNIT pUnit = ssmR3Find(pVM, pszName, UnitHdr.u32Instance);
-                    if (pUnit)
-                    {
-                        /*
-                         * Call the execute handler.
-                         */
-                        pSSM->cbUnitLeftV1 = UnitHdr.cbUnit - RT_OFFSETOF(SSMFILEUNITHDRV1, szName[UnitHdr.cchName]);
-                        pSSM->offUnit = 0;
-                        pSSM->u.Read.uCurUnitVer  = UnitHdr.u32Version;
-                        pSSM->u.Read.uCurUnitPass = SSM_PASS_FINAL;
-                        pSSM->u.Read.pCurUnit     = pUnit;
-                        if (!pUnit->u.Common.pfnLoadExec)
-                        {
-                            LogRel(("SSM: No load exec callback for unit '%s'!\n", pszName));
-                            pSSM->rc = rc = VERR_SSM_NO_LOAD_EXEC;
-                            break;
-                        }
-                        switch (pUnit->enmType)
-                        {
-                            case SSMUNITTYPE_DEV:
-                                rc = pUnit->u.Dev.pfnLoadExec(pUnit->u.Dev.pDevIns, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
-                                break;
-                            case SSMUNITTYPE_DRV:
-                                rc = pUnit->u.Drv.pfnLoadExec(pUnit->u.Drv.pDrvIns, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
-                                break;
-                            case SSMUNITTYPE_INTERNAL:
-                                rc = pUnit->u.Internal.pfnLoadExec(pVM, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
-                                break;
-                            case SSMUNITTYPE_EXTERNAL:
-                                rc = pUnit->u.External.pfnLoadExec(pSSM, pUnit->u.External.pvUser, UnitHdr.u32Version, SSM_PASS_FINAL);
-                                break;
-                            default:
-                                rc = VERR_INTERNAL_ERROR;
-                                break;
-                        }
-                        pUnit->fCalled = true;
-                        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
-                            pSSM->rc = rc;
-
-                        /*
-                         * Close the reader stream.
-                         */
-                        rc = ssmR3DataReadFinishV1(pSSM);
-                        if (RT_SUCCESS(rc))
-                        {
-                            /*
-                             * Now, we'll check the current position to see if all, or
-                             * more than all, the data was read.
-                             *
-                             * Note! Because of buffering / compression we'll only see the
-                             * really bad ones here.
-                             */
-                            uint64_t off = ssmR3StrmTell(&pSSM->Strm);
-                            int64_t i64Diff = off - (offUnit + UnitHdr.cbUnit);
-                            if (i64Diff < 0)
-                            {
-                                Log(("SSM: Unit '%s' left %lld bytes unread!\n", pszName, -i64Diff));
-                                rc = ssmR3StrmSkipTo(&pSSM->Strm, offUnit + UnitHdr.cbUnit);
-                                ssmR3ProgressByByte(pSSM, offUnit + UnitHdr.cbUnit - pSSM->offEst);
-                            }
-                            else if (i64Diff > 0)
-                            {
-                                LogRel(("SSM: Unit '%s' read %lld bytes too much!\n", pszName, i64Diff));
-                                if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
-                                    rc = VMSetError(pVM, VERR_SSM_LOADED_TOO_MUCH, RT_SRC_POS,
-                                                    N_("Unit '%s' read %lld bytes too much"), pszName, i64Diff);
-                                break;
-                            }
-
-                            pSSM->offUnit = UINT64_MAX;
-                        }
-                        else
-                        {
-                            LogRel(("SSM: Load exec failed for '%s' instance #%u ! (version %u)\n",
-                                    pszName, UnitHdr.u32Instance, UnitHdr.u32Version));
-                            if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
-                            {
-                                if (rc == VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION)
-                                    VMSetError(pVM, rc, RT_SRC_POS, N_("Unsupported version %u of data unit '%s' (instance #%u)"),
-                                               UnitHdr.u32Version, UnitHdr.szName, UnitHdr.u32Instance);
-                                else
-                                    VMSetError(pVM, rc, RT_SRC_POS, N_("Load exec failed for '%s' instance #%u (version %u)"),
-                                               pszName, UnitHdr.u32Instance, UnitHdr.u32Version);
-                            }
-                            break;
-                        }
-
-                        pSSM->u.Read.pCurUnit       = NULL;
-                        pSSM->u.Read.uCurUnitVer    = UINT32_MAX;
-                        pSSM->u.Read.uCurUnitPass   = 0;
-                    }
-                    else
-                    {
-                        /*
-                         * SSM unit wasn't found - ignore this when loading for the debugger.
-                         */
-                        LogRel(("SSM: Found no handler for unit '%s'!\n", pszName));
-                        rc = VERR_SSM_INTEGRITY_UNIT_NOT_FOUND;
-                        if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
-                            break;
-                        rc = ssmR3StrmSkipTo(&pSSM->Strm, offUnit + UnitHdr.cbUnit);
-                    }
-                }
-            }
-            else
-                rc = VERR_NO_TMP_MEMORY;
-        }
-
-        /*
-         * I/O errors ends up here (yea, I know, very nice programming).
-         */
-        if (RT_FAILURE(rc))
-        {
-            LogRel(("SSM: I/O error. rc=%Rrc\n", rc));
-            break;
-        }
-
-        /*
-         * Check for cancellation.
-         */
-        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
-        {
-            LogRel(("SSM: Cancelled!n"));
-            rc = pSSM->rc;
-            if (RT_SUCCESS(pSSM->rc))
-                pSSM->rc = rc = VERR_SSM_CANCELLED;
-            break;
-        }
-    }
-
-    RTMemTmpFree(pszName);
-    return rc;
-}
-
-
-/**
- * Reads and verifies the directory and footer.
- *
- * @returns VBox status code.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3LoadDirectoryAndFooter(PSSMHANDLE pSSM)
-{
-    /*
-     * The directory.
-     *
-     * Get the header containing the number of entries first.  Then read the
-     * entries and pass the combined block to the validation function.
-     */
-    uint64_t        off      = ssmR3StrmTell(&pSSM->Strm);
-    size_t const    cbDirHdr = RT_OFFSETOF(SSMFILEDIR, aEntries);
-    SSMFILEDIR      DirHdr;
-    int rc = ssmR3StrmRead(&pSSM->Strm, &DirHdr, cbDirHdr);
-    if (RT_FAILURE(rc))
-        return rc;
-    AssertLogRelMsgReturn(!memcmp(DirHdr.szMagic, SSMFILEDIR_MAGIC, sizeof(DirHdr.szMagic)),
-                          ("Invalid directory magic at %#llx (%lld): %.*Rhxs\n", off, off, sizeof(DirHdr.szMagic), DirHdr.szMagic),
-                          VERR_SSM_INTEGRITY_DIR_MAGIC);
-    AssertLogRelMsgReturn(DirHdr.cEntries < _64K,
-                          ("Too many directory entries at %#llx (%lld): %#x\n", off, off, DirHdr.cEntries),
-                          VERR_SSM_INTEGRITY_DIR);
-
-    size_t      cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[DirHdr.cEntries]);
-    PSSMFILEDIR pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
-    if (!pDir)
-        return VERR_NO_TMP_MEMORY;
-    memcpy(pDir, &DirHdr, cbDirHdr);
-    rc = ssmR3StrmRead(&pSSM->Strm, (uint8_t *)pDir + cbDirHdr, cbDir - cbDirHdr);
-    if (RT_SUCCESS(rc))
-        rc = ssmR3ValidateDirectory(pDir, cbDir, off, DirHdr.cEntries, pSSM->u.Read.cbFileHdr, pSSM->u.Read.u32SvnRev);
-    RTMemTmpFree(pDir);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Read and validate the footer.
-     */
-    off = ssmR3StrmTell(&pSSM->Strm);
-    uint32_t    u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
-    SSMFILEFTR  Footer;
-    rc = ssmR3StrmRead(&pSSM->Strm, &Footer, sizeof(Footer));
-    if (RT_FAILURE(rc))
-        return rc;
-    return ssmR3ValidateFooter(&Footer, off, DirHdr.cEntries, pSSM->u.Read.fStreamCrc32, u32StreamCRC);
-}
-
-
-/**
- * Executes the loading of a V2.X file.
- *
- * @returns VBox status code.  May or may not set pSSM->rc, the returned
- *          status code is ALWAYS the more accurate of the two.
- * @param   pVM                 The VM handle.
- * @param   pSSM                The saved state handle.
- */
-static int ssmR3LoadExecV2(PVM pVM, PSSMHANDLE pSSM)
-{
-    pSSM->enmOp = SSMSTATE_LOAD_EXEC;
-    for (;;)
-    {
-        /*
-         * Read the unit header and check its integrity.
-         */
-        uint64_t            offUnit         = ssmR3StrmTell(&pSSM->Strm);
-        uint32_t            u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
-        SSMFILEUNITHDRV2    UnitHdr;
-        int rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName));
-        if (RT_FAILURE(rc))
-            return rc;
-        if (RT_UNLIKELY(    memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic))
-                        &&  memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END,   sizeof(UnitHdr.szMagic))))
-        {
-            LogRel(("SSM: Unit at %#llx (%lld): Invalid unit magic: %.*Rhxs!\n",
-                    offUnit, offUnit, sizeof(UnitHdr.szMagic) - 1, &UnitHdr.szMagic[0]));
-            pSSM->u.Read.fHaveSetError = true;
-            return VMSetError(pVM, VERR_SSM_INTEGRITY_UNIT_MAGIC, RT_SRC_POS,
-                              N_("Unit at %#llx (%lld): Invalid unit magic"), offUnit, offUnit);
-        }
-        if (UnitHdr.cbName)
-        {
-            AssertLogRelMsgReturn(UnitHdr.cbName <= sizeof(UnitHdr.szName),
-                                  ("Unit at %#llx (%lld): UnitHdr.cbName=%u > %u\n",
-                                   offUnit, offUnit, UnitHdr.cbName, sizeof(UnitHdr.szName)),
-                                  VERR_SSM_INTEGRITY_UNIT);
-            rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr.szName[0], UnitHdr.cbName);
-            if (RT_FAILURE(rc))
-                return rc;
-            AssertLogRelMsgReturn(!UnitHdr.szName[UnitHdr.cbName - 1],
-                                  ("Unit at %#llx (%lld): Name %.*Rhxs was not properly terminated.\n",
-                                   offUnit, offUnit, UnitHdr.cbName, UnitHdr.szName),
-                                  VERR_SSM_INTEGRITY_UNIT);
-        }
-        SSM_CHECK_CRC32_RET(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]),
-                            ("Unit at %#llx (%lld): CRC mismatch: %08x, correct is %08x\n", offUnit, offUnit, u32CRC, u32ActualCRC));
-        AssertLogRelMsgReturn(UnitHdr.offStream == offUnit,
-                              ("Unit at %#llx (%lld): offStream=%#llx, expected %#llx\n", offUnit, offUnit, UnitHdr.offStream, offUnit),
-                              VERR_SSM_INTEGRITY_UNIT);
-        AssertLogRelMsgReturn(UnitHdr.u32CurStreamCRC == u32CurStreamCRC || !pSSM->Strm.fChecksummed,
-                              ("Unit at %#llx (%lld): Stream CRC mismatch: %08x, correct is %08x\n", offUnit, offUnit, UnitHdr.u32CurStreamCRC, u32CurStreamCRC),
-                              VERR_SSM_INTEGRITY_UNIT);
-        AssertLogRelMsgReturn(!UnitHdr.fFlags, ("Unit at %#llx (%lld): fFlags=%08x\n", offUnit, offUnit, UnitHdr.fFlags),
-                              VERR_SSM_INTEGRITY_UNIT);
-        if (!memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END,   sizeof(UnitHdr.szMagic)))
-        {
-            AssertLogRelMsgReturn(   UnitHdr.cbName       == 0
-                                  && UnitHdr.u32Instance  == 0
-                                  && UnitHdr.u32Version   == 0
-                                  && UnitHdr.u32Pass      == SSM_PASS_FINAL,
-                                  ("Unit at %#llx (%lld): Malformed END unit\n", offUnit, offUnit),
-                                  VERR_SSM_INTEGRITY_UNIT);
-
-            /*
-             * Complete the progress bar (pending 99% afterwards) and RETURN.
-             */
-            Log(("SSM: Unit at %#9llx: END UNIT\n", offUnit));
-            ssmR3ProgressByByte(pSSM, pSSM->cbEstTotal - pSSM->offEst);
-            return ssmR3LoadDirectoryAndFooter(pSSM);
-        }
-        AssertLogRelMsgReturn(UnitHdr.cbName > 1, ("Unit at %#llx (%lld): No name\n", offUnit, offUnit), VERR_SSM_INTEGRITY);
-
-        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
-             offUnit, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
-
-        /*
-         * Find the data unit in our internal table.
-         */
-        PSSMUNIT pUnit = ssmR3Find(pVM, UnitHdr.szName, UnitHdr.u32Instance);
-        if (pUnit)
-        {
-            /*
-             * Call the execute handler.
-             */
-            AssertLogRelMsgReturn(pUnit->u.Common.pfnLoadExec,
-                                  ("SSM: No load exec callback for unit '%s'!\n", UnitHdr.szName),
-                                  VERR_SSM_NO_LOAD_EXEC);
-            pSSM->u.Read.uCurUnitVer  = UnitHdr.u32Version;
-            pSSM->u.Read.uCurUnitPass = UnitHdr.u32Pass;
-            pSSM->u.Read.pCurUnit     = pUnit;
-            ssmR3DataReadBeginV2(pSSM);
-            switch (pUnit->enmType)
-            {
-                case SSMUNITTYPE_DEV:
-                    rc = pUnit->u.Dev.pfnLoadExec(pUnit->u.Dev.pDevIns, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
-                    break;
-                case SSMUNITTYPE_DRV:
-                    rc = pUnit->u.Drv.pfnLoadExec(pUnit->u.Drv.pDrvIns, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
-                    break;
-                case SSMUNITTYPE_INTERNAL:
-                    rc = pUnit->u.Internal.pfnLoadExec(pVM, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
-                    break;
-                case SSMUNITTYPE_EXTERNAL:
-                    rc = pUnit->u.External.pfnLoadExec(pSSM, pUnit->u.External.pvUser, UnitHdr.u32Version, UnitHdr.u32Pass);
-                    break;
-                default:
-                    rc = VERR_INTERNAL_ERROR;
-                    break;
-            }
-            pUnit->fCalled = true;
-            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
-                pSSM->rc = rc;
-            rc = ssmR3DataReadFinishV2(pSSM);
-            if (RT_SUCCESS(rc))
-                pSSM->offUnit = UINT64_MAX;
-            else
-            {
-                LogRel(("SSM: LoadExec failed for '%s' instance #%u (version %u, pass %#x): %Rrc\n",
-                        UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Version, UnitHdr.u32Pass, rc));
-                if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
-                {
-                    if (rc == VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION)
-                        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Unsupported version %u of data unit '%s' (instance #%u, pass %#x)"),
-                                        UnitHdr.u32Version, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass);
-                    else
-                        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to load unit '%s'"), UnitHdr.szName);
-                }
-                return rc;
-            }
-        }
-        else
-        {
-            /*
-             * SSM unit wasn't found - ignore this when loading for the debugger.
-             */
-            LogRel(("SSM: Found no handler for unit '%s' instance #%u!\n", UnitHdr.szName, UnitHdr.u32Instance));
-            if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
-            {
-                pSSM->u.Read.fHaveSetError = true;
-                return VMSetError(pVM, VERR_SSM_INTEGRITY_UNIT_NOT_FOUND, RT_SRC_POS,
-                                  N_("Found no handler for unit '%s' instance #%u"), UnitHdr.szName, UnitHdr.u32Instance);
-            }
-            SSMR3SkipToEndOfUnit(pSSM);
-            ssmR3DataReadFinishV2(pSSM);
-        }
-
-        /*
-         * Check for cancellation.
-         */
-        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
-        {
-            LogRel(("SSM: Cancelled!\n"));
-            if (RT_SUCCESS(pSSM->rc))
-                pSSM->rc = VERR_SSM_CANCELLED;
-            return pSSM->rc;
-        }
-    }
-    /* won't get here */
-}
-
-
-
-
-/**
- * Load VM save operation.
- *
- * @returns VBox status.
- *
- * @param   pVM             The VM handle.
- * @param   pszFilename     The name of the saved state file. NULL if pStreamOps
- *                          is used.
- * @param   pStreamOps      The stream method table. NULL if pszFilename is
- *                          used.
- * @param   pvStreamOpsUser The user argument for the stream methods.
- * @param   enmAfter        What is planned after a successful load operation.
- *                          Only acceptable values are SSMAFTER_RESUME and SSMAFTER_DEBUG_IT.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvProgressUser  User argument for the progress callback.
- *
- * @thread  EMT
- */
-VMMR3DECL(int) SSMR3Load(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                         SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser)
-{
-    LogFlow(("SSMR3Load: pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p\n",
-             pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser));
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * Validate input.
-     */
-    AssertMsgReturn(   enmAfter == SSMAFTER_RESUME
-                    || enmAfter == SSMAFTER_TELEPORT
-                    || enmAfter == SSMAFTER_DEBUG_IT,
-                    ("%d\n", enmAfter),
-                    VERR_INVALID_PARAMETER);
-    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
-    if (pStreamOps)
-    {
-        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
-        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
-        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
-        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
-    }
-
-    /*
-     * Create the handle and open the file.
-     */
-    SSMHANDLE Handle;
-    int rc = ssmR3OpenFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser, false /* fChecksumIt */,
-                           true /* fChecksumOnRead */, 8 /*cBuffers*/, &Handle);
-    if (RT_SUCCESS(rc))
-    {
-        ssmR3StrmStartIoThread(&Handle.Strm);
-        ssmR3SetCancellable(pVM, &Handle, true);
-
-        Handle.enmAfter         = enmAfter;
-        Handle.pfnProgress      = pfnProgress;
-        Handle.pvUser           = pvProgressUser;
-        Handle.uPercentLive     = 0;
-        Handle.uPercentPrepare  = 2;
-        Handle.uPercentDone     = 2;
-
-        if (Handle.u.Read.u16VerMajor)
-            LogRel(("SSM: File header: Format %u.%u, VirtualBox Version %u.%u.%u r%u, %u-bit host, cbGCPhys=%u, cbGCPtr=%u\n",
-                    Handle.u.Read.uFmtVerMajor, Handle.u.Read.uFmtVerMinor,
-                    Handle.u.Read.u16VerMajor, Handle.u.Read.u16VerMinor, Handle.u.Read.u32VerBuild, Handle.u.Read.u32SvnRev,
-                    Handle.u.Read.cHostBits, Handle.u.Read.cbGCPhys, Handle.u.Read.cbGCPtr));
-        else
-            LogRel(("SSM: File header: Format %u.%u, %u-bit host, cbGCPhys=%u, cbGCPtr=%u\n" ,
-                    Handle.u.Read.uFmtVerMajor, Handle.u.Read.uFmtVerMinor,
-                    Handle.u.Read.cHostBits, Handle.u.Read.cbGCPhys, Handle.u.Read.cbGCPtr));
-
-        if (pfnProgress)
-            pfnProgress(pVM, Handle.uPercent, pvProgressUser);
-
-        /*
-         * Clear the per unit flags.
-         */
-        PSSMUNIT pUnit;
-        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-            pUnit->fCalled = false;
-
-        /*
-         * Do the prepare run.
-         */
-        Handle.rc = VINF_SUCCESS;
-        Handle.enmOp = SSMSTATE_LOAD_PREP;
-        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-        {
-            if (pUnit->u.Common.pfnLoadPrep)
-            {
-                Handle.u.Read.pCurUnit = pUnit;
-                pUnit->fCalled = true;
-                switch (pUnit->enmType)
-                {
-                    case SSMUNITTYPE_DEV:
-                        rc = pUnit->u.Dev.pfnLoadPrep(pUnit->u.Dev.pDevIns, &Handle);
-                        break;
-                    case SSMUNITTYPE_DRV:
-                        rc = pUnit->u.Drv.pfnLoadPrep(pUnit->u.Drv.pDrvIns, &Handle);
-                        break;
-                    case SSMUNITTYPE_INTERNAL:
-                        rc = pUnit->u.Internal.pfnLoadPrep(pVM, &Handle);
-                        break;
-                    case SSMUNITTYPE_EXTERNAL:
-                        rc = pUnit->u.External.pfnLoadPrep(&Handle, pUnit->u.External.pvUser);
-                        break;
-                    default:
-                        rc = VERR_INTERNAL_ERROR;
-                        break;
-                }
-                Handle.u.Read.pCurUnit = NULL;
-                if (RT_FAILURE(rc) && RT_SUCCESS_NP(Handle.rc))
-                    Handle.rc = rc;
-                else
-                    rc = Handle.rc;
-                if (RT_FAILURE(rc))
-                {
-                    LogRel(("SSM: Prepare load failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
-                    break;
-                }
-            }
-        }
-
-        /* end of prepare % */
-        if (pfnProgress)
-            pfnProgress(pVM, Handle.uPercentPrepare - 1, pvProgressUser);
-        Handle.uPercent      = Handle.uPercentPrepare;
-        Handle.cbEstTotal    = Handle.u.Read.cbLoadFile;
-        Handle.offEstUnitEnd = Handle.u.Read.cbLoadFile;
-
-        /*
-         * Do the execute run.
-         */
-        if (RT_SUCCESS(rc))
-        {
-            if (Handle.u.Read.uFmtVerMajor >= 2)
-                rc = ssmR3LoadExecV2(pVM, &Handle);
-            else
-                rc = ssmR3LoadExecV1(pVM, &Handle);
-            Handle.u.Read.pCurUnit       = NULL;
-            Handle.u.Read.uCurUnitVer    = UINT32_MAX;
-            Handle.u.Read.uCurUnitPass   = 0;
-
-            /* (progress should be pending 99% now) */
-            AssertMsg(   Handle.fLiveSave
-                      || RT_FAILURE(rc)
-                      || Handle.uPercent == 101 - Handle.uPercentDone, ("%d\n", Handle.uPercent));
-        }
-
-        /*
-         * Do the done run.
-         */
-        Handle.rc = rc;
-        Handle.enmOp = SSMSTATE_LOAD_DONE;
-        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
-        {
-            if (    pUnit->u.Common.pfnLoadDone
-                && (   pUnit->fCalled
-                    || (!pUnit->u.Common.pfnLoadPrep && !pUnit->u.Common.pfnLoadExec)))
-            {
-                Handle.u.Read.pCurUnit = pUnit;
-                int const rcOld = Handle.rc;
-                rc = VINF_SUCCESS;
-                switch (pUnit->enmType)
-                {
-                    case SSMUNITTYPE_DEV:
-                        rc = pUnit->u.Dev.pfnLoadDone(pUnit->u.Dev.pDevIns, &Handle);
-                        break;
-                    case SSMUNITTYPE_DRV:
-                        rc = pUnit->u.Drv.pfnLoadDone(pUnit->u.Drv.pDrvIns, &Handle);
-                        break;
-                    case SSMUNITTYPE_INTERNAL:
-                        rc = pUnit->u.Internal.pfnLoadDone(pVM, &Handle);
-                        break;
-                    case SSMUNITTYPE_EXTERNAL:
-                        rc = pUnit->u.External.pfnLoadDone(&Handle, pUnit->u.External.pvUser);
-                        break;
-                    default:
-                        rc = VERR_INTERNAL_ERROR;
-                        break;
-                }
-                Handle.u.Read.pCurUnit = NULL;
-                if (RT_SUCCESS(rc) && Handle.rc != rcOld)
-                    rc = Handle.rc;
-                if (RT_FAILURE(rc))
-                {
-                    LogRel(("SSM: LoadDone failed with rc=%Rrc for data unit '%s' instance #%u.\n",
-                            rc, pUnit->szName, pUnit->u32Instance));
-                    if (!ASMAtomicXchgBool(&Handle.u.Read.fHaveSetError, true))
-                        VMSetError(pVM, rc, RT_SRC_POS, N_("LoadDone failed with rc=%Rrc for data unit '%s' instance #%u."),
-                                   rc, pUnit->szName, pUnit->u32Instance);
-                    if (RT_SUCCESS_NP(Handle.rc))
-                        Handle.rc = rc;
-                }
-            }
-        }
-
-        /* progress */
-        if (pfnProgress)
-            pfnProgress(pVM, 99, pvProgressUser);
-
-        ssmR3SetCancellable(pVM, &Handle, false);
-        ssmR3StrmClose(&Handle.Strm, Handle.rc == VERR_SSM_CANCELLED);
-        rc = Handle.rc;
-    }
-
-    /*
-     * Done
-     */
-    if (RT_SUCCESS(rc))
-    {
-        /* progress */
-        if (pfnProgress)
-            pfnProgress(pVM, 100, pvProgressUser);
-        Log(("SSM: Load of '%s' completed!\n", pszFilename));
-    }
-    return rc;
-}
-
-
-/**
- * VMSetError wrapper for load errors that inserts the saved state details.
- *
- * @returns rc.
- * @param   pSSM                The saved state handle.
- * @param   rc                  The status code of the error. Use RT_SRC_POS.
- * @param   RT_SRC_POS_DECL     The source location.
- * @param   pszFormat           The message format string.
- * @param   ...                 Variable argument list.
- */
-VMMR3DECL(int) SSMR3SetLoadError(PSSMHANDLE pSSM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    rc = SSMR3SetLoadErrorV(pSSM, rc, RT_SRC_POS_ARGS, pszFormat, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * VMSetError wrapper for load errors that inserts the saved state details.
- *
- * @returns rc.
- * @param   pSSM                The saved state handle.
- * @param   rc                  The status code of the error.
- * @param   RT_SRC_POS_DECL     The error location, use RT_SRC_POS.
- * @param   pszFormat           The message format string.
- * @param   va                  Variable argument list.
- */
-VMMR3DECL(int) SSMR3SetLoadErrorV(PSSMHANDLE pSSM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
-{
-    /*
-     * Input validations.
-     */
-    SSM_ASSERT_READABLE_RET(pSSM);
-    AssertPtr(pszFormat);
-    Assert(RT_FAILURE_NP(rc));
-
-    /*
-     * Format the incoming error.
-     */
-    char *pszMsg;
-    RTStrAPrintfV(&pszMsg, pszFormat, va);
-    if (!pszMsg)
-    {
-        VMSetError(pSSM->pVM, VERR_NO_MEMORY, RT_SRC_POS,
-                   N_("SSMR3SetLoadErrorV ran out of memory formatting: %s\n"), pszFormat);
-        return rc;
-    }
-
-    /*
-     * Forward to VMSetError with the additional info.
-     */
-    PSSMUNIT    pUnit       = pSSM->u.Read.pCurUnit;
-    const char *pszName     = pUnit ? pUnit->szName      : "unknown";
-    uint32_t    uInstance   = pUnit ? pUnit->u32Instance : 0;
-    if (   pSSM->enmOp == SSMSTATE_LOAD_EXEC
-        && pSSM->u.Read.uCurUnitPass == SSM_PASS_FINAL)
-        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [ver=%u pass=final]"),
-                        pszName, uInstance, pszMsg, pSSM->u.Read.uCurUnitVer);
-    else if (pSSM->enmOp == SSMSTATE_LOAD_EXEC)
-        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [ver=%u pass=#%u]"),
-                        pszName, uInstance, pszMsg, pSSM->u.Read.uCurUnitVer, pSSM->u.Read.uCurUnitPass);
-    else if (pSSM->enmOp == SSMSTATE_LOAD_PREP)
-        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [prep]"),
-                        pszName, uInstance, pszMsg);
-    else if (pSSM->enmOp == SSMSTATE_LOAD_DONE)
-        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [done]"),
-                        pszName, uInstance, pszMsg);
-    else if (pSSM->enmOp == SSMSTATE_OPEN_READ)
-        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [read]"),
-                        pszName, uInstance, pszMsg);
-    else
-        AssertFailed();
-    pSSM->u.Read.fHaveSetError = true;
-    RTStrFree(pszMsg);
-    return rc;
-}
-
-
-/**
- * SSMR3SetLoadError wrapper that returns VERR_SSM_LOAD_CONFIG_MISMATCH.
- *
- * @returns VERR_SSM_LOAD_CONFIG_MISMATCH.
- * @param   pSSM                The saved state handle.
- * @param   RT_SRC_POS_DECL     The error location, use RT_SRC_POS.
- * @param   pszFormat           The message format string.
- * @param   va                  Variable argument list.
- */
-VMMR3DECL(int) SSMR3SetCfgError(PSSMHANDLE pSSM, RT_SRC_POS_DECL, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    int rc = SSMR3SetLoadErrorV(pSSM, VERR_SSM_LOAD_CONFIG_MISMATCH, RT_SRC_POS_ARGS, pszFormat, va);
-    va_end(va);
-    return rc;
-}
-
-#endif /* !SSM_STANDALONE */
-
-/**
- * Validates a file as a validate SSM saved state.
- *
- * This will only verify the file format, the format and content of individual
- * data units are not inspected.
- *
- * @returns VINF_SUCCESS if valid.
- * @returns VBox status code on other failures.
- *
- * @param   pszFilename     The path to the file to validate.
- * @param   fChecksumIt     Whether to checksum the file or not.
- *
- * @thread  Any.
- */
-VMMR3DECL(int) SSMR3ValidateFile(const char *pszFilename, bool fChecksumIt)
-{
-    LogFlow(("SSMR3ValidateFile: pszFilename=%p:{%s} fChecksumIt=%RTbool\n", pszFilename, pszFilename, fChecksumIt));
-
-    /*
-     * Try open the file and validate it.
-     */
-    SSMHANDLE Handle;
-    int rc = ssmR3OpenFile(NULL, pszFilename, NULL /*pStreamOps*/, NULL /*pvUser*/, fChecksumIt,
-                           false /*fChecksumOnRead*/, 1 /*cBuffers*/, &Handle);
-    if (RT_SUCCESS(rc))
-        ssmR3StrmClose(&Handle.Strm, false /*fCancelled*/);
-    else
-        Log(("SSM: Failed to open saved state file '%s', rc=%Rrc.\n",  pszFilename, rc));
-    return rc;
-}
-
-
-/**
- * Opens a saved state file for reading.
- *
- * @returns VBox status code.
- *
- * @param   pszFilename     The path to the saved state file.
- * @param   fFlags          Open flags. Reserved, must be 0.
- * @param   ppSSM           Where to store the SSM handle.
- *
- * @thread  Any.
- */
-VMMR3DECL(int) SSMR3Open(const char *pszFilename, unsigned fFlags, PSSMHANDLE *ppSSM)
-{
-    LogFlow(("SSMR3Open: pszFilename=%p:{%s} fFlags=%#x ppSSM=%p\n", pszFilename, pszFilename, fFlags, ppSSM));
-
-    /*
-     * Validate input.
-     */
-    AssertMsgReturn(VALID_PTR(pszFilename), ("%p\n", pszFilename), VERR_INVALID_PARAMETER);
-    AssertMsgReturn(!fFlags, ("%#x\n", fFlags), VERR_INVALID_PARAMETER);
-    AssertMsgReturn(VALID_PTR(ppSSM), ("%p\n", ppSSM), VERR_INVALID_PARAMETER);
-
-    /*
-     * Allocate a handle.
-     */
-    PSSMHANDLE pSSM = (PSSMHANDLE)RTMemAllocZ(sizeof(*pSSM));
-    AssertReturn(pSSM, VERR_NO_MEMORY);
-
-    /*
-     * Try open the file and validate it.
-     */
-    int rc = ssmR3OpenFile(NULL, pszFilename, NULL /*pStreamOps*/, NULL /*pvUser*/, false /*fChecksumIt*/,
-                           true /*fChecksumOnRead*/, 1 /*cBuffers*/, pSSM);
-    if (RT_SUCCESS(rc))
-    {
-        pSSM->enmAfter = SSMAFTER_OPENED;
-        pSSM->enmOp    = SSMSTATE_OPEN_READ;
-        *ppSSM = pSSM;
-        LogFlow(("SSMR3Open: returns VINF_SUCCESS *ppSSM=%p\n", *ppSSM));
-        return VINF_SUCCESS;
-    }
-
-    Log(("SSMR3Open: Failed to open saved state file '%s', rc=%Rrc.\n",  pszFilename, rc));
-    RTMemFree(pSSM);
-    return rc;
-
-}
-
-
-/**
- * Closes a saved state file opened by SSMR3Open().
- *
- * @returns VBox status code.
- *
- * @param   pSSM            The SSM handle returned by SSMR3Open().
- *
- * @thread  Any, but the caller is responsible for serializing calls per handle.
- */
-VMMR3DECL(int) SSMR3Close(PSSMHANDLE pSSM)
-{
-    LogFlow(("SSMR3Close: pSSM=%p\n", pSSM));
-
-    /*
-     * Validate input.
-     */
-    AssertMsgReturn(VALID_PTR(pSSM), ("%p\n", pSSM), VERR_INVALID_PARAMETER);
-    AssertMsgReturn(pSSM->enmAfter == SSMAFTER_OPENED, ("%d\n", pSSM->enmAfter),VERR_INVALID_PARAMETER);
-    AssertMsgReturn(pSSM->enmOp == SSMSTATE_OPEN_READ, ("%d\n", pSSM->enmOp), VERR_INVALID_PARAMETER);
-    Assert(pSSM->fCancelled == SSMHANDLE_OK);
-
-    /*
-     * Close the stream and free the handle.
-     */
-    int rc = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
-    if (pSSM->u.Read.pZipDecompV1)
-    {
-        RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
-        pSSM->u.Read.pZipDecompV1 = NULL;
-    }
-    RTMemFree(pSSM);
-    return rc;
-}
-
-
-/**
- * Worker for SSMR3Seek that seeks version 1 saved state files.
- *
- * @returns VBox status code.
- * @param   pSSM                The SSM handle.
- * @param   pszUnit             The unit to seek to.
- * @param   iInstance           The particular instance we seek.
- * @param   piVersion           Where to store the unit version number.
- */
-static int ssmR3FileSeekV1(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
-{
-    /*
-     * Walk the data units until we find EOF or a match.
-     */
-    size_t              cbUnitNm = strlen(pszUnit) + 1;
-    AssertLogRelReturn(cbUnitNm <= SSM_MAX_NAME_SIZE, VERR_SSM_UNIT_NOT_FOUND);
-    char                szName[SSM_MAX_NAME_SIZE];
-    SSMFILEUNITHDRV1    UnitHdr;
-    for (RTFOFF off = pSSM->u.Read.cbFileHdr; ; off += UnitHdr.cbUnit)
-    {
-        /*
-         * Read the unit header and verify it.
-         */
-        int rc = ssmR3StrmPeekAt(&pSSM->Strm, off, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV1, szName), NULL);
-        AssertRCReturn(rc, rc);
-        if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(SSMFILEUNITHDR_MAGIC)))
-        {
-            /*
-             * Does what we've got match, if so read the name.
-             */
-            if (    UnitHdr.u32Instance == iInstance
-                &&  UnitHdr.cchName     == cbUnitNm)
-            {
-                rc = ssmR3StrmPeekAt(&pSSM->Strm, off + RT_OFFSETOF(SSMFILEUNITHDRV1, szName), szName, cbUnitNm, NULL);
-                AssertRCReturn(rc, rc);
-                AssertLogRelMsgReturn(!szName[UnitHdr.cchName - 1],
-                                      (" Unit name '%.*s' was not properly terminated.\n", cbUnitNm, szName),
-                                      VERR_SSM_INTEGRITY_UNIT);
-
-                /*
-                 * Does the name match?
-                 */
-                if (!memcmp(szName, pszUnit, cbUnitNm))
-                {
-                    rc = ssmR3StrmSeek(&pSSM->Strm, off + RT_OFFSETOF(SSMFILEUNITHDRV1, szName) + cbUnitNm, RTFILE_SEEK_BEGIN, 0);
-                    pSSM->cbUnitLeftV1 = UnitHdr.cbUnit - RT_OFFSETOF(SSMFILEUNITHDRV1, szName[cbUnitNm]);
-                    pSSM->offUnit = 0;
-                    if (piVersion)
-                        *piVersion = UnitHdr.u32Version;
-                    return VINF_SUCCESS;
-                }
-            }
-        }
-        else if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_END, sizeof(SSMFILEUNITHDR_END)))
-            return VERR_SSM_UNIT_NOT_FOUND;
-        else
-            AssertLogRelMsgFailedReturn(("Invalid unit magic at offset %RTfoff, '%.*s'!\n",
-                                         off, sizeof(UnitHdr.achMagic) - 1, &UnitHdr.achMagic[0]),
-                                        VERR_SSM_INTEGRITY_UNIT_MAGIC);
-    }
-    /* won't get here. */
-}
-
-
-/**
- * Worker for ssmR3FileSeekV2 for simplifying memory cleanup.
- *
- * @returns VBox status code.
- * @param   pSSM                The SSM handle.
- * @param   pDir                The directory buffer.
- * @param   cbDir               The size of the directory.
- * @param   cDirEntries         The number of directory entries.
- * @param   offDir              The directory offset in the file.
- * @param   pszUnit             The unit to seek to.
- * @param   iInstance           The particular instance we seek.
- * @param   piVersion           Where to store the unit version number.
- */
-static int ssmR3FileSeekSubV2(PSSMHANDLE pSSM, PSSMFILEDIR pDir, size_t cbDir, uint32_t cDirEntries, uint64_t offDir,
-                              const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
-{
-    /*
-     * Read it.
-     */
-    int rc = ssmR3StrmPeekAt(&pSSM->Strm, offDir, pDir, cbDir, NULL);
-    AssertLogRelRCReturn(rc, rc);
-    rc = ssmR3ValidateDirectory(pDir, (uint32_t)cbDir, offDir, cDirEntries, pSSM->u.Read.cbFileHdr, pSSM->u.Read.u32SvnRev);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Search the directory.
-     */
-    size_t          cbUnitNm   = strlen(pszUnit) + 1;
-    uint32_t const  u32NameCRC = RTCrc32(pszUnit, cbUnitNm - 1);
-    for (uint32_t i = 0; i < cDirEntries; i++)
-    {
-        if (    pDir->aEntries[i].u32NameCRC  == u32NameCRC
-            &&  pDir->aEntries[i].u32Instance == iInstance
-            &&  pDir->aEntries[i].off != 0 /* bug in unreleased code */
-           )
-        {
-            /*
-             * Read and validate the unit header.
-             */
-            SSMFILEUNITHDRV2    UnitHdr;
-            size_t              cbToRead = sizeof(UnitHdr);
-            if (pDir->aEntries[i].off + cbToRead > offDir)
-            {
-                cbToRead = offDir - pDir->aEntries[i].off;
-                RT_ZERO(UnitHdr);
-            }
-            rc = ssmR3StrmPeekAt(&pSSM->Strm, pDir->aEntries[i].off, &UnitHdr, cbToRead, NULL);
-            AssertLogRelRCReturn(rc, rc);
-
-            AssertLogRelMsgReturn(!memcmp(UnitHdr.szMagic, SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic)),
-                                  ("Bad unit header or dictionary offset: i=%u off=%lld\n", i, pDir->aEntries[i].off),
-                                  VERR_SSM_INTEGRITY_UNIT);
-            AssertLogRelMsgReturn(UnitHdr.offStream == pDir->aEntries[i].off,
-                                  ("Bad unit header: i=%d off=%lld offStream=%lld\n", i, pDir->aEntries[i].off, UnitHdr.offStream),
-                                  VERR_SSM_INTEGRITY_UNIT);
-            AssertLogRelMsgReturn(UnitHdr.u32Instance == pDir->aEntries[i].u32Instance,
-                                  ("Bad unit header: i=%d off=%lld u32Instance=%u Dir.u32Instance=%u\n",
-                                   i, pDir->aEntries[i].off, UnitHdr.u32Instance, pDir->aEntries[i].u32Instance),
-                                  VERR_SSM_INTEGRITY_UNIT);
-            uint32_t cbUnitHdr = RT_UOFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]);
-            AssertLogRelMsgReturn(   UnitHdr.cbName > 0
-                                  && UnitHdr.cbName < sizeof(UnitHdr)
-                                  && cbUnitHdr <= cbToRead,
-                                  ("Bad unit header: i=%u off=%lld cbName=%#x cbToRead=%#x\n", i, pDir->aEntries[i].off, UnitHdr.cbName, cbToRead),
-                                  VERR_SSM_INTEGRITY_UNIT);
-            SSM_CHECK_CRC32_RET(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]),
-                                ("Bad unit header CRC: i=%u off=%lld u32CRC=%#x u32ActualCRC=%#x\n",
-                                 i, pDir->aEntries[i].off, u32CRC, u32ActualCRC));
-
-            /*
-             * Ok, it is valid, get on with the comparing now.
-             */
-            if (    UnitHdr.cbName == cbUnitNm
-                &&  !memcmp(UnitHdr.szName, pszUnit, cbUnitNm))
-            {
-                if (piVersion)
-                    *piVersion = UnitHdr.u32Version;
-                rc = ssmR3StrmSeek(&pSSM->Strm, pDir->aEntries[i].off + cbUnitHdr, RTFILE_SEEK_BEGIN,
-                                   RTCrc32Process(UnitHdr.u32CurStreamCRC, &UnitHdr, cbUnitHdr));
-                AssertLogRelRCReturn(rc, rc);
-                ssmR3DataReadBeginV2(pSSM);
-                return VINF_SUCCESS;
-            }
-        }
-    }
-
-    return VERR_SSM_UNIT_NOT_FOUND;
-}
-
-
-/**
- * Worker for SSMR3Seek that seeks version 2 saved state files.
- *
- * @returns VBox status code.
- * @param   pSSM                The SSM handle.
- * @param   pszUnit             The unit to seek to.
- * @param   iInstance           The particular instance we seek.
- * @param   piVersion           Where to store the unit version number.
- */
-static int ssmR3FileSeekV2(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
-{
-    /*
-     * Read the footer, allocate a temporary buffer for the dictionary and
-     * pass it down to a worker to simplify cleanup.
-     */
-    uint64_t        offFooter;
-    SSMFILEFTR      Footer;
-    int rc = ssmR3StrmPeekAt(&pSSM->Strm, -(RTFOFF)sizeof(Footer), &Footer, sizeof(Footer), &offFooter);
-    AssertLogRelRCReturn(rc, rc);
-    AssertLogRelReturn(!memcmp(Footer.szMagic, SSMFILEFTR_MAGIC, sizeof(Footer.szMagic)), VERR_SSM_INTEGRITY);
-    SSM_CHECK_CRC32_RET(&Footer, sizeof(Footer), ("Bad footer CRC: %08x, actual %08x\n", u32CRC, u32ActualCRC));
-
-    size_t const    cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[Footer.cDirEntries]);
-    PSSMFILEDIR     pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
-    if (RT_UNLIKELY(!pDir))
-        return VERR_NO_TMP_MEMORY;
-    rc = ssmR3FileSeekSubV2(pSSM, pDir, cbDir, Footer.cDirEntries, offFooter - cbDir,
-                            pszUnit, iInstance, piVersion);
-    RTMemTmpFree(pDir);
-
-    return rc;
-}
-
-
-/**
- * Seeks to a specific data unit.
- *
- * After seeking it's possible to use the getters to on
- * that data unit.
- *
- * @returns VBox status code.
- * @returns VERR_SSM_UNIT_NOT_FOUND if the unit+instance wasn't found.
- *
- * @param   pSSM            The SSM handle returned by SSMR3Open().
- * @param   pszUnit         The name of the data unit.
- * @param   iInstance       The instance number.
- * @param   piVersion       Where to store the version number. (Optional)
- *
- * @thread  Any, but the caller is responsible for serializing calls per handle.
- */
-VMMR3DECL(int) SSMR3Seek(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
-{
-    LogFlow(("SSMR3Seek: pSSM=%p pszUnit=%p:{%s} iInstance=%RU32 piVersion=%p\n",
-             pSSM, pszUnit, pszUnit, iInstance, piVersion));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pSSM, VERR_INVALID_PARAMETER);
-    AssertMsgReturn(pSSM->enmAfter == SSMAFTER_OPENED, ("%d\n", pSSM->enmAfter),VERR_INVALID_PARAMETER);
-    AssertMsgReturn(pSSM->enmOp == SSMSTATE_OPEN_READ, ("%d\n", pSSM->enmOp), VERR_INVALID_PARAMETER);
-    AssertPtrReturn(pszUnit, VERR_INVALID_POINTER);
-    AssertMsgReturn(!piVersion || VALID_PTR(piVersion), ("%p\n", piVersion), VERR_INVALID_POINTER);
-
-    /*
-     * Reset the state.
-     */
-    if (pSSM->u.Read.pZipDecompV1)
-    {
-        RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
-        pSSM->u.Read.pZipDecompV1 = NULL;
-    }
-    pSSM->cbUnitLeftV1  = 0;
-    pSSM->offUnit       = UINT64_MAX;
-
-    /*
-     * Call the version specific workers.
-     */
-    if (pSSM->u.Read.uFmtVerMajor >= 2)
-        pSSM->rc = ssmR3FileSeekV2(pSSM, pszUnit, iInstance, piVersion);
-    else
-        pSSM->rc = ssmR3FileSeekV1(pSSM, pszUnit, iInstance, piVersion);
-    return pSSM->rc;
-}
-
-
-
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-/* ... Misc APIs ... */
-
-
-
-/**
- * Query what the VBox status code of the operation is.
- *
- * This can be used for putting and getting a batch of values
- * without bother checking the result till all the calls have
- * been made.
- *
- * @returns SSMAFTER enum value.
- * @param   pSSM            The saved state handle.
- */
-VMMR3DECL(int) SSMR3HandleGetStatus(PSSMHANDLE pSSM)
-{
-    SSM_ASSERT_VALID_HANDLE(pSSM);
-    return pSSM->rc;
-}
-
-
-/**
- * Fail the load operation.
- *
- * This is mainly intended for sub item loaders (like timers) which
- * return code isn't necessarily heeded by the caller but is important
- * to SSM.
- *
- * @returns VBox status code of the handle, or VERR_INVALID_PARAMETER.
- * @param   pSSM            The saved state handle.
- * @param   iStatus         Failure status code. This MUST be a VERR_*.
- */
-VMMR3DECL(int) SSMR3HandleSetStatus(PSSMHANDLE pSSM, int iStatus)
-{
-    SSM_ASSERT_VALID_HANDLE(pSSM);
-    Assert(pSSM->enmOp != SSMSTATE_LIVE_VOTE);
-    if (RT_FAILURE(iStatus))
-    {
-        int rc = pSSM->rc;
-        if (RT_SUCCESS(rc))
-            pSSM->rc = rc = iStatus;
-        return rc;
-    }
-    AssertMsgFailed(("iStatus=%d %Rrc\n", iStatus, iStatus));
-    return VERR_INVALID_PARAMETER;
-}
-
-
-/**
- * Get what to do after this operation.
- *
- * @returns SSMAFTER enum value.
- * @param   pSSM            The saved state handle.
- */
-VMMR3DECL(SSMAFTER) SSMR3HandleGetAfter(PSSMHANDLE pSSM)
-{
-    SSM_ASSERT_VALID_HANDLE(pSSM);
-    return pSSM->enmAfter;
-}
-
-
-/**
- * Checks if it is a live save operation or not.
- *
- * @returns True if it is, false if it isn't.
- * @param   pSSM            The saved state handle.
- */
-VMMR3DECL(bool) SSMR3HandleIsLiveSave(PSSMHANDLE pSSM)
-{
-    SSM_ASSERT_VALID_HANDLE(pSSM);
-    return pSSM->fLiveSave;
-}
-
-
-/**
- * Gets the maximum downtime for a live operation.
- *
- * @returns The max downtime in milliseconds.  Can be anything from 0 thru
- *          UINT32_MAX.
- *
- * @param   pSSM            The saved state handle.
- */
-VMMR3DECL(uint32_t) SSMR3HandleMaxDowntime(PSSMHANDLE pSSM)
-{
-    SSM_ASSERT_VALID_HANDLE(pSSM);
-    if (pSSM->enmOp <= SSMSTATE_SAVE_DONE)
-        return pSSM->u.Write.cMsMaxDowntime;
-    return UINT32_MAX;
-}
-
-
-/**
- * Gets the host bit count of a saved state.
- *
- * @returns 32 or 64. If pSSM is invalid, 0 is returned.
- * @param   pSSM            The saved state handle.
- *
- * @remarks This method should ONLY be used for hacks when loading OLDER saved
- *          state that have data layout or semantic changes without the
- *          compulsory version number change.
- */
-VMMR3DECL(uint32_t) SSMR3HandleHostBits(PSSMHANDLE pSSM)
-{
-    SSM_ASSERT_VALID_HANDLE(pSSM);
-    return ssmR3GetHostBits(pSSM);
-}
-
-
-/**
- * Get the VirtualBox SVN revision that created the saved state.
- *
- * @returns The revision number on success.
- *          form.  If we don't know, it's 0.
- * @param   pSSM            The saved state handle.
- *
- * @remarks This method should ONLY be used for hacks when loading OLDER saved
- *          state that have data layout or semantic changes without the
- *          compulsory version number change.  Be VERY careful with this
- *          function since it will return different values for OSE builds!
- */
-VMMR3DECL(uint32_t)     SSMR3HandleRevision(PSSMHANDLE pSSM)
-{
-    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
-        return pSSM->u.Read.u32SvnRev;
-#ifdef SSM_STANDALONE
-    return 0;
-#else
-    return VMMGetSvnRev();
-#endif
-}
-
-
-/**
- * Gets the VirtualBox version that created the saved state.
- *
- * @returns VBOX_FULL_VERSION style version number.
- *          Returns UINT32_MAX if unknown or somehow out of range.
- *
- * @param   pSSM            The saved state handle.
- *
- * @remarks This method should ONLY be used for hacks when loading OLDER saved
- *          state that have data layout or semantic changes without the
- *          compulsory version number change.
- */
-VMMR3DECL(uint32_t)     SSMR3HandleVersion(PSSMHANDLE pSSM)
-{
-    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
-    {
-        if (    !pSSM->u.Read.u16VerMajor
-            &&  !pSSM->u.Read.u16VerMinor
-            &&  !pSSM->u.Read.u32VerBuild)
-            return UINT32_MAX;
-        AssertReturn(pSSM->u.Read.u16VerMajor <=   0xff, UINT32_MAX);
-        AssertReturn(pSSM->u.Read.u16VerMinor <=   0xff, UINT32_MAX);
-        AssertReturn(pSSM->u.Read.u32VerBuild <= 0xffff, UINT32_MAX);
-        return VBOX_FULL_VERSION_MAKE(pSSM->u.Read.u16VerMajor, pSSM->u.Read.u16VerMinor, pSSM->u.Read.u32VerBuild);
-    }
-    return VBOX_FULL_VERSION;
-}
-
-
-/**
- * Get the host OS and architecture where the saved state was created.
- *
- * @returns Pointer to a read only string.  When known, this is on the os.arch
- *          form.  If we don't know, it's an empty string.
- * @param   pSSM            The saved state handle.
- *
- * @remarks This method should ONLY be used for hacks when loading OLDER saved
- *          state that have data layout or semantic changes without the
- *          compulsory version number change.
- */
-VMMR3DECL(const char *) SSMR3HandleHostOSAndArch(PSSMHANDLE pSSM)
-{
-    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
-        return pSSM->u.Read.szHostOSAndArch;
-    return KBUILD_TARGET "." KBUILD_TARGET_ARCH;
-}
-
-
-#ifndef SSM_STANDALONE
-/**
- * Asynchronously cancels the current SSM operation ASAP.
- *
- * @returns VBox status code.
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_SSM_NO_PENDING_OPERATION if nothing around that can be
- *          cancelled.
- * @retval  VERR_SSM_ALREADY_CANCELLED if the operation as already been
- *          cancelled.
- *
- * @param   pVM                 The VM handle.
- *
- * @thread  Any.
- */
-VMMR3DECL(int) SSMR3Cancel(PVM pVM)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    int rc = RTCritSectEnter(&pVM->ssm.s.CancelCritSect);
-    AssertRCReturn(rc, rc);
-
-    PSSMHANDLE pSSM = pVM->ssm.s.pSSM;
-    if (pSSM)
-    {
-        uint32_t u32Old;
-        if (ASMAtomicCmpXchgExU32(&pSSM->fCancelled, SSMHANDLE_CANCELLED, SSMHANDLE_OK, &u32Old))
-        {
-            LogRel(("SSM: Cancelled pending operation\n"));
-            rc = VINF_SUCCESS;
-        }
-        else if (u32Old == SSMHANDLE_CANCELLED)
-            rc = VERR_SSM_ALREADY_CANCELLED;
-        else
-        {
-            AssertLogRelMsgFailed(("fCancelled=%RX32 enmOp=%d\n", u32Old, pSSM->enmOp));
-            rc = VERR_INTERNAL_ERROR_2;
-        }
-    }
-    else
-        rc = VERR_SSM_NO_PENDING_OPERATION;
-
-    RTCritSectLeave(&pVM->ssm.s.CancelCritSect);
-    return rc;
-}
-#endif /* !SSM_STANDALONE */
-
Index: unk/src/VBox/VMM/SSMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/SSMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,288 +1,0 @@
-/* $Id$ */
-/** @file
- * SSM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___SSMInternal_h
-#define ___SSMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/ssm.h>
-#include <iprt/critsect.h>
-
-RT_C_DECLS_BEGIN
-
-/** @defgroup grp_ssm_int       Internals
- * @ingroup grp_ssm
- * @internal
- * @{
- */
-
-
-/**
- * Data unit callback type.
- */
-typedef enum SSMUNITTYPE
-{
-    /** PDM Device . */
-    SSMUNITTYPE_DEV = 1,
-    /** PDM Driver. */
-    SSMUNITTYPE_DRV,
-    /** VM Internal. */
-    SSMUNITTYPE_INTERNAL,
-    /** External Wrapper. */
-    SSMUNITTYPE_EXTERNAL
-} SSMUNITTYPE;
-
-/** Pointer to a data unit descriptor. */
-typedef struct SSMUNIT *PSSMUNIT;
-
-/**
- * Data unit descriptor.
- */
-typedef struct SSMUNIT
-{
-    /** Pointer ot the next one in the list. */
-    PSSMUNIT                pNext;
-
-    /** Called in this save/load operation.
-     * The flag is used to determine whether there is need for a call to
-     * done or not. */
-    bool                    fCalled;
-    /** Finished its live part.
-     * This is used to handle VERR_SSM_VOTE_FOR_GIVING_UP.  */
-    bool                    fDoneLive;
-    /** Callback interface type. */
-    SSMUNITTYPE             enmType;
-    /** Type specific data. */
-    union
-    {
-        /** SSMUNITTYPE_DEV. */
-        struct
-        {
-            /** Prepare live save. */
-            PFNSSMDEVLIVEPREP   pfnLivePrep;
-            /** Execute live save. */
-            PFNSSMDEVLIVEEXEC   pfnLiveExec;
-            /** Vote live save complete. */
-            PFNSSMDEVLIVEVOTE   pfnLiveVote;
-            /** Prepare save. */
-            PFNSSMDEVSAVEPREP   pfnSavePrep;
-            /** Execute save. */
-            PFNSSMDEVSAVEEXEC   pfnSaveExec;
-            /** Done save. */
-            PFNSSMDEVSAVEDONE   pfnSaveDone;
-            /** Prepare load. */
-            PFNSSMDEVLOADPREP   pfnLoadPrep;
-            /** Execute load. */
-            PFNSSMDEVLOADEXEC   pfnLoadExec;
-            /** Done load. */
-            PFNSSMDEVLOADDONE   pfnLoadDone;
-            /** Device instance. */
-            PPDMDEVINS          pDevIns;
-        } Dev;
-
-        /** SSMUNITTYPE_DRV. */
-        struct
-        {
-            /** Prepare live save. */
-            PFNSSMDRVLIVEPREP   pfnLivePrep;
-            /** Execute live save. */
-            PFNSSMDRVLIVEEXEC   pfnLiveExec;
-            /** Vote live save complete. */
-            PFNSSMDRVLIVEVOTE   pfnLiveVote;
-            /** Prepare save. */
-            PFNSSMDRVSAVEPREP   pfnSavePrep;
-            /** Execute save. */
-            PFNSSMDRVSAVEEXEC   pfnSaveExec;
-            /** Done save. */
-            PFNSSMDRVSAVEDONE   pfnSaveDone;
-            /** Prepare load. */
-            PFNSSMDRVLOADPREP   pfnLoadPrep;
-            /** Execute load. */
-            PFNSSMDRVLOADEXEC   pfnLoadExec;
-            /** Done load. */
-            PFNSSMDRVLOADDONE   pfnLoadDone;
-            /** Driver instance. */
-            PPDMDRVINS          pDrvIns;
-        } Drv;
-
-        /** SSMUNITTYPE_INTERNAL. */
-        struct
-        {
-            /** Prepare live save. */
-            PFNSSMINTLIVEPREP   pfnLivePrep;
-            /** Execute live save. */
-            PFNSSMINTLIVEEXEC   pfnLiveExec;
-            /** Vote live save complete. */
-            PFNSSMINTLIVEVOTE   pfnLiveVote;
-            /** Prepare save. */
-            PFNSSMINTSAVEPREP   pfnSavePrep;
-            /** Execute save. */
-            PFNSSMINTSAVEEXEC   pfnSaveExec;
-            /** Done save. */
-            PFNSSMINTSAVEDONE   pfnSaveDone;
-            /** Prepare load. */
-            PFNSSMINTLOADPREP   pfnLoadPrep;
-            /** Execute load. */
-            PFNSSMINTLOADEXEC   pfnLoadExec;
-            /** Done load. */
-            PFNSSMINTLOADDONE   pfnLoadDone;
-        } Internal;
-
-        /** SSMUNITTYPE_EXTERNAL. */
-        struct
-        {
-            /** Prepare live save. */
-            PFNSSMEXTLIVEPREP   pfnLivePrep;
-            /** Execute live save. */
-            PFNSSMEXTLIVEEXEC   pfnLiveExec;
-            /** Vote live save complete. */
-            PFNSSMEXTLIVEVOTE   pfnLiveVote;
-            /** Prepare save. */
-            PFNSSMEXTSAVEPREP   pfnSavePrep;
-            /** Execute save. */
-            PFNSSMEXTSAVEEXEC   pfnSaveExec;
-            /** Done save. */
-            PFNSSMEXTSAVEDONE   pfnSaveDone;
-            /** Prepare load. */
-            PFNSSMEXTLOADPREP   pfnLoadPrep;
-            /** Execute load. */
-            PFNSSMEXTLOADEXEC   pfnLoadExec;
-            /** Done load. */
-            PFNSSMEXTLOADDONE   pfnLoadDone;
-            /** User data. */
-            void               *pvUser;
-        } External;
-
-        struct
-        {
-            /** Prepare live save. */
-            PFNRT               pfnLivePrep;
-            /** Execute live save. */
-            PFNRT               pfnLiveExec;
-            /** Vote live save complete. */
-            PFNRT               pfnLiveVote;
-            /** Prepare save. */
-            PFNRT               pfnSavePrep;
-            /** Execute save. */
-            PFNRT               pfnSaveExec;
-            /** Done save. */
-            PFNRT               pfnSaveDone;
-            /** Prepare load. */
-            PFNRT               pfnLoadPrep;
-            /** Execute load. */
-            PFNRT               pfnLoadExec;
-            /** Done load. */
-            PFNRT               pfnLoadDone;
-            /** User data. */
-            void               *pvKey;
-        } Common;
-    } u;
-    /** Data layout version. */
-    uint32_t                u32Version;
-    /** Instance number. */
-    uint32_t                u32Instance;
-    /** The offset of the final data unit.
-     * This is used for constructing the directory. */
-    RTFOFF                  offStream;
-    /** The guessed size of the data unit - used only for progress indication. */
-    size_t                  cbGuess;
-    /** Name size. (bytes) */
-    size_t                  cchName;
-    /** Name of this unit. (extends beyond the defined size) */
-    char                    szName[1];
-} SSMUNIT;
-
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Dev.pfnLivePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Dev.pfnLiveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Dev.pfnLiveVote);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Dev.pfnSavePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Dev.pfnSaveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Dev.pfnSaveDone);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Dev.pfnLoadPrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Dev.pfnLoadExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Dev.pfnLoadDone);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.Dev.pDevIns);
-
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Drv.pfnLivePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Drv.pfnLiveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Drv.pfnLiveVote);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Drv.pfnSavePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Drv.pfnSaveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Drv.pfnSaveDone);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Drv.pfnLoadPrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Drv.pfnLoadExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Drv.pfnLoadDone);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.Drv.pDrvIns);
-
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Internal.pfnLivePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Internal.pfnLiveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Internal.pfnLiveVote);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Internal.pfnSavePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Internal.pfnSaveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Internal.pfnSaveDone);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Internal.pfnLoadPrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Internal.pfnLoadExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Internal.pfnLoadDone);
-
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.External.pfnLivePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.External.pfnLiveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.External.pfnLiveVote);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.External.pfnSavePrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.External.pfnSaveExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.External.pfnSaveDone);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.External.pfnLoadPrep);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.External.pfnLoadExec);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.External.pfnLoadDone);
-AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.External.pvUser);
-
-
-/**
- * SSM VM Instance data.
- * Changes to this must checked against the padding of the cfgm union in VM!
- *
- * @todo Move this to UVM.
- */
-typedef struct SSM
-{
-    /** Critical section for serializing cancellation (pSSM). */
-    RTCRITSECT              CancelCritSect;
-    /** The handle of the current save or load operation.
-     * This is used by SSMR3Cancel.  */
-    PSSMHANDLE volatile     pSSM;
-
-    /** FIFO of data entity descriptors. */
-    R3PTRTYPE(PSSMUNIT)     pHead;
-    /** The number of register units. */
-    uint32_t                cUnits;
-    /** For lazy init. */
-    bool                    fInitialized;
-    /** Current pass (for STAM). */
-    uint32_t                uPass;
-    uint32_t                u32Alignment;
-} SSM;
-/** Pointer to SSM VM instance data. */
-typedef SSM *PSSM;
-
-
-
-/** @} */
-
-RT_C_DECLS_END
-
-#endif /* !___SSMInternal_h */
-
Index: unk/src/VBox/VMM/STAM.cpp
===================================================================
--- /trunk/src/VBox/VMM/STAM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1993 +1,0 @@
-/* $Id$ */
-/** @file
- * STAM - The Statistics Manager.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_stam       STAM - The Statistics Manager
- *
- * The purpose for the statistics manager is to present the rest of the system
- * with a somewhat uniform way of accessing VMM statistics.  STAM sports a
- * couple of different APIs for accessing them: STAMR3EnumU, STAMR3SnapshotU,
- * STAMR3DumpU, STAMR3DumpToReleaseLogU and the debugger.  Main is exposing the
- * XML based one, STAMR3SnapshotU.
- *
- * The rest of the VMM together with the devices and drivers registers their
- * statistics with STAM giving them a name.  The name is hierarchical, the
- * components separated by slashes ('/') and must start with a slash.
- *
- * Each item registered with STAM - also, half incorrectly, called a sample -
- * has a type, unit, visibility, data pointer and description associated with it
- * in addition to the name (described above).  The type tells STAM what kind of
- * structure the pointer is pointing to.  The visibility allows unused
- * statistics from cluttering the output or showing up in the GUI.  All the bits
- * together makes STAM able to present the items in a sensible way to the user.
- * Some types also allows STAM to reset the data, which is very convenient when
- * digging into specific operations and such.
- *
- * PS. The VirtualBox Debugger GUI has a viewer for inspecting the statistics
- * STAM provides.  You will also find statistics in the release and debug logs.
- * And as mentioned in the introduction, the debugger console features a couple
- * of command: .stats and .statsreset.
- *
- * @see grp_stam
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_STAM
-#include <VBox/stam.h>
-#include "STAMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-#include <VBox/err.h>
-#include <VBox/dbg.h>
-#include <VBox/log.h>
-
-#include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <iprt/alloc.h>
-#include <iprt/stream.h>
-#include <iprt/string.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Argument structure for stamR3PrintOne().
- */
-typedef struct STAMR3PRINTONEARGS
-{
-    PVM pVM;
-    void *pvArg;
-    DECLCALLBACKMEMBER(void, pfnPrintf)(struct STAMR3PRINTONEARGS *pvArg, const char *pszFormat, ...);
-} STAMR3PRINTONEARGS, *PSTAMR3PRINTONEARGS;
-
-
-/**
- * Argument structure to stamR3EnumOne().
- */
-typedef struct STAMR3ENUMONEARGS
-{
-    PVM             pVM;
-    PFNSTAMR3ENUM   pfnEnum;
-    void           *pvUser;
-} STAMR3ENUMONEARGS, *PSTAMR3ENUMONEARGS;
-
-
-/**
- * The snapshot status structure.
- * Argument package passed to stamR3SnapshotOne, stamR3SnapshotPrintf and stamR3SnapshotOutput.
- */
-typedef struct STAMR3SNAPSHOTONE
-{
-    /** Pointer to the buffer start. */
-    char           *pszStart;
-    /** Pointer to the buffer end. */
-    char           *pszEnd;
-    /** Pointer to the current buffer position. */
-    char           *psz;
-    /** The VM handle. */
-    PVM             pVM;
-    /** The number of bytes allocated. */
-    size_t          cbAllocated;
-    /** The status code. */
-    int             rc;
-    /** Whether to include the description strings. */
-    bool            fWithDesc;
-} STAMR3SNAPSHOTONE, *PSTAMR3SNAPSHOTONE;
-
-
-/**
- * Init record for a ring-0 statistic sample.
- */
-typedef struct STAMR0SAMPLE
-{
-    /** The GVMMSTATS structure offset of the variable. */
-    unsigned        offVar;
-    /** The type. */
-    STAMTYPE        enmType;
-    /** The unit. */
-    STAMUNIT        enmUnit;
-    /** The name. */
-    const char     *pszName;
-    /** The description. */
-    const char     *pszDesc;
-} STAMR0SAMPLE;
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int                  stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
-                                            STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc);
-static int                  stamR3ResetOne(PSTAMDESC pDesc, void *pvArg);
-static DECLCALLBACK(void)   stamR3EnumLogPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
-static DECLCALLBACK(void)   stamR3EnumRelLogPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
-static DECLCALLBACK(void)   stamR3EnumPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
-static int                  stamR3SnapshotOne(PSTAMDESC pDesc, void *pvArg);
-static int                  stamR3SnapshotPrintf(PSTAMR3SNAPSHOTONE pThis, const char *pszFormat, ...);
-static int                  stamR3PrintOne(PSTAMDESC pDesc, void *pvArg);
-static int                  stamR3EnumOne(PSTAMDESC pDesc, void *pvArg);
-static bool                 stamR3MultiMatch(const char * const *papszExpressions, unsigned cExpressions, unsigned *piExpression, const char *pszName);
-static char **              stamR3SplitPattern(const char *pszPat, unsigned *pcExpressions, char **ppszCopy);
-static int                  stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0, int (pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg);
-static void                 stamR3Ring0StatsRegisterU(PUVM pUVM);
-static void                 stamR3Ring0StatsUpdateU(PUVM pUVM, const char *pszPat);
-static void                 stamR3Ring0StatsUpdateMultiU(PUVM pUVM, const char * const *papszExpressions, unsigned cExpressions);
-
-#ifdef VBOX_WITH_DEBUGGER
-static DECLCALLBACK(int)    stamR3CmdStats(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-static DECLCALLBACK(void)   stamR3EnumDbgfPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...);
-static DECLCALLBACK(int)    stamR3CmdStatsReset(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
-#endif
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-#ifdef VBOX_WITH_DEBUGGER
-/** Pattern argument. */
-static const DBGCVARDESC    g_aArgPat[] =
-{
-    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
-    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "pattern",      "Which samples the command shall be applied to. Use '*' as wildcard. Use ';' to separate expression." }
-};
-
-/** Command descriptors. */
-static const DBGCCMD    g_aCmds[] =
-{
-    /* pszCmd,      cArgsMin, cArgsMax, paArgDesc,          cArgDescs,                  pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
-    { "stats",      0,        1,        &g_aArgPat[0],      RT_ELEMENTS(g_aArgPat),        NULL,               0,          stamR3CmdStats,     "[pattern]",        "Display statistics." },
-    { "statsreset", 0,        1,        &g_aArgPat[0],      RT_ELEMENTS(g_aArgPat),        NULL,               0,          stamR3CmdStatsReset,"[pattern]",        "Resets statistics." }
-};
-#endif
-
-
-/**
- * The GVMM mapping records - sans the host cpus.
- */
-static const STAMR0SAMPLE g_aGVMMStats[] =
-{
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltCalls", "The number of calls to GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltBlocking),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltBlocking", "The number of times we did go to sleep in GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltTimeouts),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltTimeouts", "The number of times we timed out in GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltNotBlocking),  STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltNotBlocking", "The number of times we didn't go to sleep in GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltWakeUps),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltWakeUps", "The number of wake ups done during GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpCalls),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpCalls", "The number of calls to GVMMR0WakeUp." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpNotHalted),  STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpNotHalted", "The number of times the EMT thread wasn't actually halted when GVMMR0WakeUp was called." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpWakeUps),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpWakeUps", "The number of wake ups done during GVMMR0WakeUp (not counting the explicit one)." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPokeCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PokeCalls", "The number of calls to GVMMR0Poke." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPokeNotBusy),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PokeNotBusy", "The number of times the EMT thread wasn't actually busy when GVMMR0Poke was called." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollCalls", "The number of calls to GVMMR0SchedPoll." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollHalts),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollHalts", "The number of times the EMT has halted in a GVMMR0SchedPoll call." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollWakeUps),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollWakeUps", "The number of wake ups done during GVMMR0SchedPoll." },
-
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltCalls", "The number of calls to GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltBlocking),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltBlocking", "The number of times we did go to sleep in GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltTimeouts),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltTimeouts", "The number of times we timed out in GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltNotBlocking), STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltNotBlocking", "The number of times we didn't go to sleep in GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltWakeUps),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltWakeUps", "The number of wake ups done during GVMMR0SchedHalt." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpCalls),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpCalls", "The number of calls to GVMMR0WakeUp." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpNotHalted), STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpNotHalted", "The number of times the EMT thread wasn't actually halted when GVMMR0WakeUp was called." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpWakeUps),   STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpWakeUps", "The number of wake ups done during GVMMR0WakeUp (not counting the explicit one)." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPokeCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PokeCalls", "The number of calls to GVMMR0Poke." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPokeNotBusy),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PokeNotBusy", "The number of times the EMT thread wasn't actually busy when GVMMR0Poke was called." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollCalls", "The number of calls to GVMMR0SchedPoll." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollHalts),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollHalts", "The number of times the EMT has halted in a GVMMR0SchedPoll call." },
-    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollWakeUps),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollWakeUps", "The number of wake ups done during GVMMR0SchedPoll." },
-
-    { RT_UOFFSETOF(GVMMSTATS, cVMs),                      STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/VMs", "The number of VMs accessible to the caller." },
-    { RT_UOFFSETOF(GVMMSTATS, cEMTs),                     STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/EMTs", "The number of emulation threads." },
-    { RT_UOFFSETOF(GVMMSTATS, cHostCpus),                 STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/HostCPUs", "The number of host CPUs." },
-};
-
-
-/**
- * Initializes the STAM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) STAMR3InitUVM(PUVM pUVM)
-{
-    LogFlow(("STAMR3Init\n"));
-
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompile(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
-    AssertRelease(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
-
-    /*
-     * Setup any fixed pointers and offsets.
-     */
-    int rc = RTSemRWCreate(&pUVM->stam.s.RWSem);
-    AssertRCReturn(rc, rc);
-
-    /*
-     * Register the ring-0 statistics (GVMM/GMM).
-     */
-    stamR3Ring0StatsRegisterU(pUVM);
-
-#ifdef VBOX_WITH_DEBUGGER
-    /*
-     * Register debugger commands.
-     */
-    static bool fRegisteredCmds = false;
-    if (!fRegisteredCmds)
-    {
-        rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
-        if (RT_SUCCESS(rc))
-            fRegisteredCmds = true;
-    }
-#endif
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Terminates the STAM.
- *
- * @param   pUVM        Pointer to the user mode VM structure.
- */
-VMMR3DECL(void) STAMR3TermUVM(PUVM pUVM)
-{
-    /*
-     * Free used memory and the RWLock.
-     */
-    PSTAMDESC pCur = pUVM->stam.s.pHead;
-    while (pCur)
-    {
-        void *pvFree = pCur;
-        pCur = pCur->pNext;
-        RTMemFree(pvFree);
-    }
-    pUVM->stam.s.pHead = NULL;
-
-    Assert(pUVM->stam.s.RWSem != NIL_RTSEMRW);
-    RTSemRWDestroy(pUVM->stam.s.RWSem);
-    pUVM->stam.s.RWSem = NIL_RTSEMRW;
-}
-
-
-/**
- * Registers a sample with the statistics manager.
- *
- * Statistics are maintained on a per VM basis and is normally registered
- * during the VM init stage, but there is nothing preventing you from
- * register them at runtime.
- *
- * Use STAMR3Deregister() to deregister statistics at runtime, however do
- * not bother calling at termination time.
- *
- * It is not possible to register the same sample twice.
- *
- * @returns VBox status.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
- *                      Further nesting is possible.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- */
-VMMR3DECL(int)  STAMR3RegisterU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
-{
-    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
-    return stamR3RegisterU(pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc);
-}
-
-
-/**
- * Registers a sample with the statistics manager.
- *
- * Statistics are maintained on a per VM basis and is normally registered
- * during the VM init stage, but there is nothing preventing you from
- * register them at runtime.
- *
- * Use STAMR3Deregister() to deregister statistics at runtime, however do
- * not bother calling at termination time.
- *
- * It is not possible to register the same sample twice.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
- *                      Further nesting is possible.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- */
-VMMR3DECL(int)  STAMR3Register(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
-{
-    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
-    return stamR3RegisterU(pVM->pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc);
-}
-
-
-/**
- * Same as STAMR3RegisterU except that the name is specified in a
- * RTStrPrintf like fashion.
- *
- * @returns VBox status.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   ...         Arguments to the format string.
- */
-VMMR3DECL(int)  STAMR3RegisterFU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
-                                 const char *pszDesc, const char *pszName, ...)
-{
-    va_list args;
-    va_start(args, pszName);
-    int rc = STAMR3RegisterVU(pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-    va_end(args);
-    return rc;
-}
-
-
-/**
- * Same as STAMR3Register except that the name is specified in a
- * RTStrPrintf like fashion.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   ...         Arguments to the format string.
- */
-VMMR3DECL(int)  STAMR3RegisterF(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
-                                const char *pszDesc, const char *pszName, ...)
-{
-    va_list args;
-    va_start(args, pszName);
-    int rc = STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-    va_end(args);
-    return rc;
-}
-
-
-/**
- * Same as STAMR3Register except that the name is specified in a
- * RTStrPrintfV like fashion.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   args        Arguments to the format string.
- */
-VMMR3DECL(int)  STAMR3RegisterVU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
-                                 const char *pszDesc, const char *pszName, va_list args)
-{
-    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
-
-    char *pszFormattedName;
-    RTStrAPrintfV(&pszFormattedName, pszName, args);
-    if (!pszFormattedName)
-        return VERR_NO_MEMORY;
-
-    int rc = STAMR3RegisterU(pUVM, pvSample, enmType, enmVisibility, pszFormattedName, enmUnit, pszDesc);
-    RTStrFree(pszFormattedName);
-    return rc;
-}
-
-
-/**
- * Same as STAMR3Register except that the name is specified in a
- * RTStrPrintfV like fashion.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   args        Arguments to the format string.
- */
-VMMR3DECL(int)  STAMR3RegisterV(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
-                                const char *pszDesc, const char *pszName, va_list args)
-{
-    return STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
-}
-
-
-/**
- * Similar to STAMR3Register except for the two callbacks, the implied type (STAMTYPE_CALLBACK),
- * and name given in an RTStrPrintf like fashion.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
- * @param   pfnPrint    Print the sample.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   ...         Arguments to the format string.
- * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
- */
-VMMR3DECL(int)  STAMR3RegisterCallback(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
-                                       PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
-                                       const char *pszDesc, const char *pszName, ...)
-{
-    va_list args;
-    va_start(args, pszName);
-    int rc = STAMR3RegisterCallbackV(pVM, pvSample, enmVisibility, enmUnit, pfnReset, pfnPrint, pszDesc, pszName, args);
-    va_end(args);
-    return rc;
-}
-
-
-/**
- * Same as STAMR3RegisterCallback() except for the ellipsis which is a va_list here.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
- * @param   pfnPrint    Print the sample.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   args        Arguments to the format string.
- * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
- */
-VMMR3DECL(int)  STAMR3RegisterCallbackV(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
-                                        PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
-                                        const char *pszDesc, const char *pszName, va_list args)
-{
-    char *pszFormattedName;
-    RTStrAPrintfV(&pszFormattedName, pszName, args);
-    if (!pszFormattedName)
-        return VERR_NO_MEMORY;
-
-    int rc = stamR3RegisterU(pVM->pUVM, pvSample, pfnReset, pfnPrint, STAMTYPE_CALLBACK, enmVisibility, pszFormattedName, enmUnit, pszDesc);
-    RTStrFree(pszFormattedName);
-    return rc;
-}
-
-
-/**
- * Divide the strings into sub-strings using '/' as delimiter
- * and then compare them in strcmp fashion.
- *
- * @returns Difference.
- * @retval  0 if equal.
- * @retval  < 0 if psz1 is less than psz2.
- * @retval  > 0 if psz1 greater than psz2.
- *
- * @param   psz1        The first string.
- * @param   psz2        The second string.
- */
-static int stamR3SlashCompare(const char *psz1, const char *psz2)
-{
-    for (;;)
-    {
-        unsigned int ch1 = *psz1++;
-        unsigned int ch2 = *psz2++;
-        if (ch1 != ch2)
-        {
-            /* slash is end-of-sub-string, so it trumps everything but '\0'. */
-            if (ch1 == '/')
-                return ch2 ? -1 : 1;
-            if (ch2 == '/')
-                return ch1 ? 1 : -1;
-            return ch1 - ch2;
-        }
-
-        /* done? */
-        if (ch1 == '\0')
-            return 0;
-    }
-}
-
-
-/**
- * Internal worker for the different register calls.
- *
- * @returns VBox status.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pvSample    Pointer to the sample.
- * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
- * @param   pfnPrint    Print the sample.
- * @param   enmType     Sample type. This indicates what pvSample is pointing at.
- * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
- * @param   enmUnit     Sample unit.
- * @param   pszDesc     Sample description.
- * @param   pszName     The sample name format string.
- * @param   args        Arguments to the format string.
- * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
- */
-static int stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
-                           STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
-{
-    STAM_LOCK_WR(pUVM);
-
-    /*
-     * Check if exists.
-     */
-    PSTAMDESC   pPrev = NULL;
-    PSTAMDESC   pCur = pUVM->stam.s.pHead;
-    while (pCur)
-    {
-        int iDiff = strcmp(pCur->pszName, pszName);
-        /* passed it */
-        if (iDiff > 0)
-            break;
-        /* found it. */
-        if (!iDiff)
-        {
-            STAM_UNLOCK_WR(pUVM);
-            AssertMsgFailed(("Duplicate sample name: %s\n", pszName));
-            return VERR_ALREADY_EXISTS;
-        }
-
-        /* next */
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-
-    /*
-     * Check that the name doesn't screw up sorting order when taking
-     * slashes into account. The QT4 GUI makes some assumptions.
-     * Problematic chars are: !"#$%&'()*+,-.
-     */
-    Assert(pszName[0] == '/');
-    if (pPrev)
-        Assert(stamR3SlashCompare(pPrev->pszName, pszName) < 0);
-    if (pCur)
-        Assert(stamR3SlashCompare(pCur->pszName, pszName) > 0);
-
-#ifdef VBOX_STRICT
-    /*
-     * Check alignment requirements.
-     */
-    switch (enmType)
-    {
-            /* 8 byte / 64-bit */
-        case STAMTYPE_U64:
-        case STAMTYPE_U64_RESET:
-        case STAMTYPE_X64:
-        case STAMTYPE_X64_RESET:
-        case STAMTYPE_COUNTER:
-        case STAMTYPE_PROFILE:
-        case STAMTYPE_PROFILE_ADV:
-            AssertMsg(!((uintptr_t)pvSample & 7), ("%p - %s\n", pvSample, pszName));
-            break;
-
-            /* 4 byte / 32-bit */
-        case STAMTYPE_RATIO_U32:
-        case STAMTYPE_RATIO_U32_RESET:
-        case STAMTYPE_U32:
-        case STAMTYPE_U32_RESET:
-        case STAMTYPE_X32:
-        case STAMTYPE_X32_RESET:
-            AssertMsg(!((uintptr_t)pvSample & 3), ("%p - %s\n", pvSample, pszName));
-            break;
-
-            /* 2 byte / 32-bit */
-        case STAMTYPE_U16:
-        case STAMTYPE_U16_RESET:
-        case STAMTYPE_X16:
-        case STAMTYPE_X16_RESET:
-            AssertMsg(!((uintptr_t)pvSample & 1), ("%p - %s\n", pvSample, pszName));
-            break;
-
-            /* 1 byte / 8-bit / unaligned */
-        case STAMTYPE_U8:
-        case STAMTYPE_U8_RESET:
-        case STAMTYPE_X8:
-        case STAMTYPE_X8_RESET:
-        case STAMTYPE_CALLBACK:
-            break;
-
-        default:
-            AssertMsgFailed(("%d\n", enmType));
-            break;
-    }
-#endif /* VBOX_STRICT */
-
-    /*
-     * Create a new node and insert it at the current location.
-     */
-    int rc;
-    size_t cchName = strlen(pszName) + 1;
-    size_t cchDesc = pszDesc ? strlen(pszDesc) + 1 : 0;
-    PSTAMDESC pNew = (PSTAMDESC)RTMemAlloc(sizeof(*pNew) + cchName + cchDesc);
-    if (pNew)
-    {
-        pNew->pszName       = (char *)memcpy((char *)(pNew + 1), pszName, cchName);
-        pNew->enmType       = enmType;
-        pNew->enmVisibility = enmVisibility;
-        if (enmType != STAMTYPE_CALLBACK)
-            pNew->u.pv      = pvSample;
-        else
-        {
-            pNew->u.Callback.pvSample = pvSample;
-            pNew->u.Callback.pfnReset = pfnReset;
-            pNew->u.Callback.pfnPrint = pfnPrint;
-        }
-        pNew->enmUnit       = enmUnit;
-        pNew->pszDesc       = NULL;
-        if (pszDesc)
-            pNew->pszDesc   = (char *)memcpy((char *)(pNew + 1) + cchName,  pszDesc,  cchDesc);
-
-        pNew->pNext         = pCur;
-        if (pPrev)
-            pPrev->pNext    = pNew;
-        else
-            pUVM->stam.s.pHead = pNew;
-
-        stamR3ResetOne(pNew, pUVM->pVM);
-        rc = VINF_SUCCESS;
-    }
-    else
-        rc = VERR_NO_MEMORY;
-
-    STAM_UNLOCK_WR(pUVM);
-    return rc;
-}
-
-
-/**
- * Deregisters a sample previously registered by STAR3Register().
- *
- * This is intended used for devices which can be unplugged and for
- * temporary samples.
- *
- * @returns VBox status.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pvSample    Pointer to the sample registered with STAMR3Register().
- */
-VMMR3DECL(int)  STAMR3DeregisterU(PUVM pUVM, void *pvSample)
-{
-    STAM_LOCK_WR(pUVM);
-
-    /*
-     * Search for it.
-     */
-    int         rc = VERR_INVALID_HANDLE;
-    PSTAMDESC   pPrev = NULL;
-    PSTAMDESC   pCur = pUVM->stam.s.pHead;
-    while (pCur)
-    {
-        if (pCur->u.pv == pvSample)
-        {
-            void *pvFree = pCur;
-            pCur = pCur->pNext;
-            if (pPrev)
-                pPrev->pNext = pCur;
-            else
-                pUVM->stam.s.pHead = pCur;
-
-            RTMemFree(pvFree);
-            rc = VINF_SUCCESS;
-            continue;
-        }
-
-        /* next */
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-
-    STAM_UNLOCK_WR(pUVM);
-    return rc;
-}
-
-
-/**
- * Deregisters a sample previously registered by STAR3Register().
- *
- * This is intended used for devices which can be unplugged and for
- * temporary samples.
- *
- * @returns VBox status.
- * @param   pVM         The VM handle.
- * @param   pvSample    Pointer to the sample registered with STAMR3Register().
- */
-VMMR3DECL(int)  STAMR3Deregister(PVM pVM, void *pvSample)
-{
-    return STAMR3DeregisterU(pVM->pUVM, pvSample);
-}
-
-
-/**
- * Resets statistics for the specified VM.
- * It's possible to select a subset of the samples.
- *
- * @returns VBox status. (Basically, it cannot fail.)
- * @param   pVM         The VM handle.
- * @param   pszPat      The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                      If NULL all samples are reset.
- * @remarks Don't confuse this with the other 'XYZR3Reset' methods, it's not called at VM reset.
- */
-VMMR3DECL(int)  STAMR3ResetU(PUVM pUVM, const char *pszPat)
-{
-    int rc = VINF_SUCCESS;
-
-    /* ring-0 */
-    GVMMRESETSTATISTICSSREQ GVMMReq;
-    //GMMRESETSTATISTICSSREQ GMMReq;
-    bool fGVMMMatched = !pszPat || !*pszPat;
-    //bool fGMMMatched = fGVMMMatched;
-    if (fGVMMMatched)
-        memset(&GVMMReq.Stats, 0xff, sizeof(GVMMReq.Stats));
-    else
-    {
-        char *pszCopy;
-        unsigned cExpressions;
-        char **papszExpressions = stamR3SplitPattern(pszPat, &cExpressions, &pszCopy);
-        if (!papszExpressions)
-            return VERR_NO_MEMORY;
-
-        /* GVMM */
-        memset(&GVMMReq.Stats, 0, sizeof(GVMMReq.Stats));
-        for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
-            if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGVMMStats[i].pszName))
-            {
-                *((uint8_t *)&GVMMReq.Stats + g_aGVMMStats[i].offVar) = 0xff;
-                fGVMMMatched = true;
-            }
-        if (!fGVMMMatched)
-        {
-            /** @todo match cpu leaves some rainy day.  */
-        }
-
-        /* GMM */
-//        memset(&GMMReq.Stats, 0, sizeof(GMMReq.Stats));
-//        for (unsigned i = 0; i < RT_ELEMENTS(g_aGMMStats); i++)
-//            if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGMMStats[i].pszName))
-//            {
-//                 *((uint8_t *)&GMMReq.Stats + g_aGMMStats[i].offVar) = 0xff;
-//                 fGMMMatched = true;
-//            }
-
-        RTMemTmpFree(papszExpressions);
-        RTStrFree(pszCopy);
-    }
-
-    STAM_LOCK_WR(pUVM);
-    if (fGVMMMatched)
-    {
-        PVM pVM = pUVM->pVM;
-        GVMMReq.Hdr.cbReq = sizeof(GVMMReq);
-        GVMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-        GVMMReq.pSession = pVM->pSession;
-        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GVMM_RESET_STATISTICS, 0, &GVMMReq.Hdr);
-    }
-
-//    if (fGMMMatched)
-//    {
-//        PVM pVM = pUVM->pVM;
-//        GMMReq.Hdr.cbReq = sizeof(Req);
-//        GMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-//        GMMReq.pSession = pVM->pSession;
-//        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, VMMR0_DO_GMM_RESET_STATISTICS, 0, &Req.Hdr);
-//    }
-
-    /* and the reset */
-    stamR3EnumU(pUVM, pszPat, false /* fUpdateRing0 */, stamR3ResetOne, pUVM->pVM);
-
-    STAM_UNLOCK_WR(pUVM);
-    return rc;
-}
-
-/**
- * Resets statistics for the specified VM.
- * It's possible to select a subset of the samples.
- *
- * @returns VBox status. (Basically, it cannot fail.)
- * @param   pVM         The VM handle.
- * @param   pszPat      The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                      If NULL all samples are reset.
- * @remarks Don't confuse this with the other 'XYZR3Reset' methods, it's not called at VM reset.
- */
-VMMR3DECL(int)  STAMR3Reset(PVM pVM, const char *pszPat)
-{
-    return STAMR3ResetU(pVM->pUVM, pszPat);
-}
-
-
-/**
- * Resets one statistics sample.
- * Callback for stamR3EnumU().
- *
- * @returns VINF_SUCCESS
- * @param   pDesc   Pointer to the current descriptor.
- * @param   pvArg   User argument - The VM handle.
- */
-static int stamR3ResetOne(PSTAMDESC pDesc, void *pvArg)
-{
-    switch (pDesc->enmType)
-    {
-        case STAMTYPE_COUNTER:
-            ASMAtomicXchgU64(&pDesc->u.pCounter->c, 0);
-            break;
-
-        case STAMTYPE_PROFILE:
-        case STAMTYPE_PROFILE_ADV:
-            ASMAtomicXchgU64(&pDesc->u.pProfile->cPeriods, 0);
-            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicks, 0);
-            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicksMax, 0);
-            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicksMin, ~0);
-            break;
-
-        case STAMTYPE_RATIO_U32_RESET:
-            ASMAtomicXchgU32(&pDesc->u.pRatioU32->u32A, 0);
-            ASMAtomicXchgU32(&pDesc->u.pRatioU32->u32B, 0);
-            break;
-
-        case STAMTYPE_CALLBACK:
-            if (pDesc->u.Callback.pfnReset)
-                pDesc->u.Callback.pfnReset((PVM)pvArg, pDesc->u.Callback.pvSample);
-            break;
-
-        case STAMTYPE_U8_RESET:
-        case STAMTYPE_X8_RESET:
-            ASMAtomicXchgU8(pDesc->u.pu8, 0);
-            break;
-
-        case STAMTYPE_U16_RESET:
-        case STAMTYPE_X16_RESET:
-            ASMAtomicXchgU16(pDesc->u.pu16, 0);
-            break;
-
-        case STAMTYPE_U32_RESET:
-        case STAMTYPE_X32_RESET:
-            ASMAtomicXchgU32(pDesc->u.pu32, 0);
-            break;
-
-        case STAMTYPE_U64_RESET:
-        case STAMTYPE_X64_RESET:
-            ASMAtomicXchgU64(pDesc->u.pu64, 0);
-            break;
-
-        /* These are custom and will not be touched. */
-        case STAMTYPE_U8:
-        case STAMTYPE_X8:
-        case STAMTYPE_U16:
-        case STAMTYPE_X16:
-        case STAMTYPE_U32:
-        case STAMTYPE_X32:
-        case STAMTYPE_U64:
-        case STAMTYPE_X64:
-        case STAMTYPE_RATIO_U32:
-            break;
-
-        default:
-            AssertMsgFailed(("enmType=%d\n", pDesc->enmType));
-            break;
-    }
-    NOREF(pvArg);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Get a snapshot of the statistics.
- * It's possible to select a subset of the samples.
- *
- * @returns VBox status. (Basically, it cannot fail.)
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are reset.
- * @param   fWithDesc       Whether to include the descriptions.
- * @param   ppszSnapshot    Where to store the pointer to the snapshot data.
- *                          The format of the snapshot should be XML, but that will have to be discussed
- *                          when this function is implemented.
- *                          The returned pointer must be freed by calling STAMR3SnapshotFree().
- * @param   pcchSnapshot    Where to store the size of the snapshot data. (Excluding the trailing '\0')
- */
-VMMR3DECL(int) STAMR3SnapshotU(PUVM pUVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc)
-{
-    STAMR3SNAPSHOTONE State = { NULL, NULL, NULL, pUVM->pVM, 0, VINF_SUCCESS, fWithDesc };
-
-    /*
-     * Write the XML header.
-     */
-    /** @todo Make this proper & valid XML. */
-    stamR3SnapshotPrintf(&State, "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n");
-
-    /*
-     * Write the content.
-     */
-    stamR3SnapshotPrintf(&State, "<Statistics>\n");
-    int rc = stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3SnapshotOne, &State);
-    stamR3SnapshotPrintf(&State, "</Statistics>\n");
-
-    if (RT_SUCCESS(rc))
-        rc = State.rc;
-    else
-    {
-        RTMemFree(State.pszStart);
-        State.pszStart = State.pszEnd = State.psz = NULL;
-        State.cbAllocated = 0;
-    }
-
-    /*
-     * Done.
-     */
-    *ppszSnapshot = State.pszStart;
-    if (pcchSnapshot)
-        *pcchSnapshot = State.psz - State.pszStart;
-    return rc;
-}
-
-
-/**
- * Get a snapshot of the statistics.
- * It's possible to select a subset of the samples.
- *
- * @returns VBox status. (Basically, it cannot fail.)
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are reset.
- * @param   fWithDesc       Whether to include the descriptions.
- * @param   ppszSnapshot    Where to store the pointer to the snapshot data.
- *                          The format of the snapshot should be XML, but that will have to be discussed
- *                          when this function is implemented.
- *                          The returned pointer must be freed by calling STAMR3SnapshotFree().
- * @param   pcchSnapshot    Where to store the size of the snapshot data.
- *                          (Excluding the trailing '\\0')
- */
-VMMR3DECL(int) STAMR3Snapshot(PVM pVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc)
-{
-    return STAMR3SnapshotU(pVM->pUVM, pszPat, ppszSnapshot, pcchSnapshot, fWithDesc);
-}
-
-
-/**
- * stamR3EnumU callback employed by STAMR3Snapshot.
- *
- * @returns VBox status code, but it's interpreted as 0 == success / !0 == failure by enmR3Enum.
- * @param   pDesc       The sample.
- * @param   pvArg       The snapshot status structure.
- */
-static int stamR3SnapshotOne(PSTAMDESC pDesc, void *pvArg)
-{
-    PSTAMR3SNAPSHOTONE pThis = (PSTAMR3SNAPSHOTONE)pvArg;
-
-    switch (pDesc->enmType)
-    {
-        case STAMTYPE_COUNTER:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pCounter->c == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<Counter c=\"%lld\"", pDesc->u.pCounter->c);
-            break;
-
-        case STAMTYPE_PROFILE:
-        case STAMTYPE_PROFILE_ADV:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pProfile->cPeriods == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<Profile cPeriods=\"%lld\" cTicks=\"%lld\" cTicksMin=\"%lld\" cTicksMax=\"%lld\"",
-                                 pDesc->u.pProfile->cPeriods, pDesc->u.pProfile->cTicks, pDesc->u.pProfile->cTicksMin,
-                                 pDesc->u.pProfile->cTicksMax);
-            break;
-
-        case STAMTYPE_RATIO_U32:
-        case STAMTYPE_RATIO_U32_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && !pDesc->u.pRatioU32->u32A && !pDesc->u.pRatioU32->u32B)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<Ratio32 u32A=\"%lld\" u32B=\"%lld\"",
-                                 pDesc->u.pRatioU32->u32A, pDesc->u.pRatioU32->u32B);
-            break;
-
-        case STAMTYPE_CALLBACK:
-        {
-            char szBuf[512];
-            pDesc->u.Callback.pfnPrint(pThis->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
-            stamR3SnapshotPrintf(pThis, "<Callback val=\"%s\"", szBuf);
-            break;
-        }
-
-        case STAMTYPE_U8:
-        case STAMTYPE_U8_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<U8 val=\"%u\"", *pDesc->u.pu8);
-            break;
-
-        case STAMTYPE_X8:
-        case STAMTYPE_X8_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<X8 val=\"%#x\"", *pDesc->u.pu8);
-            break;
-
-        case STAMTYPE_U16:
-        case STAMTYPE_U16_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<U16 val=\"%u\"", *pDesc->u.pu16);
-            break;
-
-        case STAMTYPE_X16:
-        case STAMTYPE_X16_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<X16 val=\"%#x\"", *pDesc->u.pu16);
-            break;
-
-        case STAMTYPE_U32:
-        case STAMTYPE_U32_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<U32 val=\"%u\"", *pDesc->u.pu32);
-            break;
-
-        case STAMTYPE_X32:
-        case STAMTYPE_X32_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<X32 val=\"%#x\"", *pDesc->u.pu32);
-            break;
-
-        case STAMTYPE_U64:
-        case STAMTYPE_U64_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<U64 val=\"%llu\"", *pDesc->u.pu64);
-            break;
-
-        case STAMTYPE_X64:
-        case STAMTYPE_X64_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
-                return VINF_SUCCESS;
-            stamR3SnapshotPrintf(pThis, "<X64 val=\"%#llx\"", *pDesc->u.pu64);
-            break;
-
-        default:
-            AssertMsgFailed(("%d\n", pDesc->enmType));
-            return 0;
-    }
-
-    stamR3SnapshotPrintf(pThis, " unit=\"%s\"", STAMR3GetUnit(pDesc->enmUnit));
-
-    switch (pDesc->enmVisibility)
-    {
-        default:
-        case STAMVISIBILITY_ALWAYS:
-            break;
-        case STAMVISIBILITY_USED:
-            stamR3SnapshotPrintf(pThis, " vis=\"used\"");
-            break;
-        case STAMVISIBILITY_NOT_GUI:
-            stamR3SnapshotPrintf(pThis, " vis=\"not-gui\"");
-            break;
-    }
-
-    stamR3SnapshotPrintf(pThis, " name=\"%s\"", pDesc->pszName);
-
-    if (pThis->fWithDesc && pDesc->pszDesc)
-    {
-        /*
-         * The description is a bit tricky as it may include chars that
-         * xml requires to be escaped.
-         */
-        const char *pszBadChar = strpbrk(pDesc->pszDesc, "&<>\"'");
-        if (!pszBadChar)
-            return stamR3SnapshotPrintf(pThis, " desc=\"%s\"/>\n", pDesc->pszDesc);
-
-        stamR3SnapshotPrintf(pThis, " desc=\"");
-        const char *pszCur = pDesc->pszDesc;
-        do
-        {
-            stamR3SnapshotPrintf(pThis, "%.*s", pszBadChar - pszCur, pszCur);
-            switch (*pszBadChar)
-            {
-                case '&':   stamR3SnapshotPrintf(pThis, "&amp;");   break;
-                case '<':   stamR3SnapshotPrintf(pThis, "&lt;");    break;
-                case '>':   stamR3SnapshotPrintf(pThis, "&gt;");    break;
-                case '"':   stamR3SnapshotPrintf(pThis, "&quot;");  break;
-                case '\'':  stamR3SnapshotPrintf(pThis, "&apos;");  break;
-                default:    AssertMsgFailed(("%c", *pszBadChar));    break;
-            }
-            pszCur = pszBadChar + 1;
-            pszBadChar = strpbrk(pszCur, "&<>\"'");
-        } while (pszBadChar);
-        return stamR3SnapshotPrintf(pThis, "%s\"/>\n", pszCur);
-    }
-    return stamR3SnapshotPrintf(pThis, "/>\n");
-}
-
-
-/**
- * Output callback for stamR3SnapshotPrintf.
- *
- * @returns number of bytes written.
- * @param   pvArg       The snapshot status structure.
- * @param   pach        Pointer to an array of characters (bytes).
- * @param   cch         The number or chars (bytes) to write from the array.
- */
-static DECLCALLBACK(size_t) stamR3SnapshotOutput(void *pvArg, const char *pach, size_t cch)
-{
-    PSTAMR3SNAPSHOTONE pThis = (PSTAMR3SNAPSHOTONE)pvArg;
-
-    /*
-     * Make sure we've got space for it.
-     */
-    if (RT_UNLIKELY((uintptr_t)pThis->pszEnd - (uintptr_t)pThis->psz < cch + 1))
-    {
-        if (RT_FAILURE(pThis->rc))
-            return 0;
-
-        size_t cbNewSize = pThis->cbAllocated;
-        if (cbNewSize > cch)
-            cbNewSize *= 2;
-        else
-            cbNewSize += RT_ALIGN(cch + 1, 0x1000);
-        char *pszNew = (char *)RTMemRealloc(pThis->pszStart, cbNewSize);
-        if (!pszNew)
-        {
-            /*
-             * Free up immediately, out-of-memory is bad news and this
-             * isn't an important allocations / API.
-             */
-            pThis->rc = VERR_NO_MEMORY;
-            RTMemFree(pThis->pszStart);
-            pThis->pszStart = pThis->pszEnd = pThis->psz = NULL;
-            pThis->cbAllocated = 0;
-            return 0;
-        }
-
-        pThis->psz = pszNew + (pThis->psz - pThis->pszStart);
-        pThis->pszStart = pszNew;
-        pThis->pszEnd = pszNew + cbNewSize;
-        pThis->cbAllocated = cbNewSize;
-    }
-
-    /*
-     * Copy the chars to the buffer and terminate it.
-     */
-    memcpy(pThis->psz, pach, cch);
-    pThis->psz += cch;
-    *pThis->psz = '\0';
-    return cch;
-}
-
-
-/**
- * Wrapper around RTStrFormatV for use by the snapshot API.
- *
- * @returns VBox status code.
- * @param   pThis       The snapshot status structure.
- * @param   pszFormat   The format string.
- * @param   ...         Optional arguments.
- */
-static int stamR3SnapshotPrintf(PSTAMR3SNAPSHOTONE pThis, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    RTStrFormatV(stamR3SnapshotOutput, pThis, NULL, NULL, pszFormat, va);
-    va_end(va);
-    return pThis->rc;
-}
-
-
-/**
- * Releases a statistics snapshot returned by STAMR3Snapshot().
- *
- * @returns VBox status.
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
- *                          NULL is allowed.
- */
-VMMR3DECL(int)  STAMR3SnapshotFreeU(PUVM pUVM, char *pszSnapshot)
-{
-    if (!pszSnapshot)
-        RTMemFree(pszSnapshot);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Releases a statistics snapshot returned by STAMR3Snapshot().
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
- *                          NULL is allowed.
- */
-VMMR3DECL(int)  STAMR3SnapshotFree(PVM pVM, char *pszSnapshot)
-{
-    return STAMR3SnapshotFreeU(pVM->pUVM, pszSnapshot);
-}
-
-
-/**
- * Dumps the selected statistics to the log.
- *
- * @returns VBox status.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are written to the log.
- */
-VMMR3DECL(int)  STAMR3DumpU(PUVM pUVM, const char *pszPat)
-{
-    STAMR3PRINTONEARGS Args;
-    Args.pVM = pUVM->pVM;
-    Args.pvArg = NULL;
-    Args.pfnPrintf = stamR3EnumLogPrintf;
-
-    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dumps the selected statistics to the log.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are written to the log.
- */
-VMMR3DECL(int)  STAMR3Dump(PVM pVM, const char *pszPat)
-{
-    return STAMR3DumpU(pVM->pUVM, pszPat);
-}
-
-
-/**
- * Prints to the log.
- *
- * @param   pArgs       Pointer to the print one argument structure.
- * @param   pszFormat   Format string.
- * @param   ...         Format arguments.
- */
-static DECLCALLBACK(void) stamR3EnumLogPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    RTLogPrintfV(pszFormat, va);
-    va_end(va);
-    NOREF(pArgs);
-}
-
-
-/**
- * Dumps the selected statistics to the release log.
- *
- * @returns VBox status.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are written to the log.
- */
-VMMR3DECL(int)  STAMR3DumpToReleaseLogU(PUVM pUVM, const char *pszPat)
-{
-    STAMR3PRINTONEARGS Args;
-    Args.pVM = pUVM->pVM;
-    Args.pvArg = NULL;
-    Args.pfnPrintf = stamR3EnumRelLogPrintf;
-
-    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Dumps the selected statistics to the release log.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are written to the log.
- */
-VMMR3DECL(int)  STAMR3DumpToReleaseLog(PVM pVM, const char *pszPat)
-{
-    return STAMR3DumpToReleaseLogU(pVM->pUVM, pszPat);
-}
-
-
-/**
- * Prints to the release log.
- *
- * @param   pArgs       Pointer to the print one argument structure.
- * @param   pszFormat   Format string.
- * @param   ...         Format arguments.
- */
-static DECLCALLBACK(void) stamR3EnumRelLogPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    RTLogRelPrintfV(pszFormat, va);
-    va_end(va);
-    NOREF(pArgs);
-}
-
-
-/**
- * Prints the selected statistics to standard out.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are reset.
- */
-VMMR3DECL(int)  STAMR3PrintU(PUVM pUVM, const char *pszPat)
-{
-    STAMR3PRINTONEARGS Args;
-    Args.pVM = pUVM->pVM;
-    Args.pvArg = NULL;
-    Args.pfnPrintf = stamR3EnumPrintf;
-
-    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Prints the selected statistics to standard out.
- *
- * @returns VBox status.
- * @param   pVM             The VM handle.
- * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
- *                          If NULL all samples are reset.
- */
-VMMR3DECL(int)  STAMR3Print(PVM pVM, const char *pszPat)
-{
-    return STAMR3PrintU(pVM->pUVM, pszPat);
-}
-
-
-/**
- * Prints to stdout.
- *
- * @param   pArgs       Pointer to the print one argument structure.
- * @param   pszFormat   Format string.
- * @param   ...         Format arguments.
- */
-static DECLCALLBACK(void) stamR3EnumPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    RTPrintfV(pszFormat, va);
-    va_end(va);
-    NOREF(pArgs);
-}
-
-
-/**
- * Prints one sample.
- * Callback for stamR3EnumU().
- *
- * @returns VINF_SUCCESS
- * @param   pDesc   Pointer to the current descriptor.
- * @param   pvArg   User argument - STAMR3PRINTONEARGS.
- */
-static int stamR3PrintOne(PSTAMDESC pDesc, void *pvArg)
-{
-    PSTAMR3PRINTONEARGS pArgs = (PSTAMR3PRINTONEARGS)pvArg;
-
-    switch (pDesc->enmType)
-    {
-        case STAMTYPE_COUNTER:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pCounter->c == 0)
-                return VINF_SUCCESS;
-
-            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s\n", pDesc->pszName, pDesc->u.pCounter->c, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_PROFILE:
-        case STAMTYPE_PROFILE_ADV:
-        {
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pProfile->cPeriods == 0)
-                return VINF_SUCCESS;
-
-            uint64_t u64 = pDesc->u.pProfile->cPeriods ? pDesc->u.pProfile->cPeriods : 1;
-            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s (%12llu ticks, %7llu times, max %9llu, min %7lld)\n", pDesc->pszName,
-                             pDesc->u.pProfile->cTicks / u64, STAMR3GetUnit(pDesc->enmUnit),
-                             pDesc->u.pProfile->cTicks, pDesc->u.pProfile->cPeriods, pDesc->u.pProfile->cTicksMax, pDesc->u.pProfile->cTicksMin);
-            break;
-        }
-
-        case STAMTYPE_RATIO_U32:
-        case STAMTYPE_RATIO_U32_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && !pDesc->u.pRatioU32->u32A && !pDesc->u.pRatioU32->u32B)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8u:%-8u %s\n", pDesc->pszName,
-                             pDesc->u.pRatioU32->u32A, pDesc->u.pRatioU32->u32B, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_CALLBACK:
-        {
-            char szBuf[512];
-            pDesc->u.Callback.pfnPrint(pArgs->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
-            pArgs->pfnPrintf(pArgs, "%-32s %s %s\n", pDesc->pszName, szBuf, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-        }
-
-        case STAMTYPE_U8:
-        case STAMTYPE_U8_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu8, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_X8:
-        case STAMTYPE_X8_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu8, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_U16:
-        case STAMTYPE_U16_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu16, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_X16:
-        case STAMTYPE_X16_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu16, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_U32:
-        case STAMTYPE_U32_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu32, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_X32:
-        case STAMTYPE_X32_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu32, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_U64:
-        case STAMTYPE_U64_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s\n", pDesc->pszName, *pDesc->u.pu64, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        case STAMTYPE_X64:
-        case STAMTYPE_X64_RESET:
-            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
-                return VINF_SUCCESS;
-            pArgs->pfnPrintf(pArgs, "%-32s %8llx %s\n", pDesc->pszName, *pDesc->u.pu64, STAMR3GetUnit(pDesc->enmUnit));
-            break;
-
-        default:
-            AssertMsgFailed(("enmType=%d\n", pDesc->enmType));
-            break;
-    }
-    NOREF(pvArg);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Enumerate the statistics by the means of a callback function.
- *
- * @returns Whatever the callback returns.
- *
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pszPat      The pattern to match samples.
- * @param   pfnEnum     The callback function.
- * @param   pvUser      The pvUser argument of the callback function.
- */
-VMMR3DECL(int) STAMR3EnumU(PUVM pUVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser)
-{
-    STAMR3ENUMONEARGS Args;
-    Args.pVM     = pUVM->pVM;
-    Args.pfnEnum = pfnEnum;
-    Args.pvUser  = pvUser;
-
-    return stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3EnumOne, &Args);
-}
-
-
-/**
- * Enumerate the statistics by the means of a callback function.
- *
- * @returns Whatever the callback returns.
- *
- * @param   pVM         The VM handle.
- * @param   pszPat      The pattern to match samples.
- * @param   pfnEnum     The callback function.
- * @param   pvUser      The pvUser argument of the callback function.
- */
-VMMR3DECL(int) STAMR3Enum(PVM pVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser)
-{
-    return STAMR3EnumU(pVM->pUVM, pszPat, pfnEnum, pvUser);
-}
-
-
-/**
- * Callback function for STARTR3Enum().
- *
- * @returns whatever the callback returns.
- * @param   pDesc       Pointer to the current descriptor.
- * @param   pvArg       Points to a STAMR3ENUMONEARGS structure.
- */
-static int stamR3EnumOne(PSTAMDESC pDesc, void *pvArg)
-{
-    PSTAMR3ENUMONEARGS pArgs = (PSTAMR3ENUMONEARGS)pvArg;
-    int rc;
-    if (pDesc->enmType == STAMTYPE_CALLBACK)
-    {
-        /* Give the enumerator something useful. */
-        char szBuf[512];
-        pDesc->u.Callback.pfnPrint(pArgs->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
-        rc = pArgs->pfnEnum(pDesc->pszName, pDesc->enmType, szBuf, pDesc->enmUnit,
-                            pDesc->enmVisibility, pDesc->pszDesc, pArgs->pvUser);
-    }
-    else
-        rc = pArgs->pfnEnum(pDesc->pszName, pDesc->enmType, pDesc->u.pv, pDesc->enmUnit,
-                            pDesc->enmVisibility, pDesc->pszDesc, pArgs->pvUser);
-    return rc;
-}
-
-
-/**
- * Match a name against an array of patterns.
- *
- * @returns true if it matches, false if it doesn't match.
- * @param   papszExpressions    The array of pattern expressions.
- * @param   cExpressions        The number of array entries.
- * @param   piExpression        Where to read/store the current skip index. Optional.
- * @param   pszName             The name to match.
- */
-static bool stamR3MultiMatch(const char * const *papszExpressions, unsigned cExpressions,
-                             unsigned *piExpression, const char *pszName)
-{
-    for (unsigned i = piExpression ? *piExpression : 0; i < cExpressions; i++)
-    {
-        const char *pszPat = papszExpressions[i];
-        if (RTStrSimplePatternMatch(pszPat, pszName))
-        {
-            /* later:
-            if (piExpression && i > *piExpression)
-            {
-                check if we can skip some expressions
-            }*/
-            return true;
-        }
-    }
-    return false;
-}
-
-
-/**
- * Splits a multi pattern into single ones.
- *
- * @returns Pointer to an array of single patterns. Free it with RTMemTmpFree.
- * @param   pszPat          The pattern to split.
- * @param   pcExpressions   The number of array elements.
- * @param   pszCopy         The pattern copy to free using RTStrFree.
- */
-static char **stamR3SplitPattern(const char *pszPat, unsigned *pcExpressions, char **ppszCopy)
-{
-    Assert(pszPat && *pszPat);
-
-    char *pszCopy = RTStrDup(pszPat);
-    if (!pszCopy)
-        return NULL;
-
-    /* count them & allocate array. */
-    char *psz = pszCopy;
-    unsigned cExpressions = 1;
-    while ((psz = strchr(psz, '|')) != NULL)
-        cExpressions++, psz++;
-
-    char **papszExpressions = (char **)RTMemTmpAllocZ((cExpressions + 1) * sizeof(char *));
-    if (!papszExpressions)
-    {
-        RTStrFree(pszCopy);
-        return NULL;
-    }
-
-    /* split */
-    psz = pszCopy;
-    for (unsigned i = 0;;)
-    {
-        papszExpressions[i] = psz;
-        if (++i >= cExpressions)
-            break;
-        psz = strchr(psz, '|');
-        *psz++ = '\0';
-    }
-
-    /* sort the array, putting '*' last. */
-    /** @todo sort it... */
-
-    *pcExpressions = cExpressions;
-    *ppszCopy = pszCopy;
-    return papszExpressions;
-}
-
-
-/**
- * Enumerates the nodes selected by a pattern or all nodes if no pattern
- * is specified.
- *
- * The call may lock STAM for writing before calling this function, however do
- * not lock it for reading as this function may need to write lock STAM.
- *
- * @returns The rc from the callback.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   pszPat          Pattern.
- * @param   fUpdateRing0    Update the ring-0 .
- * @param   pfnCallback     Callback function which shall be called for matching nodes.
- *                          If it returns anything but VINF_SUCCESS the enumeration is
- *                          terminated and the status code returned to the caller.
- * @param   pvArg           User parameter for the callback.
- */
-static int stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0,
-                       int (*pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg)
-{
-    int rc = VINF_SUCCESS;
-
-    /*
-     * All
-     */
-    if (!pszPat || !*pszPat || !strcmp(pszPat, "*"))
-    {
-        if (fUpdateRing0)
-            stamR3Ring0StatsUpdateU(pUVM, "*");
-
-        STAM_LOCK_RD(pUVM);
-        PSTAMDESC pCur = pUVM->stam.s.pHead;
-        while (pCur)
-        {
-            rc = pfnCallback(pCur, pvArg);
-            if (rc)
-                break;
-
-            /* next */
-            pCur = pCur->pNext;
-        }
-        STAM_UNLOCK_RD(pUVM);
-    }
-
-    /*
-     * Single expression pattern.
-     */
-    else if (!strchr(pszPat, '|'))
-    {
-        if (fUpdateRing0)
-            stamR3Ring0StatsUpdateU(pUVM, pszPat);
-
-        STAM_LOCK_RD(pUVM);
-        /** @todo This needs to be optimized since the GUI is using this path for the VM info dialog.
-         * Note that it's doing exact matching. Organizing the samples in a tree would speed up thing
-         * no end (at least for debug and profile builds). */
-        for (PSTAMDESC pCur = pUVM->stam.s.pHead; pCur; pCur = pCur->pNext)
-            if (RTStrSimplePatternMatch(pszPat, pCur->pszName))
-            {
-                rc = pfnCallback(pCur, pvArg);
-                if (rc)
-                    break;
-            }
-        STAM_UNLOCK_RD(pUVM);
-    }
-
-    /*
-     * Multi expression pattern.
-     */
-    else
-    {
-        /*
-         * Split up the pattern first.
-         */
-        char *pszCopy;
-        unsigned cExpressions;
-        char **papszExpressions = stamR3SplitPattern(pszPat, &cExpressions, &pszCopy);
-        if (!papszExpressions)
-            return VERR_NO_MEMORY;
-
-        /*
-         * Perform the enumeration.
-         */
-        if (fUpdateRing0)
-            stamR3Ring0StatsUpdateMultiU(pUVM, papszExpressions, cExpressions);
-
-        STAM_LOCK_RD(pUVM);
-        unsigned iExpression = 0;
-        for (PSTAMDESC pCur = pUVM->stam.s.pHead; pCur; pCur = pCur->pNext)
-            if (stamR3MultiMatch(papszExpressions, cExpressions, &iExpression, pCur->pszName))
-            {
-                rc = pfnCallback(pCur, pvArg);
-                if (rc)
-                    break;
-            }
-        STAM_UNLOCK_RD(pUVM);
-
-        RTMemTmpFree(papszExpressions);
-        RTStrFree(pszCopy);
-    }
-
-    return rc;
-}
-
-
-/**
- * Registers the ring-0 statistics.
- *
- * @param   pUVM        Pointer to the user mode VM structure.
- */
-static void stamR3Ring0StatsRegisterU(PUVM pUVM)
-{
-    /* GVMM */
-    for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
-        stamR3RegisterU(pUVM, (uint8_t *)&pUVM->stam.s.GVMMStats + g_aGVMMStats[i].offVar, NULL, NULL,
-                        g_aGVMMStats[i].enmType, STAMVISIBILITY_ALWAYS, g_aGVMMStats[i].pszName,
-                        g_aGVMMStats[i].enmUnit, g_aGVMMStats[i].pszDesc);
-    pUVM->stam.s.cRegisteredHostCpus = 0;
-}
-
-
-/**
- * Updates the ring-0 statistics (the copy).
- *
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pszPat      The pattern.
- */
-static void stamR3Ring0StatsUpdateU(PUVM pUVM, const char *pszPat)
-{
-    stamR3Ring0StatsUpdateMultiU(pUVM, &pszPat, 1);
-}
-
-
-/**
- * Updates the ring-0 statistics.
- *
- * The ring-0 statistics aren't directly addressable from ring-3 and must be
- * copied when needed.
- *
- * @param   pUVM        Pointer to the user mode VM structure.
- * @param   pszPat      The pattern (for knowing when to skip).
- */
-static void stamR3Ring0StatsUpdateMultiU(PUVM pUVM, const char * const *papszExpressions, unsigned cExpressions)
-{
-    PVM pVM = pUVM->pVM;
-    if (!pVM || !pVM->pSession)
-        return;
-
-    /* GVMM */
-    bool fUpdate = false;
-    for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
-        if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGVMMStats[i].pszName))
-        {
-            fUpdate = true;
-            break;
-        }
-    if (!fUpdate)
-    {
-        /** @todo check the cpu leaves - rainy day.   */
-    }
-    if (fUpdate)
-    {
-        GVMMQUERYSTATISTICSSREQ Req;
-        Req.Hdr.cbReq = sizeof(Req);
-        Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
-        Req.pSession = pVM->pSession;
-        int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GVMM_QUERY_STATISTICS, 0, &Req.Hdr);
-        if (RT_SUCCESS(rc))
-        {
-            pUVM->stam.s.GVMMStats = Req.Stats;
-
-            /*
-             * Check if the number of host CPUs has changed (it will the first
-             * time around and normally never again).
-             */
-            if (RT_UNLIKELY(pUVM->stam.s.GVMMStats.cHostCpus > pUVM->stam.s.cRegisteredHostCpus))
-            {
-                STAM_LOCK_WR(pUVM);
-                if (RT_UNLIKELY(pUVM->stam.s.GVMMStats.cHostCpus > pUVM->stam.s.cRegisteredHostCpus))
-                {
-                    uint32_t cCpus = pUVM->stam.s.GVMMStats.cHostCpus;
-                    for (uint32_t iCpu  = pUVM->stam.s.cRegisteredHostCpus; iCpu < cCpus; iCpu++)
-                    {
-                        char   szName[120];
-                        size_t cchBase = RTStrPrintf(szName, sizeof(szName), "/GVMM/HostCpus/%u", iCpu);
-                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].idCpu, NULL, NULL,
-                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE, "Host CPU ID");
-                        strcpy(&szName[cchBase], "/idxCpuSet");
-                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].idxCpuSet, NULL, NULL,
-                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE, "CPU Set index");
-                        strcpy(&szName[cchBase], "/DesiredHz");
-                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].uDesiredHz, NULL, NULL,
-                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_HZ, "The desired frequency");
-                        strcpy(&szName[cchBase], "/CurTimerHz");
-                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].uTimerHz, NULL, NULL,
-                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_HZ, "The current timer frequency");
-                        strcpy(&szName[cchBase], "/PPTChanges");
-                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].cChanges, NULL, NULL,
-                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_OCCURENCES, "RTTimerChangeInterval calls");
-                        strcpy(&szName[cchBase], "/PPTStarts");
-                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].cStarts, NULL, NULL,
-                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_OCCURENCES, "RTTimerStart calls");
-                    }
-                    pUVM->stam.s.cRegisteredHostCpus = cCpus;
-                }
-                STAM_UNLOCK_WR(pUVM);
-            }
-        }
-    }
-}
-
-
-/**
- * Get the unit string.
- *
- * @returns Pointer to read only unit string.
- * @param   enmUnit     The unit.
- */
-VMMR3DECL(const char *) STAMR3GetUnit(STAMUNIT enmUnit)
-{
-    switch (enmUnit)
-    {
-        case STAMUNIT_NONE:                 return "";
-        case STAMUNIT_CALLS:                return "calls";
-        case STAMUNIT_COUNT:                return "count";
-        case STAMUNIT_BYTES:                return "bytes";
-        case STAMUNIT_PAGES:                return "pages";
-        case STAMUNIT_ERRORS:               return "errors";
-        case STAMUNIT_OCCURENCES:           return "times";
-        case STAMUNIT_TICKS:                return "ticks";
-        case STAMUNIT_TICKS_PER_CALL:       return "ticks/call";
-        case STAMUNIT_TICKS_PER_OCCURENCE:  return "ticks/time";
-        case STAMUNIT_GOOD_BAD:             return "good:bad";
-        case STAMUNIT_MEGABYTES:            return "megabytes";
-        case STAMUNIT_KILOBYTES:            return "kilobytes";
-        case STAMUNIT_NS:                   return "ns";
-        case STAMUNIT_NS_PER_CALL:          return "ns/call";
-        case STAMUNIT_NS_PER_OCCURENCE:     return "ns/time";
-        case STAMUNIT_PCT:                  return "%";
-        case STAMUNIT_HZ:                   return "Hz";
-
-        default:
-            AssertMsgFailed(("Unknown unit %d\n", enmUnit));
-            return "(?unit?)";
-    }
-}
-
-#ifdef VBOX_WITH_DEBUGGER
-
-/**
- * The '.stats' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) stamR3CmdStats(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires VM to be selected.\n");
-    PUVM pUVM = pVM->pUVM;
-    if (!pUVM->stam.s.pHead)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no statistics present.\n");
-
-    /*
-     * Do the printing.
-     */
-    STAMR3PRINTONEARGS Args;
-    Args.pVM = pVM;
-    Args.pvArg = pCmdHlp;
-    Args.pfnPrintf = stamR3EnumDbgfPrintf;
-
-    return stamR3EnumU(pUVM, cArgs ? paArgs[0].u.pszString : NULL, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
-}
-
-
-/**
- * Display one sample in the debugger.
- *
- * @param   pArgs       Pointer to the print one argument structure.
- * @param   pszFormat   Format string.
- * @param   ...         Format arguments.
- */
-static DECLCALLBACK(void) stamR3EnumDbgfPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
-{
-    PDBGCCMDHLP pCmdHlp = (PDBGCCMDHLP)pArgs->pvArg;
-
-    va_list va;
-    va_start(va, pszFormat);
-    pCmdHlp->pfnPrintfV(pCmdHlp, NULL, pszFormat, va);
-    va_end(va);
-    NOREF(pArgs);
-}
-
-
-/**
- * The '.statsreset' command.
- *
- * @returns VBox status.
- * @param   pCmd        Pointer to the command descriptor (as registered).
- * @param   pCmdHlp     Pointer to command helper functions.
- * @param   pVM         Pointer to the current VM (if any).
- * @param   paArgs      Pointer to (readonly) array of arguments.
- * @param   cArgs       Number of arguments in the array.
- */
-static DECLCALLBACK(int) stamR3CmdStatsReset(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
-{
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires VM to be selected.\n");
-    PUVM pUVM = pVM->pUVM;
-    if (!pUVM->stam.s.pHead)
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no statistics present.\n");
-
-    /*
-     * Execute reset.
-     */
-    int rc = STAMR3ResetU(pUVM, cArgs ? paArgs[0].u.pszString : NULL);
-    if (RT_SUCCESS(rc))
-        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "info: Statistics reset.\n");
-
-    return pCmdHlp->pfnVBoxError(pCmdHlp, rc, "Resetting statistics.\n");
-}
-
-#endif /* VBOX_WITH_DEBUGGER */
-
Index: unk/src/VBox/VMM/STAMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/STAMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,125 +1,0 @@
-/* $Id$ */
-/** @file
- * STAM Internal Header.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___STAMInternal_h
-#define ___STAMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/stam.h>
-#include <VBox/gvmm.h>
-#include <iprt/semaphore.h>
-
-
-
-RT_C_DECLS_BEGIN
-
-/** @defgroup grp_stam_int   Internals
- * @ingroup grp_stam
- * @internal
- * @{
- */
-
-/**
- * Sample descriptor.
- */
-typedef struct STAMDESC
-{
-    /** Pointer to the next sample. */
-    struct STAMDESC    *pNext;
-    /** Sample name. */
-    const char         *pszName;
-    /** Sample type. */
-    STAMTYPE            enmType;
-    /** Visibility type. */
-    STAMVISIBILITY      enmVisibility;
-    /** Pointer to the sample data. */
-    union STAMDESCSAMPLEDATA
-    {
-        /** Counter. */
-        PSTAMCOUNTER    pCounter;
-        /** Profile. */
-        PSTAMPROFILE    pProfile;
-        /** Advanced profile. */
-        PSTAMPROFILEADV pProfileAdv;
-        /** Ratio, unsigned 32-bit. */
-        PSTAMRATIOU32   pRatioU32;
-        /** unsigned 8-bit. */
-        uint8_t        *pu8;
-        /** unsigned 16-bit. */
-        uint16_t       *pu16;
-        /** unsigned 32-bit. */
-        uint32_t       *pu32;
-        /** unsigned 64-bit. */
-        uint64_t       *pu64;
-        /** Simple void pointer. */
-        void           *pv;
-        /** */
-        struct STAMDESCSAMPLEDATACALLBACKS
-        {
-            /** The same pointer. */
-            void                   *pvSample;
-            /** Pointer to the reset callback. */
-            PFNSTAMR3CALLBACKRESET  pfnReset;
-            /** Pointer to the print callback. */
-            PFNSTAMR3CALLBACKPRINT  pfnPrint;
-        }               Callback;
-    }                   u;
-    /** Unit. */
-    STAMUNIT            enmUnit;
-    /** Description. */
-    const char         *pszDesc;
-} STAMDESC;
-/** Pointer to sample descriptor. */
-typedef STAMDESC        *PSTAMDESC;
-/** Pointer to const sample descriptor. */
-typedef const STAMDESC  *PCSTAMDESC;
-
-
-/**
- * STAM data kept in the UVM.
- */
-typedef struct STAMUSERPERVM
-{
-    /** Pointer to the first sample. */
-    R3PTRTYPE(PSTAMDESC)    pHead;
-    /** RW Lock for the list. */
-    RTSEMRW                 RWSem;
-
-    /** The copy of the GVMM statistics. */
-    GVMMSTATS               GVMMStats;
-    /** The number of registered host CPU leaves. */
-    uint32_t                cRegisteredHostCpus;
-} STAMUSERPERVM;
-/** Pointer to the STAM data kept in the UVM. */
-typedef STAMUSERPERVM *PSTAMUSERPERVM;
-
-
-/** Locks the sample descriptors for reading. */
-#define STAM_LOCK_RD(pUVM)      do { int rcSem = RTSemRWRequestRead(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT);  AssertRC(rcSem); } while (0)
-/** Locks the sample descriptors for writing. */
-#define STAM_LOCK_WR(pUVM)      do { int rcSem = RTSemRWRequestWrite(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT); AssertRC(rcSem); } while (0)
-/** UnLocks the sample descriptors after reading. */
-#define STAM_UNLOCK_RD(pUVM)    do { int rcSem = RTSemRWReleaseRead(pUVM->stam.s.RWSem);  AssertRC(rcSem); } while (0)
-/** UnLocks the sample descriptors after writing. */
-#define STAM_UNLOCK_WR(pUVM)    do { int rcSem = RTSemRWReleaseWrite(pUVM->stam.s.RWSem); AssertRC(rcSem); } while (0)
-
-/** @} */
-
-RT_C_DECLS_END
-
-#endif
Index: unk/src/VBox/VMM/TM.cpp
===================================================================
--- /trunk/src/VBox/VMM/TM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,3021 +1,0 @@
-/* $Id$ */
-/** @file
- * TM - Time Manager.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_tm        TM - The Time Manager
- *
- * The Time Manager abstracts the CPU clocks and manages timers used by the VMM,
- * device and drivers.
- *
- * @see grp_tm
- *
- *
- * @section sec_tm_clocks   Clocks
- *
- * There are currently 4 clocks:
- *   - Virtual (guest).
- *   - Synchronous virtual (guest).
- *   - CPU Tick (TSC) (guest).  Only current use is rdtsc emulation. Usually a
- *     function of the virtual clock.
- *   - Real (host).  This is only used for display updates atm.
- *
- * The most important clocks are the three first ones and of these the second is
- * the most interesting.
- *
- *
- * The synchronous virtual clock is tied to the virtual clock except that it
- * will take into account timer delivery lag caused by host scheduling.  It will
- * normally never advance beyond the head timer, and when lagging too far behind
- * it will gradually speed up to catch up with the virtual clock.  All devices
- * implementing time sources accessible to and used by the guest is using this
- * clock (for timers and other things).  This ensures consistency between the
- * time sources.
- *
- * The virtual clock is implemented as an offset to a monotonic, high
- * resolution, wall clock.  The current time source is using the RTTimeNanoTS()
- * machinery based upon the Global Info Pages (GIP), that is, we're using TSC
- * deltas (usually 10 ms) to fill the gaps between GIP updates.  The result is
- * a fairly high res clock that works in all contexts and on all hosts.  The
- * virtual clock is paused when the VM isn't in the running state.
- *
- * The CPU tick (TSC) is normally virtualized as a function of the synchronous
- * virtual clock, where the frequency defaults to the host cpu frequency (as we
- * measure it). In this mode it is possible to configure the frequency. Another
- * (non-default) option is to use the raw unmodified host TSC values. And yet
- * another, to tie it to time spent executing guest code.  All these things are
- * configurable should non-default behavior be desirable.
- *
- * The real clock is a monotonic clock (when available) with relatively low
- * resolution, though this a bit host specific.  Note that we're currently not
- * servicing timers using the real clock when the VM is not running, this is
- * simply because it has not been needed yet therefore not implemented.
- *
- *
- * @subsection subsec_tm_timesync Guest Time Sync / UTC time
- *
- * Guest time syncing is primarily taken care of by the VMM device.  The
- * principle is very simple, the guest additions periodically asks the VMM
- * device what the current UTC time is and makes adjustments accordingly.
- *
- * A complicating factor is that the synchronous virtual clock might be doing
- * catchups and the guest perception is currently a little bit behind the world
- * but it will (hopefully) be catching up soon as we're feeding timer interrupts
- * at a slightly higher rate.  Adjusting the guest clock to the current wall
- * time in the real world would be a bad idea then because the guest will be
- * advancing too fast and run ahead of world time (if the catchup works out).
- * To solve this problem TM provides the VMM device with an UTC time source that
- * gets adjusted with the current lag, so that when the guest eventually catches
- * up the lag it will be showing correct real world time.
- *
- *
- * @section sec_tm_timers   Timers
- *
- * The timers can use any of the TM clocks described in the previous section.
- * Each clock has its own scheduling facility, or timer queue if you like.
- * There are a few factors which makes it a bit complex.  First, there is the
- * usual R0 vs R3 vs. RC thing.  Then there are multiple threads, and then there
- * is the timer thread that periodically checks whether any timers has expired
- * without EMT noticing.  On the API level, all but the create and save APIs
- * must be multithreaded.  EMT will always run the timers.
- *
- * The design is using a doubly linked list of active timers which is ordered
- * by expire date.  This list is only modified by the EMT thread.  Updates to
- * the list are batched in a singly linked list, which is then processed by the
- * EMT thread at the first opportunity (immediately, next time EMT modifies a
- * timer on that clock, or next timer timeout).  Both lists are offset based and
- * all the elements are therefore allocated from the hyper heap.
- *
- * For figuring out when there is need to schedule and run timers TM will:
- *    - Poll whenever somebody queries the virtual clock.
- *    - Poll the virtual clocks from the EM and REM loops.
- *    - Poll the virtual clocks from trap exit path.
- *    - Poll the virtual clocks and calculate first timeout from the halt loop.
- *    - Employ a thread which periodically (100Hz) polls all the timer queues.
- *
- *
- * @image html TMTIMER-Statechart-Diagram.gif
- *
- * @section sec_tm_timer    Logging
- *
- * Level 2: Logs a most of the timer state transitions and queue servicing.
- * Level 3: Logs a few oddments.
- * Level 4: Logs TMCLOCK_VIRTUAL_SYNC catch-up events.
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_TM
-#include <VBox/tm.h>
-#include <VBox/vmm.h>
-#include <VBox/mm.h>
-#include <VBox/ssm.h>
-#include <VBox/dbgf.h>
-#include <VBox/rem.h>
-#include <VBox/pdmapi.h>
-#include <VBox/iom.h>
-#include "TMInternal.h"
-#include <VBox/vm.h>
-
-#include <VBox/pdmdev.h>
-#include <VBox/param.h>
-#include <VBox/err.h>
-
-#include <VBox/log.h>
-#include <iprt/asm.h>
-#include <iprt/asm-math.h>
-#include <iprt/asm-amd64-x86.h>
-#include <iprt/assert.h>
-#include <iprt/thread.h>
-#include <iprt/time.h>
-#include <iprt/timer.h>
-#include <iprt/semaphore.h>
-#include <iprt/string.h>
-#include <iprt/env.h>
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The current saved state version.*/
-#define TM_SAVED_STATE_VERSION  3
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static bool                 tmR3HasFixedTSC(PVM pVM);
-static uint64_t             tmR3CalibrateTSC(PVM pVM);
-static DECLCALLBACK(int)    tmR3Save(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int)    tmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static DECLCALLBACK(void)   tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
-static void                 tmR3TimerQueueRun(PVM pVM, PTMTIMERQUEUE pQueue);
-static void                 tmR3TimerQueueRunVirtualSync(PVM pVM);
-static DECLCALLBACK(int)    tmR3SetWarpDrive(PVM pVM, uint32_t u32Percent);
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-static DECLCALLBACK(void)   tmR3CpuLoadTimer(PVM pVM, PTMTIMER pTimer, void *pvUser);
-#endif
-static DECLCALLBACK(void)   tmR3TimerInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void)   tmR3TimerInfoActive(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-static DECLCALLBACK(void)   tmR3InfoClocks(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-
-
-/**
- * Initializes the TM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMM_INT_DECL(int) TMR3Init(PVM pVM)
-{
-    LogFlow(("TMR3Init:\n"));
-
-    /*
-     * Assert alignment and sizes.
-     */
-    AssertCompileMemberAlignment(VM, tm.s, 32);
-    AssertCompile(sizeof(pVM->tm.s) <= sizeof(pVM->tm.padding));
-    AssertCompileMemberAlignment(TM, TimerCritSect, 8);
-    AssertCompileMemberAlignment(TM, VirtualSyncLock, 8);
-
-    /*
-     * Init the structure.
-     */
-    void *pv;
-    int rc = MMHyperAlloc(pVM, sizeof(pVM->tm.s.paTimerQueuesR3[0]) * TMCLOCK_MAX, 0, MM_TAG_TM, &pv);
-    AssertRCReturn(rc, rc);
-    pVM->tm.s.paTimerQueuesR3 = (PTMTIMERQUEUE)pv;
-    pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pv);
-    pVM->tm.s.paTimerQueuesRC = MMHyperR3ToRC(pVM, pv);
-
-    pVM->tm.s.offVM = RT_OFFSETOF(VM, tm.s);
-    pVM->tm.s.idTimerCpu = pVM->cCpus - 1; /* The last CPU. */
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].enmClock        = TMCLOCK_VIRTUAL;
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].u64Expire       = INT64_MAX;
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].enmClock   = TMCLOCK_VIRTUAL_SYNC;
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].u64Expire  = INT64_MAX;
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].enmClock           = TMCLOCK_REAL;
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].u64Expire          = INT64_MAX;
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].enmClock            = TMCLOCK_TSC;
-    pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].u64Expire           = INT64_MAX;
-
-
-    /*
-     * We directly use the GIP to calculate the virtual time. We map the
-     * the GIP into the guest context so we can do this calculation there
-     * as well and save costly world switches.
-     */
-    pVM->tm.s.pvGIPR3 = (void *)g_pSUPGlobalInfoPage;
-    AssertMsgReturn(pVM->tm.s.pvGIPR3, ("GIP support is now required!\n"), VERR_INTERNAL_ERROR);
-    RTHCPHYS HCPhysGIP;
-    rc = SUPR3GipGetPhys(&HCPhysGIP);
-    AssertMsgRCReturn(rc, ("Failed to get GIP physical address!\n"), rc);
-
-    RTGCPTR GCPtr;
-    rc = MMR3HyperMapHCPhys(pVM, pVM->tm.s.pvGIPR3, NIL_RTR0PTR, HCPhysGIP, PAGE_SIZE, "GIP", &GCPtr);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsgFailed(("Failed to map GIP into GC, rc=%Rrc!\n", rc));
-        return rc;
-    }
-    pVM->tm.s.pvGIPRC = GCPtr;
-    LogFlow(("TMR3Init: HCPhysGIP=%RHp at %RRv\n", HCPhysGIP, pVM->tm.s.pvGIPRC));
-    MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-
-    /* Check assumptions made in TMAllVirtual.cpp about the GIP update interval. */
-    if (    g_pSUPGlobalInfoPage->u32Magic == SUPGLOBALINFOPAGE_MAGIC
-        &&  g_pSUPGlobalInfoPage->u32UpdateIntervalNS >= 250000000 /* 0.25s */)
-        return VMSetError(pVM, VERR_INTERNAL_ERROR, RT_SRC_POS,
-                          N_("The GIP update interval is too big. u32UpdateIntervalNS=%RU32 (u32UpdateHz=%RU32)"),
-                          g_pSUPGlobalInfoPage->u32UpdateIntervalNS, g_pSUPGlobalInfoPage->u32UpdateHz);
-    LogRel(("TM: GIP - u32Mode=%d (%s) u32UpdateHz=%u\n", g_pSUPGlobalInfoPage->u32Mode,
-            g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_SYNC_TSC ? "SyncTSC"
-            : g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_ASYNC_TSC ? "AsyncTSC" : "Unknown",
-            g_pSUPGlobalInfoPage->u32UpdateHz));
-
-    /*
-     * Setup the VirtualGetRaw backend.
-     */
-    pVM->tm.s.VirtualGetRawDataR3.pu64Prev = &pVM->tm.s.u64VirtualRawPrev;
-    pVM->tm.s.VirtualGetRawDataR3.pfnBad = tmVirtualNanoTSBad;
-    pVM->tm.s.VirtualGetRawDataR3.pfnRediscover = tmVirtualNanoTSRediscover;
-    if (ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_SSE2)
-    {
-        if (g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_SYNC_TSC)
-            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLFenceSync;
-        else
-            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLFenceAsync;
-    }
-    else
-    {
-        if (g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_SYNC_TSC)
-            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLegacySync;
-        else
-            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLegacyAsync;
-    }
-
-    pVM->tm.s.VirtualGetRawDataRC.pu64Prev = MMHyperR3ToRC(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
-    pVM->tm.s.VirtualGetRawDataR0.pu64Prev = MMHyperR3ToR0(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
-    AssertReturn(pVM->tm.s.VirtualGetRawDataR0.pu64Prev, VERR_INTERNAL_ERROR);
-    /* The rest is done in TMR3InitFinalize since it's too early to call PDM. */
-
-    /*
-     * Init the locks.
-     */
-    rc = PDMR3CritSectInit(pVM, &pVM->tm.s.TimerCritSect, RT_SRC_POS, "TM Timer Lock");
-    if (RT_FAILURE(rc))
-        return rc;
-    rc = PDMR3CritSectInit(pVM, &pVM->tm.s.VirtualSyncLock, RT_SRC_POS, "TM VirtualSync Lock");
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Get our CFGM node, create it if necessary.
-     */
-    PCFGMNODE pCfgHandle = CFGMR3GetChild(CFGMR3GetRoot(pVM), "TM");
-    if (!pCfgHandle)
-    {
-        rc = CFGMR3InsertNode(CFGMR3GetRoot(pVM), "TM", &pCfgHandle);
-        AssertRCReturn(rc, rc);
-    }
-
-    /*
-     * Determine the TSC configuration and frequency.
-     */
-    /* mode */
-    /** @cfgm{/TM/TSCVirtualized,bool,true}
-     * Use a virtualize TSC, i.e. trap all TSC access. */
-    rc = CFGMR3QueryBool(pCfgHandle, "TSCVirtualized", &pVM->tm.s.fTSCVirtualized);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        pVM->tm.s.fTSCVirtualized = true; /* trap rdtsc */
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying bool value \"UseRealTSC\""));
-
-    /* source */
-    /** @cfgm{/TM/UseRealTSC,bool,false}
-     * Use the real TSC as time source for the TSC instead of the synchronous
-     * virtual clock (false, default). */
-    rc = CFGMR3QueryBool(pCfgHandle, "UseRealTSC", &pVM->tm.s.fTSCUseRealTSC);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        pVM->tm.s.fTSCUseRealTSC = false; /* use virtual time */
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying bool value \"UseRealTSC\""));
-    if (!pVM->tm.s.fTSCUseRealTSC)
-        pVM->tm.s.fTSCVirtualized = true;
-
-    /* TSC reliability */
-    /** @cfgm{/TM/MaybeUseOffsettedHostTSC,bool,detect}
-     * Whether the CPU has a fixed TSC rate and may be used in offsetted mode with
-     * VT-x/AMD-V execution.  This is autodetected in a very restrictive way by
-     * default. */
-    rc = CFGMR3QueryBool(pCfgHandle, "MaybeUseOffsettedHostTSC", &pVM->tm.s.fMaybeUseOffsettedHostTSC);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-    {
-        if (!pVM->tm.s.fTSCUseRealTSC)
-            pVM->tm.s.fMaybeUseOffsettedHostTSC = tmR3HasFixedTSC(pVM);
-        else
-            pVM->tm.s.fMaybeUseOffsettedHostTSC = true;
-    }
-
-    /** @cfgm{TM/TSCTicksPerSecond, uint32_t, Current TSC frequency from GIP}
-     * The number of TSC ticks per second (i.e. the TSC frequency). This will
-     * override TSCUseRealTSC, TSCVirtualized and MaybeUseOffsettedHostTSC.
-     */
-    rc = CFGMR3QueryU64(pCfgHandle, "TSCTicksPerSecond", &pVM->tm.s.cTSCTicksPerSecond);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-    {
-        pVM->tm.s.cTSCTicksPerSecond = tmR3CalibrateTSC(pVM);
-        if (    !pVM->tm.s.fTSCUseRealTSC
-            &&  pVM->tm.s.cTSCTicksPerSecond >= _4G)
-        {
-            pVM->tm.s.cTSCTicksPerSecond = _4G - 1; /* (A limitation of our math code) */
-            pVM->tm.s.fMaybeUseOffsettedHostTSC = false;
-        }
-    }
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint64_t value \"TSCTicksPerSecond\""));
-    else if (   pVM->tm.s.cTSCTicksPerSecond < _1M
-             || pVM->tm.s.cTSCTicksPerSecond >= _4G)
-        return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
-                          N_("Configuration error: \"TSCTicksPerSecond\" = %RI64 is not in the range 1MHz..4GHz-1"),
-                          pVM->tm.s.cTSCTicksPerSecond);
-    else
-    {
-        pVM->tm.s.fTSCUseRealTSC = pVM->tm.s.fMaybeUseOffsettedHostTSC = false;
-        pVM->tm.s.fTSCVirtualized = true;
-    }
-
-    /** @cfgm{TM/TSCTiedToExecution, bool, false}
-     * Whether the TSC should be tied to execution. This will exclude most of the
-     * virtualization overhead, but will by default include the time spent in the
-     * halt state (see TM/TSCNotTiedToHalt). This setting will override all other
-     * TSC settings except for TSCTicksPerSecond and TSCNotTiedToHalt, which should
-     * be used avoided or used with great care. Note that this will only work right
-     * together with VT-x or AMD-V, and with a single virtual CPU. */
-    rc = CFGMR3QueryBoolDef(pCfgHandle, "TSCTiedToExecution", &pVM->tm.s.fTSCTiedToExecution, false);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying bool value \"TSCTiedToExecution\""));
-    if (pVM->tm.s.fTSCTiedToExecution)
-    {
-        /* tied to execution, override all other settings. */
-        pVM->tm.s.fTSCVirtualized = true;
-        pVM->tm.s.fTSCUseRealTSC = true;
-        pVM->tm.s.fMaybeUseOffsettedHostTSC = false;
-    }
-
-    /** @cfgm{TM/TSCNotTiedToHalt, bool, true}
-     * For overriding the default of TM/TSCTiedToExecution, i.e. set this to false
-     * to make the TSC freeze during HLT. */
-    rc = CFGMR3QueryBoolDef(pCfgHandle, "TSCNotTiedToHalt", &pVM->tm.s.fTSCNotTiedToHalt, false);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying bool value \"TSCNotTiedToHalt\""));
-
-    /* setup and report */
-    if (pVM->tm.s.fTSCVirtualized)
-        CPUMR3SetCR4Feature(pVM, X86_CR4_TSD, ~X86_CR4_TSD);
-    else
-        CPUMR3SetCR4Feature(pVM, 0, ~X86_CR4_TSD);
-    LogRel(("TM: cTSCTicksPerSecond=%#RX64 (%'RU64) fTSCVirtualized=%RTbool fTSCUseRealTSC=%RTbool\n"
-            "TM: fMaybeUseOffsettedHostTSC=%RTbool TSCTiedToExecution=%RTbool TSCNotTiedToHalt=%RTbool\n",
-            pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.fTSCVirtualized, pVM->tm.s.fTSCUseRealTSC,
-            pVM->tm.s.fMaybeUseOffsettedHostTSC, pVM->tm.s.fTSCTiedToExecution, pVM->tm.s.fTSCNotTiedToHalt));
-
-    /*
-     * Configure the timer synchronous virtual time.
-     */
-    /** @cfgm{TM/ScheduleSlack, uint32_t, ns, 0, UINT32_MAX, 100000}
-     * Scheduling slack when processing timers. */
-    rc = CFGMR3QueryU32(pCfgHandle, "ScheduleSlack", &pVM->tm.s.u32VirtualSyncScheduleSlack);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        pVM->tm.s.u32VirtualSyncScheduleSlack           =   100000; /* 0.100ms (ASSUMES virtual time is nanoseconds) */
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying 32-bit integer value \"ScheduleSlack\""));
-
-    /** @cfgm{TM/CatchUpStopThreshold, uint64_t, ns, 0, UINT64_MAX, 500000}
-     * When to stop a catch-up, considering it successful. */
-    rc = CFGMR3QueryU64(pCfgHandle, "CatchUpStopThreshold", &pVM->tm.s.u64VirtualSyncCatchUpStopThreshold);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        pVM->tm.s.u64VirtualSyncCatchUpStopThreshold    =   500000; /* 0.5ms */
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpStopThreshold\""));
-
-    /** @cfgm{TM/CatchUpGiveUpThreshold, uint64_t, ns, 0, UINT64_MAX, 60000000000}
-     * When to give up a catch-up attempt. */
-    rc = CFGMR3QueryU64(pCfgHandle, "CatchUpGiveUpThreshold", &pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold  = UINT64_C(60000000000); /* 60 sec */
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpGiveUpThreshold\""));
-
-
-    /** @cfgm{TM/CatchUpPrecentage[0..9], uint32_t, %, 1, 2000, various}
-     * The catch-up percent for a given period.  */
-    /** @cfgm{TM/CatchUpStartThreshold[0..9], uint64_t, ns, 0, UINT64_MAX,
-     * The catch-up period threshold, or if you like, when a period starts.  */
-#define TM_CFG_PERIOD(iPeriod, DefStart, DefPct) \
-    do \
-    { \
-        uint64_t u64; \
-        rc = CFGMR3QueryU64(pCfgHandle, "CatchUpStartThreshold" #iPeriod, &u64); \
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND) \
-            u64 = UINT64_C(DefStart); \
-        else if (RT_FAILURE(rc)) \
-            return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpThreshold" #iPeriod "\"")); \
-        if (    (iPeriod > 0 && u64 <= pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod - 1].u64Start) \
-            ||  u64 >= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold) \
-            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, N_("Configuration error: Invalid start of period #" #iPeriod ": %'RU64"), u64); \
-        pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u64Start = u64; \
-        rc = CFGMR3QueryU32(pCfgHandle, "CatchUpPrecentage" #iPeriod, &pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u32Percentage); \
-        if (rc == VERR_CFGM_VALUE_NOT_FOUND) \
-            pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u32Percentage = (DefPct); \
-        else if (RT_FAILURE(rc)) \
-            return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying 32-bit integer value \"CatchUpPrecentage" #iPeriod "\"")); \
-    } while (0)
-    /* This needs more tuning. Not sure if we really need so many period and be so gentle. */
-    TM_CFG_PERIOD(0,     750000,   5); /* 0.75ms at 1.05x */
-    TM_CFG_PERIOD(1,    1500000,  10); /* 1.50ms at 1.10x */
-    TM_CFG_PERIOD(2,    8000000,  25); /*    8ms at 1.25x */
-    TM_CFG_PERIOD(3,   30000000,  50); /*   30ms at 1.50x */
-    TM_CFG_PERIOD(4,   75000000,  75); /*   75ms at 1.75x */
-    TM_CFG_PERIOD(5,  175000000, 100); /*  175ms at 2x */
-    TM_CFG_PERIOD(6,  500000000, 200); /*  500ms at 3x */
-    TM_CFG_PERIOD(7, 3000000000, 300); /*    3s  at 4x */
-    TM_CFG_PERIOD(8,30000000000, 400); /*   30s  at 5x */
-    TM_CFG_PERIOD(9,55000000000, 500); /*   55s  at 6x */
-    AssertCompile(RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods) == 10);
-#undef TM_CFG_PERIOD
-
-    /*
-     * Configure real world time (UTC).
-     */
-    /** @cfgm{TM/UTCOffset, int64_t, ns, INT64_MIN, INT64_MAX, 0}
-     * The UTC offset. This is used to put the guest back or forwards in time.  */
-    rc = CFGMR3QueryS64(pCfgHandle, "UTCOffset", &pVM->tm.s.offUTC);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        pVM->tm.s.offUTC = 0; /* ns */
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying 64-bit integer value \"UTCOffset\""));
-
-    /*
-     * Setup the warp drive.
-     */
-    /** @cfgm{TM/WarpDrivePercentage, uint32_t, %, 0, 20000, 100}
-     * The warp drive percentage, 100% is normal speed.  This is used to speed up
-     * or slow down the virtual clock, which can be useful for fast forwarding
-     * borring periods during tests. */
-    rc = CFGMR3QueryU32(pCfgHandle, "WarpDrivePercentage", &pVM->tm.s.u32VirtualWarpDrivePercentage);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        rc = CFGMR3QueryU32(CFGMR3GetRoot(pVM), "WarpDrivePercentage", &pVM->tm.s.u32VirtualWarpDrivePercentage); /* legacy */
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        pVM->tm.s.u32VirtualWarpDrivePercentage = 100;
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint32_t value \"WarpDrivePercent\""));
-    else if (   pVM->tm.s.u32VirtualWarpDrivePercentage < 2
-             || pVM->tm.s.u32VirtualWarpDrivePercentage > 20000)
-        return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
-                          N_("Configuration error: \"WarpDrivePercent\" = %RI32 is not in the range 2..20000"),
-                          pVM->tm.s.u32VirtualWarpDrivePercentage);
-    pVM->tm.s.fVirtualWarpDrive = pVM->tm.s.u32VirtualWarpDrivePercentage != 100;
-    if (pVM->tm.s.fVirtualWarpDrive)
-        LogRel(("TM: u32VirtualWarpDrivePercentage=%RI32\n", pVM->tm.s.u32VirtualWarpDrivePercentage));
-
-    /*
-     * Gather the Host Hz configuration values.
-     */
-    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzMax", &pVM->tm.s.cHostHzMax, 20000);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint32_t value \"HostHzMax\""));
-
-    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorTimerCpu", &pVM->tm.s.cPctHostHzFudgeFactorTimerCpu, 111);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorTimerCpu\""));
-
-    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorOtherCpu", &pVM->tm.s.cPctHostHzFudgeFactorOtherCpu, 110);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorOtherCpu\""));
-
-    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp100", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp100, 300);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp100\""));
-
-    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp200", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp200, 250);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp200\""));
-
-    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp400", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp400, 200);
-    if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp400\""));
-
-    /*
-     * Start the timer (guard against REM not yielding).
-     */
-    /** @cfgm{TM/TimerMillies, uint32_t, ms, 1, 1000, 10}
-     * The watchdog timer interval.  */
-    uint32_t u32Millies;
-    rc = CFGMR3QueryU32(pCfgHandle, "TimerMillies", &u32Millies);
-    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
-        u32Millies = 10;
-    else if (RT_FAILURE(rc))
-        return VMSetError(pVM, rc, RT_SRC_POS,
-                          N_("Configuration error: Failed to query uint32_t value \"TimerMillies\""));
-    rc = RTTimerCreate(&pVM->tm.s.pTimer, u32Millies, tmR3TimerCallback, pVM);
-    if (RT_FAILURE(rc))
-    {
-        AssertMsgFailed(("Failed to create timer, u32Millies=%d rc=%Rrc.\n", u32Millies, rc));
-        return rc;
-    }
-    Log(("TM: Created timer %p firing every %d milliseconds\n", pVM->tm.s.pTimer, u32Millies));
-    pVM->tm.s.u32TimerMillies = u32Millies;
-
-    /*
-     * Register saved state.
-     */
-    rc = SSMR3RegisterInternal(pVM, "tm", 1, TM_SAVED_STATE_VERSION, sizeof(uint64_t) * 8,
-                               NULL, NULL, NULL,
-                               NULL, tmR3Save, NULL,
-                               NULL, tmR3Load, NULL);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Register statistics.
-     */
-    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR3.c1nsSteps,STAMTYPE_U32, "/TM/R3/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
-    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR3.cBadPrev, STAMTYPE_U32, "/TM/R3/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
-    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR0.c1nsSteps,STAMTYPE_U32, "/TM/R0/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
-    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR0.cBadPrev, STAMTYPE_U32, "/TM/R0/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
-    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataRC.c1nsSteps,STAMTYPE_U32, "/TM/RC/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
-    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataRC.cBadPrev, STAMTYPE_U32, "/TM/RC/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
-    STAM_REL_REG(     pVM,(void*)&pVM->tm.s.offVirtualSync,               STAMTYPE_U64, "/TM/VirtualSync/CurrentOffset",               STAMUNIT_NS, "The current offset. (subtract GivenUp to get the lag)");
-    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.offVirtualSyncGivenUp,        STAMTYPE_U64, "/TM/VirtualSync/GivenUp",                     STAMUNIT_NS, "Nanoseconds of the 'CurrentOffset' that's been given up and won't ever be attempted caught up with.");
-    STAM_REL_REG(     pVM,(void*)&pVM->tm.s.uMaxHzHint,                   STAMTYPE_U32, "/TM/MaxHzHint",                               STAMUNIT_HZ, "Max guest timer frequency hint.");
-
-#ifdef VBOX_WITH_STATISTICS
-    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR3.cExpired,    STAMTYPE_U32, "/TM/R3/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
-    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR3.cUpdateRaces,STAMTYPE_U32, "/TM/R3/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
-    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR0.cExpired,    STAMTYPE_U32, "/TM/R0/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
-    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR0.cUpdateRaces,STAMTYPE_U32, "/TM/R0/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
-    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataRC.cExpired,    STAMTYPE_U32, "/TM/RC/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
-    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataRC.cUpdateRaces,STAMTYPE_U32, "/TM/RC/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
-    STAM_REG(pVM, &pVM->tm.s.StatDoQueues,                            STAMTYPE_PROFILE, "/TM/DoQueues",                    STAMUNIT_TICKS_PER_CALL, "Profiling timer TMR3TimerQueuesDo.");
-    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL],      STAMTYPE_PROFILE_ADV, "/TM/DoQueues/Virtual",            STAMUNIT_TICKS_PER_CALL, "Time spent on the virtual clock queue.");
-    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], STAMTYPE_PROFILE_ADV, "/TM/DoQueues/VirtualSync",        STAMUNIT_TICKS_PER_CALL, "Time spent on the virtual sync clock queue.");
-    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_REAL],         STAMTYPE_PROFILE_ADV, "/TM/DoQueues/Real",               STAMUNIT_TICKS_PER_CALL, "Time spent on the real clock queue.");
-
-    STAM_REG(pVM, &pVM->tm.s.StatPoll,                                STAMTYPE_COUNTER, "/TM/Poll",                            STAMUNIT_OCCURENCES, "TMTimerPoll calls.");
-    STAM_REG(pVM, &pVM->tm.s.StatPollAlreadySet,                      STAMTYPE_COUNTER, "/TM/Poll/AlreadySet",                 STAMUNIT_OCCURENCES, "TMTimerPoll calls where the FF was already set.");
-    STAM_REG(pVM, &pVM->tm.s.StatPollELoop,                           STAMTYPE_COUNTER, "/TM/Poll/ELoop",                      STAMUNIT_OCCURENCES, "Times TMTimerPoll has given up getting a consistent virtual sync data set.");
-    STAM_REG(pVM, &pVM->tm.s.StatPollMiss,                            STAMTYPE_COUNTER, "/TM/Poll/Miss",                       STAMUNIT_OCCURENCES, "TMTimerPoll calls where nothing had expired.");
-    STAM_REG(pVM, &pVM->tm.s.StatPollRunning,                         STAMTYPE_COUNTER, "/TM/Poll/Running",                    STAMUNIT_OCCURENCES, "TMTimerPoll calls where the queues were being run.");
-    STAM_REG(pVM, &pVM->tm.s.StatPollSimple,                          STAMTYPE_COUNTER, "/TM/Poll/Simple",                     STAMUNIT_OCCURENCES, "TMTimerPoll calls where we could take the simple path.");
-    STAM_REG(pVM, &pVM->tm.s.StatPollVirtual,                         STAMTYPE_COUNTER, "/TM/Poll/HitsVirtual",                STAMUNIT_OCCURENCES, "The number of times TMTimerPoll found an expired TMCLOCK_VIRTUAL queue.");
-    STAM_REG(pVM, &pVM->tm.s.StatPollVirtualSync,                     STAMTYPE_COUNTER, "/TM/Poll/HitsVirtualSync",            STAMUNIT_OCCURENCES, "The number of times TMTimerPoll found an expired TMCLOCK_VIRTUAL_SYNC queue.");
-
-    STAM_REG(pVM, &pVM->tm.s.StatPostponedR3,                         STAMTYPE_COUNTER, "/TM/PostponedR3",                     STAMUNIT_OCCURENCES, "Postponed due to unschedulable state, in ring-3.");
-    STAM_REG(pVM, &pVM->tm.s.StatPostponedRZ,                         STAMTYPE_COUNTER, "/TM/PostponedRZ",                     STAMUNIT_OCCURENCES, "Postponed due to unschedulable state, in ring-0 / RC.");
-
-    STAM_REG(pVM, &pVM->tm.s.StatScheduleOneR3,                       STAMTYPE_PROFILE, "/TM/ScheduleOneR3",               STAMUNIT_TICKS_PER_CALL, "Profiling the scheduling of one queue during a TMTimer* call in EMT.");
-    STAM_REG(pVM, &pVM->tm.s.StatScheduleOneRZ,                       STAMTYPE_PROFILE, "/TM/ScheduleOneRZ",               STAMUNIT_TICKS_PER_CALL, "Profiling the scheduling of one queue during a TMTimer* call in EMT.");
-    STAM_REG(pVM, &pVM->tm.s.StatScheduleSetFF,                       STAMTYPE_COUNTER, "/TM/ScheduleSetFF",                   STAMUNIT_OCCURENCES, "The number of times the timer FF was set instead of doing scheduling.");
-
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSet,                            STAMTYPE_COUNTER, "/TM/TimerSet",                        STAMUNIT_OCCURENCES, "Calls");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetOpt,                         STAMTYPE_COUNTER, "/TM/TimerSet/Opt",                    STAMUNIT_OCCURENCES, "Optimized path taken.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetR3,                          STAMTYPE_PROFILE, "/TM/TimerSet/R3",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-3.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRZ,                          STAMTYPE_PROFILE, "/TM/TimerSet/RZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-0 / RC.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStActive,                    STAMTYPE_COUNTER, "/TM/TimerSet/StActive",               STAMUNIT_OCCURENCES, "ACTIVE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStExpDeliver,                STAMTYPE_COUNTER, "/TM/TimerSet/StExpDeliver",           STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStOther,                     STAMTYPE_COUNTER, "/TM/TimerSet/StOther",                STAMUNIT_OCCURENCES, "Other states");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendStop,                  STAMTYPE_COUNTER, "/TM/TimerSet/StPendStop",             STAMUNIT_OCCURENCES, "PENDING_STOP");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendStopSched,             STAMTYPE_COUNTER, "/TM/TimerSet/StPendStopSched",        STAMUNIT_OCCURENCES, "PENDING_STOP_SCHEDULE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendSched,                 STAMTYPE_COUNTER, "/TM/TimerSet/StPendSched",            STAMUNIT_OCCURENCES, "PENDING_SCHEDULE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendResched,               STAMTYPE_COUNTER, "/TM/TimerSet/StPendResched",          STAMUNIT_OCCURENCES, "PENDING_RESCHEDULE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStStopped,                   STAMTYPE_COUNTER, "/TM/TimerSet/StStopped",              STAMUNIT_OCCURENCES, "STOPPED");
-
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelative,                    STAMTYPE_COUNTER, "/TM/TimerSetRelative",                STAMUNIT_OCCURENCES, "Calls");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeOpt,                 STAMTYPE_COUNTER, "/TM/TimerSetRelative/Opt",            STAMUNIT_OCCURENCES, "Optimized path taken.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeR3,                  STAMTYPE_PROFILE, "/TM/TimerSetRelative/R3",         STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetRelative calls made in ring-3.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeRZ,                  STAMTYPE_PROFILE, "/TM/TimerSetRelative/RZ",         STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetReltaive calls made in ring-0 / RC.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeRacyVirtSync,        STAMTYPE_COUNTER, "/TM/TimerSetRelative/RacyVirtSync",   STAMUNIT_OCCURENCES, "Potentially racy virtual sync timer update.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStActive,            STAMTYPE_COUNTER, "/TM/TimerSetRelative/StActive",       STAMUNIT_OCCURENCES, "ACTIVE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStExpDeliver,        STAMTYPE_COUNTER, "/TM/TimerSetRelative/StExpDeliver",   STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStOther,             STAMTYPE_COUNTER, "/TM/TimerSetRelative/StOther",        STAMUNIT_OCCURENCES, "Other states");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendStop,          STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendStop",     STAMUNIT_OCCURENCES, "PENDING_STOP");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendStopSched,     STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendStopSched",STAMUNIT_OCCURENCES, "PENDING_STOP_SCHEDULE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendSched,         STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendSched",    STAMUNIT_OCCURENCES, "PENDING_SCHEDULE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendResched,       STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendResched",  STAMUNIT_OCCURENCES, "PENDING_RESCHEDULE");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStStopped,           STAMTYPE_COUNTER, "/TM/TimerSetRelative/StStopped",      STAMUNIT_OCCURENCES, "STOPPED");
-
-    STAM_REG(pVM, &pVM->tm.s.StatTimerStopR3,                         STAMTYPE_PROFILE, "/TM/TimerStopR3",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerStop calls made in ring-3.");
-    STAM_REG(pVM, &pVM->tm.s.StatTimerStopRZ,                         STAMTYPE_PROFILE, "/TM/TimerStopRZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerStop calls made in ring-0 / RC.");
-
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualGet,                          STAMTYPE_COUNTER, "/TM/VirtualGet",                      STAMUNIT_OCCURENCES, "The number of times TMTimerGet was called when the clock was running.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualGetSetFF,                     STAMTYPE_COUNTER, "/TM/VirtualGetSetFF",                 STAMUNIT_OCCURENCES, "Times we set the FF when calling TMTimerGet.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGet,                      STAMTYPE_COUNTER, "/TM/VirtualSyncGet",                  STAMUNIT_OCCURENCES, "The number of times tmVirtualSyncGetEx was called.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetELoop,                 STAMTYPE_COUNTER, "/TM/VirtualSyncGet/ELoop",            STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx has given up getting a consistent virtual sync data set.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetExpired,               STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Expired",          STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx encountered an expired timer stopping the clock.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetLocked,                STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Locked",           STAMUNIT_OCCURENCES, "Times we successfully acquired the lock in tmVirtualSyncGetEx.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetLockless,              STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Lockless",         STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx returned without needing to take the lock.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetSetFF,                 STAMTYPE_COUNTER, "/TM/VirtualSyncGet/SetFF",            STAMUNIT_OCCURENCES, "Times we set the FF when calling tmVirtualSyncGetEx.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualPause,                        STAMTYPE_COUNTER, "/TM/VirtualPause",                    STAMUNIT_OCCURENCES, "The number of times TMR3TimerPause was called.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualResume,                       STAMTYPE_COUNTER, "/TM/VirtualResume",                   STAMUNIT_OCCURENCES, "The number of times TMR3TimerResume was called.");
-
-    STAM_REG(pVM, &pVM->tm.s.StatTimerCallbackSetFF,                  STAMTYPE_COUNTER, "/TM/CallbackSetFF",                   STAMUNIT_OCCURENCES, "The number of times the timer callback set FF.");
-
-    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE010,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE010",      STAMUNIT_OCCURENCES, "In catch-up mode, 10% or lower.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE025,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE025",      STAMUNIT_OCCURENCES, "In catch-up mode, 25%-11%.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE100,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE100",      STAMUNIT_OCCURENCES, "In catch-up mode, 100%-26%.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupOther,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupOther",      STAMUNIT_OCCURENCES, "In catch-up mode, > 100%.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCNotFixed,                         STAMTYPE_COUNTER, "/TM/TSC/Intercept/NotFixed",          STAMUNIT_OCCURENCES, "TSC is not fixed, it may run at variable speed.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCNotTicking,                       STAMTYPE_COUNTER, "/TM/TSC/Intercept/NotTicking",        STAMUNIT_OCCURENCES, "TSC is not ticking.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCSyncNotTicking,                   STAMTYPE_COUNTER, "/TM/TSC/Intercept/SyncNotTicking",    STAMUNIT_OCCURENCES, "VirtualSync isn't ticking.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCWarp,                             STAMTYPE_COUNTER, "/TM/TSC/Intercept/Warp",              STAMUNIT_OCCURENCES, "Warpdrive is active.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCSet,                              STAMTYPE_COUNTER, "/TM/TSC/Sets",                        STAMUNIT_OCCURENCES, "Calls to TMCpuTickSet.");
-    STAM_REG(pVM, &pVM->tm.s.StatTSCUnderflow,                        STAMTYPE_COUNTER, "/TM/TSC/Underflow",                   STAMUNIT_OCCURENCES, "TSC underflow; corrected with last seen value .");
-#endif /* VBOX_WITH_STATISTICS */
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.offTSCRawSrc,          STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS, "TSC offset relative the raw source",           "/TM/TSC/offCPU%u", i);
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsTotal,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,               "Resettable: Total CPU run time.",   "/TM/CPU/%02u", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecuting,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code.",    "/TM/CPU/%02u/PrfExecuting", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecLong,    STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - long hauls.",    "/TM/CPU/%02u/PrfExecLong", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecShort,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - short stretches.",    "/TM/CPU/%02u/PrfExecShort", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecTiny,    STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - tiny bits.",    "/TM/CPU/%02u/PrfExecTiny", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsHalted,      STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent halted.",                  "/TM/CPU/%02u/PrfHalted", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsOther,       STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent in the VMM or preempted.", "/TM/CPU/%02u/PrfOther", i);
-# endif
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsTotal,              STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Total CPU run time.",                          "/TM/CPU/%02u/cNsTotal", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsExecuting,          STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent executing guest code.",             "/TM/CPU/%02u/cNsExecuting", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsHalted,             STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent halted.",                           "/TM/CPU/%02u/cNsHalted", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsOther,              STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent in the VMM or preempted.",          "/TM/CPU/%02u/cNsOther", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cPeriodsExecuting,     STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Times executed guest code.",                   "/TM/CPU/%02u/cPeriodsExecuting", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cPeriodsHalted,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Times halted.",                                "/TM/CPU/%02u/cPeriodsHalted", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.CpuLoad.cPctExecuting, STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent executing guest code recently.",    "/TM/CPU/%02u/pctExecuting", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.CpuLoad.cPctHalted,    STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent halted recently.",                  "/TM/CPU/%02u/pctHalted", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.CpuLoad.cPctOther,     STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent in the VMM or preempted recently.", "/TM/CPU/%02u/pctOther", i);
-#endif
-    }
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctExecuting, STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent executing guest code recently.",    "/TM/CPU/pctExecuting");
-    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctHalted,    STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent halted recently.",                  "/TM/CPU/pctHalted");
-    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctOther,     STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent in the VMM or preempted recently.", "/TM/CPU/pctOther");
-#endif
-
-#ifdef VBOX_WITH_STATISTICS
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncCatchup,              STAMTYPE_PROFILE_ADV, "/TM/VirtualSync/CatchUp",    STAMUNIT_TICKS_PER_OCCURENCE, "Counting and measuring the times spent catching up.");
-    STAM_REG(pVM, (void *)&pVM->tm.s.fVirtualSyncCatchUp,                  STAMTYPE_U8, "/TM/VirtualSync/CatchUpActive",             STAMUNIT_NONE, "Catch-Up active indicator.");
-    STAM_REG(pVM, (void *)&pVM->tm.s.u32VirtualSyncCatchUpPercentage,     STAMTYPE_U32, "/TM/VirtualSync/CatchUpPercentage",          STAMUNIT_PCT, "The catch-up percentage. (+100/100 to get clock multiplier)");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncFF,                       STAMTYPE_PROFILE, "/TM/VirtualSync/FF",         STAMUNIT_TICKS_PER_OCCURENCE, "Time spent in TMR3VirtualSyncFF by all but the dedicate timer EMT.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGiveUp,                   STAMTYPE_COUNTER, "/TM/VirtualSync/GiveUp",              STAMUNIT_OCCURENCES, "Times the catch-up was abandoned.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGiveUpBeforeStarting,     STAMTYPE_COUNTER, "/TM/VirtualSync/GiveUpBeforeStarting",STAMUNIT_OCCURENCES, "Times the catch-up was abandoned before even starting. (Typically debugging++.)");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRun,                      STAMTYPE_COUNTER, "/TM/VirtualSync/Run",                 STAMUNIT_OCCURENCES, "Times the virtual sync timer queue was considered.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunRestart,               STAMTYPE_COUNTER, "/TM/VirtualSync/Run/Restarts",        STAMUNIT_OCCURENCES, "Times the clock was restarted after a run.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunStop,                  STAMTYPE_COUNTER, "/TM/VirtualSync/Run/Stop",            STAMUNIT_OCCURENCES, "Times the clock was stopped when calculating the current time before examining the timers.");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunStoppedAlready,        STAMTYPE_COUNTER, "/TM/VirtualSync/Run/StoppedAlready",  STAMUNIT_OCCURENCES, "Times the clock was already stopped elsewhere (TMVirtualSyncGet).");
-    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunSlack,                 STAMTYPE_PROFILE, "/TM/VirtualSync/Run/Slack",     STAMUNIT_NS_PER_OCCURENCE, "The scheduling slack. (Catch-up handed out when running timers.)");
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods); i++)
-    {
-        STAMR3RegisterF(pVM, &pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage,    STAMTYPE_U32, STAMVISIBILITY_ALWAYS,          STAMUNIT_PCT, "The catch-up percentage.",         "/TM/VirtualSync/Periods/%u", i);
-        STAMR3RegisterF(pVM, &pVM->tm.s.aStatVirtualSyncCatchupAdjust[i],           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,   STAMUNIT_OCCURENCES, "Times adjusted to this period.",   "/TM/VirtualSync/Periods/%u/Adjust", i);
-        STAMR3RegisterF(pVM, &pVM->tm.s.aStatVirtualSyncCatchupInitial[i],          STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,   STAMUNIT_OCCURENCES, "Times started in this period.",    "/TM/VirtualSync/Periods/%u/Initial", i);
-        STAMR3RegisterF(pVM, &pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u64Start,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,           STAMUNIT_NS, "Start of this period (lag).",      "/TM/VirtualSync/Periods/%u/Start", i);
-    }
-#endif /* VBOX_WITH_STATISTICS */
-
-    /*
-     * Register info handlers.
-     */
-    DBGFR3InfoRegisterInternalEx(pVM, "timers",       "Dumps all timers. No arguments.",          tmR3TimerInfo,        DBGFINFO_FLAGS_RUN_ON_EMT);
-    DBGFR3InfoRegisterInternalEx(pVM, "activetimers", "Dumps active all timers. No arguments.",   tmR3TimerInfoActive,  DBGFINFO_FLAGS_RUN_ON_EMT);
-    DBGFR3InfoRegisterInternalEx(pVM, "clocks",       "Display the time of the various clocks.",  tmR3InfoClocks,       DBGFINFO_FLAGS_RUN_ON_EMT);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Checks if the host CPU has a fixed TSC frequency.
- *
- * @returns true if it has, false if it hasn't.
- *
- * @remark  This test doesn't bother with very old CPUs that don't do power
- *          management or any other stuff that might influence the TSC rate.
- *          This isn't currently relevant.
- */
-static bool tmR3HasFixedTSC(PVM pVM)
-{
-    if (ASMHasCpuId())
-    {
-        uint32_t uEAX, uEBX, uECX, uEDX;
-
-        if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_AMD)
-        {
-            /*
-             * AuthenticAMD - Check for APM support and that TscInvariant is set.
-             *
-             * This test isn't correct with respect to fixed/non-fixed TSC and
-             * older models, but this isn't relevant since the result is currently
-             * only used for making a decision on AMD-V models.
-             */
-            ASMCpuId(0x80000000, &uEAX, &uEBX, &uECX, &uEDX);
-            if (uEAX >= 0x80000007)
-            {
-                PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
-
-                ASMCpuId(0x80000007, &uEAX, &uEBX, &uECX, &uEDX);
-                if (   (uEDX & X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR) /* TscInvariant */
-                    && pGip->u32Mode == SUPGIPMODE_SYNC_TSC /* no fixed tsc if the gip timer is in async mode */)
-                    return true;
-            }
-        }
-        else if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_INTEL)
-        {
-            /*
-             * GenuineIntel - Check the model number.
-             *
-             * This test is lacking in the same way and for the same reasons
-             * as the AMD test above.
-             */
-            ASMCpuId(1, &uEAX, &uEBX, &uECX, &uEDX);
-            unsigned uModel  = (uEAX >> 4) & 0x0f;
-            unsigned uFamily = (uEAX >> 8) & 0x0f;
-            if (uFamily == 0x0f)
-                uFamily += (uEAX >> 20) & 0xff;
-            if (uFamily >= 0x06)
-                uModel += ((uEAX >> 16) & 0x0f) << 4;
-            if (    (uFamily == 0x0f /*P4*/     && uModel >= 0x03)
-                ||  (uFamily == 0x06 /*P2/P3*/  && uModel >= 0x0e))
-                return true;
-        }
-    }
-    return false;
-}
-
-
-/**
- * Calibrate the CPU tick.
- *
- * @returns Number of ticks per second.
- */
-static uint64_t tmR3CalibrateTSC(PVM pVM)
-{
-    /*
-     * Use GIP when available present.
-     */
-    uint64_t    u64Hz;
-    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
-    if (    pGip
-        &&  pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC)
-    {
-        unsigned iCpu = pGip->u32Mode != SUPGIPMODE_ASYNC_TSC ? 0 : ASMGetApicId();
-        if (iCpu >= RT_ELEMENTS(pGip->aCPUs))
-            AssertReleaseMsgFailed(("iCpu=%d - the ApicId is too high. send VBox.log and hardware specs!\n", iCpu));
-        else
-        {
-            if (tmR3HasFixedTSC(pVM))
-                /* Sleep a bit to get a more reliable CpuHz value. */
-                RTThreadSleep(32);
-            else
-            {
-                /* Spin for 40ms to try push up the CPU frequency and get a more reliable CpuHz value. */
-                const uint64_t u64 = RTTimeMilliTS();
-                while ((RTTimeMilliTS() - u64) < 40 /*ms*/)
-                    /* nothing */;
-            }
-
-            pGip = g_pSUPGlobalInfoPage;
-            if (    pGip
-                &&  pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC
-                &&  (u64Hz = pGip->aCPUs[iCpu].u64CpuHz)
-                &&  u64Hz != ~(uint64_t)0)
-                return u64Hz;
-        }
-    }
-
-    /* call this once first to make sure it's initialized. */
-    RTTimeNanoTS();
-
-    /*
-     * Yield the CPU to increase our chances of getting
-     * a correct value.
-     */
-    RTThreadYield();                    /* Try avoid interruptions between TSC and NanoTS samplings. */
-    static const unsigned   s_auSleep[5] = { 50, 30, 30, 40, 40 };
-    uint64_t                au64Samples[5];
-    unsigned                i;
-    for (i = 0; i < RT_ELEMENTS(au64Samples); i++)
-    {
-        RTMSINTERVAL    cMillies;
-        int             cTries   = 5;
-        uint64_t        u64Start = ASMReadTSC();
-        uint64_t        u64End;
-        uint64_t        StartTS  = RTTimeNanoTS();
-        uint64_t        EndTS;
-        do
-        {
-            RTThreadSleep(s_auSleep[i]);
-            u64End = ASMReadTSC();
-            EndTS  = RTTimeNanoTS();
-            cMillies = (RTMSINTERVAL)((EndTS - StartTS + 500000) / 1000000);
-        } while (   cMillies == 0       /* the sleep may be interrupted... */
-                 || (cMillies < 20 && --cTries > 0));
-        uint64_t    u64Diff = u64End - u64Start;
-
-        au64Samples[i] = (u64Diff * 1000) / cMillies;
-        AssertMsg(cTries > 0, ("cMillies=%d i=%d\n", cMillies, i));
-    }
-
-    /*
-     * Discard the highest and lowest results and calculate the average.
-     */
-    unsigned iHigh = 0;
-    unsigned iLow = 0;
-    for (i = 1; i < RT_ELEMENTS(au64Samples); i++)
-    {
-        if (au64Samples[i] < au64Samples[iLow])
-            iLow = i;
-        if (au64Samples[i] > au64Samples[iHigh])
-            iHigh = i;
-    }
-    au64Samples[iLow] = 0;
-    au64Samples[iHigh] = 0;
-
-    u64Hz = au64Samples[0];
-    for (i = 1; i < RT_ELEMENTS(au64Samples); i++)
-        u64Hz += au64Samples[i];
-    u64Hz /= RT_ELEMENTS(au64Samples) - 2;
-
-    return u64Hz;
-}
-
-
-/**
- * Finalizes the TM initialization.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMM_INT_DECL(int) TMR3InitFinalize(PVM pVM)
-{
-    int rc;
-
-    /*
-     * Resolve symbols.
-     */
-    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSBad",          &pVM->tm.s.VirtualGetRawDataRC.pfnBad);
-    AssertRCReturn(rc, rc);
-    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSRediscover",   &pVM->tm.s.VirtualGetRawDataRC.pfnRediscover);
-    AssertRCReturn(rc, rc);
-    if (pVM->tm.s.pfnVirtualGetRawR3       == RTTimeNanoTSLFenceSync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceSync",  &pVM->tm.s.pfnVirtualGetRawRC);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLFenceAsync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceAsync", &pVM->tm.s.pfnVirtualGetRawRC);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacySync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacySync",  &pVM->tm.s.pfnVirtualGetRawRC);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacyAsync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacyAsync", &pVM->tm.s.pfnVirtualGetRawRC);
-    else
-        AssertFatalFailed();
-    AssertRCReturn(rc, rc);
-
-    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "tmVirtualNanoTSBad",          &pVM->tm.s.VirtualGetRawDataR0.pfnBad);
-    AssertRCReturn(rc, rc);
-    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "tmVirtualNanoTSRediscover",   &pVM->tm.s.VirtualGetRawDataR0.pfnRediscover);
-    AssertRCReturn(rc, rc);
-    if (pVM->tm.s.pfnVirtualGetRawR3       == RTTimeNanoTSLFenceSync)
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLFenceSync",  &pVM->tm.s.pfnVirtualGetRawR0);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLFenceAsync)
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLFenceAsync", &pVM->tm.s.pfnVirtualGetRawR0);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacySync)
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLegacySync",  &pVM->tm.s.pfnVirtualGetRawR0);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacyAsync)
-        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLegacyAsync", &pVM->tm.s.pfnVirtualGetRawR0);
-    else
-        AssertFatalFailed();
-    AssertRCReturn(rc, rc);
-
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-    /*
-     * Create a timer for refreshing the CPU load stats.
-     */
-    PTMTIMER pTimer;
-    rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, tmR3CpuLoadTimer, NULL, "CPU Load Timer", &pTimer);
-    if (RT_SUCCESS(rc))
-        rc = TMTimerSetMillies(pTimer, 1000);
-#endif
-
-    return rc;
-}
-
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * @param   pVM     The VM.
- * @param   offDelta    Relocation delta relative to old location.
- */
-VMM_INT_DECL(void) TMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    int rc;
-    LogFlow(("TMR3Relocate\n"));
-
-    pVM->tm.s.pvGIPRC = MMHyperR3ToRC(pVM, pVM->tm.s.pvGIPR3);
-    pVM->tm.s.paTimerQueuesRC = MMHyperR3ToRC(pVM, pVM->tm.s.paTimerQueuesR3);
-    pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pVM->tm.s.paTimerQueuesR3);
-
-    pVM->tm.s.VirtualGetRawDataRC.pu64Prev = MMHyperR3ToRC(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
-    AssertFatal(pVM->tm.s.VirtualGetRawDataRC.pu64Prev);
-    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSBad",          &pVM->tm.s.VirtualGetRawDataRC.pfnBad);
-    AssertFatalRC(rc);
-    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSRediscover",   &pVM->tm.s.VirtualGetRawDataRC.pfnRediscover);
-    AssertFatalRC(rc);
-
-    if (pVM->tm.s.pfnVirtualGetRawR3       == RTTimeNanoTSLFenceSync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceSync",  &pVM->tm.s.pfnVirtualGetRawRC);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLFenceAsync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceAsync", &pVM->tm.s.pfnVirtualGetRawRC);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacySync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacySync",  &pVM->tm.s.pfnVirtualGetRawRC);
-    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacyAsync)
-        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacyAsync", &pVM->tm.s.pfnVirtualGetRawRC);
-    else
-        AssertFatalFailed();
-    AssertFatalRC(rc);
-
-    /*
-     * Iterate the timers updating the pVMRC pointers.
-     */
-    for (PTMTIMER pTimer = pVM->tm.s.pCreated; pTimer; pTimer = pTimer->pBigNext)
-    {
-        pTimer->pVMRC = pVM->pVMRC;
-        pTimer->pVMR0 = pVM->pVMR0;
-    }
-}
-
-
-/**
- * Terminates the TM.
- *
- * Termination means cleaning up and freeing all resources,
- * the VM it self is at this point powered off or suspended.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMM_INT_DECL(int) TMR3Term(PVM pVM)
-{
-    AssertMsg(pVM->tm.s.offVM, ("bad init order!\n"));
-    if (pVM->tm.s.pTimer)
-    {
-        int rc = RTTimerDestroy(pVM->tm.s.pTimer);
-        AssertRC(rc);
-        pVM->tm.s.pTimer = NULL;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * The VM is being reset.
- *
- * For the TM component this means that a rescheduling is preformed,
- * the FF is cleared and but without running the queues. We'll have to
- * check if this makes sense or not, but it seems like a good idea now....
- *
- * @param   pVM     VM handle.
- */
-VMM_INT_DECL(void) TMR3Reset(PVM pVM)
-{
-    LogFlow(("TMR3Reset:\n"));
-    VM_ASSERT_EMT(pVM);
-    tmTimerLock(pVM);
-
-    /*
-     * Abort any pending catch up.
-     * This isn't perfect...
-     */
-    if (pVM->tm.s.fVirtualSyncCatchUp)
-    {
-        const uint64_t offVirtualNow = TMVirtualGetNoCheck(pVM);
-        const uint64_t offVirtualSyncNow = TMVirtualSyncGetNoCheck(pVM);
-        if (pVM->tm.s.fVirtualSyncCatchUp)
-        {
-            STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
-
-            const uint64_t offOld = pVM->tm.s.offVirtualSyncGivenUp;
-            const uint64_t offNew = offVirtualNow - offVirtualSyncNow;
-            Assert(offOld <= offNew);
-            ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
-            ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSync, offNew);
-            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
-            LogRel(("TM: Aborting catch-up attempt on reset with a %'RU64 ns lag on reset; new total: %'RU64 ns\n", offNew - offOld, offNew));
-        }
-    }
-
-    /*
-     * Process the queues.
-     */
-    for (int i = 0; i < TMCLOCK_MAX; i++)
-        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[i]);
-#ifdef VBOX_STRICT
-    tmTimerQueuesSanityChecks(pVM, "TMR3Reset");
-#endif
-
-    PVMCPU pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
-    VMCPU_FF_CLEAR(pVCpuDst, VMCPU_FF_TIMER); /** @todo FIXME: this isn't right. */
-    tmTimerUnlock(pVM);
-}
-
-
-/**
- * Resolve a builtin RC symbol.
- * Called by PDM when loading or relocating GC modules.
- *
- * @returns VBox status
- * @param   pVM             VM Handle.
- * @param   pszSymbol       Symbol to resolve.
- * @param   pRCPtrValue     Where to store the symbol value.
- * @remark  This has to     work before TMR3Relocate() is called.
- */
-VMM_INT_DECL(int) TMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
-{
-    if (!strcmp(pszSymbol, "g_pSUPGlobalInfoPage"))
-        *pRCPtrValue = MMHyperR3ToRC(pVM, &pVM->tm.s.pvGIPRC);
-    //else if (..)
-    else
-        return VERR_SYMBOL_NOT_FOUND;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) tmR3Save(PVM pVM, PSSMHANDLE pSSM)
-{
-    LogFlow(("tmR3Save:\n"));
-#ifdef VBOX_STRICT
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-        Assert(!pVCpu->tm.s.fTSCTicking);
-    }
-    Assert(!pVM->tm.s.cVirtualTicking);
-    Assert(!pVM->tm.s.fVirtualSyncTicking);
-#endif
-
-    /*
-     * Save the virtual clocks.
-     */
-    /* the virtual clock. */
-    SSMR3PutU64(pSSM, TMCLOCK_FREQ_VIRTUAL);
-    SSMR3PutU64(pSSM, pVM->tm.s.u64Virtual);
-
-    /* the virtual timer synchronous clock. */
-    SSMR3PutU64(pSSM, pVM->tm.s.u64VirtualSync);
-    SSMR3PutU64(pSSM, pVM->tm.s.offVirtualSync);
-    SSMR3PutU64(pSSM, pVM->tm.s.offVirtualSyncGivenUp);
-    SSMR3PutU64(pSSM, pVM->tm.s.u64VirtualSyncCatchUpPrev);
-    SSMR3PutBool(pSSM, pVM->tm.s.fVirtualSyncCatchUp);
-
-    /* real time clock */
-    SSMR3PutU64(pSSM, TMCLOCK_FREQ_REAL);
-
-    /* the cpu tick clock. */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-        SSMR3PutU64(pSSM, TMCpuTickGet(pVCpu));
-    }
-    return SSMR3PutU64(pSSM, pVM->tm.s.cTSCTicksPerSecond);
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) tmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    LogFlow(("tmR3Load:\n"));
-
-    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
-#ifdef VBOX_STRICT
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-        Assert(!pVCpu->tm.s.fTSCTicking);
-    }
-    Assert(!pVM->tm.s.cVirtualTicking);
-    Assert(!pVM->tm.s.fVirtualSyncTicking);
-#endif
-
-    /*
-     * Validate version.
-     */
-    if (uVersion != TM_SAVED_STATE_VERSION)
-    {
-        AssertMsgFailed(("tmR3Load: Invalid version uVersion=%d!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    /*
-     * Load the virtual clock.
-     */
-    pVM->tm.s.cVirtualTicking = 0;
-    /* the virtual clock. */
-    uint64_t u64Hz;
-    int rc = SSMR3GetU64(pSSM, &u64Hz);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (u64Hz != TMCLOCK_FREQ_VIRTUAL)
-    {
-        AssertMsgFailed(("The virtual clock frequency differs! Saved: %'RU64 Binary: %'RU64\n",
-                         u64Hz, TMCLOCK_FREQ_VIRTUAL));
-        return VERR_SSM_VIRTUAL_CLOCK_HZ;
-    }
-    SSMR3GetU64(pSSM, &pVM->tm.s.u64Virtual);
-    pVM->tm.s.u64VirtualOffset = 0;
-
-    /* the virtual timer synchronous clock. */
-    pVM->tm.s.fVirtualSyncTicking = false;
-    uint64_t u64;
-    SSMR3GetU64(pSSM, &u64);
-    pVM->tm.s.u64VirtualSync = u64;
-    SSMR3GetU64(pSSM, &u64);
-    pVM->tm.s.offVirtualSync = u64;
-    SSMR3GetU64(pSSM, &u64);
-    pVM->tm.s.offVirtualSyncGivenUp = u64;
-    SSMR3GetU64(pSSM, &u64);
-    pVM->tm.s.u64VirtualSyncCatchUpPrev = u64;
-    bool f;
-    SSMR3GetBool(pSSM, &f);
-    pVM->tm.s.fVirtualSyncCatchUp = f;
-
-    /* the real clock  */
-    rc = SSMR3GetU64(pSSM, &u64Hz);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (u64Hz != TMCLOCK_FREQ_REAL)
-    {
-        AssertMsgFailed(("The real clock frequency differs! Saved: %'RU64 Binary: %'RU64\n",
-                         u64Hz, TMCLOCK_FREQ_REAL));
-        return VERR_SSM_VIRTUAL_CLOCK_HZ; /* misleading... */
-    }
-
-    /* the cpu tick clock. */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        pVCpu->tm.s.fTSCTicking = false;
-        SSMR3GetU64(pSSM, &pVCpu->tm.s.u64TSC);
-
-        if (pVM->tm.s.fTSCUseRealTSC)
-            pVCpu->tm.s.offTSCRawSrc = 0; /** @todo TSC restore stuff and HWACC. */
-    }
-
-    rc = SSMR3GetU64(pSSM, &u64Hz);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (!pVM->tm.s.fTSCUseRealTSC)
-        pVM->tm.s.cTSCTicksPerSecond = u64Hz;
-
-    LogRel(("TM: cTSCTicksPerSecond=%#RX64 (%'RU64) fTSCVirtualized=%RTbool fTSCUseRealTSC=%RTbool (state load)\n",
-            pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.fTSCVirtualized, pVM->tm.s.fTSCUseRealTSC));
-
-    /*
-     * Make sure timers get rescheduled immediately.
-     */
-    PVMCPU pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
-    VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Internal TMR3TimerCreate worker.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   enmClock    The timer clock.
- * @param   pszDesc     The timer description.
- * @param   ppTimer     Where to store the timer pointer on success.
- */
-static int tmr3TimerCreate(PVM pVM, TMCLOCK enmClock, const char *pszDesc, PPTMTIMERR3 ppTimer)
-{
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Allocate the timer.
-     */
-    PTMTIMERR3 pTimer = NULL;
-    if (pVM->tm.s.pFree && VM_IS_EMT(pVM))
-    {
-        pTimer = pVM->tm.s.pFree;
-        pVM->tm.s.pFree = pTimer->pBigNext;
-        Log3(("TM: Recycling timer %p, new free head %p.\n", pTimer, pTimer->pBigNext));
-    }
-
-    if (!pTimer)
-    {
-        int rc = MMHyperAlloc(pVM, sizeof(*pTimer), 0, MM_TAG_TM, (void **)&pTimer);
-        if (RT_FAILURE(rc))
-            return rc;
-        Log3(("TM: Allocated new timer %p\n", pTimer));
-    }
-
-    /*
-     * Initialize it.
-     */
-    pTimer->u64Expire       = 0;
-    pTimer->enmClock        = enmClock;
-    pTimer->pVMR3           = pVM;
-    pTimer->pVMR0           = pVM->pVMR0;
-    pTimer->pVMRC           = pVM->pVMRC;
-    pTimer->enmState        = TMTIMERSTATE_STOPPED;
-    pTimer->offScheduleNext = 0;
-    pTimer->offNext         = 0;
-    pTimer->offPrev         = 0;
-    pTimer->pvUser          = NULL;
-    pTimer->pCritSect       = NULL;
-    pTimer->pszDesc         = pszDesc;
-
-    /* insert into the list of created timers. */
-    tmTimerLock(pVM);
-    pTimer->pBigPrev        = NULL;
-    pTimer->pBigNext        = pVM->tm.s.pCreated;
-    pVM->tm.s.pCreated      = pTimer;
-    if (pTimer->pBigNext)
-        pTimer->pBigNext->pBigPrev = pTimer;
-#ifdef VBOX_STRICT
-    tmTimerQueuesSanityChecks(pVM, "tmR3TimerCreate");
-#endif
-    tmTimerUnlock(pVM);
-
-    *ppTimer = pTimer;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Creates a device timer.
- *
- * @returns VBox status.
- * @param   pVM             The VM to create the timer in.
- * @param   pDevIns         Device instance.
- * @param   enmClock        The clock to use on this timer.
- * @param   pfnCallback     Callback function.
- * @param   pvUser          The user argument to the callback.
- * @param   fFlags          Timer creation flags, see grp_tm_timer_flags.
- * @param   pszDesc         Pointer to description string which must stay around
- *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
- * @param   ppTimer         Where to store the timer on success.
- */
-VMM_INT_DECL(int) TMR3TimerCreateDevice(PVM pVM, PPDMDEVINS pDevIns, TMCLOCK enmClock, PFNTMTIMERDEV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
-{
-    AssertReturn(!(fFlags & ~(TMTIMER_FLAGS_NO_CRIT_SECT)), VERR_INVALID_PARAMETER);
-
-    /*
-     * Allocate and init stuff.
-     */
-    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, ppTimer);
-    if (RT_SUCCESS(rc))
-    {
-        (*ppTimer)->enmType         = TMTIMERTYPE_DEV;
-        (*ppTimer)->u.Dev.pfnTimer  = pfnCallback;
-        (*ppTimer)->u.Dev.pDevIns   = pDevIns;
-        (*ppTimer)->pvUser          = pvUser;
-        if (fFlags & TMTIMER_FLAGS_DEFAULT_CRIT_SECT)
-        {
-            if (pDevIns->pCritSectR3)
-                (*ppTimer)->pCritSect = pDevIns->pCritSectR3;
-            else
-                (*ppTimer)->pCritSect = IOMR3GetCritSect(pVM);
-        }
-        Log(("TM: Created device timer %p clock %d callback %p '%s'\n", (*ppTimer), enmClock, pfnCallback, pszDesc));
-    }
-
-    return rc;
-}
-
-
-/**
- * Creates a driver timer.
- *
- * @returns VBox status.
- * @param   pVM             The VM to create the timer in.
- * @param   pDrvIns         Driver instance.
- * @param   enmClock        The clock to use on this timer.
- * @param   pfnCallback     Callback function.
- * @param   pvUser          The user argument to the callback.
- * @param   fFlags          Timer creation flags, see grp_tm_timer_flags.
- * @param   pszDesc         Pointer to description string which must stay around
- *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
- * @param   ppTimer         Where to store the timer on success.
- */
-VMM_INT_DECL(int) TMR3TimerCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, TMCLOCK enmClock, PFNTMTIMERDRV pfnCallback, void *pvUser,
-                                        uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
-{
-    AssertReturn(!(fFlags & ~(TMTIMER_FLAGS_NO_CRIT_SECT)), VERR_INVALID_PARAMETER);
-
-    /*
-     * Allocate and init stuff.
-     */
-    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, ppTimer);
-    if (RT_SUCCESS(rc))
-    {
-        (*ppTimer)->enmType         = TMTIMERTYPE_DRV;
-        (*ppTimer)->u.Drv.pfnTimer  = pfnCallback;
-        (*ppTimer)->u.Drv.pDrvIns   = pDrvIns;
-        (*ppTimer)->pvUser          = pvUser;
-        Log(("TM: Created device timer %p clock %d callback %p '%s'\n", (*ppTimer), enmClock, pfnCallback, pszDesc));
-    }
-
-    return rc;
-}
-
-
-/**
- * Creates an internal timer.
- *
- * @returns VBox status.
- * @param   pVM             The VM to create the timer in.
- * @param   enmClock        The clock to use on this timer.
- * @param   pfnCallback     Callback function.
- * @param   pvUser          User argument to be passed to the callback.
- * @param   pszDesc         Pointer to description string which must stay around
- *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
- * @param   ppTimer         Where to store the timer on success.
- */
-VMMR3DECL(int) TMR3TimerCreateInternal(PVM pVM, TMCLOCK enmClock, PFNTMTIMERINT pfnCallback, void *pvUser, const char *pszDesc, PPTMTIMERR3 ppTimer)
-{
-    /*
-     * Allocate and init  stuff.
-     */
-    PTMTIMER pTimer;
-    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, &pTimer);
-    if (RT_SUCCESS(rc))
-    {
-        pTimer->enmType             = TMTIMERTYPE_INTERNAL;
-        pTimer->u.Internal.pfnTimer = pfnCallback;
-        pTimer->pvUser              = pvUser;
-        *ppTimer = pTimer;
-        Log(("TM: Created internal timer %p clock %d callback %p '%s'\n", pTimer, enmClock, pfnCallback, pszDesc));
-    }
-
-    return rc;
-}
-
-/**
- * Creates an external timer.
- *
- * @returns Timer handle on success.
- * @returns NULL on failure.
- * @param   pVM             The VM to create the timer in.
- * @param   enmClock        The clock to use on this timer.
- * @param   pfnCallback     Callback function.
- * @param   pvUser          User argument.
- * @param   pszDesc         Pointer to description string which must stay around
- *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
- */
-VMMR3DECL(PTMTIMERR3) TMR3TimerCreateExternal(PVM pVM, TMCLOCK enmClock, PFNTMTIMEREXT pfnCallback, void *pvUser, const char *pszDesc)
-{
-    /*
-     * Allocate and init stuff.
-     */
-    PTMTIMERR3 pTimer;
-    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, &pTimer);
-    if (RT_SUCCESS(rc))
-    {
-        pTimer->enmType             = TMTIMERTYPE_EXTERNAL;
-        pTimer->u.External.pfnTimer = pfnCallback;
-        pTimer->pvUser              = pvUser;
-        Log(("TM: Created external timer %p clock %d callback %p '%s'\n", pTimer, enmClock, pfnCallback, pszDesc));
-        return pTimer;
-    }
-
-    return NULL;
-}
-
-
-/**
- * Destroy a timer
- *
- * @returns VBox status.
- * @param   pTimer          Timer handle as returned by one of the create functions.
- */
-VMMR3DECL(int) TMR3TimerDestroy(PTMTIMER pTimer)
-{
-    /*
-     * Be extra careful here.
-     */
-    if (!pTimer)
-        return VINF_SUCCESS;
-    AssertPtr(pTimer);
-    Assert((unsigned)pTimer->enmClock < (unsigned)TMCLOCK_MAX);
-
-    PVM             pVM      = pTimer->CTX_SUFF(pVM);
-    PTMTIMERQUEUE   pQueue   = &pVM->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock];
-    bool            fActive  = false;
-    bool            fPending = false;
-
-    AssertMsg(   !pTimer->pCritSect
-              || VMR3GetState(pVM) != VMSTATE_RUNNING
-              || PDMCritSectIsOwner(pTimer->pCritSect), ("%s\n", pTimer->pszDesc));
-
-    /*
-     * The rest of the game happens behind the lock, just
-     * like create does. All the work is done here.
-     */
-    tmTimerLock(pVM);
-    for (int cRetries = 1000;; cRetries--)
-    {
-        /*
-         * Change to the DESTROY state.
-         */
-        TMTIMERSTATE enmState    = pTimer->enmState;
-        TMTIMERSTATE enmNewState = enmState;
-        Log2(("TMTimerDestroy: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d\n",
-              pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), cRetries));
-        switch (enmState)
-        {
-            case TMTIMERSTATE_STOPPED:
-            case TMTIMERSTATE_EXPIRED_DELIVER:
-                break;
-
-            case TMTIMERSTATE_ACTIVE:
-                fActive     = true;
-                break;
-
-            case TMTIMERSTATE_PENDING_STOP:
-            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
-            case TMTIMERSTATE_PENDING_RESCHEDULE:
-                fActive     = true;
-                fPending    = true;
-                break;
-
-            case TMTIMERSTATE_PENDING_SCHEDULE:
-                fPending    = true;
-                break;
-
-            /*
-             * This shouldn't happen as the caller should make sure there are no races.
-             */
-            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
-            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
-            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
-                AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->pszDesc));
-                tmTimerUnlock(pVM);
-                if (!RTThreadYield())
-                    RTThreadSleep(1);
-                AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->pszDesc),
-                                VERR_TM_UNSTABLE_STATE);
-                tmTimerLock(pVM);
-                continue;
-
-            /*
-             * Invalid states.
-             */
-            case TMTIMERSTATE_FREE:
-            case TMTIMERSTATE_DESTROY:
-                tmTimerUnlock(pVM);
-                AssertLogRelMsgFailedReturn(("pTimer=%p %s\n", pTimer, tmTimerState(enmState)), VERR_TM_INVALID_STATE);
-
-            default:
-                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
-                tmTimerUnlock(pVM);
-                return VERR_TM_UNKNOWN_STATE;
-        }
-
-        /*
-         * Try switch to the destroy state.
-         * This should always succeed as the caller should make sure there are no race.
-         */
-        bool fRc;
-        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_DESTROY, enmState, fRc);
-        if (fRc)
-            break;
-        AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->pszDesc));
-        tmTimerUnlock(pVM);
-        AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->pszDesc),
-                        VERR_TM_UNSTABLE_STATE);
-        tmTimerLock(pVM);
-    }
-
-    /*
-     * Unlink from the active list.
-     */
-    if (fActive)
-    {
-        const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
-        const PTMTIMER pNext = TMTIMER_GET_NEXT(pTimer);
-        if (pPrev)
-            TMTIMER_SET_NEXT(pPrev, pNext);
-        else
-        {
-            TMTIMER_SET_HEAD(pQueue, pNext);
-            pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
-        }
-        if (pNext)
-            TMTIMER_SET_PREV(pNext, pPrev);
-        pTimer->offNext = 0;
-        pTimer->offPrev = 0;
-    }
-
-    /*
-     * Unlink from the schedule list by running it.
-     */
-    if (fPending)
-    {
-        Log3(("TMR3TimerDestroy: tmTimerQueueSchedule\n"));
-        STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
-        Assert(pQueue->offSchedule);
-        tmTimerQueueSchedule(pVM, pQueue);
-    }
-
-    /*
-     * Read to move the timer from the created list and onto the free list.
-     */
-    Assert(!pTimer->offNext); Assert(!pTimer->offPrev); Assert(!pTimer->offScheduleNext);
-
-    /* unlink from created list */
-    if (pTimer->pBigPrev)
-        pTimer->pBigPrev->pBigNext = pTimer->pBigNext;
-    else
-        pVM->tm.s.pCreated         = pTimer->pBigNext;
-    if (pTimer->pBigNext)
-        pTimer->pBigNext->pBigPrev = pTimer->pBigPrev;
-    pTimer->pBigNext = 0;
-    pTimer->pBigPrev = 0;
-
-    /* free */
-    Log2(("TM: Inserting %p into the free list ahead of %p!\n", pTimer, pVM->tm.s.pFree));
-    TM_SET_STATE(pTimer, TMTIMERSTATE_FREE);
-    pTimer->pBigNext = pVM->tm.s.pFree;
-    pVM->tm.s.pFree = pTimer;
-
-#ifdef VBOX_STRICT
-    tmTimerQueuesSanityChecks(pVM, "TMR3TimerDestroy");
-#endif
-    tmTimerUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Destroy all timers owned by a device.
- *
- * @returns VBox status.
- * @param   pVM             VM handle.
- * @param   pDevIns         Device which timers should be destroyed.
- */
-VMM_INT_DECL(int) TMR3TimerDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
-{
-    LogFlow(("TMR3TimerDestroyDevice: pDevIns=%p\n", pDevIns));
-    if (!pDevIns)
-        return VERR_INVALID_PARAMETER;
-
-    tmTimerLock(pVM);
-    PTMTIMER    pCur = pVM->tm.s.pCreated;
-    while (pCur)
-    {
-        PTMTIMER pDestroy = pCur;
-        pCur = pDestroy->pBigNext;
-        if (    pDestroy->enmType == TMTIMERTYPE_DEV
-            &&  pDestroy->u.Dev.pDevIns == pDevIns)
-        {
-            int rc = TMR3TimerDestroy(pDestroy);
-            AssertRC(rc);
-        }
-    }
-    tmTimerUnlock(pVM);
-
-    LogFlow(("TMR3TimerDestroyDevice: returns VINF_SUCCESS\n"));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Destroy all timers owned by a driver.
- *
- * @returns VBox status.
- * @param   pVM             VM handle.
- * @param   pDrvIns         Driver which timers should be destroyed.
- */
-VMM_INT_DECL(int) TMR3TimerDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
-{
-    LogFlow(("TMR3TimerDestroyDriver: pDrvIns=%p\n", pDrvIns));
-    if (!pDrvIns)
-        return VERR_INVALID_PARAMETER;
-
-    tmTimerLock(pVM);
-    PTMTIMER    pCur = pVM->tm.s.pCreated;
-    while (pCur)
-    {
-        PTMTIMER pDestroy = pCur;
-        pCur = pDestroy->pBigNext;
-        if (    pDestroy->enmType == TMTIMERTYPE_DRV
-            &&  pDestroy->u.Drv.pDrvIns == pDrvIns)
-        {
-            int rc = TMR3TimerDestroy(pDestroy);
-            AssertRC(rc);
-        }
-    }
-    tmTimerUnlock(pVM);
-
-    LogFlow(("TMR3TimerDestroyDriver: returns VINF_SUCCESS\n"));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Internal function for getting the clock time.
- *
- * @returns clock time.
- * @param   pVM         The VM handle.
- * @param   enmClock    The clock.
- */
-DECLINLINE(uint64_t) tmClock(PVM pVM, TMCLOCK enmClock)
-{
-    switch (enmClock)
-    {
-        case TMCLOCK_VIRTUAL:       return TMVirtualGet(pVM);
-        case TMCLOCK_VIRTUAL_SYNC:  return TMVirtualSyncGet(pVM);
-        case TMCLOCK_REAL:          return TMRealGet(pVM);
-        case TMCLOCK_TSC:           return TMCpuTickGet(&pVM->aCpus[0] /* just take VCPU 0 */);
-        default:
-            AssertMsgFailed(("enmClock=%d\n", enmClock));
-            return ~(uint64_t)0;
-    }
-}
-
-
-/**
- * Checks if the sync queue has one or more expired timers.
- *
- * @returns true / false.
- *
- * @param   pVM         The VM handle.
- * @param   enmClock    The queue.
- */
-DECLINLINE(bool) tmR3HasExpiredTimer(PVM pVM, TMCLOCK enmClock)
-{
-    const uint64_t u64Expire = pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock].u64Expire;
-    return u64Expire != INT64_MAX && u64Expire <= tmClock(pVM, enmClock);
-}
-
-
-/**
- * Checks for expired timers in all the queues.
- *
- * @returns true / false.
- * @param   pVM         The VM handle.
- */
-DECLINLINE(bool) tmR3AnyExpiredTimers(PVM pVM)
-{
-    /*
-     * Combine the time calculation for the first two since we're not on EMT
-     * TMVirtualSyncGet only permits EMT.
-     */
-    uint64_t u64Now = TMVirtualGetNoCheck(pVM);
-    if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64Now)
-        return true;
-    u64Now = pVM->tm.s.fVirtualSyncTicking
-           ? u64Now - pVM->tm.s.offVirtualSync
-           : pVM->tm.s.u64VirtualSync;
-    if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64Now)
-        return true;
-
-    /*
-     * The remaining timers.
-     */
-    if (tmR3HasExpiredTimer(pVM, TMCLOCK_REAL))
-        return true;
-    if (tmR3HasExpiredTimer(pVM, TMCLOCK_TSC))
-        return true;
-    return false;
-}
-
-
-/**
- * Schedule timer callback.
- *
- * @param   pTimer      Timer handle.
- * @param   pvUser      VM handle.
- * @thread  Timer thread.
- *
- * @remark  We cannot do the scheduling and queues running from a timer handler
- *          since it's not executing in EMT, and even if it was it would be async
- *          and we wouldn't know the state of the affairs.
- *          So, we'll just raise the timer FF and force any REM execution to exit.
- */
-static DECLCALLBACK(void) tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t /*iTick*/)
-{
-    PVM     pVM      = (PVM)pvUser;
-    PVMCPU  pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
-
-    AssertCompile(TMCLOCK_MAX == 4);
-#ifdef DEBUG_Sander /* very annoying, keep it private. */
-    if (VMCPU_FF_ISSET(pVCpuDst, VMCPU_FF_TIMER))
-        Log(("tmR3TimerCallback: timer event still pending!!\n"));
-#endif
-    if (    !VMCPU_FF_ISSET(pVCpuDst, VMCPU_FF_TIMER)
-        &&  (   pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule /** @todo FIXME - reconsider offSchedule as a reason for running the timer queues. */
-            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule
-            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule
-            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offSchedule
-            ||  tmR3AnyExpiredTimers(pVM)
-            )
-        && !VMCPU_FF_ISSET(pVCpuDst, VMCPU_FF_TIMER)
-        && !pVM->tm.s.fRunningQueues
-       )
-    {
-        Log5(("TM(%u): FF: 0 -> 1\n", __LINE__));
-        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
-        REMR3NotifyTimerPending(pVM, pVCpuDst);
-        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM /** @todo | VMNOTIFYFF_FLAGS_POKE ?*/);
-        STAM_COUNTER_INC(&pVM->tm.s.StatTimerCallbackSetFF);
-    }
-}
-
-
-/**
- * Schedules and runs any pending timers.
- *
- * This is normally called from a forced action handler in EMT.
- *
- * @param   pVM             The VM to run the timers for.
- *
- * @thread  EMT (actually EMT0, but we fend off the others)
- */
-VMMR3DECL(void) TMR3TimerQueuesDo(PVM pVM)
-{
-    /*
-     * Only the dedicated timer EMT should do stuff here.
-     * (fRunningQueues is only used as an indicator.)
-     */
-    Assert(pVM->tm.s.idTimerCpu < pVM->cCpus);
-    PVMCPU pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
-    if (VMMGetCpu(pVM) != pVCpuDst)
-    {
-        Assert(pVM->cCpus > 1);
-        return;
-    }
-    STAM_PROFILE_START(&pVM->tm.s.StatDoQueues, a);
-    Log2(("TMR3TimerQueuesDo:\n"));
-    Assert(!pVM->tm.s.fRunningQueues);
-    ASMAtomicWriteBool(&pVM->tm.s.fRunningQueues, true);
-    tmTimerLock(pVM);
-
-    /*
-     * Process the queues.
-     */
-    AssertCompile(TMCLOCK_MAX == 4);
-
-    /* TMCLOCK_VIRTUAL_SYNC (see also TMR3VirtualSyncFF) */
-    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], s1);
-    tmVirtualSyncLock(pVM);
-    ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, true);
-    VMCPU_FF_CLEAR(pVCpuDst, VMCPU_FF_TIMER);   /* Clear the FF once we started working for real. */
-
-    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule)
-        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC]);
-    tmR3TimerQueueRunVirtualSync(pVM);
-    if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */
-        VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
-
-    ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, false);
-    tmVirtualSyncUnlock(pVM);
-    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], s1);
-
-    /* TMCLOCK_VIRTUAL */
-    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2);
-    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule)
-        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]);
-    tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]);
-    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2);
-
-    /* TMCLOCK_TSC */
-    Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offActive); /* not used */
-
-    /* TMCLOCK_REAL */
-    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3);
-    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule)
-        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]);
-    tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]);
-    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3);
-
-#ifdef VBOX_STRICT
-    /* check that we didn't screw up. */
-    tmTimerQueuesSanityChecks(pVM, "TMR3TimerQueuesDo");
-#endif
-
-    /* done */
-    Log2(("TMR3TimerQueuesDo: returns void\n"));
-    ASMAtomicWriteBool(&pVM->tm.s.fRunningQueues, false);
-    tmTimerUnlock(pVM);
-    STAM_PROFILE_STOP(&pVM->tm.s.StatDoQueues, a);
-}
-
-//RT_C_DECLS_BEGIN
-//int     iomLock(PVM pVM);
-//void    iomUnlock(PVM pVM);
-//RT_C_DECLS_END
-
-
-/**
- * Schedules and runs any pending times in the specified queue.
- *
- * This is normally called from a forced action handler in EMT.
- *
- * @param   pVM             The VM to run the timers for.
- * @param   pQueue          The queue to run.
- */
-static void tmR3TimerQueueRun(PVM pVM, PTMTIMERQUEUE pQueue)
-{
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Run timers.
-     *
-     * We check the clock once and run all timers which are ACTIVE
-     * and have an expire time less or equal to the time we read.
-     *
-     * N.B. A generic unlink must be applied since other threads
-     *      are allowed to mess with any active timer at any time.
-     *      However, we only allow EMT to handle EXPIRED_PENDING
-     *      timers, thus enabling the timer handler function to
-     *      arm the timer again.
-     */
-    PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue);
-    if (!pNext)
-        return;
-    const uint64_t u64Now = tmClock(pVM, pQueue->enmClock);
-    while (pNext && pNext->u64Expire <= u64Now)
-    {
-        PTMTIMER        pTimer    = pNext;
-        pNext = TMTIMER_GET_NEXT(pTimer);
-        PPDMCRITSECT    pCritSect = pTimer->pCritSect;
-        if (pCritSect)
-            PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
-        Log2(("tmR3TimerQueueRun: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, u64Expire=%llx (now=%llx) .pszDesc=%s}\n",
-              pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, pTimer->u64Expire, u64Now, pTimer->pszDesc));
-        bool fRc;
-        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_GET_UNLINK, TMTIMERSTATE_ACTIVE, fRc);
-        if (fRc)
-        {
-            Assert(!pTimer->offScheduleNext); /* this can trigger falsely */
-
-            /* unlink */
-            const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
-            if (pPrev)
-                TMTIMER_SET_NEXT(pPrev, pNext);
-            else
-            {
-                TMTIMER_SET_HEAD(pQueue, pNext);
-                pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
-            }
-            if (pNext)
-                TMTIMER_SET_PREV(pNext, pPrev);
-            pTimer->offNext = 0;
-            pTimer->offPrev = 0;
-
-            /* fire */
-            TM_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_DELIVER);
-            switch (pTimer->enmType)
-            {
-                case TMTIMERTYPE_DEV:       pTimer->u.Dev.pfnTimer(pTimer->u.Dev.pDevIns, pTimer, pTimer->pvUser); break;
-                case TMTIMERTYPE_DRV:       pTimer->u.Drv.pfnTimer(pTimer->u.Drv.pDrvIns, pTimer, pTimer->pvUser); break;
-                case TMTIMERTYPE_INTERNAL:  pTimer->u.Internal.pfnTimer(pVM, pTimer, pTimer->pvUser); break;
-                case TMTIMERTYPE_EXTERNAL:  pTimer->u.External.pfnTimer(pTimer->pvUser); break;
-                default:
-                    AssertMsgFailed(("Invalid timer type %d (%s)\n", pTimer->enmType, pTimer->pszDesc));
-                    break;
-            }
-
-            /* change the state if it wasn't changed already in the handler. */
-            TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_EXPIRED_DELIVER, fRc);
-            Log2(("tmR3TimerQueueRun: new state %s\n", tmTimerState(pTimer->enmState)));
-        }
-        if (pCritSect)
-            PDMCritSectLeave(pCritSect);
-    } /* run loop */
-}
-
-
-/**
- * Schedules and runs any pending times in the timer queue for the
- * synchronous virtual clock.
- *
- * This scheduling is a bit different from the other queues as it need
- * to implement the special requirements of the timer synchronous virtual
- * clock, thus this 2nd queue run function.
- *
- * @param   pVM             The VM to run the timers for.
- *
- * @remarks The caller must own both the TM/EMT and the Virtual Sync locks.
- */
-static void tmR3TimerQueueRunVirtualSync(PVM pVM)
-{
-    PTMTIMERQUEUE const pQueue = &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC];
-    VM_ASSERT_EMT(pVM);
-
-    /*
-     * Any timers?
-     */
-    PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue);
-    if (RT_UNLIKELY(!pNext))
-    {
-        Assert(pVM->tm.s.fVirtualSyncTicking || !pVM->tm.s.cVirtualTicking);
-        return;
-    }
-    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRun);
-
-    /*
-     * Calculate the time frame for which we will dispatch timers.
-     *
-     * We use a time frame ranging from the current sync time (which is most likely the
-     * same as the head timer) and some configurable period (100000ns) up towards the
-     * current virtual time. This period might also need to be restricted by the catch-up
-     * rate so frequent calls to this function won't accelerate the time too much, however
-     * this will be implemented at a later point if necessary.
-     *
-     * Without this frame we would 1) having to run timers much more frequently
-     * and 2) lag behind at a steady rate.
-     */
-    const uint64_t  u64VirtualNow  = TMVirtualGetNoCheck(pVM);
-    uint64_t const  offSyncGivenUp = pVM->tm.s.offVirtualSyncGivenUp;
-    uint64_t        u64Now;
-    if (!pVM->tm.s.fVirtualSyncTicking)
-    {
-        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunStoppedAlready);
-        u64Now = pVM->tm.s.u64VirtualSync;
-#ifdef DEBUG_bird
-        Assert(u64Now <= pNext->u64Expire);
-#endif
-    }
-    else
-    {
-        /* Calc 'now'. */
-        bool        fStopCatchup   = false;
-        bool        fUpdateStuff   = false;
-        uint64_t    off            = pVM->tm.s.offVirtualSync;
-        if (pVM->tm.s.fVirtualSyncCatchUp)
-        {
-            uint64_t u64Delta = u64VirtualNow - pVM->tm.s.u64VirtualSyncCatchUpPrev;
-            if (RT_LIKELY(!(u64Delta >> 32)))
-            {
-                uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, pVM->tm.s.u32VirtualSyncCatchUpPercentage, 100);
-                if (off > u64Sub + offSyncGivenUp)
-                {
-                    off -= u64Sub;
-                    Log4(("TM: %'RU64/-%'8RU64: sub %'RU64 [tmR3TimerQueueRunVirtualSync]\n", u64VirtualNow - off, off - offSyncGivenUp, u64Sub));
-                }
-                else
-                {
-                    STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
-                    fStopCatchup = true;
-                    off = offSyncGivenUp;
-                }
-                fUpdateStuff = true;
-            }
-        }
-        u64Now = u64VirtualNow - off;
-
-        /* Check if stopped by expired timer. */
-        uint64_t u64Expire = pNext->u64Expire;
-        if (u64Now >= pNext->u64Expire)
-        {
-            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunStop);
-            u64Now = pNext->u64Expire;
-            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64Now);
-            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
-            Log4(("TM: %'RU64/-%'8RU64: exp tmr [tmR3TimerQueueRunVirtualSync]\n", u64Now, u64VirtualNow - u64Now - offSyncGivenUp));
-        }
-        else if (fUpdateStuff)
-        {
-            ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, off);
-            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev, u64VirtualNow);
-            if (fStopCatchup)
-            {
-                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
-                Log4(("TM: %'RU64/0: caught up [tmR3TimerQueueRunVirtualSync]\n", u64VirtualNow));
-            }
-        }
-    }
-
-    /* calc end of frame. */
-    uint64_t u64Max = u64Now + pVM->tm.s.u32VirtualSyncScheduleSlack;
-    if (u64Max > u64VirtualNow - offSyncGivenUp)
-        u64Max = u64VirtualNow - offSyncGivenUp;
-
-    /* assert sanity */
-#ifdef DEBUG_bird
-    Assert(u64Now <= u64VirtualNow - offSyncGivenUp);
-    Assert(u64Max <= u64VirtualNow - offSyncGivenUp);
-    Assert(u64Now <= u64Max);
-    Assert(offSyncGivenUp == pVM->tm.s.offVirtualSyncGivenUp);
-#endif
-
-    /*
-     * Process the expired timers moving the clock along as we progress.
-     */
-#ifdef DEBUG_bird
-#ifdef VBOX_STRICT
-    uint64_t u64Prev = u64Now; NOREF(u64Prev);
-#endif
-#endif
-    while (pNext && pNext->u64Expire <= u64Max)
-    {
-        PTMTIMER        pTimer    = pNext;
-        pNext = TMTIMER_GET_NEXT(pTimer);
-        PPDMCRITSECT    pCritSect = pTimer->pCritSect;
-        if (pCritSect)
-            PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
-        Log2(("tmR3TimerQueueRun: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, u64Expire=%llx (now=%llx) .pszDesc=%s}\n",
-              pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, pTimer->u64Expire, u64Now, pTimer->pszDesc));
-        bool fRc;
-        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_GET_UNLINK, TMTIMERSTATE_ACTIVE, fRc);
-        if (fRc)
-        {
-            /* unlink */
-            const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
-            if (pPrev)
-                TMTIMER_SET_NEXT(pPrev, pNext);
-            else
-            {
-                TMTIMER_SET_HEAD(pQueue, pNext);
-                pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
-            }
-            if (pNext)
-                TMTIMER_SET_PREV(pNext, pPrev);
-            pTimer->offNext = 0;
-            pTimer->offPrev = 0;
-
-            /* advance the clock - don't permit timers to be out of order or armed in the 'past'. */
-#ifdef DEBUG_bird
-#ifdef VBOX_STRICT
-            AssertMsg(pTimer->u64Expire >= u64Prev, ("%'RU64 < %'RU64 %s\n", pTimer->u64Expire, u64Prev, pTimer->pszDesc));
-            u64Prev = pTimer->u64Expire;
-#endif
-#endif
-            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, pTimer->u64Expire);
-            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
-
-            /* fire */
-            TM_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_DELIVER);
-            switch (pTimer->enmType)
-            {
-                case TMTIMERTYPE_DEV:       pTimer->u.Dev.pfnTimer(pTimer->u.Dev.pDevIns, pTimer, pTimer->pvUser); break;
-                case TMTIMERTYPE_DRV:       pTimer->u.Drv.pfnTimer(pTimer->u.Drv.pDrvIns, pTimer, pTimer->pvUser); break;
-                case TMTIMERTYPE_INTERNAL:  pTimer->u.Internal.pfnTimer(pVM, pTimer, pTimer->pvUser); break;
-                case TMTIMERTYPE_EXTERNAL:  pTimer->u.External.pfnTimer(pTimer->pvUser); break;
-                default:
-                    AssertMsgFailed(("Invalid timer type %d (%s)\n", pTimer->enmType, pTimer->pszDesc));
-                    break;
-            }
-
-            /* Change the state if it wasn't changed already in the handler.
-               Reset the Hz hint too since this is the same as TMTimerStop. */
-            TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_EXPIRED_DELIVER, fRc);
-            if (fRc && pTimer->uHzHint)
-            {
-                if (pTimer->uHzHint >= pVM->tm.s.uMaxHzHint)
-                    ASMAtomicWriteBool(&pVM->tm.s.fHzHintNeedsUpdating, true);
-                pTimer->uHzHint = 0;
-            }
-            Log2(("tmR3TimerQueueRun: new state %s\n", tmTimerState(pTimer->enmState)));
-        }
-        if (pCritSect)
-            PDMCritSectLeave(pCritSect);
-    } /* run loop */
-
-    /*
-     * Restart the clock if it was stopped to serve any timers,
-     * and start/adjust catch-up if necessary.
-     */
-    if (    !pVM->tm.s.fVirtualSyncTicking
-        &&  pVM->tm.s.cVirtualTicking)
-    {
-        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunRestart);
-
-        /* calc the slack we've handed out. */
-        const uint64_t u64VirtualNow2 = TMVirtualGetNoCheck(pVM);
-        Assert(u64VirtualNow2 >= u64VirtualNow);
-#ifdef DEBUG_bird
-        AssertMsg(pVM->tm.s.u64VirtualSync >= u64Now, ("%'RU64 < %'RU64\n", pVM->tm.s.u64VirtualSync, u64Now));
-#endif
-        const uint64_t offSlack = pVM->tm.s.u64VirtualSync - u64Now;
-        STAM_STATS({
-            if (offSlack)
-            {
-                PSTAMPROFILE p = &pVM->tm.s.StatVirtualSyncRunSlack;
-                p->cPeriods++;
-                p->cTicks += offSlack;
-                if (p->cTicksMax < offSlack) p->cTicksMax = offSlack;
-                if (p->cTicksMin > offSlack) p->cTicksMin = offSlack;
-            }
-        });
-
-        /* Let the time run a little bit while we were busy running timers(?). */
-        uint64_t u64Elapsed;
-#define MAX_ELAPSED 30000U /* ns */
-        if (offSlack > MAX_ELAPSED)
-            u64Elapsed = 0;
-        else
-        {
-            u64Elapsed = u64VirtualNow2 - u64VirtualNow;
-            if (u64Elapsed > MAX_ELAPSED)
-                u64Elapsed = MAX_ELAPSED;
-            u64Elapsed = u64Elapsed > offSlack ? u64Elapsed - offSlack : 0;
-        }
-#undef MAX_ELAPSED
-
-        /* Calc the current offset. */
-        uint64_t offNew = u64VirtualNow2 - pVM->tm.s.u64VirtualSync - u64Elapsed;
-        Assert(!(offNew & RT_BIT_64(63)));
-        uint64_t offLag = offNew - pVM->tm.s.offVirtualSyncGivenUp;
-        Assert(!(offLag & RT_BIT_64(63)));
-
-        /*
-         * Deal with starting, adjusting and stopping catchup.
-         */
-        if (pVM->tm.s.fVirtualSyncCatchUp)
-        {
-            if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpStopThreshold)
-            {
-                /* stop */
-                STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
-                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
-                Log4(("TM: %'RU64/-%'8RU64: caught up [pt]\n", u64VirtualNow2 - offNew, offLag));
-            }
-            else if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold)
-            {
-                /* adjust */
-                unsigned i = 0;
-                while (     i + 1 < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods)
-                       &&   offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[i + 1].u64Start)
-                    i++;
-                if (pVM->tm.s.u32VirtualSyncCatchUpPercentage < pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage)
-                {
-                    STAM_COUNTER_INC(&pVM->tm.s.aStatVirtualSyncCatchupAdjust[i]);
-                    ASMAtomicWriteU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage, pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage);
-                    Log4(("TM: %'RU64/%'8RU64: adj %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
-                }
-                pVM->tm.s.u64VirtualSyncCatchUpPrev = u64VirtualNow2;
-            }
-            else
-            {
-                /* give up */
-                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGiveUp);
-                STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
-                ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
-                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
-                Log4(("TM: %'RU64/%'8RU64: give up %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
-                LogRel(("TM: Giving up catch-up attempt at a %'RU64 ns lag; new total: %'RU64 ns\n", offLag, offNew));
-            }
-        }
-        else if (offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[0].u64Start)
-        {
-            if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold)
-            {
-                /* start */
-                STAM_PROFILE_ADV_START(&pVM->tm.s.StatVirtualSyncCatchup, c);
-                unsigned i = 0;
-                while (     i + 1 < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods)
-                       &&   offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[i + 1].u64Start)
-                    i++;
-                STAM_COUNTER_INC(&pVM->tm.s.aStatVirtualSyncCatchupInitial[i]);
-                ASMAtomicWriteU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage, pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage);
-                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, true);
-                Log4(("TM: %'RU64/%'8RU64: catch-up %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
-            }
-            else
-            {
-                /* don't bother */
-                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGiveUpBeforeStarting);
-                ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
-                Log4(("TM: %'RU64/%'8RU64: give up\n", u64VirtualNow2 - offNew, offLag));
-                LogRel(("TM: Not bothering to attempt catching up a %'RU64 ns lag; new total: %'RU64\n", offLag, offNew));
-            }
-        }
-
-        /*
-         * Update the offset and restart the clock.
-         */
-        Assert(!(offNew & RT_BIT_64(63)));
-        ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, offNew);
-        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, true);
-    }
-}
-
-
-/**
- * Deals with stopped Virtual Sync clock.
- *
- * This is called by the forced action flag handling code in EM when it
- * encounters the VM_FF_TM_VIRTUAL_SYNC flag. It is called by all VCPUs and they
- * will block on the VirtualSyncLock until the pending timers has been executed
- * and the clock restarted.
- *
- * @param   pVM             The VM to run the timers for.
- * @param   pVCpu           The virtual CPU we're running at.
- *
- * @thread  EMTs
- */
-VMM_INT_DECL(void) TMR3VirtualSyncFF(PVM pVM, PVMCPU pVCpu)
-{
-    Log2(("TMR3VirtualSyncFF:\n"));
-
-    /*
-     * The EMT doing the timers is diverted to them.
-     */
-    if (pVCpu->idCpu == pVM->tm.s.idTimerCpu)
-        TMR3TimerQueuesDo(pVM);
-    /*
-     * The other EMTs will block on the virtual sync lock and the first owner
-     * will run the queue and thus restarting the clock.
-     *
-     * Note! This is very suboptimal code wrt to resuming execution when there
-     *       are more than two Virtual CPUs, since they will all have to enter
-     *       the critical section one by one. But it's a very simple solution
-     *       which will have to do the job for now.
-     */
-    else
-    {
-        STAM_PROFILE_START(&pVM->tm.s.StatVirtualSyncFF, a);
-        tmVirtualSyncLock(pVM);
-        if (pVM->tm.s.fVirtualSyncTicking)
-        {
-            STAM_PROFILE_STOP(&pVM->tm.s.StatVirtualSyncFF, a); /* before the unlock! */
-            tmVirtualSyncUnlock(pVM);
-            Log2(("TMR3VirtualSyncFF: ticking\n"));
-        }
-        else
-        {
-            tmVirtualSyncUnlock(pVM);
-
-            /* try run it. */
-            tmTimerLock(pVM);
-            tmVirtualSyncLock(pVM);
-            if (pVM->tm.s.fVirtualSyncTicking)
-                Log2(("TMR3VirtualSyncFF: ticking (2)\n"));
-            else
-            {
-                ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, true);
-                Log2(("TMR3VirtualSyncFF: running queue\n"));
-
-                if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule)
-                    tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC]);
-                tmR3TimerQueueRunVirtualSync(pVM);
-                if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */
-                    VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
-
-                ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, false);
-            }
-            STAM_PROFILE_STOP(&pVM->tm.s.StatVirtualSyncFF, a); /* before the unlock! */
-            tmVirtualSyncUnlock(pVM);
-            tmTimerUnlock(pVM);
-        }
-    }
-}
-
-
-/** @name Saved state values
- * @{ */
-#define TMTIMERSTATE_SAVED_PENDING_STOP         4
-#define TMTIMERSTATE_SAVED_PENDING_SCHEDULE     7
-/** @}  */
-
-
-/**
- * Saves the state of a timer to a saved state.
- *
- * @returns VBox status.
- * @param   pTimer          Timer to save.
- * @param   pSSM            Save State Manager handle.
- */
-VMMR3DECL(int) TMR3TimerSave(PTMTIMERR3 pTimer, PSSMHANDLE pSSM)
-{
-    LogFlow(("TMR3TimerSave: %p:{enmState=%s, .pszDesc={%s}} pSSM=%p\n", pTimer, tmTimerState(pTimer->enmState), pTimer->pszDesc, pSSM));
-    switch (pTimer->enmState)
-    {
-        case TMTIMERSTATE_STOPPED:
-        case TMTIMERSTATE_PENDING_STOP:
-        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
-            return SSMR3PutU8(pSSM, TMTIMERSTATE_SAVED_PENDING_STOP);
-
-        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
-        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
-            AssertMsgFailed(("u64Expire is being updated! (%s)\n", pTimer->pszDesc));
-            if (!RTThreadYield())
-                RTThreadSleep(1);
-            /* fall thru */
-        case TMTIMERSTATE_ACTIVE:
-        case TMTIMERSTATE_PENDING_SCHEDULE:
-        case TMTIMERSTATE_PENDING_RESCHEDULE:
-            SSMR3PutU8(pSSM, TMTIMERSTATE_SAVED_PENDING_SCHEDULE);
-            return SSMR3PutU64(pSSM, pTimer->u64Expire);
-
-        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
-        case TMTIMERSTATE_EXPIRED_DELIVER:
-        case TMTIMERSTATE_DESTROY:
-        case TMTIMERSTATE_FREE:
-            AssertMsgFailed(("Invalid timer state %d %s (%s)\n", pTimer->enmState, tmTimerState(pTimer->enmState), pTimer->pszDesc));
-            return SSMR3HandleSetStatus(pSSM, VERR_TM_INVALID_STATE);
-    }
-
-    AssertMsgFailed(("Unknown timer state %d (%s)\n", pTimer->enmState, pTimer->pszDesc));
-    return SSMR3HandleSetStatus(pSSM, VERR_TM_UNKNOWN_STATE);
-}
-
-
-/**
- * Loads the state of a timer from a saved state.
- *
- * @returns VBox status.
- * @param   pTimer          Timer to restore.
- * @param   pSSM            Save State Manager handle.
- */
-VMMR3DECL(int) TMR3TimerLoad(PTMTIMERR3 pTimer, PSSMHANDLE pSSM)
-{
-    Assert(pTimer); Assert(pSSM); VM_ASSERT_EMT(pTimer->pVMR3);
-    LogFlow(("TMR3TimerLoad: %p:{enmState=%s, .pszDesc={%s}} pSSM=%p\n", pTimer, tmTimerState(pTimer->enmState), pTimer->pszDesc, pSSM));
-
-    /*
-     * Load the state and validate it.
-     */
-    uint8_t u8State;
-    int rc = SSMR3GetU8(pSSM, &u8State);
-    if (RT_FAILURE(rc))
-        return rc;
-#if 1 /* Workaround for accidental state shift in r47786 (2009-05-26 19:12:12). */  /** @todo remove this in a few weeks! */
-    if (    u8State == TMTIMERSTATE_SAVED_PENDING_STOP + 1
-        ||  u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE + 1)
-        u8State--;
-#endif
-    if (    u8State != TMTIMERSTATE_SAVED_PENDING_STOP
-        &&  u8State != TMTIMERSTATE_SAVED_PENDING_SCHEDULE)
-    {
-        AssertLogRelMsgFailed(("u8State=%d\n", u8State));
-        return SSMR3HandleSetStatus(pSSM, VERR_TM_LOAD_STATE);
-    }
-
-    /* Enter the critical section to make TMTimerSet/Stop happy. */
-    PPDMCRITSECT pCritSect = pTimer->pCritSect;
-    if (pCritSect)
-        PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
-
-    if (u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE)
-    {
-        /*
-         * Load the expire time.
-         */
-        uint64_t u64Expire;
-        rc = SSMR3GetU64(pSSM, &u64Expire);
-        if (RT_FAILURE(rc))
-            return rc;
-
-        /*
-         * Set it.
-         */
-        Log(("u8State=%d u64Expire=%llu\n", u8State, u64Expire));
-        rc = TMTimerSet(pTimer, u64Expire);
-    }
-    else
-    {
-        /*
-         * Stop it.
-         */
-        Log(("u8State=%d\n", u8State));
-        rc = TMTimerStop(pTimer);
-    }
-
-    if (pCritSect)
-        PDMCritSectLeave(pCritSect);
-
-    /*
-     * On failure set SSM status.
-     */
-    if (RT_FAILURE(rc))
-        rc = SSMR3HandleSetStatus(pSSM, rc);
-    return rc;
-}
-
-
-/**
- * Associates a critical section with a timer.
- *
- * The critical section will be entered prior to doing the timer call back, thus
- * avoiding potential races between the timer thread and other threads trying to
- * stop or adjust the timer expiration while it's being delivered. The timer
- * thread will leave the critical section when the timer callback returns.
- *
- * In strict builds, ownership of the critical section will be asserted by
- * TMTimerSet, TMTimerStop, TMTimerGetExpire and TMTimerDestroy (when called at
- * runtime).
- *
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_INVALID_HANDLE if the timer handle is NULL or invalid
- *          (asserted).
- * @retval  VERR_INVALID_PARAMETER if pCritSect is NULL or has an invalid magic
- *          (asserted).
- * @retval  VERR_ALREADY_EXISTS if a critical section was already associated
- *          with the timer (asserted).
- * @retval  VERR_INVALID_STATE if the timer isn't stopped.
- *
- * @param   pTimer          The timer handle.
- * @param   pCritSect       The critical section. The caller must make sure this
- *                          is around for the life time of the timer.
- *
- * @thread  Any, but the caller is responsible for making sure the timer is not
- *          active.
- */
-VMMR3DECL(int) TMR3TimerSetCritSect(PTMTIMERR3 pTimer, PPDMCRITSECT pCritSect)
-{
-    AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
-    AssertPtrReturn(pCritSect, VERR_INVALID_PARAMETER);
-    const char *pszName = PDMR3CritSectName(pCritSect); /* exploited for validation */
-    AssertReturn(pszName, VERR_INVALID_PARAMETER);
-    AssertReturn(!pTimer->pCritSect, VERR_ALREADY_EXISTS);
-    AssertReturn(pTimer->enmState == TMTIMERSTATE_STOPPED, VERR_INVALID_STATE);
-    LogFlow(("pTimer=%p (%s) pCritSect=%p (%s)\n", pTimer, pTimer->pszDesc, pCritSect, pszName));
-
-    pTimer->pCritSect = pCritSect;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Get the real world UTC time adjusted for VM lag.
- *
- * @returns pTime.
- * @param   pVM             The VM instance.
- * @param   pTime           Where to store the time.
- */
-VMM_INT_DECL(PRTTIMESPEC) TMR3UtcNow(PVM pVM, PRTTIMESPEC pTime)
-{
-    RTTimeNow(pTime);
-    RTTimeSpecSubNano(pTime, ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) - ASMAtomicReadU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp));
-    RTTimeSpecAddNano(pTime, pVM->tm.s.offUTC);
-    return pTime;
-}
-
-
-/**
- * Pauses all clocks except TMCLOCK_REAL.
- *
- * @returns VBox status code, all errors are asserted.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The virtual CPU handle.
- * @thread  EMT corresponding to the virtual CPU handle.
- */
-VMMR3DECL(int) TMR3NotifySuspend(PVM pVM, PVMCPU pVCpu)
-{
-    VMCPU_ASSERT_EMT(pVCpu);
-
-    /*
-     * The shared virtual clock (includes virtual sync which is tied to it).
-     */
-    tmTimerLock(pVM);                           /* Paranoia: Exploiting the timer lock here. */
-    int rc = tmVirtualPauseLocked(pVM);
-    tmTimerUnlock(pVM);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Pause the TSC last since it is normally linked to the virtual
-     * sync clock, so the above code may actually stop both clock.
-     */
-    rc = tmCpuTickPause(pVM, pVCpu);
-    if (RT_FAILURE(rc))
-        return rc;
-
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-    /*
-     * Update cNsTotal.
-     */
-    uint32_t uGen = ASMAtomicIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1);
-    pVCpu->tm.s.cNsTotal = RTTimeNanoTS() - pVCpu->tm.s.u64NsTsStartTotal;
-    pVCpu->tm.s.cNsOther = pVCpu->tm.s.cNsTotal - pVCpu->tm.s.cNsExecuting - pVCpu->tm.s.cNsHalted;
-    ASMAtomicWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1);
-#endif
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Resumes all clocks except TMCLOCK_REAL.
- *
- * @returns VBox status code, all errors are asserted.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The virtual CPU handle.
- * @thread  EMT corresponding to the virtual CPU handle.
- */
-VMMR3DECL(int) TMR3NotifyResume(PVM pVM, PVMCPU pVCpu)
-{
-    VMCPU_ASSERT_EMT(pVCpu);
-    int rc;
-
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-    /*
-     * Set u64NsTsStartTotal.  There is no need to back this out if either of
-     * the two calls below fail.
-     */
-    pVCpu->tm.s.u64NsTsStartTotal = RTTimeNanoTS() - pVCpu->tm.s.cNsTotal;
-#endif
-
-    /*
-     * Resume the TSC first since it is normally linked to the virtual sync
-     * clock, so it may actually not be resumed until we've executed the code
-     * below.
-     */
-    if (!pVM->tm.s.fTSCTiedToExecution)
-    {
-        rc = tmCpuTickResume(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * The shared virtual clock (includes virtual sync which is tied to it).
-     */
-    tmTimerLock(pVM);                           /* Paranoia: Exploiting the timer lock here. */
-    rc = tmVirtualResumeLocked(pVM);
-    tmTimerUnlock(pVM);
-
-    return rc;
-}
-
-
-/**
- * Sets the warp drive percent of the virtual time.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   u32Percent  The new percentage. 100 means normal operation.
- */
-VMMDECL(int) TMR3SetWarpDrive(PVM pVM, uint32_t u32Percent)
-{
-    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)tmR3SetWarpDrive, 2, pVM, u32Percent);
-}
-
-
-/**
- * EMT worker for TMR3SetWarpDrive.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   u32Percent  See TMR3SetWarpDrive().
- * @internal
- */
-static DECLCALLBACK(int) tmR3SetWarpDrive(PVM pVM, uint32_t u32Percent)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-
-    /*
-     * Validate it.
-     */
-    AssertMsgReturn(u32Percent >= 2 && u32Percent <= 20000,
-                    ("%RX32 is not between 2 and 20000 (inclusive).\n", u32Percent),
-                    VERR_INVALID_PARAMETER);
-
-/** @todo This isn't a feature specific to virtual time, move the variables to
- * TM level and make it affect TMR3UTCNow as well! */
-
-    /*
-     * If the time is running we'll have to pause it before we can change
-     * the warp drive settings.
-     */
-    tmTimerLock(pVM);                           /* Paranoia: Exploiting the timer lock here. */
-    bool fPaused = !!pVM->tm.s.cVirtualTicking;
-    if (fPaused) /** @todo this isn't really working, but wtf. */
-        TMR3NotifySuspend(pVM, pVCpu);
-
-    pVM->tm.s.u32VirtualWarpDrivePercentage = u32Percent;
-    pVM->tm.s.fVirtualWarpDrive = u32Percent != 100;
-    LogRel(("TM: u32VirtualWarpDrivePercentage=%RI32 fVirtualWarpDrive=%RTbool\n",
-            pVM->tm.s.u32VirtualWarpDrivePercentage, pVM->tm.s.fVirtualWarpDrive));
-
-    if (fPaused)
-        TMR3NotifyResume(pVM, pVCpu);
-    tmTimerUnlock(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets the performance information for one virtual CPU as seen by the VMM.
- *
- * The returned times covers the period where the VM is running and will be
- * reset when restoring a previous VM state (at least for the time being).
- *
- * @retval  VINF_SUCCESS on success.
- * @retval  VERR_NOT_IMPLEMENTED if not compiled in.
- * @retval  VERR_INVALID_STATE if the VM handle is bad.
- * @retval  VERR_INVALID_PARAMETER if idCpu is out of range.
- *
- * @param   pVM             The VM handle.
- * @param   idCpu           The ID of the virtual CPU which times to get.
- * @param   pcNsTotal       Where to store the total run time (nano seconds) of
- *                          the CPU, i.e. the sum of the three other returns.
- *                          Optional.
- * @param   pcNsExecuting   Where to store the time (nano seconds) spent
- *                          executing guest code.  Optional.
- * @param   pcNsHalted      Where to store the time (nano seconds) spent
- *                          halted.  Optional
- * @param   pcNsOther       Where to store the time (nano seconds) spent
- *                          preempted by the host scheduler, on virtualization
- *                          overhead and on other tasks.
- */
-VMMR3DECL(int) TMR3GetCpuLoadTimes(PVM pVM, VMCPUID idCpu, uint64_t *pcNsTotal, uint64_t *pcNsExecuting,
-                                   uint64_t *pcNsHalted, uint64_t *pcNsOther)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
-
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-    /*
-     * Get a stable result set.
-     * This should be way quicker than an EMT request.
-     */
-    PVMCPU      pVCpu        = &pVM->aCpus[idCpu];
-    uint32_t    uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
-    uint64_t    cNsTotal     = pVCpu->tm.s.cNsTotal;
-    uint64_t    cNsExecuting = pVCpu->tm.s.cNsExecuting;
-    uint64_t    cNsHalted    = pVCpu->tm.s.cNsHalted;
-    uint64_t    cNsOther     = pVCpu->tm.s.cNsOther;
-    while (   (uTimesGen & 1) /* update in progress */
-           || uTimesGen != ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen))
-    {
-        RTThreadYield();
-        uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
-        cNsTotal     = pVCpu->tm.s.cNsTotal;
-        cNsExecuting = pVCpu->tm.s.cNsExecuting;
-        cNsHalted    = pVCpu->tm.s.cNsHalted;
-        cNsOther     = pVCpu->tm.s.cNsOther;
-    }
-
-    /*
-     * Fill in the return values.
-     */
-    if (pcNsTotal)
-        *pcNsTotal = cNsTotal;
-    if (pcNsExecuting)
-        *pcNsExecuting = cNsExecuting;
-    if (pcNsHalted)
-        *pcNsHalted = cNsHalted;
-    if (pcNsOther)
-        *pcNsOther = cNsOther;
-
-    return VINF_SUCCESS;
-
-#else
-    return VERR_NOT_IMPLEMENTED;
-#endif
-}
-
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-
-/**
- * Helper for tmR3CpuLoadTimer.
- * @returns
- * @param   pState              The state to update.
- * @param   cNsTotalDelta       Total time.
- * @param   cNsExecutingDelta   Time executing.
- * @param   cNsHaltedDelta      Time halted.
- */
-DECLINLINE(void) tmR3CpuLoadTimerMakeUpdate(PTMCPULOADSTATE pState,
-                                            uint64_t cNsTotal,
-                                            uint64_t cNsExecuting,
-                                            uint64_t cNsHalted)
-{
-    /* Calc deltas */
-    uint64_t cNsTotalDelta      = cNsTotal     - pState->cNsPrevTotal;
-    pState->cNsPrevTotal        = cNsTotal;
-
-    uint64_t cNsExecutingDelta  = cNsExecuting - pState->cNsPrevExecuting;
-    pState->cNsPrevExecuting    = cNsExecuting;
-
-    uint64_t cNsHaltedDelta     = cNsHalted    - pState->cNsPrevHalted;
-    pState->cNsPrevHalted       = cNsHalted;
-
-    /* Calc pcts. */
-    if (!cNsTotalDelta)
-    {
-        pState->cPctExecuting   = 0;
-        pState->cPctHalted      = 100;
-        pState->cPctOther       = 0;
-    }
-    else if (cNsTotalDelta < UINT64_MAX / 4)
-    {
-        pState->cPctExecuting   = (uint8_t)(cNsExecutingDelta    * 100 / cNsTotalDelta);
-        pState->cPctHalted      = (uint8_t)(cNsHaltedDelta       * 100 / cNsTotalDelta);
-        pState->cPctOther       = (uint8_t)((cNsTotalDelta - cNsExecutingDelta - cNsHaltedDelta) * 100 / cNsTotalDelta);
-    }
-    else
-    {
-        pState->cPctExecuting   = 0;
-        pState->cPctHalted      = 100;
-        pState->cPctOther       = 0;
-    }
-}
-
-
-/**
- * Timer callback that calculates the CPU load since the last time it was
- * called.
- *
- * @param   pVM                 The VM handle.
- * @param   pTimer              The timer.
- * @param   pvUser              NULL, unused.
- */
-static DECLCALLBACK(void) tmR3CpuLoadTimer(PVM pVM, PTMTIMER pTimer, void *pvUser)
-{
-    /*
-     * Re-arm the timer first.
-     */
-    int rc = TMTimerSetMillies(pTimer, 1000);
-    AssertLogRelRC(rc);
-    NOREF(pvUser);
-
-    /*
-     * Update the values for each CPU.
-     */
-    uint64_t cNsTotalAll       = 0;
-    uint64_t cNsExecutingAll   = 0;
-    uint64_t cNsHaltedAll      = 0;
-    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
-    {
-        PVMCPU      pVCpu = &pVM->aCpus[iCpu];
-
-        /* Try get a stable data set. */
-        uint32_t    cTries       = 3;
-        uint32_t    uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
-        uint64_t    cNsTotal     = pVCpu->tm.s.cNsTotal;
-        uint64_t    cNsExecuting = pVCpu->tm.s.cNsExecuting;
-        uint64_t    cNsHalted    = pVCpu->tm.s.cNsHalted;
-        while (RT_UNLIKELY(   (uTimesGen & 1) /* update in progress */
-                           || uTimesGen != ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen)))
-        {
-            if (!--cTries)
-                break;
-            ASMNopPause();
-            uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
-            cNsTotal     = pVCpu->tm.s.cNsTotal;
-            cNsExecuting = pVCpu->tm.s.cNsExecuting;
-            cNsHalted    = pVCpu->tm.s.cNsHalted;
-        }
-
-        /* Totals */
-        cNsTotalAll     += cNsTotal;
-        cNsExecutingAll += cNsExecuting;
-        cNsHaltedAll    += cNsHalted;
-
-        /* Calc the PCTs and update the state. */
-        tmR3CpuLoadTimerMakeUpdate(&pVCpu->tm.s.CpuLoad, cNsTotal, cNsExecuting, cNsHalted);
-    }
-
-    /*
-     * Update the value for all the CPUs.
-     */
-    tmR3CpuLoadTimerMakeUpdate(&pVM->tm.s.CpuLoad, cNsTotalAll, cNsExecutingAll, cNsHaltedAll);
-
-    /** @todo Try add 1, 5 and 15 min load stats. */
-
-}
-
-#endif /* !VBOX_WITHOUT_NS_ACCOUNTING */
-
-/**
- * Gets the 5 char clock name for the info tables.
- *
- * @returns The name.
- * @param   enmClock        The clock.
- */
-DECLINLINE(const char *) tmR3Get5CharClockName(TMCLOCK enmClock)
-{
-    switch (enmClock)
-    {
-        case TMCLOCK_REAL:          return "Real ";
-        case TMCLOCK_VIRTUAL:       return "Virt ";
-        case TMCLOCK_VIRTUAL_SYNC:  return "VrSy ";
-        case TMCLOCK_TSC:           return "TSC  ";
-        default:                    return "Bad  ";
-    }
-}
-
-
-/**
- * Display all timers.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) tmR3TimerInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-    pHlp->pfnPrintf(pHlp,
-                    "Timers (pVM=%p)\n"
-                    "%.*s %.*s %.*s %.*s Clock %18s %18s %6s %-25s Description\n",
-                    pVM,
-                    sizeof(RTR3PTR) * 2,        "pTimerR3        ",
-                    sizeof(int32_t) * 2,        "offNext         ",
-                    sizeof(int32_t) * 2,        "offPrev         ",
-                    sizeof(int32_t) * 2,        "offSched        ",
-                                                "Time",
-                                                "Expire",
-                                                "HzHint",
-                                                "State");
-    tmTimerLock(pVM);
-    for (PTMTIMERR3 pTimer = pVM->tm.s.pCreated; pTimer; pTimer = pTimer->pBigNext)
-    {
-        pHlp->pfnPrintf(pHlp,
-                        "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n",
-                        pTimer,
-                        pTimer->offNext,
-                        pTimer->offPrev,
-                        pTimer->offScheduleNext,
-                        tmR3Get5CharClockName(pTimer->enmClock),
-                        TMTimerGet(pTimer),
-                        pTimer->u64Expire,
-                        pTimer->uHzHint,
-                        tmTimerState(pTimer->enmState),
-                        pTimer->pszDesc);
-    }
-    tmTimerUnlock(pVM);
-}
-
-
-/**
- * Display all active timers.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) tmR3TimerInfoActive(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-    pHlp->pfnPrintf(pHlp,
-                    "Active Timers (pVM=%p)\n"
-                    "%.*s %.*s %.*s %.*s Clock %18s %18s %6s %-25s Description\n",
-                    pVM,
-                    sizeof(RTR3PTR) * 2,        "pTimerR3        ",
-                    sizeof(int32_t) * 2,        "offNext         ",
-                    sizeof(int32_t) * 2,        "offPrev         ",
-                    sizeof(int32_t) * 2,        "offSched        ",
-                                                "Time",
-                                                "Expire",
-                                                "HzHint",
-                                                "State");
-    for (unsigned iQueue = 0; iQueue < TMCLOCK_MAX; iQueue++)
-    {
-        tmTimerLock(pVM);
-        for (PTMTIMERR3 pTimer = TMTIMER_GET_HEAD(&pVM->tm.s.paTimerQueuesR3[iQueue]);
-             pTimer;
-             pTimer = TMTIMER_GET_NEXT(pTimer))
-        {
-            pHlp->pfnPrintf(pHlp,
-                            "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n",
-                            pTimer,
-                            pTimer->offNext,
-                            pTimer->offPrev,
-                            pTimer->offScheduleNext,
-                            tmR3Get5CharClockName(pTimer->enmClock),
-                            TMTimerGet(pTimer),
-                            pTimer->u64Expire,
-                            pTimer->uHzHint,
-                            tmTimerState(pTimer->enmState),
-                            pTimer->pszDesc);
-        }
-        tmTimerUnlock(pVM);
-    }
-}
-
-
-/**
- * Display all clocks.
- *
- * @param   pVM         VM Handle.
- * @param   pHlp        The info helpers.
- * @param   pszArgs     Arguments, ignored.
- */
-static DECLCALLBACK(void) tmR3InfoClocks(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    NOREF(pszArgs);
-
-    /*
-     * Read the times first to avoid more than necessary time variation.
-     */
-    const uint64_t u64Virtual     = TMVirtualGet(pVM);
-    const uint64_t u64VirtualSync = TMVirtualSyncGet(pVM);
-    const uint64_t u64Real        = TMRealGet(pVM);
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU   pVCpu  = &pVM->aCpus[i];
-        uint64_t u64TSC = TMCpuTickGet(pVCpu);
-
-        /*
-         * TSC
-         */
-        pHlp->pfnPrintf(pHlp,
-                        "Cpu Tick: %18RU64 (%#016RX64) %RU64Hz %s%s",
-                        u64TSC, u64TSC, TMCpuTicksPerSecond(pVM),
-                        pVCpu->tm.s.fTSCTicking ? "ticking" : "paused",
-                        pVM->tm.s.fTSCVirtualized ? " - virtualized" : "");
-        if (pVM->tm.s.fTSCUseRealTSC)
-        {
-            pHlp->pfnPrintf(pHlp, " - real tsc");
-            if (pVCpu->tm.s.offTSCRawSrc)
-                pHlp->pfnPrintf(pHlp, "\n          offset %RU64", pVCpu->tm.s.offTSCRawSrc);
-        }
-        else
-            pHlp->pfnPrintf(pHlp, " - virtual clock");
-        pHlp->pfnPrintf(pHlp, "\n");
-    }
-
-    /*
-     * virtual
-     */
-    pHlp->pfnPrintf(pHlp,
-                    " Virtual: %18RU64 (%#016RX64) %RU64Hz %s",
-                    u64Virtual, u64Virtual, TMVirtualGetFreq(pVM),
-                    pVM->tm.s.cVirtualTicking ? "ticking" : "paused");
-    if (pVM->tm.s.fVirtualWarpDrive)
-        pHlp->pfnPrintf(pHlp, " WarpDrive %RU32 %%", pVM->tm.s.u32VirtualWarpDrivePercentage);
-    pHlp->pfnPrintf(pHlp, "\n");
-
-    /*
-     * virtual sync
-     */
-    pHlp->pfnPrintf(pHlp,
-                    "VirtSync: %18RU64 (%#016RX64) %s%s",
-                    u64VirtualSync, u64VirtualSync,
-                    pVM->tm.s.fVirtualSyncTicking ? "ticking" : "paused",
-                    pVM->tm.s.fVirtualSyncCatchUp ? " - catchup" : "");
-    if (pVM->tm.s.offVirtualSync)
-    {
-        pHlp->pfnPrintf(pHlp, "\n          offset %RU64", pVM->tm.s.offVirtualSync);
-        if (pVM->tm.s.u32VirtualSyncCatchUpPercentage)
-            pHlp->pfnPrintf(pHlp, "  catch-up rate %u %%", pVM->tm.s.u32VirtualSyncCatchUpPercentage);
-    }
-    pHlp->pfnPrintf(pHlp, "\n");
-
-    /*
-     * real
-     */
-    pHlp->pfnPrintf(pHlp,
-                    "    Real: %18RU64 (%#016RX64) %RU64Hz\n",
-                    u64Real, u64Real, TMRealGetFreq(pVM));
-}
-
Index: unk/src/VBox/VMM/TMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/TMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,726 +1,0 @@
-/* $Id$ */
-/** @file
- * TM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___TMInternal_h
-#define ___TMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <iprt/time.h>
-#include <iprt/timer.h>
-#include <iprt/assert.h>
-#include <VBox/stam.h>
-#include <VBox/pdmcritsect.h>
-
-RT_C_DECLS_BEGIN
-
-
-/** @defgroup grp_tm_int       Internal
- * @ingroup grp_tm
- * @internal
- * @{
- */
-
-/** Frequency of the real clock. */
-#define TMCLOCK_FREQ_REAL       UINT32_C(1000)
-/** Frequency of the virtual clock. */
-#define TMCLOCK_FREQ_VIRTUAL    UINT32_C(1000000000)
-
-
-/**
- * Timer type.
- */
-typedef enum TMTIMERTYPE
-{
-    /** Device timer. */
-    TMTIMERTYPE_DEV = 1,
-    /** Driver timer. */
-    TMTIMERTYPE_DRV,
-    /** Internal timer . */
-    TMTIMERTYPE_INTERNAL,
-    /** External timer. */
-    TMTIMERTYPE_EXTERNAL
-} TMTIMERTYPE;
-
-/**
- * Timer state
- */
-typedef enum TMTIMERSTATE
-{
-    /** Timer is stopped. */
-    TMTIMERSTATE_STOPPED = 1,
-    /** Timer is active. */
-    TMTIMERSTATE_ACTIVE,
-    /** Timer is expired, getting expire and unlinking. */
-    TMTIMERSTATE_EXPIRED_GET_UNLINK,
-    /** Timer is expired and is being delivered. */
-    TMTIMERSTATE_EXPIRED_DELIVER,
-
-    /** Timer is stopped but still in the active list.
-     * Currently in the ScheduleTimers list. */
-    TMTIMERSTATE_PENDING_STOP,
-    /** Timer is stopped but needs unlinking from the ScheduleTimers list.
-     * Currently in the ScheduleTimers list. */
-    TMTIMERSTATE_PENDING_STOP_SCHEDULE,
-    /** Timer is being modified and will soon be pending scheduling.
-     * Currently in the ScheduleTimers list. */
-    TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE,
-    /** Timer is pending scheduling.
-     * Currently in the ScheduleTimers list. */
-    TMTIMERSTATE_PENDING_SCHEDULE,
-    /** Timer is being modified and will soon be pending rescheduling.
-     * Currently in the ScheduleTimers list and the active list. */
-    TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE,
-    /** Timer is modified and is now pending rescheduling.
-     * Currently in the ScheduleTimers list and the active list. */
-    TMTIMERSTATE_PENDING_RESCHEDULE,
-    /** Timer is being destroyed. */
-    TMTIMERSTATE_DESTROY,
-    /** Timer is free. */
-    TMTIMERSTATE_FREE
-} TMTIMERSTATE;
-
-/** Predicate that returns true if the give state is pending scheduling or
- *  rescheduling of any kind. Will reference the argument more than once! */
-#define TMTIMERSTATE_IS_PENDING_SCHEDULING(enmState) \
-    (   (enmState) <= TMTIMERSTATE_PENDING_RESCHEDULE \
-     && (enmState) >= TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE)
-
-
-/**
- * Internal representation of a timer.
- *
- * For correct serialization (without the use of semaphores and
- * other blocking/slow constructs) certain rules applies to updating
- * this structure:
- *      - For thread other than EMT only u64Expire, enmState and pScheduleNext*
- *        are changeable. Everything else is out of bounds.
- *      - Updating of u64Expire timer can only happen in the TMTIMERSTATE_STOPPED
- *        and TMTIMERSTATE_PENDING_RESCHEDULING_SET_EXPIRE states.
- *      - Timers in the TMTIMERSTATE_EXPIRED state are only accessible from EMT.
- *      - Actual destruction of a timer can only be done at scheduling time.
- */
-typedef struct TMTIMER
-{
-    /** Expire time. */
-    volatile uint64_t       u64Expire;
-    /** Clock to apply to u64Expire. */
-    TMCLOCK                 enmClock;
-    /** Timer callback type. */
-    TMTIMERTYPE             enmType;
-    /** Type specific data. */
-    union
-    {
-        /** TMTIMERTYPE_DEV. */
-        struct
-        {
-            /** Callback. */
-            R3PTRTYPE(PFNTMTIMERDEV)    pfnTimer;
-            /** Device instance. */
-            PPDMDEVINSR3                pDevIns;
-        } Dev;
-
-        /** TMTIMERTYPE_DRV. */
-        struct
-        {
-            /** Callback. */
-            R3PTRTYPE(PFNTMTIMERDRV)    pfnTimer;
-            /** Device instance. */
-            R3PTRTYPE(PPDMDRVINS)       pDrvIns;
-        } Drv;
-
-        /** TMTIMERTYPE_INTERNAL. */
-        struct
-        {
-            /** Callback. */
-            R3PTRTYPE(PFNTMTIMERINT)    pfnTimer;
-        } Internal;
-
-        /** TMTIMERTYPE_EXTERNAL. */
-        struct
-        {
-            /** Callback. */
-            R3PTRTYPE(PFNTMTIMEREXT)    pfnTimer;
-        } External;
-    } u;
-
-    /** Timer state. */
-    volatile TMTIMERSTATE   enmState;
-    /** Timer relative offset to the next timer in the schedule list. */
-    int32_t volatile        offScheduleNext;
-
-    /** Timer relative offset to the next timer in the chain. */
-    int32_t                 offNext;
-    /** Timer relative offset to the previous timer in the chain. */
-    int32_t                 offPrev;
-
-    /** Pointer to the VM the timer belongs to - R3 Ptr. */
-    PVMR3                   pVMR3;
-    /** Pointer to the VM the timer belongs to - R0 Ptr. */
-    PVMR0                   pVMR0;
-    /** Pointer to the VM the timer belongs to - RC Ptr. */
-    PVMRC                   pVMRC;
-    /** The timer frequency hint.  This is 0 if not hint was given. */
-    uint32_t volatile       uHzHint;
-
-    /** User argument. */
-    RTR3PTR                 pvUser;
-    /** The critical section associated with the lock. */
-    R3PTRTYPE(PPDMCRITSECT) pCritSect;
-
-    /** Pointer to the next timer in the list of created or free timers. (TM::pTimers or TM::pFree) */
-    PTMTIMERR3              pBigNext;
-    /** Pointer to the previous timer in the list of all created timers. (TM::pTimers) */
-    PTMTIMERR3              pBigPrev;
-    /** Pointer to the timer description. */
-    R3PTRTYPE(const char *) pszDesc;
-#if HC_ARCH_BITS == 32
-    uint32_t                padding0; /**< pad structure to multiple of 8 bytes. */
-#endif
-} TMTIMER;
-AssertCompileMemberSize(TMTIMER, enmState, sizeof(uint32_t));
-
-
-/**
- * Updates a timer state in the correct atomic manner.
- */
-#if 1
-# define TM_SET_STATE(pTimer, state) \
-    ASMAtomicWriteU32((uint32_t volatile *)&(pTimer)->enmState, state)
-#else
-# define TM_SET_STATE(pTimer, state) \
-    do { \
-        uint32_t uOld1 = (pTimer)->enmState; \
-        Log(("%s: %p: %d -> %d\n", __FUNCTION__, (pTimer), (pTimer)->enmState, state)); \
-        uint32_t uOld2 = ASMAtomicXchgU32((uint32_t volatile *)&(pTimer)->enmState, state); \
-        Assert(uOld1 == uOld2); \
-    } while (0)
-#endif
-
-/**
- * Tries to updates a timer state in the correct atomic manner.
- */
-#if 1
-# define TM_TRY_SET_STATE(pTimer, StateNew, StateOld, fRc) \
-    (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)&(pTimer)->enmState, StateNew, StateOld)
-#else
-# define TM_TRY_SET_STATE(pTimer, StateNew, StateOld, fRc) \
-    do { (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)&(pTimer)->enmState, StateNew, StateOld); \
-         Log(("%s: %p: %d -> %d %RTbool\n", __FUNCTION__, (pTimer), StateOld, StateNew, fRc)); \
-    } while (0)
-#endif
-
-/** Get the previous timer. */
-#define TMTIMER_GET_PREV(pTimer) ((PTMTIMER)((pTimer)->offPrev ? (intptr_t)(pTimer) + (pTimer)->offPrev : 0))
-/** Get the next timer. */
-#define TMTIMER_GET_NEXT(pTimer) ((PTMTIMER)((pTimer)->offNext ? (intptr_t)(pTimer) + (pTimer)->offNext : 0))
-/** Set the previous timer link. */
-#define TMTIMER_SET_PREV(pTimer, pPrev) ((pTimer)->offPrev = (pPrev) ? (intptr_t)(pPrev) - (intptr_t)(pTimer) : 0)
-/** Set the next timer link. */
-#define TMTIMER_SET_NEXT(pTimer, pNext) ((pTimer)->offNext = (pNext) ? (intptr_t)(pNext) - (intptr_t)(pTimer) : 0)
-
-
-/**
- * A timer queue.
- *
- * This is allocated on the hyper heap.
- */
-typedef struct TMTIMERQUEUE
-{
-    /** The cached expire time for this queue.
-     * Updated by EMT when scheduling the queue or modifying the head timer.
-     * Assigned UINT64_MAX when there is no head timer. */
-    uint64_t                u64Expire;
-    /** Doubly linked list of active timers.
-     *
-     * When no scheduling is pending, this list is will be ordered by expire time (ascending).
-     * Access is serialized by only letting the emulation thread (EMT) do changes.
-     *
-     * The offset is relative to the queue structure.
-     */
-    int32_t                 offActive;
-    /** List of timers pending scheduling of some kind.
-     *
-     * Timer stats allowed in the list are TMTIMERSTATE_PENDING_STOPPING,
-     * TMTIMERSTATE_PENDING_DESTRUCTION, TMTIMERSTATE_PENDING_STOPPING_DESTRUCTION,
-     * TMTIMERSTATE_PENDING_RESCHEDULING and TMTIMERSTATE_PENDING_SCHEDULE.
-     *
-     * The offset is relative to the queue structure.
-     */
-    int32_t volatile        offSchedule;
-    /** The clock for this queue. */
-    TMCLOCK                 enmClock;
-    /** Pad the structure up to 32 bytes. */
-    uint32_t                au32Padding[3];
-} TMTIMERQUEUE;
-
-/** Pointer to a timer queue. */
-typedef TMTIMERQUEUE *PTMTIMERQUEUE;
-
-/** Get the head of the active timer list. */
-#define TMTIMER_GET_HEAD(pQueue)        ((PTMTIMER)((pQueue)->offActive ? (intptr_t)(pQueue) + (pQueue)->offActive : 0))
-/** Set the head of the active timer list. */
-#define TMTIMER_SET_HEAD(pQueue, pHead) ((pQueue)->offActive = pHead ? (intptr_t)pHead - (intptr_t)(pQueue) : 0)
-
-
-/**
- * CPU load data set.
- * Mainly used by tmR3CpuLoadTimer.
- */
-typedef struct TMCPULOADSTATE
-{
-    /** The percent of the period spent executing guest code. */
-    uint8_t                 cPctExecuting;
-    /** The percent of the period spent halted. */
-    uint8_t                 cPctHalted;
-    /** The percent of the period spent on other things. */
-    uint8_t                 cPctOther;
-    /** Explicit alignment padding */
-    uint8_t                 au8Alignment[5];
-
-    /** Previous cNsTotal value. */
-    uint64_t                cNsPrevTotal;
-    /** Previous cNsExecuting value. */
-    uint64_t                cNsPrevExecuting;
-    /** Previous cNsHalted value. */
-    uint64_t                cNsPrevHalted;
-} TMCPULOADSTATE;
-AssertCompileSizeAlignment(TMCPULOADSTATE, 8);
-AssertCompileMemberAlignment(TMCPULOADSTATE, cNsPrevTotal, 8);
-/** Pointer to a CPU load data set. */
-typedef TMCPULOADSTATE *PTMCPULOADSTATE;
-
-/**
- * Converts a TM pointer into a VM pointer.
- * @returns Pointer to the VM structure the TM is part of.
- * @param   pTM   Pointer to TM instance data.
- */
-#define TM2VM(pTM)  ( (PVM)((char*)pTM - pTM->offVM) )
-
-
-/**
- * TM VM Instance data.
- * Changes to this must checked against the padding of the cfgm union in VM!
- */
-typedef struct TM
-{
-    /** Offset to the VM structure.
-     * See TM2VM(). */
-    RTUINT                      offVM;
-
-    /** Set if we fully virtualize the TSC, i.e. intercept all rdtsc instructions.
-     * Config variable: TSCVirtualized (bool) */
-    bool                        fTSCVirtualized;
-    /** Set if we use the real TSC as time source or if we use the virtual clock.
-     * If fTSCVirtualized is set we maintain a offset to the TSC and pausing/resuming the
-     * ticking. fTSCVirtualized = false implies fTSCUseRealTSC = true.
-     * Config variable: TSCUseRealTSC (bool) */
-    bool                        fTSCUseRealTSC;
-    /** Flag indicating that the host TSC is suitable for use in AMD-V and VT-x mode.
-     * Config variable: MaybeUseOffsettedHostTSC (boolean) */
-    bool                        fMaybeUseOffsettedHostTSC;
-    /** Whether the TSC is tied to the execution of code.
-     * Config variable: TSCTiedToExecution (bool) */
-    bool                        fTSCTiedToExecution;
-    /** Modifier for fTSCTiedToExecution which pauses the TSC while halting if true.
-     * Config variable: TSCNotTiedToHalt (bool) */
-    bool                        fTSCNotTiedToHalt;
-    bool                        afAlignment0[2]; /**< alignment padding */
-    /** The ID of the virtual CPU that normally runs the timers. */
-    VMCPUID                     idTimerCpu;
-    /** The number of CPU clock ticks per second (TMCLOCK_TSC).
-     * Config variable: TSCTicksPerSecond (64-bit unsigned int)
-     * The config variable implies fTSCVirtualized = true and fTSCUseRealTSC = false. */
-    uint64_t                    cTSCTicksPerSecond;
-
-    /** Virtual time ticking enabled indicator (counter for each VCPU). (TMCLOCK_VIRTUAL) */
-    uint32_t volatile           cVirtualTicking;
-    /** Virtual time is not running at 100%. */
-    bool                        fVirtualWarpDrive;
-    /** Virtual timer synchronous time ticking enabled indicator (bool). (TMCLOCK_VIRTUAL_SYNC) */
-    bool volatile               fVirtualSyncTicking;
-    /** Virtual timer synchronous time catch-up active. */
-    bool volatile               fVirtualSyncCatchUp;
-    bool                        afAlignment1[5]; /**< alignment padding */
-    /** WarpDrive percentage.
-     * 100% is normal (fVirtualSyncNormal == true). When other than 100% we apply
-     * this percentage to the raw time source for the period it's been valid in,
-     * i.e. since u64VirtualWarpDriveStart. */
-    uint32_t                    u32VirtualWarpDrivePercentage;
-
-    /** The offset of the virtual clock relative to it's timesource.
-     * Only valid if fVirtualTicking is set. */
-    uint64_t                    u64VirtualOffset;
-    /** The guest virtual time when fVirtualTicking is cleared. */
-    uint64_t                    u64Virtual;
-    /** When the Warp drive was started or last adjusted.
-     * Only valid when fVirtualWarpDrive is set. */
-    uint64_t                    u64VirtualWarpDriveStart;
-    /** The previously returned nano TS.
-     * This handles TSC drift on SMP systems and expired interval.
-     * This is a valid range u64NanoTS to u64NanoTS + 1000000000 (ie. 1sec). */
-    uint64_t volatile           u64VirtualRawPrev;
-    /** The ring-3 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
-    RTTIMENANOTSDATAR3          VirtualGetRawDataR3;
-    /** The ring-0 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
-    RTTIMENANOTSDATAR0          VirtualGetRawDataR0;
-    /** The ring-0 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
-    RTTIMENANOTSDATARC          VirtualGetRawDataRC;
-    /** Pointer to the ring-3 tmVirtualGetRawNanoTS worker function. */
-    R3PTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawR3;
-    /** Pointer to the ring-3 tmVirtualGetRawNanoTS worker function. */
-    R0PTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawR0;
-    /** Pointer to the ring-3 tmVirtualGetRawNanoTS worker function. */
-    RCPTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawRC;
-    /** Alignment. */
-    RTRCPTR                     AlignmentRCPtr;
-    /** The guest virtual timer synchronous time when fVirtualSyncTicking is cleared. */
-    uint64_t volatile           u64VirtualSync;
-    /** The offset of the timer synchronous virtual clock (TMCLOCK_VIRTUAL_SYNC) relative
-     * to the virtual clock (TMCLOCK_VIRTUAL).
-     * (This is accessed by the timer thread and must be updated atomically.) */
-    uint64_t volatile           offVirtualSync;
-    /** The offset into offVirtualSync that's been irrevocably given up by failed catch-up attempts.
-     * Thus the current lag is offVirtualSync - offVirtualSyncGivenUp. */
-    uint64_t                    offVirtualSyncGivenUp;
-    /** The TMCLOCK_VIRTUAL at the previous TMVirtualGetSync call when catch-up is active. */
-    uint64_t volatile           u64VirtualSyncCatchUpPrev;
-    /** The current catch-up percentage. */
-    uint32_t volatile           u32VirtualSyncCatchUpPercentage;
-    /** How much slack when processing timers. */
-    uint32_t                    u32VirtualSyncScheduleSlack;
-    /** When to stop catch-up. */
-    uint64_t                    u64VirtualSyncCatchUpStopThreshold;
-    /** When to give up catch-up. */
-    uint64_t                    u64VirtualSyncCatchUpGiveUpThreshold;
-/** @def TM_MAX_CATCHUP_PERIODS
- * The number of catchup rates. */
-#define TM_MAX_CATCHUP_PERIODS  10
-    /** The aggressiveness of the catch-up relative to how far we've lagged behind.
-     * The idea is to have increasing catch-up percentage as the lag increases. */
-    struct TMCATCHUPPERIOD
-    {
-        uint64_t                u64Start;       /**< When this period starts. (u64VirtualSyncOffset). */
-        uint32_t                u32Percentage;  /**< The catch-up percent to apply. */
-        uint32_t                u32Alignment;   /**< Structure alignment */
-    }                           aVirtualSyncCatchUpPeriods[TM_MAX_CATCHUP_PERIODS];
-
-    /** The current max timer Hz hint. */
-    uint32_t volatile           uMaxHzHint;
-    /** Whether to recalulate the HzHint next time its queried. */
-    bool volatile               fHzHintNeedsUpdating;
-    /** Alignment */
-    bool                        afAlignment2[3];
-    /** @cfgm{TM/HostHzMax, uint32_t, Hz, 0, UINT32_MAX, 20000}
-     * The max host Hz frequency hint returned by TMCalcHostTimerFrequency.  */
-    uint32_t                    cHostHzMax;
-    /** @cfgm{TM/HostHzFudgeFactorTimerCpu, uint32_t, Hz, 0, UINT32_MAX, 111}
-     * The number of Hz TMCalcHostTimerFrequency adds for the timer CPU.  */
-    uint32_t                    cPctHostHzFudgeFactorTimerCpu;
-    /** @cfgm{TM/HostHzFudgeFactorOtherCpu, uint32_t, Hz, 0, UINT32_MAX, 110}
-     * The number of Hz TMCalcHostTimerFrequency adds for the other CPUs. */
-    uint32_t                    cPctHostHzFudgeFactorOtherCpu;
-    /** @cfgm{TM/HostHzFudgeFactorCatchUp100, uint32_t, Hz, 0, UINT32_MAX, 300}
-     *  The fudge factor (expressed in percent) that catch-up percentages below
-     * 100% is multiplied by. */
-    uint32_t                    cPctHostHzFudgeFactorCatchUp100;
-    /** @cfgm{TM/HostHzFudgeFactorCatchUp200, uint32_t, Hz, 0, UINT32_MAX, 250}
-     * The fudge factor (expressed in percent) that catch-up percentages
-     * 100%-199% is multiplied by. */
-    uint32_t                    cPctHostHzFudgeFactorCatchUp200;
-    /** @cfgm{TM/HostHzFudgeFactorCatchUp400, uint32_t, Hz, 0, UINT32_MAX, 200}
-     * The fudge factor (expressed in percent) that catch-up percentages
-     * 200%-399% is multiplied by. */
-    uint32_t                    cPctHostHzFudgeFactorCatchUp400;
-
-    /** The UTC offset in ns.
-     * This is *NOT* for converting UTC to local time. It is for converting real
-     * world UTC time to VM UTC time. This feature is indented for doing date
-     * testing of software and similar.
-     * @todo Implement warpdrive on UTC. */
-    int64_t                     offUTC;
-
-    /** Timer queues for the different clock types - R3 Ptr */
-    R3PTRTYPE(PTMTIMERQUEUE)    paTimerQueuesR3;
-    /** Timer queues for the different clock types - R0 Ptr */
-    R0PTRTYPE(PTMTIMERQUEUE)    paTimerQueuesR0;
-    /** Timer queues for the different clock types - RC Ptr */
-    RCPTRTYPE(PTMTIMERQUEUE)    paTimerQueuesRC;
-
-    /** Pointer to our RC mapping of the GIP. */
-    RCPTRTYPE(void *)           pvGIPRC;
-    /** Pointer to our R3 mapping of the GIP. */
-    R3PTRTYPE(void *)           pvGIPR3;
-
-    /** Pointer to a singly linked list of free timers.
-     * This chain is using the TMTIMER::pBigNext members.
-     * Only accessible from the emulation thread. */
-    PTMTIMERR3                  pFree;
-
-    /** Pointer to a doubly linked list of created timers.
-     * This chain is using the TMTIMER::pBigNext and TMTIMER::pBigPrev members.
-     * Only accessible from the emulation thread. */
-    PTMTIMERR3                  pCreated;
-
-    /** The schedule timer timer handle (runtime timer).
-     * This timer will do frequent check on pending queue schedules and
-     * raise VM_FF_TIMER to pull EMTs attention to them.
-     */
-    R3PTRTYPE(PRTTIMER)         pTimer;
-    /** Interval in milliseconds of the pTimer timer. */
-    uint32_t                    u32TimerMillies;
-
-    /** Indicates that queues are being run. */
-    bool volatile               fRunningQueues;
-    /** Indicates that the virtual sync queue is being run. */
-    bool volatile               fRunningVirtualSyncQueue;
-    /** Alignment */
-    bool                        afAlignment3[2];
-
-    /** Lock serializing access to the timer lists. */
-    PDMCRITSECT                 TimerCritSect;
-    /** Lock serializing access to the VirtualSync clock. */
-    PDMCRITSECT                 VirtualSyncLock;
-
-    /** CPU load state for all the virtual CPUs (tmR3CpuLoadTimer). */
-    TMCPULOADSTATE              CpuLoad;
-
-    /** TMR3TimerQueuesDo
-     * @{ */
-    STAMPROFILE                 StatDoQueues;
-    STAMPROFILEADV              aStatDoQueues[TMCLOCK_MAX];
-    /** @} */
-    /** tmSchedule
-     * @{ */
-    STAMPROFILE                 StatScheduleOneRZ;
-    STAMPROFILE                 StatScheduleOneR3;
-    STAMCOUNTER                 StatScheduleSetFF;
-    STAMCOUNTER                 StatPostponedR3;
-    STAMCOUNTER                 StatPostponedRZ;
-    /** @} */
-    /** Read the time
-     * @{ */
-    STAMCOUNTER                 StatVirtualGet;
-    STAMCOUNTER                 StatVirtualGetSetFF;
-    STAMCOUNTER                 StatVirtualSyncGet;
-    STAMCOUNTER                 StatVirtualSyncGetELoop;
-    STAMCOUNTER                 StatVirtualSyncGetExpired;
-    STAMCOUNTER                 StatVirtualSyncGetLockless;
-    STAMCOUNTER                 StatVirtualSyncGetLocked;
-    STAMCOUNTER                 StatVirtualSyncGetSetFF;
-    STAMCOUNTER                 StatVirtualPause;
-    STAMCOUNTER                 StatVirtualResume;
-    /* @} */
-    /** TMTimerPoll
-     * @{ */
-    STAMCOUNTER                 StatPoll;
-    STAMCOUNTER                 StatPollAlreadySet;
-    STAMCOUNTER                 StatPollELoop;
-    STAMCOUNTER                 StatPollMiss;
-    STAMCOUNTER                 StatPollRunning;
-    STAMCOUNTER                 StatPollSimple;
-    STAMCOUNTER                 StatPollVirtual;
-    STAMCOUNTER                 StatPollVirtualSync;
-    /** @} */
-    /** TMTimerSet
-     * @{ */
-    STAMCOUNTER                 StatTimerSet;
-    STAMCOUNTER                 StatTimerSetOpt;
-    STAMPROFILE                 StatTimerSetRZ;
-    STAMPROFILE                 StatTimerSetR3;
-    STAMCOUNTER                 StatTimerSetStStopped;
-    STAMCOUNTER                 StatTimerSetStExpDeliver;
-    STAMCOUNTER                 StatTimerSetStActive;
-    STAMCOUNTER                 StatTimerSetStPendStop;
-    STAMCOUNTER                 StatTimerSetStPendStopSched;
-    STAMCOUNTER                 StatTimerSetStPendSched;
-    STAMCOUNTER                 StatTimerSetStPendResched;
-    STAMCOUNTER                 StatTimerSetStOther;
-    /** @} */
-    /** TMTimerSetRelative
-     * @{ */
-    STAMCOUNTER                 StatTimerSetRelative;
-    STAMPROFILE                 StatTimerSetRelativeRZ;
-    STAMPROFILE                 StatTimerSetRelativeR3;
-    STAMCOUNTER                 StatTimerSetRelativeOpt;
-    STAMCOUNTER                 StatTimerSetRelativeRacyVirtSync;
-    STAMCOUNTER                 StatTimerSetRelativeStStopped;
-    STAMCOUNTER                 StatTimerSetRelativeStExpDeliver;
-    STAMCOUNTER                 StatTimerSetRelativeStActive;
-    STAMCOUNTER                 StatTimerSetRelativeStPendStop;
-    STAMCOUNTER                 StatTimerSetRelativeStPendStopSched;
-    STAMCOUNTER                 StatTimerSetRelativeStPendSched;
-    STAMCOUNTER                 StatTimerSetRelativeStPendResched;
-    STAMCOUNTER                 StatTimerSetRelativeStOther;
-    /** @} */
-    /** TMTimerStop
-     * @{ */
-    STAMPROFILE                 StatTimerStopRZ;
-    STAMPROFILE                 StatTimerStopR3;
-    /** @} */
-    /** VirtualSync - Running and Catching Up
-     * @{ */
-    STAMCOUNTER                 StatVirtualSyncRun;
-    STAMCOUNTER                 StatVirtualSyncRunRestart;
-    STAMPROFILE                 StatVirtualSyncRunSlack;
-    STAMCOUNTER                 StatVirtualSyncRunStop;
-    STAMCOUNTER                 StatVirtualSyncRunStoppedAlready;
-    STAMCOUNTER                 StatVirtualSyncGiveUp;
-    STAMCOUNTER                 StatVirtualSyncGiveUpBeforeStarting;
-    STAMPROFILEADV              StatVirtualSyncCatchup;
-    STAMCOUNTER                 aStatVirtualSyncCatchupInitial[TM_MAX_CATCHUP_PERIODS];
-    STAMCOUNTER                 aStatVirtualSyncCatchupAdjust[TM_MAX_CATCHUP_PERIODS];
-    /** @} */
-    /** TMR3VirtualSyncFF (non dedicated EMT). */
-    STAMPROFILE                 StatVirtualSyncFF;
-    /** The timer callback. */
-    STAMCOUNTER                 StatTimerCallbackSetFF;
-
-    /** Calls to TMCpuTickSet. */
-    STAMCOUNTER                 StatTSCSet;
-
-    /** @name Reasons for refusing TSC offsetting in TMCpuTickCanUseRealTSC.
-     * @{ */
-    STAMCOUNTER                 StatTSCNotFixed;
-    STAMCOUNTER                 StatTSCNotTicking;
-    STAMCOUNTER                 StatTSCCatchupLE010;
-    STAMCOUNTER                 StatTSCCatchupLE025;
-    STAMCOUNTER                 StatTSCCatchupLE100;
-    STAMCOUNTER                 StatTSCCatchupOther;
-    STAMCOUNTER                 StatTSCWarp;
-    STAMCOUNTER                 StatTSCUnderflow;
-    STAMCOUNTER                 StatTSCSyncNotTicking;
-    /** @} */
-} TM;
-/** Pointer to TM VM instance data. */
-typedef TM *PTM;
-
-/**
- * TM VMCPU Instance data.
- * Changes to this must checked against the padding of the tm union in VM!
- */
-typedef struct TMCPU
-{
-    /** Offset to the VMCPU structure.
-     * See TMCPU2VM(). */
-    RTUINT                      offVMCPU;
-
-    /** CPU timestamp ticking enabled indicator (bool). (RDTSC) */
-    bool                        fTSCTicking;
-    bool                        afAlignment0[3]; /**< alignment padding */
-
-    /** The offset between the raw TSC source and the Guest TSC.
-     * Only valid if fTicking is set and and fTSCUseRealTSC is clear. */
-    uint64_t                    offTSCRawSrc;
-
-    /** The guest TSC when fTicking is cleared. */
-    uint64_t                    u64TSC;
-
-    /** The last seen TSC by the guest. */
-    uint64_t                    u64TSCLastSeen;
-
-#ifndef VBOX_WITHOUT_NS_ACCOUNTING
-    /** The nanosecond timestamp of the CPU start or resume.
-     * This is recalculated when the VM is started so that
-     * cNsTotal = RTTimeNanoTS() - u64NsTsStartCpu. */
-    uint64_t                    u64NsTsStartTotal;
-    /** The nanosecond timestamp of the last start-execute notification. */
-    uint64_t                    u64NsTsStartExecuting;
-    /** The nanosecond timestamp of the last start-halt notification. */
-    uint64_t                    u64NsTsStartHalting;
-    /** The cNsXXX generation. */
-    uint32_t volatile           uTimesGen;
-    /** Explicit alignment padding.  */
-    uint32_t                    u32Alignment;
-    /** The number of nanoseconds total run time.
-     * @remarks This is updated when cNsExecuting and cNsHalted are updated. */
-    uint64_t                    cNsTotal;
-    /** The number of nanoseconds spent executing. */
-    uint64_t                    cNsExecuting;
-    /** The number of nanoseconds being halted. */
-    uint64_t                    cNsHalted;
-    /** The number of nanoseconds spent on other things.
-     * @remarks This is updated when cNsExecuting and cNsHalted are updated. */
-    uint64_t                    cNsOther;
-    /** The number of halts. */
-    uint64_t                    cPeriodsHalted;
-    /** The number of guest execution runs. */
-    uint64_t                    cPeriodsExecuting;
-# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
-    /** Resettable version of cNsTotal. */
-    STAMCOUNTER                 StatNsTotal;
-    /** Resettable version of cNsExecuting. */
-    STAMPROFILE                 StatNsExecuting;
-    /** Long execution intervals. */
-    STAMPROFILE                 StatNsExecLong;
-    /** Short execution intervals . */
-    STAMPROFILE                 StatNsExecShort;
-    /** Tiny execution intervals . */
-    STAMPROFILE                 StatNsExecTiny;
-    /** Resettable version of cNsHalted. */
-    STAMPROFILE                 StatNsHalted;
-    /** Resettable version of cNsOther. */
-    STAMPROFILE                 StatNsOther;
-# endif
-
-    /** CPU load state for this virtual CPU (tmR3CpuLoadTimer). */
-    TMCPULOADSTATE              CpuLoad;
-#endif
-} TMCPU;
-/** Pointer to TM VMCPU instance data. */
-typedef TMCPU *PTMCPU;
-
-#if 0 /* enable this to rule out locking bugs on single cpu guests. */
-# define tmTimerLock(pVM)                VINF_SUCCESS
-# define tmTimerTryLock(pVM)             VINF_SUCCESS
-# define tmTimerUnlock(pVM)              ((void)0)
-# define tmVirtualSyncLock(pVM)     VINF_SUCCESS
-# define tmVirtualSyncTryLock(pVM)  VINF_SUCCESS
-# define tmVirtualSyncUnlock(pVM)   ((void)0)
-# define TM_ASSERT_LOCK(pVM)        VM_ASSERT_EMT(pVM)
-#else
-int                     tmTimerLock(PVM pVM);
-int                     tmTimerTryLock(PVM pVM);
-void                    tmTimerUnlock(PVM pVM);
-/** Checks that the caller owns the timer lock.  */
-#define TM_ASSERT_LOCK(pVM) Assert(PDMCritSectIsOwner(&pVM->tm.s.TimerCritSect))
-int                     tmVirtualSyncLock(PVM pVM);
-int                     tmVirtualSyncTryLock(PVM pVM);
-void                    tmVirtualSyncUnlock(PVM pVM);
-#endif
-
-const char             *tmTimerState(TMTIMERSTATE enmState);
-void                    tmTimerQueueSchedule(PVM pVM, PTMTIMERQUEUE pQueue);
-#ifdef VBOX_STRICT
-void                    tmTimerQueuesSanityChecks(PVM pVM, const char *pszWhere);
-#endif
-
-int                     tmCpuTickPause(PVM pVM, PVMCPU pVCpu);
-int                     tmCpuTickResume(PVM pVM, PVMCPU pVCpu);
-
-int                     tmVirtualPauseLocked(PVM pVM);
-int                     tmVirtualResumeLocked(PVM pVM);
-DECLEXPORT(void)        tmVirtualNanoTSBad(PRTTIMENANOTSDATA pData, uint64_t u64NanoTS, uint64_t u64DeltaPrev, uint64_t u64PrevNanoTS);
-DECLEXPORT(uint64_t)    tmVirtualNanoTSRediscover(PRTTIMENANOTSDATA pData);
-
-
-/** @} */
-
-RT_C_DECLS_END
-
-#endif
-
Index: unk/src/VBox/VMM/TRPM.cpp
===================================================================
--- /trunk/src/VBox/VMM/TRPM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1536 +1,0 @@
-/* $Id$ */
-/** @file
- * TRPM - The Trap Monitor.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_trpm   TRPM - The Trap Monitor
- *
- * The Trap Monitor (TRPM) is responsible for all trap and interrupt handling in
- * the VMM.  It plays a major role in raw-mode execution and a lesser one in the
- * hardware assisted mode.
- *
- * Note first, the following will use trap as a collective term for faults,
- * aborts and traps.
- *
- * @see grp_trpm
- *
- *
- * @section sec_trpm_rc     Raw-Mode Context
- *
- * When executing in the raw-mode context, TRPM will be managing the IDT and
- * processing all traps and interrupts.  It will also monitor the guest IDT
- * because CSAM wishes to know about changes to it (trap/interrupt/syscall
- * handler patching) and TRPM needs to keep the \#BP gate in sync (ring-3
- * considerations).  See TRPMR3SyncIDT and CSAMR3CheckGates.
- *
- * External interrupts will be forwarded to the host context by the quickest
- * possible route where they will be reasserted.  The other events will be
- * categorized into virtualization traps, genuine guest traps and hypervisor
- * traps.  The latter group may be recoverable depending on when they happen and
- * whether there is a handler for it, otherwise it will cause a guru meditation.
- *
- * TRPM distinguishes the between the first two (virt and guest traps) and the
- * latter (hyper) by checking the CPL of the trapping code, if CPL == 0 then
- * it's a hyper trap otherwise it's a virt/guest trap.  There are three trap
- * dispatcher tables, one ad-hoc for one time traps registered via
- * TRPMGCSetTempHandler(), one for hyper traps and one for virt/guest traps.
- * The latter two live in TRPMGCHandlersA.asm, the former in the VM structure.
- *
- * The raw-mode context trap handlers found in TRPMGCHandlers.cpp (for the most
- * part), will call up the other VMM sub-systems depending on what it things
- * happens.  The two most busy traps are page faults (\#PF) and general
- * protection fault/trap (\#GP).
- *
- * Before resuming guest code after having taken a virtualization trap or
- * injected a guest trap, TRPM will check for pending forced action and
- * every now and again let TM check for timed out timers.  This allows code that
- * is being executed as part of virtualization traps to signal ring-3 exits,
- * page table resyncs and similar without necessarily using the status code.  It
- * also make sure we're more responsive to timers and requests from other
- * threads (necessarily running on some different core/cpu in most cases).
- *
- *
- * @section sec_trpm_all    All Contexts
- *
- * TRPM will also dispatch / inject interrupts and traps to the guest, both when
- * in raw-mode and when in hardware assisted mode.  See TRPMInject().
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_TRPM
-#include <VBox/trpm.h>
-#include <VBox/cpum.h>
-#include <VBox/selm.h>
-#include <VBox/ssm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pgm.h>
-#include <include/internal/pgm.h>
-#include <VBox/dbgf.h>
-#include <VBox/mm.h>
-#include <VBox/stam.h>
-#include <VBox/csam.h>
-#include <VBox/patm.h>
-#include "TRPMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/em.h>
-#include <VBox/rem.h>
-#include <VBox/hwaccm.h>
-
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <iprt/string.h>
-#include <iprt/alloc.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Trap handler function.
- * @todo need to specialize this as we go along.
- */
-typedef enum TRPMHANDLER
-{
-    /** Generic Interrupt handler. */
-    TRPM_HANDLER_INT = 0,
-    /** Generic Trap handler. */
-    TRPM_HANDLER_TRAP,
-    /** Trap 8 (\#DF) handler. */
-    TRPM_HANDLER_TRAP_08,
-    /** Trap 12 (\#MC) handler. */
-    TRPM_HANDLER_TRAP_12,
-    /** Max. */
-    TRPM_HANDLER_MAX
-} TRPMHANDLER, *PTRPMHANDLER;
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-/** Preinitialized IDT.
- * The u16OffsetLow is a value of the TRPMHANDLER enum which TRPMR3Relocate()
- * will use to pick the right address. The u16SegSel is always VMM CS.
- */
-static VBOXIDTE_GENERIC     g_aIdt[256] =
-{
-/* special trap handler - still, this is an interrupt gate not a trap gate... */
-#define IDTE_TRAP(enm)  { (unsigned)enm,                    0, 0, VBOX_IDTE_TYPE1, VBOX_IDTE_TYPE2_INT_32, 0, 1, 0 }
-/* generic trap handler. */
-#define IDTE_TRAP_GEN() IDTE_TRAP(TRPM_HANDLER_TRAP)
-/* special interrupt handler. */
-#define IDTE_INT(enm)   { (unsigned)enm,                    0, 0, VBOX_IDTE_TYPE1, VBOX_IDTE_TYPE2_INT_32,  0, 1, 0 }
-/* generic interrupt handler. */
-#define IDTE_INT_GEN()  IDTE_INT(TRPM_HANDLER_INT)
-/* special task gate IDT entry (for critical exceptions like #DF). */
-#define IDTE_TASK(enm)  { (unsigned)enm,                    0, 0, VBOX_IDTE_TYPE1, VBOX_IDTE_TYPE2_TASK,    0, 1, 0 }
-/* draft, fixme later when the handler is written. */
-#define IDTE_RESERVED() { 0,                                0, 0,               0,                      0,  0, 0, 0 }
-
-                                        /* N  - M M -  T  - C - D i */
-                                        /* o  - n o -  y  - o - e p */
-                                        /*    - e n -  p  - d - s t */
-                                        /*    -   i -  e  - e - c . */
-                                        /*    -   c -     -   - r   */
-                                        /* ============================================================= */
-    IDTE_TRAP_GEN(),                    /*  0 - #DE - F   - N - Divide error */
-    IDTE_TRAP_GEN(),                    /*  1 - #DB - F/T - N - Single step, INT 1 instruction */
-#ifdef VBOX_WITH_NMI
-    IDTE_TRAP_GEN(),                    /*  2 -     - I   - N - Non-Maskable Interrupt (NMI) */
-#else
-    IDTE_INT_GEN(),                     /*  2 -     - I   - N - Non-Maskable Interrupt (NMI) */
-#endif
-    IDTE_TRAP_GEN(),                    /*  3 - #BP - T   - N - Breakpoint, INT 3 instruction. */
-    IDTE_TRAP_GEN(),                    /*  4 - #OF - T   - N - Overflow, INTO instruction. */
-    IDTE_TRAP_GEN(),                    /*  5 - #BR - F   - N - BOUND Range Exceeded, BOUND instruction. */
-    IDTE_TRAP_GEN(),                    /*  6 - #UD - F   - N - Undefined(/Invalid) Opcode. */
-    IDTE_TRAP_GEN(),                    /*  7 - #NM - F   - N - Device not available, FP or (F)WAIT instruction. */
-    IDTE_TASK(TRPM_HANDLER_TRAP_08),    /*  8 - #DF - A   - 0 - Double fault. */
-    IDTE_TRAP_GEN(),                    /*  9 -     - F   - N - Coprocessor Segment Overrun (obsolete). */
-    IDTE_TRAP_GEN(),                    /*  a - #TS - F   - Y - Invalid TSS, Taskswitch or TSS access. */
-    IDTE_TRAP_GEN(),                    /*  b - #NP - F   - Y - Segment not present. */
-    IDTE_TRAP_GEN(),                    /*  c - #SS - F   - Y - Stack-Segment fault. */
-    IDTE_TRAP_GEN(),                    /*  d - #GP - F   - Y - General protection fault. */
-    IDTE_TRAP_GEN(),                    /*  e - #PF - F   - Y - Page fault. - interrupt gate!!! */
-    IDTE_RESERVED(),                    /*  f -     -     -   - Intel Reserved. Do not use. */
-    IDTE_TRAP_GEN(),                    /* 10 - #MF - F   - N - x86 FPU Floating-Point Error (Math fault), FP or (F)WAIT instruction. */
-    IDTE_TRAP_GEN(),                    /* 11 - #AC - F   - 0 - Alignment Check. */
-    IDTE_TRAP(TRPM_HANDLER_TRAP_12),    /* 12 - #MC - A   - N - Machine Check. */
-    IDTE_TRAP_GEN(),                    /* 13 - #XF - F   - N - SIMD Floating-Point Exception. */
-    IDTE_RESERVED(),                    /* 14 -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 15 -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 16 -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 17 -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 18 -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 19 -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 1a -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 1b -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 1c -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 1d -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 1e -     -     -   - Intel Reserved. Do not use. */
-    IDTE_RESERVED(),                    /* 1f -     -     -   - Intel Reserved. Do not use. */
-    IDTE_INT_GEN(),                     /* 20 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 21 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 22 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 23 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 24 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 25 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 26 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 27 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 28 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 29 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 2a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 2b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 2c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 2d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 2e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 2f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 30 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 31 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 32 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 33 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 34 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 35 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 36 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 37 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 38 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 39 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 3a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 3b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 3c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 3d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 3e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 3f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 40 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 41 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 42 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 43 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 44 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 45 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 46 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 47 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 48 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 49 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 4a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 4b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 4c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 4d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 4e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 4f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 50 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 51 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 52 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 53 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 54 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 55 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 56 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 57 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 58 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 59 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 5a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 5b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 5c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 5d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 5e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 5f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 60 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 61 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 62 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 63 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 64 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 65 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 66 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 67 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 68 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 69 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 6a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 6b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 6c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 6d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 6e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 6f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 70 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 71 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 72 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 73 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 74 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 75 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 76 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 77 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 78 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 79 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 7a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 7b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 7c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 7d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 7e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 7f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 80 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 81 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 82 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 83 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 84 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 85 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 86 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 87 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 88 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 89 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 8a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 8b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 8c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 8d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 8e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 8f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 90 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 91 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 92 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 93 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 94 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 95 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 96 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 97 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 98 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 99 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 9a -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 9b -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 9c -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 9d -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 9e -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* 9f -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a0 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a1 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a2 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a3 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a4 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a5 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a6 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a7 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a8 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* a9 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* aa -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ab -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ac -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ad -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ae -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* af -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b0 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b1 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b2 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b3 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b4 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b5 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b6 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b7 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b8 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* b9 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ba -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* bb -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* bc -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* bd -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* be -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* bf -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c0 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c1 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c2 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c3 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c4 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c5 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c6 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c7 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c8 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* c9 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ca -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* cb -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* cc -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* cd -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ce -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* cf -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d0 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d1 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d2 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d3 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d4 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d5 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d6 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d7 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d8 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* d9 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* da -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* db -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* dc -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* dd -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* de -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* df -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e0 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e1 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e2 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e3 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e4 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e5 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e6 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e7 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e8 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* e9 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ea -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* eb -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ec -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ed -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ee -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ef -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f0 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f1 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f2 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f3 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f4 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f5 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f6 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f7 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f8 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* f9 -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* fa -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* fb -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* fc -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* fd -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* fe -     - I   -   - User defined Interrupts, external of INT n. */
-    IDTE_INT_GEN(),                     /* ff -     - I   -   - User defined Interrupts, external of INT n. */
-#undef IDTE_TRAP
-#undef IDTE_TRAP_GEN
-#undef IDTE_INT
-#undef IDTE_INT_GEN
-#undef IDTE_TASK
-#undef IDTE_UNUSED
-#undef IDTE_RESERVED
-};
-
-
-/** Enable or disable tracking of Guest's IDT. */
-#define TRPM_TRACK_GUEST_IDT_CHANGES
-
-/** Enable or disable tracking of Shadow IDT. */
-#define TRPM_TRACK_SHADOW_IDT_CHANGES
-
-/** TRPM saved state version. */
-#define TRPM_SAVED_STATE_VERSION        9
-#define TRPM_SAVED_STATE_VERSION_UNI    8   /* SMP support bumped the version */
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static DECLCALLBACK(int) trpmR3Save(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int) trpmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static DECLCALLBACK(int) trpmR3GuestIDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
-
-
-/**
- * Initializes the Trap Manager
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) TRPMR3Init(PVM pVM)
-{
-    LogFlow(("TRPMR3Init\n"));
-
-    /*
-     * Assert sizes and alignments.
-     */
-    AssertRelease(!(RT_OFFSETOF(VM, trpm.s) & 31));
-    AssertRelease(!(RT_OFFSETOF(VM, trpm.s.aIdt) & 15));
-    AssertRelease(sizeof(pVM->trpm.s) <= sizeof(pVM->trpm.padding));
-    AssertRelease(RT_ELEMENTS(pVM->trpm.s.aGuestTrapHandler) == sizeof(pVM->trpm.s.au32IdtPatched)*8);
-
-    /*
-     * Initialize members.
-     */
-    pVM->trpm.s.offVM              = RT_OFFSETOF(VM, trpm);
-    pVM->trpm.s.offTRPMCPU         = RT_OFFSETOF(VM, aCpus[0].trpm) - RT_OFFSETOF(VM, trpm);
-
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        pVCpu->trpm.s.offVM         = RT_OFFSETOF(VM, aCpus[i].trpm);
-        pVCpu->trpm.s.offVMCpu      = RT_OFFSETOF(VMCPU, trpm);
-        pVCpu->trpm.s.uActiveVector = ~0;
-    }
-
-    pVM->trpm.s.GuestIdtr.pIdt     = RTRCPTR_MAX;
-    pVM->trpm.s.pvMonShwIdtRC            = RTRCPTR_MAX;
-    pVM->trpm.s.fDisableMonitoring = false;
-    pVM->trpm.s.fSafeToDropGuestIDTMonitoring = false;
-
-    /*
-     * Read the configuration (if any).
-     */
-    PCFGMNODE pTRPMNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "TRPM");
-    if (pTRPMNode)
-    {
-        bool f;
-        int rc = CFGMR3QueryBool(pTRPMNode, "SafeToDropGuestIDTMonitoring", &f);
-        if (RT_SUCCESS(rc))
-            pVM->trpm.s.fSafeToDropGuestIDTMonitoring = f;
-    }
-
-    /* write config summary to log */
-    if (pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
-        LogRel(("TRPM: Dropping Guest IDT Monitoring.\n"));
-
-    /*
-     * Initialize the IDT.
-     * The handler addresses will be set in the TRPMR3Relocate() function.
-     */
-    Assert(sizeof(pVM->trpm.s.aIdt) == sizeof(g_aIdt));
-    memcpy(&pVM->trpm.s.aIdt[0], &g_aIdt[0], sizeof(pVM->trpm.s.aIdt));
-
-    /*
-     * Register the saved state data unit.
-     */
-    int rc = SSMR3RegisterInternal(pVM, "trpm", 1, TRPM_SAVED_STATE_VERSION, sizeof(TRPM),
-                                   NULL, NULL, NULL,
-                                   NULL, trpmR3Save, NULL,
-                                   NULL, trpmR3Load, NULL);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Statistics.
-     */
-    STAM_REG(pVM, &pVM->trpm.s.StatRCWriteGuestIDTFault,    STAMTYPE_COUNTER, "/TRPM/RC/IDTWritesFault",    STAMUNIT_OCCURENCES,     "Guest IDT writes the we returned to R3 to handle.");
-    STAM_REG(pVM, &pVM->trpm.s.StatRCWriteGuestIDTHandled,  STAMTYPE_COUNTER, "/TRPM/RC/IDTWritesHandled",  STAMUNIT_OCCURENCES,     "Guest IDT writes that we handled successfully.");
-    STAM_REG(pVM, &pVM->trpm.s.StatSyncIDT,                 STAMTYPE_PROFILE, "/PROF/TRPM/SyncIDT",         STAMUNIT_TICKS_PER_CALL, "Profiling of TRPMR3SyncIDT().");
-
-    /* traps */
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x00],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/00",          STAMUNIT_TICKS_PER_CALL, "#DE - Divide error.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x01],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/01",          STAMUNIT_TICKS_PER_CALL, "#DB - Debug (single step and more).");
-    //STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x02],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/02",          STAMUNIT_TICKS_PER_CALL, "NMI");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x03],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/03",          STAMUNIT_TICKS_PER_CALL, "#BP - Breakpoint.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x04],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/04",          STAMUNIT_TICKS_PER_CALL, "#OF - Overflow.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x05],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/05",          STAMUNIT_TICKS_PER_CALL, "#BR - Bound range exceeded.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x06],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/06",          STAMUNIT_TICKS_PER_CALL, "#UD - Undefined opcode.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x07],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/07",          STAMUNIT_TICKS_PER_CALL, "#NM - Device not available (FPU).");
-    //STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x08],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/08",          STAMUNIT_TICKS_PER_CALL, "#DF - Double fault.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x09],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/09",          STAMUNIT_TICKS_PER_CALL, "#?? - Coprocessor segment overrun (obsolete).");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0a],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0a",          STAMUNIT_TICKS_PER_CALL, "#TS - Task switch fault.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0b],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0b",          STAMUNIT_TICKS_PER_CALL, "#NP - Segment not present.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0c],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0c",          STAMUNIT_TICKS_PER_CALL, "#SS - Stack segment fault.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0d],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0d",          STAMUNIT_TICKS_PER_CALL, "#GP - General protection fault.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0e],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0e",          STAMUNIT_TICKS_PER_CALL, "#PF - Page fault.");
-    //STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0f],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0f",          STAMUNIT_TICKS_PER_CALL, "Reserved.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x10],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/10",          STAMUNIT_TICKS_PER_CALL, "#MF - Math fault..");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x11],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/11",          STAMUNIT_TICKS_PER_CALL, "#AC - Alignment check.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x12],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/12",          STAMUNIT_TICKS_PER_CALL, "#MC - Machine check.");
-    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x13],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/13",          STAMUNIT_TICKS_PER_CALL, "#XF - SIMD Floating-Point Exception.");
-
-#ifdef VBOX_WITH_STATISTICS
-    rc = MMHyperAlloc(pVM, sizeof(STAMCOUNTER) * 255, 8, MM_TAG_STAM, (void **)&pVM->trpm.s.paStatForwardedIRQR3);
-    AssertRCReturn(rc, rc);
-    pVM->trpm.s.paStatForwardedIRQRC = MMHyperR3ToRC(pVM, pVM->trpm.s.paStatForwardedIRQR3);
-    pVM->trpm.s.paStatForwardedIRQR0 = MMHyperR3ToR0(pVM, pVM->trpm.s.paStatForwardedIRQR3);
-    for (unsigned i = 0; i < 255; i++)
-        STAMR3RegisterF(pVM, &pVM->trpm.s.paStatForwardedIRQR3[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Forwarded interrupts.",
-                        i < 0x20 ? "/TRPM/ForwardRaw/TRAP/%02X" : "/TRPM/ForwardRaw/IRQ/%02X", i);
-#endif
-
-    STAM_REG(pVM, &pVM->trpm.s.StatForwardProfR3,       STAMTYPE_PROFILE_ADV, "/TRPM/ForwardRaw/ProfR3",   STAMUNIT_TICKS_PER_CALL, "Profiling TRPMForwardTrap.");
-    STAM_REG(pVM, &pVM->trpm.s.StatForwardProfRZ,       STAMTYPE_PROFILE_ADV, "/TRPM/ForwardRaw/ProfRZ",   STAMUNIT_TICKS_PER_CALL, "Profiling TRPMForwardTrap.");
-    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailNoHandler,    STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailNoHandler", STAMUNIT_OCCURENCES,"Failure to forward interrupt in raw mode.");
-    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailPatchAddr,    STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailPatchAddr", STAMUNIT_OCCURENCES,"Failure to forward interrupt in raw mode.");
-    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailR3,           STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailR3",       STAMUNIT_OCCURENCES, "Failure to forward interrupt in raw mode.");
-    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailRZ,           STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailRZ",       STAMUNIT_OCCURENCES, "Failure to forward interrupt in raw mode.");
-
-    STAM_REG(pVM, &pVM->trpm.s.StatTrap0dDisasm,            STAMTYPE_PROFILE, "/TRPM/RC/Traps/0d/Disasm",   STAMUNIT_TICKS_PER_CALL, "Profiling disassembly part of trpmGCTrap0dHandler.");
-    STAM_REG(pVM, &pVM->trpm.s.StatTrap0dRdTsc,             STAMTYPE_COUNTER, "/TRPM/RC/Traps/0d/RdTsc",        STAMUNIT_OCCURENCES, "Number of RDTSC #GPs.");
-
-    /*
-     * Default action when entering raw mode for the first time
-     */
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-    return 0;
-}
-
-
-/**
- * Applies relocations to data and code managed by this component.
- *
- * This function will be called at init and whenever the VMM need
- * to relocate itself inside the GC.
- *
- * @param   pVM         The VM handle.
- * @param   offDelta    Relocation delta relative to old location.
- */
-VMMR3DECL(void) TRPMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    /* Only applies to raw mode which supports only 1 VCPU. */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    LogFlow(("TRPMR3Relocate\n"));
-    /*
-     * Get the trap handler addresses.
-     *
-     * If VMMGC.gc is screwed, so are we. We'll assert here since it elsewise
-     * would make init order impossible if we should assert the presence of these
-     * exports in TRPMR3Init().
-     */
-    RTRCPTR aRCPtrs[TRPM_HANDLER_MAX];
-    RT_ZERO(aRCPtrs);
-    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerInterupt", &aRCPtrs[TRPM_HANDLER_INT]);
-    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerInterupt in VMMGC.gc!\n"));
-
-    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerGeneric",  &aRCPtrs[TRPM_HANDLER_TRAP]);
-    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerGeneric in VMMGC.gc!\n"));
-
-    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerTrap08",   &aRCPtrs[TRPM_HANDLER_TRAP_08]);
-    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerTrap08 in VMMGC.gc!\n"));
-
-    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerTrap12",   &aRCPtrs[TRPM_HANDLER_TRAP_12]);
-    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerTrap12 in VMMGC.gc!\n"));
-
-    RTSEL SelCS = CPUMGetHyperCS(pVCpu);
-
-    /*
-     * Iterate the idt and set the addresses.
-     */
-    PVBOXIDTE pIdte = &pVM->trpm.s.aIdt[0];
-    PVBOXIDTE_GENERIC pIdteTemplate = &g_aIdt[0];
-    for (unsigned i = 0; i < RT_ELEMENTS(pVM->trpm.s.aIdt); i++, pIdte++, pIdteTemplate++)
-    {
-        if (    pIdte->Gen.u1Present
-            &&  !ASMBitTest(&pVM->trpm.s.au32IdtPatched[0], i)
-           )
-        {
-            Assert(pIdteTemplate->u16OffsetLow < TRPM_HANDLER_MAX);
-            RTGCPTR Offset = aRCPtrs[pIdteTemplate->u16OffsetLow];
-            switch (pIdteTemplate->u16OffsetLow)
-            {
-                /*
-                 * Generic handlers have different entrypoints for each possible
-                 * vector number. These entrypoints makes a sort of an array with
-                 * 8 byte entries where the vector number is the index.
-                 * See TRPMGCHandlersA.asm for details.
-                 */
-                case TRPM_HANDLER_INT:
-                case TRPM_HANDLER_TRAP:
-                    Offset += i * 8;
-                    break;
-                case TRPM_HANDLER_TRAP_12:
-                    break;
-                case TRPM_HANDLER_TRAP_08:
-                    /* Handle #DF Task Gate in special way. */
-                    pIdte->Gen.u16SegSel     = SELMGetTrap8Selector(pVM);
-                    pIdte->Gen.u16OffsetLow  = 0;
-                    pIdte->Gen.u16OffsetHigh = 0;
-                    SELMSetTrap8EIP(pVM, Offset);
-                    continue;
-            }
-            /* (non-task gates only ) */
-            pIdte->Gen.u16OffsetLow  = Offset & 0xffff;
-            pIdte->Gen.u16OffsetHigh = Offset >> 16;
-            pIdte->Gen.u16SegSel     = SelCS;
-        }
-    }
-
-    /*
-     * Update IDTR (limit is including!).
-     */
-    CPUMSetHyperIDTR(pVCpu, VM_RC_ADDR(pVM, &pVM->trpm.s.aIdt[0]), sizeof(pVM->trpm.s.aIdt)-1);
-
-    if (    !pVM->trpm.s.fDisableMonitoring
-        &&  !VMMIsHwVirtExtForced(pVM))
-    {
-#ifdef TRPM_TRACK_SHADOW_IDT_CHANGES
-        if (pVM->trpm.s.pvMonShwIdtRC != RTRCPTR_MAX)
-        {
-            rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.pvMonShwIdtRC);
-            AssertRC(rc);
-        }
-        pVM->trpm.s.pvMonShwIdtRC = VM_RC_ADDR(pVM, &pVM->trpm.s.aIdt[0]);
-        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->trpm.s.pvMonShwIdtRC, pVM->trpm.s.pvMonShwIdtRC + sizeof(pVM->trpm.s.aIdt) - 1,
-                                         0, 0, "trpmRCShadowIDTWriteHandler", 0, "Shadow IDT write access handler");
-        AssertRC(rc);
-#endif
-    }
-
-    /* Relocate IDT handlers for forwarding guest traps/interrupts. */
-    for (uint32_t iTrap = 0; iTrap < RT_ELEMENTS(pVM->trpm.s.aGuestTrapHandler); iTrap++)
-    {
-        if (pVM->trpm.s.aGuestTrapHandler[iTrap] != TRPM_INVALID_HANDLER)
-        {
-            Log(("TRPMR3Relocate: iGate=%2X Handler %RRv -> %RRv\n", iTrap, pVM->trpm.s.aGuestTrapHandler[iTrap], pVM->trpm.s.aGuestTrapHandler[iTrap] + offDelta));
-            pVM->trpm.s.aGuestTrapHandler[iTrap] += offDelta;
-        }
-
-        if (ASMBitTest(&pVM->trpm.s.au32IdtPatched[0], iTrap))
-        {
-            PVBOXIDTE   pIdteCur = &pVM->trpm.s.aIdt[iTrap];
-            RTGCPTR     pHandler = VBOXIDTE_OFFSET(*pIdteCur);
-
-            Log(("TRPMR3Relocate: *iGate=%2X Handler %RGv -> %RGv\n", iTrap, pHandler, pHandler + offDelta));
-            pHandler += offDelta;
-
-            pIdteCur->Gen.u16OffsetHigh = pHandler >> 16;
-            pIdteCur->Gen.u16OffsetLow  = pHandler & 0xFFFF;
-        }
-    }
-
-#ifdef VBOX_WITH_STATISTICS
-    pVM->trpm.s.paStatForwardedIRQRC += offDelta;
-    pVM->trpm.s.paStatForwardedIRQR0 = MMHyperR3ToR0(pVM, pVM->trpm.s.paStatForwardedIRQR3);
-#endif
-}
-
-
-/**
- * Terminates the Trap Manager
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(int) TRPMR3Term(PVM pVM)
-{
-    NOREF(pVM);
-    return 0;
-}
-
-
-/**
- * Resets a virtual CPU.
- *
- * Used by TRPMR3Reset and CPU hot plugging.
- *
- * @param   pVCpu               The virtual CPU handle.
- */
-VMMR3DECL(void) TRPMR3ResetCpu(PVMCPU pVCpu)
-{
-    pVCpu->trpm.s.uActiveVector = ~0;
-}
-
-
-/**
- * The VM is being reset.
- *
- * For the TRPM component this means that any IDT write monitors
- * needs to be removed, any pending trap cleared, and the IDT reset.
- *
- * @param   pVM     VM handle.
- */
-VMMR3DECL(void) TRPMR3Reset(PVM pVM)
-{
-    /*
-     * Deregister any virtual handlers.
-     */
-#ifdef TRPM_TRACK_GUEST_IDT_CHANGES
-    if (pVM->trpm.s.GuestIdtr.pIdt != RTRCPTR_MAX)
-    {
-        if (!pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
-        {
-            int rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.GuestIdtr.pIdt);
-            AssertRC(rc);
-        }
-        pVM->trpm.s.GuestIdtr.pIdt = RTRCPTR_MAX;
-    }
-    pVM->trpm.s.GuestIdtr.cbIdt = 0;
-#endif
-
-    /*
-     * Reinitialize other members calling the relocator to get things right.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        TRPMR3ResetCpu(&pVM->aCpus[i]);
-    memcpy(&pVM->trpm.s.aIdt[0], &g_aIdt[0], sizeof(pVM->trpm.s.aIdt));
-    memset(pVM->trpm.s.aGuestTrapHandler, 0, sizeof(pVM->trpm.s.aGuestTrapHandler));
-    TRPMR3Relocate(pVM, 0);
-
-    /*
-     * Default action when entering raw mode for the first time
-     */
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
-    VMCPU_FF_SET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) trpmR3Save(PVM pVM, PSSMHANDLE pSSM)
-{
-    PTRPM pTrpm = &pVM->trpm.s;
-    LogFlow(("trpmR3Save:\n"));
-
-    /*
-     * Active and saved traps.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PTRPMCPU pTrpmCpu = &pVM->aCpus[i].trpm.s;
-        SSMR3PutUInt(pSSM,      pTrpmCpu->uActiveVector);
-        SSMR3PutUInt(pSSM,      pTrpmCpu->enmActiveType);
-        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uActiveErrorCode);
-        SSMR3PutGCUIntPtr(pSSM, pTrpmCpu->uActiveCR2);
-        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uSavedVector);
-        SSMR3PutUInt(pSSM,      pTrpmCpu->enmSavedType);
-        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uSavedErrorCode);
-        SSMR3PutGCUIntPtr(pSSM, pTrpmCpu->uSavedCR2);
-        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uPrevVector);
-    }
-    SSMR3PutBool(pSSM,      pTrpm->fDisableMonitoring);
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies 1 VCPU */
-    SSMR3PutUInt(pSSM,      VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT));
-    SSMR3PutMem(pSSM,       &pTrpm->au32IdtPatched[0], sizeof(pTrpm->au32IdtPatched));
-    SSMR3PutU32(pSSM, ~0);              /* separator. */
-
-    /*
-     * Save any trampoline gates.
-     */
-    for (uint32_t iTrap = 0; iTrap < RT_ELEMENTS(pTrpm->aGuestTrapHandler); iTrap++)
-    {
-        if (pTrpm->aGuestTrapHandler[iTrap])
-        {
-            SSMR3PutU32(pSSM, iTrap);
-            SSMR3PutGCPtr(pSSM, pTrpm->aGuestTrapHandler[iTrap]);
-            SSMR3PutMem(pSSM, &pTrpm->aIdt[iTrap], sizeof(pTrpm->aIdt[iTrap]));
-        }
-    }
-
-    return SSMR3PutU32(pSSM, ~0);       /* terminator */
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) trpmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    LogFlow(("trpmR3Load:\n"));
-    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
-
-    /*
-     * Validate version.
-     */
-    if (    uVersion != TRPM_SAVED_STATE_VERSION
-        &&  uVersion != TRPM_SAVED_STATE_VERSION_UNI)
-    {
-        AssertMsgFailed(("trpmR3Load: Invalid version uVersion=%d!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    /*
-     * Call the reset function to kick out any handled gates and other potential trouble.
-     */
-    TRPMR3Reset(pVM);
-
-    /*
-     * Active and saved traps.
-     */
-    PTRPM pTrpm = &pVM->trpm.s;
-
-    if (uVersion == TRPM_SAVED_STATE_VERSION)
-    {
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            PTRPMCPU pTrpmCpu = &pVM->aCpus[i].trpm.s;
-            SSMR3GetUInt(pSSM,      &pTrpmCpu->uActiveVector);
-            SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmActiveType);
-            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uActiveErrorCode);
-            SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uActiveCR2);
-            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedVector);
-            SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmSavedType);
-            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedErrorCode);
-            SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uSavedCR2);
-            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uPrevVector);
-        }
-
-        SSMR3GetBool(pSSM,      &pVM->trpm.s.fDisableMonitoring);
-    }
-    else
-    {
-        PTRPMCPU pTrpmCpu = &pVM->aCpus[0].trpm.s;
-        SSMR3GetUInt(pSSM,      &pTrpmCpu->uActiveVector);
-        SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmActiveType);
-        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uActiveErrorCode);
-        SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uActiveCR2);
-        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedVector);
-        SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmSavedType);
-        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedErrorCode);
-        SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uSavedCR2);
-        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uPrevVector);
-
-        RTGCUINT fDisableMonitoring;
-        SSMR3GetGCUInt(pSSM,    &fDisableMonitoring);
-        pTrpm->fDisableMonitoring = !!fDisableMonitoring;
-    }
-
-    RTUINT fSyncIDT;
-    int rc = SSMR3GetUInt(pSSM, &fSyncIDT);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (fSyncIDT & ~1)
-    {
-        AssertMsgFailed(("fSyncIDT=%#x\n", fSyncIDT));
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    }
-    if (fSyncIDT)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies 1 VCPU */
-        VMCPU_FF_SET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-    }
-    /* else: cleared by reset call above. */
-
-    SSMR3GetMem(pSSM, &pTrpm->au32IdtPatched[0], sizeof(pTrpm->au32IdtPatched));
-
-    /* check the separator */
-    uint32_t u32Sep;
-    rc = SSMR3GetU32(pSSM, &u32Sep);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (u32Sep != (uint32_t)~0)
-    {
-        AssertMsgFailed(("u32Sep=%#x (first)\n", u32Sep));
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    }
-
-    /*
-     * Restore any trampoline gates.
-     */
-    for (;;)
-    {
-        /* gate number / terminator */
-        uint32_t iTrap;
-        rc = SSMR3GetU32(pSSM, &iTrap);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (iTrap == (uint32_t)~0)
-            break;
-        if (    iTrap >= RT_ELEMENTS(pTrpm->aIdt)
-            ||  pTrpm->aGuestTrapHandler[iTrap])
-        {
-            AssertMsgFailed(("iTrap=%#x\n", iTrap));
-            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-        }
-
-        /* restore the IDT entry. */
-        RTGCPTR GCPtrHandler;
-        SSMR3GetGCPtr(pSSM, &GCPtrHandler);
-        VBOXIDTE Idte;
-        rc = SSMR3GetMem(pSSM, &Idte, sizeof(Idte));
-        if (RT_FAILURE(rc))
-            return rc;
-        Assert(GCPtrHandler);
-        pTrpm->aIdt[iTrap] = Idte;
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Check if gate handlers were updated
- * (callback for the VMCPU_FF_TRPM_SYNC_IDT forced action).
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   pVCpu       The VMCPU handle.
- */
-VMMR3DECL(int) TRPMR3SyncIDT(PVM pVM, PVMCPU pVCpu)
-{
-    STAM_PROFILE_START(&pVM->trpm.s.StatSyncIDT, a);
-    const bool  fRawRing0 = EMIsRawRing0Enabled(pVM);
-    int         rc;
-
-    if (pVM->trpm.s.fDisableMonitoring)
-    {
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-        return VINF_SUCCESS;    /* Nothing to do */
-    }
-
-    if (fRawRing0 && CSAMIsEnabled(pVM))
-    {
-        /* Clear all handlers */
-        Log(("TRPMR3SyncIDT: Clear all trap handlers.\n"));
-        /** @todo inefficient, but simple */
-        for (unsigned iGate = 0; iGate < 256; iGate++)
-            trpmClearGuestTrapHandler(pVM, iGate);
-
-        /* Scan them all (only the first time) */
-        CSAMR3CheckGates(pVM, 0, 256);
-    }
-
-    /*
-     * Get the IDTR.
-     */
-    VBOXIDTR IDTR;
-    IDTR.pIdt = CPUMGetGuestIDTR(pVCpu, &IDTR.cbIdt);
-    if (!IDTR.cbIdt)
-    {
-        Log(("No IDT entries...\n"));
-        return DBGFSTOP(pVM);
-    }
-
-#ifdef TRPM_TRACK_GUEST_IDT_CHANGES
-    /*
-     * Check if Guest's IDTR has changed.
-     */
-    if (    IDTR.pIdt  != pVM->trpm.s.GuestIdtr.pIdt
-        ||  IDTR.cbIdt != pVM->trpm.s.GuestIdtr.cbIdt)
-    {
-        Log(("TRPMR3UpdateFromCPUM: Guest's IDT is changed to pIdt=%08X cbIdt=%08X\n", IDTR.pIdt, IDTR.cbIdt));
-        if (!pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
-        {
-            /*
-             * [Re]Register write virtual handler for guest's IDT.
-             */
-            if (pVM->trpm.s.GuestIdtr.pIdt != RTRCPTR_MAX)
-            {
-                rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.GuestIdtr.pIdt);
-                AssertRCReturn(rc, rc);
-            }
-            /* limit is including */
-            rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, IDTR.pIdt, IDTR.pIdt + IDTR.cbIdt /* already inclusive */,
-                                             0, trpmR3GuestIDTWriteHandler, "trpmRCGuestIDTWriteHandler", 0, "Guest IDT write access handler");
-
-            if (rc == VERR_PGM_HANDLER_VIRTUAL_CONFLICT)
-            {
-                /* Could be a conflict with CSAM */
-                CSAMR3RemovePage(pVM, IDTR.pIdt);
-                if (PAGE_ADDRESS(IDTR.pIdt) != PAGE_ADDRESS(IDTR.pIdt + IDTR.cbIdt))
-                    CSAMR3RemovePage(pVM, IDTR.pIdt + IDTR.cbIdt);
-
-                rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, IDTR.pIdt, IDTR.pIdt + IDTR.cbIdt /* already inclusive */,
-                                                 0, trpmR3GuestIDTWriteHandler, "trpmRCGuestIDTWriteHandler", 0, "Guest IDT write access handler");
-            }
-
-            AssertRCReturn(rc, rc);
-        }
-
-        /* Update saved Guest IDTR. */
-        pVM->trpm.s.GuestIdtr = IDTR;
-    }
-#endif
-
-    /*
-     * Sync the interrupt gate.
-     * Should probably check/sync the others too, but for now we'll handle that in #GP.
-     */
-    X86DESC  Idte3;
-    rc = PGMPhysSimpleReadGCPtr(pVCpu, &Idte3, IDTR.pIdt + sizeof(Idte3) * 3,  sizeof(Idte3));
-    if (RT_FAILURE(rc))
-    {
-        AssertMsgRC(rc, ("Failed to read IDT[3]! rc=%Rrc\n", rc));
-        return DBGFSTOP(pVM);
-    }
-    AssertRCReturn(rc, rc);
-    if (fRawRing0)
-        pVM->trpm.s.aIdt[3].Gen.u2DPL = RT_MAX(Idte3.Gen.u2Dpl, 1);
-    else
-        pVM->trpm.s.aIdt[3].Gen.u2DPL = Idte3.Gen.u2Dpl;
-
-    /*
-     * Clear the FF and we're done.
-     */
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-    STAM_PROFILE_STOP(&pVM->trpm.s.StatSyncIDT, a);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Disable IDT monitoring and syncing
- *
- * @param   pVM         The VM to operate on.
- */
-VMMR3DECL(void) TRPMR3DisableMonitoring(PVM pVM)
-{
-    /*
-     * Deregister any virtual handlers.
-     */
-#ifdef TRPM_TRACK_GUEST_IDT_CHANGES
-    if (pVM->trpm.s.GuestIdtr.pIdt != RTRCPTR_MAX)
-    {
-        if (!pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
-        {
-            int rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.GuestIdtr.pIdt);
-            AssertRC(rc);
-        }
-        pVM->trpm.s.GuestIdtr.pIdt = RTRCPTR_MAX;
-    }
-    pVM->trpm.s.GuestIdtr.cbIdt = 0;
-#endif
-
-#ifdef TRPM_TRACK_SHADOW_IDT_CHANGES
-    if (pVM->trpm.s.pvMonShwIdtRC != RTRCPTR_MAX)
-    {
-        int rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.pvMonShwIdtRC);
-        AssertRC(rc);
-        pVM->trpm.s.pvMonShwIdtRC = RTRCPTR_MAX;
-    }
-#endif
-
-    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-
-    pVM->trpm.s.fDisableMonitoring = true;
-}
-
-
-/**
- * \#PF Handler callback for virtual access handler ranges.
- *
- * Important to realize that a physical page in a range can have aliases, and
- * for ALL and WRITE handlers these will also trigger.
- *
- * @returns VINF_SUCCESS if the handler have carried out the operation.
- * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
- * @param   pVM             VM Handle.
- * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
- * @param   pvPtr           The HC mapping of that address.
- * @param   pvBuf           What the guest is reading/writing.
- * @param   cbBuf           How much it's reading/writing.
- * @param   enmAccessType   The access type.
- * @param   pvUser          User argument.
- */
-static DECLCALLBACK(int) trpmR3GuestIDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
-{
-    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
-    Log(("trpmR3GuestIDTWriteHandler: write to %RGv size %d\n", GCPtr, cbBuf));
-    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TRPM_SYNC_IDT);
-    return VINF_PGM_HANDLER_DO_DEFAULT;
-}
-
-
-/**
- * Clear passthrough interrupt gate handler (reset to default handler)
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   iTrap       Trap/interrupt gate number.
- */
-VMMR3DECL(int) trpmR3ClearPassThroughHandler(PVM pVM, unsigned iTrap)
-{
-    /* Only applies to raw mode which supports only 1 VCPU. */
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    /** @todo cleanup trpmR3ClearPassThroughHandler()! */
-    RTRCPTR aGCPtrs[TRPM_HANDLER_MAX];
-    int rc;
-
-    memset(aGCPtrs, 0, sizeof(aGCPtrs));
-
-    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerInterupt", &aGCPtrs[TRPM_HANDLER_INT]);
-    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerInterupt in VMMGC.gc!\n"));
-
-    if (    iTrap < TRPM_HANDLER_INT_BASE
-        ||  iTrap >= RT_ELEMENTS(pVM->trpm.s.aIdt))
-    {
-        AssertMsg(iTrap < TRPM_HANDLER_INT_BASE, ("Illegal gate number %#x!\n", iTrap));
-        return VERR_INVALID_PARAMETER;
-    }
-    memcpy(&pVM->trpm.s.aIdt[iTrap], &g_aIdt[iTrap], sizeof(pVM->trpm.s.aIdt[0]));
-
-    /* Unmark it for relocation purposes. */
-    ASMBitClear(&pVM->trpm.s.au32IdtPatched[0], iTrap);
-
-    RTSEL               SelCS         = CPUMGetHyperCS(pVCpu);
-    PVBOXIDTE           pIdte         = &pVM->trpm.s.aIdt[iTrap];
-    PVBOXIDTE_GENERIC   pIdteTemplate = &g_aIdt[iTrap];
-    if (pIdte->Gen.u1Present)
-    {
-        Assert(pIdteTemplate->u16OffsetLow == TRPM_HANDLER_INT);
-        Assert(sizeof(RTRCPTR) == sizeof(aGCPtrs[0]));
-        RTRCPTR Offset = (RTRCPTR)aGCPtrs[pIdteTemplate->u16OffsetLow];
-
-        /*
-         * Generic handlers have different entrypoints for each possible
-         * vector number. These entrypoints make a sort of an array with
-         * 8 byte entries where the vector number is the index.
-         * See TRPMGCHandlersA.asm for details.
-         */
-        Offset += iTrap * 8;
-
-        if (pIdte->Gen.u5Type2 != VBOX_IDTE_TYPE2_TASK)
-        {
-            pIdte->Gen.u16OffsetLow  = Offset & 0xffff;
-            pIdte->Gen.u16OffsetHigh = Offset >> 16;
-            pIdte->Gen.u16SegSel     = SelCS;
-        }
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Check if address is a gate handler (interrupt or trap).
- *
- * @returns gate nr or ~0 is not found
- *
- * @param   pVM         VM handle.
- * @param   GCPtr       GC address to check.
- */
-VMMR3DECL(uint32_t) TRPMR3QueryGateByHandler(PVM pVM, RTRCPTR GCPtr)
-{
-    for (uint32_t iTrap = 0; iTrap < RT_ELEMENTS(pVM->trpm.s.aGuestTrapHandler); iTrap++)
-    {
-        if (pVM->trpm.s.aGuestTrapHandler[iTrap] == GCPtr)
-            return iTrap;
-
-        /* redundant */
-        if (ASMBitTest(&pVM->trpm.s.au32IdtPatched[0], iTrap))
-        {
-            PVBOXIDTE   pIdte = &pVM->trpm.s.aIdt[iTrap];
-            RTGCPTR     pHandler = VBOXIDTE_OFFSET(*pIdte);
-
-            if (pHandler == GCPtr)
-                return iTrap;
-        }
-    }
-    return ~0;
-}
-
-
-/**
- * Get guest trap/interrupt gate handler
- *
- * @returns Guest trap handler address or TRPM_INVALID_HANDLER if none installed
- * @param   pVM         The VM to operate on.
- * @param   iTrap       Interrupt/trap number.
- */
-VMMR3DECL(RTRCPTR) TRPMR3GetGuestTrapHandler(PVM pVM, unsigned iTrap)
-{
-    AssertReturn(iTrap < RT_ELEMENTS(pVM->trpm.s.aIdt), TRPM_INVALID_HANDLER);
-
-    return pVM->trpm.s.aGuestTrapHandler[iTrap];
-}
-
-
-/**
- * Set guest trap/interrupt gate handler
- * Used for setting up trap gates used for kernel calls.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   iTrap       Interrupt/trap number.
- * @param   pHandler    GC handler pointer
- */
-VMMR3DECL(int) TRPMR3SetGuestTrapHandler(PVM pVM, unsigned iTrap, RTRCPTR pHandler)
-{
-    /* Only valid in raw mode which implies 1 VCPU */
-    Assert(PATMIsEnabled(pVM) && pVM->cCpus == 1);
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    /*
-     * Validate.
-     */
-    if (iTrap >= RT_ELEMENTS(pVM->trpm.s.aIdt))
-    {
-        AssertMsg(iTrap < TRPM_HANDLER_INT_BASE, ("Illegal gate number %d!\n", iTrap));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    AssertReturn(pHandler == TRPM_INVALID_HANDLER || PATMIsPatchGCAddr(pVM, pHandler), VERR_INVALID_PARAMETER);
-
-    uint16_t    cbIDT;
-    RTGCPTR     GCPtrIDT = CPUMGetGuestIDTR(pVCpu, &cbIDT);
-    if (iTrap * sizeof(VBOXIDTE) >= cbIDT)
-        return VERR_INVALID_PARAMETER;  /* Silently ignore out of range requests. */
-
-    if (pHandler == TRPM_INVALID_HANDLER)
-    {
-        /* clear trap handler */
-        Log(("TRPMR3SetGuestTrapHandler: clear handler %x\n", iTrap));
-        return trpmClearGuestTrapHandler(pVM, iTrap);
-    }
-
-    /*
-     * Read the guest IDT entry.
-     */
-    VBOXIDTE GuestIdte;
-    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &GuestIdte, GCPtrIDT + iTrap * sizeof(GuestIdte),  sizeof(GuestIdte));
-    if (RT_FAILURE(rc))
-    {
-        AssertMsgRC(rc, ("Failed to read IDTE! rc=%Rrc\n", rc));
-        return rc;
-    }
-
-    if (EMIsRawRing0Enabled(pVM))
-    {
-        /*
-         * Only replace handlers for which we are 100% certain there won't be
-         * any host interrupts.
-         *
-         * 0x2E is safe on Windows because it's the system service interrupt gate. Not
-         * quite certain if this is safe or not on 64-bit Vista, it probably is.
-         *
-         * 0x80 is safe on Linux because it's the syscall vector and is part of the
-         * 32-bit usermode ABI. 64-bit Linux (usually) supports 32-bit processes
-         * and will therefor never assign hardware interrupts to 0x80.
-         *
-         * Exactly why 0x80 is safe on 32-bit Windows is a bit hazy, but it seems
-         * to work ok... However on 64-bit Vista (SMP?) is doesn't work reliably.
-         * Booting Linux/BSD guest will cause system lockups on most of the computers.
-         * -> Update: It seems gate 0x80 is not safe on 32-bits Windows either. See
-         *            defect #3604.
-         *
-         * PORTME - Check if your host keeps any of these gates free from hw ints.
-         *
-         * Note! SELMR3SyncTSS also has code related to this interrupt handler replacing.
-         */
-        /** @todo handle those dependencies better! */
-        /** @todo Solve this in a proper manner. see defect #1186 */
-#if defined(RT_OS_WINDOWS) && defined(RT_ARCH_X86)
-        if (iTrap == 0x2E)
-#elif defined(RT_OS_LINUX)
-        if (iTrap == 0x80)
-#else
-        if (0)
-#endif
-        {
-            if (     GuestIdte.Gen.u1Present
-                &&  (   GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_TRAP_32
-                     || GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_INT_32)
-                &&   GuestIdte.Gen.u2DPL == 3)
-            {
-                PVBOXIDTE   pIdte = &pVM->trpm.s.aIdt[iTrap];
-
-                GuestIdte.Gen.u5Type2 = VBOX_IDTE_TYPE2_TRAP_32;
-                GuestIdte.Gen.u16OffsetHigh = pHandler >> 16;
-                GuestIdte.Gen.u16OffsetLow  = pHandler & 0xFFFF;
-                GuestIdte.Gen.u16SegSel |= 1;  //ring 1
-                *pIdte = GuestIdte;
-
-                /* Mark it for relocation purposes. */
-                ASMBitSet(&pVM->trpm.s.au32IdtPatched[0], iTrap);
-
-                /* Also store it in our guest trap array. */
-                pVM->trpm.s.aGuestTrapHandler[iTrap] = pHandler;
-
-                Log(("Setting trap handler %x to %08X (direct)\n", iTrap, pHandler));
-                return VINF_SUCCESS;
-            }
-            /* ok, let's try to install a trampoline handler then. */
-        }
-    }
-
-    if (     GuestIdte.Gen.u1Present
-        &&  (   GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_TRAP_32
-             || GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_INT_32)
-        &&  (GuestIdte.Gen.u2DPL == 3 || GuestIdte.Gen.u2DPL == 0))
-    {
-        /*
-         * Save handler which can be used for a trampoline call inside the GC
-         */
-        Log(("Setting trap handler %x to %08X\n", iTrap, pHandler));
-        pVM->trpm.s.aGuestTrapHandler[iTrap] = pHandler;
-        return VINF_SUCCESS;
-    }
-    return VERR_INVALID_PARAMETER;
-}
-
-
-/**
- * Check if address is a gate handler (interrupt/trap/task/anything).
- *
- * @returns True is gate handler, false if not.
- *
- * @param   pVM         VM handle.
- * @param   GCPtr       GC address to check.
- */
-VMMR3DECL(bool) TRPMR3IsGateHandler(PVM pVM, RTRCPTR GCPtr)
-{
-    /* Only valid in raw mode which implies 1 VCPU */
-    Assert(PATMIsEnabled(pVM) && pVM->cCpus == 1);
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    /*
-     * Read IDTR and calc last entry.
-     */
-    uint16_t    cbIDT;
-    RTGCPTR     GCPtrIDTE = CPUMGetGuestIDTR(pVCpu, &cbIDT);
-    unsigned    cEntries = (cbIDT + 1) / sizeof(VBOXIDTE);
-    if (!cEntries)
-        return false;
-    RTGCPTR   GCPtrIDTELast = GCPtrIDTE + (cEntries - 1) * sizeof(VBOXIDTE);
-
-    /*
-     * Outer loop: iterate pages.
-     */
-    while (GCPtrIDTE <= GCPtrIDTELast)
-    {
-        /*
-         * Convert this page to a HC address.
-         * (This function checks for not-present pages.)
-         */
-        PCVBOXIDTE      pIDTE;
-        PGMPAGEMAPLOCK  Lock;
-        int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrIDTE, (const void **)&pIDTE, &Lock);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Inner Loop: Iterate the data on this page looking for an entry equal to GCPtr.
-             * N.B. Member of the Flat Earth Society...
-             */
-            while (GCPtrIDTE <= GCPtrIDTELast)
-            {
-                if (pIDTE->Gen.u1Present)
-                {
-                    RTRCPTR GCPtrHandler = VBOXIDTE_OFFSET(*pIDTE);
-                    if (GCPtr == GCPtrHandler)
-                    {
-                        PGMPhysReleasePageMappingLock(pVM, &Lock);
-                        return true;
-                    }
-                }
-
-                /* next entry */
-                if ((GCPtrIDTE & PAGE_OFFSET_MASK) + sizeof(VBOXIDTE) >= PAGE_SIZE)
-                {
-                    AssertMsg(!(GCPtrIDTE & (sizeof(VBOXIDTE) - 1)),
-                              ("IDT is crossing pages and it's not aligned! GCPtrIDTE=%#x cbIDT=%#x\n", GCPtrIDTE, cbIDT));
-                    GCPtrIDTE += sizeof(VBOXIDTE);
-                    break;
-                }
-                GCPtrIDTE += sizeof(VBOXIDTE);
-                pIDTE++;
-            }
-            PGMPhysReleasePageMappingLock(pVM, &Lock);
-        }
-        else
-        {
-            /* Skip to the next page (if any). Take care not to wrap around the address space. */
-            if ((GCPtrIDTELast >> PAGE_SHIFT) == (GCPtrIDTE >> PAGE_SHIFT))
-                return false;
-            GCPtrIDTE = RT_ALIGN_T(GCPtrIDTE, PAGE_SIZE, RTGCPTR) + PAGE_SIZE + (GCPtrIDTE & (sizeof(VBOXIDTE) - 1));
-        }
-    }
-    return false;
-}
-
-
-/**
- * Inject event (such as external irq or trap)
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pVCpu       The VMCPU to operate on.
- * @param   enmEvent    Trpm event type
- */
-VMMR3DECL(int) TRPMR3InjectEvent(PVM pVM, PVMCPU pVCpu, TRPMEVENT enmEvent)
-{
-    PCPUMCTX pCtx;
-    int      rc;
-
-    pCtx = CPUMQueryGuestCtxPtr(pVCpu);
-    Assert(!PATMIsPatchGCAddr(pVM, pCtx->eip));
-    Assert(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
-
-    /* Currently only useful for external hardware interrupts. */
-    Assert(enmEvent == TRPM_HARDWARE_INT);
-
-    if (REMR3QueryPendingInterrupt(pVM, pVCpu) == REM_NO_PENDING_IRQ)
-    {
-#ifdef TRPM_FORWARD_TRAPS_IN_GC
-
-# ifdef LOG_ENABLED
-        DBGFR3InfoLog(pVM, "cpumguest", "TRPMInject");
-        DBGFR3DisasInstrCurrentLog(pVCpu, "TRPMInject");
-# endif
-
-        uint8_t u8Interrupt;
-        rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
-        Log(("TRPMR3InjectEvent: CPU%d u8Interrupt=%d (%#x) rc=%Rrc\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc));
-        if (RT_SUCCESS(rc))
-        {
-            if (HWACCMIsEnabled(pVM))
-            {
-                rc = TRPMAssertTrap(pVCpu, u8Interrupt, enmEvent);
-                AssertRC(rc);
-                STAM_COUNTER_INC(&pVM->trpm.s.paStatForwardedIRQR3[u8Interrupt]);
-                return HWACCMR3IsActive(pVCpu) ? VINF_EM_RESCHEDULE_HWACC : VINF_EM_RESCHEDULE_REM;
-            }
-            /* If the guest gate is not patched, then we will check (again) if we can patch it. */
-            if (pVM->trpm.s.aGuestTrapHandler[u8Interrupt] == TRPM_INVALID_HANDLER)
-            {
-                CSAMR3CheckGates(pVM, u8Interrupt, 1);
-                Log(("TRPMR3InjectEvent: recheck gate %x -> valid=%d\n", u8Interrupt, TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) != TRPM_INVALID_HANDLER));
-            }
-
-            if (pVM->trpm.s.aGuestTrapHandler[u8Interrupt] != TRPM_INVALID_HANDLER)
-            {
-                /* Must check pending forced actions as our IDT or GDT might be out of sync */
-                rc = EMR3CheckRawForcedActions(pVM, pVCpu);
-                if (rc == VINF_SUCCESS)
-                {
-                    /* There's a handler -> let's execute it in raw mode */
-                    rc = TRPMForwardTrap(pVCpu, CPUMCTX2CORE(pCtx), u8Interrupt, 0, TRPM_TRAP_NO_ERRORCODE, enmEvent, -1);
-                    if (rc == VINF_SUCCESS /* Don't use RT_SUCCESS */)
-                    {
-                        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_TSS));
-
-                        STAM_COUNTER_INC(&pVM->trpm.s.paStatForwardedIRQR3[u8Interrupt]);
-                        return VINF_EM_RESCHEDULE_RAW;
-                    }
-                }
-            }
-            else
-                STAM_COUNTER_INC(&pVM->trpm.s.StatForwardFailNoHandler);
-            REMR3NotifyPendingInterrupt(pVM, pVCpu, u8Interrupt);
-        }
-        else
-        {
-            AssertRC(rc);
-            return HWACCMR3IsActive(pVCpu) ? VINF_EM_RESCHEDULE_HWACC : VINF_EM_RESCHEDULE_REM; /* (Heed the halted state if this is changed!) */
-        }
-#else
-        if (HWACCMR3IsActive(pVM))
-        {
-            uint8_t u8Interrupt;
-            rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
-            Log(("TRPMR3InjectEvent: u8Interrupt=%d (%#x) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
-            if (RT_SUCCESS(rc))
-            {
-                rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
-                AssertRC(rc);
-                STAM_COUNTER_INC(&pVM->trpm.s.paStatForwardedIRQR3[u8Interrupt]);
-                return VINF_EM_RESCHEDULE_HWACC;
-            }
-        }
-        else
-            AssertRC(rc);
-#endif
-    }
-    /** @todo check if it's safe to translate the patch address to the original guest address.
-     *        this implies a safe state in translated instructions and should take sti successors into account (instruction fusing)
-     */
-    /* Note: if it's a PATM address, then we'll go back to raw mode regardless of the return code below. */
-
-    /* Fall back to the recompiler */
-    return VINF_EM_RESCHEDULE_REM; /* (Heed the halted state if this is changed!) */
-}
-
Index: unk/src/VBox/VMM/TRPMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/TRPMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,287 +1,0 @@
-/* $Id$ */
-/** @file
- * TRPM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___TRPMInternal_h
-#define ___TRPMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/types.h>
-#include <VBox/stam.h>
-#include <VBox/cpum.h>
-
-
-
-/* Enable to allow trap forwarding in GC. */
-#define TRPM_FORWARD_TRAPS_IN_GC
-
-/** First interrupt handler. Used for validating input. */
-#define TRPM_HANDLER_INT_BASE  0x20
-
-RT_C_DECLS_BEGIN
-
-
-/** @defgroup grp_trpm_int   Internals
- * @ingroup grp_trpm
- * @internal
- * @{
- */
-
-/** @name   TRPMGCTrapIn* flags.
- * The lower bits are offsets into the CPUMCTXCORE structure.
- * @{ */
-/** The mask for the operation. */
-#define TRPM_TRAP_IN_OP_MASK    0xffff
-/** Traps on MOV GS, eax. */
-#define TRPM_TRAP_IN_MOV_GS     1
-/** Traps on MOV FS, eax. */
-#define TRPM_TRAP_IN_MOV_FS     2
-/** Traps on MOV ES, eax. */
-#define TRPM_TRAP_IN_MOV_ES     3
-/** Traps on MOV DS, eax. */
-#define TRPM_TRAP_IN_MOV_DS     4
-/** Traps on IRET. */
-#define TRPM_TRAP_IN_IRET       5
-/** Set if this is a V86 resume. */
-#define TRPM_TRAP_IN_V86        RT_BIT(30)
-/** If set this is a hypervisor register set. If cleared it's a guest set. */
-#define TRPM_TRAP_IN_HYPER      RT_BIT(31)
-/** @} */
-
-
-#if 0 /* not used */
-/**
- * Converts a TRPM pointer into a VM pointer.
- * @returns Pointer to the VM structure the TRPM is part of.
- * @param   pTRPM   Pointer to TRPM instance data.
- */
-#define TRPM_2_VM(pTRPM)            ( (PVM)((uint8_t *)(pTRPM) - (pTRPM)->offVM) )
-#endif
-
-/**
- * Converts a TRPM pointer into a TRPMCPU pointer.
- * @returns Pointer to the VM structure the TRPMCPU is part of.
- * @param   pTRPM   Pointer to TRPMCPU instance data.
- * @remarks Raw-mode only, not SMP safe.
- */
-#define TRPM_2_TRPMCPU(pTrpmCpu)     ( (PTRPMCPU)((uint8_t *)(pTrpmCpu) + (pTrpmCpu)->offTRPMCPU) )
-
-
-/**
- * TRPM Data (part of VM)
- *
- * IMPORTANT! Keep the nasm version of this struct up-to-date.
- */
-#pragma pack(4)
-typedef struct TRPM
-{
-    /** Offset to the VM structure.
-     * See TRPM_2_VM(). */
-    RTINT                   offVM;
-    /** Offset to the TRPMCPU structure.
-     * See TRPM2TRPMCPU(). */
-    RTINT                   offTRPMCPU;
-
-    /** IDT monitoring and sync flag (HWACC). */
-    bool                    fDisableMonitoring; /** @todo r=bird: bool and 7 byte achPadding1. */
-
-    /** Whether monitoring of the guest IDT is enabled or not.
-     *
-     * This configuration option is provided for speeding up guest like Solaris
-     * that put the IDT on the same page as a whole lot of other data that is
-     * frequently updated. The updates will cause #PFs and have to be interpreted
-     * by PGMInterpretInstruction which is slow compared to raw execution.
-     *
-     * If the guest is well behaved and doesn't change the IDT after loading it,
-     * there is no problem with dropping the IDT monitoring.
-     *
-     * @cfgm    /TRPM/SafeToDropGuestIDTMonitoring   boolean     defaults to false.
-     */
-    bool                    fSafeToDropGuestIDTMonitoring;
-
-    /** Padding to get the IDTs at a 16 byte alignment. */
-    uint8_t                 abPadding1[6];
-    /** IDTs. Aligned at 16 byte offset for speed. */
-    VBOXIDTE                aIdt[256];
-
-    /** Bitmap for IDTEs that contain PATM handlers. (needed for relocation) */
-    uint32_t                au32IdtPatched[8];
-
-    /** Temporary Hypervisor trap handlers.
-     * NULL means default action. */
-    RCPTRTYPE(void *)       aTmpTrapHandlers[256];
-
-    /** RC Pointer to the IDT shadow area (aIdt) in HMA. */
-    RCPTRTYPE(void *)       pvMonShwIdtRC;
-    /** Current (last) Guest's IDTR. */
-    VBOXIDTR                GuestIdtr;
-
-    /** padding. */
-    uint8_t                 au8Padding[2];
-
-    /** Checked trap & interrupt handler array */
-    RCPTRTYPE(void *)       aGuestTrapHandler[256];
-
-    /** RC: The number of times writes to the Guest IDT were detected. */
-    STAMCOUNTER             StatRCWriteGuestIDTFault;
-    STAMCOUNTER             StatRCWriteGuestIDTHandled;
-
-    /** HC: Profiling of the TRPMR3SyncIDT() method. */
-    STAMPROFILE             StatSyncIDT;
-    /** GC: Statistics for the trap handlers. */
-    STAMPROFILEADV          aStatGCTraps[0x14];
-
-    STAMPROFILEADV          StatForwardProfR3;
-    STAMPROFILEADV          StatForwardProfRZ;
-    STAMCOUNTER             StatForwardFailNoHandler;
-    STAMCOUNTER             StatForwardFailPatchAddr;
-    STAMCOUNTER             StatForwardFailR3;
-    STAMCOUNTER             StatForwardFailRZ;
-
-    STAMPROFILE             StatTrap0dDisasm;
-    STAMCOUNTER             StatTrap0dRdTsc;    /**< Number of RDTSC #GPs. */
-
-#ifdef VBOX_WITH_STATISTICS
-    /* R3: Statistics for interrupt handlers (allocated on the hypervisor heap). */
-    R3PTRTYPE(PSTAMCOUNTER) paStatForwardedIRQR3;
-    /* R0: Statistics for interrupt handlers (allocated on the hypervisor heap). */
-    R0PTRTYPE(PSTAMCOUNTER) paStatForwardedIRQR0;
-    /* RC: Statistics for interrupt handlers (allocated on the hypervisor heap). */
-    RCPTRTYPE(PSTAMCOUNTER) paStatForwardedIRQRC;
-#endif
-} TRPM;
-
-/** Pointer to TRPM Data. */
-typedef TRPM *PTRPM;
-
-
-/**
- * Converts a TRPMCPU pointer into a VM pointer.
- * @returns Pointer to the VM structure the TRPMCPU is part of.
- * @param   pTRPM   Pointer to TRPMCPU instance data.
- */
-#define TRPMCPU_2_VM(pTrpmCpu)      ( (PVM)((uint8_t *)(pTrpmCpu) - (pTrpmCpu)->offVM) )
-
-/**
- * Converts a TRPMCPU pointer into a VMCPU pointer.
- * @returns Pointer to the VMCPU structure the TRPMCPU is part of.
- * @param   pTRPM   Pointer to TRPMCPU instance data.
- */
-#define TRPMCPU_2_VMCPU(pTrpmCpu)   ( (PVMCPU)((uint8_t *)(pTrpmCpu) - (pTrpmCpu)->offVMCpu) )
-
-
-/**
- * Per CPU data for TRPM.
- */
-typedef struct TRPMCPU
-{
-    /** Offset into the VM structure.
-     * See TRPMCPU_2_VM(). */
-    uint32_t                offVM;
-    /** Offset into the VMCPU structure.
-     * See TRPMCPU_2_VMCPU().  */
-    uint32_t                offVMCpu;
-
-    /** Active Interrupt or trap vector number.
-     * If not ~0U this indicates that we're currently processing
-     * a interrupt, trap, fault, abort, whatever which have arrived
-     * at that vector number.
-     */
-    RTUINT                  uActiveVector;
-
-    /** Active trap type. */
-    TRPMEVENT               enmActiveType;
-
-    /** Errorcode for the active interrupt/trap. */
-    RTGCUINT                uActiveErrorCode; /**< @todo don't use RTGCUINT */
-
-    /** CR2 at the time of the active exception. */
-    RTGCUINTPTR             uActiveCR2;
-
-    /** Saved trap vector number. */
-    RTGCUINT                uSavedVector; /**< @todo don't use RTGCUINT */
-
-    /** Saved trap type. */
-    TRPMEVENT               enmSavedType;
-
-    /** Saved errorcode. */
-    RTGCUINT                uSavedErrorCode;
-
-    /** Saved cr2. */
-    RTGCUINTPTR             uSavedCR2;
-
-    /** Previous trap vector # - for debugging. */
-    RTGCUINT                uPrevVector;
-} TRPMCPU;
-
-/** Pointer to TRPMCPU Data. */
-typedef TRPMCPU *PTRPMCPU;
-
-#pragma pack()
-
-
-VMMRCDECL(int) trpmRCGuestIDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-VMMRCDECL(int) trpmRCShadowIDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
-
-/**
- * Clear guest trap/interrupt gate handler
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   iTrap       Interrupt/trap number.
- */
-VMMDECL(int) trpmClearGuestTrapHandler(PVM pVM, unsigned iTrap);
-
-
-#ifdef IN_RING3
-
-/**
- * Clear passthrough interrupt gate handler (reset to default handler)
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   iTrap       Trap/interrupt gate number.
- */
-VMMR3DECL(int) trpmR3ClearPassThroughHandler(PVM pVM, unsigned iTrap);
-
-#endif
-
-
-#ifdef IN_RING0
-
-/**
- * Calls the interrupt gate as if we received an interrupt while in Ring-0.
- *
- * @param   uIP     The interrupt gate IP.
- * @param   SelCS   The interrupt gate CS.
- * @param   RSP     The interrupt gate RSP. ~0 if no stack switch should take place. (only AMD64)
- */
-DECLASM(void) trpmR0DispatchHostInterrupt(RTR0UINTPTR uIP, RTSEL SelCS, RTR0UINTPTR RSP);
-
-/**
- * Issues a software interrupt to the specified interrupt vector.
- *
- * @param   uActiveVector   The vector number.
- */
-DECLASM(void) trpmR0DispatchHostInterruptSimple(RTUINT uActiveVector);
-
-#endif /* IN_RING0 */
-
-/** @} */
-
-RT_C_DECLS_END
-
-#endif
Index: unk/src/VBox/VMM/TRPMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/TRPMInternal.mac	(revision 35332)
+++ 	(revision )
@@ -1,123 +1,0 @@
-; $Id$
-;; @file
-; TRPM - Internal header file.
-;
-
-;
-; Copyright (C) 2006-2007 Oracle Corporation
-;
-; This file is part of VirtualBox Open Source Edition (OSE), as
-; available from http://www.virtualbox.org. This file is free software;
-; you can redistribute it and/or modify it under the terms of the GNU
-; General Public License (GPL) as published by the Free Software
-; Foundation, in version 2 as it comes in the "COPYING" file of the
-; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
-; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
-;
-
-%include "VBox/asmdefs.mac"
-%include "VBox/stam.mac"
-
-%define TRPM_TRAP_IN_OP_MASK    0xffff
-%define TRPM_TRAP_IN_MOV_GS     1
-%define TRPM_TRAP_IN_MOV_FS     2
-%define TRPM_TRAP_IN_MOV_ES     3
-%define TRPM_TRAP_IN_MOV_DS     4
-%define TRPM_TRAP_IN_IRET       5
-%define TRPM_TRAP_IN_V86        RT_BIT(30)
-%define TRPM_TRAP_IN_HYPER      RT_BIT(31)
-
-struc TRPM
-    .offVM              resd 1
-    .offTRPMCPU         resd 1
-    .fDisableMonitoring resb 1
-    .fSafeToDropGuestIDTMonitoring resb 1
-    .abPadding1         resb 6
-    .aIdt               resd 512
-    .au32IdtPatched     resd 8
-    .aTmpTrapHandlers   RTRCPTR_RES 256
-
-    .pvMonShwIdtRC            RTRCPTR_RES 1
-    .GuestIdtr          resb 10
-    .au8Padding         resb 2
-
-    .aGuestTrapHandler  RTRCPTR_RES 256
-
-    .StatRCWriteGuestIDTFault  resb STAMCOUNTER_size
-    .StatRCWriteGuestIDTHandled  resb STAMCOUNTER_size
-    .StatSyncIDT        resb STAMPROFILE_size
-    .aStatGCTraps       resb STAMPROFILEADV_size * 0x14
-
-    .StatForwardProfR3        resb STAMPROFILEADV_size
-    .StatForwardProfRZ        resb STAMPROFILEADV_size
-    .StatForwardFailNoHandler resb STAMCOUNTER_size
-    .StatForwardFailPatchAddr resb STAMCOUNTER_size
-    .StatForwardFailR3        resb STAMCOUNTER_size
-    .StatForwardFailRZ        resb STAMCOUNTER_size
-
-    .StatTrap0dDisasm         resb STAMPROFILE_size
-    .StatTrap0dRdTsc          resb STAMCOUNTER_size
-
-%ifdef VBOX_WITH_STATISTICS
-    .paStatForwardedIRQR3  RTR3PTR_RES 1
-    .paStatForwardedIRQR0  RTR0PTR_RES 1
-    .paStatForwardedIRQRC  RTRCPTR_RES 1
-%endif
-endstruc
-
-struc TRPMCPU
-    .offVM              resd 1
-    .offVMCpu           resd 1
-    .uActiveVector      resd 1
-    .enmActiveType      resd 1
-    .uActiveErrorCode   RTGCPTR_RES 1
-    .uActiveCR2         RTGCPTR_RES 1
-    .uSavedVector       RTGCPTR_RES 1
-    .enmSavedType       resd 1
-    .uSavedErrorCode    RTGCPTR_RES 1
-    .uSavedCR2          RTGCPTR_RES 1
-    .uPrevVector        RTGCPTR_RES 1
-endstruc
-
-struc VBOXTSS
-    .selPrev            resw 1
-    .padding1           resw 1
-    .esp0               resd 1
-    .ss0                resw 1
-    .padding_ss0        resw 1
-    .esp1               resd 1
-    .ss1                resw 1
-    .padding_ss1        resw 1
-    .esp2               resd 1
-    .ss2                resw 1
-    .padding_ss2        resw 1
-    .cr3                resd 1
-    .eip                resd 1
-    .eflags             resd 1
-    .eax                resd 1
-    .ecx                resd 1
-    .edx                resd 1
-    .ebx                resd 1
-    .esp                resd 1
-    .ebp                resd 1
-    .esi                resd 1
-    .edi                resd 1
-    .es                 resw 1
-    .padding_es         resw 1
-    .cs                 resw 1
-    .padding_cs         resw 1
-    .ss                 resw 1
-    .padding_ss         resw 1
-    .ds                 resw 1
-    .padding_ds         resw 1
-    .fs                 resw 1
-    .padding_fs         resw 1
-    .gs                 resw 1
-    .padding_gs         resw 1
-    .selLdt             resw 1
-    .padding_ldt        resw 1
-    .fDebugTrap         resw 1
-    .offIoBitmap        resw 1
-    .IntRedirBitmap     resb 32
-endstruc
-
Index: unk/src/VBox/VMM/VBoxVMMDeps.cpp
===================================================================
--- /trunk/src/VBox/VMM/VBoxVMMDeps.cpp	(revision 35332)
+++ 	(revision )
@@ -1,64 +1,0 @@
-/* $Id$ */
-/** @file
- * VBoxVMM link dependencies - drag all we want into the link!
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#include <VBox/pgm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/pdmqueue.h>
-#include <VBox/vm.h>
-#include <VBox/em.h>
-#include <VBox/iom.h>
-#include <VBox/dbgf.h>
-#include <VBox/dbg.h>
-#include <VBox/ftm.h>
-
-VMMR3DECL(int) VMMDoTest(PVM pVM);
-
-/** Just a dummy global structure containing a bunch of
- * function pointers to code which is wanted in the link.
- */
-PFNRT g_apfnDeps[] =
-{
-    (PFNRT)DBGFR3DisasInstrEx,
-    (PFNRT)DBGFR3LogModifyFlags,
-    (PFNRT)DBGFR3StackWalkEnd,
-    (PFNRT)DBGFR3AsSymbolByAddr,
-    (PFNRT)DBGFR3CpuGetMode,
-    (PFNRT)DBGFR3CoreWrite,
-    (PFNRT)DBGFR3MemScan,
-    (PFNRT)DBGFR3RegQueryU8,
-    (PFNRT)EMInterpretInstruction,
-    (PFNRT)IOMIOPortRead,
-    (PFNRT)PDMQueueInsert,
-    (PFNRT)PDMCritSectEnter,
-    (PFNRT)PGMInvalidatePage,
-    (PFNRT)PGMR3DbgR3Ptr2GCPhys,
-    (PFNRT)VMR3Create,
-    (PFNRT)VMMDoTest,
-    (PFNRT)FTMR3PowerOn,
-#ifdef VBOX_WITH_DEBUGGER
-    (PFNRT)DBGCCreate,
-#endif
-#ifdef VBOX_WITH_PAGE_SHARING
-    (PFNRT)PGMR3SharedModuleRegister,
-#endif
-    NULL
-};
Index: unk/src/VBox/VMM/VM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,4252 +1,0 @@
-/* $Id$ */
-/** @file
- * VM - Virtual Machine
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/** @page pg_vm     VM API
- *
- * This is the encapsulating bit.  It provides the APIs that Main and VBoxBFE
- * use to create a VMM instance for running a guest in.  It also provides
- * facilities for queuing request for execution in EMT (serialization purposes
- * mostly) and for reporting error back to the VMM user (Main/VBoxBFE).
- *
- *
- * @section sec_vm_design   Design Critique / Things To Do
- *
- * In hindsight this component is a big design mistake, all this stuff really
- * belongs in the VMM component.  It just seemed like a kind of ok idea at a
- * time when the VMM bit was a kind of vague.  'VM' also happened to be the name
- * of the per-VM instance structure (see vm.h), so it kind of made sense.
- * However as it turned out, VMM(.cpp) is almost empty all it provides in ring-3
- * is some minor functionally and some "routing" services.
- *
- * Fixing this is just a matter of some more or less straight forward
- * refactoring, the question is just when someone will get to it. Moving the EMT
- * would be a good start.
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_VM
-#include <VBox/cfgm.h>
-#include <VBox/vmm.h>
-#include <VBox/gvmm.h>
-#include <VBox/mm.h>
-#include <VBox/cpum.h>
-#include <VBox/selm.h>
-#include <VBox/trpm.h>
-#include <VBox/dbgf.h>
-#include <VBox/pgm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/em.h>
-#include <VBox/rem.h>
-#include <VBox/tm.h>
-#include <VBox/stam.h>
-#include <VBox/patm.h>
-#include <VBox/csam.h>
-#include <VBox/iom.h>
-#include <VBox/ssm.h>
-#include <VBox/ftm.h>
-#include <VBox/hwaccm.h>
-#include "VMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-
-#include <VBox/sup.h>
-#include <VBox/dbg.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-#include <iprt/asm.h>
-#include <iprt/env.h>
-#include <iprt/string.h>
-#include <iprt/time.h>
-#include <iprt/semaphore.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * VM destruction callback registration record.
- */
-typedef struct VMATDTOR
-{
-    /** Pointer to the next record in the list. */
-    struct VMATDTOR        *pNext;
-    /** Pointer to the callback function. */
-    PFNVMATDTOR             pfnAtDtor;
-    /** The user argument. */
-    void                   *pvUser;
-} VMATDTOR;
-/** Pointer to a VM destruction callback registration record. */
-typedef VMATDTOR *PVMATDTOR;
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-/** Pointer to the list of VMs. */
-static PUVM         g_pUVMsHead = NULL;
-
-/** Pointer to the list of at VM destruction callbacks. */
-static PVMATDTOR    g_pVMAtDtorHead = NULL;
-/** Lock the g_pVMAtDtorHead list. */
-#define VM_ATDTOR_LOCK() do { } while (0)
-/** Unlock the g_pVMAtDtorHead list. */
-#define VM_ATDTOR_UNLOCK() do { } while (0)
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int                  vmR3CreateUVM(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods, PUVM *ppUVM);
-static int                  vmR3CreateU(PUVM pUVM, uint32_t cCpus, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM);
-static int                  vmR3InitRing3(PVM pVM, PUVM pUVM);
-static int                  vmR3InitRing0(PVM pVM);
-static int                  vmR3InitGC(PVM pVM);
-static int                  vmR3InitDoCompleted(PVM pVM, VMINITCOMPLETED enmWhat);
-static DECLCALLBACK(size_t) vmR3LogPrefixCallback(PRTLOGGER pLogger, char *pchBuf, size_t cchBuf, void *pvUser);
-static void                 vmR3DestroyUVM(PUVM pUVM, uint32_t cMilliesEMTWait);
-static void                 vmR3AtDtor(PVM pVM);
-static bool                 vmR3ValidateStateTransition(VMSTATE enmStateOld, VMSTATE enmStateNew);
-static void                 vmR3DoAtState(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
-static int                  vmR3TrySetState(PVM pVM, const char *pszWho, unsigned cTransitions, ...);
-static void                 vmR3SetStateLocked(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
-static void                 vmR3SetState(PVM pVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
-static int                  vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...);
-
-
-/**
- * Do global VMM init.
- *
- * @returns VBox status code.
- */
-VMMR3DECL(int)   VMR3GlobalInit(void)
-{
-    /*
-     * Only once.
-     */
-    static bool volatile s_fDone = false;
-    if (s_fDone)
-        return VINF_SUCCESS;
-
-    /*
-     * We're done.
-     */
-    s_fDone = true;
-    return VINF_SUCCESS;
-}
-
-
-
-/**
- * Creates a virtual machine by calling the supplied configuration constructor.
- *
- * On successful returned the VM is powered, i.e. VMR3PowerOn() should be
- * called to start the execution.
- *
- * @returns 0 on success.
- * @returns VBox error code on failure.
- * @param   cCpus               Number of virtual CPUs for the new VM.
- * @param   pVmm2UserMethods    An optional method table that the VMM can use to
- *                              make the user perform various action, like for
- *                              instance state saving.
- * @param   pfnVMAtError        Pointer to callback function for setting VM
- *                              errors. This was added as an implicit call to
- *                              VMR3AtErrorRegister() since there is no way the
- *                              caller can get to the VM handle early enough to
- *                              do this on its own.
- *                              This is called in the context of an EMT.
- * @param   pvUserVM            The user argument passed to pfnVMAtError.
- * @param   pfnCFGMConstructor  Pointer to callback function for constructing the VM configuration tree.
- *                              This is called in the context of an EMT0.
- * @param   pvUserCFGM          The user argument passed to pfnCFGMConstructor.
- * @param   ppVM                Where to store the 'handle' of the created VM.
- */
-VMMR3DECL(int)   VMR3Create(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods,
-                            PFNVMATERROR pfnVMAtError, void *pvUserVM,
-                            PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM,
-                            PVM *ppVM)
-{
-    LogFlow(("VMR3Create: cCpus=%RU32 pVmm2UserMethods=%p pfnVMAtError=%p pvUserVM=%p  pfnCFGMConstructor=%p pvUserCFGM=%p ppVM=%p\n",
-             cCpus, pVmm2UserMethods, pfnVMAtError, pvUserVM, pfnCFGMConstructor, pvUserCFGM, ppVM));
-
-    if (pVmm2UserMethods)
-    {
-        AssertPtrReturn(pVmm2UserMethods, VERR_INVALID_POINTER);
-        AssertReturn(pVmm2UserMethods->u32Magic    == VMM2USERMETHODS_MAGIC,   VERR_INVALID_PARAMETER);
-        AssertReturn(pVmm2UserMethods->u32Version  == VMM2USERMETHODS_VERSION, VERR_INVALID_PARAMETER);
-        AssertPtrReturn(pVmm2UserMethods->pfnSaveState, VERR_INVALID_POINTER);
-        AssertReturn(pVmm2UserMethods->u32EndMagic == VMM2USERMETHODS_MAGIC,   VERR_INVALID_PARAMETER);
-    }
-    AssertPtrNullReturn(pfnVMAtError, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(pfnCFGMConstructor, VERR_INVALID_POINTER);
-    AssertPtrReturn(ppVM, VERR_INVALID_POINTER);
-
-    /*
-     * Because of the current hackiness of the applications
-     * we'll have to initialize global stuff from here.
-     * Later the applications will take care of this in a proper way.
-     */
-    static bool fGlobalInitDone = false;
-    if (!fGlobalInitDone)
-    {
-        int rc = VMR3GlobalInit();
-        if (RT_FAILURE(rc))
-            return rc;
-        fGlobalInitDone = true;
-    }
-
-    /*
-     * Validate input.
-     */
-    AssertLogRelMsgReturn(cCpus > 0 && cCpus <= VMM_MAX_CPU_COUNT, ("%RU32\n", cCpus), VERR_TOO_MANY_CPUS);
-
-    /*
-     * Create the UVM so we can register the at-error callback
-     * and consolidate a bit of cleanup code.
-     */
-    PUVM pUVM = NULL;                   /* shuts up gcc */
-    int rc = vmR3CreateUVM(cCpus, pVmm2UserMethods, &pUVM);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (pfnVMAtError)
-        rc = VMR3AtErrorRegisterU(pUVM, pfnVMAtError, pvUserVM);
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * Initialize the support library creating the session for this VM.
-         */
-        rc = SUPR3Init(&pUVM->vm.s.pSession);
-        if (RT_SUCCESS(rc))
-        {
-            /*
-             * Call vmR3CreateU in the EMT thread and wait for it to finish.
-             *
-             * Note! VMCPUID_ANY is used here because VMR3ReqQueueU would have trouble
-             *       submitting a request to a specific VCPU without a pVM. So, to make
-             *       sure init is running on EMT(0), vmR3EmulationThreadWithId makes sure
-             *       that only EMT(0) is servicing VMCPUID_ANY requests when pVM is NULL.
-             */
-            PVMREQ pReq;
-            rc = VMR3ReqCallU(pUVM, VMCPUID_ANY, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
-                              (PFNRT)vmR3CreateU, 4, pUVM, cCpus, pfnCFGMConstructor, pvUserCFGM);
-            if (RT_SUCCESS(rc))
-            {
-                rc = pReq->iStatus;
-                VMR3ReqFree(pReq);
-                if (RT_SUCCESS(rc))
-                {
-                    /*
-                     * Success!
-                     */
-                    *ppVM = pUVM->pVM;
-                    LogFlow(("VMR3Create: returns VINF_SUCCESS *ppVM=%p\n", *ppVM));
-                    return VINF_SUCCESS;
-                }
-            }
-            else
-                AssertMsgFailed(("VMR3ReqCallU failed rc=%Rrc\n", rc));
-
-            /*
-             * An error occurred during VM creation. Set the error message directly
-             * using the initial callback, as the callback list doesn't exist yet.
-             */
-            const char *pszError = NULL;
-            switch (rc)
-            {
-                case VERR_VMX_IN_VMX_ROOT_MODE:
-#ifdef RT_OS_LINUX
-                    pszError = N_("VirtualBox can't operate in VMX root mode. "
-                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
-#else
-                    pszError = N_("VirtualBox can't operate in VMX root mode. Please close all other virtualization programs.");
-#endif
-                    break;
-
-#ifndef RT_OS_DARWIN
-                case VERR_HWACCM_CONFIG_MISMATCH:
-                    pszError = N_("VT-x/AMD-V is either not available on your host or disabled. "
-                                  "This hardware extension is required by the VM configuration");
-                    break;
-#endif
-
-                case VERR_SVM_IN_USE:
-#ifdef RT_OS_LINUX
-                    pszError = N_("VirtualBox can't enable the AMD-V extension. "
-                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
-#else
-                    pszError = N_("VirtualBox can't enable the AMD-V extension. Please close all other virtualization programs.");
-#endif
-                    break;
-
-                case VERR_VERSION_MISMATCH:
-                    pszError = N_("VMMR0 driver version mismatch. Please terminate all VMs, make sure that "
-                                  "VBoxNetDHCP is not running and try again. If you still get this error, "
-                                  "re-install VirtualBox");
-                    break;
-
-#ifdef RT_OS_LINUX
-                case VERR_SUPDRV_COMPONENT_NOT_FOUND:
-                    pszError = N_("One of the kernel modules was not successfully loaded. Make sure "
-                                  "that no kernel modules from an older version of VirtualBox exist. "
-                                  "Then try to recompile and reload the kernel modules by executing "
-                                  "'/etc/init.d/vboxdrv setup' as root");
-                    break;
-#endif
-
-                case VERR_RAW_MODE_INVALID_SMP:
-                    pszError = N_("VT-x/AMD-V is either not available on your host or disabled. "
-                                  "VirtualBox requires this hardware extension to emulate more than one "
-                                  "guest CPU");
-                    break;
-
-                case VERR_SUPDRV_KERNEL_TOO_OLD_FOR_VTX:
-#ifdef RT_OS_LINUX
-                    pszError = N_("Because the host kernel is too old, VirtualBox cannot enable the VT-x "
-                                  "extension. Either upgrade your kernel to Linux 2.6.13 or later or disable "
-                                  "the VT-x extension in the VM settings. Note that without VT-x you have "
-                                  "to reduce the number of guest CPUs to one");
-#else
-                    pszError = N_("Because the host kernel is too old, VirtualBox cannot enable the VT-x "
-                                  "extension. Either upgrade your kernel or disable the VT-x extension in the "
-                                  "VM settings. Note that without VT-x you have to reduce the number of guest "
-                                  "CPUs to one");
-#endif
-                    break;
-
-                case VERR_PDM_DEVICE_NOT_FOUND:
-                    pszError = N_("A virtual device is configured in the VM settings but the device "
-                                  "implementation is missing.\n"
-                                  "A possible reason for this error is a missing extension pack. Note "
-                                  "that as of VirtualBox 4.0, certain features (for example USB 2.0 "
-                                  "support and remote desktop) are only available from an 'extension "
-                                  "pack' which must be downloaded and installed separately");
-                    break;
-
-                default:
-                    pszError = N_("Unknown error creating VM");
-                    break;
-            }
-            vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
-        }
-        else
-        {
-            /*
-             * An error occurred at support library initialization time (before the
-             * VM could be created). Set the error message directly using the
-             * initial callback, as the callback list doesn't exist yet.
-             */
-            const char *pszError;
-            switch (rc)
-            {
-                case VERR_VM_DRIVER_LOAD_ERROR:
-#ifdef RT_OS_LINUX
-                    pszError = N_("VirtualBox kernel driver not loaded. The vboxdrv kernel module "
-                                  "was either not loaded or /dev/vboxdrv is not set up properly. "
-                                  "Re-setup the kernel module by executing "
-                                  "'/etc/init.d/vboxdrv setup' as root");
-#else
-                    pszError = N_("VirtualBox kernel driver not loaded");
-#endif
-                    break;
-                case VERR_VM_DRIVER_OPEN_ERROR:
-                    pszError = N_("VirtualBox kernel driver cannot be opened");
-                    break;
-                case VERR_VM_DRIVER_NOT_ACCESSIBLE:
-#ifdef VBOX_WITH_HARDENING
-                    /* This should only happen if the executable wasn't hardened - bad code/build. */
-                    pszError = N_("VirtualBox kernel driver not accessible, permission problem. "
-                                  "Re-install VirtualBox. If you are building it yourself, you "
-                                  "should make sure it installed correctly and that the setuid "
-                                  "bit is set on the executables calling VMR3Create.");
-#else
-                    /* This should only happen when mixing builds or with the usual /dev/vboxdrv access issues. */
-# if defined(RT_OS_DARWIN)
-                    pszError = N_("VirtualBox KEXT is not accessible, permission problem. "
-                                  "If you have built VirtualBox yourself, make sure that you do not "
-                                  "have the vboxdrv KEXT from a different build or installation loaded.");
-# elif defined(RT_OS_LINUX)
-                    pszError = N_("VirtualBox kernel driver is not accessible, permission problem. "
-                                  "If you have built VirtualBox yourself, make sure that you do "
-                                  "not have the vboxdrv kernel module from a different build or "
-                                  "installation loaded. Also, make sure the vboxdrv udev rule gives "
-                                  "you the permission you need to access the device.");
-# elif defined(RT_OS_WINDOWS)
-                    pszError = N_("VirtualBox kernel driver is not accessible, permission problem.");
-# else /* solaris, freebsd, ++. */
-                    pszError = N_("VirtualBox kernel module is not accessible, permission problem. "
-                                  "If you have built VirtualBox yourself, make sure that you do "
-                                  "not have the vboxdrv kernel module from a different install loaded.");
-# endif
-#endif
-                    break;
-                case VERR_INVALID_HANDLE: /** @todo track down and fix this error. */
-                case VERR_VM_DRIVER_NOT_INSTALLED:
-#ifdef RT_OS_LINUX
-                    pszError = N_("VirtualBox kernel driver not installed. The vboxdrv kernel module "
-                                  "was either not loaded or /dev/vboxdrv was not created for some "
-                                  "reason. Re-setup the kernel module by executing "
-                                  "'/etc/init.d/vboxdrv setup' as root");
-#else
-                    pszError = N_("VirtualBox kernel driver not installed");
-#endif
-                    break;
-                case VERR_NO_MEMORY:
-                    pszError = N_("VirtualBox support library out of memory");
-                    break;
-                case VERR_VERSION_MISMATCH:
-                case VERR_VM_DRIVER_VERSION_MISMATCH:
-                    pszError = N_("The VirtualBox support driver which is running is from a different "
-                                  "version of VirtualBox.  You can correct this by stopping all "
-                                  "running instances of VirtualBox and reinstalling the software.");
-                    break;
-                default:
-                    pszError = N_("Unknown error initializing kernel driver");
-                    AssertMsgFailed(("Add error message for rc=%d (%Rrc)\n", rc, rc));
-            }
-            vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
-        }
-    }
-
-    /* cleanup */
-    vmR3DestroyUVM(pUVM, 2000);
-    LogFlow(("VMR3Create: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Creates the UVM.
- *
- * This will not initialize the support library even if vmR3DestroyUVM
- * will terminate that.
- *
- * @returns VBox status code.
- * @param   cCpus               Number of virtual CPUs
- * @param   pVmm2UserMethods    Pointer to the optional VMM -> User method
- *                              table.
- * @param   ppUVM               Where to store the UVM pointer.
- */
-static int vmR3CreateUVM(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods, PUVM *ppUVM)
-{
-    uint32_t i;
-
-    /*
-     * Create and initialize the UVM.
-     */
-    PUVM pUVM = (PUVM)RTMemPageAllocZ(RT_OFFSETOF(UVM, aCpus[cCpus]));
-    AssertReturn(pUVM, VERR_NO_MEMORY);
-    pUVM->u32Magic          = UVM_MAGIC;
-    pUVM->cCpus             = cCpus;
-    pUVM->pVmm2UserMethods  = pVmm2UserMethods;
-
-    AssertCompile(sizeof(pUVM->vm.s) <= sizeof(pUVM->vm.padding));
-
-    pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
-    pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
-    pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
-
-    pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_BOOTSTRAP;
-
-    /* Initialize the VMCPU array in the UVM. */
-    for (i = 0; i < cCpus; i++)
-    {
-        pUVM->aCpus[i].pUVM   = pUVM;
-        pUVM->aCpus[i].idCpu  = i;
-    }
-
-    /* Allocate a TLS entry to store the VMINTUSERPERVMCPU pointer. */
-    int rc = RTTlsAllocEx(&pUVM->vm.s.idxTLS, NULL);
-    AssertRC(rc);
-    if (RT_SUCCESS(rc))
-    {
-        /* Allocate a halt method event semaphore for each VCPU. */
-        for (i = 0; i < cCpus; i++)
-            pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
-        for (i = 0; i < cCpus; i++)
-        {
-            rc = RTSemEventCreate(&pUVM->aCpus[i].vm.s.EventSemWait);
-            if (RT_FAILURE(rc))
-                break;
-        }
-        if (RT_SUCCESS(rc))
-        {
-            rc = RTCritSectInit(&pUVM->vm.s.AtStateCritSect);
-            if (RT_SUCCESS(rc))
-            {
-                rc = RTCritSectInit(&pUVM->vm.s.AtErrorCritSect);
-                if (RT_SUCCESS(rc))
-                {
-                    /*
-                     * Init fundamental (sub-)components - STAM, MMR3Heap and PDMLdr.
-                     */
-                    rc = STAMR3InitUVM(pUVM);
-                    if (RT_SUCCESS(rc))
-                    {
-                        rc = MMR3InitUVM(pUVM);
-                        if (RT_SUCCESS(rc))
-                        {
-                            rc = PDMR3InitUVM(pUVM);
-                            if (RT_SUCCESS(rc))
-                            {
-                                /*
-                                 * Start the emulation threads for all VMCPUs.
-                                 */
-                                for (i = 0; i < cCpus; i++)
-                                {
-                                    rc = RTThreadCreateF(&pUVM->aCpus[i].vm.s.ThreadEMT, vmR3EmulationThread, &pUVM->aCpus[i], _1M,
-                                                         RTTHREADTYPE_EMULATION, RTTHREADFLAGS_WAITABLE,
-                                                         cCpus > 1 ? "EMT-%u" : "EMT", i);
-                                    if (RT_FAILURE(rc))
-                                        break;
-
-                                    pUVM->aCpus[i].vm.s.NativeThreadEMT = RTThreadGetNative(pUVM->aCpus[i].vm.s.ThreadEMT);
-                                }
-
-                                if (RT_SUCCESS(rc))
-                                {
-                                    *ppUVM = pUVM;
-                                    return VINF_SUCCESS;
-                                }
-
-                                /* bail out. */
-                                while (i-- > 0)
-                                {
-                                    /** @todo rainy day: terminate the EMTs. */
-                                }
-                                PDMR3TermUVM(pUVM);
-                            }
-                            MMR3TermUVM(pUVM);
-                        }
-                        STAMR3TermUVM(pUVM);
-                    }
-                    RTCritSectDelete(&pUVM->vm.s.AtErrorCritSect);
-                }
-                RTCritSectDelete(&pUVM->vm.s.AtStateCritSect);
-            }
-        }
-        for (i = 0; i < cCpus; i++)
-        {
-            RTSemEventDestroy(pUVM->aCpus[i].vm.s.EventSemWait);
-            pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
-        }
-        RTTlsFree(pUVM->vm.s.idxTLS);
-    }
-    RTMemPageFree(pUVM, RT_OFFSETOF(UVM, aCpus[pUVM->cCpus]));
-    return rc;
-}
-
-
-/**
- * Creates and initializes the VM.
- *
- * @thread EMT
- */
-static int vmR3CreateU(PUVM pUVM, uint32_t cCpus, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM)
-{
-    int rc = VINF_SUCCESS;
-
-    /*
-     * Load the VMMR0.r0 module so that we can call GVMMR0CreateVM.
-     */
-    rc = PDMR3LdrLoadVMMR0U(pUVM);
-    if (RT_FAILURE(rc))
-    {
-        /** @todo we need a cleaner solution for this (VERR_VMX_IN_VMX_ROOT_MODE).
-          * bird: what about moving the message down here? Main picks the first message, right? */
-        if (rc == VERR_VMX_IN_VMX_ROOT_MODE)
-            return rc;  /* proper error message set later on */
-        return vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("Failed to load VMMR0.r0"));
-    }
-
-    /*
-     * Request GVMM to create a new VM for us.
-     */
-    GVMMCREATEVMREQ CreateVMReq;
-    CreateVMReq.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
-    CreateVMReq.Hdr.cbReq       = sizeof(CreateVMReq);
-    CreateVMReq.pSession        = pUVM->vm.s.pSession;
-    CreateVMReq.pVMR0           = NIL_RTR0PTR;
-    CreateVMReq.pVMR3           = NULL;
-    CreateVMReq.cCpus           = cCpus;
-    rc = SUPR3CallVMMR0Ex(NIL_RTR0PTR, NIL_VMCPUID, VMMR0_DO_GVMM_CREATE_VM, 0, &CreateVMReq.Hdr);
-    if (RT_SUCCESS(rc))
-    {
-        PVM pVM = pUVM->pVM = CreateVMReq.pVMR3;
-        AssertRelease(VALID_PTR(pVM));
-        AssertRelease(pVM->pVMR0 == CreateVMReq.pVMR0);
-        AssertRelease(pVM->pSession == pUVM->vm.s.pSession);
-        AssertRelease(pVM->cCpus == cCpus);
-        AssertRelease(pVM->uCpuExecutionCap == 100);
-        AssertRelease(pVM->offVMCPU == RT_UOFFSETOF(VM, aCpus));
-
-        Log(("VMR3Create: Created pUVM=%p pVM=%p pVMR0=%p hSelf=%#x cCpus=%RU32\n",
-             pUVM, pVM, pVM->pVMR0, pVM->hSelf, pVM->cCpus));
-
-        /*
-         * Initialize the VM structure and our internal data (VMINT).
-         */
-        pVM->pUVM = pUVM;
-
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            pVM->aCpus[i].pUVCpu            = &pUVM->aCpus[i];
-            pVM->aCpus[i].idCpu             = i;
-            pVM->aCpus[i].hNativeThread     = pUVM->aCpus[i].vm.s.NativeThreadEMT;
-            Assert(pVM->aCpus[i].hNativeThread != NIL_RTNATIVETHREAD);
-            /* hNativeThreadR0 is initialized on EMT registration. */
-            pUVM->aCpus[i].pVCpu            = &pVM->aCpus[i];
-            pUVM->aCpus[i].pVM              = pVM;
-        }
-
-
-        /*
-         * Init the configuration.
-         */
-        rc = CFGMR3Init(pVM, pfnCFGMConstructor, pvUserCFGM);
-        if (RT_SUCCESS(rc))
-        {
-            PCFGMNODE pRoot = CFGMR3GetRoot(pVM);
-            rc = CFGMR3QueryBoolDef(pRoot, "HwVirtExtForced", &pVM->fHwVirtExtForced, false);
-            if (RT_SUCCESS(rc) && pVM->fHwVirtExtForced)
-                pVM->fHWACCMEnabled = true;
-
-            /*
-             * If executing in fake suplib mode disable RR3 and RR0 in the config.
-             */
-            const char *psz = RTEnvGet("VBOX_SUPLIB_FAKE");
-            if (psz && !strcmp(psz, "fake"))
-            {
-                CFGMR3RemoveValue(pRoot, "RawR3Enabled");
-                CFGMR3InsertInteger(pRoot, "RawR3Enabled", 0);
-                CFGMR3RemoveValue(pRoot, "RawR0Enabled");
-                CFGMR3InsertInteger(pRoot, "RawR0Enabled", 0);
-            }
-
-            /*
-             * Make sure the CPU count in the config data matches.
-             */
-            if (RT_SUCCESS(rc))
-            {
-                uint32_t cCPUsCfg;
-                rc = CFGMR3QueryU32Def(pRoot, "NumCPUs", &cCPUsCfg, 1);
-                AssertLogRelMsgRC(rc, ("Configuration error: Querying \"NumCPUs\" as integer failed, rc=%Rrc\n", rc));
-                if (RT_SUCCESS(rc) && cCPUsCfg != cCpus)
-                {
-                    AssertLogRelMsgFailed(("Configuration error: \"NumCPUs\"=%RU32 and VMR3CreateVM::cCpus=%RU32 does not match!\n",
-                                           cCPUsCfg, cCpus));
-                    rc = VERR_INVALID_PARAMETER;
-                }
-            }
-            if (RT_SUCCESS(rc))
-            {
-                rc = CFGMR3QueryU32Def(pRoot, "CpuExecutionCap", &pVM->uCpuExecutionCap, 100);
-                AssertLogRelMsgRC(rc, ("Configuration error: Querying \"CpuExecutionCap\" as integer failed, rc=%Rrc\n", rc));
-
-                /*
-                 * Init the ring-3 components and ring-3 per cpu data, finishing it off
-                 * by a relocation round (intermediate context finalization will do this).
-                 */
-                rc = vmR3InitRing3(pVM, pUVM);
-                if (RT_SUCCESS(rc))
-                {
-                    rc = PGMR3FinalizeMappings(pVM);
-                    if (RT_SUCCESS(rc))
-                    {
-
-                        LogFlow(("Ring-3 init succeeded\n"));
-
-                        /*
-                         * Init the Ring-0 components.
-                         */
-                        rc = vmR3InitRing0(pVM);
-                        if (RT_SUCCESS(rc))
-                        {
-                            /* Relocate again, because some switcher fixups depends on R0 init results. */
-                            VMR3Relocate(pVM, 0);
-
-#ifdef VBOX_WITH_DEBUGGER
-                            /*
-                             * Init the tcp debugger console if we're building
-                             * with debugger support.
-                             */
-                            void *pvUser = NULL;
-                            rc = DBGCTcpCreate(pVM, &pvUser);
-                            if (    RT_SUCCESS(rc)
-                                ||  rc == VERR_NET_ADDRESS_IN_USE)
-                            {
-                                pUVM->vm.s.pvDBGC = pvUser;
-#endif
-                                /*
-                                 * Init the Guest Context components.
-                                 */
-                                rc = vmR3InitGC(pVM);
-                                if (RT_SUCCESS(rc))
-                                {
-                                    /*
-                                     * Now we can safely set the VM halt method to default.
-                                     */
-                                    rc = vmR3SetHaltMethodU(pUVM, VMHALTMETHOD_DEFAULT);
-                                    if (RT_SUCCESS(rc))
-                                    {
-                                        /*
-                                         * Set the state and link into the global list.
-                                         */
-                                        vmR3SetState(pVM, VMSTATE_CREATED, VMSTATE_CREATING);
-                                        pUVM->pNext = g_pUVMsHead;
-                                        g_pUVMsHead = pUVM;
-
-#ifdef LOG_ENABLED
-                                        RTLogSetCustomPrefixCallback(NULL, vmR3LogPrefixCallback, pUVM);
-#endif
-                                        return VINF_SUCCESS;
-                                    }
-                                }
-#ifdef VBOX_WITH_DEBUGGER
-                                DBGCTcpTerminate(pVM, pUVM->vm.s.pvDBGC);
-                                pUVM->vm.s.pvDBGC = NULL;
-                            }
-#endif
-                            //..
-                        }
-                    }
-                    vmR3Destroy(pVM);
-                }
-            }
-            //..
-
-            /* Clean CFGM. */
-            int rc2 = CFGMR3Term(pVM);
-            AssertRC(rc2);
-        }
-
-        /*
-         * Do automatic cleanups while the VM structure is still alive and all
-         * references to it are still working.
-         */
-        PDMR3CritSectTerm(pVM);
-
-        /*
-         * Drop all references to VM and the VMCPU structures, then
-         * tell GVMM to destroy the VM.
-         */
-        pUVM->pVM = NULL;
-        for (VMCPUID i = 0; i < pUVM->cCpus; i++)
-        {
-            pUVM->aCpus[i].pVM = NULL;
-            pUVM->aCpus[i].pVCpu = NULL;
-        }
-        Assert(pUVM->vm.s.enmHaltMethod == VMHALTMETHOD_BOOTSTRAP);
-
-        if (pUVM->cCpus > 1)
-        {
-            /* Poke the other EMTs since they may have stale pVM and pVCpu references
-               on the stack (see VMR3WaitU for instance) if they've been awakened after
-               VM creation. */
-            for (VMCPUID i = 1; i < pUVM->cCpus; i++)
-                VMR3NotifyCpuFFU(&pUVM->aCpus[i], 0);
-            RTThreadSleep(RT_MIN(100 + 25 *(pUVM->cCpus - 1), 500)); /* very sophisticated */
-        }
-
-        int rc2 = SUPR3CallVMMR0Ex(CreateVMReq.pVMR0, 0 /*idCpu*/, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
-        AssertRC(rc2);
-    }
-    else
-        vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("VM creation failed (GVMM)"));
-
-    LogFlow(("vmR3CreateU: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Register the calling EMT with GVM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   idCpu       The Virtual CPU ID.
- */
-static DECLCALLBACK(int) vmR3RegisterEMT(PVM pVM, VMCPUID idCpu)
-{
-    Assert(VMMGetCpuId(pVM) == idCpu);
-    int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, idCpu, VMMR0_DO_GVMM_REGISTER_VMCPU, 0, NULL);
-    if (RT_FAILURE(rc))
-        LogRel(("idCpu=%u rc=%Rrc\n", idCpu, rc));
-    return rc;
-}
-
-
-/**
- * Initializes all R3 components of the VM
- */
-static int vmR3InitRing3(PVM pVM, PUVM pUVM)
-{
-    int rc;
-
-    /*
-     * Register the other EMTs with GVM.
-     */
-    for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
-    {
-        rc = VMR3ReqCallWaitU(pUVM, idCpu, (PFNRT)vmR3RegisterEMT, 2, pVM, idCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Init all R3 components, the order here might be important.
-     */
-    rc = MMR3Init(pVM);
-    if (RT_SUCCESS(rc))
-    {
-        STAM_REG(pVM, &pVM->StatTotalInGC,          STAMTYPE_PROFILE_ADV, "/PROF/VM/InGC",          STAMUNIT_TICKS_PER_CALL,    "Profiling the total time spent in GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherToGC,       STAMTYPE_PROFILE_ADV, "/PROF/VM/SwitchToGC",    STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherToHC,       STAMTYPE_PROFILE_ADV, "/PROF/VM/SwitchToHC",    STAMUNIT_TICKS_PER_CALL,    "Profiling switching to HC.");
-        STAM_REG(pVM, &pVM->StatSwitcherSaveRegs,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/SaveRegs", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherSysEnter,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/SysEnter", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherDebug,      STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Debug",    STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherCR0,        STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/CR0",  STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherCR4,        STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/CR4",  STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherLgdt,       STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Lgdt", STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherLidt,       STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Lidt", STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherLldt,       STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Lldt", STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherTSS,        STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/TSS",  STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherJmpCR3,     STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/JmpCR3",   STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
-        STAM_REG(pVM, &pVM->StatSwitcherRstrRegs,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/RstrRegs", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
-
-        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-        {
-            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltYield,           STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state yielding.",  "/PROF/VM/CPU%d/Halt/Yield", idCpu);
-            AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlock,           STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state blocking.",  "/PROF/VM/CPU%d/Halt/Block", idCpu);
-            AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockOverslept,  STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time wasted by blocking too long.", "/PROF/VM/CPU%d/Halt/BlockOverslept", idCpu);
-            AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockInsomnia,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time slept when returning to early.","/PROF/VM/CPU%d/Halt/BlockInsomnia", idCpu);
-            AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockOnTime,     STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time slept on time.",                "/PROF/VM/CPU%d/Halt/BlockOnTime", idCpu);
-            AssertRC(rc);
-            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltTimers,          STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state timer tasks.", "/PROF/VM/CPU%d/Halt/Timers", idCpu);
-            AssertRC(rc);
-        }
-
-        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocNew,   STAMTYPE_COUNTER,     "/VM/Req/AllocNew",       STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a new packet.");
-        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRaces, STAMTYPE_COUNTER,     "/VM/Req/AllocRaces",     STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc causing races.");
-        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRecycled, STAMTYPE_COUNTER,  "/VM/Req/AllocRecycled",  STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a recycled packet.");
-        STAM_REG(pVM, &pUVM->vm.s.StatReqFree,       STAMTYPE_COUNTER,     "/VM/Req/Free",           STAMUNIT_OCCURENCES,        "Number of VMR3ReqFree calls.");
-        STAM_REG(pVM, &pUVM->vm.s.StatReqFreeOverflow, STAMTYPE_COUNTER,   "/VM/Req/FreeOverflow",   STAMUNIT_OCCURENCES,        "Number of times the request was actually freed.");
-        STAM_REG(pVM, &pUVM->vm.s.StatReqProcessed,  STAMTYPE_COUNTER,     "/VM/Req/Processed",      STAMUNIT_OCCURENCES,        "Number of processed requests (any queue).");
-        STAM_REG(pVM, &pUVM->vm.s.StatReqMoreThan1,  STAMTYPE_COUNTER,     "/VM/Req/MoreThan1",      STAMUNIT_OCCURENCES,        "Number of times there are more than one request on the queue when processing it.");
-        STAM_REG(pVM, &pUVM->vm.s.StatReqPushBackRaces, STAMTYPE_COUNTER,  "/VM/Req/PushBackRaces",  STAMUNIT_OCCURENCES,        "Number of push back races.");
-
-        rc = CPUMR3Init(pVM);
-        if (RT_SUCCESS(rc))
-        {
-            rc = HWACCMR3Init(pVM);
-            if (RT_SUCCESS(rc))
-            {
-                rc = PGMR3Init(pVM);
-                if (RT_SUCCESS(rc))
-                {
-                    rc = REMR3Init(pVM);
-                    if (RT_SUCCESS(rc))
-                    {
-                        rc = MMR3InitPaging(pVM);
-                        if (RT_SUCCESS(rc))
-                            rc = TMR3Init(pVM);
-                        if (RT_SUCCESS(rc))
-                        {
-                            rc = FTMR3Init(pVM);
-                            if (RT_SUCCESS(rc))
-                            {
-                                rc = VMMR3Init(pVM);
-                                if (RT_SUCCESS(rc))
-                                {
-                                    rc = SELMR3Init(pVM);
-                                    if (RT_SUCCESS(rc))
-                                    {
-                                        rc = TRPMR3Init(pVM);
-                                        if (RT_SUCCESS(rc))
-                                        {
-                                            rc = CSAMR3Init(pVM);
-                                            if (RT_SUCCESS(rc))
-                                            {
-                                                rc = PATMR3Init(pVM);
-                                                if (RT_SUCCESS(rc))
-                                                {
-                                                    rc = IOMR3Init(pVM);
-                                                    if (RT_SUCCESS(rc))
-                                                    {
-                                                        rc = EMR3Init(pVM);
-                                                        if (RT_SUCCESS(rc))
-                                                        {
-                                                            rc = DBGFR3Init(pVM);
-                                                            if (RT_SUCCESS(rc))
-                                                            {
-                                                                rc = PDMR3Init(pVM);
-                                                                if (RT_SUCCESS(rc))
-                                                                {
-                                                                    rc = PGMR3InitDynMap(pVM);
-                                                                    if (RT_SUCCESS(rc))
-                                                                        rc = MMR3HyperInitFinalize(pVM);
-                                                                    if (RT_SUCCESS(rc))
-                                                                        rc = PATMR3InitFinalize(pVM);
-                                                                    if (RT_SUCCESS(rc))
-                                                                        rc = PGMR3InitFinalize(pVM);
-                                                                    if (RT_SUCCESS(rc))
-                                                                        rc = SELMR3InitFinalize(pVM);
-                                                                    if (RT_SUCCESS(rc))
-                                                                        rc = TMR3InitFinalize(pVM);
-                                                                    if (RT_SUCCESS(rc))
-                                                                        rc = REMR3InitFinalize(pVM);
-                                                                    if (RT_SUCCESS(rc))
-                                                                        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_RING3);
-                                                                    if (RT_SUCCESS(rc))
-                                                                    {
-                                                                        LogFlow(("vmR3InitRing3: returns %Rrc\n", VINF_SUCCESS));
-                                                                        return VINF_SUCCESS;
-                                                                    }
-                                                                    int rc2 = PDMR3Term(pVM);
-                                                                    AssertRC(rc2);
-                                                                }
-                                                                int rc2 = DBGFR3Term(pVM);
-                                                                AssertRC(rc2);
-                                                            }
-                                                            int rc2 = EMR3Term(pVM);
-                                                            AssertRC(rc2);
-                                                        }
-                                                        int rc2 = IOMR3Term(pVM);
-                                                        AssertRC(rc2);
-                                                    }
-                                                    int rc2 = PATMR3Term(pVM);
-                                                    AssertRC(rc2);
-                                                }
-                                                int rc2 = CSAMR3Term(pVM);
-                                                AssertRC(rc2);
-                                            }
-                                            int rc2 = TRPMR3Term(pVM);
-                                            AssertRC(rc2);
-                                        }
-                                        int rc2 = SELMR3Term(pVM);
-                                        AssertRC(rc2);
-                                    }
-                                    int rc2 = VMMR3Term(pVM);
-                                    AssertRC(rc2);
-                                }
-                                int rc2 = FTMR3Term(pVM);
-                                AssertRC(rc2);
-                            }
-                            int rc2 = TMR3Term(pVM);
-                            AssertRC(rc2);
-                        }
-                        int rc2 = REMR3Term(pVM);
-                        AssertRC(rc2);
-                    }
-                    int rc2 = PGMR3Term(pVM);
-                    AssertRC(rc2);
-                }
-                int rc2 = HWACCMR3Term(pVM);
-                AssertRC(rc2);
-            }
-            //int rc2 = CPUMR3Term(pVM);
-            //AssertRC(rc2);
-        }
-        /* MMR3Term is not called here because it'll kill the heap. */
-    }
-
-    LogFlow(("vmR3InitRing3: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Initializes all R0 components of the VM
- */
-static int vmR3InitRing0(PVM pVM)
-{
-    LogFlow(("vmR3InitRing0:\n"));
-
-    /*
-     * Check for FAKE suplib mode.
-     */
-    int rc = VINF_SUCCESS;
-    const char *psz = RTEnvGet("VBOX_SUPLIB_FAKE");
-    if (!psz || strcmp(psz, "fake"))
-    {
-        /*
-         * Call the VMMR0 component and let it do the init.
-         */
-        rc = VMMR3InitR0(pVM);
-    }
-    else
-        Log(("vmR3InitRing0: skipping because of VBOX_SUPLIB_FAKE=fake\n"));
-
-    /*
-     * Do notifications and return.
-     */
-    if (RT_SUCCESS(rc))
-        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_RING0);
-
-    /** @todo Move this to the VMINITCOMPLETED_RING0 notification handler. */
-    if (RT_SUCCESS(rc))
-    {
-        rc = HWACCMR3InitFinalizeR0(pVM);
-        CPUMR3SetHWVirtEx(pVM, HWACCMIsEnabled(pVM));
-    }
-
-    LogFlow(("vmR3InitRing0: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Initializes all GC components of the VM
- */
-static int vmR3InitGC(PVM pVM)
-{
-    LogFlow(("vmR3InitGC:\n"));
-
-    /*
-     * Check for FAKE suplib mode.
-     */
-    int rc = VINF_SUCCESS;
-    const char *psz = RTEnvGet("VBOX_SUPLIB_FAKE");
-    if (!psz || strcmp(psz, "fake"))
-    {
-        /*
-         * Call the VMMR0 component and let it do the init.
-         */
-        rc = VMMR3InitRC(pVM);
-    }
-    else
-        Log(("vmR3InitGC: skipping because of VBOX_SUPLIB_FAKE=fake\n"));
-
-    /*
-     * Do notifications and return.
-     */
-    if (RT_SUCCESS(rc))
-        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_GC);
-    LogFlow(("vmR3InitGC: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Do init completed notifications.
- *
- * @returns VBox status code.
- * @param   pVM         The VM handle.
- * @param   enmWhat     What's completed.
- */
-static int vmR3InitDoCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
-{
-    int rc = VMMR3InitCompleted(pVM, enmWhat);
-    if (RT_SUCCESS(rc))
-        rc = HWACCMR3InitCompleted(pVM, enmWhat);
-    return rc;
-}
-
-
-/**
- * Logger callback for inserting a custom prefix.
- *
- * @returns Number of chars written.
- * @param   pLogger             The logger.
- * @param   pchBuf              The output buffer.
- * @param   cchBuf              The output buffer size.
- * @param   pvUser              Pointer to the UVM structure.
- */
-static DECLCALLBACK(size_t) vmR3LogPrefixCallback(PRTLOGGER pLogger, char *pchBuf, size_t cchBuf, void *pvUser)
-{
-    AssertReturn(cchBuf >= 2, 0);
-    PUVM        pUVM   = (PUVM)pvUser;
-    PUVMCPU     pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
-    if (pUVCpu)
-    {
-        static const char s_szHex[17] = "0123456789abcdef";
-        VMCPUID const     idCpu       = pUVCpu->idCpu;
-        pchBuf[1] = s_szHex[ idCpu       & 15];
-        pchBuf[0] = s_szHex[(idCpu >> 4) & 15];
-    }
-    else
-    {
-        pchBuf[0] = 'x';
-        pchBuf[1] = 'y';
-    }
-
-    return 2;
-}
-
-
-/**
- * Calls the relocation functions for all VMM components so they can update
- * any GC pointers. When this function is called all the basic VM members
- * have been updated  and the actual memory relocation have been done
- * by the PGM/MM.
- *
- * This is used both on init and on runtime relocations.
- *
- * @param   pVM         VM handle.
- * @param   offDelta    Relocation delta relative to old location.
- */
-VMMR3DECL(void)   VMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    LogFlow(("VMR3Relocate: offDelta=%RGv\n", offDelta));
-
-    /*
-     * The order here is very important!
-     */
-    PGMR3Relocate(pVM, offDelta);
-    PDMR3LdrRelocateU(pVM->pUVM, offDelta);
-    PGMR3Relocate(pVM, 0);              /* Repeat after PDM relocation. */
-    CPUMR3Relocate(pVM);
-    HWACCMR3Relocate(pVM);
-    SELMR3Relocate(pVM);
-    VMMR3Relocate(pVM, offDelta);
-    SELMR3Relocate(pVM);                /* !hack! fix stack! */
-    TRPMR3Relocate(pVM, offDelta);
-    PATMR3Relocate(pVM);
-    CSAMR3Relocate(pVM, offDelta);
-    IOMR3Relocate(pVM, offDelta);
-    EMR3Relocate(pVM);
-    TMR3Relocate(pVM, offDelta);
-    DBGFR3Relocate(pVM, offDelta);
-    PDMR3Relocate(pVM, offDelta);
-}
-
-
-/**
- * EMT rendezvous worker for VMR3PowerOn.
- *
- * @returns VERR_VM_INVALID_VM_STATE or VINF_SUCCESS. (This is a strict return
- *          code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          Ignored.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3PowerOn(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    LogFlow(("vmR3PowerOn: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
-    Assert(!pvUser); NOREF(pvUser);
-
-    /*
-     * The first thread thru here tries to change the state.  We shouldn't be
-     * called again if this fails.
-     */
-    if (pVCpu->idCpu == pVM->cCpus - 1)
-    {
-        int rc = vmR3TrySetState(pVM, "VMR3PowerOn", 1, VMSTATE_POWERING_ON, VMSTATE_CREATED);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    AssertMsgReturn(enmVMState == VMSTATE_POWERING_ON,
-                    ("%s\n", VMR3GetStateName(enmVMState)),
-                    VERR_INTERNAL_ERROR_4);
-
-    /*
-     * All EMTs changes their state to started.
-     */
-    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
-
-    /*
-     * EMT(0) is last thru here and it will make the notification calls
-     * and advance the state.
-     */
-    if (pVCpu->idCpu == 0)
-    {
-        PDMR3PowerOn(pVM);
-        vmR3SetState(pVM, VMSTATE_RUNNING, VMSTATE_POWERING_ON);
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Powers on the virtual machine.
- *
- * @returns VBox status code.
- *
- * @param   pVM         The VM to power on.
- *
- * @thread      Any thread.
- * @vmstate     Created
- * @vmstateto   PoweringOn+Running
- */
-VMMR3DECL(int) VMR3PowerOn(PVM pVM)
-{
-    LogFlow(("VMR3PowerOn: pVM=%p\n", pVM));
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Gather all the EMTs to reduce the init TSC drift and keep
-     * the state changing APIs a bit uniform.
-     */
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
-                                vmR3PowerOn, NULL);
-    LogFlow(("VMR3PowerOn: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Does the suspend notifications.
- *
- * @param  pVM      The VM handle.
- * @thread  EMT(0)
- */
-static void vmR3SuspendDoWork(PVM pVM)
-{
-    PDMR3Suspend(pVM);
-}
-
-
-/**
- * EMT rendezvous worker for VMR3Suspend.
- *
- * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_SUSPEND. (This is a strict
- *          return code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          Ignored.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3Suspend(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    LogFlow(("vmR3Suspend: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
-    Assert(!pvUser); NOREF(pvUser);
-
-    /*
-     * The first EMT switches the state to suspending.  If this fails because
-     * something was racing us in one way or the other, there will be no more
-     * calls and thus the state assertion below is not going to annoy anyone.
-     */
-    if (pVCpu->idCpu == pVM->cCpus - 1)
-    {
-        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 2,
-                                 VMSTATE_SUSPENDING,        VMSTATE_RUNNING,
-                                 VMSTATE_SUSPENDING_EXT_LS, VMSTATE_RUNNING_LS);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    AssertMsgReturn(    enmVMState == VMSTATE_SUSPENDING
-                    ||  enmVMState == VMSTATE_SUSPENDING_EXT_LS,
-                    ("%s\n", VMR3GetStateName(enmVMState)),
-                    VERR_INTERNAL_ERROR_4);
-
-    /*
-     * EMT(0) does the actually suspending *after* all the other CPUs have
-     * been thru here.
-     */
-    if (pVCpu->idCpu == 0)
-    {
-        vmR3SuspendDoWork(pVM);
-
-        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 2,
-                                 VMSTATE_SUSPENDED,        VMSTATE_SUSPENDING,
-                                 VMSTATE_SUSPENDED_EXT_LS, VMSTATE_SUSPENDING_EXT_LS);
-        if (RT_FAILURE(rc))
-            return VERR_INTERNAL_ERROR_3;
-    }
-
-    return VINF_EM_SUSPEND;
-}
-
-
-/**
- * Suspends a running VM.
- *
- * @returns VBox status code. When called on EMT, this will be a strict status
- *          code that has to be propagated up the call stack.
- *
- * @param   pVM     The VM to suspend.
- *
- * @thread      Any thread.
- * @vmstate     Running or RunningLS
- * @vmstateto   Suspending + Suspended or SuspendingExtLS + SuspendedExtLS
- */
-VMMR3DECL(int) VMR3Suspend(PVM pVM)
-{
-    LogFlow(("VMR3Suspend: pVM=%p\n", pVM));
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Gather all the EMTs to make sure there are no races before
-     * changing the VM state.
-     */
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
-                                vmR3Suspend, NULL);
-    LogFlow(("VMR3Suspend: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * EMT rendezvous worker for VMR3Resume.
- *
- * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_RESUME. (This is a strict
- *          return code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          Ignored.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3Resume(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    LogFlow(("vmR3Resume: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
-    Assert(!pvUser); NOREF(pvUser);
-
-    /*
-     * The first thread thru here tries to change the state.  We shouldn't be
-     * called again if this fails.
-     */
-    if (pVCpu->idCpu == pVM->cCpus - 1)
-    {
-        int rc = vmR3TrySetState(pVM, "VMR3Resume", 1, VMSTATE_RESUMING, VMSTATE_SUSPENDED);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    AssertMsgReturn(enmVMState == VMSTATE_RESUMING,
-                    ("%s\n", VMR3GetStateName(enmVMState)),
-                    VERR_INTERNAL_ERROR_4);
-
-#if 0
-    /*
-     * All EMTs changes their state to started.
-     */
-    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
-#endif
-
-    /*
-     * EMT(0) is last thru here and it will make the notification calls
-     * and advance the state.
-     */
-    if (pVCpu->idCpu == 0)
-    {
-        PDMR3Resume(pVM);
-        vmR3SetState(pVM, VMSTATE_RUNNING, VMSTATE_RESUMING);
-        pVM->vm.s.fTeleportedAndNotFullyResumedYet = false;
-    }
-
-    return VINF_EM_RESUME;
-}
-
-
-/**
- * Resume VM execution.
- *
- * @returns VBox status code. When called on EMT, this will be a strict status
- *          code that has to be propagated up the call stack.
- *
- * @param   pVM         The VM to resume.
- *
- * @thread      Any thread.
- * @vmstate     Suspended
- * @vmstateto   Running
- */
-VMMR3DECL(int) VMR3Resume(PVM pVM)
-{
-    LogFlow(("VMR3Resume: pVM=%p\n", pVM));
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Gather all the EMTs to make sure there are no races before
-     * changing the VM state.
-     */
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
-                                vmR3Resume, NULL);
-    LogFlow(("VMR3Resume: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * EMT rendezvous worker for VMR3Save and VMR3Teleport that suspends the VM
- * after the live step has been completed.
- *
- * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_RESUME. (This is a strict
- *          return code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          The pfSuspended argument of vmR3SaveTeleport.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3LiveDoSuspend(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    LogFlow(("vmR3LiveDoSuspend: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
-    bool *pfSuspended = (bool *)pvUser;
-
-    /*
-     * The first thread thru here tries to change the state.  We shouldn't be
-     * called again if this fails.
-     */
-    if (pVCpu->idCpu == pVM->cCpus - 1U)
-    {
-        PUVM     pUVM = pVM->pUVM;
-        int      rc;
-
-        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-        VMSTATE enmVMState = pVM->enmVMState;
-        switch (enmVMState)
-        {
-            case VMSTATE_RUNNING_LS:
-                vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDING_LS, VMSTATE_RUNNING_LS);
-                rc = VINF_SUCCESS;
-                break;
-
-            case VMSTATE_SUSPENDED_EXT_LS:
-            case VMSTATE_SUSPENDED_LS:          /* (via reset) */
-                rc = VINF_SUCCESS;
-                break;
-
-            case VMSTATE_DEBUGGING_LS:
-                rc = VERR_TRY_AGAIN;
-                break;
-
-            case VMSTATE_OFF_LS:
-                vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF, VMSTATE_OFF_LS);
-                rc = VERR_SSM_LIVE_POWERED_OFF;
-                break;
-
-            case VMSTATE_FATAL_ERROR_LS:
-                vmR3SetStateLocked(pVM, pUVM, VMSTATE_FATAL_ERROR, VMSTATE_FATAL_ERROR_LS);
-                rc = VERR_SSM_LIVE_FATAL_ERROR;
-                break;
-
-            case VMSTATE_GURU_MEDITATION_LS:
-                vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION, VMSTATE_GURU_MEDITATION_LS);
-                rc = VERR_SSM_LIVE_GURU_MEDITATION;
-                break;
-
-            case VMSTATE_POWERING_OFF_LS:
-            case VMSTATE_SUSPENDING_EXT_LS:
-            case VMSTATE_RESETTING_LS:
-            default:
-                AssertMsgFailed(("%s\n", VMR3GetStateName(enmVMState)));
-                rc = VERR_INTERNAL_ERROR_3;
-                break;
-        }
-        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-        if (RT_FAILURE(rc))
-        {
-            LogFlow(("vmR3LiveDoSuspend: returns %Rrc (state was %s)\n", rc, VMR3GetStateName(enmVMState)));
-            return rc;
-        }
-    }
-
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    AssertMsgReturn(enmVMState == VMSTATE_SUSPENDING_LS,
-                    ("%s\n", VMR3GetStateName(enmVMState)),
-                    VERR_INTERNAL_ERROR_4);
-
-    /*
-     * Only EMT(0) have work to do since it's last thru here.
-     */
-    if (pVCpu->idCpu == 0)
-    {
-        vmR3SuspendDoWork(pVM);
-        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 1,
-                                 VMSTATE_SUSPENDED_LS, VMSTATE_SUSPENDING_LS);
-        if (RT_FAILURE(rc))
-            return VERR_INTERNAL_ERROR_3;
-
-        *pfSuspended = true;
-    }
-
-    return VINF_EM_SUSPEND;
-}
-
-
-/**
- * EMT rendezvous worker that VMR3Save and VMR3Teleport uses to clean up a
- * SSMR3LiveDoStep1 failure.
- *
- * Doing this as a rendezvous operation avoids all annoying transition
- * states.
- *
- * @returns VERR_VM_INVALID_VM_STATE, VINF_SUCCESS or some specific VERR_SSM_*
- *          status code. (This is a strict return code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          The pfSuspended argument of vmR3SaveTeleport.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3LiveDoStep1Cleanup(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    LogFlow(("vmR3LiveDoStep1Cleanup: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
-    bool *pfSuspended = (bool *)pvUser;
-    NOREF(pVCpu);
-
-    int rc = vmR3TrySetState(pVM, "vmR3LiveDoStep1Cleanup", 8,
-                             VMSTATE_OFF,               VMSTATE_OFF_LS,                     /* 1 */
-                             VMSTATE_FATAL_ERROR,       VMSTATE_FATAL_ERROR_LS,             /* 2 */
-                             VMSTATE_GURU_MEDITATION,   VMSTATE_GURU_MEDITATION_LS,         /* 3 */
-                             VMSTATE_SUSPENDED,         VMSTATE_SUSPENDED_LS,               /* 4 */
-                             VMSTATE_SUSPENDED,         VMSTATE_SAVING,
-                             VMSTATE_SUSPENDED,         VMSTATE_SUSPENDED_EXT_LS,
-                             VMSTATE_RUNNING,           VMSTATE_RUNNING_LS,
-                             VMSTATE_DEBUGGING,         VMSTATE_DEBUGGING_LS);
-    if (rc == 1)
-        rc = VERR_SSM_LIVE_POWERED_OFF;
-    else if (rc == 2)
-        rc = VERR_SSM_LIVE_FATAL_ERROR;
-    else if (rc == 3)
-        rc = VERR_SSM_LIVE_GURU_MEDITATION;
-    else if (rc == 4)
-    {
-        *pfSuspended = true;
-        rc = VINF_SUCCESS;
-    }
-    else if (rc > 0)
-        rc = VINF_SUCCESS;
-    return rc;
-}
-
-
-/**
- * EMT(0) worker for VMR3Save and VMR3Teleport that completes the live save.
- *
- * @returns VBox status code.
- * @retval  VINF_SSM_LIVE_SUSPENDED if VMR3Suspend was called.
- *
- * @param   pVM             The VM handle.
- * @param   pSSM            The handle of saved state operation.
- *
- * @thread  EMT(0)
- */
-static DECLCALLBACK(int) vmR3LiveDoStep2(PVM pVM, PSSMHANDLE pSSM)
-{
-    LogFlow(("vmR3LiveDoStep2: pVM=%p pSSM=%p\n", pVM, pSSM));
-    VM_ASSERT_EMT0(pVM);
-
-    /*
-     * Advance the state and mark if VMR3Suspend was called.
-     */
-    int rc = VINF_SUCCESS;
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    if (enmVMState == VMSTATE_SUSPENDED_LS)
-        vmR3SetState(pVM, VMSTATE_SAVING, VMSTATE_SUSPENDED_LS);
-    else
-    {
-        if (enmVMState != VMSTATE_SAVING)
-            vmR3SetState(pVM, VMSTATE_SAVING, VMSTATE_SUSPENDED_EXT_LS);
-        rc = VINF_SSM_LIVE_SUSPENDED;
-    }
-
-    /*
-     * Finish up and release the handle. Careful with the status codes.
-     */
-    int rc2 = SSMR3LiveDoStep2(pSSM);
-    if (rc == VINF_SUCCESS || (RT_FAILURE(rc2) && RT_SUCCESS(rc)))
-        rc = rc2;
-
-    rc2 = SSMR3LiveDone(pSSM);
-    if (rc == VINF_SUCCESS || (RT_FAILURE(rc2) && RT_SUCCESS(rc)))
-        rc = rc2;
-
-    /*
-     * Advance to the final state and return.
-     */
-    vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_SAVING);
-    Assert(rc > VINF_EM_LAST || rc < VINF_EM_FIRST);
-    return rc;
-}
-
-
-/**
- * Worker for vmR3SaveTeleport that validates the state and calls SSMR3Save or
- * SSMR3LiveSave.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
- * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
- * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
- * @param   pvStreamOpsUser     The user argument to the stream methods.
- * @param   enmAfter            What to do afterwards.
- * @param   pfnProgress         Progress callback. Optional.
- * @param   pvProgressUser      User argument for the progress callback.
- * @param   ppSSM               Where to return the saved state handle in case of a
- *                              live snapshot scenario.
- * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
- *
- * @thread  EMT
- */
-static DECLCALLBACK(int) vmR3Save(PVM pVM, uint32_t cMsMaxDowntime, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                                  SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, PSSMHANDLE *ppSSM,
-                                  bool fSkipStateChanges)
-{
-    int rc = VINF_SUCCESS;
-
-    LogFlow(("vmR3Save: pVM=%p cMsMaxDowntime=%u pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p ppSSM=%p\n",
-             pVM, cMsMaxDowntime, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser, ppSSM));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrNull(pszFilename);
-    AssertPtrNull(pStreamOps);
-    AssertPtr(pVM);
-    Assert(   enmAfter == SSMAFTER_DESTROY
-           || enmAfter == SSMAFTER_CONTINUE
-           || enmAfter == SSMAFTER_TELEPORT);
-    AssertPtr(ppSSM);
-    *ppSSM = NULL;
-
-    /*
-     * Change the state and perform/start the saving.
-     */
-    if (!fSkipStateChanges)
-    {
-        rc = vmR3TrySetState(pVM, "VMR3Save", 2,
-                             VMSTATE_SAVING,     VMSTATE_SUSPENDED,
-                             VMSTATE_RUNNING_LS, VMSTATE_RUNNING);
-    }
-    else
-    {
-        Assert(enmAfter != SSMAFTER_TELEPORT);
-        rc = 1;
-    }
-
-    if (rc == 1 && enmAfter != SSMAFTER_TELEPORT)
-    {
-        rc = SSMR3Save(pVM, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser);
-        if (!fSkipStateChanges)
-            vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_SAVING);
-    }
-    else if (rc == 2 || enmAfter == SSMAFTER_TELEPORT)
-    {
-        Assert(!fSkipStateChanges);
-        if (enmAfter == SSMAFTER_TELEPORT)
-            pVM->vm.s.fTeleportedAndNotFullyResumedYet = true;
-        rc = SSMR3LiveSave(pVM, cMsMaxDowntime, pszFilename, pStreamOps, pvStreamOpsUser,
-                           enmAfter, pfnProgress, pvProgressUser, ppSSM);
-        /* (We're not subject to cancellation just yet.) */
-    }
-    else
-        Assert(RT_FAILURE(rc));
-    return rc;
-}
-
-
-/**
- * Common worker for VMR3Save and VMR3Teleport.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
- * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
- * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
- * @param   pvStreamOpsUser     The user argument to the stream methods.
- * @param   enmAfter            What to do afterwards.
- * @param   pfnProgress         Progress callback. Optional.
- * @param   pvProgressUser      User argument for the progress callback.
- * @param   pfSuspended         Set if we suspended the VM.
- * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
- *
- * @thread  Non-EMT
- */
-static int vmR3SaveTeleport(PVM pVM, uint32_t cMsMaxDowntime,
-                            const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                            SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool *pfSuspended,
-                            bool fSkipStateChanges)
-{
-    /*
-     * Request the operation in EMT(0).
-     */
-    PSSMHANDLE pSSM;
-    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/,
-                              (PFNRT)vmR3Save, 10, pVM, cMsMaxDowntime, pszFilename, pStreamOps, pvStreamOpsUser,
-                              enmAfter, pfnProgress, pvProgressUser, &pSSM, fSkipStateChanges);
-    if (    RT_SUCCESS(rc)
-        &&  pSSM)
-    {
-        Assert(!fSkipStateChanges);
-
-        /*
-         * Live snapshot.
-         *
-         * The state handling here is kind of tricky, doing it on EMT(0) helps
-         * a bit. See the VMSTATE diagram for details.
-         */
-        rc = SSMR3LiveDoStep1(pSSM);
-        if (RT_SUCCESS(rc))
-        {
-            if (VMR3GetState(pVM) != VMSTATE_SAVING)
-                for (;;)
-                {
-                    /* Try suspend the VM. */
-                    rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
-                                            vmR3LiveDoSuspend, pfSuspended);
-                    if (rc != VERR_TRY_AGAIN)
-                        break;
-
-                    /* Wait for the state to change. */
-                    RTThreadSleep(250); /** @todo Live Migration: fix this polling wait by some smart use of multiple release event  semaphores.. */
-                }
-            if (RT_SUCCESS(rc))
-                rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3LiveDoStep2, 2, pVM, pSSM);
-            else
-            {
-                int rc2 = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)SSMR3LiveDone, 1, pSSM);
-                AssertMsg(rc2 == rc, ("%Rrc != %Rrc\n", rc2, rc));
-            }
-        }
-        else
-        {
-            int rc2 = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)SSMR3LiveDone, 1, pSSM);
-            AssertMsg(rc2 == rc, ("%Rrc != %Rrc\n", rc2, rc));
-
-            rc2 = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, vmR3LiveDoStep1Cleanup, pfSuspended);
-            if (RT_FAILURE(rc2) && rc == VERR_SSM_CANCELLED)
-                rc = rc2;
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * Save current VM state.
- *
- * Can be used for both saving the state and creating snapshots.
- *
- * When called for a VM in the Running state, the saved state is created live
- * and the VM is only suspended when the final part of the saving is preformed.
- * The VM state will not be restored to Running in this case and it's up to the
- * caller to call VMR3Resume if this is desirable.  (The rational is that the
- * caller probably wish to reconfigure the disks before resuming the VM.)
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM which state should be saved.
- * @param   pszFilename         The name of the save state file.
- * @param   pStreamOps          The stream methods.
- * @param   pvStreamOpsUser     The user argument to the stream methods.
- * @param   fContinueAfterwards Whether continue execution afterwards or not.
- *                              When in doubt, set this to true.
- * @param   pfnProgress         Progress callback. Optional.
- * @param   pvUser              User argument for the progress callback.
- * @param   pfSuspended         Set if we suspended the VM.
- *
- * @thread      Non-EMT.
- * @vmstate     Suspended or Running
- * @vmstateto   Saving+Suspended or
- *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
- */
-VMMR3DECL(int) VMR3Save(PVM pVM, const char *pszFilename, bool fContinueAfterwards, PFNVMPROGRESS pfnProgress, void *pvUser, bool *pfSuspended)
-{
-    LogFlow(("VMR3Save: pVM=%p pszFilename=%p:{%s} fContinueAfterwards=%RTbool pfnProgress=%p pvUser=%p pfSuspended=%p\n",
-             pVM, pszFilename, pszFilename, fContinueAfterwards, pfnProgress, pvUser, pfSuspended));
-
-    /*
-     * Validate input.
-     */
-    AssertPtr(pfSuspended);
-    *pfSuspended = false;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    VM_ASSERT_OTHER_THREAD(pVM);
-    AssertReturn(VALID_PTR(pszFilename), VERR_INVALID_POINTER);
-    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
-    AssertPtrNullReturn(pfnProgress, VERR_INVALID_POINTER);
-
-    /*
-     * Join paths with VMR3Teleport.
-     */
-    SSMAFTER enmAfter = fContinueAfterwards ? SSMAFTER_CONTINUE : SSMAFTER_DESTROY;
-    int rc = vmR3SaveTeleport(pVM, 250 /*cMsMaxDowntime*/,
-                              pszFilename, NULL /* pStreamOps */, NULL /* pvStreamOpsUser */,
-                              enmAfter, pfnProgress, pvUser, pfSuspended,
-                              false /* fSkipStateChanges */);
-    LogFlow(("VMR3Save: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
-    return rc;
-}
-
-/**
- * Save current VM state (used by FTM)
- *
- * Can be used for both saving the state and creating snapshots.
- *
- * When called for a VM in the Running state, the saved state is created live
- * and the VM is only suspended when the final part of the saving is preformed.
- * The VM state will not be restored to Running in this case and it's up to the
- * caller to call VMR3Resume if this is desirable.  (The rational is that the
- * caller probably wish to reconfigure the disks before resuming the VM.)
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM which state should be saved.
- * @param   pStreamOps          The stream methods.
- * @param   pvStreamOpsUser     The user argument to the stream methods.
- * @param   pfSuspended         Set if we suspended the VM.
- * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
- *
- * @thread      Any
- * @vmstate     Suspended or Running
- * @vmstateto   Saving+Suspended or
- *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
- */
-VMMR3DECL(int) VMR3SaveFT(PVM pVM, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser, bool *pfSuspended,
-                          bool fSkipStateChanges)
-{
-    LogFlow(("VMR3SaveFT: pVM=%p pStreamOps=%p pvSteamOpsUser=%p pfSuspended=%p\n",
-             pVM, pStreamOps, pvStreamOpsUser, pfSuspended));
-
-    /*
-     * Validate input.
-     */
-    AssertPtr(pfSuspended);
-    *pfSuspended = false;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertReturn(pStreamOps, VERR_INVALID_PARAMETER);
-
-    /*
-     * Join paths with VMR3Teleport.
-     */
-    int rc = vmR3SaveTeleport(pVM, 250 /*cMsMaxDowntime*/,
-                              NULL, pStreamOps, pvStreamOpsUser,
-                              SSMAFTER_CONTINUE, NULL, NULL, pfSuspended,
-                              fSkipStateChanges);
-    LogFlow(("VMR3SaveFT: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
-    return rc;
-}
-
-
-/**
- * Teleport the VM (aka live migration).
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM which state should be saved.
- * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
- * @param   pStreamOps          The stream methods.
- * @param   pvStreamOpsUser     The user argument to the stream methods.
- * @param   pfnProgress         Progress callback. Optional.
- * @param   pvProgressUser      User argument for the progress callback.
- * @param   pfSuspended         Set if we suspended the VM.
- *
- * @thread      Non-EMT.
- * @vmstate     Suspended or Running
- * @vmstateto   Saving+Suspended or
- *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
- */
-VMMR3DECL(int) VMR3Teleport(PVM pVM, uint32_t cMsMaxDowntime, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                            PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool *pfSuspended)
-{
-    LogFlow(("VMR3Teleport: pVM=%p cMsMaxDowntime=%u pStreamOps=%p pvStreamOps=%p pfnProgress=%p pvProgressUser=%p\n",
-             pVM, cMsMaxDowntime, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser));
-
-    /*
-     * Validate input.
-     */
-    AssertPtr(pfSuspended);
-    *pfSuspended = false;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    VM_ASSERT_OTHER_THREAD(pVM);
-    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
-    AssertPtrNullReturn(pfnProgress, VERR_INVALID_POINTER);
-
-    /*
-     * Join paths with VMR3Save.
-     */
-    int rc = vmR3SaveTeleport(pVM, cMsMaxDowntime,
-                              NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser,
-                              SSMAFTER_TELEPORT, pfnProgress, pvProgressUser, pfSuspended,
-                              false /* fSkipStateChanges */);
-    LogFlow(("VMR3Teleport: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
-    return rc;
-}
-
-
-
-/**
- * EMT(0) worker for VMR3LoadFromFile and VMR3LoadFromStream.
- *
- * @returns VBox status code.
- *
- * @param   pVM                 The VM handle.
- * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
- * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
- * @param   pvStreamOpsUser     The user argument to the stream methods.
- * @param   pfnProgress         Progress callback. Optional.
- * @param   pvUser              User argument for the progress callback.
- * @param   fTeleporting        Indicates whether we're teleporting or not.
- * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
- *
- * @thread  EMT.
- */
-static DECLCALLBACK(int) vmR3Load(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                                  PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool fTeleporting,
-                                  bool fSkipStateChanges)
-{
-    int rc = VINF_SUCCESS;
-
-    LogFlow(("vmR3Load: pVM=%p pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p pfnProgress=%p pvProgressUser=%p fTeleporting=%RTbool\n",
-             pVM, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser, fTeleporting));
-
-    /*
-     * Validate input (paranoia).
-     */
-    AssertPtr(pVM);
-    AssertPtrNull(pszFilename);
-    AssertPtrNull(pStreamOps);
-    AssertPtrNull(pfnProgress);
-
-    if (!fSkipStateChanges)
-    {
-        /*
-         * Change the state and perform the load.
-         *
-         * Always perform a relocation round afterwards to make sure hypervisor
-         * selectors and such are correct.
-         */
-        rc = vmR3TrySetState(pVM, "VMR3Load", 2,
-                                 VMSTATE_LOADING, VMSTATE_CREATED,
-                                 VMSTATE_LOADING, VMSTATE_SUSPENDED);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-    pVM->vm.s.fTeleportedAndNotFullyResumedYet = fTeleporting;
-
-    uint32_t cErrorsPriorToSave = VMR3GetErrorCount(pVM);
-    rc = SSMR3Load(pVM, pszFilename, pStreamOps, pvStreamOpsUser, SSMAFTER_RESUME, pfnProgress, pvProgressUser);
-    if (RT_SUCCESS(rc))
-    {
-        VMR3Relocate(pVM, 0 /*offDelta*/);
-        if (!fSkipStateChanges)
-            vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_LOADING);
-    }
-    else
-    {
-        pVM->vm.s.fTeleportedAndNotFullyResumedYet = false;
-        if (!fSkipStateChanges)
-            vmR3SetState(pVM, VMSTATE_LOAD_FAILURE, VMSTATE_LOADING);
-
-        if (cErrorsPriorToSave == VMR3GetErrorCount(pVM))
-            rc = VMSetError(pVM, rc, RT_SRC_POS,
-                            N_("Unable to restore the virtual machine's saved state from '%s'. "
-                               "It may be damaged or from an older version of VirtualBox.  "
-                               "Please discard the saved state before starting the virtual machine"),
-                            pszFilename);
-    }
-
-    return rc;
-}
-
-
-/**
- * Loads a VM state into a newly created VM or a one that is suspended.
- *
- * To restore a saved state on VM startup, call this function and then resume
- * the VM instead of powering it on.
- *
- * @returns VBox status code.
- *
- * @param   pVM             The VM handle.
- * @param   pszFilename     The name of the save state file.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvUser          User argument for the progress callback.
- *
- * @thread      Any thread.
- * @vmstate     Created, Suspended
- * @vmstateto   Loading+Suspended
- */
-VMMR3DECL(int) VMR3LoadFromFile(PVM pVM, const char *pszFilename, PFNVMPROGRESS pfnProgress, void *pvUser)
-{
-    LogFlow(("VMR3LoadFromFile: pVM=%p pszFilename=%p:{%s} pfnProgress=%p pvUser=%p\n",
-             pVM, pszFilename, pszFilename, pfnProgress, pvUser));
-
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
-
-    /*
-     * Forward the request to EMT(0).  No need to setup a rendezvous here
-     * since there is no execution taking place when this call is allowed.
-     */
-    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 8,
-                              pVM, pszFilename, (uintptr_t)NULL /*pStreamOps*/, (uintptr_t)NULL /*pvStreamOpsUser*/, pfnProgress, pvUser,
-                              false /*fTeleporting*/, false /* fSkipStateChanges */);
-    LogFlow(("VMR3LoadFromFile: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * VMR3LoadFromFile for arbitrary file streams.
- *
- * @returns VBox status code.
- *
- * @param   pVM             The VM handle.
- * @param   pStreamOps      The stream methods.
- * @param   pvStreamOpsUser The user argument to the stream methods.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvProgressUser  User argument for the progress callback.
- *
- * @thread      Any thread.
- * @vmstate     Created, Suspended
- * @vmstateto   Loading+Suspended
- */
-VMMR3DECL(int) VMR3LoadFromStream(PVM pVM, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
-                                  PFNVMPROGRESS pfnProgress, void *pvProgressUser)
-{
-    LogFlow(("VMR3LoadFromStream: pVM=%p pStreamOps=%p pvStreamOpsUser=%p pfnProgress=%p pvProgressUser=%p\n",
-             pVM, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser));
-
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
-
-    /*
-     * Forward the request to EMT(0).  No need to setup a rendezvous here
-     * since there is no execution taking place when this call is allowed.
-     */
-    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 8,
-                              pVM, (uintptr_t)NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser,
-                              true /*fTeleporting*/, false /* fSkipStateChanges */);
-    LogFlow(("VMR3LoadFromStream: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * VMR3LoadFromFileFT for arbitrary file streams.
- *
- * @returns VBox status code.
- *
- * @param   pVM             The VM handle.
- * @param   pStreamOps      The stream methods.
- * @param   pvStreamOpsUser The user argument to the stream methods.
- * @param   pfnProgress     Progress callback. Optional.
- * @param   pvProgressUser  User argument for the progress callback.
- *
- * @thread      Any thread.
- * @vmstate     Created, Suspended
- * @vmstateto   Loading+Suspended
- */
-VMMR3DECL(int) VMR3LoadFromStreamFT(PVM pVM, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser)
-{
-    LogFlow(("VMR3LoadFromStreamFT: pVM=%p pStreamOps=%p pvStreamOpsUser=%p\n",
-             pVM, pStreamOps, pvStreamOpsUser));
-
-    /*
-     * Validate input.
-     */
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
-
-    /*
-     * Forward the request to EMT(0).  No need to setup a rendezvous here
-     * since there is no execution taking place when this call is allowed.
-     */
-    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 8,
-                              pVM, (uintptr_t)NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser, NULL, NULL,
-                              true /*fTeleporting*/, true /* fSkipStateChanges */);
-    LogFlow(("VMR3LoadFromStream: returns %Rrc\n", rc));
-    return rc;
-}
-
-/**
- * EMT rendezvous worker for VMR3PowerOff.
- *
- * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_OFF. (This is a strict
- *          return code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          Ignored.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3PowerOff(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    LogFlow(("vmR3PowerOff: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
-    Assert(!pvUser); NOREF(pvUser);
-
-    /*
-     * The first EMT thru here will change the state to PoweringOff.
-     */
-    if (pVCpu->idCpu == pVM->cCpus - 1)
-    {
-        int rc = vmR3TrySetState(pVM, "VMR3PowerOff", 11,
-                                 VMSTATE_POWERING_OFF,    VMSTATE_RUNNING,           /* 1 */
-                                 VMSTATE_POWERING_OFF,    VMSTATE_SUSPENDED,         /* 2 */
-                                 VMSTATE_POWERING_OFF,    VMSTATE_DEBUGGING,         /* 3 */
-                                 VMSTATE_POWERING_OFF,    VMSTATE_LOAD_FAILURE,      /* 4 */
-                                 VMSTATE_POWERING_OFF,    VMSTATE_GURU_MEDITATION,   /* 5 */
-                                 VMSTATE_POWERING_OFF,    VMSTATE_FATAL_ERROR,       /* 6 */
-                                 VMSTATE_POWERING_OFF,    VMSTATE_CREATED,           /* 7 */   /** @todo update the diagram! */
-                                 VMSTATE_POWERING_OFF_LS, VMSTATE_RUNNING_LS,        /* 8 */
-                                 VMSTATE_POWERING_OFF_LS, VMSTATE_DEBUGGING_LS,      /* 9 */
-                                 VMSTATE_POWERING_OFF_LS, VMSTATE_GURU_MEDITATION_LS,/* 10 */
-                                 VMSTATE_POWERING_OFF_LS, VMSTATE_FATAL_ERROR_LS);   /* 11 */
-        if (RT_FAILURE(rc))
-            return rc;
-        if (rc >= 7)
-            SSMR3Cancel(pVM);
-    }
-
-    /*
-     * Check the state.
-     */
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    AssertMsgReturn(   enmVMState == VMSTATE_POWERING_OFF
-                    || enmVMState == VMSTATE_POWERING_OFF_LS,
-                    ("%s\n", VMR3GetStateName(enmVMState)),
-                    VERR_VM_INVALID_VM_STATE);
-
-    /*
-     * EMT(0) does the actual power off work here *after* all the other EMTs
-     * have been thru and entered the STOPPED state.
-     */
-    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STOPPED);
-    if (pVCpu->idCpu == 0)
-    {
-        /*
-         * For debugging purposes, we will log a summary of the guest state at this point.
-         */
-        if (enmVMState != VMSTATE_GURU_MEDITATION)
-        {
-            /** @todo SMP support? */
-            /** @todo make the state dumping at VMR3PowerOff optional. */
-            bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
-            RTLogRelPrintf("****************** Guest state at power off ******************\n");
-            DBGFR3Info(pVM, "cpumguest", "verbose", DBGFR3InfoLogRelHlp());
-            RTLogRelPrintf("***\n");
-            DBGFR3Info(pVM, "mode", NULL, DBGFR3InfoLogRelHlp());
-            RTLogRelPrintf("***\n");
-            DBGFR3Info(pVM, "activetimers", NULL, DBGFR3InfoLogRelHlp());
-            RTLogRelPrintf("***\n");
-            DBGFR3Info(pVM, "gdt", NULL, DBGFR3InfoLogRelHlp());
-            /** @todo dump guest call stack. */
-#if 1 // "temporary" while debugging #1589
-            RTLogRelPrintf("***\n");
-            uint32_t esp = CPUMGetGuestESP(pVCpu);
-            if (    CPUMGetGuestSS(pVCpu) == 0
-                &&  esp < _64K)
-            {
-                uint8_t abBuf[PAGE_SIZE];
-                RTLogRelPrintf("***\n"
-                               "ss:sp=0000:%04x ", esp);
-                uint32_t Start = esp & ~(uint32_t)63;
-                int rc = PGMPhysSimpleReadGCPhys(pVM, abBuf, Start, 0x100);
-                if (RT_SUCCESS(rc))
-                    RTLogRelPrintf("0000:%04x TO 0000:%04x:\n"
-                                   "%.*Rhxd\n",
-                                   Start, Start + 0x100 - 1,
-                                   0x100, abBuf);
-                else
-                    RTLogRelPrintf("rc=%Rrc\n", rc);
-
-                /* grub ... */
-                if (esp < 0x2000 && esp > 0x1fc0)
-                {
-                    rc = PGMPhysSimpleReadGCPhys(pVM, abBuf, 0x8000, 0x800);
-                    if (RT_SUCCESS(rc))
-                        RTLogRelPrintf("0000:8000 TO 0000:87ff:\n"
-                                       "%.*Rhxd\n",
-                                       0x800, abBuf);
-                }
-                /* microsoft cdrom hang ... */
-                if (true)
-                {
-                    rc = PGMPhysSimpleReadGCPhys(pVM, abBuf, 0x8000, 0x200);
-                    if (RT_SUCCESS(rc))
-                        RTLogRelPrintf("2000:0000 TO 2000:01ff:\n"
-                                       "%.*Rhxd\n",
-                                       0x200, abBuf);
-                }
-            }
-#endif
-            RTLogRelSetBuffering(fOldBuffered);
-            RTLogRelPrintf("************** End of Guest state at power off ***************\n");
-        }
-
-        /*
-         * Perform the power off notifications and advance the state to
-         * Off or OffLS.
-         */
-        PDMR3PowerOff(pVM);
-
-        PUVM pUVM = pVM->pUVM;
-        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-        enmVMState = pVM->enmVMState;
-        if (enmVMState == VMSTATE_POWERING_OFF_LS)
-            vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF_LS, VMSTATE_POWERING_OFF_LS);
-        else
-            vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF,    VMSTATE_POWERING_OFF);
-        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-    }
-    return VINF_EM_OFF;
-}
-
-
-/**
- * Power off the VM.
- *
- * @returns VBox status code. When called on EMT, this will be a strict status
- *          code that has to be propagated up the call stack.
- *
- * @param   pVM     The handle of the VM to be powered off.
- *
- * @thread      Any thread.
- * @vmstate     Suspended, Running, Guru Meditation, Load Failure
- * @vmstateto   Off or OffLS
- */
-VMMR3DECL(int)   VMR3PowerOff(PVM pVM)
-{
-    LogFlow(("VMR3PowerOff: pVM=%p\n", pVM));
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Gather all the EMTs to make sure there are no races before
-     * changing the VM state.
-     */
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
-                                vmR3PowerOff, NULL);
-    LogFlow(("VMR3PowerOff: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Destroys the VM.
- *
- * The VM must be powered off (or never really powered on) to call this
- * function. The VM handle is destroyed and can no longer be used up successful
- * return.
- *
- * @returns VBox status code.
- *
- * @param   pVM     The handle of the VM which should be destroyed.
- *
- * @thread      Any none emulation thread.
- * @vmstate     Off, Created
- * @vmstateto   N/A
- */
-VMMR3DECL(int) VMR3Destroy(PVM pVM)
-{
-    LogFlow(("VMR3Destroy: pVM=%p\n", pVM));
-
-    /*
-     * Validate input.
-     */
-    if (!pVM)
-        return VERR_INVALID_PARAMETER;
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-    AssertLogRelReturn(!VM_IS_EMT(pVM), VERR_VM_THREAD_IS_EMT);
-
-    /*
-     * Change VM state to destroying and unlink the VM.
-     */
-    int rc = vmR3TrySetState(pVM, "VMR3Destroy", 1, VMSTATE_DESTROYING, VMSTATE_OFF);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /** @todo lock this when we start having multiple machines in a process... */
-    PUVM pUVM = pVM->pUVM; AssertPtr(pUVM);
-    if (g_pUVMsHead == pUVM)
-        g_pUVMsHead = pUVM->pNext;
-    else
-    {
-        PUVM pPrev = g_pUVMsHead;
-        while (pPrev && pPrev->pNext != pUVM)
-            pPrev = pPrev->pNext;
-        AssertMsgReturn(pPrev, ("pUVM=%p / pVM=%p  is INVALID!\n", pUVM, pVM), VERR_INVALID_PARAMETER);
-
-        pPrev->pNext = pUVM->pNext;
-    }
-    pUVM->pNext = NULL;
-
-    /*
-     * Notify registered at destruction listeners.
-     */
-    vmR3AtDtor(pVM);
-
-    /*
-     * Call vmR3Destroy on each of the EMTs ending with EMT(0) doing the bulk
-     * of the cleanup.
-     */
-    /* vmR3Destroy on all EMTs, ending with EMT(0). */
-    rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ALL_REVERSE, (PFNRT)vmR3Destroy, 1, pVM);
-    AssertLogRelRC(rc);
-
-    /* Wait for EMTs and destroy the UVM. */
-    vmR3DestroyUVM(pUVM, 30000);
-
-    LogFlow(("VMR3Destroy: returns VINF_SUCCESS\n"));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Internal destruction worker.
- *
- * This is either called from VMR3Destroy via VMR3ReqCallU or from
- * vmR3EmulationThreadWithId when EMT(0) terminates after having called
- * VMR3Destroy().
- *
- * When called on EMT(0), it will performed the great bulk of the destruction.
- * When called on the other EMTs, they will do nothing and the whole purpose is
- * to return VINF_EM_TERMINATE so they break out of their run loops.
- *
- * @returns VINF_EM_TERMINATE.
- * @param   pVM     The VM handle.
- */
-DECLCALLBACK(int) vmR3Destroy(PVM pVM)
-{
-    PUVM   pUVM  = pVM->pUVM;
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    Assert(pVCpu);
-    LogFlow(("vmR3Destroy: pVM=%p pUVM=%p pVCpu=%p idCpu=%u\n", pVM, pUVM, pVCpu, pVCpu->idCpu));
-
-    /*
-     * Only VCPU 0 does the full cleanup (last).
-     */
-    if (pVCpu->idCpu == 0)
-    {
-        /*
-         * Dump statistics to the log.
-         */
-#if defined(VBOX_WITH_STATISTICS) || defined(LOG_ENABLED)
-        RTLogFlags(NULL, "nodisabled nobuffered");
-#endif
-#ifdef VBOX_WITH_STATISTICS
-        STAMR3Dump(pVM, "*");
-#else
-        LogRel(("************************* Statistics *************************\n"));
-        STAMR3DumpToReleaseLog(pVM, "*");
-        LogRel(("********************* End of statistics **********************\n"));
-#endif
-
-        /*
-         * Destroy the VM components.
-         */
-        int rc = TMR3Term(pVM);
-        AssertRC(rc);
-#ifdef VBOX_WITH_DEBUGGER
-        rc = DBGCTcpTerminate(pVM, pUVM->vm.s.pvDBGC);
-        pUVM->vm.s.pvDBGC = NULL;
-#endif
-        AssertRC(rc);
-        rc = FTMR3Term(pVM);
-        AssertRC(rc);
-        rc = DBGFR3Term(pVM);
-        AssertRC(rc);
-        rc = PDMR3Term(pVM);
-        AssertRC(rc);
-        rc = EMR3Term(pVM);
-        AssertRC(rc);
-        rc = IOMR3Term(pVM);
-        AssertRC(rc);
-        rc = CSAMR3Term(pVM);
-        AssertRC(rc);
-        rc = PATMR3Term(pVM);
-        AssertRC(rc);
-        rc = TRPMR3Term(pVM);
-        AssertRC(rc);
-        rc = SELMR3Term(pVM);
-        AssertRC(rc);
-        rc = REMR3Term(pVM);
-        AssertRC(rc);
-        rc = HWACCMR3Term(pVM);
-        AssertRC(rc);
-        rc = PGMR3Term(pVM);
-        AssertRC(rc);
-        rc = VMMR3Term(pVM); /* Terminates the ring-0 code! */
-        AssertRC(rc);
-        rc = CPUMR3Term(pVM);
-        AssertRC(rc);
-        SSMR3Term(pVM);
-        rc = PDMR3CritSectTerm(pVM);
-        AssertRC(rc);
-        rc = MMR3Term(pVM);
-        AssertRC(rc);
-
-        /*
-         * We're done, tell the other EMTs to quit.
-         */
-        ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
-        ASMAtomicWriteU32(&pVM->fGlobalForcedActions, VM_FF_CHECK_VM_STATE); /* Can't hurt... */
-        LogFlow(("vmR3Destroy: returning %Rrc\n", VINF_EM_TERMINATE));
-    }
-    return VINF_EM_TERMINATE;
-}
-
-
-/**
- * Destroys the UVM portion.
- *
- * This is called as the final step in the VM destruction or as the cleanup
- * in case of a creation failure.
- *
- * @param   pVM             VM Handle.
- * @param   cMilliesEMTWait The number of milliseconds to wait for the emulation
- *                          threads.
- */
-static void vmR3DestroyUVM(PUVM pUVM, uint32_t cMilliesEMTWait)
-{
-    /*
-     * Signal termination of each the emulation threads and
-     * wait for them to complete.
-     */
-    /* Signal them. */
-    ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
-    if (pUVM->pVM)
-        VM_FF_SET(pUVM->pVM, VM_FF_CHECK_VM_STATE); /* Can't hurt... */
-    for (VMCPUID i = 0; i < pUVM->cCpus; i++)
-    {
-        VMR3NotifyGlobalFFU(pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
-        RTSemEventSignal(pUVM->aCpus[i].vm.s.EventSemWait);
-    }
-
-    /* Wait for them. */
-    uint64_t    NanoTS = RTTimeNanoTS();
-    RTTHREAD    hSelf  = RTThreadSelf();
-    ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
-    for (VMCPUID i = 0; i < pUVM->cCpus; i++)
-    {
-        RTTHREAD hThread = pUVM->aCpus[i].vm.s.ThreadEMT;
-        if (    hThread != NIL_RTTHREAD
-            &&  hThread != hSelf)
-        {
-            uint64_t cMilliesElapsed = (RTTimeNanoTS() - NanoTS) / 1000000;
-            int rc2 = RTThreadWait(hThread,
-                                   cMilliesElapsed < cMilliesEMTWait
-                                   ? RT_MAX(cMilliesEMTWait - cMilliesElapsed, 2000)
-                                   : 2000,
-                                   NULL);
-            if (rc2 == VERR_TIMEOUT) /* avoid the assertion when debugging. */
-                rc2 = RTThreadWait(hThread, 1000, NULL);
-            AssertLogRelMsgRC(rc2, ("i=%u rc=%Rrc\n", i, rc2));
-            if (RT_SUCCESS(rc2))
-                pUVM->aCpus[0].vm.s.ThreadEMT = NIL_RTTHREAD;
-        }
-    }
-
-    /* Cleanup the semaphores. */
-    for (VMCPUID i = 0; i < pUVM->cCpus; i++)
-    {
-        RTSemEventDestroy(pUVM->aCpus[i].vm.s.EventSemWait);
-        pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
-    }
-
-    /*
-     * Free the event semaphores associated with the request packets.
-     */
-    unsigned cReqs = 0;
-    for (unsigned i = 0; i < RT_ELEMENTS(pUVM->vm.s.apReqFree); i++)
-    {
-        PVMREQ pReq = pUVM->vm.s.apReqFree[i];
-        pUVM->vm.s.apReqFree[i] = NULL;
-        for (; pReq; pReq = pReq->pNext, cReqs++)
-        {
-            pReq->enmState = VMREQSTATE_INVALID;
-            RTSemEventDestroy(pReq->EventSem);
-        }
-    }
-    Assert(cReqs == pUVM->vm.s.cReqFree); NOREF(cReqs);
-
-    /*
-     * Kill all queued requests. (There really shouldn't be any!)
-     */
-    for (unsigned i = 0; i < 10; i++)
-    {
-        PVMREQ pReqHead = ASMAtomicXchgPtrT(&pUVM->vm.s.pReqs, NULL, PVMREQ);
-        AssertMsg(!pReqHead, ("This isn't supposed to happen! VMR3Destroy caller has to serialize this.\n"));
-        if (!pReqHead)
-            break;
-        for (PVMREQ pReq = pReqHead; pReq; pReq = pReq->pNext)
-        {
-            ASMAtomicUoWriteSize(&pReq->iStatus, VERR_INTERNAL_ERROR);
-            ASMAtomicWriteSize(&pReq->enmState, VMREQSTATE_INVALID);
-            RTSemEventSignal(pReq->EventSem);
-            RTThreadSleep(2);
-            RTSemEventDestroy(pReq->EventSem);
-        }
-        /* give them a chance to respond before we free the request memory. */
-        RTThreadSleep(32);
-    }
-
-    /*
-     * Now all queued VCPU requests (again, there shouldn't be any).
-     */
-    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
-    {
-        PUVMCPU pUVCpu = &pUVM->aCpus[idCpu];
-
-        for (unsigned i = 0; i < 10; i++)
-        {
-            PVMREQ pReqHead = ASMAtomicXchgPtrT(&pUVCpu->vm.s.pReqs, NULL, PVMREQ);
-            AssertMsg(!pReqHead, ("This isn't supposed to happen! VMR3Destroy caller has to serialize this.\n"));
-            if (!pReqHead)
-                break;
-            for (PVMREQ pReq = pReqHead; pReq; pReq = pReq->pNext)
-            {
-                ASMAtomicUoWriteSize(&pReq->iStatus, VERR_INTERNAL_ERROR);
-                ASMAtomicWriteSize(&pReq->enmState, VMREQSTATE_INVALID);
-                RTSemEventSignal(pReq->EventSem);
-                RTThreadSleep(2);
-                RTSemEventDestroy(pReq->EventSem);
-            }
-            /* give them a chance to respond before we free the request memory. */
-            RTThreadSleep(32);
-        }
-    }
-
-    /*
-     * Make sure the VMMR0.r0 module and whatever else is unloaded.
-     */
-    PDMR3TermUVM(pUVM);
-
-    /*
-     * Terminate the support library if initialized.
-     */
-    if (pUVM->vm.s.pSession)
-    {
-        int rc = SUPR3Term(false /*fForced*/);
-        AssertRC(rc);
-        pUVM->vm.s.pSession = NIL_RTR0PTR;
-    }
-
-    /*
-     * Destroy the MM heap and free the UVM structure.
-     */
-    MMR3TermUVM(pUVM);
-    STAMR3TermUVM(pUVM);
-
-#ifdef LOG_ENABLED
-    RTLogSetCustomPrefixCallback(NULL, NULL, NULL);
-#endif
-    RTTlsFree(pUVM->vm.s.idxTLS);
-
-    ASMAtomicUoWriteU32(&pUVM->u32Magic, UINT32_MAX);
-    RTMemPageFree(pUVM, RT_OFFSETOF(UVM, aCpus[pUVM->cCpus]));
-
-    RTLogFlush(NULL);
-}
-
-
-/**
- * Enumerates the VMs in this process.
- *
- * @returns Pointer to the next VM.
- * @returns NULL when no more VMs.
- * @param   pVMPrev     The previous VM
- *                      Use NULL to start the enumeration.
- */
-VMMR3DECL(PVM) VMR3EnumVMs(PVM pVMPrev)
-{
-    /*
-     * This is quick and dirty. It has issues with VM being
-     * destroyed during the enumeration.
-     */
-    PUVM pNext;
-    if (pVMPrev)
-        pNext = pVMPrev->pUVM->pNext;
-    else
-        pNext = g_pUVMsHead;
-    return pNext ? pNext->pVM : NULL;
-}
-
-
-/**
- * Registers an at VM destruction callback.
- *
- * @returns VBox status code.
- * @param   pfnAtDtor       Pointer to callback.
- * @param   pvUser          User argument.
- */
-VMMR3DECL(int) VMR3AtDtorRegister(PFNVMATDTOR pfnAtDtor, void *pvUser)
-{
-    /*
-     * Check if already registered.
-     */
-    VM_ATDTOR_LOCK();
-    PVMATDTOR   pCur = g_pVMAtDtorHead;
-    while (pCur)
-    {
-        if (pfnAtDtor == pCur->pfnAtDtor)
-        {
-            VM_ATDTOR_UNLOCK();
-            AssertMsgFailed(("Already registered at destruction callback %p!\n", pfnAtDtor));
-            return VERR_INVALID_PARAMETER;
-        }
-
-        /* next */
-        pCur = pCur->pNext;
-    }
-    VM_ATDTOR_UNLOCK();
-
-    /*
-     * Allocate new entry.
-     */
-    PVMATDTOR   pVMAtDtor = (PVMATDTOR)RTMemAlloc(sizeof(*pVMAtDtor));
-    if (!pVMAtDtor)
-        return VERR_NO_MEMORY;
-
-    VM_ATDTOR_LOCK();
-    pVMAtDtor->pfnAtDtor = pfnAtDtor;
-    pVMAtDtor->pvUser    = pvUser;
-    pVMAtDtor->pNext     = g_pVMAtDtorHead;
-    g_pVMAtDtorHead      = pVMAtDtor;
-    VM_ATDTOR_UNLOCK();
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Deregisters an at VM destruction callback.
- *
- * @returns VBox status code.
- * @param   pfnAtDtor       Pointer to callback.
- */
-VMMR3DECL(int) VMR3AtDtorDeregister(PFNVMATDTOR pfnAtDtor)
-{
-    /*
-     * Find it, unlink it and free it.
-     */
-    VM_ATDTOR_LOCK();
-    PVMATDTOR   pPrev = NULL;
-    PVMATDTOR   pCur = g_pVMAtDtorHead;
-    while (pCur)
-    {
-        if (pfnAtDtor == pCur->pfnAtDtor)
-        {
-            if (pPrev)
-                pPrev->pNext = pCur->pNext;
-            else
-                g_pVMAtDtorHead = pCur->pNext;
-            pCur->pNext = NULL;
-            VM_ATDTOR_UNLOCK();
-
-            RTMemFree(pCur);
-            return VINF_SUCCESS;
-        }
-
-        /* next */
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-    VM_ATDTOR_UNLOCK();
-
-    return VERR_INVALID_PARAMETER;
-}
-
-
-/**
- * Walks the list of at VM destructor callbacks.
- * @param   pVM     The VM which is about to be destroyed.
- */
-static void vmR3AtDtor(PVM pVM)
-{
-    /*
-     * Find it, unlink it and free it.
-     */
-    VM_ATDTOR_LOCK();
-    for (PVMATDTOR pCur = g_pVMAtDtorHead; pCur; pCur = pCur->pNext)
-        pCur->pfnAtDtor(pVM, pCur->pvUser);
-    VM_ATDTOR_UNLOCK();
-}
-
-
-/**
- * Worker which checks integrity of some internal structures.
- * This is yet another attempt to track down that AVL tree crash.
- */
-static void vmR3CheckIntegrity(PVM pVM)
-{
-#ifdef VBOX_STRICT
-    int rc = PGMR3CheckIntegrity(pVM);
-    AssertReleaseRC(rc);
-#endif
-}
-
-
-/**
- * EMT rendezvous worker for VMR3Reset.
- *
- * This is called by the emulation threads as a response to the reset request
- * issued by VMR3Reset().
- *
- * @returns VERR_VM_INVALID_VM_STATE, VINF_EM_RESET or VINF_EM_SUSPEND. (This
- *          is a strict return code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          Ignored.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3Reset(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    Assert(!pvUser); NOREF(pvUser);
-
-    /*
-     * The first EMT will try change the state to resetting.  If this fails,
-     * we won't get called for the other EMTs.
-     */
-    if (pVCpu->idCpu == pVM->cCpus - 1)
-    {
-        int rc = vmR3TrySetState(pVM, "VMR3Reset", 3,
-                                 VMSTATE_RESETTING,     VMSTATE_RUNNING,
-                                 VMSTATE_RESETTING,     VMSTATE_SUSPENDED,
-                                 VMSTATE_RESETTING_LS,  VMSTATE_RUNNING_LS);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-
-    /*
-     * Check the state.
-     */
-    VMSTATE enmVMState = VMR3GetState(pVM);
-    AssertLogRelMsgReturn(   enmVMState == VMSTATE_RESETTING
-                          || enmVMState == VMSTATE_RESETTING_LS,
-                          ("%s\n", VMR3GetStateName(enmVMState)),
-                          VERR_INTERNAL_ERROR_4);
-
-    /*
-     * EMT(0) does the full cleanup *after* all the other EMTs has been
-     * thru here and been told to enter the EMSTATE_WAIT_SIPI state.
-     *
-     * Because there are per-cpu reset routines and order may/is important,
-     * the following sequence looks a bit ugly...
-     */
-    if (pVCpu->idCpu == 0)
-        vmR3CheckIntegrity(pVM);
-
-    /* Reset the VCpu state. */
-    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED);
-
-    /* Clear all pending forced actions. */
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_ALL_MASK & ~VMCPU_FF_REQUEST);
-
-    /*
-     * Reset the VM components.
-     */
-    if (pVCpu->idCpu == 0)
-    {
-        PATMR3Reset(pVM);
-        CSAMR3Reset(pVM);
-        PGMR3Reset(pVM);                    /* We clear VM RAM in PGMR3Reset. It's vital PDMR3Reset is executed
-                                             * _afterwards_. E.g. ACPI sets up RAM tables during init/reset. */
-/** @todo PGMR3Reset should be called after PDMR3Reset really, because we'll trash OS <-> hardware
- * communication structures residing in RAM when done in the other order.  I.e. the device must be
- * quiesced first, then we clear the memory and plan tables. Probably have to make these things
- * explicit in some way, some memory setup pass or something.
- * (Example: DevAHCI may assert if memory is zeroed before it has read the FIS.)
- *
- * @bugref{4467}
- */
-        MMR3Reset(pVM);
-        PDMR3Reset(pVM);
-        SELMR3Reset(pVM);
-        TRPMR3Reset(pVM);
-        REMR3Reset(pVM);
-        IOMR3Reset(pVM);
-        CPUMR3Reset(pVM);
-    }
-    CPUMR3ResetCpu(pVCpu);
-    if (pVCpu->idCpu == 0)
-    {
-        TMR3Reset(pVM);
-        EMR3Reset(pVM);
-        HWACCMR3Reset(pVM);                 /* This must come *after* PATM, CSAM, CPUM, SELM and TRPM. */
-
-#ifdef LOG_ENABLED
-        /*
-         * Debug logging.
-         */
-        RTLogPrintf("\n\nThe VM was reset:\n");
-        DBGFR3Info(pVM, "cpum", "verbose", NULL);
-#endif
-
-        /*
-         * Since EMT(0) is the last to go thru here, it will advance the state.
-         * When a live save is active, we will move on to SuspendingLS but
-         * leave it for VMR3Reset to do the actual suspending due to deadlock risks.
-         */
-        PUVM pUVM = pVM->pUVM;
-        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-        enmVMState = pVM->enmVMState;
-        if (enmVMState == VMSTATE_RESETTING)
-        {
-            if (pUVM->vm.s.enmPrevVMState == VMSTATE_SUSPENDED)
-                vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDED, VMSTATE_RESETTING);
-            else
-                vmR3SetStateLocked(pVM, pUVM, VMSTATE_RUNNING,   VMSTATE_RESETTING);
-        }
-        else
-            vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDING_LS, VMSTATE_RESETTING_LS);
-        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-
-        vmR3CheckIntegrity(pVM);
-
-        /*
-         * Do the suspend bit as well.
-         * It only requires some EMT(0) work at present.
-         */
-        if (enmVMState != VMSTATE_RESETTING)
-        {
-            vmR3SuspendDoWork(pVM);
-            vmR3SetState(pVM, VMSTATE_SUSPENDED_LS, VMSTATE_SUSPENDING_LS);
-        }
-    }
-
-    return enmVMState == VMSTATE_RESETTING
-         ? VINF_EM_RESET
-         : VINF_EM_SUSPEND; /** @todo VINF_EM_SUSPEND has lower priority than VINF_EM_RESET, so fix races. Perhaps add a new code for this combined case. */
-}
-
-
-/**
- * Reset the current VM.
- *
- * @returns VBox status code.
- * @param   pVM     VM to reset.
- */
-VMMR3DECL(int) VMR3Reset(PVM pVM)
-{
-    LogFlow(("VMR3Reset:\n"));
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Gather all the EMTs to make sure there are no races before
-     * changing the VM state.
-     */
-    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
-                                vmR3Reset, NULL);
-    LogFlow(("VMR3Reset: returns %Rrc\n", rc));
-    return rc;
-}
-
-
-/**
- * Gets the current VM state.
- *
- * @returns The current VM state.
- * @param   pVM             VM handle.
- * @thread  Any
- */
-VMMR3DECL(VMSTATE) VMR3GetState(PVM pVM)
-{
-    return pVM->enmVMState;
-}
-
-
-/**
- * Gets the state name string for a VM state.
- *
- * @returns Pointer to the state name. (readonly)
- * @param   enmState        The state.
- */
-VMMR3DECL(const char *) VMR3GetStateName(VMSTATE enmState)
-{
-    switch (enmState)
-    {
-        case VMSTATE_CREATING:          return "CREATING";
-        case VMSTATE_CREATED:           return "CREATED";
-        case VMSTATE_LOADING:           return "LOADING";
-        case VMSTATE_POWERING_ON:       return "POWERING_ON";
-        case VMSTATE_RESUMING:          return "RESUMING";
-        case VMSTATE_RUNNING:           return "RUNNING";
-        case VMSTATE_RUNNING_LS:        return "RUNNING_LS";
-        case VMSTATE_RUNNING_FT:        return "RUNNING_FT";
-        case VMSTATE_RESETTING:         return "RESETTING";
-        case VMSTATE_RESETTING_LS:      return "RESETTING_LS";
-        case VMSTATE_SUSPENDED:         return "SUSPENDED";
-        case VMSTATE_SUSPENDED_LS:      return "SUSPENDED_LS";
-        case VMSTATE_SUSPENDED_EXT_LS:  return "SUSPENDED_EXT_LS";
-        case VMSTATE_SUSPENDING:        return "SUSPENDING";
-        case VMSTATE_SUSPENDING_LS:     return "SUSPENDING_LS";
-        case VMSTATE_SUSPENDING_EXT_LS: return "SUSPENDING_EXT_LS";
-        case VMSTATE_SAVING:            return "SAVING";
-        case VMSTATE_DEBUGGING:         return "DEBUGGING";
-        case VMSTATE_DEBUGGING_LS:      return "DEBUGGING_LS";
-        case VMSTATE_POWERING_OFF:      return "POWERING_OFF";
-        case VMSTATE_POWERING_OFF_LS:   return "POWERING_OFF_LS";
-        case VMSTATE_FATAL_ERROR:       return "FATAL_ERROR";
-        case VMSTATE_FATAL_ERROR_LS:    return "FATAL_ERROR_LS";
-        case VMSTATE_GURU_MEDITATION:   return "GURU_MEDITATION";
-        case VMSTATE_GURU_MEDITATION_LS:return "GURU_MEDITATION_LS";
-        case VMSTATE_LOAD_FAILURE:      return "LOAD_FAILURE";
-        case VMSTATE_OFF:               return "OFF";
-        case VMSTATE_OFF_LS:            return "OFF_LS";
-        case VMSTATE_DESTROYING:        return "DESTROYING";
-        case VMSTATE_TERMINATED:        return "TERMINATED";
-
-        default:
-            AssertMsgFailed(("Unknown state %d\n", enmState));
-            return "Unknown!\n";
-    }
-}
-
-
-/**
- * Validates the state transition in strict builds.
- *
- * @returns true if valid, false if not.
- *
- * @param   enmStateOld         The old (current) state.
- * @param   enmStateNew         The proposed new state.
- *
- * @remarks The reference for this is found in doc/vp/VMM.vpp, the VMSTATE
- *          diagram (under State Machine Diagram).
- */
-static bool vmR3ValidateStateTransition(VMSTATE enmStateOld, VMSTATE enmStateNew)
-{
-#ifdef VBOX_STRICT
-    switch (enmStateOld)
-    {
-        case VMSTATE_CREATING:
-            AssertMsgReturn(enmStateNew == VMSTATE_CREATED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_CREATED:
-            AssertMsgReturn(   enmStateNew == VMSTATE_LOADING
-                            || enmStateNew == VMSTATE_POWERING_ON
-                            || enmStateNew == VMSTATE_POWERING_OFF
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_LOADING:
-            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
-                            || enmStateNew == VMSTATE_LOAD_FAILURE
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_POWERING_ON:
-            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
-                            /*|| enmStateNew == VMSTATE_FATAL_ERROR ?*/
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_RESUMING:
-            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
-                            /*|| enmStateNew == VMSTATE_FATAL_ERROR ?*/
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_RUNNING:
-            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
-                            || enmStateNew == VMSTATE_SUSPENDING
-                            || enmStateNew == VMSTATE_RESETTING
-                            || enmStateNew == VMSTATE_RUNNING_LS
-                            || enmStateNew == VMSTATE_RUNNING_FT
-                            || enmStateNew == VMSTATE_DEBUGGING
-                            || enmStateNew == VMSTATE_FATAL_ERROR
-                            || enmStateNew == VMSTATE_GURU_MEDITATION
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_RUNNING_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF_LS
-                            || enmStateNew == VMSTATE_SUSPENDING_LS
-                            || enmStateNew == VMSTATE_SUSPENDING_EXT_LS
-                            || enmStateNew == VMSTATE_RESETTING_LS
-                            || enmStateNew == VMSTATE_RUNNING
-                            || enmStateNew == VMSTATE_DEBUGGING_LS
-                            || enmStateNew == VMSTATE_FATAL_ERROR_LS
-                            || enmStateNew == VMSTATE_GURU_MEDITATION_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_RUNNING_FT:
-            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
-                            || enmStateNew == VMSTATE_FATAL_ERROR
-                            || enmStateNew == VMSTATE_GURU_MEDITATION
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_RESETTING:
-            AssertMsgReturn(enmStateNew == VMSTATE_RUNNING, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_RESETTING_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_SUSPENDING:
-            AssertMsgReturn(enmStateNew == VMSTATE_SUSPENDED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_SUSPENDING_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING
-                            || enmStateNew == VMSTATE_SUSPENDED_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_SUSPENDING_EXT_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING
-                            || enmStateNew == VMSTATE_SUSPENDED_EXT_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_SUSPENDED:
-            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
-                            || enmStateNew == VMSTATE_SAVING
-                            || enmStateNew == VMSTATE_RESETTING
-                            || enmStateNew == VMSTATE_RESUMING
-                            || enmStateNew == VMSTATE_LOADING
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_SUSPENDED_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
-                            || enmStateNew == VMSTATE_SAVING
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_SUSPENDED_EXT_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
-                            || enmStateNew == VMSTATE_SAVING
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_SAVING:
-            AssertMsgReturn(enmStateNew == VMSTATE_SUSPENDED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_DEBUGGING:
-            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
-                            || enmStateNew == VMSTATE_POWERING_OFF
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_DEBUGGING_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_DEBUGGING
-                            || enmStateNew == VMSTATE_RUNNING_LS
-                            || enmStateNew == VMSTATE_POWERING_OFF_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_POWERING_OFF:
-            AssertMsgReturn(enmStateNew == VMSTATE_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_POWERING_OFF_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
-                            || enmStateNew == VMSTATE_OFF_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_OFF:
-            AssertMsgReturn(enmStateNew == VMSTATE_DESTROYING, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_OFF_LS:
-            AssertMsgReturn(enmStateNew == VMSTATE_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_FATAL_ERROR:
-            AssertMsgReturn(enmStateNew == VMSTATE_POWERING_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_FATAL_ERROR_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_FATAL_ERROR
-                            || enmStateNew == VMSTATE_POWERING_OFF_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_GURU_MEDITATION:
-            AssertMsgReturn(   enmStateNew == VMSTATE_DEBUGGING
-                            || enmStateNew == VMSTATE_POWERING_OFF
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_GURU_MEDITATION_LS:
-            AssertMsgReturn(   enmStateNew == VMSTATE_GURU_MEDITATION
-                            || enmStateNew == VMSTATE_DEBUGGING_LS
-                            || enmStateNew == VMSTATE_POWERING_OFF_LS
-                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_LOAD_FAILURE:
-            AssertMsgReturn(enmStateNew == VMSTATE_POWERING_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_DESTROYING:
-            AssertMsgReturn(enmStateNew == VMSTATE_TERMINATED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-
-        case VMSTATE_TERMINATED:
-        default:
-            AssertMsgFailedReturn(("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
-            break;
-    }
-#endif /* VBOX_STRICT */
-    return true;
-}
-
-
-/**
- * Does the state change callouts.
- *
- * The caller owns the AtStateCritSect.
- *
- * @param   pVM                 The VM handle.
- * @param   pUVM                The UVM handle.
- * @param   enmStateNew         The New state.
- * @param   enmStateOld         The old state.
- */
-static void vmR3DoAtState(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
-{
-    LogRel(("Changing the VM state from '%s' to '%s'.\n", VMR3GetStateName(enmStateOld),  VMR3GetStateName(enmStateNew)));
-
-    for (PVMATSTATE pCur = pUVM->vm.s.pAtState; pCur; pCur = pCur->pNext)
-    {
-        pCur->pfnAtState(pVM, enmStateNew, enmStateOld, pCur->pvUser);
-        if (    enmStateNew     != VMSTATE_DESTROYING
-            &&  pVM->enmVMState == VMSTATE_DESTROYING)
-            break;
-        AssertMsg(pVM->enmVMState == enmStateNew,
-                  ("You are not allowed to change the state while in the change callback, except "
-                   "from destroying the VM. There are restrictions in the way the state changes "
-                   "are propagated up to the EM execution loop and it makes the program flow very "
-                   "difficult to follow. (%s, expected %s, old %s)\n",
-                   VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateNew),
-                   VMR3GetStateName(enmStateOld)));
-    }
-}
-
-
-/**
- * Sets the current VM state, with the AtStatCritSect already entered.
- *
- * @param   pVM                 The VM handle.
- * @param   pUVM                The UVM handle.
- * @param   enmStateNew         The new state.
- * @param   enmStateOld         The old state.
- */
-static void vmR3SetStateLocked(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
-{
-    vmR3ValidateStateTransition(enmStateOld, enmStateNew);
-
-    AssertMsg(pVM->enmVMState == enmStateOld,
-              ("%s != %s\n", VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateOld)));
-    pUVM->vm.s.enmPrevVMState = enmStateOld;
-    pVM->enmVMState           = enmStateNew;
-    VM_FF_CLEAR(pVM, VM_FF_CHECK_VM_STATE);
-
-    vmR3DoAtState(pVM, pUVM, enmStateNew, enmStateOld);
-}
-
-
-/**
- * Sets the current VM state.
- *
- * @param   pVM             VM handle.
- * @param   enmStateNew     The new state.
- * @param   enmStateOld     The old state (for asserting only).
- */
-static void vmR3SetState(PVM pVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
-{
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-
-    AssertMsg(pVM->enmVMState == enmStateOld,
-              ("%s != %s\n", VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateOld)));
-    vmR3SetStateLocked(pVM, pUVM, enmStateNew, pVM->enmVMState);
-
-    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-}
-
-
-/**
- * Tries to perform a state transition.
- *
- * @returns The 1-based ordinal of the succeeding transition.
- *          VERR_VM_INVALID_VM_STATE and Assert+LogRel on failure.
- *
- * @param   pVM                 The VM handle.
- * @param   pszWho              Who is trying to change it.
- * @param   cTransitions        The number of transitions in the ellipsis.
- * @param   ...                 Transition pairs; new, old.
- */
-static int vmR3TrySetState(PVM pVM, const char *pszWho, unsigned cTransitions, ...)
-{
-    va_list va;
-    VMSTATE enmStateNew = VMSTATE_CREATED;
-    VMSTATE enmStateOld = VMSTATE_CREATED;
-
-#ifdef VBOX_STRICT
-    /*
-     * Validate the input first.
-     */
-    va_start(va, cTransitions);
-    for (unsigned i = 0; i < cTransitions; i++)
-    {
-        enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
-        enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
-        vmR3ValidateStateTransition(enmStateOld, enmStateNew);
-    }
-    va_end(va);
-#endif
-
-    /*
-     * Grab the lock and see if any of the proposed transitions works out.
-     */
-    va_start(va, cTransitions);
-    int     rc          = VERR_VM_INVALID_VM_STATE;
-    PUVM    pUVM        = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-
-    VMSTATE enmStateCur = pVM->enmVMState;
-
-    for (unsigned i = 0; i < cTransitions; i++)
-    {
-        enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
-        enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
-        if (enmStateCur == enmStateOld)
-        {
-            vmR3SetStateLocked(pVM, pUVM, enmStateNew, enmStateOld);
-            rc = i + 1;
-            break;
-        }
-    }
-
-    if (RT_FAILURE(rc))
-    {
-        /*
-         * Complain about it.
-         */
-        if (cTransitions == 1)
-        {
-            LogRel(("%s: %s -> %s failed, because the VM state is actually %s\n",
-                    pszWho, VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew), VMR3GetStateName(enmStateCur)));
-            VMSetError(pVM, VERR_VM_INVALID_VM_STATE, RT_SRC_POS,
-                       N_("%s failed because the VM state is %s instead of %s"),
-                       pszWho, VMR3GetStateName(enmStateCur), VMR3GetStateName(enmStateOld));
-            AssertMsgFailed(("%s: %s -> %s failed, because the VM state is actually %s\n",
-                             pszWho, VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew), VMR3GetStateName(enmStateCur)));
-        }
-        else
-        {
-            va_end(va);
-            va_start(va, cTransitions);
-            LogRel(("%s:\n", pszWho));
-            for (unsigned i = 0; i < cTransitions; i++)
-            {
-                enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
-                enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
-                LogRel(("%s%s -> %s",
-                        i ? ", " : " ", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)));
-            }
-            LogRel((" failed, because the VM state is actually %s\n", VMR3GetStateName(enmStateCur)));
-            VMSetError(pVM, VERR_VM_INVALID_VM_STATE, RT_SRC_POS,
-                       N_("%s failed because the current VM state, %s, was not found in the state transition table"),
-                       pszWho, VMR3GetStateName(enmStateCur), VMR3GetStateName(enmStateOld));
-            AssertMsgFailed(("%s - state=%s, see release log for full details. Check the cTransitions passed us.\n",
-                             pszWho, VMR3GetStateName(enmStateCur)));
-        }
-    }
-
-    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-    va_end(va);
-    Assert(rc > 0 || rc < 0);
-    return rc;
-}
-
-
-/**
- * Flag a guru meditation ... a hack.
- *
- * @param   pVM             The VM handle
- *
- * @todo    Rewrite this part. The guru meditation should be flagged
- *          immediately by the VMM and not by VMEmt.cpp when it's all over.
- */
-void vmR3SetGuruMeditation(PVM pVM)
-{
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-
-    VMSTATE enmStateCur = pVM->enmVMState;
-    if (enmStateCur == VMSTATE_RUNNING)
-        vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION, VMSTATE_RUNNING);
-    else if (enmStateCur == VMSTATE_RUNNING_LS)
-    {
-        vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION_LS, VMSTATE_RUNNING_LS);
-        SSMR3Cancel(pVM);
-    }
-
-    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-}
-
-
-/**
- * Called by vmR3EmulationThreadWithId just before the VM structure is freed.
- *
- * @param   pVM             The VM handle.
- */
-void vmR3SetTerminated(PVM pVM)
-{
-    vmR3SetState(pVM, VMSTATE_TERMINATED, VMSTATE_DESTROYING);
-}
-
-
-/**
- * Checks if the VM was teleported and hasn't been fully resumed yet.
- *
- * This applies to both sides of the teleportation since we may leave a working
- * clone behind and the user is allowed to resume this...
- *
- * @returns true / false.
- * @param   pVM                 The VM handle.
- * @thread  Any thread.
- */
-VMMR3DECL(bool) VMR3TeleportedAndNotFullyResumedYet(PVM pVM)
-{
-    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
-    return pVM->vm.s.fTeleportedAndNotFullyResumedYet;
-}
-
-
-/**
- * Registers a VM state change callback.
- *
- * You are not allowed to call any function which changes the VM state from a
- * state callback.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pfnAtState      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
-VMMR3DECL(int) VMR3AtStateRegister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser)
-{
-    LogFlow(("VMR3AtStateRegister: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pfnAtState, VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Allocate a new record.
-     */
-    PUVM pUVM = pVM->pUVM;
-    PVMATSTATE pNew = (PVMATSTATE)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
-    if (!pNew)
-        return VERR_NO_MEMORY;
-
-    /* fill */
-    pNew->pfnAtState = pfnAtState;
-    pNew->pvUser     = pvUser;
-
-    /* insert */
-    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-    pNew->pNext      = *pUVM->vm.s.ppAtStateNext;
-    *pUVM->vm.s.ppAtStateNext = pNew;
-    pUVM->vm.s.ppAtStateNext = &pNew->pNext;
-    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Deregisters a VM state change callback.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   pfnAtState      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
-VMMR3DECL(int) VMR3AtStateDeregister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser)
-{
-    LogFlow(("VMR3AtStateDeregister: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pfnAtState, VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
-
-    /*
-     * Search the list for the entry.
-     */
-    PVMATSTATE pPrev = NULL;
-    PVMATSTATE pCur = pUVM->vm.s.pAtState;
-    while (     pCur
-           &&   (   pCur->pfnAtState != pfnAtState
-                 || pCur->pvUser != pvUser))
-    {
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-    if (!pCur)
-    {
-        AssertMsgFailed(("pfnAtState=%p was not found\n", pfnAtState));
-        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-        return VERR_FILE_NOT_FOUND;
-    }
-
-    /*
-     * Unlink it.
-     */
-    if (pPrev)
-    {
-        pPrev->pNext = pCur->pNext;
-        if (!pCur->pNext)
-            pUVM->vm.s.ppAtStateNext = &pPrev->pNext;
-    }
-    else
-    {
-        pUVM->vm.s.pAtState = pCur->pNext;
-        if (!pCur->pNext)
-            pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
-    }
-
-    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
-
-    /*
-     * Free it.
-     */
-    pCur->pfnAtState = NULL;
-    pCur->pNext = NULL;
-    MMR3HeapFree(pCur);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Registers a VM error callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pfnAtError      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
-VMMR3DECL(int)   VMR3AtErrorRegister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser)
-{
-    return VMR3AtErrorRegisterU(pVM->pUVM, pfnAtError, pvUser);
-}
-
-
-/**
- * Registers a VM error callback.
- *
- * @returns VBox status code.
- * @param   pUVM            The VM handle.
- * @param   pfnAtError      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
-VMMR3DECL(int)   VMR3AtErrorRegisterU(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser)
-{
-    LogFlow(("VMR3AtErrorRegister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pfnAtError, VERR_INVALID_PARAMETER);
-    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Allocate a new record.
-     */
-    PVMATERROR pNew = (PVMATERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
-    if (!pNew)
-        return VERR_NO_MEMORY;
-
-    /* fill */
-    pNew->pfnAtError = pfnAtError;
-    pNew->pvUser     = pvUser;
-
-    /* insert */
-    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
-    pNew->pNext      = *pUVM->vm.s.ppAtErrorNext;
-    *pUVM->vm.s.ppAtErrorNext = pNew;
-    pUVM->vm.s.ppAtErrorNext = &pNew->pNext;
-    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Deregisters a VM error callback.
- *
- * @returns VBox status code.
- * @param   pVM             The VM handle.
- * @param   pfnAtError      Pointer to callback.
- * @param   pvUser          User argument.
- * @thread  Any.
- */
-VMMR3DECL(int) VMR3AtErrorDeregister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser)
-{
-    LogFlow(("VMR3AtErrorDeregister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pfnAtError, VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
-
-    /*
-     * Search the list for the entry.
-     */
-    PVMATERROR pPrev = NULL;
-    PVMATERROR pCur = pUVM->vm.s.pAtError;
-    while (     pCur
-           &&   (   pCur->pfnAtError != pfnAtError
-                 || pCur->pvUser != pvUser))
-    {
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-    if (!pCur)
-    {
-        AssertMsgFailed(("pfnAtError=%p was not found\n", pfnAtError));
-        RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-        return VERR_FILE_NOT_FOUND;
-    }
-
-    /*
-     * Unlink it.
-     */
-    if (pPrev)
-    {
-        pPrev->pNext = pCur->pNext;
-        if (!pCur->pNext)
-            pUVM->vm.s.ppAtErrorNext = &pPrev->pNext;
-    }
-    else
-    {
-        pUVM->vm.s.pAtError = pCur->pNext;
-        if (!pCur->pNext)
-            pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
-    }
-
-    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-
-    /*
-     * Free it.
-     */
-    pCur->pfnAtError = NULL;
-    pCur->pNext = NULL;
-    MMR3HeapFree(pCur);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Ellipsis to va_list wrapper for calling pfnAtError.
- */
-static void vmR3SetErrorWorkerDoCall(PVM pVM, PVMATERROR pCur, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    pCur->pfnAtError(pVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va);
-    va_end(va);
-}
-
-
-/**
- * This is a worker function for GC and Ring-0 calls to VMSetError and VMSetErrorV.
- * The message is found in VMINT.
- *
- * @param   pVM             The VM handle.
- * @thread  EMT.
- */
-VMMR3DECL(void) VMR3SetErrorWorker(PVM pVM)
-{
-    VM_ASSERT_EMT(pVM);
-    AssertReleaseMsgFailed(("And we have a winner! You get to implement Ring-0 and GC VMSetErrorV! Contracts!\n"));
-
-    /*
-     * Unpack the error (if we managed to format one).
-     */
-    PVMERROR pErr = pVM->vm.s.pErrorR3;
-    const char *pszFile = NULL;
-    const char *pszFunction = NULL;
-    uint32_t    iLine = 0;
-    const char *pszMessage;
-    int32_t     rc = VERR_MM_HYPER_NO_MEMORY;
-    if (pErr)
-    {
-        AssertCompile(sizeof(const char) == sizeof(uint8_t));
-        if (pErr->offFile)
-            pszFile = (const char *)pErr + pErr->offFile;
-        iLine = pErr->iLine;
-        if (pErr->offFunction)
-            pszFunction = (const char *)pErr + pErr->offFunction;
-        if (pErr->offMessage)
-            pszMessage = (const char *)pErr + pErr->offMessage;
-        else
-            pszMessage = "No message!";
-    }
-    else
-        pszMessage = "No message! (Failed to allocate memory to put the error message in!)";
-
-    /*
-     * Call the at error callbacks.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
-    ASMAtomicIncU32(&pUVM->vm.s.cRuntimeErrors);
-    for (PVMATERROR pCur = pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
-        vmR3SetErrorWorkerDoCall(pVM, pCur, rc, RT_SRC_POS_ARGS, "%s", pszMessage);
-    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-}
-
-
-/**
- * Gets the number of errors raised via VMSetError.
- *
- * This can be used avoid double error messages.
- *
- * @returns The error count.
- * @param   pVM             The VM handle.
- */
-VMMR3DECL(uint32_t) VMR3GetErrorCount(PVM pVM)
-{
-    return pVM->pUVM->vm.s.cErrors;
-}
-
-
-/**
- * Creation time wrapper for vmR3SetErrorUV.
- *
- * @returns rc.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   rc              The VBox status code.
- * @param   RT_SRC_POS_DECL The source position of this error.
- * @param   pszFormat       Format string.
- * @param   ...             The arguments.
- * @thread  Any thread.
- */
-static int vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    vmR3SetErrorUV(pUVM, rc, pszFile, iLine, pszFunction, pszFormat, &va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Worker which calls everyone listening to the VM error messages.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   rc              The VBox status code.
- * @param   RT_SRC_POS_DECL The source position of this error.
- * @param   pszFormat       Format string.
- * @param   pArgs           Pointer to the format arguments.
- * @thread  EMT
- */
-DECLCALLBACK(void) vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *pArgs)
-{
-    /*
-     * Log the error.
-     */
-    va_list va3;
-    va_copy(va3, *pArgs);
-    RTLogRelPrintf("VMSetError: %s(%d) %s; rc=%Rrc\n"
-                   "VMSetError: %N\n",
-                   pszFile, iLine, pszFunction, rc,
-                   pszFormat, &va3);
-    va_end(va3);
-
-#ifdef LOG_ENABLED
-    va_copy(va3, *pArgs);
-    RTLogPrintf("VMSetError: %s(%d) %s; rc=%Rrc\n"
-                "%N\n",
-                pszFile, iLine, pszFunction, rc,
-                pszFormat, &va3);
-    va_end(va3);
-#endif
-
-    /*
-     * Make a copy of the message.
-     */
-    if (pUVM->pVM)
-        vmSetErrorCopy(pUVM->pVM, rc, RT_SRC_POS_ARGS, pszFormat, *pArgs);
-
-    /*
-     * Call the at error callbacks.
-     */
-    bool fCalledSomeone = false;
-    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
-    ASMAtomicIncU32(&pUVM->vm.s.cErrors);
-    for (PVMATERROR pCur = pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
-    {
-        va_list va2;
-        va_copy(va2, *pArgs);
-        pCur->pfnAtError(pUVM->pVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va2);
-        va_end(va2);
-        fCalledSomeone = true;
-    }
-    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-}
-
-
-/**
- * Registers a VM runtime error callback.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pfnAtRuntimeError   Pointer to callback.
- * @param   pvUser              User argument.
- * @thread  Any.
- */
-VMMR3DECL(int)   VMR3AtRuntimeErrorRegister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
-{
-    LogFlow(("VMR3AtRuntimeErrorRegister: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pfnAtRuntimeError, VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    /*
-     * Allocate a new record.
-     */
-    PUVM pUVM = pVM->pUVM;
-    PVMATRUNTIMEERROR pNew = (PVMATRUNTIMEERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
-    if (!pNew)
-        return VERR_NO_MEMORY;
-
-    /* fill */
-    pNew->pfnAtRuntimeError = pfnAtRuntimeError;
-    pNew->pvUser            = pvUser;
-
-    /* insert */
-    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
-    pNew->pNext             = *pUVM->vm.s.ppAtRuntimeErrorNext;
-    *pUVM->vm.s.ppAtRuntimeErrorNext = pNew;
-    pUVM->vm.s.ppAtRuntimeErrorNext = &pNew->pNext;
-    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Deregisters a VM runtime error callback.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   pfnAtRuntimeError   Pointer to callback.
- * @param   pvUser              User argument.
- * @thread  Any.
- */
-VMMR3DECL(int)   VMR3AtRuntimeErrorDeregister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
-{
-    LogFlow(("VMR3AtRuntimeErrorDeregister: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pfnAtRuntimeError, VERR_INVALID_PARAMETER);
-    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
-
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
-
-    /*
-     * Search the list for the entry.
-     */
-    PVMATRUNTIMEERROR pPrev = NULL;
-    PVMATRUNTIMEERROR pCur = pUVM->vm.s.pAtRuntimeError;
-    while (     pCur
-           &&   (   pCur->pfnAtRuntimeError != pfnAtRuntimeError
-                 || pCur->pvUser != pvUser))
-    {
-        pPrev = pCur;
-        pCur = pCur->pNext;
-    }
-    if (!pCur)
-    {
-        AssertMsgFailed(("pfnAtRuntimeError=%p was not found\n", pfnAtRuntimeError));
-        RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-        return VERR_FILE_NOT_FOUND;
-    }
-
-    /*
-     * Unlink it.
-     */
-    if (pPrev)
-    {
-        pPrev->pNext = pCur->pNext;
-        if (!pCur->pNext)
-            pUVM->vm.s.ppAtRuntimeErrorNext = &pPrev->pNext;
-    }
-    else
-    {
-        pUVM->vm.s.pAtRuntimeError = pCur->pNext;
-        if (!pCur->pNext)
-            pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
-    }
-
-    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-
-    /*
-     * Free it.
-     */
-    pCur->pfnAtRuntimeError = NULL;
-    pCur->pNext = NULL;
-    MMR3HeapFree(pCur);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * EMT rendezvous worker that vmR3SetRuntimeErrorCommon uses to safely change
- * the state to FatalError(LS).
- *
- * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_SUSPEND.  (This is a strict
- *          return code, see FNVMMEMTRENDEZVOUS.)
- *
- * @param   pVM             The VM handle.
- * @param   pVCpu           The VMCPU handle of the EMT.
- * @param   pvUser          Ignored.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3SetRuntimeErrorChangeState(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    NOREF(pVCpu);
-    Assert(!pvUser); NOREF(pvUser);
-
-    /*
-     * The first EMT thru here changes the state.
-     */
-    if (pVCpu->idCpu == pVM->cCpus - 1)
-    {
-        int rc = vmR3TrySetState(pVM, "VMSetRuntimeError", 2,
-                                 VMSTATE_FATAL_ERROR,    VMSTATE_RUNNING,
-                                 VMSTATE_FATAL_ERROR_LS, VMSTATE_RUNNING_LS);
-        if (RT_FAILURE(rc))
-            return rc;
-        if (rc == 2)
-            SSMR3Cancel(pVM);
-
-        VM_FF_SET(pVM, VM_FF_CHECK_VM_STATE);
-    }
-
-    /* This'll make sure we get out of whereever we are (e.g. REM). */
-    return VINF_EM_SUSPEND;
-}
-
-
-/**
- * Worker for VMR3SetRuntimeErrorWorker and vmR3SetRuntimeErrorV.
- *
- * This does the common parts after the error has been saved / retrieved.
- *
- * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
- *
- * @param   pVM             The VM handle.
- * @param   fFlags          The error flags.
- * @param   pszErrorId      Error ID string.
- * @param   pszFormat       Format string.
- * @param   pVa             Pointer to the format arguments.
- */
-static int vmR3SetRuntimeErrorCommon(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa)
-{
-    LogRel(("VM: Raising runtime error '%s' (fFlags=%#x)\n", pszErrorId, fFlags));
-
-    /*
-     * Take actions before the call.
-     */
-    int rc;
-    if (fFlags & VMSETRTERR_FLAGS_FATAL)
-        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
-                                vmR3SetRuntimeErrorChangeState, NULL);
-    else if (fFlags & VMSETRTERR_FLAGS_SUSPEND)
-        rc = VMR3Suspend(pVM);
-    else
-        rc = VINF_SUCCESS;
-
-    /*
-     * Do the callback round.
-     */
-    PUVM pUVM = pVM->pUVM;
-    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
-    ASMAtomicIncU32(&pUVM->vm.s.cRuntimeErrors);
-    for (PVMATRUNTIMEERROR pCur = pUVM->vm.s.pAtRuntimeError; pCur; pCur = pCur->pNext)
-    {
-        va_list va;
-        va_copy(va, *pVa);
-        pCur->pfnAtRuntimeError(pVM, pCur->pvUser, fFlags, pszErrorId, pszFormat, va);
-        va_end(va);
-    }
-    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
-
-    return rc;
-}
-
-
-/**
- * Ellipsis to va_list wrapper for calling vmR3SetRuntimeErrorCommon.
- */
-static int vmR3SetRuntimeErrorCommonF(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
-{
-    va_list va;
-    va_start(va, pszFormat);
-    int rc = vmR3SetRuntimeErrorCommon(pVM, fFlags, pszErrorId, pszFormat, &va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * This is a worker function for RC and Ring-0 calls to VMSetError and
- * VMSetErrorV.
- *
- * The message is found in VMINT.
- *
- * @returns VBox status code, see VMSetRuntimeError.
- * @param   pVM             The VM handle.
- * @thread  EMT.
- */
-VMMR3DECL(int) VMR3SetRuntimeErrorWorker(PVM pVM)
-{
-    VM_ASSERT_EMT(pVM);
-    AssertReleaseMsgFailed(("And we have a winner! You get to implement Ring-0 and GC VMSetRuntimeErrorV! Congrats!\n"));
-
-    /*
-     * Unpack the error (if we managed to format one).
-     */
-    const char     *pszErrorId = "SetRuntimeError";
-    const char     *pszMessage = "No message!";
-    uint32_t        fFlags     = VMSETRTERR_FLAGS_FATAL;
-    PVMRUNTIMEERROR pErr       = pVM->vm.s.pRuntimeErrorR3;
-    if (pErr)
-    {
-        AssertCompile(sizeof(const char) == sizeof(uint8_t));
-        if (pErr->offErrorId)
-            pszErrorId = (const char *)pErr + pErr->offErrorId;
-        if (pErr->offMessage)
-            pszMessage = (const char *)pErr + pErr->offMessage;
-        fFlags = pErr->fFlags;
-    }
-
-    /*
-     * Join cause with vmR3SetRuntimeErrorV.
-     */
-    return vmR3SetRuntimeErrorCommonF(pVM, fFlags, pszErrorId, "%s", pszMessage);
-}
-
-
-/**
- * Worker for VMSetRuntimeErrorV for doing the job on EMT in ring-3.
- *
- * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
- *
- * @param   pVM             The VM handle.
- * @param   fFlags          The error flags.
- * @param   pszErrorId      Error ID string.
- * @param   pszMessage      The error message residing the MM heap.
- *
- * @thread  EMT
- */
-DECLCALLBACK(int) vmR3SetRuntimeError(PVM pVM, uint32_t fFlags, const char *pszErrorId, char *pszMessage)
-{
-#if 0 /** @todo make copy of the error msg. */
-    /*
-     * Make a copy of the message.
-     */
-    va_list va2;
-    va_copy(va2, *pVa);
-    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va2);
-    va_end(va2);
-#endif
-
-    /*
-     * Join paths with VMR3SetRuntimeErrorWorker.
-     */
-    int rc = vmR3SetRuntimeErrorCommonF(pVM, fFlags, pszErrorId, "%s", pszMessage);
-    MMR3HeapFree(pszMessage);
-    return rc;
-}
-
-
-/**
- * Worker for VMSetRuntimeErrorV for doing the job on EMT in ring-3.
- *
- * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
- *
- * @param   pVM             The VM handle.
- * @param   fFlags          The error flags.
- * @param   pszErrorId      Error ID string.
- * @param   pszFormat       Format string.
- * @param   pVa             Pointer to the format arguments.
- *
- * @thread  EMT
- */
-DECLCALLBACK(int) vmR3SetRuntimeErrorV(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa)
-{
-    /*
-     * Make a copy of the message.
-     */
-    va_list va2;
-    va_copy(va2, *pVa);
-    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va2);
-    va_end(va2);
-
-    /*
-     * Join paths with VMR3SetRuntimeErrorWorker.
-     */
-    return vmR3SetRuntimeErrorCommon(pVM, fFlags, pszErrorId, pszFormat, pVa);
-}
-
-
-/**
- * Gets the number of runtime errors raised via VMR3SetRuntimeError.
- *
- * This can be used avoid double error messages.
- *
- * @returns The runtime error count.
- * @param   pVM             The VM handle.
- */
-VMMR3DECL(uint32_t) VMR3GetRuntimeErrorCount(PVM pVM)
-{
-    return pVM->pUVM->vm.s.cRuntimeErrors;
-}
-
-
-/**
- * Gets the ID virtual of the virtual CPU associated with the calling thread.
- *
- * @returns The CPU ID. NIL_VMCPUID if the thread isn't an EMT.
- *
- * @param   pVM             The VM handle.
- */
-VMMR3DECL(RTCPUID) VMR3GetVMCPUId(PVM pVM)
-{
-    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
-    return pUVCpu
-         ? pUVCpu->idCpu
-         : NIL_VMCPUID;
-}
-
-
-/**
- * Returns the native handle of the current EMT VMCPU thread.
- *
- * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
- * @param   pVM             The VM handle.
- * @thread  EMT
- */
-VMMR3DECL(RTNATIVETHREAD) VMR3GetVMCPUNativeThread(PVM pVM)
-{
-    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
-
-    if (!pUVCpu)
-        return NIL_RTNATIVETHREAD;
-
-    return pUVCpu->vm.s.NativeThreadEMT;
-}
-
-
-/**
- * Returns the native handle of the current EMT VMCPU thread.
- *
- * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
- * @param   pVM             The VM handle.
- * @thread  EMT
- */
-VMMR3DECL(RTNATIVETHREAD) VMR3GetVMCPUNativeThreadU(PUVM pUVM)
-{
-    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
-
-    if (!pUVCpu)
-        return NIL_RTNATIVETHREAD;
-
-    return pUVCpu->vm.s.NativeThreadEMT;
-}
-
-
-/**
- * Returns the handle of the current EMT VMCPU thread.
- *
- * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
- * @param   pVM             The VM handle.
- * @thread  EMT
- */
-VMMR3DECL(RTTHREAD) VMR3GetVMCPUThread(PVM pVM)
-{
-    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
-
-    if (!pUVCpu)
-        return NIL_RTTHREAD;
-
-    return pUVCpu->vm.s.ThreadEMT;
-}
-
-
-/**
- * Returns the handle of the current EMT VMCPU thread.
- *
- * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
- * @param   pVM             The VM handle.
- * @thread  EMT
- */
-VMMR3DECL(RTTHREAD) VMR3GetVMCPUThreadU(PUVM pUVM)
-{
-    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
-
-    if (!pUVCpu)
-        return NIL_RTTHREAD;
-
-    return pUVCpu->vm.s.ThreadEMT;
-}
-
-
-/**
- * Return the package and core id of a CPU.
- *
- * @returns VBOX status code.
- * @param   pVM              The VM to operate on.
- * @param   idCpu            Virtual CPU to get the ID from.
- * @param   pidCpuCore       Where to store the core ID of the virtual CPU.
- * @param   pidCpuPackage    Where to store the package ID of the virtual CPU.
- *
- */
-VMMR3DECL(int) VMR3GetCpuCoreAndPackageIdFromCpuId(PVM pVM, VMCPUID idCpu, uint32_t *pidCpuCore, uint32_t *pidCpuPackage)
-{
-    if (idCpu >= pVM->cCpus)
-        return VERR_INVALID_CPU_ID;
-
-#ifdef VBOX_WITH_MULTI_CORE
-    *pidCpuCore    = idCpu;
-    *pidCpuPackage = 0;
-#else
-    *pidCpuCore    = 0;
-    *pidCpuPackage = idCpu;
-#endif
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Worker for VMR3HotUnplugCpu.
- *
- * @returns VINF_EM_WAIT_SPIP (strict status code).
- * @param   pVM                 The VM handle.
- * @param   idCpu               The current CPU.
- */
-static DECLCALLBACK(int) vmR3HotUnplugCpu(PVM pVM, VMCPUID idCpu)
-{
-    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-    VMCPU_ASSERT_EMT(pVCpu);
-
-    /*
-     * Reset per CPU resources.
-     *
-     * Actually only needed for VT-x because the CPU seems to be still in some
-     * paged mode and startup fails after a new hot plug event. SVM works fine
-     * even without this.
-     */
-    Log(("vmR3HotUnplugCpu for VCPU %u\n", idCpu));
-    PGMR3ResetUnpluggedCpu(pVM, pVCpu);
-    PDMR3ResetCpu(pVCpu);
-    TRPMR3ResetCpu(pVCpu);
-    CPUMR3ResetCpu(pVCpu);
-    EMR3ResetCpu(pVCpu);
-    HWACCMR3ResetCpu(pVCpu);
-    return VINF_EM_WAIT_SIPI;
-}
-
-
-/**
- * Hot-unplugs a CPU from the guest.
- *
- * @returns VBox status code.
- * @param   pVM     The VM to operate on.
- * @param   idCpu   Virtual CPU to perform the hot unplugging operation on.
- */
-VMMR3DECL(int) VMR3HotUnplugCpu(PVM pVM, VMCPUID idCpu)
-{
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-
-    /** @todo r=bird: Don't destroy the EMT, it'll break VMMR3EmtRendezvous and
-     *        broadcast requests.  Just note down somewhere that the CPU is
-     *        offline and send it to SPIP wait.  Maybe modify VMCPUSTATE and push
-     *        it out of the EM loops when offline. */
-    return VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmR3HotUnplugCpu, 2, pVM, idCpu);
-}
-
-
-/**
- * Hot-plugs a CPU on the guest.
- *
- * @returns VBox status code.
- * @param   pVM     The VM to operate on.
- * @param   idCpu   Virtual CPU to perform the hot plugging operation on.
- */
-VMMR3DECL(int) VMR3HotPlugCpu(PVM pVM, VMCPUID idCpu)
-{
-    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
-
-    /** @todo r-bird: Just mark it online and make sure it waits on SPIP. */
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Changes the VMM execution cap.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM to operate on.
- * @param   ulCpuExecutionCap   New CPU execution cap
- */
-VMMR3DECL(int) VMR3SetCpuExecutionCap(PVM pVM, unsigned ulCpuExecutionCap)
-{
-    AssertReturn(ulCpuExecutionCap > 0 && ulCpuExecutionCap <= 100, VERR_INVALID_PARAMETER);
-
-    Log(("VMR3SetCpuExecutionCap: new priority = %d\n", ulCpuExecutionCap));
-    /* Note: not called from EMT. */
-    pVM->uCpuExecutionCap = ulCpuExecutionCap;
-    return VINF_SUCCESS;
-}
-
Index: unk/src/VBox/VMM/VMEmt.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMEmt.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1278 +1,0 @@
-/* $Id$ */
-/** @file
- * VM - Virtual Machine, The Emulation Thread.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_VM
-#include <VBox/tm.h>
-#include <VBox/dbgf.h>
-#include <VBox/em.h>
-#include <VBox/pdmapi.h>
-#include <VBox/rem.h>
-#include <VBox/tm.h>
-#include "VMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-
-#include <VBox/err.h>
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <iprt/asm-math.h>
-#include <iprt/semaphore.h>
-#include <iprt/string.h>
-#include <iprt/thread.h>
-#include <iprt/time.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-int vmR3EmulationThreadWithId(RTTHREAD ThreadSelf, PUVMCPU pUVCpu, VMCPUID idCpu);
-
-
-/**
- * The emulation thread main function.
- *
- * @returns Thread exit code.
- * @param   ThreadSelf  The handle to the executing thread.
- * @param   pvArgs      Pointer to the user mode per-VCpu structure (UVMPCU).
- */
-DECLCALLBACK(int) vmR3EmulationThread(RTTHREAD ThreadSelf, void *pvArgs)
-{
-    PUVMCPU pUVCpu = (PUVMCPU)pvArgs;
-    return vmR3EmulationThreadWithId(ThreadSelf, pUVCpu, pUVCpu->idCpu);
-}
-
-
-/**
- * The emulation thread main function, with Virtual CPU ID for debugging.
- *
- * @returns Thread exit code.
- * @param   ThreadSelf  The handle to the executing thread.
- * @param   pUVCpu      Pointer to the user mode per-VCpu structure.
- * @param   idCpu       The virtual CPU ID, for backtrace purposes.
- */
-int vmR3EmulationThreadWithId(RTTHREAD ThreadSelf, PUVMCPU pUVCpu, VMCPUID idCpu)
-{
-    PUVM    pUVM = pUVCpu->pUVM;
-    int     rc;
-
-    AssertReleaseMsg(VALID_PTR(pUVM) && pUVM->u32Magic == UVM_MAGIC,
-                     ("Invalid arguments to the emulation thread!\n"));
-
-    rc = RTTlsSet(pUVM->vm.s.idxTLS, pUVCpu);
-    AssertReleaseMsgRCReturn(rc, ("RTTlsSet %x failed with %Rrc\n", pUVM->vm.s.idxTLS, rc), rc);
-
-    /*
-     * The request loop.
-     */
-    rc = VINF_SUCCESS;
-    Log(("vmR3EmulationThread: Emulation thread starting the days work... Thread=%#x pUVM=%p\n", ThreadSelf, pUVM));
-    VMSTATE enmBefore = VMSTATE_CREATED; /* (only used for logging atm.) */
-    for (;;)
-    {
-        /*
-         * During early init there is no pVM, so make a special path
-         * for that to keep things clearly separate.
-         */
-        if (!pUVM->pVM)
-        {
-            /*
-             * Check for termination first.
-             */
-            if (pUVM->vm.s.fTerminateEMT)
-            {
-                rc = VINF_EM_TERMINATE;
-                break;
-            }
-
-            /*
-             * Only the first VCPU may initialize the VM during early init
-             * and must therefore service all VMCPUID_ANY requests.
-             * See also VMR3Create
-             */
-            if (    pUVM->vm.s.pReqs
-                &&  pUVCpu->idCpu == 0)
-            {
-                /*
-                 * Service execute in any EMT request.
-                 */
-                rc = VMR3ReqProcessU(pUVM, VMCPUID_ANY);
-                Log(("vmR3EmulationThread: Req rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), pUVM->pVM ? VMR3GetStateName(pUVM->pVM->enmVMState) : "CREATING"));
-            }
-            else if (pUVCpu->vm.s.pReqs)
-            {
-                /*
-                 * Service execute in specific EMT request.
-                 */
-                rc = VMR3ReqProcessU(pUVM, pUVCpu->idCpu);
-                Log(("vmR3EmulationThread: Req (cpu=%u) rc=%Rrc, VM state %s -> %s\n", pUVCpu->idCpu, rc, VMR3GetStateName(enmBefore), pUVM->pVM ? VMR3GetStateName(pUVM->pVM->enmVMState) : "CREATING"));
-            }
-            else
-            {
-                /*
-                 * Nothing important is pending, so wait for something.
-                 */
-                rc = VMR3WaitU(pUVCpu);
-                if (RT_FAILURE(rc))
-                {
-                    AssertLogRelMsgFailed(("VMR3WaitU failed with %Rrc\n", rc));
-                    break;
-                }
-            }
-        }
-        else
-        {
-            /*
-             * Pending requests which needs servicing?
-             *
-             * We check for state changes in addition to status codes when
-             * servicing requests. (Look after the ifs.)
-             */
-            PVM pVM = pUVM->pVM;
-            enmBefore = pVM->enmVMState;
-            if (pUVM->vm.s.fTerminateEMT)
-            {
-                rc = VINF_EM_TERMINATE;
-                break;
-            }
-
-            if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
-            {
-                rc = VMMR3EmtRendezvousFF(pVM, &pVM->aCpus[idCpu]);
-                Log(("vmR3EmulationThread: Rendezvous rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
-            }
-            else if (pUVM->vm.s.pReqs)
-            {
-                /*
-                 * Service execute in any EMT request.
-                 */
-                rc = VMR3ReqProcessU(pUVM, VMCPUID_ANY);
-                Log(("vmR3EmulationThread: Req rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
-            }
-            else if (pUVCpu->vm.s.pReqs)
-            {
-                /*
-                 * Service execute in specific EMT request.
-                 */
-                rc = VMR3ReqProcessU(pUVM, pUVCpu->idCpu);
-                Log(("vmR3EmulationThread: Req (cpu=%u) rc=%Rrc, VM state %s -> %s\n", pUVCpu->idCpu, rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
-            }
-            else if (VM_FF_ISSET(pVM, VM_FF_DBGF))
-            {
-                /*
-                 * Service the debugger request.
-                 */
-                rc = DBGFR3VMMForcedAction(pVM);
-                Log(("vmR3EmulationThread: Dbg rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
-            }
-            else if (VM_FF_TESTANDCLEAR(pVM, VM_FF_RESET))
-            {
-                /*
-                 * Service a delayed reset request.
-                 */
-                rc = VMR3Reset(pVM);
-                VM_FF_CLEAR(pVM, VM_FF_RESET);
-                Log(("vmR3EmulationThread: Reset rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
-            }
-            else
-            {
-                /*
-                 * Nothing important is pending, so wait for something.
-                 */
-                rc = VMR3WaitU(pUVCpu);
-                if (RT_FAILURE(rc))
-                {
-                    AssertLogRelMsgFailed(("VMR3WaitU failed with %Rrc\n", rc));
-                    break;
-                }
-            }
-
-            /*
-             * Check for termination requests, these have extremely high priority.
-             */
-            if (    rc == VINF_EM_TERMINATE
-                ||  pUVM->vm.s.fTerminateEMT)
-                break;
-        }
-
-        /*
-         * Some requests (both VMR3Req* and the DBGF) can potentially resume
-         * or start the VM, in that case we'll get a change in VM status
-         * indicating that we're now running.
-         */
-        if (    RT_SUCCESS(rc)
-            &&  pUVM->pVM)
-        {
-            PVM     pVM   = pUVM->pVM;
-            PVMCPU  pVCpu = &pVM->aCpus[idCpu];
-            if (    pVM->enmVMState == VMSTATE_RUNNING
-                &&  VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(pVCpu)))
-            {
-                rc = EMR3ExecuteVM(pVM, pVCpu);
-                Log(("vmR3EmulationThread: EMR3ExecuteVM() -> rc=%Rrc, enmVMState=%d\n", rc, pVM->enmVMState));
-                if (EMGetState(pVCpu) == EMSTATE_GURU_MEDITATION)
-                    vmR3SetGuruMeditation(pVM);
-            }
-        }
-
-    } /* forever */
-
-
-    /*
-     * Cleanup and exit.
-     */
-    Log(("vmR3EmulationThread: Terminating emulation thread! Thread=%#x pUVM=%p rc=%Rrc enmBefore=%d enmVMState=%d\n",
-         ThreadSelf, pUVM, rc, enmBefore, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_TERMINATED));
-    if (   idCpu == 0
-        && pUVM->pVM)
-    {
-        PVM pVM = pUVM->pVM;
-        vmR3SetTerminated(pVM);
-        pUVM->pVM = NULL;
-
-        /** @todo SMP: This isn't 100% safe. We should wait for the other
-         *        threads to finish before destroy the VM. */
-        int rc2 = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
-        AssertLogRelRC(rc2);
-    }
-
-    pUVCpu->vm.s.NativeThreadEMT = NIL_RTNATIVETHREAD;
-    Log(("vmR3EmulationThread: EMT is terminated.\n"));
-    return rc;
-}
-
-
-/**
- * Gets the name of a halt method.
- *
- * @returns Pointer to a read only string.
- * @param   enmMethod   The method.
- */
-static const char *vmR3GetHaltMethodName(VMHALTMETHOD enmMethod)
-{
-    switch (enmMethod)
-    {
-        case VMHALTMETHOD_BOOTSTRAP:    return "bootstrap";
-        case VMHALTMETHOD_DEFAULT:      return "default";
-        case VMHALTMETHOD_OLD:          return "old";
-        case VMHALTMETHOD_1:            return "method1";
-        //case VMHALTMETHOD_2:            return "method2";
-        case VMHALTMETHOD_GLOBAL_1:     return "global1";
-        default:                        return "unknown";
-    }
-}
-
-
-/**
- * Signal a fatal wait error.
- *
- * @returns Fatal error code to be propagated up the call stack.
- * @param   pUVCpu              The user mode per CPU structure of the calling
- *                              EMT.
- * @param   pszFmt              The error format with a single %Rrc in it.
- * @param   rcFmt               The status code to format.
- */
-static int vmR3FatalWaitError(PUVMCPU pUVCpu, const char *pszFmt, int rcFmt)
-{
-    /** @todo This is wrong ... raise a fatal error / guru meditation
-     *        instead. */
-    AssertLogRelMsgFailed((pszFmt, rcFmt));
-    ASMAtomicUoWriteBool(&pUVCpu->pUVM->vm.s.fTerminateEMT, true);
-    if (pUVCpu->pVM)
-        VM_FF_SET(pUVCpu->pVM, VM_FF_CHECK_VM_STATE);
-    return VERR_INTERNAL_ERROR;
-}
-
-
-/**
- * The old halt loop.
- */
-static DECLCALLBACK(int) vmR3HaltOldDoHalt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t /* u64Now*/)
-{
-    /*
-     * Halt loop.
-     */
-    PVM    pVM   = pUVCpu->pVM;
-    PVMCPU pVCpu = pUVCpu->pVCpu;
-
-    int rc = VINF_SUCCESS;
-    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
-    //unsigned cLoops = 0;
-    for (;;)
-    {
-        /*
-         * Work the timers and check if we can exit.
-         * The poll call gives us the ticks left to the next event in
-         * addition to perhaps set an FF.
-         */
-        uint64_t const u64StartTimers   = RTTimeNanoTS();
-        TMR3TimerQueuesDo(pVM);
-        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
-        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-            break;
-        uint64_t u64NanoTS;
-        TMTimerPollGIP(pVM, pVCpu, &u64NanoTS);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-            break;
-
-        /*
-         * Wait for a while. Someone will wake us up or interrupt the call if
-         * anything needs our attention.
-         */
-        if (u64NanoTS < 50000)
-        {
-            //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d spin\n", u64NanoTS, cLoops++);
-            /* spin */;
-        }
-        else
-        {
-            VMMR3YieldStop(pVM);
-            //uint64_t u64Start = RTTimeNanoTS();
-            if (u64NanoTS <  870000) /* this is a bit speculative... works fine on linux. */
-            {
-                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d yield", u64NanoTS, cLoops++);
-                uint64_t const u64StartSchedYield   = RTTimeNanoTS();
-                RTThreadYield(); /* this is the best we can do here */
-                uint64_t const cNsElapsedSchedYield = RTTimeNanoTS() - u64StartSchedYield;
-                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltYield, cNsElapsedSchedYield);
-            }
-            else if (u64NanoTS < 2000000)
-            {
-                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep 1ms", u64NanoTS, cLoops++);
-                uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
-                rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1);
-                uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
-                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
-            }
-            else
-            {
-                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep %dms", u64NanoTS, cLoops++, (uint32_t)RT_MIN((u64NanoTS - 500000) / 1000000, 15));
-                uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
-                rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, RT_MIN((u64NanoTS - 1000000) / 1000000, 15));
-                uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
-                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
-            }
-            //uint64_t u64Slept = RTTimeNanoTS() - u64Start;
-            //RTLogPrintf(" -> rc=%Rrc in %RU64 ns / %RI64 ns delta\n", rc, u64Slept, u64NanoTS - u64Slept);
-        }
-        if (rc == VERR_TIMEOUT)
-            rc = VINF_SUCCESS;
-        else if (RT_FAILURE(rc))
-        {
-            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
-            break;
-        }
-    }
-
-    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
-    return rc;
-}
-
-
-/**
- * Initialize the configuration of halt method 1 & 2.
- *
- * @return VBox status code. Failure on invalid CFGM data.
- * @param   pVM     The VM handle.
- */
-static int vmR3HaltMethod12ReadConfigU(PUVM pUVM)
-{
-    /*
-     * The defaults.
-     */
-#if 1 /* DEBUGGING STUFF - REMOVE LATER */
-    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
-    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   2*1000000;
-    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg =  75*1000000;
-    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  30*1000000;
-    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =  20*1000000;
-#else
-    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
-    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   5*1000000;
-    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = 200*1000000;
-    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  20*1000000;
-    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =   2*1000000;
-#endif
-
-    /*
-     * Query overrides.
-     *
-     * I don't have time to bother with niceties such as invalid value checks
-     * here right now. sorry.
-     */
-    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pUVM->pVM), "/VMM/HaltedMethod1");
-    if (pCfg)
-    {
-        uint32_t u32;
-        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "LagBlockIntervalDivisor", &u32)))
-            pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = u32;
-        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MinBlockInterval", &u32)))
-            pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg = u32;
-        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MaxBlockInterval", &u32)))
-            pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = u32;
-        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StartSpinning", &u32)))
-            pUVM->vm.s.Halt.Method12.u32StartSpinningCfg = u32;
-        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StopSpinning", &u32)))
-            pUVM->vm.s.Halt.Method12.u32StopSpinningCfg = u32;
-        LogRel(("HaltedMethod1 config: %d/%d/%d/%d/%d\n",
-                pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
-                pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
-                pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg,
-                pUVM->vm.s.Halt.Method12.u32StartSpinningCfg,
-                pUVM->vm.s.Halt.Method12.u32StopSpinningCfg));
-    }
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Initialize halt method 1.
- *
- * @return VBox status code.
- * @param   pUVM            Pointer to the user mode VM structure.
- */
-static DECLCALLBACK(int) vmR3HaltMethod1Init(PUVM pUVM)
-{
-    return vmR3HaltMethod12ReadConfigU(pUVM);
-}
-
-
-/**
- * Method 1 - Block whenever possible, and when lagging behind
- * switch to spinning for 10-30ms with occasional blocking until
- * the lag has been eliminated.
- */
-static DECLCALLBACK(int) vmR3HaltMethod1Halt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now)
-{
-    PUVM    pUVM    = pUVCpu->pUVM;
-    PVMCPU  pVCpu   = pUVCpu->pVCpu;
-    PVM     pVM     = pUVCpu->pVM;
-
-    /*
-     * To simplify things, we decide up-front whether we should switch to spinning or
-     * not. This makes some ASSUMPTIONS about the cause of the spinning (PIT/RTC/PCNet)
-     * and that it will generate interrupts or other events that will cause us to exit
-     * the halt loop.
-     */
-    bool fBlockOnce = false;
-    bool fSpinning = false;
-    uint32_t u32CatchUpPct = TMVirtualSyncGetCatchUpPct(pVM);
-    if (u32CatchUpPct /* non-zero if catching up */)
-    {
-        if (pUVCpu->vm.s.Halt.Method12.u64StartSpinTS)
-        {
-            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StopSpinningCfg;
-            if (fSpinning)
-            {
-                uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
-                fBlockOnce = u64Now - pUVCpu->vm.s.Halt.Method12.u64LastBlockTS
-                           > RT_MAX(pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
-                                    RT_MIN(u64Lag / pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
-                                           pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg));
-            }
-            else
-            {
-                //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVCpu->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
-                pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = 0;
-            }
-        }
-        else
-        {
-            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StartSpinningCfg;
-            if (fSpinning)
-                pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = u64Now;
-        }
-    }
-    else if (pUVCpu->vm.s.Halt.Method12.u64StartSpinTS)
-    {
-        //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVCpu->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
-        pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = 0;
-    }
-
-    /*
-     * Halt loop.
-     */
-    int rc = VINF_SUCCESS;
-    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
-    unsigned cLoops = 0;
-    for (;; cLoops++)
-    {
-        /*
-         * Work the timers and check if we can exit.
-         */
-        uint64_t const u64StartTimers   = RTTimeNanoTS();
-        TMR3TimerQueuesDo(pVM);
-        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
-        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-            break;
-
-        /*
-         * Estimate time left to the next event.
-         */
-        uint64_t u64NanoTS;
-        TMTimerPollGIP(pVM, pVCpu, &u64NanoTS);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-            break;
-
-        /*
-         * Block if we're not spinning and the interval isn't all that small.
-         */
-        if (    (   !fSpinning
-                 || fBlockOnce)
-#if 1 /* DEBUGGING STUFF - REMOVE LATER */
-            &&  u64NanoTS >= 100000) /* 0.100 ms */
-#else
-            &&  u64NanoTS >= 250000) /* 0.250 ms */
-#endif
-        {
-            const uint64_t Start = pUVCpu->vm.s.Halt.Method12.u64LastBlockTS = RTTimeNanoTS();
-            VMMR3YieldStop(pVM);
-
-            uint32_t cMilliSecs = RT_MIN(u64NanoTS / 1000000, 15);
-            if (cMilliSecs <= pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg)
-                cMilliSecs = 1;
-            else
-                cMilliSecs -= pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg;
-
-            //RTLogRelPrintf("u64NanoTS=%RI64 cLoops=%3d sleep %02dms (%7RU64) ", u64NanoTS, cLoops, cMilliSecs, u64NanoTS);
-            uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
-            rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, cMilliSecs);
-            uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
-            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
-
-            if (rc == VERR_TIMEOUT)
-                rc = VINF_SUCCESS;
-            else if (RT_FAILURE(rc))
-            {
-                rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
-                break;
-            }
-
-            /*
-             * Calc the statistics.
-             * Update averages every 16th time, and flush parts of the history every 64th time.
-             */
-            const uint64_t Elapsed = RTTimeNanoTS() - Start;
-            pUVCpu->vm.s.Halt.Method12.cNSBlocked += Elapsed;
-            if (Elapsed > u64NanoTS)
-                pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong += Elapsed - u64NanoTS;
-            pUVCpu->vm.s.Halt.Method12.cBlocks++;
-            if (!(pUVCpu->vm.s.Halt.Method12.cBlocks & 0xf))
-            {
-                pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg = pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong / pUVCpu->vm.s.Halt.Method12.cBlocks;
-                if (!(pUVCpu->vm.s.Halt.Method12.cBlocks & 0x3f))
-                {
-                    pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong = pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg * 0x40;
-                    pUVCpu->vm.s.Halt.Method12.cBlocks = 0x40;
-                }
-            }
-            //RTLogRelPrintf(" -> %7RU64 ns / %7RI64 ns delta%s\n", Elapsed, Elapsed - u64NanoTS, fBlockOnce ? " (block once)" : "");
-
-            /*
-             * Clear the block once flag if we actually blocked.
-             */
-            if (    fBlockOnce
-                &&  Elapsed > 100000 /* 0.1 ms */)
-                fBlockOnce = false;
-        }
-    }
-    //if (fSpinning) RTLogRelPrintf("spun for %RU64 ns %u loops; lag=%RU64 pct=%d\n", RTTimeNanoTS() - u64Now, cLoops, TMVirtualSyncGetLag(pVM), u32CatchUpPct);
-
-    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
-    return rc;
-}
-
-
-/**
- * Initialize the global 1 halt method.
- *
- * @return VBox status code.
- * @param   pUVM            Pointer to the user mode VM structure.
- */
-static DECLCALLBACK(int) vmR3HaltGlobal1Init(PUVM pUVM)
-{
-    /*
-     * The defaults.
-     */
-    uint32_t cNsResolution = SUPSemEventMultiGetResolution(pUVM->vm.s.pSession);
-    if (cNsResolution > 5*RT_NS_100US)
-        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = 50000;
-    else if (cNsResolution > RT_NS_100US)
-        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = cNsResolution / 4;
-    else
-        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = 2000;
-
-    /*
-     * Query overrides.
-     *
-     * I don't have time to bother with niceties such as invalid value checks
-     * here right now. sorry.
-     */
-    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pUVM->pVM), "/VMM/HaltedGlobal1");
-    if (pCfg)
-    {
-        uint32_t u32;
-        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "SpinBlockThreshold", &u32)))
-            pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = u32;
-    }
-    LogRel(("HaltedGlobal1 config: cNsSpinBlockThresholdCfg=%u\n",
-            pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * The global 1 halt method - Block in GMM (ring-0) and let it
- * try take care of the global scheduling of EMT threads.
- */
-static DECLCALLBACK(int) vmR3HaltGlobal1Halt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now)
-{
-    PUVM    pUVM  = pUVCpu->pUVM;
-    PVMCPU  pVCpu = pUVCpu->pVCpu;
-    PVM     pVM   = pUVCpu->pVM;
-    Assert(VMMGetCpu(pVM) == pVCpu);
-
-    /*
-     * Halt loop.
-     */
-    //uint64_t u64NowLog, u64Start;
-    //u64Start = u64NowLog = RTTimeNanoTS();
-    int rc = VINF_SUCCESS;
-    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
-    unsigned cLoops = 0;
-    for (;; cLoops++)
-    {
-        /*
-         * Work the timers and check if we can exit.
-         */
-        uint64_t const u64StartTimers   = RTTimeNanoTS();
-        TMR3TimerQueuesDo(pVM);
-        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
-        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-            break;
-
-        /*
-         * Estimate time left to the next event.
-         */
-        //u64NowLog = RTTimeNanoTS();
-        uint64_t u64Delta;
-        uint64_t u64GipTime = TMTimerPollGIP(pVM, pVCpu, &u64Delta);
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-            break;
-
-        /*
-         * Block if we're not spinning and the interval isn't all that small.
-         */
-        if (u64Delta >= pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg)
-        {
-            VMMR3YieldStop(pVM);
-            if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-                ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-                break;
-
-            //RTLogPrintf("loop=%-3d  u64GipTime=%'llu / %'llu   now=%'llu / %'llu\n", cLoops, u64GipTime, u64Delta, u64NowLog, u64GipTime - u64NowLog);
-            uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
-            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_HALT, u64GipTime, NULL);
-            uint64_t const u64EndSchedHalt     = RTTimeNanoTS();
-            uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
-            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
-
-            if (rc == VERR_INTERRUPTED)
-                rc = VINF_SUCCESS;
-            else if (RT_FAILURE(rc))
-            {
-                rc = vmR3FatalWaitError(pUVCpu, "VMMR0_DO_GVMM_SCHED_HALT->%Rrc\n", rc);
-                break;
-            }
-            else
-            {
-                int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
-                if (cNsOverslept > 50000)
-                    STAM_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockOverslept, cNsOverslept);
-                else if (cNsOverslept < -50000)
-                    STAM_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockInsomnia,  cNsElapsedSchedHalt);
-                else
-                    STAM_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockOnTime,    cNsElapsedSchedHalt);
-            }
-        }
-        /*
-         * When spinning call upon the GVMM and do some wakups once
-         * in a while, it's not like we're actually busy or anything.
-         */
-        else if (!(cLoops & 0x1fff))
-        {
-            uint64_t const u64StartSchedYield   = RTTimeNanoTS();
-            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_POLL, false /* don't yield */, NULL);
-            uint64_t const cNsElapsedSchedYield = RTTimeNanoTS() - u64StartSchedYield;
-            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltYield, cNsElapsedSchedYield);
-        }
-    }
-    //RTLogPrintf("*** %u loops %'llu;  lag=%RU64\n", cLoops, u64NowLog - u64Start, TMVirtualSyncGetLag(pVM));
-
-    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
-    return rc;
-}
-
-
-/**
- * The global 1 halt method - VMR3Wait() worker.
- *
- * @returns VBox status code.
- * @param   pUVCpu            Pointer to the user mode VMCPU structure.
- */
-static DECLCALLBACK(int) vmR3HaltGlobal1Wait(PUVMCPU pUVCpu)
-{
-    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
-
-    PVM    pVM   = pUVCpu->pUVM->pVM;
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    Assert(pVCpu->idCpu == pUVCpu->idCpu);
-
-    int rc = VINF_SUCCESS;
-    for (;;)
-    {
-        /*
-         * Check Relevant FFs.
-         */
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK))
-            break;
-
-        /*
-         * Wait for a while. Someone will wake us up or interrupt the call if
-         * anything needs our attention.
-         */
-        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_HALT, RTTimeNanoTS() + 1000000000 /* +1s */, NULL);
-        if (rc == VERR_INTERRUPTED)
-            rc = VINF_SUCCESS;
-        else if (RT_FAILURE(rc))
-        {
-            rc = vmR3FatalWaitError(pUVCpu, "VMMR0_DO_GVMM_SCHED_HALT->%Rrc\n", rc);
-            break;
-        }
-    }
-
-    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
-    return rc;
-}
-
-
-/**
- * The global 1 halt method - VMR3NotifyFF() worker.
- *
- * @param   pUVCpu          Pointer to the user mode VMCPU structure.
- * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
- */
-static DECLCALLBACK(void) vmR3HaltGlobal1NotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
-{
-    if (pUVCpu->vm.s.fWait)
-    {
-        int rc = SUPR3CallVMMR0Ex(pUVCpu->pVM->pVMR0, pUVCpu->idCpu, VMMR0_DO_GVMM_SCHED_WAKE_UP, 0, NULL);
-        AssertRC(rc);
-    }
-    else if (   (   (fFlags & VMNOTIFYFF_FLAGS_POKE)
-                 || !(fFlags & VMNOTIFYFF_FLAGS_DONE_REM))
-             && pUVCpu->pVCpu)
-    {
-        VMCPUSTATE enmState = VMCPU_GET_STATE(pUVCpu->pVCpu);
-        if (enmState == VMCPUSTATE_STARTED_EXEC)
-        {
-            if (fFlags & VMNOTIFYFF_FLAGS_POKE)
-            {
-                int rc = SUPR3CallVMMR0Ex(pUVCpu->pVM->pVMR0, pUVCpu->idCpu, VMMR0_DO_GVMM_SCHED_POKE, 0, NULL);
-                AssertRC(rc);
-            }
-        }
-        else if (enmState == VMCPUSTATE_STARTED_EXEC_REM)
-        {
-            if (!(fFlags & VMNOTIFYFF_FLAGS_DONE_REM))
-                REMR3NotifyFF(pUVCpu->pVM);
-        }
-    }
-}
-
-
-/**
- * Bootstrap VMR3Wait() worker.
- *
- * @returns VBox status code.
- * @param   pUVMCPU            Pointer to the user mode VMCPU structure.
- */
-static DECLCALLBACK(int) vmR3BootstrapWait(PUVMCPU pUVCpu)
-{
-    PUVM pUVM = pUVCpu->pUVM;
-
-    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
-
-    int rc = VINF_SUCCESS;
-    for (;;)
-    {
-        /*
-         * Check Relevant FFs.
-         */
-        if (pUVM->vm.s.pReqs)   /* global requests pending? */
-            break;
-        if (pUVCpu->vm.s.pReqs) /* local requests pending? */
-            break;
-
-        if (    pUVCpu->pVM
-            &&  (   VM_FF_ISPENDING(pUVCpu->pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
-                 || VMCPU_FF_ISPENDING(VMMGetCpu(pUVCpu->pVM), VMCPU_FF_EXTERNAL_SUSPENDED_MASK)
-                )
-            )
-            break;
-        if (pUVM->vm.s.fTerminateEMT)
-            break;
-
-        /*
-         * Wait for a while. Someone will wake us up or interrupt the call if
-         * anything needs our attention.
-         */
-        rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1000);
-        if (rc == VERR_TIMEOUT)
-            rc = VINF_SUCCESS;
-        else if (RT_FAILURE(rc))
-        {
-            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
-            break;
-        }
-    }
-
-    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
-    return rc;
-}
-
-
-/**
- * Bootstrap VMR3NotifyFF() worker.
- *
- * @param   pUVCpu          Pointer to the user mode VMCPU structure.
- * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
- */
-static DECLCALLBACK(void) vmR3BootstrapNotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
-{
-    if (pUVCpu->vm.s.fWait)
-    {
-        int rc = RTSemEventSignal(pUVCpu->vm.s.EventSemWait);
-        AssertRC(rc);
-    }
-    NOREF(fFlags);
-}
-
-
-/**
- * Default VMR3Wait() worker.
- *
- * @returns VBox status code.
- * @param   pUVMCPU            Pointer to the user mode VMCPU structure.
- */
-static DECLCALLBACK(int) vmR3DefaultWait(PUVMCPU pUVCpu)
-{
-    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
-
-    PVM    pVM   = pUVCpu->pVM;
-    PVMCPU pVCpu = pUVCpu->pVCpu;
-    int    rc    = VINF_SUCCESS;
-    for (;;)
-    {
-        /*
-         * Check Relevant FFs.
-         */
-        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
-            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK))
-            break;
-
-        /*
-         * Wait for a while. Someone will wake us up or interrupt the call if
-         * anything needs our attention.
-         */
-        rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1000);
-        if (rc == VERR_TIMEOUT)
-            rc = VINF_SUCCESS;
-        else if (RT_FAILURE(rc))
-        {
-            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc", rc);
-            break;
-        }
-    }
-
-    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
-    return rc;
-}
-
-
-/**
- * Default VMR3NotifyFF() worker.
- *
- * @param   pUVCpu          Pointer to the user mode VMCPU structure.
- * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
- */
-static DECLCALLBACK(void) vmR3DefaultNotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
-{
-    if (pUVCpu->vm.s.fWait)
-    {
-        int rc = RTSemEventSignal(pUVCpu->vm.s.EventSemWait);
-        AssertRC(rc);
-    }
-    else if (   !(fFlags & VMNOTIFYFF_FLAGS_DONE_REM)
-             && pUVCpu->pVCpu
-             && pUVCpu->pVCpu->enmState == VMCPUSTATE_STARTED_EXEC_REM)
-        REMR3NotifyFF(pUVCpu->pVM);
-}
-
-
-/**
- * Array with halt method descriptors.
- * VMINT::iHaltMethod contains an index into this array.
- */
-static const struct VMHALTMETHODDESC
-{
-    /** The halt method id. */
-    VMHALTMETHOD enmHaltMethod;
-    /** The init function for loading config and initialize variables. */
-    DECLR3CALLBACKMEMBER(int,  pfnInit,(PUVM pUVM));
-    /** The term function. */
-    DECLR3CALLBACKMEMBER(void, pfnTerm,(PUVM pUVM));
-    /** The VMR3WaitHaltedU function. */
-    DECLR3CALLBACKMEMBER(int,  pfnHalt,(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now));
-    /** The VMR3WaitU function. */
-    DECLR3CALLBACKMEMBER(int,  pfnWait,(PUVMCPU pUVCpu));
-    /** The VMR3NotifyCpuFFU function. */
-    DECLR3CALLBACKMEMBER(void, pfnNotifyCpuFF,(PUVMCPU pUVCpu, uint32_t fFlags));
-    /** The VMR3NotifyGlobalFFU function. */
-    DECLR3CALLBACKMEMBER(void, pfnNotifyGlobalFF,(PUVM pUVM, uint32_t fFlags));
-} g_aHaltMethods[] =
-{
-    { VMHALTMETHOD_BOOTSTRAP, NULL,                NULL,   NULL,                vmR3BootstrapWait,   vmR3BootstrapNotifyCpuFF,   NULL },
-    { VMHALTMETHOD_OLD,       NULL,                NULL,   vmR3HaltOldDoHalt,   vmR3DefaultWait,     vmR3DefaultNotifyCpuFF,     NULL },
-    { VMHALTMETHOD_1,         vmR3HaltMethod1Init, NULL,   vmR3HaltMethod1Halt, vmR3DefaultWait,     vmR3DefaultNotifyCpuFF,     NULL },
-    { VMHALTMETHOD_GLOBAL_1,  vmR3HaltGlobal1Init, NULL,   vmR3HaltGlobal1Halt, vmR3HaltGlobal1Wait, vmR3HaltGlobal1NotifyCpuFF, NULL },
-};
-
-
-/**
- * Notify the emulation thread (EMT) about pending Forced Action (FF).
- *
- * This function is called by thread other than EMT to make
- * sure EMT wakes up and promptly service an FF request.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
- */
-VMMR3DECL(void) VMR3NotifyGlobalFFU(PUVM pUVM, uint32_t fFlags)
-{
-    LogFlow(("VMR3NotifyGlobalFFU:\n"));
-    uint32_t iHaldMethod = pUVM->vm.s.iHaltMethod;
-
-    if (g_aHaltMethods[iHaldMethod].pfnNotifyGlobalFF) /** @todo make mandatory. */
-        g_aHaltMethods[iHaldMethod].pfnNotifyGlobalFF(pUVM, fFlags);
-    else
-        for (VMCPUID iCpu = 0; iCpu < pUVM->cCpus; iCpu++)
-            g_aHaltMethods[iHaldMethod].pfnNotifyCpuFF(&pUVM->aCpus[iCpu], fFlags);
-}
-
-
-/**
- * Notify the emulation thread (EMT) about pending Forced Action (FF).
- *
- * This function is called by thread other than EMT to make
- * sure EMT wakes up and promptly service an FF request.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
- */
-VMMR3DECL(void) VMR3NotifyCpuFFU(PUVMCPU pUVCpu, uint32_t fFlags)
-{
-    PUVM pUVM = pUVCpu->pUVM;
-
-    LogFlow(("VMR3NotifyCpuFFU:\n"));
-    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyCpuFF(pUVCpu, fFlags);
-}
-
-
-/**
- * Halted VM Wait.
- * Any external event will unblock the thread.
- *
- * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
- *          case an appropriate status code is returned.
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- * @param   fIgnoreInterrupts   If set the VM_FF_INTERRUPT flags is ignored.
- * @thread  The emulation thread.
- */
-VMMR3DECL(int) VMR3WaitHalted(PVM pVM, PVMCPU pVCpu, bool fIgnoreInterrupts)
-{
-    LogFlow(("VMR3WaitHalted: fIgnoreInterrupts=%d\n", fIgnoreInterrupts));
-
-    /*
-     * Check Relevant FFs.
-     */
-    const uint32_t fMask = !fIgnoreInterrupts
-        ? VMCPU_FF_EXTERNAL_HALTED_MASK
-        : VMCPU_FF_EXTERNAL_HALTED_MASK & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC);
-    if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
-        ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
-    {
-        LogFlow(("VMR3WaitHalted: returns VINF_SUCCESS (FF %#x FFCPU %#x)\n", pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * The yielder is suspended while we're halting, while TM might have clock(s) running
-     * only at certain times and need to be notified..
-     */
-    if (pVCpu->idCpu == 0)
-        VMMR3YieldSuspend(pVM);
-    TMNotifyStartOfHalt(pVCpu);
-
-    /*
-     * Record halt averages for the last second.
-     */
-    PUVMCPU pUVCpu = pVCpu->pUVCpu;
-    uint64_t u64Now = RTTimeNanoTS();
-    int64_t off = u64Now - pUVCpu->vm.s.u64HaltsStartTS;
-    if (off > 1000000000)
-    {
-        if (off > _4G || !pUVCpu->vm.s.cHalts)
-        {
-            pUVCpu->vm.s.HaltInterval = 1000000000 /* 1 sec */;
-            pUVCpu->vm.s.HaltFrequency = 1;
-        }
-        else
-        {
-            pUVCpu->vm.s.HaltInterval = (uint32_t)off / pUVCpu->vm.s.cHalts;
-            pUVCpu->vm.s.HaltFrequency = ASMMultU64ByU32DivByU32(pUVCpu->vm.s.cHalts, 1000000000, (uint32_t)off);
-        }
-        pUVCpu->vm.s.u64HaltsStartTS = u64Now;
-        pUVCpu->vm.s.cHalts = 0;
-    }
-    pUVCpu->vm.s.cHalts++;
-
-    /*
-     * Do the halt.
-     */
-    Assert(VMCPU_GET_STATE(pVCpu) == VMCPUSTATE_STARTED);
-    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HALTED);
-    PUVM pUVM = pUVCpu->pUVM;
-    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnHalt(pUVCpu, fMask, u64Now);
-    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
-
-    /*
-     * Notify TM and resume the yielder
-     */
-    TMNotifyEndOfHalt(pVCpu);
-    if (pVCpu->idCpu == 0)
-        VMMR3YieldResume(pVM);
-
-    LogFlow(("VMR3WaitHalted: returns %Rrc (FF %#x)\n", rc, pVM->fGlobalForcedActions));
-    return rc;
-}
-
-
-/**
- * Suspended VM Wait.
- * Only a handful of forced actions will cause the function to
- * return to the caller.
- *
- * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
- *          case an appropriate status code is returned.
- * @param   pUVCpu          Pointer to the user mode VMCPU structure.
- * @thread  The emulation thread.
- */
-VMMR3DECL(int) VMR3WaitU(PUVMCPU pUVCpu)
-{
-    LogFlow(("VMR3WaitU:\n"));
-
-    /*
-     * Check Relevant FFs.
-     */
-    PVM    pVM   = pUVCpu->pVM;
-    PVMCPU pVCpu = pUVCpu->pVCpu;
-
-    if (    pVM
-        &&  (   VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
-             || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK)
-            )
-        )
-    {
-        LogFlow(("VMR3Wait: returns VINF_SUCCESS (FF %#x)\n", pVM->fGlobalForcedActions));
-        return VINF_SUCCESS;
-    }
-
-    /*
-     * Do waiting according to the halt method (so VMR3NotifyFF
-     * doesn't have to special case anything).
-     */
-    PUVM pUVM = pUVCpu->pUVM;
-    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnWait(pUVCpu);
-    LogFlow(("VMR3WaitU: returns %Rrc (FF %#x)\n", rc, pVM ? pVM->fGlobalForcedActions : 0));
-    return rc;
-}
-
-
-/**
- * Interface that PDMR3Suspend, PDMR3PowerOff and PDMR3Reset uses when they wait
- * for the handling of asynchronous notifications to complete.
- *
- * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
- *          case an appropriate status code is returned.
- * @param   pUVCpu              Pointer to the user mode VMCPU structure.
- * @thread  The emulation thread.
- */
-VMMR3_INT_DECL(int) VMR3AsyncPdmNotificationWaitU(PUVMCPU pUVCpu)
-{
-    LogFlow(("VMR3AsyncPdmNotificationWaitU:\n"));
-    return VMR3WaitU(pUVCpu);
-}
-
-
-/**
- * Interface that PDM the helper asynchronous notification completed methods
- * uses for EMT0 when it is waiting inside VMR3AsyncPdmNotificationWaitU().
- *
- * @param   pUVM                Pointer to the user mode VM structure.
- */
-VMMR3_INT_DECL(void) VMR3AsyncPdmNotificationWakeupU(PUVM pUVM)
-{
-    LogFlow(("VMR3AsyncPdmNotificationWakeupU:\n"));
-    VM_FF_SET(pUVM->pVM, VM_FF_REQUEST); /* this will have to do for now. */
-    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyCpuFF(&pUVM->aCpus[0], 0 /*fFlags*/);
-}
-
-
-/**
- * Rendezvous callback that will be called once.
- *
- * @returns VBox strict status code.
- * @param   pVM                 VM handle.
- * @param   pVCpu               The VMCPU handle for the calling EMT.
- * @param   pvUser              The new g_aHaltMethods index.
- */
-static DECLCALLBACK(VBOXSTRICTRC) vmR3SetHaltMethodCallback(PVM pVM, PVMCPU pVCpu, void *pvUser)
-{
-    PUVM        pUVM = pVM->pUVM;
-    uintptr_t   i    = (uintptr_t)pvUser;
-    Assert(i < RT_ELEMENTS(g_aHaltMethods));
-    NOREF(pVCpu);
-
-    /*
-     * Terminate the old one.
-     */
-    if (    pUVM->vm.s.enmHaltMethod != VMHALTMETHOD_INVALID
-        &&  g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm)
-    {
-        g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm(pUVM);
-        pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_INVALID;
-    }
-
-    /* Assert that the failure fallback is where we expect. */
-    Assert(g_aHaltMethods[0].enmHaltMethod == VMHALTMETHOD_BOOTSTRAP);
-    Assert(!g_aHaltMethods[0].pfnTerm && !g_aHaltMethods[0].pfnInit);
-
-    /*
-     * Init the new one.
-     */
-    int rc = VINF_SUCCESS;
-    memset(&pUVM->vm.s.Halt, 0, sizeof(pUVM->vm.s.Halt));
-    if (g_aHaltMethods[i].pfnInit)
-    {
-        rc = g_aHaltMethods[i].pfnInit(pUVM);
-        if (RT_FAILURE(rc))
-        {
-            /* Fall back on the bootstrap method. This requires no
-               init/term (see assertion above), and will always work. */
-            AssertLogRelRC(rc);
-            i = 0;
-        }
-    }
-
-    /*
-     * Commit it.
-     */
-    pUVM->vm.s.enmHaltMethod = g_aHaltMethods[i].enmHaltMethod;
-    ASMAtomicWriteU32(&pUVM->vm.s.iHaltMethod, i);
-
-    return rc;
-}
-
-
-/**
- * Changes the halt method.
- *
- * @returns VBox status code.
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   enmHaltMethod   The new halt method.
- * @thread  EMT.
- */
-int vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod)
-{
-    PVM pVM = pUVM->pVM; Assert(pVM);
-    VM_ASSERT_EMT(pVM);
-    AssertReturn(enmHaltMethod > VMHALTMETHOD_INVALID && enmHaltMethod < VMHALTMETHOD_END, VERR_INVALID_PARAMETER);
-
-    /*
-     * Resolve default (can be overridden in the configuration).
-     */
-    if (enmHaltMethod == VMHALTMETHOD_DEFAULT)
-    {
-        uint32_t u32;
-        int rc = CFGMR3QueryU32(CFGMR3GetChild(CFGMR3GetRoot(pVM), "VM"), "HaltMethod", &u32);
-        if (RT_SUCCESS(rc))
-        {
-            enmHaltMethod = (VMHALTMETHOD)u32;
-            if (enmHaltMethod <= VMHALTMETHOD_INVALID || enmHaltMethod >= VMHALTMETHOD_END)
-                return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, N_("Invalid VM/HaltMethod value %d"), enmHaltMethod);
-        }
-        else if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_CHILD_NOT_FOUND)
-            return VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to Query VM/HaltMethod as uint32_t"));
-        else
-            enmHaltMethod = VMHALTMETHOD_GLOBAL_1;
-            //enmHaltMethod = VMHALTMETHOD_1;
-            //enmHaltMethod = VMHALTMETHOD_OLD;
-    }
-    LogRel(("VM: Halt method %s (%d)\n", vmR3GetHaltMethodName(enmHaltMethod), enmHaltMethod));
-
-    /*
-     * Find the descriptor.
-     */
-    unsigned i = 0;
-    while (     i < RT_ELEMENTS(g_aHaltMethods)
-           &&   g_aHaltMethods[i].enmHaltMethod != enmHaltMethod)
-        i++;
-    AssertReturn(i < RT_ELEMENTS(g_aHaltMethods), VERR_INVALID_PARAMETER);
-
-    /*
-     * This needs to be done while the other EMTs are not sleeping or otherwise messing around.
-     */
-    return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, vmR3SetHaltMethodCallback, (void *)(uintptr_t)i);
-}
-
Index: unk/src/VBox/VMM/VMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/VMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,444 +1,0 @@
-/* $Id$ */
-/** @file
- * VM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___VMInternal_h
-#define ___VMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/vmapi.h>
-#include <iprt/assert.h>
-#include <iprt/critsect.h>
-#include <setjmp.h>
-
-
-
-/** @defgroup grp_vm_int   Internals
- * @ingroup grp_vm
- * @internal
- * @{
- */
-
-
-/**
- * VM state change callback.
- */
-typedef struct VMATSTATE
-{
-    /** Pointer to the next one. */
-    struct VMATSTATE               *pNext;
-    /** Pointer to the callback. */
-    PFNVMATSTATE                    pfnAtState;
-    /** The user argument. */
-    void                           *pvUser;
-} VMATSTATE;
-/** Pointer to a VM state change callback. */
-typedef VMATSTATE *PVMATSTATE;
-
-
-/**
- * VM error callback.
- */
-typedef struct VMATERROR
-{
-    /** Pointer to the next one. */
-    struct VMATERROR               *pNext;
-    /** Pointer to the callback. */
-    PFNVMATERROR                    pfnAtError;
-    /** The user argument. */
-    void                           *pvUser;
-} VMATERROR;
-/** Pointer to a VM error callback. */
-typedef VMATERROR *PVMATERROR;
-
-
-/**
- * Chunk of memory allocated off the hypervisor heap in which
- * we copy the error details.
- */
-typedef struct VMERROR
-{
-    /** The size of the chunk. */
-    uint32_t                        cbAllocated;
-    /** The current offset into the chunk.
-     * We start by putting the filename and function immediately
-     * after the end of the buffer. */
-    uint32_t                        off;
-    /** Offset from the start of this structure to the file name. */
-    uint32_t                        offFile;
-    /** The line number. */
-    uint32_t                        iLine;
-    /** Offset from the start of this structure to the function name. */
-    uint32_t                        offFunction;
-    /** Offset from the start of this structure to the formatted message text. */
-    uint32_t                        offMessage;
-    /** The VBox status code. */
-    int32_t                         rc;
-} VMERROR, *PVMERROR;
-
-
-/**
- * VM runtime error callback.
- */
-typedef struct VMATRUNTIMEERROR
-{
-    /** Pointer to the next one. */
-    struct VMATRUNTIMEERROR         *pNext;
-    /** Pointer to the callback. */
-    PFNVMATRUNTIMEERROR              pfnAtRuntimeError;
-    /** The user argument. */
-    void                            *pvUser;
-} VMATRUNTIMEERROR;
-/** Pointer to a VM error callback. */
-typedef VMATRUNTIMEERROR *PVMATRUNTIMEERROR;
-
-
-/**
- * Chunk of memory allocated off the hypervisor heap in which
- * we copy the runtime error details.
- */
-typedef struct VMRUNTIMEERROR
-{
-    /** The size of the chunk. */
-    uint32_t                        cbAllocated;
-    /** The current offset into the chunk.
-     * We start by putting the error ID immediately
-     * after the end of the buffer. */
-    uint32_t                        off;
-    /** Offset from the start of this structure to the error ID. */
-    uint32_t                        offErrorId;
-    /** Offset from the start of this structure to the formatted message text. */
-    uint32_t                        offMessage;
-    /** Error flags. */
-    uint32_t                        fFlags;
-} VMRUNTIMEERROR, *PVMRUNTIMEERROR;
-
-/** The halt method. */
-typedef enum
-{
-    /** The usual invalid value. */
-    VMHALTMETHOD_INVALID = 0,
-    /** Use the method used during bootstrapping. */
-    VMHALTMETHOD_BOOTSTRAP,
-    /** Use the default method. */
-    VMHALTMETHOD_DEFAULT,
-    /** The old spin/yield/block method. */
-    VMHALTMETHOD_OLD,
-    /** The first go at a block/spin method. */
-    VMHALTMETHOD_1,
-    /** The first go at a more global approach. */
-    VMHALTMETHOD_GLOBAL_1,
-    /** The end of valid methods. (not inclusive of course) */
-    VMHALTMETHOD_END,
-    /** The usual 32-bit max value. */
-    VMHALTMETHOD_32BIT_HACK = 0x7fffffff
-} VMHALTMETHOD;
-
-
-/**
- * VM Internal Data (part of the VM structure).
- *
- * @todo Move this and all related things to VMM. The VM component was, to some
- *       extent at least, a bad ad hoc design which should all have been put in
- *       VMM. @see pg_vm.
- */
-typedef struct VMINT
-{
-    /** VM Error Message. */
-    R3PTRTYPE(PVMERROR)             pErrorR3;
-    /** VM Runtime Error Message. */
-    R3PTRTYPE(PVMRUNTIMEERROR)      pRuntimeErrorR3;
-    /** The VM was/is-being teleported and has not yet been fully resumed. */
-    bool                            fTeleportedAndNotFullyResumedYet;
-} VMINT;
-/** Pointer to the VM Internal Data (part of the VM structure). */
-typedef VMINT *PVMINT;
-
-
-/**
- * VM internal data kept in the UVM.
- */
-typedef struct VMINTUSERPERVM
-{
-    /** Head of the request queue. Atomic. */
-    volatile PVMREQ                 pReqs;
-    /** The last index used during alloc/free. */
-    volatile uint32_t               iReqFree;
-    /** Number of free request packets. */
-    volatile uint32_t               cReqFree;
-    /** Array of pointers to lists of free request packets. Atomic. */
-    volatile PVMREQ                 apReqFree[9];
-
-#ifdef VBOX_WITH_STATISTICS
-    /** Number of VMR3ReqAlloc returning a new packet. */
-    STAMCOUNTER                     StatReqAllocNew;
-    /** Number of VMR3ReqAlloc causing races. */
-    STAMCOUNTER                     StatReqAllocRaces;
-    /** Number of VMR3ReqAlloc returning a recycled packet. */
-    STAMCOUNTER                     StatReqAllocRecycled;
-    /** Number of VMR3ReqFree calls. */
-    STAMCOUNTER                     StatReqFree;
-    /** Number of times the request was actually freed. */
-    STAMCOUNTER                     StatReqFreeOverflow;
-    /** Number of requests served. */
-    STAMCOUNTER                     StatReqProcessed;
-    /** Number of times there are more than one request and the others needed to be
-     * pushed back onto the list. */
-    STAMCOUNTER                     StatReqMoreThan1;
-    /** Number of times we've raced someone when pushing the other requests back
-     * onto the list. */
-    STAMCOUNTER                     StatReqPushBackRaces;
-#endif
-
-    /** Pointer to the support library session.
-     * Mainly for creation and destruction. */
-    PSUPDRVSESSION                  pSession;
-
-    /** Force EMT to terminate. */
-    bool volatile                   fTerminateEMT;
-
-    /** Critical section for pAtState and enmPrevVMState. */
-    RTCRITSECT                      AtStateCritSect;
-    /** List of registered state change callbacks. */
-    PVMATSTATE                      pAtState;
-    /** List of registered state change callbacks. */
-    PVMATSTATE                     *ppAtStateNext;
-    /** The previous VM state.
-     * This is mainly used for the 'Resetting' state, but may come in handy later
-     * and when debugging. */
-    VMSTATE                         enmPrevVMState;
-
-    /** Critical section for pAtError and pAtRuntimeError. */
-    RTCRITSECT                      AtErrorCritSect;
-
-    /** List of registered error callbacks. */
-    PVMATERROR                      pAtError;
-    /** List of registered error callbacks. */
-    PVMATERROR                     *ppAtErrorNext;
-    /** The error message count.
-     * This is incremented every time an error is raised.  */
-    uint32_t volatile               cErrors;
-
-    /** The runtime error message count.
-     * This is incremented every time a runtime error is raised.  */
-    uint32_t volatile               cRuntimeErrors;
-    /** List of registered error callbacks. */
-    PVMATRUNTIMEERROR               pAtRuntimeError;
-    /** List of registered error callbacks. */
-    PVMATRUNTIMEERROR              *ppAtRuntimeErrorNext;
-
-    /** @name Generic Halt data
-     * @{
-     */
-    /** The current halt method.
-     * Can be selected by CFGM option 'VM/HaltMethod'. */
-    VMHALTMETHOD                    enmHaltMethod;
-    /** The index into g_aHaltMethods of the current halt method. */
-    uint32_t volatile               iHaltMethod;
-    /** @} */
-
-    /** @todo Do NOT add new members here or reuse the current, we need to store the config for
-     *  each halt method separately because we're racing on SMP guest rigs. */
-    union
-    {
-       /**
-        * Method 1 & 2 - Block whenever possible, and when lagging behind
-        * switch to spinning with regular blocking every 5-200ms (defaults)
-        * depending on the accumulated lag. The blocking interval is adjusted
-        * with the average oversleeping of the last 64 times.
-        *
-        * The difference between 1 and 2 is that we use native absolute
-        * time APIs for the blocking instead of the millisecond based IPRT
-        * interface.
-        */
-        struct
-        {
-            /** The max interval without blocking (when spinning). */
-            uint32_t                u32MinBlockIntervalCfg;
-            /** The minimum interval between blocking (when spinning). */
-            uint32_t                u32MaxBlockIntervalCfg;
-            /** The value to divide the current lag by to get the raw blocking interval (when spinning). */
-            uint32_t                u32LagBlockIntervalDivisorCfg;
-            /** When to start spinning (lag / nano secs). */
-            uint32_t                u32StartSpinningCfg;
-            /** When to stop spinning (lag / nano secs). */
-            uint32_t                u32StopSpinningCfg;
-        }                           Method12;
-
-       /**
-        * The GVMM manages halted and waiting EMTs.
-        */
-        struct
-        {
-            /** The threshold between spinning and blocking. */
-            uint32_t                cNsSpinBlockThresholdCfg;
-        }                           Global1;
-    }                               Halt;
-
-    /** Pointer to the DBGC instance data. */
-    void                           *pvDBGC;
-
-    /** TLS index for the VMINTUSERPERVMCPU pointer. */
-    RTTLS                           idxTLS;
-} VMINTUSERPERVM;
-
-/** Pointer to the VM internal data kept in the UVM. */
-typedef VMINTUSERPERVM *PVMINTUSERPERVM;
-
-
-/**
- * VMCPU internal data kept in the UVM.
- *
- * Almost a copy of VMINTUSERPERVM. Separate data properly later on.
- */
-typedef struct VMINTUSERPERVMCPU
-{
-    /** Head of the request queue. Atomic. */
-    volatile PVMREQ                 pReqs;
-
-    /** The handle to the EMT thread. */
-    RTTHREAD                        ThreadEMT;
-    /** The native of the EMT thread. */
-    RTNATIVETHREAD                  NativeThreadEMT;
-    /** Wait event semaphore. */
-    RTSEMEVENT                      EventSemWait;
-    /** Wait/Idle indicator. */
-    bool volatile                   fWait;
-    /** Align the next bit. */
-    bool                            afAlignment[7];
-
-    /** @name Generic Halt data
-     * @{
-     */
-    /** The average time (ns) between two halts in the last second. (updated once per second) */
-    uint32_t                        HaltInterval;
-    /** The average halt frequency for the last second. (updated once per second) */
-    uint32_t                        HaltFrequency;
-    /** The number of halts in the current period. */
-    uint32_t                        cHalts;
-    uint32_t                        padding; /**< alignment padding. */
-    /** When we started counting halts in cHalts (RTTimeNanoTS). */
-    uint64_t                        u64HaltsStartTS;
-    /** @} */
-
-    /** Union containing data and config for the different halt algorithms. */
-    union
-    {
-       /**
-        * Method 1 & 2 - Block whenever possible, and when lagging behind
-        * switch to spinning with regular blocking every 5-200ms (defaults)
-        * depending on the accumulated lag. The blocking interval is adjusted
-        * with the average oversleeping of the last 64 times.
-        *
-        * The difference between 1 and 2 is that we use native absolute
-        * time APIs for the blocking instead of the millisecond based IPRT
-        * interface.
-        */
-        struct
-        {
-            /** How many times we've blocked while cBlockedNS and cBlockedTooLongNS has been accumulating. */
-            uint32_t                cBlocks;
-            /** Align the next member. */
-            uint32_t                u32Alignment;
-            /** Avg. time spend oversleeping when blocking. (Re-calculated every so often.) */
-            uint64_t                cNSBlockedTooLongAvg;
-            /** Total time spend oversleeping when blocking. */
-            uint64_t                cNSBlockedTooLong;
-            /** Total time spent blocking. */
-            uint64_t                cNSBlocked;
-            /** The timestamp (RTTimeNanoTS) of the last block. */
-            uint64_t                u64LastBlockTS;
-
-            /** When we started spinning relentlessly in order to catch up some of the oversleeping.
-             * This is 0 when we're not spinning. */
-            uint64_t                u64StartSpinTS;
-        }                           Method12;
-
-#if 0
-       /**
-        * Method 3 & 4 - Same as method 1 & 2 respectivly, except that we
-        * sprinkle it with yields.
-        */
-       struct
-       {
-           /** How many times we've blocked while cBlockedNS and cBlockedTooLongNS has been accumulating. */
-           uint32_t                 cBlocks;
-           /** Avg. time spend oversleeping when blocking. (Re-calculated every so often.) */
-           uint64_t                 cBlockedTooLongNSAvg;
-           /** Total time spend oversleeping when blocking. */
-           uint64_t                 cBlockedTooLongNS;
-           /** Total time spent blocking. */
-           uint64_t                 cBlockedNS;
-           /** The timestamp (RTTimeNanoTS) of the last block. */
-           uint64_t                 u64LastBlockTS;
-
-           /** How many times we've yielded while cBlockedNS and cBlockedTooLongNS has been accumulating. */
-           uint32_t                 cYields;
-           /** Avg. time spend oversleeping when yielding. */
-           uint32_t                 cYieldTooLongNSAvg;
-           /** Total time spend oversleeping when yielding. */
-           uint64_t                 cYieldTooLongNS;
-           /** Total time spent yielding. */
-           uint64_t                 cYieldedNS;
-           /** The timestamp (RTTimeNanoTS) of the last block. */
-           uint64_t                 u64LastYieldTS;
-
-           /** When we started spinning relentlessly in order to catch up some of the oversleeping. */
-           uint64_t                 u64StartSpinTS;
-       }                            Method34;
-#endif
-    }                               Halt;
-
-    /** Profiling the halted state; yielding vs blocking.
-     * @{ */
-    STAMPROFILE                     StatHaltYield;
-    STAMPROFILE                     StatHaltBlock;
-    STAMPROFILE                     StatHaltBlockOverslept;
-    STAMPROFILE                     StatHaltBlockInsomnia;
-    STAMPROFILE                     StatHaltBlockOnTime;
-    STAMPROFILE                     StatHaltTimers;
-    STAMPROFILE                     StatHaltPoll;
-    /** @} */
-} VMINTUSERPERVMCPU;
-#ifdef IN_RING3
-AssertCompileMemberAlignment(VMINTUSERPERVMCPU, u64HaltsStartTS, 8);
-AssertCompileMemberAlignment(VMINTUSERPERVMCPU, Halt.Method12.cNSBlockedTooLongAvg, 8);
-AssertCompileMemberAlignment(VMINTUSERPERVMCPU, StatHaltYield, 8);
-#endif
-
-/** Pointer to the VM internal data kept in the UVM. */
-typedef VMINTUSERPERVMCPU *PVMINTUSERPERVMCPU;
-
-RT_C_DECLS_BEGIN
-
-DECLCALLBACK(int)   vmR3EmulationThread(RTTHREAD ThreadSelf, void *pvArg);
-int                 vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod);
-DECLCALLBACK(int)   vmR3Destroy(PVM pVM);
-DECLCALLBACK(void)  vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *args);
-void                vmSetErrorCopy(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args);
-DECLCALLBACK(int)   vmR3SetRuntimeError(PVM pVM, uint32_t fFlags, const char *pszErrorId, char *pszMessage);
-DECLCALLBACK(int)   vmR3SetRuntimeErrorV(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa);
-void                vmSetRuntimeErrorCopy(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va);
-void                vmR3SetGuruMeditation(PVM pVM);
-void                vmR3SetTerminated(PVM pVM);
-
-RT_C_DECLS_END
-
-
-/** @} */
-
-#endif
-
Index: unk/src/VBox/VMM/VMM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMM.cpp	(revision 35332)
+++ 	(revision )
@@ -1,2315 +1,0 @@
-/* $Id$ */
-/** @file
- * VMM - The Virtual Machine Monitor Core.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-//#define NO_SUPCALLR0VMM
-
-/** @page pg_vmm        VMM - The Virtual Machine Monitor
- *
- * The VMM component is two things at the moment, it's a component doing a few
- * management and routing tasks, and it's the whole virtual machine monitor
- * thing.  For hysterical reasons, it is not doing all the management that one
- * would expect, this is instead done by @ref pg_vm.  We'll address this
- * misdesign eventually.
- *
- * @see grp_vmm, grp_vm
- *
- *
- * @section sec_vmmstate        VMM State
- *
- * @image html VM_Statechart_Diagram.gif
- *
- * To be written.
- *
- *
- * @subsection  subsec_vmm_init     VMM Initialization
- *
- * To be written.
- *
- *
- * @subsection  subsec_vmm_term     VMM Termination
- *
- * To be written.
- *
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_VMM
-#include <VBox/vmm.h>
-#include <VBox/vmapi.h>
-#include <VBox/pgm.h>
-#include <VBox/cfgm.h>
-#include <VBox/pdmqueue.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/pdmapi.h>
-#include <VBox/cpum.h>
-#include <VBox/mm.h>
-#include <VBox/iom.h>
-#include <VBox/trpm.h>
-#include <VBox/selm.h>
-#include <VBox/em.h>
-#include <VBox/sup.h>
-#include <VBox/dbgf.h>
-#include <VBox/csam.h>
-#include <VBox/patm.h>
-#include <VBox/rem.h>
-#include <VBox/ssm.h>
-#include <VBox/tm.h>
-#include "VMMInternal.h"
-#include "VMMSwitcher/VMMSwitcher.h"
-#include <VBox/vm.h>
-#include <VBox/ftm.h>
-
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/version.h>
-#include <VBox/x86.h>
-#include <VBox/hwaccm.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-#include <iprt/asm.h>
-#include <iprt/time.h>
-#include <iprt/semaphore.h>
-#include <iprt/stream.h>
-#include <iprt/string.h>
-#include <iprt/stdarg.h>
-#include <iprt/ctype.h>
-
-
-
-/*******************************************************************************
-*   Defined Constants And Macros                                               *
-*******************************************************************************/
-/** The saved state version. */
-#define VMM_SAVED_STATE_VERSION     4
-/** The saved state version used by v3.0 and earlier. (Teleportation) */
-#define VMM_SAVED_STATE_VERSION_3_0 3
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int                  vmmR3InitStacks(PVM pVM);
-static int                  vmmR3InitLoggers(PVM pVM);
-static void                 vmmR3InitRegisterStats(PVM pVM);
-static DECLCALLBACK(int)    vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
-static DECLCALLBACK(int)    vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
-static DECLCALLBACK(void)   vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
-static int                  vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
-static DECLCALLBACK(void)   vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
-
-
-/**
- * Initializes the VMM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3_INT_DECL(int) VMMR3Init(PVM pVM)
-{
-    LogFlow(("VMMR3Init\n"));
-
-    /*
-     * Assert alignment, sizes and order.
-     */
-    AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
-    AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
-    AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
-
-    /*
-     * Init basic VM VMM members.
-     */
-    pVM->vmm.s.offVM = RT_OFFSETOF(VM, vmm);
-    pVM->vmm.s.pahEvtRendezvousEnterOrdered     = NULL;
-    pVM->vmm.s.hEvtRendezvousEnterOneByOne      = NIL_RTSEMEVENT;
-    pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce  = NIL_RTSEMEVENTMULTI;
-    pVM->vmm.s.hEvtMulRendezvousDone            = NIL_RTSEMEVENTMULTI;
-    pVM->vmm.s.hEvtRendezvousDoneCaller         = NIL_RTSEMEVENT;
-
-    /** @cfgm{YieldEMTInterval, uint32_t, 1, UINT32_MAX, 23, ms}
-     * The EMT yield interval.  The EMT yielding is a hack we employ to play a
-     * bit nicer with the rest of the system (like for instance the GUI).
-     */
-    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies,
-                               23 /* Value arrived at after experimenting with the grub boot prompt. */);
-    AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
-
-
-    /** @cfgm{VMM/UsePeriodicPreemptionTimers, boolean, true}
-     * Controls whether we employ per-cpu preemption timers to limit the time
-     * spent executing guest code.  This option is not available on all
-     * platforms and we will silently ignore this setting then.  If we are
-     * running in VT-x mode, we will use the VMX-preemption timer instead of
-     * this one when possible.
-     */
-    PCFGMNODE pCfgVMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "VMM");
-    rc = CFGMR3QueryBoolDef(pCfgVMM, "UsePeriodicPreemptionTimers", &pVM->vmm.s.fUsePeriodicPreemptionTimers, true);
-    AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"VMM/UsePeriodicPreemptionTimers\", rc=%Rrc\n", rc), rc);
-
-    /*
-     * Initialize the VMM sync critical section and semaphores.
-     */
-    rc = RTCritSectInit(&pVM->vmm.s.CritSectSync);
-    AssertRCReturn(rc, rc);
-    pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
-    if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
-        return VERR_NO_MEMORY;
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
-        AssertRCReturn(rc, rc);
-    }
-    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
-    AssertRCReturn(rc, rc);
-    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
-    AssertRCReturn(rc, rc);
-    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
-    AssertRCReturn(rc, rc);
-    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
-    AssertRCReturn(rc, rc);
-
-    /* GC switchers are enabled by default. Turned off by HWACCM. */
-    pVM->vmm.s.fSwitcherDisabled = false;
-
-    /*
-     * Register the saved state data unit.
-     */
-    rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
-                               NULL, NULL, NULL,
-                               NULL, vmmR3Save, NULL,
-                               NULL, vmmR3Load, NULL);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Register the Ring-0 VM handle with the session for fast ioctl calls.
-     */
-    rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Init various sub-components.
-     */
-    rc = vmmR3SwitcherInit(pVM);
-    if (RT_SUCCESS(rc))
-    {
-        rc = vmmR3InitStacks(pVM);
-        if (RT_SUCCESS(rc))
-        {
-            rc = vmmR3InitLoggers(pVM);
-
-#ifdef VBOX_WITH_NMI
-            /*
-             * Allocate mapping for the host APIC.
-             */
-            if (RT_SUCCESS(rc))
-            {
-                rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
-                AssertRC(rc);
-            }
-#endif
-            if (RT_SUCCESS(rc))
-            {
-                /*
-                 * Debug info and statistics.
-                 */
-                DBGFR3InfoRegisterInternal(pVM, "ff", "Displays the current Forced actions Flags.", vmmR3InfoFF);
-                vmmR3InitRegisterStats(pVM);
-
-                return VINF_SUCCESS;
-            }
-        }
-        /** @todo: Need failure cleanup. */
-
-        //more todo in here?
-        //if (RT_SUCCESS(rc))
-        //{
-        //}
-        //int rc2 = vmmR3TermCoreCode(pVM);
-        //AssertRC(rc2));
-    }
-
-    return rc;
-}
-
-
-/**
- * Allocate & setup the VMM RC stack(s) (for EMTs).
- *
- * The stacks are also used for long jumps in Ring-0.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- *
- * @remarks The optional guard page gets it protection setup up during R3 init
- *          completion because of init order issues.
- */
-static int vmmR3InitStacks(PVM pVM)
-{
-    int rc = VINF_SUCCESS;
-#ifdef VMM_R0_SWITCH_STACK
-    uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
-#else
-    uint32_t fFlags = 0;
-#endif
-
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[idCpu];
-
-#ifdef VBOX_STRICT_VMM_STACK
-        rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
-#else
-        rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
-#endif
-                                       PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
-        if (RT_SUCCESS(rc))
-        {
-#ifdef VBOX_STRICT_VMM_STACK
-            pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
-#endif
-#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
-            /* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
-            if (!VMMIsHwVirtExtForced(pVM))
-                pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
-            else
-#endif
-                pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
-            pVCpu->vmm.s.pbEMTStackRC       = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
-            pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
-            AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
-
-            CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
-        }
-    }
-
-    return rc;
-}
-
-
-/**
- * Initialize the loggers.
- *
- * @returns VBox status code.
- * @param   pVM         Pointer to the shared VM structure.
- */
-static int vmmR3InitLoggers(PVM pVM)
-{
-    int rc;
-
-    /*
-     * Allocate RC & R0 Logger instances (they are finalized in the relocator).
-     */
-#ifdef LOG_ENABLED
-    PRTLOGGER pLogger = RTLogDefaultInstance();
-    if (pLogger)
-    {
-        pVM->vmm.s.cbRCLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pLogger->cGroups]);
-        rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
-        if (RT_FAILURE(rc))
-            return rc;
-        pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
-
-# ifdef VBOX_WITH_R0_LOGGING
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            PVMCPU pVCpu = &pVM->aCpus[i];
-
-            rc = MMR3HyperAllocOnceNoRelEx(pVM, RT_OFFSETOF(VMMR0LOGGER, Logger.afGroups[pLogger->cGroups]),
-                                           0, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
-                                           (void **)&pVCpu->vmm.s.pR0LoggerR3);
-            if (RT_FAILURE(rc))
-                return rc;
-            pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
-            //pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
-            pVCpu->vmm.s.pR0LoggerR3->cbLogger = RT_OFFSETOF(RTLOGGER, afGroups[pLogger->cGroups]);
-            pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
-        }
-# endif
-    }
-#endif /* LOG_ENABLED */
-
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-    /*
-     * Allocate RC release logger instances (finalized in the relocator).
-     */
-    PRTLOGGER pRelLogger = RTLogRelDefaultInstance();
-    if (pRelLogger)
-    {
-        pVM->vmm.s.cbRCRelLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
-        rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
-        if (RT_FAILURE(rc))
-            return rc;
-        pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
-    }
-#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
-    return VINF_SUCCESS;
-}
-
-
-/**
- * VMMR3Init worker that register the statistics with STAM.
- *
- * @param   pVM         The shared VM structure.
- */
-static void vmmR3InitRegisterStats(PVM pVM)
-{
-    /*
-     * Statistics.
-     */
-    STAM_REG(pVM, &pVM->vmm.s.StatRunRC,                    STAMTYPE_COUNTER, "/VMM/RunRC",                     STAMUNIT_OCCURENCES, "Number of context switches.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal,              STAMTYPE_COUNTER, "/VMM/RZRet/Normal",              STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt,           STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt",           STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper,      STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper",      STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap,           STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap",           STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch,          STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt,       STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector,       STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap,            STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate,             STAMTYPE_COUNTER, "/VMM/RZRet/Emulate",             STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOBlockEmulate,      STAMTYPE_COUNTER, "/VMM/RZRet/EmulateIOBlock",      STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_EMULATE_IO_BLOCK returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate,        STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate",        STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead,              STAMTYPE_COUNTER, "/VMM/RZRet/IORead",              STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_READ returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite,             STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite",             STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_WRITE returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead,            STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead",            STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite,           STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite",           STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_WRITE returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite,       STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite",       STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ_WRITE returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead,       STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead",       STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite,      STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite",      STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault,            STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPDFault,             STAMTYPE_COUNTER, "/VMM/RZRet/PDFault",             STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_PD_FAULT returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask,            STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask",            STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3,             STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR",              STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc,                STAMTYPE_COUNTER, "/VMM/RZRet/Misc",                STAMUNIT_OCCURENCES, "Number of misc returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3,           STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3",           STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF,             STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF",             STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP,             STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP",             STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ,        STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret",           STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM,       STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM",         STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3,                STAMTYPE_COUNTER, "/VMM/RZRet/ToR3",                STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Unknown,         STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Unknown",        STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3TMVirt,          STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/TMVirt",         STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3HandyPages,      STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Handy",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3PDMQueues,       STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/PDMQueue",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Rendezvous,      STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Rendezvous",     STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Timer,           STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Timer",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3DMA,             STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/DMA",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3CritSect,        STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/CritSect",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending,        STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending",        STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending,    STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending",    STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn,     STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn",     STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode,       STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode",       STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending,     STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending",     STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest,      STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest",      STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR,            STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR",            STAMUNIT_OCCURENCES, "Number of VINF_EM_HWACCM_PATCH_TPR_INSTR returns.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3,           STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc",             STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock,            STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock",          STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock,            STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock",          STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow,        STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk,        STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy,      STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy",    STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay,          STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay",        STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush,           STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError,         STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError",       STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
-    STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError,  STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError",   STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
-
-#ifdef VBOX_WITH_STATISTICS
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax,  STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,      "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg,  STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,      "Average stack usage.",      "/VMM/Stack/CPU%u/Avg", i);
-        STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64,       STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.",   "/VMM/Stack/CPU%u/Uses", i);
-    }
-#endif
-}
-
-
-/**
- * Initializes the R0 VMM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3_INT_DECL(int) VMMR3InitR0(PVM pVM)
-{
-    int    rc;
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    Assert(pVCpu && pVCpu->idCpu == 0);
-
-#ifdef LOG_ENABLED
-    /*
-     * Initialize the ring-0 logger if we haven't done so yet.
-     */
-    if (    pVCpu->vmm.s.pR0LoggerR3
-        &&  !pVCpu->vmm.s.pR0LoggerR3->fCreated)
-    {
-        rc = VMMR3UpdateLoggers(pVM);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-#endif
-
-    /*
-     * Call Ring-0 entry with init code.
-     */
-    for (;;)
-    {
-#ifdef NO_SUPCALLR0VMM
-        //rc = VERR_GENERAL_FAILURE;
-        rc = VINF_SUCCESS;
-#else
-        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_VMMR0_INIT, VMMGetSvnRev(), NULL);
-#endif
-        /*
-         * Flush the logs.
-         */
-#ifdef LOG_ENABLED
-        if (    pVCpu->vmm.s.pR0LoggerR3
-            &&  pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
-            RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
-#endif
-        if (rc != VINF_VMM_CALL_HOST)
-            break;
-        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
-            break;
-        /* Resume R0 */
-    }
-
-    if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
-    {
-        LogRel(("R0 init failed, rc=%Rra\n", rc));
-        if (RT_SUCCESS(rc))
-            rc = VERR_INTERNAL_ERROR;
-    }
-    return rc;
-}
-
-
-/**
- * Initializes the RC VMM.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- */
-VMMR3_INT_DECL(int) VMMR3InitRC(PVM pVM)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    Assert(pVCpu && pVCpu->idCpu == 0);
-
-    /* In VMX mode, there's no need to init RC. */
-    if (pVM->vmm.s.fSwitcherDisabled)
-        return VINF_SUCCESS;
-
-    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
-
-    /*
-     * Call VMMGCInit():
-     *      -# resolve the address.
-     *      -# setup stackframe and EIP to use the trampoline.
-     *      -# do a generic hypervisor call.
-     */
-    RTRCPTR RCPtrEP;
-    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
-    if (RT_SUCCESS(rc))
-    {
-        CPUMHyperSetCtxCore(pVCpu, NULL);
-        CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
-        uint64_t u64TS = RTTimeProgramStartNanoTS();
-        CPUMPushHyper(pVCpu, (uint32_t)(u64TS >> 32));    /* Param 3: The program startup TS - Hi. */
-        CPUMPushHyper(pVCpu, (uint32_t)u64TS);            /* Param 3: The program startup TS - Lo. */
-        CPUMPushHyper(pVCpu, VMMGetSvnRev());             /* Param 2: Version argument. */
-        CPUMPushHyper(pVCpu, VMMGC_DO_VMMGC_INIT);        /* Param 1: Operation. */
-        CPUMPushHyper(pVCpu, pVM->pVMRC);                 /* Param 0: pVM */
-        CPUMPushHyper(pVCpu, 5 * sizeof(RTRCPTR));        /* trampoline param: stacksize.  */
-        CPUMPushHyper(pVCpu, RCPtrEP);                    /* Call EIP. */
-        CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
-        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
-
-        for (;;)
-        {
-#ifdef NO_SUPCALLR0VMM
-            //rc = VERR_GENERAL_FAILURE;
-            rc = VINF_SUCCESS;
-#else
-            rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
-#endif
-#ifdef LOG_ENABLED
-            PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
-            if (    pLogger
-                &&  pLogger->offScratch > 0)
-                RTLogFlushRC(NULL, pLogger);
-#endif
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-            PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
-            if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
-                RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
-#endif
-            if (rc != VINF_VMM_CALL_HOST)
-                break;
-            rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-            if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
-                break;
-        }
-
-        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
-        {
-            VMMR3FatalDump(pVM, pVCpu, rc);
-            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
-                rc = VERR_INTERNAL_ERROR;
-        }
-        AssertRC(rc);
-    }
-    return rc;
-}
-
-
-/**
- * Called when an init phase completes.
- *
- * @returns VBox status code.
- * @param   pVM                 The VM handle.
- * @param   enmWhat             Which init phase.
- */
-VMMR3_INT_DECL(int) VMMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
-{
-    int rc = VINF_SUCCESS;
-
-    switch (enmWhat)
-    {
-        case VMINITCOMPLETED_RING3:
-        {
-            /*
-             * Set page attributes to r/w for stack pages.
-             */
-            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-            {
-                rc = PGMMapSetPage(pVM, pVM->aCpus[idCpu].vmm.s.pbEMTStackRC, VMM_STACK_SIZE,
-                                   X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
-                AssertRCReturn(rc, rc);
-            }
-
-            /*
-             * Create the EMT yield timer.
-             */
-            rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
-            AssertRCReturn(rc, rc);
-
-            rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
-            AssertRCReturn(rc, rc);
-
-#ifdef VBOX_WITH_NMI
-            /*
-             * Map the host APIC into GC - This is AMD/Intel + Host OS specific!
-             */
-            rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
-                        X86_PTE_P | X86_PTE_RW | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A | X86_PTE_D);
-            AssertRCReturn(rc, rc);
-#endif
-
-#ifdef VBOX_STRICT_VMM_STACK
-            /*
-             * Setup the stack guard pages: Two inaccessible pages at each sides of the
-             * stack to catch over/under-flows.
-             */
-            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-            {
-                uint8_t *pbEMTStackR3 = pVM->aCpus[idCpu].vmm.s.pbEMTStackR3;
-
-                memset(pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
-                MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /*fSet*/);
-
-                memset(pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
-                MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /*fSet*/);
-            }
-            pVM->vmm.s.fStackGuardsStationed = true;
-#endif
-            break;
-        }
-
-        case VMINITCOMPLETED_RING0:
-        {
-            /*
-             * Disable the periodic preemption timers if we can use the
-             * VMX-preemption timer instead.
-             */
-            if (   pVM->vmm.s.fUsePeriodicPreemptionTimers
-                && HWACCMR3IsVmxPreemptionTimerUsed(pVM))
-                pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
-            LogRel(("VMM: fUsePeriodicPreemptionTimers=%RTbool\n", pVM->vmm.s.fUsePeriodicPreemptionTimers));
-            break;
-        }
-
-        default: /* shuts up gcc */
-            break;
-    }
-
-    return rc;
-}
-
-
-/**
- * Terminate the VMM bits.
- *
- * @returns VINF_SUCCESS.
- * @param   pVM         The VM handle.
- */
-VMMR3_INT_DECL(int) VMMR3Term(PVM pVM)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    Assert(pVCpu && pVCpu->idCpu == 0);
-
-    /*
-     * Call Ring-0 entry with termination code.
-     */
-    int rc;
-    for (;;)
-    {
-#ifdef NO_SUPCALLR0VMM
-        //rc = VERR_GENERAL_FAILURE;
-        rc = VINF_SUCCESS;
-#else
-        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_VMMR0_TERM, 0, NULL);
-#endif
-        /*
-         * Flush the logs.
-         */
-#ifdef LOG_ENABLED
-        if (    pVCpu->vmm.s.pR0LoggerR3
-            &&  pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
-            RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
-#endif
-        if (rc != VINF_VMM_CALL_HOST)
-            break;
-        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
-            break;
-        /* Resume R0 */
-    }
-    if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
-    {
-        LogRel(("VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
-        if (RT_SUCCESS(rc))
-            rc = VERR_INTERNAL_ERROR;
-    }
-
-    RTCritSectDelete(&pVM->vmm.s.CritSectSync);
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
-        pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
-    }
-    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
-    pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
-    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
-    pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
-    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
-    pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
-    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
-    pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
-
-#ifdef VBOX_STRICT_VMM_STACK
-    /*
-     * Make the two stack guard pages present again.
-     */
-    if (pVM->vmm.s.fStackGuardsStationed)
-    {
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            uint8_t *pbEMTStackR3 = pVM->aCpus[i].vmm.s.pbEMTStackR3;
-            MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE,      PAGE_SIZE, false /*fSet*/);
-            MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /*fSet*/);
-        }
-        pVM->vmm.s.fStackGuardsStationed = false;
-    }
-#endif
-    return rc;
-}
-
-
-/**
- * Applies relocations to data and code managed by this
- * component. This function will be called at init and
- * whenever the VMM need to relocate it self inside the GC.
- *
- * The VMM will need to apply relocations to the core code.
- *
- * @param   pVM         The VM handle.
- * @param   offDelta    The relocation delta.
- */
-VMMR3_INT_DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
-{
-    LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
-
-    /*
-     * Recalc the RC address.
-     */
-#ifdef VBOX_WITH_RAW_MODE
-    pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
-#endif
-
-    /*
-     * The stack.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU pVCpu = &pVM->aCpus[i];
-
-        CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
-
-        pVCpu->vmm.s.pbEMTStackRC       = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
-        pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
-    }
-
-    /*
-     * All the switchers.
-     */
-    vmmR3SwitcherRelocate(pVM, offDelta);
-
-    /*
-     * Get other RC entry points.
-     */
-    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
-    AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
-
-    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
-    AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
-
-    /*
-     * Update the logger.
-     */
-    VMMR3UpdateLoggers(pVM);
-}
-
-
-/**
- * Updates the settings for the RC and R0 loggers.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-VMMR3_INT_DECL(int) VMMR3UpdateLoggers(PVM pVM)
-{
-    /*
-     * Simply clone the logger instance (for RC).
-     */
-    int rc = VINF_SUCCESS;
-    RTRCPTR RCPtrLoggerFlush = 0;
-
-    if (pVM->vmm.s.pRCLoggerR3
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-        || pVM->vmm.s.pRCRelLoggerR3
-#endif
-       )
-    {
-        rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
-        AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
-    }
-
-    if (pVM->vmm.s.pRCLoggerR3)
-    {
-        RTRCPTR RCPtrLoggerWrapper = 0;
-        rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
-        AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
-
-        pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
-        rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
-                          RCPtrLoggerWrapper,  RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
-        AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
-    }
-
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-    if (pVM->vmm.s.pRCRelLoggerR3)
-    {
-        RTRCPTR RCPtrLoggerWrapper = 0;
-        rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
-        AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
-
-        pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
-        rc = RTLogCloneRC(RTLogRelDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
-                          RCPtrLoggerWrapper,  RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
-        AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
-    }
-#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
-
-#ifdef LOG_ENABLED
-    /*
-     * For the ring-0 EMT logger, we use a per-thread logger instance
-     * in ring-0. Only initialize it once.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        PVMCPU       pVCpu = &pVM->aCpus[i];
-        PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
-        if (pR0LoggerR3)
-        {
-            if (!pR0LoggerR3->fCreated)
-            {
-                RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
-                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
-                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
-
-                RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
-                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
-                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
-
-                rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
-                                      *(PFNRTLOGGER *)&pfnLoggerWrapper, *(PFNRTLOGFLUSH *)&pfnLoggerFlush,
-                                      RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY);
-                AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
-
-                RTR0PTR pfnLoggerPrefix = NIL_RTR0PTR;
-                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerPrefix", &pfnLoggerPrefix);
-                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerPrefix not found! rc=%Rra\n", rc), rc);
-                rc = RTLogSetCustomPrefixCallback(&pR0LoggerR3->Logger, *(PFNRTLOGPREFIX *)&pfnLoggerPrefix, NULL);
-                AssertReleaseMsgRCReturn(rc, ("RTLogSetCustomPrefixCallback failed! rc=%Rra\n", rc), rc);
-
-                pR0LoggerR3->idCpu = i;
-                pR0LoggerR3->fCreated = true;
-                pR0LoggerR3->fFlushingDisabled = false;
-
-            }
-
-            rc = RTLogCopyGroupsAndFlags(&pR0LoggerR3->Logger, NULL /* default */, pVM->vmm.s.pRCLoggerR3->fFlags, RTLOGFLAGS_BUFFERED);
-            AssertRC(rc);
-        }
-    }
-#endif
-    return rc;
-}
-
-
-/**
- * Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
- *
- * @returns Pointer to the buffer.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
-{
-    if (HWACCMIsEnabled(pVM))
-        return pVM->vmm.s.szRing0AssertMsg1;
-
-    RTRCPTR RCPtr;
-    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
-    if (RT_SUCCESS(rc))
-        return (const char *)MMHyperRCToR3(pVM, RCPtr);
-
-    return NULL;
-}
-
-
-/**
- * Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
- *
- * @returns Pointer to the buffer.
- * @param   pVM         The VM handle.
- */
-VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
-{
-    if (HWACCMIsEnabled(pVM))
-        return pVM->vmm.s.szRing0AssertMsg2;
-
-    RTRCPTR RCPtr;
-    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
-    if (RT_SUCCESS(rc))
-        return (const char *)MMHyperRCToR3(pVM, RCPtr);
-
-    return NULL;
-}
-
-
-/**
- * Execute state save operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- */
-static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
-{
-    LogFlow(("vmmR3Save:\n"));
-
-    /*
-     * Save the started/stopped state of all CPUs except 0 as it will always
-     * be running. This avoids breaking the saved state version. :-)
-     */
-    for (VMCPUID i = 1; i < pVM->cCpus; i++)
-        SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
-
-    return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
-}
-
-
-/**
- * Execute state load operation.
- *
- * @returns VBox status code.
- * @param   pVM             VM Handle.
- * @param   pSSM            SSM operation handle.
- * @param   uVersion        Data layout version.
- * @param   uPass           The data pass.
- */
-static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
-{
-    LogFlow(("vmmR3Load:\n"));
-    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
-
-    /*
-     * Validate version.
-     */
-    if (    uVersion != VMM_SAVED_STATE_VERSION
-        &&  uVersion != VMM_SAVED_STATE_VERSION_3_0)
-    {
-        AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
-        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
-    }
-
-    if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
-    {
-        /* Ignore the stack bottom, stack pointer and stack bits. */
-        RTRCPTR RCPtrIgnored;
-        SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
-        SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
-#ifdef RT_OS_DARWIN
-        if (   SSMR3HandleVersion(pSSM)  >= VBOX_FULL_VERSION_MAKE(3,0,0)
-            && SSMR3HandleVersion(pSSM)  <  VBOX_FULL_VERSION_MAKE(3,1,0)
-            && SSMR3HandleRevision(pSSM) >= 48858
-            && (   !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
-                || !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
-           )
-            SSMR3Skip(pSSM, 16384);
-        else
-            SSMR3Skip(pSSM, 8192);
-#else
-        SSMR3Skip(pSSM, 8192);
-#endif
-    }
-
-    /*
-     * Restore the VMCPU states. VCPU 0 is always started.
-     */
-    VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
-    for (VMCPUID i = 1; i < pVM->cCpus; i++)
-    {
-        bool fStarted;
-        int rc = SSMR3GetBool(pSSM, &fStarted);
-        if (RT_FAILURE(rc))
-            return rc;
-        VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
-    }
-
-    /* terminator */
-    uint32_t u32;
-    int rc = SSMR3GetU32(pSSM, &u32);
-    if (RT_FAILURE(rc))
-        return rc;
-    if (u32 != UINT32_MAX)
-    {
-        AssertMsgFailed(("u32=%#x\n", u32));
-        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Resolve a builtin RC symbol.
- *
- * Called by PDM when loading or relocating RC modules.
- *
- * @returns VBox status
- * @param   pVM             VM Handle.
- * @param   pszSymbol       Symbol to resolv
- * @param   pRCPtrValue     Where to store the symbol value.
- *
- * @remark  This has to work before VMMR3Relocate() is called.
- */
-VMMR3_INT_DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
-{
-    if (!strcmp(pszSymbol, "g_Logger"))
-    {
-        if (pVM->vmm.s.pRCLoggerR3)
-            pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
-        *pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
-    }
-    else if (!strcmp(pszSymbol, "g_RelLogger"))
-    {
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-        if (pVM->vmm.s.pRCRelLoggerR3)
-            pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
-        *pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
-#else
-        *pRCPtrValue = NIL_RTRCPTR;
-#endif
-    }
-    else
-        return VERR_SYMBOL_NOT_FOUND;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Suspends the CPU yielder.
- *
- * @param   pVM             The VM handle.
- */
-VMMR3_INT_DECL(void) VMMR3YieldSuspend(PVM pVM)
-{
-    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
-    if (!pVM->vmm.s.cYieldResumeMillies)
-    {
-        uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
-        uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
-        if (u64Now >= u64Expire || u64Expire == ~(uint64_t)0)
-            pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
-        else
-            pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
-        TMTimerStop(pVM->vmm.s.pYieldTimer);
-    }
-    pVM->vmm.s.u64LastYield = RTTimeNanoTS();
-}
-
-
-/**
- * Stops the CPU yielder.
- *
- * @param   pVM             The VM handle.
- */
-VMMR3_INT_DECL(void) VMMR3YieldStop(PVM pVM)
-{
-    if (!pVM->vmm.s.cYieldResumeMillies)
-        TMTimerStop(pVM->vmm.s.pYieldTimer);
-    pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
-    pVM->vmm.s.u64LastYield = RTTimeNanoTS();
-}
-
-
-/**
- * Resumes the CPU yielder when it has been a suspended or stopped.
- *
- * @param   pVM             The VM handle.
- */
-VMMR3_INT_DECL(void) VMMR3YieldResume(PVM pVM)
-{
-    if (pVM->vmm.s.cYieldResumeMillies)
-    {
-        TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
-        pVM->vmm.s.cYieldResumeMillies = 0;
-    }
-}
-
-
-/**
- * Internal timer callback function.
- *
- * @param   pVM             The VM.
- * @param   pTimer          The timer handle.
- * @param   pvUser          User argument specified upon timer creation.
- */
-static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
-{
-    /*
-     * This really needs some careful tuning. While we shouldn't be too greedy since
-     * that'll cause the rest of the system to stop up, we shouldn't be too nice either
-     * because that'll cause us to stop up.
-     *
-     * The current logic is to use the default interval when there is no lag worth
-     * mentioning, but when we start accumulating lag we don't bother yielding at all.
-     *
-     * (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
-     * so the lag is up to date.)
-     */
-    const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
-    if (    u64Lag     <   50000000 /* 50ms */
-        ||  (   u64Lag < 1000000000 /*  1s */
-             && RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
-       )
-    {
-        uint64_t u64Elapsed = RTTimeNanoTS();
-        pVM->vmm.s.u64LastYield = u64Elapsed;
-
-        RTThreadYield();
-
-#ifdef LOG_ENABLED
-        u64Elapsed = RTTimeNanoTS() - u64Elapsed;
-        Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
-#endif
-    }
-    TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
-}
-
-
-/**
- * Executes guest code in the raw-mode context.
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       The VMCPU to operate on.
- */
-VMMR3_INT_DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
-{
-    Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
-
-    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
-
-    /*
-     * Set the EIP and ESP.
-     */
-    CPUMSetHyperEIP(pVCpu, CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
-                    ? pVM->vmm.s.pfnCPUMRCResumeGuestV86
-                    : pVM->vmm.s.pfnCPUMRCResumeGuest);
-    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
-
-    /*
-     * We hide log flushes (outer) and hypervisor interrupts (inner).
-     */
-    for (;;)
-    {
-#ifdef VBOX_STRICT
-        if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) || CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
-            EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
-        PGMMapCheck(pVM);
-#endif
-        int rc;
-        do
-        {
-#ifdef NO_SUPCALLR0VMM
-            rc = VERR_GENERAL_FAILURE;
-#else
-            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
-            if (RT_LIKELY(rc == VINF_SUCCESS))
-                rc = pVCpu->vmm.s.iLastGZRc;
-#endif
-        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
-
-        /*
-         * Flush the logs.
-         */
-#ifdef LOG_ENABLED
-        PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
-        if (    pLogger
-            &&  pLogger->offScratch > 0)
-            RTLogFlushRC(NULL, pLogger);
-#endif
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-        PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
-        if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
-            RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
-#endif
-        if (rc != VINF_VMM_CALL_HOST)
-        {
-            Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
-            return rc;
-        }
-        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-        /* Resume GC */
-    }
-}
-
-
-/**
- * Executes guest code (Intel VT-x and AMD-V).
- *
- * @param   pVM         VM handle.
- * @param   pVCpu       The VMCPU to operate on.
- */
-VMMR3_INT_DECL(int) VMMR3HwAccRunGC(PVM pVM, PVMCPU pVCpu)
-{
-    Log2(("VMMR3HwAccRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
-
-    for (;;)
-    {
-        int rc;
-        do
-        {
-#ifdef NO_SUPCALLR0VMM
-            rc = VERR_GENERAL_FAILURE;
-#else
-            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HWACC_RUN, pVCpu->idCpu);
-            if (RT_LIKELY(rc == VINF_SUCCESS))
-                rc = pVCpu->vmm.s.iLastGZRc;
-#endif
-        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
-
-#if 0 /* todo triggers too often */
-        Assert(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TO_R3));
-#endif
-
-#ifdef LOG_ENABLED
-        /*
-         * Flush the log
-         */
-        PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
-        if (    pR0LoggerR3
-            &&  pR0LoggerR3->Logger.offScratch > 0)
-            RTLogFlushToLogger(&pR0LoggerR3->Logger, NULL);
-#endif /* !LOG_ENABLED */
-        if (rc != VINF_VMM_CALL_HOST)
-        {
-            Log2(("VMMR3HwAccRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
-            return rc;
-        }
-        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-        /* Resume R0 */
-    }
-}
-
-/**
- * VCPU worker for VMMSendSipi.
- *
- * @param   pVM         The VM to operate on.
- * @param   idCpu       Virtual CPU to perform SIPI on
- * @param   uVector     SIPI vector
- */
-DECLCALLBACK(int) vmmR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
-{
-    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-    VMCPU_ASSERT_EMT(pVCpu);
-
-    /** @todo what are we supposed to do if the processor is already running? */
-    if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
-        return VERR_ACCESS_DENIED;
-
-
-    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
-
-    pCtx->cs                        = uVector << 8;
-    pCtx->csHid.u64Base             = uVector << 12;
-    pCtx->csHid.u32Limit            = 0x0000ffff;
-    pCtx->rip                       = 0;
-
-    Log(("vmmR3SendSipi for VCPU %d with vector %x\n", uVector));
-
-# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
-    EMSetState(pVCpu, EMSTATE_HALTED);
-    return VINF_EM_RESCHEDULE;
-# else /* And if we go the VMCPU::enmState way it can stay here. */
-    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
-    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
-    return VINF_SUCCESS;
-# endif
-}
-
-DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
-{
-    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-    VMCPU_ASSERT_EMT(pVCpu);
-
-    Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
-    CPUMR3ResetCpu(pVCpu);
-    return VINF_EM_WAIT_SIPI;
-}
-
-/**
- * Sends SIPI to the virtual CPU by setting CS:EIP into vector-dependent state
- * and unhalting processor
- *
- * @param   pVM         The VM to operate on.
- * @param   idCpu       Virtual CPU to perform SIPI on
- * @param   uVector     SIPI vector
- */
-VMMR3_INT_DECL(void) VMMR3SendSipi(PVM pVM, VMCPUID idCpu,  uint32_t uVector)
-{
-    AssertReturnVoid(idCpu < pVM->cCpus);
-
-    int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendSipi, 3, pVM, idCpu, uVector);
-    AssertRC(rc);
-}
-
-/**
- * Sends init IPI to the virtual CPU.
- *
- * @param   pVM         The VM to operate on.
- * @param   idCpu       Virtual CPU to perform int IPI on
- */
-VMMR3_INT_DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
-{
-    AssertReturnVoid(idCpu < pVM->cCpus);
-
-    int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
-    AssertRC(rc);
-}
-
-/**
- * Registers the guest memory range that can be used for patching
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pPatchMem   Patch memory range
- * @param   cbPatchMem  Size of the memory range
- */
-VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
-{
-    if (HWACCMIsEnabled(pVM))
-        return HWACMMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
-
-    return VERR_NOT_SUPPORTED;
-}
-
-/**
- * Deregisters the guest memory range that can be used for patching
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   pPatchMem   Patch memory range
- * @param   cbPatchMem  Size of the memory range
- */
-VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
-{
-    if (HWACCMIsEnabled(pVM))
-        return HWACMMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
-
-    return VINF_SUCCESS;
-}
-
-
-/**
- * VCPU worker for VMMR3SynchronizeAllVCpus.
- *
- * @param   pVM         The VM to operate on.
- * @param   idCpu       Virtual CPU to perform SIPI on
- * @param   uVector     SIPI vector
- */
-DECLCALLBACK(int) vmmR3SyncVCpu(PVM pVM)
-{
-    /* Block until the job in the caller has finished. */
-    RTCritSectEnter(&pVM->vmm.s.CritSectSync);
-    RTCritSectLeave(&pVM->vmm.s.CritSectSync);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Atomically execute a callback handler
- * Note: This is very expensive; avoid using it frequently!
- *
- * @param   pVM         The VM to operate on.
- * @param   pfnHandler  Callback handler
- * @param   pvUser      User specified parameter
- *
- * @thread  EMT
- * @todo    Remove this if not used again soon.
- */
-VMMR3DECL(int) VMMR3AtomicExecuteHandler(PVM pVM, PFNATOMICHANDLER pfnHandler, void *pvUser)
-{
-    int    rc;
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
-
-    /* Shortcut for the uniprocessor case. */
-    if (pVM->cCpus == 1)
-        return pfnHandler(pVM, pvUser);
-
-    RTCritSectEnter(&pVM->vmm.s.CritSectSync);
-    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
-    {
-        if (idCpu != pVCpu->idCpu)
-        {
-            rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SyncVCpu, 1, pVM);
-            AssertRC(rc);
-        }
-    }
-    /* Wait until all other VCPUs are waiting for us. */
-    while (RTCritSectGetWaiters(&pVM->vmm.s.CritSectSync) != (int32_t)(pVM->cCpus - 1))
-        RTThreadSleep(1);
-
-    rc = pfnHandler(pVM, pvUser);
-    RTCritSectLeave(&pVM->vmm.s.CritSectSync);
-    return rc;
-}
-
-
-/**
- * Count returns and have the last non-caller EMT wake up the caller.
- *
- * @returns VBox strict informational status code for EM scheduling. No failures
- *          will be returned here, those are for the caller only.
- *
- * @param   pVM                 The VM handle.
- */
-DECL_FORCE_INLINE(int) vmmR3EmtRendezvousNonCallerReturn(PVM pVM)
-{
-    int rcRet = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
-    uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
-    if (cReturned == pVM->cCpus - 1U)
-    {
-        int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
-        AssertLogRelRC(rc);
-    }
-
-    AssertLogRelMsgReturn(   rcRet <= VINF_SUCCESS
-                          || (rcRet >= VINF_EM_FIRST && rcRet <= VINF_EM_LAST),
-                          ("%Rrc\n", rcRet),
-                          VERR_IPE_UNEXPECTED_INFO_STATUS);
-    return RT_SUCCESS(rcRet) ? rcRet : VINF_SUCCESS;
-}
-
-
-/**
- * Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
- *
- * @returns VBox strict informational status code for EM scheduling. No failures
- *          will be returned here, those are for the caller only.  When
- *          fIsCaller is set, VINF_SUCCESS is always returned.
- *
- * @param   pVM                 The VM handle.
- * @param   pVCpu               The VMCPU structure for the calling EMT.
- * @param   fIsCaller           Whether we're the VMMR3EmtRendezvous caller or
- *                              not.
- * @param   fFlags              The flags.
- * @param   pfnRendezvous       The callback.
- * @param   pvUser              The user argument for the callback.
- */
-static int vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
-                                    uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
-{
-    int rc;
-
-    /*
-     * Enter, the last EMT triggers the next callback phase.
-     */
-    uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
-    if (cEntered != pVM->cCpus)
-    {
-        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
-        {
-            /* Wait for our turn. */
-            rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
-            AssertLogRelRC(rc);
-        }
-        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
-        {
-            /* Wait for the last EMT to arrive and wake everyone up. */
-            rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
-            AssertLogRelRC(rc);
-        }
-        else if (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
-                 || (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
-        {
-            /* Wait for our turn. */
-            rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
-            AssertLogRelRC(rc);
-        }
-        else
-        {
-            Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
-
-            /*
-             * The execute once is handled specially to optimize the code flow.
-             *
-             * The last EMT to arrive will perform the callback and the other
-             * EMTs will wait on the Done/DoneCaller semaphores (instead of
-             * the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
-             * returns, that EMT will initiate the normal return sequence.
-             */
-            if (!fIsCaller)
-            {
-                rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
-                AssertLogRelRC(rc);
-
-                return vmmR3EmtRendezvousNonCallerReturn(pVM);
-            }
-            return VINF_SUCCESS;
-        }
-    }
-    else
-    {
-        /*
-         * All EMTs are waiting, clear the FF and take action according to the
-         * execution method.
-         */
-        VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
-
-        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
-        {
-            /* Wake up everyone. */
-            rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
-            AssertLogRelRC(rc);
-        }
-        else if (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
-                 || (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
-        {
-            /* Figure out who to wake up and wake it up. If it's ourself, then
-               it's easy otherwise wait for our turn. */
-            VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
-                           ? 0
-                           : pVM->cCpus - 1U;
-            if (pVCpu->idCpu != iFirst)
-            {
-                rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
-                AssertLogRelRC(rc);
-                rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
-                AssertLogRelRC(rc);
-            }
-        }
-        /* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
-    }
-
-
-    /*
-     * Do the callback and update the status if necessary.
-     */
-    if (    !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
-        ||  RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
-    {
-        VBOXSTRICTRC rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
-        if (rcStrict != VINF_SUCCESS)
-        {
-            AssertLogRelMsg(   rcStrict <= VINF_SUCCESS
-                            || (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
-                            ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
-            int32_t i32RendezvousStatus;
-            do
-            {
-                i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
-                if (    rcStrict == i32RendezvousStatus
-                    ||  RT_FAILURE(i32RendezvousStatus)
-                    ||  (   i32RendezvousStatus != VINF_SUCCESS
-                         && rcStrict > i32RendezvousStatus))
-                    break;
-            } while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict), i32RendezvousStatus));
-        }
-    }
-
-    /*
-     * Increment the done counter and take action depending on whether we're
-     * the last to finish callback execution.
-     */
-    uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
-    if (    cDone != pVM->cCpus
-        &&  (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
-    {
-        /* Signal the next EMT? */
-        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
-        {
-            rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
-            AssertLogRelRC(rc);
-        }
-        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
-        {
-            Assert(cDone == pVCpu->idCpu + 1U);
-            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
-            AssertLogRelRC(rc);
-        }
-        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
-        {
-            Assert(pVM->cCpus - cDone == pVCpu->idCpu);
-            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
-            AssertLogRelRC(rc);
-        }
-
-        /* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
-        if (!fIsCaller)
-        {
-            rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
-            AssertLogRelRC(rc);
-        }
-    }
-    else
-    {
-        /* Callback execution is all done, tell the rest to return. */
-        rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
-        AssertLogRelRC(rc);
-    }
-
-    if (!fIsCaller)
-        return vmmR3EmtRendezvousNonCallerReturn(pVM);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Called in response to VM_FF_EMT_RENDEZVOUS.
- *
- * @returns VBox strict status code - EM scheduling.  No errors will be returned
- *          here, nor will any non-EM scheduling status codes be returned.
- *
- * @param   pVM         The VM handle
- * @param   pVCpu       The handle of the calling EMT.
- *
- * @thread  EMT
- */
-VMMR3_INT_DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
-{
-    return vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
-                                    pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
-}
-
-
-/**
- * EMT rendezvous.
- *
- * Gathers all the EMTs and execute some code on each of them, either in a one
- * by one fashion or all at once.
- *
- * @returns VBox strict status code.  This will be the the first error,
- *          VINF_SUCCESS, or an EM scheduling status code.
- *
- * @param   pVM             The VM handle.
- * @param   fFlags          Flags indicating execution methods. See
- *                          grp_VMMR3EmtRendezvous_fFlags.
- * @param   pfnRendezvous   The callback.
- * @param   pvUser          User argument for the callback.
- *
- * @thread  Any.
- */
-VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
-{
-    /*
-     * Validate input.
-     */
-    AssertMsg(   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
-              && (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
-              && !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
-    AssertMsg(   !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
-              || (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
-                  && (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
-              ("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
-
-    VBOXSTRICTRC rcStrict;
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    if (!pVCpu)
-        /*
-         * Forward the request to an EMT thread.
-         */
-        rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY,
-                                   (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
-    else if (pVM->cCpus == 1)
-        /*
-         * Shortcut for the single EMT case.
-         */
-        rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
-    else
-    {
-        /*
-         * Spin lock. If busy, wait for the other EMT to finish while keeping a
-         * lookout of the RENDEZVOUS FF.
-         */
-        int rc;
-        rcStrict = VINF_SUCCESS;
-        if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
-        {
-            while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
-            {
-                if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
-                {
-                    rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
-                    if (    rc != VINF_SUCCESS
-                        &&  (   rcStrict == VINF_SUCCESS
-                             || rcStrict > rc))
-                        rcStrict = rc;
-                    /** @todo Perhaps deal with termination here?  */
-                }
-                ASMNopPause();
-            }
-        }
-        Assert(!VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS));
-
-        /*
-         * Clear the slate. This is a semaphore ping-pong orgy. :-)
-         */
-        for (VMCPUID i = 0; i < pVM->cCpus; i++)
-        {
-            rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
-            AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        }
-        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0);         AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);  AssertLogRelRC(rc);
-        rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone);            AssertLogRelRC(rc);
-        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0);            AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
-        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
-        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
-        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
-        ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
-        ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, (void *)(uintptr_t)pfnRendezvous);
-        ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
-        ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
-
-        /*
-         * Set the FF and poke the other EMTs.
-         */
-        VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
-        VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
-
-        /*
-         * Do the same ourselves.
-         */
-        vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
-
-        /*
-         * The caller waits for the other EMTs to be done and return before doing
-         * the cleanup. This makes away with wakeup / reset races we would otherwise
-         * risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
-         */
-        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
-        AssertLogRelRC(rc);
-
-        /*
-         * Get the return code and clean up a little bit.
-         */
-        int rcMy = pVM->vmm.s.i32RendezvousStatus;
-        ASMAtomicWriteNullPtr((void * volatile *)&pVM->vmm.s.pfnRendezvous);
-
-        ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
-
-        /*
-         * Merge rcStrict and rcMy.
-         */
-        AssertRC(VBOXSTRICTRC_VAL(rcStrict));
-        if (    rcMy != VINF_SUCCESS
-            &&  (   rcStrict == VINF_SUCCESS
-                 || rcStrict > rcMy))
-            rcStrict = rcMy;
-    }
-
-    AssertLogRelMsgReturn(   rcStrict <= VINF_SUCCESS
-                          || (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
-                          ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
-                          VERR_IPE_UNEXPECTED_INFO_STATUS);
-    return VBOXSTRICTRC_VAL(rcStrict);
-}
-
-
-/**
- * Read from the ring 0 jump buffer stack
- *
- * @returns VBox status code.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idCpu           The ID of the source CPU context (for the address).
- * @param   R0Addr          Where to start reading.
- * @param   pvBuf           Where to store the data we've read.
- * @param   cbRead          The number of bytes to read.
- */
-VMMR3_INT_DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR R0Addr, void *pvBuf, size_t cbRead)
-{
-    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
-    AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
-
-#ifdef VMM_R0_SWITCH_STACK
-    RTHCUINTPTR off = R0Addr - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
-#else
-    RTHCUINTPTR off = pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - (pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - R0Addr);
-#endif
-    if (   off          >  VMM_STACK_SIZE
-        || off + cbRead >= VMM_STACK_SIZE)
-        return VERR_INVALID_POINTER;
-
-    memcpy(pvBuf, &pVCpu->vmm.s.pbEMTStackR3[off], cbRead);
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Calls a RC function.
- *
- * @param   pVM         The VM handle.
- * @param   RCPtrEntry  The address of the RC function.
- * @param   cArgs       The number of arguments in the ....
- * @param   ...         Arguments to the function.
- */
-VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
-{
-    va_list args;
-    va_start(args, cArgs);
-    int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
-    va_end(args);
-    return rc;
-}
-
-
-/**
- * Calls a RC function.
- *
- * @param   pVM         The VM handle.
- * @param   RCPtrEntry  The address of the RC function.
- * @param   cArgs       The number of arguments in the ....
- * @param   args        Arguments to the function.
- */
-VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
-{
-    /* Raw mode implies 1 VCPU. */
-    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
-
-    /*
-     * Setup the call frame using the trampoline.
-     */
-    CPUMHyperSetCtxCore(pVCpu, NULL);
-    memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
-    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32));
-    PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
-    int i = cArgs;
-    while (i-- > 0)
-        *pFrame++ = va_arg(args, RTGCUINTPTR32);
-
-    CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32));                          /* stack frame size */
-    CPUMPushHyper(pVCpu, RCPtrEntry);                                             /* what to call */
-    CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
-
-    /*
-     * We hide log flushes (outer) and hypervisor interrupts (inner).
-     */
-    for (;;)
-    {
-        int rc;
-        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
-        do
-        {
-#ifdef NO_SUPCALLR0VMM
-            rc = VERR_GENERAL_FAILURE;
-#else
-            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
-            if (RT_LIKELY(rc == VINF_SUCCESS))
-                rc = pVCpu->vmm.s.iLastGZRc;
-#endif
-        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
-
-        /*
-         * Flush the logs.
-         */
-#ifdef LOG_ENABLED
-        PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
-        if (    pLogger
-            &&  pLogger->offScratch > 0)
-            RTLogFlushRC(NULL, pLogger);
-#endif
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-        PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
-        if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
-            RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
-#endif
-        if (rc == VERR_TRPM_PANIC || rc == VERR_TRPM_DONT_PANIC)
-            VMMR3FatalDump(pVM, pVCpu, rc);
-        if (rc != VINF_VMM_CALL_HOST)
-        {
-            Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
-            return rc;
-        }
-        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-}
-
-
-/**
- * Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
- *
- * @returns VBox status code.
- * @param   pVM         The VM to operate on.
- * @param   uOperation  Operation to execute.
- * @param   u64Arg      Constant argument.
- * @param   pReqHdr     Pointer to a request header. See SUPR3CallVMMR0Ex for
- *                      details.
- */
-VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
-{
-    PVMCPU pVCpu = VMMGetCpu(pVM);
-    AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
-
-    /*
-     * Call Ring-0 entry with init code.
-     */
-    int rc;
-    for (;;)
-    {
-#ifdef NO_SUPCALLR0VMM
-        rc = VERR_GENERAL_FAILURE;
-#else
-        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, uOperation, u64Arg, pReqHdr);
-#endif
-        /*
-         * Flush the logs.
-         */
-#ifdef LOG_ENABLED
-        if (    pVCpu->vmm.s.pR0LoggerR3
-            &&  pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
-            RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
-#endif
-        if (rc != VINF_VMM_CALL_HOST)
-            break;
-        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
-            break;
-        /* Resume R0 */
-    }
-
-    AssertLogRelMsgReturn(rc == VINF_SUCCESS || RT_FAILURE(rc),
-                          ("uOperation=%u rc=%Rrc\n", uOperation, rc),
-                          VERR_INTERNAL_ERROR);
-    return rc;
-}
-
-
-/**
- * Resumes executing hypervisor code when interrupted by a queue flush or a
- * debug event.
- *
- * @returns VBox status code.
- * @param   pVM         VM handle.
- * @param   pVCpu       VMCPU handle.
- */
-VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
-{
-    Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
-    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
-
-    /*
-     * We hide log flushes (outer) and hypervisor interrupts (inner).
-     */
-    for (;;)
-    {
-        int rc;
-        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
-        do
-        {
-#ifdef NO_SUPCALLR0VMM
-            rc = VERR_GENERAL_FAILURE;
-#else
-            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
-            if (RT_LIKELY(rc == VINF_SUCCESS))
-                rc = pVCpu->vmm.s.iLastGZRc;
-#endif
-        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
-
-        /*
-         * Flush the loggers,
-         */
-#ifdef LOG_ENABLED
-        PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
-        if (    pLogger
-            &&  pLogger->offScratch > 0)
-            RTLogFlushRC(NULL, pLogger);
-#endif
-#ifdef VBOX_WITH_RC_RELEASE_LOGGING
-        PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
-        if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
-            RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
-#endif
-        if (rc == VERR_TRPM_PANIC || rc == VERR_TRPM_DONT_PANIC)
-            VMMR3FatalDump(pVM, pVCpu, rc);
-        if (rc != VINF_VMM_CALL_HOST)
-        {
-            Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
-            return rc;
-        }
-        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
-        if (RT_FAILURE(rc))
-            return rc;
-    }
-}
-
-
-/**
- * Service a call to the ring-3 host code.
- *
- * @returns VBox status code.
- * @param   pVM     VM handle.
- * @param   pVCpu   VMCPU handle
- * @remark  Careful with critsects.
- */
-static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
-{
-    /*
-     * We must also check for pending critsect exits or else we can deadlock
-     * when entering other critsects here.
-     */
-    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
-        PDMCritSectFF(pVCpu);
-
-    switch (pVCpu->vmm.s.enmCallRing3Operation)
-    {
-        /*
-         * Acquire the PDM lock.
-         */
-        case VMMCALLRING3_PDM_LOCK:
-        {
-            pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
-            break;
-        }
-
-        /*
-         * Grow the PGM pool.
-         */
-        case VMMCALLRING3_PGM_POOL_GROW:
-        {
-            pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
-            break;
-        }
-
-        /*
-         * Maps an page allocation chunk into ring-3 so ring-0 can use it.
-         */
-        case VMMCALLRING3_PGM_MAP_CHUNK:
-        {
-            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
-            break;
-        }
-
-        /*
-         * Allocates more handy pages.
-         */
-        case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
-        {
-            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
-            break;
-        }
-
-        /*
-         * Allocates a large page.
-         */
-        case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
-        {
-            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
-            break;
-        }
-
-        /*
-         * Acquire the PGM lock.
-         */
-        case VMMCALLRING3_PGM_LOCK:
-        {
-            pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
-            break;
-        }
-
-        /*
-         * Acquire the MM hypervisor heap lock.
-         */
-        case VMMCALLRING3_MMHYPER_LOCK:
-        {
-            pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
-            break;
-        }
-
-        /*
-         * Flush REM handler notifications.
-         */
-        case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
-        {
-            REMR3ReplayHandlerNotifications(pVM);
-            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
-            break;
-        }
-
-        /*
-         * This is a noop. We just take this route to avoid unnecessary
-         * tests in the loops.
-         */
-        case VMMCALLRING3_VMM_LOGGER_FLUSH:
-            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
-            LogAlways(("*FLUSH*\n"));
-            break;
-
-        /*
-         * Set the VM error message.
-         */
-        case VMMCALLRING3_VM_SET_ERROR:
-            VMR3SetErrorWorker(pVM);
-            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
-            break;
-
-        /*
-         * Set the VM runtime error message.
-         */
-        case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
-            pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
-            break;
-
-        /*
-         * Signal a ring 0 hypervisor assertion.
-         * Cancel the longjmp operation that's in progress.
-         */
-        case VMMCALLRING3_VM_R0_ASSERTION:
-            pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
-            pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
-#ifdef RT_ARCH_X86
-            pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
-#else
-            pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
-#endif
-#ifdef VMM_R0_SWITCH_STACK
-            *(uint64_t *)pVCpu->vmm.s.pbEMTStackR3 = 0; /* clear marker  */
-#endif
-            LogRel((pVM->vmm.s.szRing0AssertMsg1));
-            LogRel((pVM->vmm.s.szRing0AssertMsg2));
-            return VERR_VMM_RING0_ASSERTION;
-
-        /*
-         * A forced switch to ring 0 for preemption purposes.
-         */
-        case VMMCALLRING3_VM_R0_PREEMPT:
-            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
-            break;
-
-        case VMMCALLRING3_FTM_SET_CHECKPOINT:
-            pVCpu->vmm.s.rcCallRing3 = FTMR3SetCheckpoint(pVM, (FTMCHECKPOINTTYPE)pVCpu->vmm.s.u64CallRing3Arg);
-            break;
-
-        default:
-            AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
-            return VERR_INTERNAL_ERROR;
-    }
-
-    pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Displays the Force action Flags.
- *
- * @param   pVM         The VM handle.
- * @param   pHlp        The output helpers.
- * @param   pszArgs     The additional arguments (ignored).
- */
-static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
-{
-    int         c;
-    uint32_t    f;
-#define PRINT_FLAG(prf,flag) do { \
-        if (f & (prf##flag)) \
-        { \
-            static const char *s_psz = #flag; \
-            if (!(c % 6)) \
-                pHlp->pfnPrintf(pHlp, "%s\n    %s", c ? "," : "", s_psz); \
-            else \
-                pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
-            c++; \
-            f &= ~(prf##flag); \
-        } \
-    } while (0)
-
-#define PRINT_GROUP(prf,grp,sfx) do { \
-        if (f & (prf##grp##sfx)) \
-        { \
-            static const char *s_psz = #grp; \
-            if (!(c % 5)) \
-                pHlp->pfnPrintf(pHlp, "%s    %s", c ? ",\n" : "  Groups:\n", s_psz); \
-            else \
-                pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
-            c++; \
-        } \
-    } while (0)
-
-    /*
-     * The global flags.
-     */
-    const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
-    pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
-
-    /* show the flag mnemonics  */
-    c = 0;
-    f = fGlobalForcedActions;
-    PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
-    PRINT_FLAG(VM_FF_,PDM_QUEUES);
-    PRINT_FLAG(VM_FF_,PDM_DMA);
-    PRINT_FLAG(VM_FF_,DBGF);
-    PRINT_FLAG(VM_FF_,REQUEST);
-    PRINT_FLAG(VM_FF_,CHECK_VM_STATE);
-    PRINT_FLAG(VM_FF_,RESET);
-    PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
-    PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
-    PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
-    PRINT_FLAG(VM_FF_,PGM_POOL_FLUSH_PENDING);
-    PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
-    PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
-    if (f)
-        pHlp->pfnPrintf(pHlp, "%s\n    Unknown bits: %#RX32\n", c ? "," : "", f);
-    else
-        pHlp->pfnPrintf(pHlp, "\n");
-
-    /* the groups */
-    c = 0;
-    f = fGlobalForcedActions;
-    PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
-    PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
-    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
-    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
-    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
-    PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
-    PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
-    PRINT_GROUP(VM_FF_,ALL_BUT_RAW,_MASK);
-    if (c)
-        pHlp->pfnPrintf(pHlp, "\n");
-
-    /*
-     * Per CPU flags.
-     */
-    for (VMCPUID i = 0; i < pVM->cCpus; i++)
-    {
-        const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
-        pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
-
-        /* show the flag mnemonics */
-        c = 0;
-        f = fLocalForcedActions;
-        PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
-        PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
-        PRINT_FLAG(VMCPU_FF_,TIMER);
-        PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
-        PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
-        PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
-        PRINT_FLAG(VMCPU_FF_,TLB_FLUSH);
-        PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
-        PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
-        PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
-        PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
-        PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
-        PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
-        PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
-        PRINT_FLAG(VMCPU_FF_,TO_R3);
-        if (f)
-            pHlp->pfnPrintf(pHlp, "%s\n    Unknown bits: %#RX32\n", c ? "," : "", f);
-        else
-            pHlp->pfnPrintf(pHlp, "\n");
-
-        /* the groups */
-        c = 0;
-        f = fLocalForcedActions;
-        PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
-        PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
-        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
-        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
-        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
-        PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
-        PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
-        PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
-        PRINT_GROUP(VMCPU_FF_,HWACCM_TO_R3,_MASK);
-        PRINT_GROUP(VMCPU_FF_,ALL_BUT_RAW,_MASK);
-        if (c)
-            pHlp->pfnPrintf(pHlp, "\n");
-    }
-
-#undef PRINT_FLAG
-#undef PRINT_GROUP
-}
-
Index: /trunk/src/VBox/VMM/VMMAll/EMAll.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/EMAll.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/EMAll.cpp	(revision 35333)
@@ -39,5 +39,5 @@
 #include <VBox/disopcode.h>
 #include <VBox/log.h>
-#include <include/internal/pgm.h>
+#include "internal/pgm.h"
 #include <iprt/assert.h>
 #include <iprt/asm.h>
Index: /trunk/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp	(revision 35333)
@@ -21,5 +21,5 @@
 *******************************************************************************/
 #define LOG_GROUP LOG_GROUP_PDM//_CRITSECT
-#include "../PDMInternal.h"
+#include "PDMInternal.h"
 #include <VBox/pdmcritsect.h>
 #include <VBox/mm.h>
Index: /trunk/src/VBox/VMM/VMMAll/PGMAll.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/PGMAll.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/PGMAll.cpp	(revision 35333)
@@ -34,7 +34,7 @@
 #include <VBox/hwaccm.h>
 #include <VBox/hwacc_vmx.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 #include <iprt/assert.h>
 #include <iprt/asm-amd64-x86.h>
Index: /trunk/src/VBox/VMM/VMMAll/PGMAllHandler.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/PGMAllHandler.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/PGMAllHandler.cpp	(revision 35333)
@@ -30,7 +30,7 @@
 #include <VBox/dbgf.h>
 #include <VBox/rem.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 
 #include <VBox/log.h>
Index: /trunk/src/VBox/VMM/VMMAll/PGMAllMap.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/PGMAllMap.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/PGMAllMap.cpp	(revision 35333)
@@ -21,7 +21,7 @@
 #define LOG_GROUP LOG_GROUP_PGM
 #include <VBox/pgm.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 #include <VBox/err.h>
 #include <iprt/asm-amd64-x86.h>
Index: /trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp	(revision 35333)
@@ -26,7 +26,7 @@
 #include <VBox/em.h>
 #include <VBox/rem.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 #include <VBox/param.h>
 #include <VBox/err.h>
Index: /trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp	(revision 35333)
@@ -28,7 +28,7 @@
 # include <VBox/patm.h>
 #endif
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 #include <VBox/disopcode.h>
 #include <VBox/hwacc_vmx.h>
Index: /trunk/src/VBox/VMM/VMMAll/TMAllCpu.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/TMAllCpu.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/TMAllCpu.cpp	(revision 35333)
@@ -23,5 +23,5 @@
 #include <VBox/tm.h>
 #include <iprt/asm-amd64-x86.h> /* for SUPGetCpuHzFromGIP */
-#include "../TMInternal.h"
+#include "TMInternal.h"
 #include <VBox/vm.h>
 #include <VBox/sup.h>
Index: /trunk/src/VBox/VMM/VMMAll/TRPMAll.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/TRPMAll.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMAll/TRPMAll.cpp	(revision 35333)
@@ -37,5 +37,5 @@
 #include <iprt/asm-amd64-x86.h>
 #include <iprt/param.h>
-#include <include/internal/pgm.h>
+#include "internal/pgm.h"
 
 
Index: unk/src/VBox/VMM/VMMCodingGuidelines.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMCodingGuidelines.cpp	(revision 35332)
+++ 	(revision )
@@ -1,93 +1,0 @@
-/* $Id$ */
-/** @file
- * VMM - Coding Guidelines.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/** @page pg_vmm_guideline                  VMM Coding Guidelines
- *
- * The guidelines extends the VBox coding guidelines (@ref pg_vbox_guideline)
- * and consists of a compulsory part and an optional part. It is very important
- * that the rules of the compulsory part is followed. That will prevent obvious
- * bugs, and it will ease porting the code to 32/64 and 64/32 bits setups.
- *
- *
- *
- * @section sec_vmm_guideline_compulsory        Compulsory
- *
- * It is of vital importance is to distinguish between addresses - both virtual
- * and physical - applying to Guest Context and Host Context. To assist the
- * coder in this, a set of types and macros have been created. Another vital
- * thing is that structures shared between the two contexts ends up with the
- * same size and member offsets in both places. There are types and macros
- * for that too.
- *
- *
- * The rules:
- *
- *      - When declaring pointers in shared structures use the RCPTRTYPE(),
- *        R0PTRTYPE() and R3PTRTYPE() macros.
- *
- *      - Use RTGCPTR and RTHCPTR when dealing with the other context in
- *        none shared structures, parameter lists, stack variables and such.
- *
- *      - Following the above rules, pointers will in a context other than the
- *        one a pointer was defined for, appear as unsigned integers.
- *
- *      - It is NOT permitted to subject a pointer from the other context to pointer
- *        types of the current context by direct cast or by definition.
- *
- *      - When doing pointer arithmetic cast using uintptr_t, intptr_t or char *.
- *        Never cast a pointer to anything else for this purpose, that will not
- *        work everywhere! (1)
- *
- *      - Physical addresses are also specific to their context. Use RTGCPHYS
- *        and RTHCPHYS when dealing when them. Both types are unsigned integers.
- *
- *      - Integers in shared structures should be using a RT integer type or
- *        any of the [u]int[0-9]+_t types. (2)
- *
- *      - If code is shared between the contexts, GCTYPE() can be used to declare
- *        things differently. If GCTYPE() usage is extensive, don't share the code.
- *
- *      - The context is part of all public symbols which are specific to a single
- *        context.
- *
- *
- * (1) Talking about porting between 32-bit and 64-bit architectures and even
- *     between 64-bit platforms. On 64-bit linux int is 32-bit, long is 64-bit.
- *     However on 64-bit windows both int and long are 32-bit - there is no
- *     standard 64 bit type (_int64 is not a standard type, it's an stupid
- *     extension).
- *
- * (2) The VBox integer types are RTINT, RTUINT, RTGCINT, RTGCUINT,
- *     RTGCINTPTR, RTGCUINTPTR, RTHCINT, RTHCUINT, RTHCINTPTR and
- *     RTHCUINTPTR.
- *
- *
- *
- * @section sec_vmm_guideline_optional          Optional
- *
- * There are the general VBox guidelines, see @ref sec_vbox_guideline_optional.
- * In addition to these for the following rules applies to the VMM:
- *
- *      - Prefixes GCPtr and HCPtr are preferred over suffixes HC and GC of
- *        pointers.
- *
- *      - Prefixes GCPhys and HCPhys are generally used for physical addresses,
- *        types RTGCPHYS and RTHCPHYS respectively.
- *
- */
-
Index: unk/src/VBox/VMM/VMMDocsRawMode.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMDocsRawMode.cpp	(revision 35332)
+++ 	(revision )
@@ -1,34 +1,0 @@
-/* $Id$ */
-/** @file
- * This file contains the documentation of the raw-mode execution.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-
-/** @page pg_raw                Raw-mode Code Execution
- *
- * This chapter describes the virtualization technique which we call raw-mode
- * and how VirtualBox makes use of it and implements it.
- *
- * @todo Write raw-mode chapter!
- *
- *
- * @section sec_rawr3           Raw-Ring3
- *
- *
- * @section sec_rawr0           Raw-Ring0
- *
- *
- */
Index: /trunk/src/VBox/VMM/VMMGC/HWACCMGCA.asm
===================================================================
--- /trunk/src/VBox/VMM/VMMGC/HWACCMGCA.asm	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMGC/HWACCMGCA.asm	(revision 35333)
@@ -26,5 +26,5 @@
 %include "VBox/cpum.mac"
 %include "VBox/x86.mac"
-%include "../HWACCMInternal.mac"
+%include "HWACCMInternal.mac"
 
 %ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely.
@@ -110,11 +110,11 @@
     mov     rax, VERR_VMX_INVALID_VMXON_PTR
     jmp     .vmstart64_vmxon_failed
-    
+
 .vmxon_success:
     jnz     .vmxon_success2
     mov     rax, VERR_VMX_GENERIC
     jmp     .vmstart64_vmxon_failed
-    
-.vmxon_success2:    
+
+.vmxon_success2:
     ; Activate the VMCS pointer
     vmptrld [rbp + 16 + 8]
@@ -122,5 +122,5 @@
     mov     rax, VERR_VMX_INVALID_VMCS_PTR
     jmp     .vmstart64_vmxoff_end
-    
+
 .vmptrld_success:
     jnz     .vmptrld_success2
@@ -159,6 +159,6 @@
     mov     rcx, 0
     jmp     .cached_write
-    
-ALIGN(16)    
+
+ALIGN(16)
 .cached_write:
     mov     eax, [rbx + VMCSCACHE.Write.aField + rcx*4]
@@ -177,5 +177,5 @@
     push    xBX
 %endif
- 
+
     ; Save the host state that's relevant in the temporary 64 bits mode
     mov     rdx, cr0
@@ -208,9 +208,9 @@
     mov     qword [rbx + VMCSCACHE.uPos], 4
 %endif
-    
+
     ; hopefully we can ignore TR (we restore it anyway on the way back to 32 bits mode)
-    
+
     ;/* First we have to save some final CPU context registers. */
-    lea     rdx, [.vmlaunch64_done wrt rip]    
+    lea     rdx, [.vmlaunch64_done wrt rip]
     mov     rax, VMX_VMCS_HOST_RIP  ;/* return address (too difficult to continue after VMLAUNCH?) */
     vmwrite rax, rdx
@@ -326,5 +326,5 @@
     mov     [rdi + VMCSCACHE.TestOut.cr8], rax
  %endif
-    
+
     mov     ecx, [rdi + VMCSCACHE.Read.cValidEntries]
     cmp     ecx, 0  ; can't happen
@@ -550,5 +550,5 @@
 
     mov     cr0, rcx                    ; and restore old CR0 again
-    
+
     mov     eax, VINF_SUCCESS
     ret
Index: /trunk/src/VBox/VMM/VMMGC/PGMGC.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMGC/PGMGC.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMGC/PGMGC.cpp	(revision 35333)
@@ -29,7 +29,7 @@
 #include <VBox/trpm.h>
 #include <VBox/rem.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 
 #include <iprt/asm.h>
Index: unk/src/VBox/VMM/VMMGuruMeditation.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMGuruMeditation.cpp	(revision 35332)
+++ 	(revision )
@@ -1,664 +1,0 @@
-/* $Id$ */
-/** @file
- * VMM - The Virtual Machine Monitor, Guru Meditation Code.
- */
-
-/*
- * Copyright (C) 2006-2010 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_VMM
-#include <VBox/vmm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/pdmcritsect.h>
-#include <VBox/trpm.h>
-#include <VBox/dbgf.h>
-#include "VMMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/mm.h>
-#include <VBox/iom.h>
-#include <VBox/em.h>
-
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/version.h>
-#include <VBox/hwaccm.h>
-#include <iprt/assert.h>
-#include <iprt/time.h>
-#include <iprt/stream.h>
-#include <iprt/string.h>
-#include <iprt/stdarg.h>
-
-
-/*******************************************************************************
-*   Structures and Typedefs                                                    *
-*******************************************************************************/
-/**
- * Structure to pass to DBGFR3Info() and for doing all other
- * output during fatal dump.
- */
-typedef struct VMMR3FATALDUMPINFOHLP
-{
-    /** The helper core. */
-    DBGFINFOHLP Core;
-    /** The release logger instance. */
-    PRTLOGGER   pRelLogger;
-    /** The saved release logger flags. */
-    uint32_t    fRelLoggerFlags;
-    /** The logger instance. */
-    PRTLOGGER   pLogger;
-    /** The saved logger flags. */
-    uint32_t    fLoggerFlags;
-    /** The saved logger destination flags. */
-    uint32_t    fLoggerDestFlags;
-    /** Whether to output to stderr or not. */
-    bool        fStdErr;
-    /** Whether we're still recording the summary or not. */
-    bool        fRecSummary;
-    /** Buffer for the summary. */
-    char        szSummary[4096-2];
-    /** The current summary offset. */
-    size_t      offSummary;
-} VMMR3FATALDUMPINFOHLP, *PVMMR3FATALDUMPINFOHLP;
-/** Pointer to a VMMR3FATALDUMPINFOHLP structure. */
-typedef const VMMR3FATALDUMPINFOHLP *PCVMMR3FATALDUMPINFOHLP;
-
-
-/**
- * Print formatted string.
- *
- * @param   pHlp        Pointer to this structure.
- * @param   pszFormat   The format string.
- * @param   ...         Arguments.
- */
-static DECLCALLBACK(void) vmmR3FatalDumpInfoHlp_pfnPrintf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
-{
-    va_list args;
-    va_start(args, pszFormat);
-    pHlp->pfnPrintfV(pHlp, pszFormat, args);
-    va_end(args);
-}
-
-
-/**
- * Print formatted string.
- *
- * @param   pHlp        Pointer to this structure.
- * @param   pszFormat   The format string.
- * @param   args        Argument list.
- */
-static DECLCALLBACK(void) vmmR3FatalDumpInfoHlp_pfnPrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
-{
-    PVMMR3FATALDUMPINFOHLP pMyHlp = (PVMMR3FATALDUMPINFOHLP)pHlp;
-
-    if (pMyHlp->pRelLogger)
-    {
-        va_list args2;
-        va_copy(args2, args);
-        RTLogLoggerV(pMyHlp->pRelLogger, pszFormat, args2);
-        va_end(args2);
-    }
-    if (pMyHlp->pLogger)
-    {
-        va_list args2;
-        va_copy(args2, args);
-        RTLogLoggerV(pMyHlp->pLogger, pszFormat, args);
-        va_end(args2);
-    }
-    if (pMyHlp->fStdErr)
-    {
-        va_list args2;
-        va_copy(args2, args);
-        RTStrmPrintfV(g_pStdErr, pszFormat, args);
-        va_end(args2);
-    }
-    if (pMyHlp->fRecSummary)
-    {
-        size_t cchLeft = sizeof(pMyHlp->szSummary) - pMyHlp->offSummary;
-        if (cchLeft > 1)
-        {
-            va_list args2;
-            va_copy(args2, args);
-            size_t cch = RTStrPrintfV(&pMyHlp->szSummary[pMyHlp->offSummary], cchLeft, pszFormat, args);
-            va_end(args2);
-            Assert(cch <= cchLeft);
-            pMyHlp->offSummary += cch;
-        }
-    }
-}
-
-
-/**
- * Initializes the fatal dump output helper.
- *
- * @param   pHlp        The structure to initialize.
- */
-static void vmmR3FatalDumpInfoHlpInit(PVMMR3FATALDUMPINFOHLP pHlp)
-{
-    memset(pHlp, 0, sizeof(*pHlp));
-
-    pHlp->Core.pfnPrintf = vmmR3FatalDumpInfoHlp_pfnPrintf;
-    pHlp->Core.pfnPrintfV = vmmR3FatalDumpInfoHlp_pfnPrintfV;
-
-    /*
-     * The loggers.
-     */
-    pHlp->pRelLogger = RTLogRelDefaultInstance();
-#ifndef LOG_ENABLED
-    if (!pHlp->pRelLogger)
-#endif
-        pHlp->pLogger = RTLogDefaultInstance();
-
-    if (pHlp->pRelLogger)
-    {
-        pHlp->fRelLoggerFlags = pHlp->pRelLogger->fFlags;
-        pHlp->pRelLogger->fFlags &= ~(RTLOGFLAGS_BUFFERED | RTLOGFLAGS_DISABLED);
-    }
-
-    if (pHlp->pLogger)
-    {
-        pHlp->fLoggerFlags     = pHlp->pLogger->fFlags;
-        pHlp->fLoggerDestFlags = pHlp->pLogger->fDestFlags;
-        pHlp->pLogger->fFlags     &= ~(RTLOGFLAGS_BUFFERED | RTLOGFLAGS_DISABLED);
-#ifndef DEBUG_sandervl
-        pHlp->pLogger->fDestFlags |= RTLOGDEST_DEBUGGER;
-#endif
-    }
-
-    /*
-     * Check if we need write to stderr.
-     */
-    pHlp->fStdErr = (!pHlp->pRelLogger || !(pHlp->pRelLogger->fDestFlags & (RTLOGDEST_STDOUT | RTLOGDEST_STDERR)))
-                 && (!pHlp->pLogger || !(pHlp->pLogger->fDestFlags & (RTLOGDEST_STDOUT | RTLOGDEST_STDERR)));
-#ifdef DEBUG_sandervl
-    pHlp->fStdErr = false; /* takes too long to display here */
-#endif
-
-    /*
-     * Init the summary recording.
-     */
-    pHlp->fRecSummary  = true;
-    pHlp->offSummary   = 0;
-    pHlp->szSummary[0] = '\0';
-}
-
-
-/**
- * Deletes the fatal dump output helper.
- *
- * @param   pHlp        The structure to delete.
- */
-static void vmmR3FatalDumpInfoHlpDelete(PVMMR3FATALDUMPINFOHLP pHlp)
-{
-    if (pHlp->pRelLogger)
-    {
-        RTLogFlush(pHlp->pRelLogger);
-        pHlp->pRelLogger->fFlags = pHlp->fRelLoggerFlags;
-    }
-
-    if (pHlp->pLogger)
-    {
-        RTLogFlush(pHlp->pLogger);
-        pHlp->pLogger->fFlags     = pHlp->fLoggerFlags;
-        pHlp->pLogger->fDestFlags = pHlp->fLoggerDestFlags;
-    }
-}
-
-
-/**
- * Dumps the VM state on a fatal error.
- *
- * @param   pVM         VM Handle.
- * @param   pVCpu       VMCPU Handle.
- * @param   rcErr       VBox status code.
- */
-VMMR3DECL(void) VMMR3FatalDump(PVM pVM, PVMCPU pVCpu, int rcErr)
-{
-    /*
-     * Create our output helper and sync it with the log settings.
-     * This helper will be used for all the output.
-     */
-    VMMR3FATALDUMPINFOHLP   Hlp;
-    PCDBGFINFOHLP           pHlp = &Hlp.Core;
-    vmmR3FatalDumpInfoHlpInit(&Hlp);
-
-    /* Release owned locks to make sure other VCPUs can continue in case they were waiting for one. */
-    PDMR3CritSectLeaveAll(pVM);
-
-    /*
-     * Header.
-     */
-    pHlp->pfnPrintf(pHlp,
-                    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
-                    "!!\n"
-                    "!!                 Guru Meditation %d (%Rrc)\n"
-                    "!!\n",
-                    rcErr, rcErr);
-
-    /*
-     * Continue according to context.
-     */
-    bool fDoneHyper = false;
-    switch (rcErr)
-    {
-        /*
-         * Hypervisor errors.
-         */
-        case VERR_VMM_RING0_ASSERTION:
-        case VINF_EM_DBG_HYPER_ASSERTION:
-        case VERR_VMM_RING3_CALL_DISABLED:
-        {
-            const char *pszMsg1 = VMMR3GetRZAssertMsg1(pVM);
-            while (pszMsg1 && *pszMsg1 == '\n')
-                pszMsg1++;
-            const char *pszMsg2 = VMMR3GetRZAssertMsg2(pVM);
-            while (pszMsg2 && *pszMsg2 == '\n')
-                pszMsg2++;
-            pHlp->pfnPrintf(pHlp,
-                            "%s"
-                            "%s",
-                            pszMsg1,
-                            pszMsg2);
-            if (    !pszMsg2
-                ||  !*pszMsg2
-                ||  strchr(pszMsg2, '\0')[-1] != '\n')
-                pHlp->pfnPrintf(pHlp, "\n");
-            /* fall thru */
-        }
-        case VERR_TRPM_DONT_PANIC:
-        case VERR_TRPM_PANIC:
-        case VINF_EM_RAW_STALE_SELECTOR:
-        case VINF_EM_RAW_IRET_TRAP:
-        case VINF_EM_DBG_HYPER_BREAKPOINT:
-        case VINF_EM_DBG_HYPER_STEPPED:
-        case VERR_VMM_HYPER_CR3_MISMATCH:
-        {
-            /*
-             * Active trap? This is only of partial interest when in hardware
-             * assisted virtualization mode, thus the different messages.
-             */
-            uint32_t        uEIP       = CPUMGetHyperEIP(pVCpu);
-            TRPMEVENT       enmType;
-            uint8_t         u8TrapNo   =       0xce;
-            RTGCUINT        uErrorCode = 0xdeadface;
-            RTGCUINTPTR     uCR2       = 0xdeadface;
-            int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
-            if (!HWACCMIsEnabled(pVM))
-            {
-                if (RT_SUCCESS(rc2))
-                    pHlp->pfnPrintf(pHlp,
-                                    "!! TRAP=%02x ERRCD=%RGv CR2=%RGv EIP=%RX32 Type=%d\n",
-                                    u8TrapNo, uErrorCode, uCR2, uEIP, enmType);
-                else
-                    pHlp->pfnPrintf(pHlp,
-                                    "!! EIP=%RX32 NOTRAP\n",
-                                    uEIP);
-            }
-            else if (RT_SUCCESS(rc2))
-                pHlp->pfnPrintf(pHlp,
-                                "!! ACTIVE TRAP=%02x ERRCD=%RGv CR2=%RGv PC=%RGr Type=%d (Guest!)\n",
-                                u8TrapNo, uErrorCode, uCR2, CPUMGetGuestRIP(pVCpu), enmType);
-
-            /*
-             * Dump the relevant hypervisor registers and stack.
-             */
-            if (HWACCMIsEnabled(pVM))
-            {
-                if (   rcErr == VERR_VMM_RING0_ASSERTION /* fInRing3Call has already been cleared here. */
-                    || pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
-                {
-                    /* Dump the jmpbuf.  */
-                    pHlp->pfnPrintf(pHlp,
-                                    "!!\n"
-                                    "!! CallRing3JmpBuf:\n"
-                                    "!!\n");
-                    pHlp->pfnPrintf(pHlp,
-                                    "SavedEsp=%RHv SavedEbp=%RHv SpResume=%RHv SpCheck=%RHv\n",
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.SavedEsp,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.SpResume,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck);
-                    pHlp->pfnPrintf(pHlp,
-                                    "pvSavedStack=%RHv cbSavedStack=%#x  fInRing3Call=%RTbool\n",
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call);
-                    pHlp->pfnPrintf(pHlp,
-                                    "cbUsedMax=%#x cbUsedAvg=%#x cbUsedTotal=%#llx cUsedTotal=%#llx\n",
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedMax,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedAvg,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedTotal,
-                                    pVCpu->vmm.s.CallRing3JmpBufR0.cUsedTotal);
-
-                    /* Dump the resume register frame on the stack. */
-                    PRTHCUINTPTR pBP;
-#ifdef VMM_R0_SWITCH_STACK
-                    pBP = (PRTHCUINTPTR)&pVCpu->vmm.s.pbEMTStackR3[  pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp
-                                                                   - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3)];
-#else
-                    pBP = (PRTHCUINTPTR)&pVCpu->vmm.s.pbEMTStackR3[  pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack
-                                                                   - pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck
-                                                                   + pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp];
-#endif
-#if HC_ARCH_BITS == 32
-                    pHlp->pfnPrintf(pHlp,
-                                    "eax=volatile ebx=%08x ecx=volatile edx=volatile esi=%08x edi=%08x\n"
-                                    "eip=%08x esp=%08x ebp=%08x efl=%08x\n"
-                                    ,
-                                    pBP[-3], pBP[-2], pBP[-1],
-                                    pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 8, pBP[0], pBP[-4]);
-#else
-# ifdef RT_OS_WINDOWS
-                    pHlp->pfnPrintf(pHlp,
-                                    "rax=volatile         rbx=%016RX64 rcx=volatile         rdx=volatile\n"
-                                    "rsi=%016RX64 rdi=%016RX64  r8=volatile          r9=volatile        \n"
-                                    "r10=volatile         r11=volatile         r12=%016RX64 r13=%016RX64\n"
-                                    "r14=%016RX64 r15=%016RX64\n"
-                                    "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 rfl=%08RX64\n"
-                                    ,
-                                    pBP[-7],
-                                    pBP[-6], pBP[-5],
-                                    pBP[-4], pBP[-3],
-                                    pBP[-2], pBP[-1],
-                                    pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 16, pBP[0], pBP[-8]);
-# else
-                    pHlp->pfnPrintf(pHlp,
-                                    "rax=volatile         rbx=%016RX64 rcx=volatile         rdx=volatile\n"
-                                    "rsi=volatile         rdi=volatile          r8=volatile          r9=volatile        \n"
-                                    "r10=volatile         r11=volatile         r12=%016RX64 r13=%016RX64\n"
-                                    "r14=%016RX64 r15=%016RX64\n"
-                                    "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 rflags=%08RX64\n"
-                                    ,
-                                    pBP[-5],
-                                    pBP[-4], pBP[-3],
-                                    pBP[-2], pBP[-1],
-                                    pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 16, pBP[0], pBP[-6]);
-# endif
-#endif
-
-                    /* Callstack. */
-                    DBGFADDRESS pc;
-                    pc.fFlags    = DBGFADDRESS_FLAGS_RING0 | DBGFADDRESS_FLAGS_VALID;
-#if HC_ARCH_BITS == 64
-                    pc.FlatPtr   = pc.off = pVCpu->vmm.s.CallRing3JmpBufR0.rip;
-#else
-                    pc.FlatPtr   = pc.off = pVCpu->vmm.s.CallRing3JmpBufR0.eip;
-#endif
-                    pc.Sel       = DBGF_SEL_FLAT;
-
-                    DBGFADDRESS ebp;
-                    ebp.fFlags   = DBGFADDRESS_FLAGS_RING0 | DBGFADDRESS_FLAGS_VALID;
-                    ebp.FlatPtr  = ebp.off = pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp;
-                    ebp.Sel      = DBGF_SEL_FLAT;
-
-                    DBGFADDRESS esp;
-                    esp.fFlags   = DBGFADDRESS_FLAGS_RING0 | DBGFADDRESS_FLAGS_VALID;
-                    esp.Sel      = DBGF_SEL_FLAT;
-                    esp.FlatPtr  = esp.off = pVCpu->vmm.s.CallRing3JmpBufR0.SavedEsp;
-
-                    PCDBGFSTACKFRAME pFirstFrame;
-                    rc2 = DBGFR3StackWalkBeginEx(pVM, pVCpu->idCpu, DBGFCODETYPE_RING0, &ebp, &esp, &pc,
-                                                 DBGFRETURNTYPE_INVALID, &pFirstFrame);
-                    if (RT_SUCCESS(rc2))
-                    {
-                        pHlp->pfnPrintf(pHlp,
-                                        "!!\n"
-                                        "!! Call Stack:\n"
-                                        "!!\n");
-#if HC_ARCH_BITS == 32
-                        pHlp->pfnPrintf(pHlp, "EBP      Ret EBP  Ret CS:EIP    Arg0     Arg1     Arg2     Arg3     CS:EIP        Symbol [line]\n");
-#else
-                        pHlp->pfnPrintf(pHlp, "RBP              Ret RBP          Ret RIP          RIP              Symbol [line]\n");
-#endif
-                        for (PCDBGFSTACKFRAME pFrame = pFirstFrame;
-                             pFrame;
-                             pFrame = DBGFR3StackWalkNext(pFrame))
-                        {
-#if HC_ARCH_BITS == 32
-                            pHlp->pfnPrintf(pHlp,
-                                            "%RHv %RHv %04RX32:%RHv %RHv %RHv %RHv %RHv",
-                                            (RTHCUINTPTR)pFrame->AddrFrame.off,
-                                            (RTHCUINTPTR)pFrame->AddrReturnFrame.off,
-                                            (RTHCUINTPTR)pFrame->AddrReturnPC.Sel,
-                                            (RTHCUINTPTR)pFrame->AddrReturnPC.off,
-                                            pFrame->Args.au32[0],
-                                            pFrame->Args.au32[1],
-                                            pFrame->Args.au32[2],
-                                            pFrame->Args.au32[3]);
-                            pHlp->pfnPrintf(pHlp, " %RTsel:%08RHv", pFrame->AddrPC.Sel, pFrame->AddrPC.off);
-#else
-                            pHlp->pfnPrintf(pHlp,
-                                            "%RHv %RHv %RHv %RHv",
-                                            (RTHCUINTPTR)pFrame->AddrFrame.off,
-                                            (RTHCUINTPTR)pFrame->AddrReturnFrame.off,
-                                            (RTHCUINTPTR)pFrame->AddrReturnPC.off,
-                                            (RTHCUINTPTR)pFrame->AddrPC.off);
-#endif
-                            if (pFrame->pSymPC)
-                            {
-                                RTGCINTPTR offDisp = pFrame->AddrPC.FlatPtr - pFrame->pSymPC->Value;
-                                if (offDisp > 0)
-                                    pHlp->pfnPrintf(pHlp, " %s+%llx", pFrame->pSymPC->szName, (int64_t)offDisp);
-                                else if (offDisp < 0)
-                                    pHlp->pfnPrintf(pHlp, " %s-%llx", pFrame->pSymPC->szName, -(int64_t)offDisp);
-                                else
-                                    pHlp->pfnPrintf(pHlp, " %s", pFrame->pSymPC->szName);
-                            }
-                            if (pFrame->pLinePC)
-                                pHlp->pfnPrintf(pHlp, " [%s @ 0i%d]", pFrame->pLinePC->szFilename, pFrame->pLinePC->uLineNo);
-                            pHlp->pfnPrintf(pHlp, "\n");
-                        }
-                        DBGFR3StackWalkEnd(pFirstFrame);
-                    }
-
-                    /* raw stack */
-                    Hlp.fRecSummary = false;
-                    pHlp->pfnPrintf(pHlp,
-                                    "!!\n"
-                                    "!! Raw stack (mind the direction). \n"
-                                    "!! pbEMTStackR0=%RHv pbEMTStackBottomR0=%RHv VMM_STACK_SIZE=%#x\n"
-                                    "!! pbEmtStackR3=%p\n"
-                                    "!!\n"
-                                    "%.*Rhxd\n",
-                                    MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3),
-                                    MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3) + VMM_STACK_SIZE,
-                                    VMM_STACK_SIZE,
-                                    pVCpu->vmm.s.pbEMTStackR3,
-                                    VMM_STACK_SIZE, pVCpu->vmm.s.pbEMTStackR3);
-                }
-                else
-                {
-                    pHlp->pfnPrintf(pHlp,
-                                    "!! Skipping ring-0 registers and stack, rcErr=%Rrc\n", rcErr);
-                }
-            }
-            else
-            {
-                /*
-                 * Try figure out where eip is.
-                 */
-                /* core code? */
-                if (uEIP - (RTGCUINTPTR)pVM->vmm.s.pvCoreCodeRC < pVM->vmm.s.cbCoreCode)
-                    pHlp->pfnPrintf(pHlp,
-                                "!! EIP is in CoreCode, offset %#x\n",
-                                uEIP - (RTGCUINTPTR)pVM->vmm.s.pvCoreCodeRC);
-                else
-                {   /* ask PDM */  /** @todo ask DBGFR3Sym later? */
-                    char        szModName[64];
-                    RTRCPTR     RCPtrMod;
-                    char        szNearSym1[260];
-                    RTRCPTR     RCPtrNearSym1;
-                    char        szNearSym2[260];
-                    RTRCPTR     RCPtrNearSym2;
-                    int rc = PDMR3LdrQueryRCModFromPC(pVM, uEIP,
-                                                      &szModName[0],  sizeof(szModName),  &RCPtrMod,
-                                                      &szNearSym1[0], sizeof(szNearSym1), &RCPtrNearSym1,
-                                                      &szNearSym2[0], sizeof(szNearSym2), &RCPtrNearSym2);
-                    if (RT_SUCCESS(rc))
-                        pHlp->pfnPrintf(pHlp,
-                                        "!! EIP in %s (%RRv) at rva %x near symbols:\n"
-                                        "!!    %RRv rva %RRv off %08x  %s\n"
-                                        "!!    %RRv rva %RRv off -%08x %s\n",
-                                        szModName,  RCPtrMod, (unsigned)(uEIP - RCPtrMod),
-                                        RCPtrNearSym1, RCPtrNearSym1 - RCPtrMod, (unsigned)(uEIP - RCPtrNearSym1), szNearSym1,
-                                        RCPtrNearSym2, RCPtrNearSym2 - RCPtrMod, (unsigned)(RCPtrNearSym2 - uEIP), szNearSym2);
-                    else
-                        pHlp->pfnPrintf(pHlp,
-                                        "!! EIP is not in any code known to VMM!\n");
-                }
-
-                /* Disassemble the instruction. */
-                char szInstr[256];
-                rc2 = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0, DBGF_DISAS_FLAGS_CURRENT_HYPER | DBGF_DISAS_FLAGS_DEFAULT_MODE,
-                                         &szInstr[0], sizeof(szInstr), NULL);
-                if (RT_SUCCESS(rc2))
-                    pHlp->pfnPrintf(pHlp,
-                                    "!! %s\n", szInstr);
-
-                /* Dump the hypervisor cpu state. */
-                pHlp->pfnPrintf(pHlp,
-                                "!!\n"
-                                "!!\n"
-                                "!!\n");
-                rc2 = DBGFR3Info(pVM, "cpumhyper", "verbose", pHlp);
-                fDoneHyper = true;
-
-                /* Callstack. */
-                PCDBGFSTACKFRAME pFirstFrame;
-                rc2 = DBGFR3StackWalkBegin(pVM, pVCpu->idCpu, DBGFCODETYPE_HYPER, &pFirstFrame);
-                if (RT_SUCCESS(rc2))
-                {
-                    pHlp->pfnPrintf(pHlp,
-                                    "!!\n"
-                                    "!! Call Stack:\n"
-                                    "!!\n"
-                                    "EBP      Ret EBP  Ret CS:EIP    Arg0     Arg1     Arg2     Arg3     CS:EIP        Symbol [line]\n");
-                    for (PCDBGFSTACKFRAME pFrame = pFirstFrame;
-                         pFrame;
-                         pFrame = DBGFR3StackWalkNext(pFrame))
-                    {
-                        pHlp->pfnPrintf(pHlp,
-                                        "%08RX32 %08RX32 %04RX32:%08RX32 %08RX32 %08RX32 %08RX32 %08RX32",
-                                        (uint32_t)pFrame->AddrFrame.off,
-                                        (uint32_t)pFrame->AddrReturnFrame.off,
-                                        (uint32_t)pFrame->AddrReturnPC.Sel,
-                                        (uint32_t)pFrame->AddrReturnPC.off,
-                                        pFrame->Args.au32[0],
-                                        pFrame->Args.au32[1],
-                                        pFrame->Args.au32[2],
-                                        pFrame->Args.au32[3]);
-                        pHlp->pfnPrintf(pHlp, " %RTsel:%08RGv", pFrame->AddrPC.Sel, pFrame->AddrPC.off);
-                        if (pFrame->pSymPC)
-                        {
-                            RTGCINTPTR offDisp = pFrame->AddrPC.FlatPtr - pFrame->pSymPC->Value;
-                            if (offDisp > 0)
-                                pHlp->pfnPrintf(pHlp, " %s+%llx", pFrame->pSymPC->szName, (int64_t)offDisp);
-                            else if (offDisp < 0)
-                                pHlp->pfnPrintf(pHlp, " %s-%llx", pFrame->pSymPC->szName, -(int64_t)offDisp);
-                            else
-                                pHlp->pfnPrintf(pHlp, " %s", pFrame->pSymPC->szName);
-                        }
-                        if (pFrame->pLinePC)
-                            pHlp->pfnPrintf(pHlp, " [%s @ 0i%d]", pFrame->pLinePC->szFilename, pFrame->pLinePC->uLineNo);
-                        pHlp->pfnPrintf(pHlp, "\n");
-                    }
-                    DBGFR3StackWalkEnd(pFirstFrame);
-                }
-
-                /* raw stack */
-                Hlp.fRecSummary = false;
-                pHlp->pfnPrintf(pHlp,
-                                "!!\n"
-                                "!! Raw stack (mind the direction). pbEMTStackRC=%RRv pbEMTStackBottomRC=%RRv\n"
-                                "!!\n"
-                                "%.*Rhxd\n",
-                                pVCpu->vmm.s.pbEMTStackRC, pVCpu->vmm.s.pbEMTStackBottomRC,
-                                VMM_STACK_SIZE, pVCpu->vmm.s.pbEMTStackR3);
-            } /* !HWACCMIsEnabled */
-            break;
-        }
-
-        default:
-        {
-            break;
-        }
-
-    } /* switch (rcErr) */
-    Hlp.fRecSummary = false;
-
-
-    /*
-     * Generic info dumper loop.
-     */
-    static struct
-    {
-        const char *pszInfo;
-        const char *pszArgs;
-    } const     aInfo[] =
-    {
-        { "mappings",       NULL },
-        { "hma",            NULL },
-        { "cpumguest",      "verbose" },
-        { "cpumguestinstr", "verbose" },
-        { "cpumhyper",      "verbose" },
-        { "cpumhost",       "verbose" },
-        { "mode",           "all" },
-        { "cpuid",          "verbose" },
-        { "handlers",       "phys virt hyper stats" },
-        { "timers",         NULL },
-        { "activetimers",   NULL },
-    };
-    for (unsigned i = 0; i < RT_ELEMENTS(aInfo); i++)
-    {
-        if (fDoneHyper && !strcmp(aInfo[i].pszInfo, "cpumhyper"))
-            continue;
-        pHlp->pfnPrintf(pHlp,
-                        "!!\n"
-                        "!! {%s, %s}\n"
-                        "!!\n",
-                        aInfo[i].pszInfo, aInfo[i].pszArgs);
-        DBGFR3Info(pVM, aInfo[i].pszInfo, aInfo[i].pszArgs, pHlp);
-    }
-
-    /* All other info items */
-    DBGFR3InfoMulti(pVM,
-                    "*",
-                    "mappings|hma|cpum|cpumguest|cpumguestinstr|cpumhyper|cpumhost|mode|cpuid"
-                    "|pgmpd|pgmcr3|timers|activetimers|handlers|help",
-                    "!!\n"
-                    "!! {%s}\n"
-                    "!!\n",
-                    pHlp);
-
-
-    /* done */
-    pHlp->pfnPrintf(pHlp,
-                    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");
-
-
-    /*
-     * Repeat the summary to stderr so we don't have to scroll half a mile up.
-     */
-    if (Hlp.szSummary[0])
-        RTStrmPrintf(g_pStdErr,
-                     "%s"
-                     "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
-                     Hlp.szSummary);
-
-    /*
-     * Delete the output instance (flushing and restoring of flags).
-     */
-    vmmR3FatalDumpInfoHlpDelete(&Hlp);
-}
-
Index: unk/src/VBox/VMM/VMMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/VMMInternal.h	(revision 35332)
+++ 	(revision )
@@ -1,644 +1,0 @@
-/* $Id$ */
-/** @file
- * VMM - Internal header file.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___VMMInternal_h
-#define ___VMMInternal_h
-
-#include <VBox/cdefs.h>
-#include <VBox/stam.h>
-#include <VBox/log.h>
-#include <iprt/critsect.h>
-
-
-#if !defined(IN_VMM_R3) && !defined(IN_VMM_R0) && !defined(IN_VMM_RC)
-# error "Not in VMM! This is an internal header!"
-#endif
-
-
-/** @defgroup grp_vmm_int   Internals
- * @ingroup grp_vmm
- * @internal
- * @{
- */
-
-/** @def VBOX_WITH_RC_RELEASE_LOGGING
- * Enables RC release logging. */
-#define VBOX_WITH_RC_RELEASE_LOGGING
-
-/** @def VBOX_WITH_R0_LOGGING
- * Enables Ring-0 logging (non-release).
- *
- * Ring-0 logging isn't 100% safe yet (thread id reuse / process exit cleanup),
- * so you have to sign up here by adding your defined(DEBUG_<userid>) to the
- * #if, or by adding VBOX_WITH_R0_LOGGING to your LocalConfig.kmk.
- */
-#if defined(DEBUG_sandervl) || defined(DEBUG_frank) || defined(DOXYGEN_RUNNING)
-# define VBOX_WITH_R0_LOGGING
-#endif
-
-/** @def VBOX_STRICT_VMM_STACK
- * Enables VMM stack guard pages to catch stack over- and underruns. */
-#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
-# define VBOX_STRICT_VMM_STACK
-#endif
-
-
-/**
- * Converts a VMM pointer into a VM pointer.
- * @returns Pointer to the VM structure the VMM is part of.
- * @param   pVMM   Pointer to VMM instance data.
- */
-#define VMM2VM(pVMM)  ( (PVM)((char*)pVMM - pVMM->offVM) )
-
-
-/**
- * Switcher function, HC to RC.
- *
- * @param   pVM         The VM handle.
- * @returns Return code indicating the action to take.
- */
-typedef DECLASMTYPE(int) FNVMMSWITCHERHC(PVM pVM);
-/** Pointer to switcher function. */
-typedef FNVMMSWITCHERHC *PFNVMMSWITCHERHC;
-
-/**
- * Switcher function, RC to HC.
- *
- * @param   rc          VBox status code.
- */
-typedef DECLASMTYPE(void) FNVMMSWITCHERRC(int rc);
-/** Pointer to switcher function. */
-typedef FNVMMSWITCHERRC *PFNVMMSWITCHERRC;
-
-
-/**
- * The ring-0 logger instance wrapper.
- *
- * We need to be able to find the VM handle from the logger instance, so we wrap
- * it in this structure.
- */
-typedef struct VMMR0LOGGER
-{
-    /** Pointer to the VM handle. */
-    R0PTRTYPE(PVM)              pVM;
-    /** Size of the allocated logger instance (Logger). */
-    uint32_t                    cbLogger;
-    /** Flag indicating whether we've create the logger Ring-0 instance yet. */
-    bool                        fCreated;
-    /** Flag indicating whether we've disabled flushing (world switch) or not. */
-    bool                        fFlushingDisabled;
-    /** Flag indicating whether we've registered the instance already. */
-    bool                        fRegistered;
-    bool                        a8Alignment;
-    /** The CPU ID. */
-    VMCPUID                     idCpu;
-#if HC_ARCH_BITS == 64
-    uint32_t                    u32Alignment;
-#endif
-    /** The ring-0 logger instance. This extends beyond the size.  */
-    RTLOGGER                    Logger;
-} VMMR0LOGGER;
-/** Pointer to a ring-0 logger instance wrapper. */
-typedef VMMR0LOGGER *PVMMR0LOGGER;
-
-
-/**
- * Jump buffer for the setjmp/longjmp like constructs used to
- * quickly 'call' back into Ring-3.
- */
-typedef struct VMMR0JMPBUF
-{
-    /** Traditional jmp_buf stuff
-     * @{ */
-#if HC_ARCH_BITS == 32
-    uint32_t                    ebx;
-    uint32_t                    esi;
-    uint32_t                    edi;
-    uint32_t                    ebp;
-    uint32_t                    esp;
-    uint32_t                    eip;
-    uint32_t                    eflags;
-#endif
-#if HC_ARCH_BITS == 64
-    uint64_t                    rbx;
-# ifdef RT_OS_WINDOWS
-    uint64_t                    rsi;
-    uint64_t                    rdi;
-# endif
-    uint64_t                    rbp;
-    uint64_t                    r12;
-    uint64_t                    r13;
-    uint64_t                    r14;
-    uint64_t                    r15;
-    uint64_t                    rsp;
-    uint64_t                    rip;
-# ifdef RT_OS_WINDOWS
-    uint128_t                   xmm6;
-    uint128_t                   xmm7;
-    uint128_t                   xmm8;
-    uint128_t                   xmm9;
-    uint128_t                   xmm10;
-    uint128_t                   xmm11;
-    uint128_t                   xmm12;
-    uint128_t                   xmm13;
-    uint128_t                   xmm14;
-    uint128_t                   xmm15;
-# endif
-    uint64_t                    rflags;
-#endif
-    /** @} */
-
-    /** Flag that indicates that we've done a ring-3 call. */
-    bool                        fInRing3Call;
-    /** The number of bytes we've saved. */
-    uint32_t                    cbSavedStack;
-    /** Pointer to the buffer used to save the stack.
-     * This is assumed to be 8KB. */
-    RTR0PTR                     pvSavedStack;
-    /** Esp we we match against esp on resume to make sure the stack wasn't relocated. */
-    RTHCUINTREG                 SpCheck;
-    /** The esp we should resume execution with after the restore. */
-    RTHCUINTREG                 SpResume;
-    /** ESP/RSP at the time of the jump to ring 3. */
-    RTHCUINTREG                 SavedEsp;
-    /** EBP/RBP at the time of the jump to ring 3. */
-    RTHCUINTREG                 SavedEbp;
-
-    /** Stats: Max amount of stack used. */
-    uint32_t                    cbUsedMax;
-    /** Stats: Average stack usage. (Avg = cbUsedTotal / cUsedTotal) */
-    uint32_t                    cbUsedAvg;
-    /** Stats: Total amount of stack used. */
-    uint64_t                    cbUsedTotal;
-    /** Stats: Number of stack usages. */
-    uint64_t                    cUsedTotal;
-} VMMR0JMPBUF;
-/** Pointer to a ring-0 jump buffer. */
-typedef VMMR0JMPBUF *PVMMR0JMPBUF;
-
-
-/**
- * VMM Data (part of VM)
- */
-typedef struct VMM
-{
-    /** Offset to the VM structure.
-     * See VMM2VM(). */
-    RTINT                       offVM;
-
-    /** @name World Switcher and Related
-     * @{
-     */
-    /** Size of the core code. */
-    RTUINT                      cbCoreCode;
-    /** Physical address of core code. */
-    RTHCPHYS                    HCPhysCoreCode;
-    /** Pointer to core code ring-3 mapping - contiguous memory.
-     * At present this only means the context switcher code. */
-    RTR3PTR                     pvCoreCodeR3;
-    /** Pointer to core code ring-0 mapping - contiguous memory.
-     * At present this only means the context switcher code. */
-    RTR0PTR                     pvCoreCodeR0;
-    /** Pointer to core code guest context mapping. */
-    RTRCPTR                     pvCoreCodeRC;
-    RTRCPTR                     pRCPadding0; /**< Alignment padding */
-#ifdef VBOX_WITH_NMI
-    /** The guest context address of the APIC (host) mapping. */
-    RTRCPTR                     GCPtrApicBase;
-    RTRCPTR                     pRCPadding1; /**< Alignment padding */
-#endif
-    /** The current switcher.
-     * This will be set before the VMM is fully initialized. */
-    VMMSWITCHER                 enmSwitcher;
-    /** Flag to disable the switcher permanently (VMX) (boolean) */
-    bool                        fSwitcherDisabled;
-    /** Array of offsets to the different switchers within the core code. */
-    RTUINT                      aoffSwitchers[VMMSWITCHER_MAX];
-
-    /** Resume Guest Execution. See CPUMGCResumeGuest(). */
-    RTRCPTR                     pfnCPUMRCResumeGuest;
-    /** Resume Guest Execution in V86 mode. See CPUMGCResumeGuestV86(). */
-    RTRCPTR                     pfnCPUMRCResumeGuestV86;
-    /** Call Trampoline. See vmmGCCallTrampoline(). */
-    RTRCPTR                     pfnCallTrampolineRC;
-    /** Guest to host switcher entry point. */
-    RCPTRTYPE(PFNVMMSWITCHERRC) pfnGuestToHostRC;
-    /** Host to guest switcher entry point. */
-    R0PTRTYPE(PFNVMMSWITCHERHC) pfnHostToGuestR0;
-    /** @}  */
-
-    /** @name Logging
-     * @{
-     */
-    /** Size of the allocated logger instance (pRCLoggerRC/pRCLoggerR3). */
-    uint32_t                    cbRCLogger;
-    /** Pointer to the RC logger instance - RC Ptr.
-     * This is NULL if logging is disabled. */
-    RCPTRTYPE(PRTLOGGERRC)      pRCLoggerRC;
-    /** Pointer to the GC logger instance - R3 Ptr.
-     * This is NULL if logging is disabled. */
-    R3PTRTYPE(PRTLOGGERRC)      pRCLoggerR3;
-    /** Pointer to the GC release logger instance - R3 Ptr. */
-    R3PTRTYPE(PRTLOGGERRC)      pRCRelLoggerR3;
-    /** Pointer to the GC release logger instance - RC Ptr. */
-    RCPTRTYPE(PRTLOGGERRC)      pRCRelLoggerRC;
-    /** Size of the allocated release logger instance (pRCRelLoggerRC/pRCRelLoggerR3).
-     * This may differ from cbRCLogger. */
-    uint32_t                    cbRCRelLogger;
-    /** Whether log flushing has been disabled or not. */
-    bool                        fRCLoggerFlushingDisabled;
-    bool                        afAlignment[5]; /**< Alignment padding. */
-    /** @} */
-
-    /** Whether the stack guard pages have been stationed or not. */
-    bool                        fStackGuardsStationed;
-    /** Whether we should use the periodic preemption timers. */
-    bool                        fUsePeriodicPreemptionTimers;
-
-    /** The EMT yield timer. */
-    PTMTIMERR3                  pYieldTimer;
-    /** The period to the next timeout when suspended or stopped.
-     * This is 0 when running. */
-    uint32_t                    cYieldResumeMillies;
-    /** The EMT yield timer interval (milliseconds). */
-    uint32_t                    cYieldEveryMillies;
-    /** The timestamp of the previous yield. (nano) */
-    uint64_t                    u64LastYield;
-
-    /** Critical section.
-     * Use for synchronizing all VCPUs
-     */
-    RTCRITSECT                  CritSectSync;
-
-    /** @name EMT Rendezvous
-     * @{ */
-    /** Semaphore to wait on upon entering ordered execution. */
-    R3PTRTYPE(PRTSEMEVENT)      pahEvtRendezvousEnterOrdered;
-    /** Semaphore to wait on upon entering for one-by-one execution. */
-    RTSEMEVENT                  hEvtRendezvousEnterOneByOne;
-    /** Semaphore to wait on upon entering for all-at-once execution. */
-    RTSEMEVENTMULTI             hEvtMulRendezvousEnterAllAtOnce;
-    /** Semaphore to wait on when done. */
-    RTSEMEVENTMULTI             hEvtMulRendezvousDone;
-    /** Semaphore the VMMR3EmtRendezvous caller waits on at the end. */
-    RTSEMEVENT                  hEvtRendezvousDoneCaller;
-    /** Callback. */
-    R3PTRTYPE(PFNVMMEMTRENDEZVOUS) volatile pfnRendezvous;
-    /** The user argument for the callback. */
-    RTR3PTR volatile            pvRendezvousUser;
-    /** Flags. */
-    volatile uint32_t           fRendezvousFlags;
-    /** The number of EMTs that has entered. */
-    volatile uint32_t           cRendezvousEmtsEntered;
-    /** The number of EMTs that has done their job. */
-    volatile uint32_t           cRendezvousEmtsDone;
-    /** The number of EMTs that has returned. */
-    volatile uint32_t           cRendezvousEmtsReturned;
-    /** The status code. */
-    volatile int32_t            i32RendezvousStatus;
-    /** Spin lock. */
-    volatile uint32_t           u32RendezvousLock;
-    /** @} */
-
-#if HC_ARCH_BITS == 32
-    uint32_t                    u32Alignment; /**< Alignment padding. */
-#endif
-
-    /** Buffer for storing the standard assertion message for a ring-0 assertion.
-     * Used for saving the assertion message text for the release log and guru
-     * meditation dump. */
-    char                        szRing0AssertMsg1[512];
-    /** Buffer for storing the custom message for a ring-0 assertion. */
-    char                        szRing0AssertMsg2[256];
-
-    /** Number of VMMR0_DO_RUN_GC calls. */
-    STAMCOUNTER                 StatRunRC;
-
-    /** Statistics for each of the RC/R0 return codes.
-     * @{ */
-    STAMCOUNTER                 StatRZRetNormal;
-    STAMCOUNTER                 StatRZRetInterrupt;
-    STAMCOUNTER                 StatRZRetInterruptHyper;
-    STAMCOUNTER                 StatRZRetGuestTrap;
-    STAMCOUNTER                 StatRZRetRingSwitch;
-    STAMCOUNTER                 StatRZRetRingSwitchInt;
-    STAMCOUNTER                 StatRZRetStaleSelector;
-    STAMCOUNTER                 StatRZRetIRETTrap;
-    STAMCOUNTER                 StatRZRetEmulate;
-    STAMCOUNTER                 StatRZRetIOBlockEmulate;
-    STAMCOUNTER                 StatRZRetPatchEmulate;
-    STAMCOUNTER                 StatRZRetIORead;
-    STAMCOUNTER                 StatRZRetIOWrite;
-    STAMCOUNTER                 StatRZRetMMIORead;
-    STAMCOUNTER                 StatRZRetMMIOWrite;
-    STAMCOUNTER                 StatRZRetMMIOPatchRead;
-    STAMCOUNTER                 StatRZRetMMIOPatchWrite;
-    STAMCOUNTER                 StatRZRetMMIOReadWrite;
-    STAMCOUNTER                 StatRZRetLDTFault;
-    STAMCOUNTER                 StatRZRetGDTFault;
-    STAMCOUNTER                 StatRZRetIDTFault;
-    STAMCOUNTER                 StatRZRetTSSFault;
-    STAMCOUNTER                 StatRZRetPDFault;
-    STAMCOUNTER                 StatRZRetCSAMTask;
-    STAMCOUNTER                 StatRZRetSyncCR3;
-    STAMCOUNTER                 StatRZRetMisc;
-    STAMCOUNTER                 StatRZRetPatchInt3;
-    STAMCOUNTER                 StatRZRetPatchPF;
-    STAMCOUNTER                 StatRZRetPatchGP;
-    STAMCOUNTER                 StatRZRetPatchIretIRQ;
-    STAMCOUNTER                 StatRZRetRescheduleREM;
-    STAMCOUNTER                 StatRZRetToR3;
-    STAMCOUNTER                 StatRZRetToR3Unknown;
-    STAMCOUNTER                 StatRZRetToR3TMVirt;
-    STAMCOUNTER                 StatRZRetToR3HandyPages;
-    STAMCOUNTER                 StatRZRetToR3PDMQueues;
-    STAMCOUNTER                 StatRZRetToR3Rendezvous;
-    STAMCOUNTER                 StatRZRetToR3Timer;
-    STAMCOUNTER                 StatRZRetToR3DMA;
-    STAMCOUNTER                 StatRZRetToR3CritSect;
-    STAMCOUNTER                 StatRZRetTimerPending;
-    STAMCOUNTER                 StatRZRetInterruptPending;
-    STAMCOUNTER                 StatRZRetCallRing3;
-    STAMCOUNTER                 StatRZRetPATMDuplicateFn;
-    STAMCOUNTER                 StatRZRetPGMChangeMode;
-    STAMCOUNTER                 StatRZRetPendingRequest;
-    STAMCOUNTER                 StatRZRetPGMFlushPending;
-    STAMCOUNTER                 StatRZRetPatchTPR;
-    STAMCOUNTER                 StatRZCallPDMLock;
-    STAMCOUNTER                 StatRZCallLogFlush;
-    STAMCOUNTER                 StatRZCallPGMPoolGrow;
-    STAMCOUNTER                 StatRZCallPGMMapChunk;
-    STAMCOUNTER                 StatRZCallPGMAllocHandy;
-    STAMCOUNTER                 StatRZCallRemReplay;
-    STAMCOUNTER                 StatRZCallVMSetError;
-    STAMCOUNTER                 StatRZCallVMSetRuntimeError;
-    STAMCOUNTER                 StatRZCallPGMLock;
-    /** @} */
-} VMM;
-/** Pointer to VMM. */
-typedef VMM *PVMM;
-
-
-/**
- * VMMCPU Data (part of VMCPU)
- */
-typedef struct VMMCPU
-{
-    /** Offset to the VMCPU structure.
-     * See VMM2VMCPU(). */
-    RTINT                       offVMCPU;
-
-    /** The last RC/R0 return code. */
-    int32_t                     iLastGZRc;
-
-    /** VMM stack, pointer to the top of the stack in R3.
-     * Stack is allocated from the hypervisor heap and is page aligned
-     * and always writable in RC. */
-    R3PTRTYPE(uint8_t *)        pbEMTStackR3;
-    /** Pointer to the bottom of the stack - needed for doing relocations. */
-    RCPTRTYPE(uint8_t *)        pbEMTStackRC;
-    /** Pointer to the bottom of the stack - needed for doing relocations. */
-    RCPTRTYPE(uint8_t *)        pbEMTStackBottomRC;
-
-#ifdef LOG_ENABLED
-    /** Pointer to the R0 logger instance - R3 Ptr.
-     * This is NULL if logging is disabled. */
-    R3PTRTYPE(PVMMR0LOGGER)     pR0LoggerR3;
-    /** Pointer to the R0 logger instance - R0 Ptr.
-     * This is NULL if logging is disabled. */
-    R0PTRTYPE(PVMMR0LOGGER)     pR0LoggerR0;
-#endif
-
-    /** @name Call Ring-3
-     * Formerly known as host calls.
-     * @{ */
-    /** The disable counter. */
-    uint32_t                    cCallRing3Disabled;
-    /** The pending operation. */
-    VMMCALLRING3                enmCallRing3Operation;
-    /** The result of the last operation. */
-    int32_t                     rcCallRing3;
-#if HC_ARCH_BITS == 64
-    uint32_t                    padding;
-#endif
-    /** The argument to the operation. */
-    uint64_t                    u64CallRing3Arg;
-    /** The Ring-0 jmp buffer. */
-    VMMR0JMPBUF                 CallRing3JmpBufR0;
-    /** @} */
-
-} VMMCPU;
-/** Pointer to VMMCPU. */
-typedef VMMCPU *PVMMCPU;
-
-
-/**
- * The VMMGCEntry() codes.
- */
-typedef enum VMMGCOPERATION
-{
-    /** Do GC module init. */
-    VMMGC_DO_VMMGC_INIT = 1,
-
-    /** The first Trap testcase. */
-    VMMGC_DO_TESTCASE_TRAP_FIRST = 0x0dead000,
-    /** Trap 0 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_0 = VMMGC_DO_TESTCASE_TRAP_FIRST,
-    /** Trap 1 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_1,
-    /** Trap 2 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_2,
-    /** Trap 3 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_3,
-    /** Trap 4 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_4,
-    /** Trap 5 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_5,
-    /** Trap 6 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_6,
-    /** Trap 7 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_7,
-    /** Trap 8 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_8,
-    /** Trap 9 testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_9,
-    /** Trap 0a testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_0A,
-    /** Trap 0b testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_0B,
-    /** Trap 0c testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_0C,
-    /** Trap 0d testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_0D,
-    /** Trap 0e testcases, uArg selects the variation. */
-    VMMGC_DO_TESTCASE_TRAP_0E,
-    /** The last trap testcase (exclusive). */
-    VMMGC_DO_TESTCASE_TRAP_LAST,
-    /** Testcase for checking interrupt forwarding. */
-    VMMGC_DO_TESTCASE_HYPER_INTERRUPT,
-    /** Switching testing and profiling stub. */
-    VMMGC_DO_TESTCASE_NOP,
-    /** Testcase for checking interrupt masking.. */
-    VMMGC_DO_TESTCASE_INTERRUPT_MASKING,
-    /** Switching testing and profiling stub. */
-    VMMGC_DO_TESTCASE_HWACCM_NOP,
-
-    /** The usual 32-bit hack. */
-    VMMGC_DO_32_BIT_HACK = 0x7fffffff
-} VMMGCOPERATION;
-
-
-RT_C_DECLS_BEGIN
-
-#ifdef IN_RING3
-int  vmmR3SwitcherInit(PVM pVM);
-void vmmR3SwitcherRelocate(PVM pVM, RTGCINTPTR offDelta);
-#endif /* IN_RING3 */
-
-#ifdef IN_RING0
-/**
- * World switcher assembly routine.
- * It will call VMMGCEntry().
- *
- * @returns return code from VMMGCEntry().
- * @param   pVM     The VM in question.
- * @param   uArg    See VMMGCEntry().
- * @internal
- */
-DECLASM(int)    vmmR0WorldSwitch(PVM pVM, unsigned uArg);
-
-/**
- * Callback function for vmmR0CallRing3SetJmp.
- *
- * @returns VBox status code.
- * @param   pVM     The VM handle.
- */
-typedef DECLCALLBACK(int) FNVMMR0SETJMP(PVM pVM, PVMCPU pVCpu);
-/** Pointer to FNVMMR0SETJMP(). */
-typedef FNVMMR0SETJMP *PFNVMMR0SETJMP;
-
-/**
- * The setjmp variant used for calling Ring-3.
- *
- * This differs from the normal setjmp in that it will resume VMMRZCallRing3 if we're
- * in the middle of a ring-3 call. Another differences is the function pointer and
- * argument. This has to do with resuming code and the stack frame of the caller.
- *
- * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
- * @param   pJmpBuf     The jmp_buf to set.
- * @param   pfn         The function to be called when not resuming..
- * @param   pVM         The argument of that function.
- */
-DECLASM(int)    vmmR0CallRing3SetJmp(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP pfn, PVM pVM, PVMCPU pVCpu);
-
-/**
- * Callback function for vmmR0CallRing3SetJmpEx.
- *
- * @returns VBox status code.
- * @param   pvUser      The user argument.
- */
-typedef DECLCALLBACK(int) FNVMMR0SETJMPEX(void *pvUser);
-/** Pointer to FNVMMR0SETJMP(). */
-typedef FNVMMR0SETJMPEX *PFNVMMR0SETJMPEX;
-
-/**
- * Same as vmmR0CallRing3SetJmp except for the function signature.
- *
- * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
- * @param   pJmpBuf     The jmp_buf to set.
- * @param   pfn         The function to be called when not resuming..
- * @param   pvUser      The argument of that function.
- */
-DECLASM(int)    vmmR0CallRing3SetJmpEx(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMPEX pfn, void *pvUser);
-
-
-/**
- * Worker for VMMRZCallRing3.
- * This will save the stack and registers.
- *
- * @returns rc.
- * @param   pJmpBuf         Pointer to the jump buffer.
- * @param   rc              The return code.
- */
-DECLASM(int)    vmmR0CallRing3LongJmp(PVMMR0JMPBUF pJmpBuf, int rc);
-
-/**
- * Internal R0 logger worker: Logger wrapper.
- */
-VMMR0DECL(void) vmmR0LoggerWrapper(const char *pszFormat, ...);
-
-/**
- * Internal R0 logger worker: Flush logger.
- *
- * @param   pLogger     The logger instance to flush.
- * @remark  This function must be exported!
- */
-VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger);
-
-/**
- * Internal R0 logger worker: Custom prefix.
- *
- * @returns Number of chars written.
- *
- * @param   pLogger     The logger instance.
- * @param   pchBuf      The output buffer.
- * @param   cchBuf      The size of the buffer.
- * @param   pvUser      User argument (ignored).
- */
-VMMR0DECL(size_t) vmmR0LoggerPrefix(PRTLOGGER pLogger, char *pchBuf, size_t cchBuf, void *pvUser);
-
-#endif /* IN_RING0 */
-#ifdef IN_RC
-
-/**
- * Internal GC logger worker: Logger wrapper.
- */
-VMMRCDECL(void) vmmGCLoggerWrapper(const char *pszFormat, ...);
-
-/**
- * Internal GC release logger worker: Logger wrapper.
- */
-VMMRCDECL(void) vmmGCRelLoggerWrapper(const char *pszFormat, ...);
-
-/**
- * Internal GC logger worker: Flush logger.
- *
- * @returns VINF_SUCCESS.
- * @param   pLogger     The logger instance to flush.
- * @remark  This function must be exported!
- */
-VMMRCDECL(int)  vmmGCLoggerFlush(PRTLOGGERRC pLogger);
-
-/** @name Trap testcases and related labels.
- * @{ */
-DECLASM(void)   vmmGCEnableWP(void);
-DECLASM(void)   vmmGCDisableWP(void);
-DECLASM(int)    vmmGCTestTrap3(void);
-DECLASM(int)    vmmGCTestTrap8(void);
-DECLASM(int)    vmmGCTestTrap0d(void);
-DECLASM(int)    vmmGCTestTrap0e(void);
-DECLASM(int)    vmmGCTestTrap0e_FaultEIP(void); /**< a label */
-DECLASM(int)    vmmGCTestTrap0e_ResumeEIP(void); /**< a label */
-/** @} */
-
-#endif /* IN_RC */
-
-RT_C_DECLS_END
-
-/** @} */
-
-#endif
Index: unk/src/VBox/VMM/VMMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/VMMInternal.mac	(revision 35332)
+++ 	(revision )
@@ -1,96 +1,0 @@
-; $Id$
-;; @file
-; VMM - Internal header file.
-;
-
-;
-; Copyright (C) 2006-2009 Oracle Corporation
-;
-; This file is part of VirtualBox Open Source Edition (OSE), as
-; available from http://www.virtualbox.org. This file is free software;
-; you can redistribute it and/or modify it under the terms of the GNU
-; General Public License (GPL) as published by the Free Software
-; Foundation, in version 2 as it comes in the "COPYING" file of the
-; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
-; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
-;
-
-%include "VBox/asmdefs.mac"
-
-;
-; Determine the default stack switching unless specified explicitly.
-;
-%ifndef VMM_R0_SWITCH_STACK
- %ifndef VMM_R0_NO_SWITCH_STACK
-  %ifdef RT_OS_DARWIN
-   %define VMM_R0_SWITCH_STACK
-  %endif
- %endif
-%endif
-
-
-struc VMMR0JMPBUF
-%ifdef RT_ARCH_X86
-    ; traditional jmp_buf
-    .ebx            resd 1
-    .esi            resd 1
-    .edi            resd 1
-    .ebp            resd 1
-    .esp            resd 1
-    .eip            resd 1
-    .eflags         resd 1
-
-    ; additional state and stack info.
-    .fInRing3Call   resd 1
-    .cbSavedStack   resd 1
-    .pvSavedStack   resd 1
-    .SpCheck        resd 1
-    .SpResume       resd 1
-    .SavedEsp       resd 1
-    .SavedEbp       resd 1
-%endif
-%ifdef RT_ARCH_AMD64
-    ; traditional jmp_buf
-    .rbx            resq 1
- %ifdef RT_OS_WINDOWS
-    .rsi            resq 1
-    .rdi            resq 1
- %endif
-    .rbp            resq 1
-    .r12            resq 1
-    .r13            resq 1
-    .r14            resq 1
-    .r15            resq 1
-    .rsp            resq 1
-    .rip            resq 1
- %ifdef RT_OS_WINDOWS
-    .xmm6           resq 2
-    .xmm7           resq 2
-    .xmm8           resq 2
-    .xmm9           resq 2
-    .xmm10          resq 2
-    .xmm11          resq 2
-    .xmm12          resq 2
-    .xmm13          resq 2
-    .xmm14          resq 2
-    .xmm15          resq 2
- %endif
-    .rflags         resq 1
-
-    ; additional state and stack info.
-    .fInRing3Call   resd 1
-    .cbSavedStack   resd 1
-    .pvSavedStack   resq 1
-    .SpCheck        resq 1
-    .SpResume       resq 1
-    .SavedEsp       resq 1
-    .SavedEbp       resq 1
-%endif
-
-    ; Statistics
-    .cbUsedMax      resd 1
-    .cbUsedAvg      resd 1
-    .cbUsedTotal    resq 1
-    .cUsedTotal     resq 1
-endstruc
-
Index: /trunk/src/VBox/VMM/VMMR0/HWACCMR0A.asm
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/HWACCMR0A.asm	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMR0/HWACCMR0A.asm	(revision 35333)
@@ -24,5 +24,5 @@
 %include "VBox/cpum.mac"
 %include "VBox/x86.mac"
-%include "../HWACCMInternal.mac"
+%include "HWACCMInternal.mac"
 
 %ifdef RT_OS_OS2 ;; @todo fix OMF support in yasm and kick nasm out completely.
Index: /trunk/src/VBox/VMM/VMMR0/PGMR0.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/PGMR0.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMR0/PGMR0.cpp	(revision 35333)
@@ -22,7 +22,7 @@
 #include <VBox/pgm.h>
 #include <VBox/gmm.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 #include <VBox/log.h>
 #include <VBox/err.h>
Index: /trunk/src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMR0/PGMR0SharedPage.cpp	(revision 35333)
@@ -22,7 +22,7 @@
 #include <VBox/pgm.h>
 #include <VBox/gmm.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 #include <VBox/log.h>
 #include <VBox/err.h>
Index: /trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm
===================================================================
--- /trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMR0/VMMR0JmpA-amd64.asm	(revision 35333)
@@ -20,5 +20,5 @@
 ;*******************************************************************************
 %include "VBox/asmdefs.mac"
-%include "../VMMInternal.mac"
+%include "VMMInternal.mac"
 %include "iprt/err.mac"
 %include "VBox/param.mac"
Index: /trunk/src/VBox/VMM/VMMR3/CFGM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/CFGM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/CFGM.cpp	(revision 35333)
@@ -0,0 +1,2812 @@
+/* $Id$ */
+/** @file
+ * CFGM - Configuration Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_cfgm       CFGM - The Configuration Manager
+ *
+ * The configuration manager is a directory containing the VM configuration at
+ * run time. It works in a manner similar to the windows registry - it's like a
+ * file system hierarchy, but the files (values) live in a separate name space
+ * and can include the path separators.
+ *
+ * The configuration is normally created via a callback passed to VMR3Create()
+ * via the pfnCFGMConstructor parameter. To make testcase writing a bit simpler,
+ * we allow the callback to be NULL, in which case a simple default
+ * configuration will be created by CFGMR3ConstructDefaultTree(). The
+ * Console::configConstructor() method in Main/ConsoleImpl2.cpp creates the
+ * configuration from the XML.
+ *
+ * Devices, drivers, services and other PDM stuff are given their own subtree
+ * where they are protected from accessing information of any parents. This is
+ * is implemented via the CFGMR3SetRestrictedRoot() API.
+ *
+ * Data validation out over the basic primitives is left to the caller. The
+ * caller is in a better position to know the proper validation rules of the
+ * individual properties.
+ *
+ * @see grp_cfgm
+ *
+ *
+ * @section sec_cfgm_primitives     Data Primitives
+ *
+ * CFGM supports the following data primitives:
+ *      - Integers. Representation is unsigned 64-bit. Boolean, unsigned and
+ *        small integers, and pointers are all represented using this primitive.
+ *      - Zero terminated character strings. These are of course UTF-8.
+ *      - Variable length byte strings. This can be used to get/put binary
+ *        objects like for instance RTMAC.
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_CFGM
+#include <VBox/cfgm.h>
+#include <VBox/dbgf.h>
+#include <VBox/mm.h>
+#include "CFGMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/uuid.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static void cfgmR3DumpPath(PCFGMNODE pNode, PCDBGFINFOHLP pHlp);
+static void cfgmR3Dump(PCFGMNODE pRoot, unsigned iLevel, PCDBGFINFOHLP pHlp);
+static DECLCALLBACK(void) cfgmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static int  cfgmR3ResolveNode(PCFGMNODE pNode, const char *pszPath, PCFGMNODE *ppChild);
+static int  cfgmR3ResolveLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf);
+static int  cfgmR3InsertLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf);
+static void cfgmR3RemoveLeaf(PCFGMNODE pNode, PCFGMLEAF pLeaf);
+static void cfgmR3FreeValue(PCFGMLEAF pLeaf);
+
+
+
+/**
+ * Constructs the configuration for the VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 Pointer to VM which configuration has not yet been loaded.
+ * @param   pfnCFGMConstructor  Pointer to callback function for constructing the VM configuration tree.
+ *                              This is called in the EM.
+ * @param   pvUser              The user argument passed to pfnCFGMConstructor.
+ * @thread  EMT.
+ */
+VMMR3DECL(int) CFGMR3Init(PVM pVM, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUser)
+{
+    LogFlow(("CFGMR3Init: pfnCFGMConstructor=%p pvUser=%p\n", pfnCFGMConstructor, pvUser));
+
+    /*
+     * Init data members.
+     */
+    pVM->cfgm.s.pRoot = NULL;
+
+    /*
+     * Register DBGF into item.
+     */
+    int rc = DBGFR3InfoRegisterInternal(pVM, "cfgm", "Dumps a part of the CFGM tree. The argument indicates where to start.", cfgmR3Info);
+    AssertRCReturn(rc,rc);
+
+    /*
+     * Root Node.
+     */
+    PCFGMNODE pRoot = (PCFGMNODE)MMR3HeapAllocZ(pVM, MM_TAG_CFGM, sizeof(*pRoot));
+    if (!pRoot)
+        return VERR_NO_MEMORY;
+    pRoot->pVM        = pVM;
+    pRoot->cchName    = 0;
+    pVM->cfgm.s.pRoot = pRoot;
+
+        /*
+         * Call the constructor if specified, if not use the default one.
+         */
+    if (pfnCFGMConstructor)
+        rc = pfnCFGMConstructor(pVM, pvUser);
+    else
+        rc = CFGMR3ConstructDefaultTree(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        Log(("CFGMR3Init: Successfully constructed the configuration\n"));
+        CFGMR3Dump(CFGMR3GetRoot(pVM));
+    }
+    else
+        AssertMsgFailed(("Constructor failed with rc=%Rrc pfnCFGMConstructor=%p\n", rc, pfnCFGMConstructor));
+
+    return rc;
+}
+
+
+/**
+ * Terminates the configuration manager.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ */
+VMMR3DECL(int) CFGMR3Term(PVM pVM)
+{
+    CFGMR3RemoveNode(pVM->cfgm.s.pRoot);
+    pVM->cfgm.s.pRoot = NULL;
+    return 0;
+}
+
+
+/**
+ * Gets the root node for the VM.
+ *
+ * @returns Pointer to root node.
+ * @param   pVM             VM handle.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetRoot(PVM pVM)
+{
+    return pVM->cfgm.s.pRoot;
+}
+
+
+/**
+ * Gets the parent of a CFGM node.
+ *
+ * @returns Pointer to the parent node.
+ * @returns NULL if pNode is Root or pNode is the start of a
+ *          restricted subtree (use CFGMr3GetParentEx() for that).
+ *
+ * @param   pNode           The node which parent we query.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetParent(PCFGMNODE pNode)
+{
+    if (pNode && !pNode->fRestrictedRoot)
+        return pNode->pParent;
+    return NULL;
+}
+
+
+/**
+ * Gets the parent of a CFGM node.
+ *
+ * @returns Pointer to the parent node.
+ * @returns NULL if pNode is Root or pVM is not correct.
+ *
+ * @param   pVM             The VM handle, used as token that the caller is trusted.
+ * @param   pNode           The node which parent we query.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetParentEx(PVM pVM, PCFGMNODE pNode)
+{
+    if (pNode && pNode->pVM == pVM)
+        return pNode->pParent;
+    return NULL;
+}
+
+
+/**
+ * Query a child node.
+ *
+ * @returns Pointer to the specified node.
+ * @returns NULL if node was not found or pNode is NULL.
+ * @param   pNode           Node pszPath is relative to.
+ * @param   pszPath         Path to the child node or pNode.
+ *                          It's good style to end this with '/'.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetChild(PCFGMNODE pNode, const char *pszPath)
+{
+    PCFGMNODE pChild;
+    int rc = cfgmR3ResolveNode(pNode, pszPath, &pChild);
+    if (RT_SUCCESS(rc))
+        return pChild;
+    return NULL;
+}
+
+
+/**
+ * Query a child node by a format string.
+ *
+ * @returns Pointer to the specified node.
+ * @returns NULL if node was not found or pNode is NULL.
+ * @param   pNode           Node pszPath is relative to.
+ * @param   pszPathFormat   Path to the child node or pNode.
+ *                          It's good style to end this with '/'.
+ * @param   ...             Arguments to pszPathFormat.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetChildF(PCFGMNODE pNode, const char *pszPathFormat, ...)
+{
+    va_list Args;
+    va_start(Args,  pszPathFormat);
+    PCFGMNODE pRet = CFGMR3GetChildFV(pNode, pszPathFormat, Args);
+    va_end(Args);
+    return pRet;
+}
+
+
+/**
+ * Query a child node by a format string.
+ *
+ * @returns Pointer to the specified node.
+ * @returns NULL if node was not found or pNode is NULL.
+ * @param   pNode           Node pszPath is relative to.
+ * @param   pszPathFormat   Path to the child node or pNode.
+ *                          It's good style to end this with '/'.
+ * @param   Args            Arguments to pszPathFormat.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetChildFV(PCFGMNODE pNode, const char *pszPathFormat, va_list Args)
+{
+    char *pszPath;
+    RTStrAPrintfV(&pszPath, pszPathFormat, Args);
+    if (pszPath)
+    {
+        PCFGMNODE pChild;
+        int rc = cfgmR3ResolveNode(pNode, pszPath, &pChild);
+        RTStrFree(pszPath);
+        if (RT_SUCCESS(rc))
+            return pChild;
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the first child node.
+ * Use this to start an enumeration of child nodes.
+ *
+ * @returns Pointer to the first child.
+ * @returns NULL if no children.
+ * @param   pNode           Node to enumerate children for.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetFirstChild(PCFGMNODE pNode)
+{
+    return pNode ? pNode->pFirstChild : NULL;
+}
+
+
+/**
+ * Gets the next sibling node.
+ * Use this to continue an enumeration.
+ *
+ * @returns Pointer to the first child.
+ * @returns NULL if no children.
+ * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
+ *                          or successive calls to this function.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3GetNextChild(PCFGMNODE pCur)
+{
+    return pCur ? pCur->pNext : NULL;
+}
+
+
+/**
+ * Gets the name of the current node.
+ * (Needed for enumeration.)
+ *
+ * @returns VBox status code.
+ * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
+ *                          or successive calls to CFGMR3GetNextChild().
+ * @param   pszName         Where to store the node name.
+ * @param   cchName         Size of the buffer pointed to by pszName (with terminator).
+ */
+VMMR3DECL(int) CFGMR3GetName(PCFGMNODE pCur, char *pszName, size_t cchName)
+{
+    int rc;
+    if (pCur)
+    {
+        if (cchName > pCur->cchName)
+        {
+            rc = VINF_SUCCESS;
+            memcpy(pszName, pCur->szName, pCur->cchName + 1);
+        }
+        else
+            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+    }
+    else
+        rc = VERR_CFGM_NO_NODE;
+    return rc;
+}
+
+
+/**
+ * Gets the length of the current node's name.
+ * (Needed for enumeration.)
+ *
+ * @returns Node name length in bytes including the terminating null char.
+ * @returns 0 if pCur is NULL.
+ * @param   pCur            Node to returned by a call to CFGMR3GetFirstChild()
+ *                          or successive calls to CFGMR3GetNextChild().
+ */
+VMMR3DECL(size_t) CFGMR3GetNameLen(PCFGMNODE pCur)
+{
+    return pCur ? pCur->cchName + 1 : 0;
+}
+
+
+/**
+ * Validates that the child nodes are within a set of valid names.
+ *
+ * @returns true if all names are found in pszzAllowed.
+ * @returns false if not.
+ * @param   pNode           The node which children should be examined.
+ * @param   pszzValid       List of valid names separated by '\\0' and ending with
+ *                          a double '\\0'.
+ *
+ * @deprecated Use CFGMR3ValidateConfig.
+ */
+VMMR3DECL(bool) CFGMR3AreChildrenValid(PCFGMNODE pNode, const char *pszzValid)
+{
+    if (pNode)
+    {
+        for (PCFGMNODE pChild = pNode->pFirstChild; pChild; pChild = pChild->pNext)
+        {
+            /* search pszzValid for the name */
+            const char *psz = pszzValid;
+            while (*psz)
+            {
+                size_t cch = strlen(psz);
+                if (    cch == pChild->cchName
+                    &&  !memcmp(psz, pChild->szName, cch))
+                    break;
+
+                /* next */
+                psz += cch + 1;
+            }
+
+            /* if at end of pszzValid we didn't find it => failure */
+            if (!*psz)
+            {
+                AssertMsgFailed(("Couldn't find '%s' in the valid values\n", pChild->szName));
+                return false;
+            }
+        }
+    }
+
+    /* all ok. */
+    return true;
+}
+
+
+/**
+ * Gets the first value of a node.
+ * Use this to start an enumeration of values.
+ *
+ * @returns Pointer to the first value.
+ * @param   pCur            The node (Key) which values to enumerate.
+ */
+VMMR3DECL(PCFGMLEAF) CFGMR3GetFirstValue(PCFGMNODE pCur)
+{
+    return pCur ? pCur->pFirstLeaf : NULL;
+}
+
+/**
+ * Gets the next value in enumeration.
+ *
+ * @returns Pointer to the next value.
+ * @param   pCur            The current value as returned by this function or CFGMR3GetFirstValue().
+ */
+VMMR3DECL(PCFGMLEAF) CFGMR3GetNextValue(PCFGMLEAF pCur)
+{
+    return pCur ? pCur->pNext : NULL;
+}
+
+/**
+ * Get the value name.
+ * (Needed for enumeration.)
+ *
+ * @returns VBox status code.
+ * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
+ *                          or successive calls to CFGMR3GetNextValue().
+ * @param   pszName         Where to store the value name.
+ * @param   cchName         Size of the buffer pointed to by pszName (with terminator).
+ */
+VMMR3DECL(int) CFGMR3GetValueName(PCFGMLEAF pCur, char *pszName, size_t cchName)
+{
+    int rc;
+    if (pCur)
+    {
+        if (cchName > pCur->cchName)
+        {
+            rc = VINF_SUCCESS;
+            memcpy(pszName, pCur->szName, pCur->cchName + 1);
+        }
+        else
+            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+    }
+    else
+        rc = VERR_CFGM_NO_NODE;
+    return rc;
+}
+
+
+/**
+ * Gets the length of the current node's name.
+ * (Needed for enumeration.)
+ *
+ * @returns Value name length in bytes including the terminating null char.
+ * @returns 0 if pCur is NULL.
+ * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
+ *                          or successive calls to CFGMR3GetNextValue().
+ */
+VMMR3DECL(size_t) CFGMR3GetValueNameLen(PCFGMLEAF pCur)
+{
+    return pCur ? pCur->cchName + 1 : 0;
+}
+
+
+/**
+ * Gets the value type.
+ * (For enumeration.)
+ *
+ * @returns VBox status code.
+ * @param   pCur            Value returned by a call to CFGMR3GetFirstValue()
+ *                          or successive calls to CFGMR3GetNextValue().
+ */
+VMMR3DECL(CFGMVALUETYPE) CFGMR3GetValueType(PCFGMLEAF pCur)
+{
+    Assert(pCur);
+    return pCur->enmType;
+}
+
+
+/**
+ * Validates that the values are within a set of valid names.
+ *
+ * @returns true if all names are found in pszzAllowed.
+ * @returns false if not.
+ * @param   pNode           The node which values should be examined.
+ * @param   pszzValid       List of valid names separated by '\\0' and ending with
+ *                          a double '\\0'.
+ * @deprecated Use CFGMR3ValidateConfig.
+ */
+VMMR3DECL(bool) CFGMR3AreValuesValid(PCFGMNODE pNode, const char *pszzValid)
+{
+    if (pNode)
+    {
+        for (PCFGMLEAF pLeaf = pNode->pFirstLeaf; pLeaf; pLeaf = pLeaf->pNext)
+        {
+            /* search pszzValid for the name */
+            const char *psz = pszzValid;
+            while (*psz)
+            {
+                size_t cch = strlen(psz);
+                if (    cch == pLeaf->cchName
+                    &&  !memcmp(psz, pLeaf->szName, cch))
+                    break;
+
+                /* next */
+                psz += cch + 1;
+            }
+
+            /* if at end of pszzValid we didn't find it => failure */
+            if (!*psz)
+            {
+                AssertMsgFailed(("Couldn't find '%s' in the valid values\n", pLeaf->szName));
+                return false;
+            }
+        }
+    }
+
+    /* all ok. */
+    return true;
+}
+
+
+
+/**
+ * Query value type.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   penmType        Where to store the type.
+ */
+VMMR3DECL(int) CFGMR3QueryType(PCFGMNODE pNode, const char *pszName, PCFGMVALUETYPE penmType)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (penmType)
+            *penmType = pLeaf->enmType;
+    }
+    return rc;
+}
+
+
+/**
+ * Query value size.
+ * This works on all types of values.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pcb             Where to store the value size.
+ */
+VMMR3DECL(int) CFGMR3QuerySize(PCFGMNODE pNode, const char *pszName, size_t *pcb)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        switch (pLeaf->enmType)
+        {
+            case CFGMVALUETYPE_INTEGER:
+                *pcb = sizeof(pLeaf->Value.Integer.u64);
+                break;
+
+            case CFGMVALUETYPE_STRING:
+                *pcb = pLeaf->Value.String.cb;
+                break;
+
+            case CFGMVALUETYPE_BYTES:
+                *pcb = pLeaf->Value.Bytes.cb;
+                break;
+
+            default:
+                rc = VERR_INTERNAL_ERROR;
+                AssertMsgFailed(("Invalid value type %d\n", pLeaf->enmType));
+                break;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Query integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value.
+ */
+VMMR3DECL(int) CFGMR3QueryInteger(PCFGMNODE pNode, const char *pszName, uint64_t *pu64)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_INTEGER)
+            *pu64 = pLeaf->Value.Integer.u64;
+        else
+            rc = VERR_CFGM_NOT_INTEGER;
+    }
+    return rc;
+}
+
+
+/**
+ * Query integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value. This is set to the default on failure.
+ * @param   u64Def          The default value. This is always set.
+ */
+VMMR3DECL(int) CFGMR3QueryIntegerDef(PCFGMNODE pNode, const char *pszName, uint64_t *pu64, uint64_t u64Def)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_INTEGER)
+            *pu64 = pLeaf->Value.Integer.u64;
+        else
+            rc = VERR_CFGM_NOT_INTEGER;
+    }
+
+    if (RT_FAILURE(rc))
+    {
+        *pu64 = u64Def;
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a zero terminate character value.
+ * @param   pszString       Where to store the string.
+ * @param   cchString       Size of the string buffer. (Includes terminator.)
+ */
+VMMR3DECL(int) CFGMR3QueryString(PCFGMNODE pNode, const char *pszName, char *pszString, size_t cchString)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
+        {
+            size_t cbSrc = pLeaf->Value.String.cb;
+            if (cchString >= cbSrc)
+            {
+                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
+                memset(pszString + cbSrc, 0, cchString - cbSrc);
+            }
+            else
+                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+        }
+        else
+            rc = VERR_CFGM_NOT_STRING;
+    }
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a zero terminate character value.
+ * @param   pszString       Where to store the string. This will not be set on overflow error.
+ * @param   cchString       Size of the string buffer. (Includes terminator.)
+ * @param   pszDef          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryStringDef(PCFGMNODE pNode, const char *pszName, char *pszString, size_t cchString, const char *pszDef)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
+        {
+            size_t cbSrc = pLeaf->Value.String.cb;
+            if (cchString >= cbSrc)
+            {
+                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
+                memset(pszString + cbSrc, 0, cchString - cbSrc);
+            }
+            else
+                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+        }
+        else
+            rc = VERR_CFGM_NOT_STRING;
+    }
+
+    if (RT_FAILURE(rc) && rc != VERR_CFGM_NOT_ENOUGH_SPACE)
+    {
+        size_t cchDef = strlen(pszDef);
+        if (cchString > cchDef)
+        {
+            memcpy(pszString, pszDef, cchDef);
+            memset(pszString + cchDef, 0, cchString - cchDef);
+            if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+                rc = VINF_SUCCESS;
+        }
+        else if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Query byte string value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a byte string value.
+ * @param   pvData          Where to store the binary data.
+ * @param   cbData          Size of buffer pvData points too.
+ */
+VMMR3DECL(int) CFGMR3QueryBytes(PCFGMNODE pNode, const char *pszName, void *pvData, size_t cbData)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_BYTES)
+        {
+            if (cbData >= pLeaf->Value.Bytes.cb)
+            {
+                memcpy(pvData, pLeaf->Value.Bytes.pau8, pLeaf->Value.Bytes.cb);
+                memset((char *)pvData + pLeaf->Value.Bytes.cb, 0, cbData - pLeaf->Value.Bytes.cb);
+            }
+            else
+                rc = VERR_CFGM_NOT_ENOUGH_SPACE;
+        }
+        else
+            rc = VERR_CFGM_NOT_BYTES;
+    }
+    return rc;
+}
+
+
+/**
+ * Validate one level of a configuration node.
+ *
+ * This replaces the CFGMR3AreChildrenValid and CFGMR3AreValuesValid APIs.
+ *
+ * @returns VBox status code.
+ *
+ *          When an error is returned, both VMSetError and AssertLogRelMsgFailed
+ *          have been called.  So, all the caller needs to do is to propagate
+ *          the error status up to PDM.
+ *
+ * @param   pNode               The node to validate.
+ * @param   pszNode             The node path, always ends with a slash.  Use
+ *                              "/" for the root config node.
+ * @param   pszValidValues      Patterns describing the valid value names.  See
+ *                              RTStrSimplePatternMultiMatch for details on the
+ *                              pattern syntax.
+ * @param   pszValidNodes       Patterns describing the valid node (key) names.
+ *                              See RTStrSimplePatternMultiMatch for details on
+ *                              the pattern syntax.
+ * @param   pszWho              Who is calling.
+ * @param   uInstance           The instance number of the caller.
+ */
+VMMR3DECL(int) CFGMR3ValidateConfig(PCFGMNODE pNode, const char *pszNode,
+                                    const char *pszValidValues, const char *pszValidNodes,
+                                    const char *pszWho, uint32_t uInstance)
+{
+    /* Input validation. */
+    AssertPtrNullReturn(pNode, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszNode, VERR_INVALID_POINTER);
+    Assert(*pszNode && pszNode[strlen(pszNode) - 1] == '/');
+    AssertPtrReturn(pszValidValues, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszValidNodes, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszWho, VERR_INVALID_POINTER);
+
+    if (pNode)
+    {
+        /*
+         * Enumerate the leafs and check them against pszValidValues.
+         */
+        for (PCFGMLEAF pLeaf = pNode->pFirstLeaf; pLeaf; pLeaf = pLeaf->pNext)
+        {
+            if (!RTStrSimplePatternMultiMatch(pszValidValues, RTSTR_MAX,
+                                              pLeaf->szName, pLeaf->cchName,
+                                              NULL))
+            {
+                AssertLogRelMsgFailed(("%s/%u: Value '%s/%s' didn't match '%s'\n",
+                                       pszWho, uInstance, pszNode, pLeaf->szName, pszValidValues));
+                return VMSetError(pNode->pVM, VERR_CFGM_CONFIG_UNKNOWN_VALUE, RT_SRC_POS,
+                                  N_("Unknown configuration value '%s/%s' found in the configuration of %s instance #%u"),
+                                  pszNode, pLeaf->szName, pszWho, uInstance);
+            }
+
+        }
+
+        /*
+         * Enumerate the child nodes and check them against pszValidNodes.
+         */
+        for (PCFGMNODE pChild = pNode->pFirstChild; pChild; pChild = pChild->pNext)
+        {
+            if (!RTStrSimplePatternMultiMatch(pszValidNodes, RTSTR_MAX,
+                                              pChild->szName, pChild->cchName,
+                                              NULL))
+            {
+                AssertLogRelMsgFailed(("%s/%u: Node '%s/%s' didn't match '%s'\n",
+                                       pszWho, uInstance, pszNode, pChild->szName, pszValidNodes));
+                return VMSetError(pNode->pVM, VERR_CFGM_CONFIG_UNKNOWN_NODE, RT_SRC_POS,
+                                  N_("Unknown configuration node '%s/%s' found in the configuration of %s instance #%u"),
+                                  pszNode, pChild->szName, pszWho, uInstance);
+            }
+        }
+    }
+
+    /* All is well. */
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Populates the CFGM tree with the default configuration.
+ *
+ * This assumes an empty tree and is intended for testcases and such that only
+ * need to do very small adjustments to the config.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(int) CFGMR3ConstructDefaultTree(PVM pVM)
+{
+    int rc;
+    int rcAll = VINF_SUCCESS;
+#define UPDATERC() do { if (RT_FAILURE(rc) && RT_SUCCESS(rcAll)) rcAll = rc; } while (0)
+
+    PCFGMNODE pRoot = CFGMR3GetRoot(pVM);
+    AssertReturn(pRoot, VERR_WRONG_ORDER);
+
+    /*
+     * Create VM default values.
+     */
+    rc = CFGMR3InsertString(pRoot,  "Name",                 "Default VM");
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "RamSize",              128U * _1M);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "RamHoleSize",          512U * _1M);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "TimerMillies",         10);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "RawR3Enabled",         1);
+    UPDATERC();
+    /** @todo CFGM Defaults: RawR0, PATMEnabled and CASMEnabled needs attention later. */
+    rc = CFGMR3InsertInteger(pRoot, "RawR0Enabled",         1);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "PATMEnabled",          1);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pRoot, "CSAMEnabled",          1);
+    UPDATERC();
+
+    /*
+     * PDM.
+     */
+    PCFGMNODE pPdm;
+    rc = CFGMR3InsertNode(pRoot, "PDM", &pPdm);
+    UPDATERC();
+    PCFGMNODE pDevices = NULL;
+    rc = CFGMR3InsertNode(pPdm, "Devices", &pDevices);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pDevices, "LoadBuiltin",       1); /* boolean */
+    UPDATERC();
+    PCFGMNODE pDrivers = NULL;
+    rc = CFGMR3InsertNode(pPdm, "Drivers", &pDrivers);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pDrivers, "LoadBuiltin",       1); /* boolean */
+    UPDATERC();
+
+
+    /*
+     * Devices
+     */
+    pDevices = NULL;
+    rc = CFGMR3InsertNode(pRoot, "Devices", &pDevices);
+    UPDATERC();
+    /* device */
+    PCFGMNODE pDev = NULL;
+    PCFGMNODE pInst = NULL;
+    PCFGMNODE pCfg = NULL;
+#if 0
+    PCFGMNODE pLunL0 = NULL;
+    PCFGMNODE pLunL1 = NULL;
+#endif
+
+    /*
+     * PC Arch.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pcarch", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * PC Bios.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pcbios", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "RamSize",              128U * _1M);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "RamHoleSize",          512U * _1M);
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice0",          "IDE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice1",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice2",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "BootDevice3",          "NONE");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "HardDiskDevice",       "piix3ide");
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "FloppyDevice",         "");
+    UPDATERC();
+    RTUUID Uuid;
+    RTUuidClear(&Uuid);
+    rc = CFGMR3InsertBytes(pCfg,    "UUID", &Uuid, sizeof(Uuid));
+    UPDATERC();
+
+    /*
+     * PCI bus.
+     */
+    rc = CFGMR3InsertNode(pDevices, "pci", &pDev); /* piix3 */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * PS/2 keyboard & mouse
+     */
+    rc = CFGMR3InsertNode(pDevices, "pckbd", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+#if 0
+    rc = CFGMR3InsertNode(pInst,    "LUN#0", &pLunL0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL0, "Driver",               "KeyboardQueue");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "QueueSize",            64);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "AttachedDriver", &pLunL1);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL1, "Driver",               "MainKeyboard");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL1,   "Config", &pCfg);
+    UPDATERC();
+#endif
+#if 0
+    rc = CFGMR3InsertNode(pInst,    "LUN#1", &pLunL0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL0, "Driver",               "MouseQueue");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "QueueSize",            128);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL0,   "AttachedDriver", &pLunL1);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL1, "Driver",               "MainMouse");
+    UPDATERC();
+    rc = CFGMR3InsertNode(pLunL1,   "Config", &pCfg);
+    UPDATERC();
+#endif
+
+    /*
+     * i8254 Programmable Interval Timer And Dummy Speaker
+     */
+    rc = CFGMR3InsertNode(pDevices, "i8254", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+#ifdef DEBUG
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+#endif
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * i8259 Programmable Interrupt Controller.
+     */
+    rc = CFGMR3InsertNode(pDevices, "i8259", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * RTC MC146818.
+     */
+    rc = CFGMR3InsertNode(pDevices, "mc146818", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+    /*
+     * VGA.
+     */
+    rc = CFGMR3InsertNode(pDevices, "vga", &pDev);
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "VRamSize",             4 * _1M);
+    UPDATERC();
+
+    /* Bios logo. */
+    rc = CFGMR3InsertInteger(pCfg,  "FadeIn",               1);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "FadeOut",              1);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pCfg,  "LogoTime",             0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pCfg,   "LogoFile",             "");
+    UPDATERC();
+
+#if 0
+    rc = CFGMR3InsertNode(pInst,    "LUN#0", &pLunL0);
+    UPDATERC();
+    rc = CFGMR3InsertString(pLunL0, "Driver",               "MainDisplay");
+    UPDATERC();
+#endif
+
+    /*
+     * IDE controller.
+     */
+    rc = CFGMR3InsertNode(pDevices, "piix3ide", &pDev); /* piix3 */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pDev,     "0", &pInst);
+    UPDATERC();
+    rc = CFGMR3InsertInteger(pInst, "Trusted",              1);         /* boolean */
+    UPDATERC();
+    rc = CFGMR3InsertNode(pInst,    "Config", &pCfg);
+    UPDATERC();
+
+
+
+    /*
+     * ...
+     */
+
+#undef UPDATERC
+    return rcAll;
+}
+
+
+
+
+/**
+ * Resolves a path reference to a child node.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszPath         Path to the child node.
+ * @param   ppChild         Where to store the pointer to the child node.
+ */
+static int cfgmR3ResolveNode(PCFGMNODE pNode, const char *pszPath, PCFGMNODE *ppChild)
+{
+    if (!pNode)
+        return VERR_CFGM_NO_PARENT;
+    PCFGMNODE pChild = NULL;
+    for (;;)
+    {
+        /* skip leading slashes. */
+        while (*pszPath == '/')
+            pszPath++;
+
+        /* End of path? */
+        if (!*pszPath)
+        {
+            if (!pChild)
+                return VERR_CFGM_INVALID_CHILD_PATH;
+            *ppChild = pChild;
+            return VINF_SUCCESS;
+        }
+
+        /* find end of component. */
+        const char *pszNext = strchr(pszPath, '/');
+        if (!pszNext)
+            pszNext = strchr(pszPath,  '\0');
+        RTUINT cchName = pszNext - pszPath;
+
+        /* search child list. */
+        pChild = pNode->pFirstChild;
+        for ( ; pChild; pChild = pChild->pNext)
+            if (pChild->cchName == cchName)
+            {
+                int iDiff = memcmp(pszPath, pChild->szName, cchName);
+                if (iDiff <= 0)
+                {
+                    if (iDiff != 0)
+                        pChild = NULL;
+                    break;
+                }
+            }
+        if (!pChild)
+            return VERR_CFGM_CHILD_NOT_FOUND;
+
+        /* next iteration */
+        pNode = pChild;
+        pszPath = pszNext;
+    }
+
+    /* won't get here */
+}
+
+
+/**
+ * Resolves a path reference to a child node.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of a byte string value.
+ * @param   ppLeaf          Where to store the pointer to the leaf node.
+ */
+static int cfgmR3ResolveLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf)
+{
+    if (!pNode)
+        return VERR_CFGM_NO_PARENT;
+
+    size_t      cchName = strlen(pszName);
+    PCFGMLEAF   pLeaf   = pNode->pFirstLeaf;
+    while (pLeaf)
+    {
+        if (cchName == pLeaf->cchName)
+        {
+            int iDiff = memcmp(pszName, pLeaf->szName, cchName);
+            if (iDiff <= 0)
+            {
+                if (iDiff != 0)
+                    break;
+                *ppLeaf = pLeaf;
+                return VINF_SUCCESS;
+            }
+        }
+
+        /* next */
+        pLeaf = pLeaf->pNext;
+    }
+    return VERR_CFGM_VALUE_NOT_FOUND;
+}
+
+
+
+/**
+ * Creates a CFGM tree.
+ *
+ * This is intended for creating device/driver configs can be
+ * passed around and later attached to the main tree in the
+ * correct location.
+ *
+ * @returns Pointer to the root node.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(PCFGMNODE) CFGMR3CreateTree(PVM pVM)
+{
+    PCFGMNODE pNew = (PCFGMNODE)MMR3HeapAlloc(pVM, MM_TAG_CFGM, sizeof(*pNew));
+    if (pNew)
+    {
+        pNew->pPrev         = NULL;
+        pNew->pNext         = NULL;
+        pNew->pParent       = NULL;
+        pNew->pFirstChild   = NULL;
+        pNew->pFirstLeaf    = NULL;
+        pNew->pVM           = pVM;
+        pNew->fRestrictedRoot = false;
+        pNew->cchName       = 0;
+        pNew->szName[0]     = 0;
+    }
+    return pNew;
+}
+
+
+/**
+ * Insert subtree.
+ *
+ * This function inserts (no duplication) a tree created by CFGMR3CreateTree()
+ * into the main tree.
+ *
+ * The root node of the inserted subtree will need to be reallocated, which
+ * effectually means that the passed in pSubTree handle becomes invalid
+ * upon successful return. Use the value returned in ppChild instead
+ * of pSubTree.
+ *
+ * @returns VBox status code.
+ * @returns VERR_CFGM_NODE_EXISTS if the final child node name component exists.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name or path of the new child node.
+ * @param   pSubTree    The subtree to insert. Must be returned by CFGMR3CreateTree().
+ * @param   ppChild     Where to store the address of the new child node. (optional)
+ */
+VMMR3DECL(int) CFGMR3InsertSubTree(PCFGMNODE pNode, const char *pszName, PCFGMNODE pSubTree, PCFGMNODE *ppChild)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSubTree, VERR_INVALID_POINTER);
+    AssertReturn(!pSubTree->pParent, VERR_INVALID_PARAMETER);
+    AssertReturn(pSubTree->pVM, VERR_INVALID_PARAMETER);
+    AssertReturn(pSubTree->pParent != pSubTree->pVM->cfgm.s.pRoot, VERR_INVALID_PARAMETER);
+    Assert(!pSubTree->pNext);
+    Assert(!pSubTree->pPrev);
+
+    /*
+     * Use CFGMR3InsertNode to create a new node and then
+     * re-attach the children and leafs of the subtree to it.
+     */
+    PCFGMNODE pNewChild;
+    int rc = CFGMR3InsertNode(pNode, pszName, &pNewChild);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(!pNewChild->pFirstChild);
+        pNewChild->pFirstChild = pSubTree->pFirstChild;
+        Assert(!pNewChild->pFirstLeaf);
+        pNewChild->pFirstLeaf = pSubTree->pFirstLeaf;
+        if (ppChild)
+            *ppChild = pNewChild;
+
+        /* free the old subtree root */
+        pSubTree->pVM = NULL;
+        pSubTree->pFirstLeaf = NULL;
+        pSubTree->pFirstChild = NULL;
+        MMR3HeapFree(pSubTree);
+    }
+    return rc;
+}
+
+
+/**
+ * Compares two names.
+ *
+ * @returns Similar to memcpy.
+ * @param   pszName1            The first name.
+ * @param   cchName1            The length of the first name.
+ * @param   pszName2            The second name.
+ * @param   cchName2            The length of the second name.
+ */
+DECLINLINE(int) cfgmR3CompareNames(const char *pszName1, size_t cchName1, const char *pszName2, size_t cchName2)
+{
+    int iDiff;
+    if (cchName1 <= cchName2)
+    {
+        iDiff = memcmp(pszName1, pszName2, cchName1);
+        if (!iDiff && cchName1 < cchName2)
+            iDiff = -1;
+    }
+    else
+    {
+        iDiff = memcmp(pszName1, pszName2, cchName2);
+        if (!iDiff)
+            iDiff = 1;
+    }
+    return iDiff;
+}
+
+
+/**
+ * Insert a node.
+ *
+ * @returns VBox status code.
+ * @returns VERR_CFGM_NODE_EXISTS if the final child node name component exists.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name or path of the new child node.
+ * @param   ppChild     Where to store the address of the new child node. (optional)
+ */
+VMMR3DECL(int) CFGMR3InsertNode(PCFGMNODE pNode, const char *pszName, PCFGMNODE *ppChild)
+{
+    int rc;
+    if (pNode)
+    {
+        /*
+         * If given a path we have to deal with it component by component.
+         */
+        while (*pszName == '/')
+            pszName++;
+        if (strchr(pszName, '/'))
+        {
+            char *pszDup = RTStrDup(pszName);
+            if (pszDup)
+            {
+                char *psz = pszDup;
+                for (;;)
+                {
+                    /* Terminate at '/' and find the next component. */
+                    char *pszNext = strchr(psz, '/');
+                    if (pszNext)
+                    {
+                        *pszNext++ = '\0';
+                        while (*pszNext == '/')
+                            pszNext++;
+                        if (*pszNext == '\0')
+                            pszNext = NULL;
+                    }
+
+                    /* does it exist? */
+                    PCFGMNODE pChild = CFGMR3GetChild(pNode, psz);
+                    if (!pChild)
+                    {
+                        /* no, insert it */
+                        rc = CFGMR3InsertNode(pNode, psz, &pChild);
+                        if (RT_FAILURE(rc))
+                            break;
+                        if (!pszNext)
+                        {
+                            if (ppChild)
+                                *ppChild = pChild;
+                            break;
+                        }
+
+                    }
+                    /* if last component fail */
+                    else if (!pszNext)
+                    {
+                        rc = VERR_CFGM_NODE_EXISTS;
+                        break;
+                    }
+
+                    /* next */
+                    pNode = pChild;
+                    psz = pszNext;
+                }
+                RTStrFree(pszDup);
+            }
+            else
+                rc = VERR_NO_TMP_MEMORY;
+        }
+        /*
+         * Not multicomponent, just make sure it's a non-zero name.
+         */
+        else if (*pszName)
+        {
+            /*
+             * Check if already exists and find last node in chain.
+             */
+            size_t    cchName = strlen(pszName);
+            PCFGMNODE pPrev   = NULL;
+            PCFGMNODE pNext   = pNode->pFirstChild;
+            if (pNext)
+            {
+                for ( ; pNext; pPrev = pNext, pNext = pNext->pNext)
+                {
+                    int iDiff = cfgmR3CompareNames(pszName, cchName, pNext->szName, pNext->cchName);
+                    if (iDiff <= 0)
+                    {
+                        if (!iDiff)
+                            return VERR_CFGM_NODE_EXISTS;
+                        break;
+                    }
+                }
+            }
+
+            /*
+             * Allocate and init node.
+             */
+            PCFGMNODE pNew = (PCFGMNODE)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM, sizeof(*pNew) + cchName);
+            if (pNew)
+            {
+                pNew->pParent       = pNode;
+                pNew->pFirstChild   = NULL;
+                pNew->pFirstLeaf    = NULL;
+                pNew->pVM           = pNode->pVM;
+                pNew->fRestrictedRoot = false;
+                pNew->cchName       = cchName;
+                memcpy(pNew->szName, pszName, cchName + 1);
+
+                /*
+                 * Insert into child list.
+                 */
+                pNew->pPrev         = pPrev;
+                if (pPrev)
+                    pPrev->pNext    = pNew;
+                else
+                    pNode->pFirstChild = pNew;
+                pNew->pNext         = pNext;
+                if (pNext)
+                    pNext->pPrev    = pNew;
+
+                if (ppChild)
+                    *ppChild = pNew;
+                rc = VINF_SUCCESS;
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+        {
+            rc = VERR_CFGM_INVALID_NODE_PATH;
+            AssertMsgFailed(("Invalid path %s\n", pszName));
+        }
+    }
+    else
+    {
+        rc = VERR_CFGM_NO_PARENT;
+        AssertMsgFailed(("No parent! path %s\n", pszName));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Insert a node, format string name.
+ *
+ * @returns VBox status     code.
+ * @param   pNode           Parent node.
+ * @param   ppChild         Where to store the address of the new child node. (optional)
+ * @param   pszNameFormat   Name of or path the new child node.
+ * @param   ...             Name format arguments.
+ */
+VMMR3DECL(int) CFGMR3InsertNodeF(PCFGMNODE pNode, PCFGMNODE *ppChild, const char *pszNameFormat, ...)
+{
+    va_list Args;
+    va_start(Args,  pszNameFormat);
+    int rc = CFGMR3InsertNodeFV(pNode, ppChild, pszNameFormat, Args);
+    va_end(Args);
+    return rc;
+}
+
+
+/**
+ * Insert a node, format string name.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   ppChild         Where to store the address of the new child node. (optional)
+ * @param   pszNameFormat   Name or path of the new child node.
+ * @param   Args            Name format arguments.
+ */
+VMMR3DECL(int) CFGMR3InsertNodeFV(PCFGMNODE pNode, PCFGMNODE *ppChild, const char *pszNameFormat, va_list Args)
+{
+    int     rc;
+    char   *pszName;
+    RTStrAPrintfV(&pszName, pszNameFormat, Args);
+    if (pszName)
+    {
+        rc = CFGMR3InsertNode(pNode, pszName, ppChild);
+        RTStrFree(pszName);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Marks the node as the root of a restricted subtree, i.e. the end of
+ * a CFGMR3GetParent() journey.
+ *
+ * @param   pNode       The node to mark.
+ */
+VMMR3DECL(void) CFGMR3SetRestrictedRoot(PCFGMNODE pNode)
+{
+    if (pNode)
+        pNode->fRestrictedRoot = true;
+}
+
+
+/**
+ * Insert a node.
+ *
+ * @returns VBox status code.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name of the new child node.
+ * @param   ppLeaf      Where to store the new leaf.
+ *                      The caller must fill in the enmType and Value fields!
+ */
+static int cfgmR3InsertLeaf(PCFGMNODE pNode, const char *pszName, PCFGMLEAF *ppLeaf)
+{
+    int rc;
+    if (*pszName)
+    {
+        if (pNode)
+        {
+            /*
+             * Check if already exists and find last node in chain.
+             */
+            size_t    cchName  = strlen(pszName);
+            PCFGMLEAF pPrev    = NULL;
+            PCFGMLEAF pNext    = pNode->pFirstLeaf;
+            if (pNext)
+            {
+                for ( ; pNext; pPrev = pNext, pNext = pNext->pNext)
+                {
+                    int iDiff = cfgmR3CompareNames(pszName, cchName, pNext->szName, pNext->cchName);
+                    if (iDiff <= 0)
+                    {
+                        if (!iDiff)
+                            return VERR_CFGM_LEAF_EXISTS;
+                        break;
+                    }
+                }
+            }
+
+            /*
+             * Allocate and init node.
+             */
+            PCFGMLEAF pNew = (PCFGMLEAF)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM, sizeof(*pNew) + cchName);
+            if (pNew)
+            {
+                pNew->cchName       = cchName;
+                memcpy(pNew->szName, pszName, cchName + 1);
+
+                /*
+                 * Insert into child list.
+                 */
+                pNew->pPrev         = pPrev;
+                if (pPrev)
+                    pPrev->pNext    = pNew;
+                else
+                    pNode->pFirstLeaf = pNew;
+                pNew->pNext         = pNext;
+                if (pNext)
+                    pNext->pPrev    = pNew;
+
+                *ppLeaf = pNew;
+                rc = VINF_SUCCESS;
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            rc = VERR_CFGM_NO_PARENT;
+    }
+    else
+        rc = VERR_CFGM_INVALID_CHILD_PATH;
+    return rc;
+}
+
+
+/**
+ * Remove a node.
+ *
+ * @param   pNode       Parent node.
+ */
+VMMR3DECL(void) CFGMR3RemoveNode(PCFGMNODE pNode)
+{
+    if (pNode)
+    {
+        /*
+         * Free children.
+         */
+        while (pNode->pFirstChild)
+            CFGMR3RemoveNode(pNode->pFirstChild);
+
+        /*
+         * Free leafs.
+         */
+        while (pNode->pFirstLeaf)
+            cfgmR3RemoveLeaf(pNode, pNode->pFirstLeaf);
+
+        /*
+         * Unlink ourselves.
+         */
+        if (pNode->pPrev)
+            pNode->pPrev->pNext = pNode->pNext;
+        else
+        {
+            if (pNode->pParent)
+                pNode->pParent->pFirstChild = pNode->pNext;
+            else if (pNode == pNode->pVM->cfgm.s.pRoot) /* might be a different tree */
+                pNode->pVM->cfgm.s.pRoot = NULL;
+        }
+        if (pNode->pNext)
+            pNode->pNext->pPrev = pNode->pPrev;
+
+        /*
+         * Free ourselves. (bit of paranoia first)
+         */
+        pNode->pVM = NULL;
+        pNode->pNext = NULL;
+        pNode->pPrev = NULL;
+        pNode->pParent = NULL;
+        MMR3HeapFree(pNode);
+    }
+}
+
+
+/**
+ * Removes a leaf.
+ *
+ * @param   pNode   Parent node.
+ * @param   pLeaf   Leaf to remove.
+ */
+static void cfgmR3RemoveLeaf(PCFGMNODE pNode, PCFGMLEAF pLeaf)
+{
+    if (pNode && pLeaf)
+    {
+        /*
+         * Unlink.
+         */
+        if (pLeaf->pPrev)
+            pLeaf->pPrev->pNext = pLeaf->pNext;
+        else
+            pNode->pFirstLeaf = pLeaf->pNext;
+        if (pLeaf->pNext)
+            pLeaf->pNext->pPrev = pLeaf->pPrev;
+
+        /*
+         * Free value and node.
+         */
+        cfgmR3FreeValue(pLeaf);
+        pLeaf->pNext = NULL;
+        pLeaf->pPrev = NULL;
+        MMR3HeapFree(pLeaf);
+    }
+}
+
+
+/**
+ * Frees whatever resources the leaf value is owning.
+ *
+ * Use this before assigning a new value to a leaf.
+ * The caller must either free the leaf or assign a new value to it.
+ *
+ * @param   pLeaf       Pointer to the leaf which value should be free.
+ */
+static void cfgmR3FreeValue(PCFGMLEAF pLeaf)
+{
+    if (pLeaf)
+    {
+        switch (pLeaf->enmType)
+        {
+            case CFGMVALUETYPE_BYTES:
+                MMR3HeapFree(pLeaf->Value.Bytes.pau8);
+                pLeaf->Value.Bytes.pau8 = NULL;
+                pLeaf->Value.Bytes.cb = 0;
+                break;
+
+            case CFGMVALUETYPE_STRING:
+                MMR3HeapFree(pLeaf->Value.String.psz);
+                pLeaf->Value.String.psz = NULL;
+                pLeaf->Value.String.cb = 0;
+                break;
+
+            case CFGMVALUETYPE_INTEGER:
+                break;
+        }
+        pLeaf->enmType = (CFGMVALUETYPE)0;
+    }
+}
+
+
+/**
+ * Inserts a new integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   u64Integer      The value.
+ */
+VMMR3DECL(int) CFGMR3InsertInteger(PCFGMNODE pNode, const char *pszName, uint64_t u64Integer)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        pLeaf->enmType = CFGMVALUETYPE_INTEGER;
+        pLeaf->Value.Integer.u64 = u64Integer;
+    }
+    return rc;
+}
+
+
+/**
+ * Inserts a new string value. This variant expects that the caller know the length
+ * of the string already so we can avoid calling strlen() here.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszString       The value. Must not be NULL.
+ * @param   cchString       The length of the string excluding the
+ *                          terminator.
+ */
+VMMR3DECL(int) CFGMR3InsertStringN(PCFGMNODE pNode, const char *pszName, const char *pszString, size_t cchString)
+{
+    Assert(RTStrNLen(pszString, cchString) == cchString);
+
+    int rc;
+    if (pNode)
+    {
+        /*
+         * Allocate string object first.
+         */
+        char *pszStringCopy = (char *)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_STRING, cchString + 1);
+        if (pszStringCopy)
+        {
+            memcpy(pszStringCopy, pszString, cchString);
+            pszStringCopy[cchString] = '\0';
+
+            /*
+             * Create value leaf and set it to string type.
+             */
+            PCFGMLEAF pLeaf;
+            rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+            if (RT_SUCCESS(rc))
+            {
+                pLeaf->enmType = CFGMVALUETYPE_STRING;
+                pLeaf->Value.String.psz = pszStringCopy;
+                pLeaf->Value.String.cb  = cchString + 1;
+            }
+            else
+                MMR3HeapFree(pszStringCopy);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    else
+        rc = VERR_CFGM_NO_PARENT;
+
+    return rc;
+}
+
+
+/**
+ * Inserts a new string value. Calls strlen(pszString) internally; if you know the
+ * length of the string, CFGMR3InsertStringLengthKnown() is faster.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszString       The value.
+ */
+VMMR3DECL(int) CFGMR3InsertString(PCFGMNODE pNode, const char *pszName, const char *pszString)
+{
+    return CFGMR3InsertStringN(pNode, pszName, pszString, strlen(pszString));
+}
+
+
+/**
+ * Same as CFGMR3InsertString except the string value given in RTStrPrintfV
+ * fashion.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszFormat       The value given as a format string.
+ * @param   va              Argument to pszFormat.
+ */
+VMMR3DECL(int) CFGMR3InsertStringFV(PCFGMNODE pNode, const char *pszName, const char *pszFormat, va_list va)
+{
+    int rc;
+    if (pNode)
+    {
+        /*
+         * Allocate string object first.
+         */
+        char *pszString = MMR3HeapAPrintfVU(pNode->pVM->pUVM, MM_TAG_CFGM_STRING, pszFormat, va);
+        if (pszString)
+        {
+            /*
+             * Create value leaf and set it to string type.
+             */
+            PCFGMLEAF pLeaf;
+            rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+            if (RT_SUCCESS(rc))
+            {
+                pLeaf->enmType = CFGMVALUETYPE_STRING;
+                pLeaf->Value.String.psz = pszString;
+                pLeaf->Value.String.cb  = strlen(pszString) + 1;
+            }
+            else
+                MMR3HeapFree(pszString);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    else
+        rc = VERR_CFGM_NO_PARENT;
+
+    return rc;
+}
+
+
+/**
+ * Same as CFGMR3InsertString except the string value given in RTStrPrintf
+ * fashion.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pszFormat       The value given as a format string.
+ * @param   ...             Argument to pszFormat.
+ */
+VMMR3DECL(int) CFGMR3InsertStringF(PCFGMNODE pNode, const char *pszName, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = CFGMR3InsertStringFV(pNode, pszName, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Same as CFGMR3InsertString except the string value given as a UTF-16 string.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pwszValue       The string value (UTF-16).
+ */
+VMMR3DECL(int) CFGMR3InsertStringW(PCFGMNODE pNode, const char *pszName, PCRTUTF16 pwszValue)
+{
+    char *pszValue;
+    int rc = RTUtf16ToUtf8(pwszValue, &pszValue);
+    if (RT_SUCCESS(rc))
+    {
+        rc = CFGMR3InsertString(pNode, pszName, pszValue);
+        RTStrFree(pszValue);
+    }
+    return rc;
+}
+
+
+/**
+ * Inserts a new integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Parent node.
+ * @param   pszName         Value name.
+ * @param   pvBytes         The value.
+ * @param   cbBytes         The value size.
+ */
+VMMR3DECL(int) CFGMR3InsertBytes(PCFGMNODE pNode, const char *pszName, const void *pvBytes, size_t cbBytes)
+{
+    int rc;
+    if (pNode)
+    {
+        if (cbBytes == (RTUINT)cbBytes)
+        {
+            /*
+             * Allocate string object first.
+             */
+            void *pvCopy = MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_STRING, cbBytes);
+            if (pvCopy || !cbBytes)
+            {
+                memcpy(pvCopy, pvBytes, cbBytes);
+
+                /*
+                 * Create value leaf and set it to string type.
+                 */
+                PCFGMLEAF pLeaf;
+                rc = cfgmR3InsertLeaf(pNode, pszName, &pLeaf);
+                if (RT_SUCCESS(rc))
+                {
+                    pLeaf->enmType = CFGMVALUETYPE_BYTES;
+                    pLeaf->Value.Bytes.cb   = cbBytes;
+                    pLeaf->Value.Bytes.pau8 = (uint8_t *)pvCopy;
+                }
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            rc = VERR_OUT_OF_RANGE;
+    }
+    else
+        rc = VERR_CFGM_NO_PARENT;
+
+    return rc;
+}
+
+
+/**
+ * Remove a value.
+ *
+ * @returns VBox status code.
+ * @param   pNode       Parent node.
+ * @param   pszName     Name of the new child node.
+ */
+VMMR3DECL(int) CFGMR3RemoveValue(PCFGMNODE pNode, const char *pszName)
+{
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+        cfgmR3RemoveLeaf(pNode, pLeaf);
+    return rc;
+}
+
+
+
+/*
+ *  -+- helper apis -+-
+ */
+
+
+/**
+ * Query unsigned 64-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value.
+ */
+VMMR3DECL(int) CFGMR3QueryU64(PCFGMNODE pNode, const char *pszName, uint64_t *pu64)
+{
+    return CFGMR3QueryInteger(pNode, pszName, pu64);
+}
+
+
+/**
+ * Query unsigned 64-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu64            Where to store the integer value. Set to default on failure.
+ * @param   u64Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU64Def(PCFGMNODE pNode, const char *pszName, uint64_t *pu64, uint64_t u64Def)
+{
+    return CFGMR3QueryIntegerDef(pNode, pszName, pu64, u64Def);
+}
+
+
+/**
+ * Query signed 64-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi64            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS64(PCFGMNODE pNode, const char *pszName, int64_t *pi64)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+        *pi64 = (int64_t)u64;
+    return rc;
+}
+
+
+/**
+ * Query signed 64-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi64            Where to store the value. Set to default on failure.
+ * @param   i64Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS64Def(PCFGMNODE pNode, const char *pszName, int64_t *pi64, int64_t i64Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i64Def);
+    *pi64 = (int64_t)u64;
+    return rc;
+}
+
+
+/**
+ * Query unsigned 32-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu32            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryU32(PCFGMNODE pNode, const char *pszName, uint32_t *pu32)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffff00000000)))
+            *pu32 = (uint32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query unsigned 32-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu32            Where to store the value. Set to default on failure.
+ * @param   u32Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU32Def(PCFGMNODE pNode, const char *pszName, uint32_t *pu32, uint32_t u32Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u32Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffff00000000)))
+            *pu32 = (uint32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pu32 = u32Def;
+    return rc;
+}
+
+
+/**
+ * Query signed 32-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi32            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS32(PCFGMNODE pNode, const char *pszName, int32_t *pi32)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffff80000000))
+            ||  (u64 & UINT64_C(0xffffffff80000000)) == UINT64_C(0xffffffff80000000))
+            *pi32 = (int32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query signed 32-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi32            Where to store the value. Set to default on failure.
+ * @param   i32Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS32Def(PCFGMNODE pNode, const char *pszName, int32_t *pi32, int32_t i32Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i32Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffff80000000))
+            ||  (u64 & UINT64_C(0xffffffff80000000)) == UINT64_C(0xffffffff80000000))
+            *pi32 = (int32_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pi32 = i32Def;
+    return rc;
+}
+
+
+/**
+ * Query unsigned 16-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu16            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryU16(PCFGMNODE pNode, const char *pszName, uint16_t *pu16)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffff0000)))
+            *pu16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query unsigned 16-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu16            Where to store the value. Set to default on failure.
+ * @param   i16Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU16Def(PCFGMNODE pNode, const char *pszName, uint16_t *pu16, uint16_t u16Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u16Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffff0000)))
+            *pu16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pu16 = u16Def;
+    return rc;
+}
+
+
+/**
+ * Query signed 16-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi16            Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS16(PCFGMNODE pNode, const char *pszName, int16_t *pi16)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffff8000))
+            ||  (u64 & UINT64_C(0xffffffffffff8000)) == UINT64_C(0xffffffffffff8000))
+            *pi16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query signed 16-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi16            Where to store the value. Set to default on failure.
+ * @param   i16Def          The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS16Def(PCFGMNODE pNode, const char *pszName, int16_t *pi16, int16_t i16Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i16Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffff8000))
+            ||  (u64 & UINT64_C(0xffffffffffff8000)) == UINT64_C(0xffffffffffff8000))
+            *pi16 = (int16_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pi16 = i16Def;
+    return rc;
+}
+
+
+/**
+ * Query unsigned 8-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu8             Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryU8(PCFGMNODE pNode, const char *pszName, uint8_t *pu8)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffffff00)))
+            *pu8 = (uint8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query unsigned 8-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu8             Where to store the value. Set to default on failure.
+ * @param   u8Def           The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryU8Def(PCFGMNODE pNode, const char *pszName, uint8_t *pu8, uint8_t u8Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, u8Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(u64 & UINT64_C(0xffffffffffffff00)))
+            *pu8 = (uint8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pu8 = u8Def;
+    return rc;
+}
+
+
+/**
+ * Query signed 8-bit integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi8             Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryS8(PCFGMNODE pNode, const char *pszName, int8_t *pi8)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffffff80))
+            ||  (u64 & UINT64_C(0xffffffffffffff80)) == UINT64_C(0xffffffffffffff80))
+            *pi8 = (int8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query signed 8-bit integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi8             Where to store the value. Set to default on failure.
+ * @param   i8Def           The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryS8Def(PCFGMNODE pNode, const char *pszName, int8_t *pi8, int8_t i8Def)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, i8Def);
+    if (RT_SUCCESS(rc))
+    {
+        if (   !(u64 & UINT64_C(0xffffffffffffff80))
+            ||  (u64 & UINT64_C(0xffffffffffffff80)) == UINT64_C(0xffffffffffffff80))
+            *pi8 = (int8_t)u64;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pi8 = i8Def;
+    return rc;
+}
+
+
+/**
+ * Query boolean integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pf              Where to store the value.
+ * @remark  This function will interpret any non-zero value as true.
+ */
+VMMR3DECL(int) CFGMR3QueryBool(PCFGMNODE pNode, const char *pszName, bool *pf)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+        *pf = u64 ? true : false;
+    return rc;
+}
+
+
+/**
+ * Query boolean integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pf              Where to store the value. Set to default on failure.
+ * @param   fDef            The default value.
+ * @remark  This function will interpret any non-zero value as true.
+ */
+VMMR3DECL(int) CFGMR3QueryBoolDef(PCFGMNODE pNode, const char *pszName, bool *pf, bool fDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, fDef);
+    *pf = u64 ? true : false;
+    return rc;
+}
+
+
+/**
+ * Query I/O port address value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pPort           Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryPort(PCFGMNODE pNode, const char *pszName, PRTIOPORT pPort)
+{
+    AssertCompileSize(RTIOPORT, 2);
+    return CFGMR3QueryU16(pNode, pszName, pPort);
+}
+
+
+/**
+ * Query I/O port address value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pPort           Where to store the value. Set to default on failure.
+ * @param   PortDef         The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryPortDef(PCFGMNODE pNode, const char *pszName, PRTIOPORT pPort, RTIOPORT PortDef)
+{
+    AssertCompileSize(RTIOPORT, 2);
+    return CFGMR3QueryU16Def(pNode, pszName, pPort, PortDef);
+}
+
+
+/**
+ * Query unsigned int address value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu              Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryUInt(PCFGMNODE pNode, const char *pszName, unsigned int *pu)
+{
+    AssertCompileSize(unsigned int, 4);
+    return CFGMR3QueryU32(pNode, pszName, (uint32_t *)pu);
+}
+
+
+/**
+ * Query unsigned int address value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pu              Where to store the value. Set to default on failure.
+ * @param   uDef            The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryUIntDef(PCFGMNODE pNode, const char *pszName, unsigned int *pu, unsigned int uDef)
+{
+    AssertCompileSize(unsigned int, 4);
+    return CFGMR3QueryU32Def(pNode, pszName, (uint32_t *)pu, uDef);
+}
+
+
+/**
+ * Query signed int address value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi              Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QuerySInt(PCFGMNODE pNode, const char *pszName, signed int *pi)
+{
+    AssertCompileSize(signed int, 4);
+    return CFGMR3QueryS32(pNode, pszName, (int32_t *)pi);
+}
+
+
+/**
+ * Query unsigned int address value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pi              Where to store the value. Set to default on failure.
+ * @param   iDef            The default value.
+ */
+VMMR3DECL(int) CFGMR3QuerySIntDef(PCFGMNODE pNode, const char *pszName, signed int *pi, signed int iDef)
+{
+    AssertCompileSize(signed int, 4);
+    return CFGMR3QueryS32Def(pNode, pszName, (int32_t *)pi, iDef);
+}
+
+
+/**
+ * Query pointer integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   ppv             Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryPtr(PCFGMNODE pNode, const char *pszName, void **ppv)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        uintptr_t u = (uintptr_t)u64;
+        if (u64 == u)
+            *ppv = (void *)u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query pointer integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   ppv             Where to store the value. Set to default on failure.
+ * @param   pvDef           The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryPtrDef(PCFGMNODE pNode, const char *pszName, void **ppv, void *pvDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, (uintptr_t)pvDef);
+    if (RT_SUCCESS(rc))
+    {
+        uintptr_t u = (uintptr_t)u64;
+        if (u64 == u)
+            *ppv = (void *)u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *ppv = pvDef;
+    return rc;
+}
+
+
+/**
+ * Query Guest Context pointer integer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtr(PCFGMNODE pNode, const char *pszName, PRTGCPTR pGCPtr)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCPTR u = (RTGCPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query Guest Context pointer integer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value. Set to default on failure.
+ * @param   GCPtrDef        The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrDef(PCFGMNODE pNode, const char *pszName, PRTGCPTR pGCPtr, RTGCPTR GCPtrDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCPTR u = (RTGCPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pGCPtr = GCPtrDef;
+    return rc;
+}
+
+
+/**
+ * Query Guest Context unsigned pointer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrU(PCFGMNODE pNode, const char *pszName, PRTGCUINTPTR pGCPtr)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCUINTPTR u = (RTGCUINTPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query Guest Context unsigned pointer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value. Set to default on failure.
+ * @param   GCPtrDef        The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrUDef(PCFGMNODE pNode, const char *pszName, PRTGCUINTPTR pGCPtr, RTGCUINTPTR GCPtrDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCUINTPTR u = (RTGCUINTPTR)u64;
+        if (u64 == u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pGCPtr = GCPtrDef;
+    return rc;
+}
+
+
+/**
+ * Query Guest Context signed pointer value.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrS(PCFGMNODE pNode, const char *pszName, PRTGCINTPTR pGCPtr)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryInteger(pNode, pszName, &u64);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCINTPTR u = (RTGCINTPTR)u64;
+        if (u64 == (uint64_t)u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    return rc;
+}
+
+
+/**
+ * Query Guest Context signed pointer value with default.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Name of an integer value.
+ * @param   pGCPtr          Where to store the value. Set to default on failure.
+ * @param   GCPtrDef        The default value.
+ */
+VMMR3DECL(int) CFGMR3QueryGCPtrSDef(PCFGMNODE pNode, const char *pszName, PRTGCINTPTR pGCPtr, RTGCINTPTR GCPtrDef)
+{
+    uint64_t u64;
+    int rc = CFGMR3QueryIntegerDef(pNode, pszName, &u64, GCPtrDef);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCINTPTR u = (RTGCINTPTR)u64;
+        if (u64 == (uint64_t)u)
+            *pGCPtr = u;
+        else
+            rc = VERR_CFGM_INTEGER_TOO_BIG;
+    }
+    if (RT_FAILURE(rc))
+        *pGCPtr = GCPtrDef;
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value storing it in a
+ * buffer allocated from the MM heap.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Value name. This value must be of zero terminated character string type.
+ * @param   ppszString      Where to store the string pointer.
+ *                          Free this using MMR3HeapFree().
+ */
+VMMR3DECL(int) CFGMR3QueryStringAlloc(PCFGMNODE pNode, const char *pszName, char **ppszString)
+{
+    size_t cbString;
+    int rc = CFGMR3QuerySize(pNode, pszName, &cbString);
+    if (RT_SUCCESS(rc))
+    {
+        char *pszString = (char *)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_USER, cbString);
+        if (pszString)
+        {
+            rc = CFGMR3QueryString(pNode, pszName, pszString, cbString);
+            if (RT_SUCCESS(rc))
+                *ppszString = pszString;
+            else
+                MMR3HeapFree(pszString);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+    return rc;
+}
+
+
+/**
+ * Query zero terminated character value storing it in a
+ * buffer allocated from the MM heap.
+ *
+ * @returns VBox status code.
+ * @param   pNode           Which node to search for pszName in.
+ * @param   pszName         Value name. This value must be of zero terminated character string type.
+ * @param   ppszString      Where to store the string pointer. Not set on failure.
+ *                          Free this using MMR3HeapFree().
+ * @param   pszDef          The default return value.  This can be NULL.
+ */
+VMMR3DECL(int) CFGMR3QueryStringAllocDef(PCFGMNODE pNode, const char *pszName, char **ppszString, const char *pszDef)
+{
+    /*
+     * (Don't call CFGMR3QuerySize and CFGMR3QueryStringDef here as the latter
+     * cannot handle pszDef being NULL.)
+     */
+    PCFGMLEAF pLeaf;
+    int rc = cfgmR3ResolveLeaf(pNode, pszName, &pLeaf);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLeaf->enmType == CFGMVALUETYPE_STRING)
+        {
+            size_t const cbSrc = pLeaf->Value.String.cb;
+            char *pszString = (char *)MMR3HeapAlloc(pNode->pVM, MM_TAG_CFGM_USER, cbSrc);
+            if (pszString)
+            {
+                memcpy(pszString, pLeaf->Value.String.psz, cbSrc);
+                *ppszString = pszString;
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            rc = VERR_CFGM_NOT_STRING;
+    }
+    if (RT_FAILURE(rc))
+    {
+        if (!pszDef)
+            *ppszString = NULL;
+        else
+            *ppszString = MMR3HeapStrDup(pNode->pVM, MM_TAG_CFGM_USER, pszDef);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Dumps the configuration (sub)tree to the release log.
+ *
+ * @param   pRoot   The root node of the dump.
+ */
+VMMR3DECL(void) CFGMR3Dump(PCFGMNODE pRoot)
+{
+    LogRel(("************************* CFGM dump *************************\n"));
+    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+    cfgmR3Dump(pRoot, 0, DBGFR3InfoLogRelHlp());
+    RTLogRelSetBuffering(fOldBuffered);
+    LogRel(("********************* End of CFGM dump **********************\n"));
+}
+
+
+/**
+ * Info handler, internal version.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pHlp        Callback functions for doing output.
+ * @param   pszArgs     Argument string. Optional and specific to the handler.
+ */
+static DECLCALLBACK(void) cfgmR3Info(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /*
+     * Figure where to start.
+     */
+    PCFGMNODE pRoot = pVM->cfgm.s.pRoot;
+    if (pszArgs && *pszArgs)
+    {
+        int rc = cfgmR3ResolveNode(pRoot, pszArgs, &pRoot);
+        if (RT_FAILURE(rc))
+        {
+            pHlp->pfnPrintf(pHlp, "Failed to resolve CFGM path '%s', %Rrc", pszArgs, rc);
+            return;
+        }
+    }
+
+    /*
+     * Dump the specified tree.
+     */
+    pHlp->pfnPrintf(pHlp, "pRoot=%p:{", pRoot);
+    cfgmR3DumpPath(pRoot, pHlp);
+    pHlp->pfnPrintf(pHlp, "}\n");
+    cfgmR3Dump(pRoot, 0, pHlp);
+}
+
+
+/**
+ * Recursively prints a path name.
+ */
+static void cfgmR3DumpPath(PCFGMNODE pNode, PCDBGFINFOHLP pHlp)
+{
+    if (pNode->pParent)
+        cfgmR3DumpPath(pNode->pParent, pHlp);
+    pHlp->pfnPrintf(pHlp, "%s/", pNode->szName);
+}
+
+
+/**
+ * Dumps a branch of a tree.
+ */
+static void cfgmR3Dump(PCFGMNODE pRoot, unsigned iLevel, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Path.
+     */
+    pHlp->pfnPrintf(pHlp, "[");
+    cfgmR3DumpPath(pRoot, pHlp);
+    pHlp->pfnPrintf(pHlp, "] (level %d)%s\n", iLevel, pRoot->fRestrictedRoot ? " (restricted root)" : "");
+
+    /*
+     * Values.
+     */
+    PCFGMLEAF pLeaf;
+    size_t cchMax = 0;
+    for (pLeaf = CFGMR3GetFirstValue(pRoot); pLeaf; pLeaf = CFGMR3GetNextValue(pLeaf))
+        cchMax = RT_MAX(cchMax, pLeaf->cchName);
+    for (pLeaf = CFGMR3GetFirstValue(pRoot); pLeaf; pLeaf = CFGMR3GetNextValue(pLeaf))
+    {
+        switch (CFGMR3GetValueType(pLeaf))
+        {
+            case CFGMVALUETYPE_INTEGER:
+                pHlp->pfnPrintf(pHlp, "  %-*s <integer> = %#018llx (%lld)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.Integer.u64, pLeaf->Value.Integer.u64);
+                break;
+
+            case CFGMVALUETYPE_STRING:
+                pHlp->pfnPrintf(pHlp, "  %-*s <string>  = \"%s\" (cb=%zu)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.String.psz, pLeaf->Value.String.cb);
+                break;
+
+            case CFGMVALUETYPE_BYTES:
+                pHlp->pfnPrintf(pHlp, "  %-*s <bytes>   = \"%.*Rhxs\" (cb=%zu)\n", (int)cchMax, pLeaf->szName, pLeaf->Value.Bytes.cb, pLeaf->Value.Bytes.pau8, pLeaf->Value.Bytes.cb);
+                break;
+
+            default:
+                AssertMsgFailed(("bad leaf!\n"));
+                break;
+        }
+    }
+    pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * Children.
+     */
+    for (PCFGMNODE pChild = CFGMR3GetFirstChild(pRoot); pChild; pChild = CFGMR3GetNextChild(pChild))
+    {
+        Assert(pChild->pNext != pChild);
+        Assert(pChild->pPrev != pChild);
+        Assert(pChild->pPrev != pChild->pNext || !pChild->pPrev);
+        Assert(pChild->pFirstChild != pChild);
+        Assert(pChild->pParent != pChild);
+        cfgmR3Dump(pChild, iLevel + 1, pHlp);
+    }
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/CPUM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/CPUM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/CPUM.cpp	(revision 35333)
@@ -0,0 +1,3981 @@
+/* $Id$ */
+/** @file
+ * CPUM - CPU Monitor / Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_cpum CPUM - CPU Monitor / Manager
+ *
+ * The CPU Monitor / Manager keeps track of all the CPU registers. It is
+ * also responsible for lazy FPU handling and some of the context loading
+ * in raw mode.
+ *
+ * There are three CPU contexts, the most important one is the guest one (GC).
+ * When running in raw-mode (RC) there is a special hyper context for the VMM
+ * part that floats around inside the guest address space. When running in
+ * raw-mode, CPUM also maintains a host context for saving and restoring
+ * registers across world switches. This latter is done in cooperation with the
+ * world switcher (@see pg_vmm).
+ *
+ * @see grp_cpum
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/cpum.h>
+#include <VBox/cpumdis.h>
+#include <VBox/pgm.h>
+#include <VBox/mm.h>
+#include <VBox/selm.h>
+#include <VBox/dbgf.h>
+#include <VBox/patm.h>
+#include <VBox/hwaccm.h>
+#include <VBox/ssm.h>
+#include "CPUMInternal.h"
+#include <VBox/vm.h>
+
+#include <VBox/param.h>
+#include <VBox/dis.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+#include <iprt/mp.h>
+#include <iprt/cpuset.h>
+#include "internal/pgm.h"
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The current saved state version. */
+#define CPUM_SAVED_STATE_VERSION                12
+/** The saved state version of 3.2, 3.1 and 3.3 trunk before the hidden
+ * selector register change (CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID). */
+#define CPUM_SAVED_STATE_VERSION_VER3_2         11
+/** The saved state version of 3.0 and 3.1 trunk before the teleportation
+ * changes. */
+#define CPUM_SAVED_STATE_VERSION_VER3_0         10
+/** The saved state version for the 2.1 trunk before the MSR changes. */
+#define CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR   9
+/** The saved state version of 2.0, used for backwards compatibility. */
+#define CPUM_SAVED_STATE_VERSION_VER2_0         8
+/** The saved state version of 1.6, used for backwards compatibility. */
+#define CPUM_SAVED_STATE_VERSION_VER1_6         6
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+
+/**
+ * What kind of cpu info dump to perform.
+ */
+typedef enum CPUMDUMPTYPE
+{
+    CPUMDUMPTYPE_TERSE,
+    CPUMDUMPTYPE_DEFAULT,
+    CPUMDUMPTYPE_VERBOSE
+} CPUMDUMPTYPE;
+/** Pointer to a cpu info dump type. */
+typedef CPUMDUMPTYPE *PCPUMDUMPTYPE;
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static CPUMCPUVENDOR cpumR3DetectVendor(uint32_t uEAX, uint32_t uEBX, uint32_t uECX, uint32_t uEDX);
+static int cpumR3CpuIdInit(PVM pVM);
+static DECLCALLBACK(int)  cpumR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
+static DECLCALLBACK(int)  cpumR3SaveExec(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)  cpumR3LoadPrep(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)  cpumR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int)  cpumR3LoadDone(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(void) cpumR3InfoAll(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoGuestInstr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoHyper(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3InfoHost(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) cpumR3CpuIdInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/**
+ * Initializes the CPUM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) CPUMR3Init(PVM pVM)
+{
+    LogFlow(("CPUMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, cpum.s, 32);
+    AssertCompile(sizeof(pVM->cpum.s) <= sizeof(pVM->cpum.padding));
+    AssertCompileSizeAlignment(CPUMCTX, 64);
+    AssertCompileSizeAlignment(CPUMCTXMSR, 64);
+    AssertCompileSizeAlignment(CPUMHOSTCTX, 64);
+    AssertCompileMemberAlignment(VM, cpum, 64);
+    AssertCompileMemberAlignment(VM, aCpus, 64);
+    AssertCompileMemberAlignment(VMCPU, cpum.s, 64);
+    AssertCompileMemberSizeAlignment(VM, aCpus[0].cpum.s, 64);
+
+    /* Calculate the offset from CPUM to CPUMCPU for the first CPU. */
+    pVM->cpum.s.offCPUMCPU0 = RT_OFFSETOF(VM, aCpus[0].cpum) - RT_OFFSETOF(VM, cpum);
+    Assert((uintptr_t)&pVM->cpum + pVM->cpum.s.offCPUMCPU0 == (uintptr_t)&pVM->aCpus[0].cpum);
+
+    /* Calculate the offset from CPUMCPU to CPUM. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        /*
+         * Setup any fixed pointers and offsets.
+         */
+        pVCpu->cpum.s.pHyperCoreR3 = CPUMCTX2CORE(&pVCpu->cpum.s.Hyper);
+        pVCpu->cpum.s.pHyperCoreR0 = VM_R0_ADDR(pVM, CPUMCTX2CORE(&pVCpu->cpum.s.Hyper));
+
+        pVCpu->cpum.s.offCPUM      = RT_OFFSETOF(VM, aCpus[i].cpum) - RT_OFFSETOF(VM, cpum);
+        Assert((uintptr_t)&pVCpu->cpum - pVCpu->cpum.s.offCPUM == (uintptr_t)&pVM->cpum);
+    }
+
+    /*
+     * Check that the CPU supports the minimum features we require.
+     */
+    if (!ASMHasCpuId())
+    {
+        Log(("The CPU doesn't support CPUID!\n"));
+        return VERR_UNSUPPORTED_CPU;
+    }
+    ASMCpuId_ECX_EDX(1, &pVM->cpum.s.CPUFeatures.ecx, &pVM->cpum.s.CPUFeatures.edx);
+    ASMCpuId_ECX_EDX(0x80000001, &pVM->cpum.s.CPUFeaturesExt.ecx, &pVM->cpum.s.CPUFeaturesExt.edx);
+
+    /* Setup the CR4 AND and OR masks used in the switcher */
+    /* Depends on the presence of FXSAVE(SSE) support on the host CPU */
+    if (!pVM->cpum.s.CPUFeatures.edx.u1FXSR)
+    {
+        Log(("The CPU doesn't support FXSAVE/FXRSTOR!\n"));
+        /* No FXSAVE implies no SSE */
+        pVM->cpum.s.CR4.AndMask = X86_CR4_PVI | X86_CR4_VME;
+        pVM->cpum.s.CR4.OrMask  = 0;
+    }
+    else
+    {
+        pVM->cpum.s.CR4.AndMask = X86_CR4_OSXMMEEXCPT | X86_CR4_PVI | X86_CR4_VME;
+        pVM->cpum.s.CR4.OrMask  = X86_CR4_OSFSXR;
+    }
+
+    if (!pVM->cpum.s.CPUFeatures.edx.u1MMX)
+    {
+        Log(("The CPU doesn't support MMX!\n"));
+        return VERR_UNSUPPORTED_CPU;
+    }
+    if (!pVM->cpum.s.CPUFeatures.edx.u1TSC)
+    {
+        Log(("The CPU doesn't support TSC!\n"));
+        return VERR_UNSUPPORTED_CPU;
+    }
+    /* Bogus on AMD? */
+    if (!pVM->cpum.s.CPUFeatures.edx.u1SEP)
+        Log(("The CPU doesn't support SYSENTER/SYSEXIT!\n"));
+
+    /*
+     * Detect the host CPU vendor.
+     * (The guest CPU vendor is re-detected later on.)
+     */
+    uint32_t uEAX, uEBX, uECX, uEDX;
+    ASMCpuId(0, &uEAX, &uEBX, &uECX, &uEDX);
+    pVM->cpum.s.enmHostCpuVendor = cpumR3DetectVendor(uEAX, uEBX, uECX, uEDX);
+    pVM->cpum.s.enmGuestCpuVendor = pVM->cpum.s.enmHostCpuVendor;
+
+    /*
+     * Setup hypervisor startup values.
+     */
+
+    /*
+     * Register saved state data item.
+     */
+    int rc = SSMR3RegisterInternal(pVM, "cpum", 1, CPUM_SAVED_STATE_VERSION, sizeof(CPUM),
+                                   NULL, cpumR3LiveExec, NULL,
+                                   NULL, cpumR3SaveExec, NULL,
+                                   cpumR3LoadPrep, cpumR3LoadExec, cpumR3LoadDone);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register info handlers.
+     */
+    DBGFR3InfoRegisterInternal(pVM, "cpum",             "Displays the all the cpu states.",         &cpumR3InfoAll);
+    DBGFR3InfoRegisterInternal(pVM, "cpumguest",        "Displays the guest cpu state.",            &cpumR3InfoGuest);
+    DBGFR3InfoRegisterInternal(pVM, "cpumhyper",        "Displays the hypervisor cpu state.",       &cpumR3InfoHyper);
+    DBGFR3InfoRegisterInternal(pVM, "cpumhost",         "Displays the host cpu state.",             &cpumR3InfoHost);
+    DBGFR3InfoRegisterInternal(pVM, "cpuid",            "Displays the guest cpuid leaves.",         &cpumR3CpuIdInfo);
+    DBGFR3InfoRegisterInternal(pVM, "cpumguestinstr",   "Displays the current guest instruction.",  &cpumR3InfoGuestInstr);
+
+    /*
+     * Initialize the Guest CPUID state.
+     */
+    rc = cpumR3CpuIdInit(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+    CPUMR3Reset(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Detect the CPU vendor give n the
+ *
+ * @returns The vendor.
+ * @param   uEAX                EAX from CPUID(0).
+ * @param   uEBX                EBX from CPUID(0).
+ * @param   uECX                ECX from CPUID(0).
+ * @param   uEDX                EDX from CPUID(0).
+ */
+static CPUMCPUVENDOR cpumR3DetectVendor(uint32_t uEAX, uint32_t uEBX, uint32_t uECX, uint32_t uEDX)
+{
+    if (    uEAX >= 1
+        &&  uEBX == X86_CPUID_VENDOR_AMD_EBX
+        &&  uECX == X86_CPUID_VENDOR_AMD_ECX
+        &&  uEDX == X86_CPUID_VENDOR_AMD_EDX)
+        return CPUMCPUVENDOR_AMD;
+
+    if (    uEAX >= 1
+        &&  uEBX == X86_CPUID_VENDOR_INTEL_EBX
+        &&  uECX == X86_CPUID_VENDOR_INTEL_ECX
+        &&  uEDX == X86_CPUID_VENDOR_INTEL_EDX)
+        return CPUMCPUVENDOR_INTEL;
+
+    /** @todo detect the other buggers... */
+    return CPUMCPUVENDOR_UNKNOWN;
+}
+
+
+/**
+ * Fetches overrides for a CPUID leaf.
+ *
+ * @returns VBox status code.
+ * @param   pLeaf               The leaf to load the overrides into.
+ * @param   pCfgNode            The CFGM node containing the overrides
+ *                              (/CPUM/HostCPUID/ or /CPUM/CPUID/).
+ * @param   iLeaf               The CPUID leaf number.
+ */
+static int cpumR3CpuIdFetchLeafOverride(PCPUMCPUID pLeaf, PCFGMNODE pCfgNode, uint32_t iLeaf)
+{
+    PCFGMNODE pLeafNode = CFGMR3GetChildF(pCfgNode, "%RX32", iLeaf);
+    if (pLeafNode)
+    {
+        uint32_t u32;
+        int rc = CFGMR3QueryU32(pLeafNode, "eax", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->eax = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+        rc = CFGMR3QueryU32(pLeafNode, "ebx", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->ebx = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+        rc = CFGMR3QueryU32(pLeafNode, "ecx", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->ecx = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+        rc = CFGMR3QueryU32(pLeafNode, "edx", &u32);
+        if (RT_SUCCESS(rc))
+            pLeaf->edx = u32;
+        else
+            AssertReturn(rc == VERR_CFGM_VALUE_NOT_FOUND, rc);
+
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Load the overrides for a set of CPUID leaves.
+ *
+ * @returns VBox status code.
+ * @param   paLeaves            The leaf array.
+ * @param   cLeaves             The number of leaves.
+ * @param   uStart              The start leaf number.
+ * @param   pCfgNode            The CFGM node containing the overrides
+ *                              (/CPUM/HostCPUID/ or /CPUM/CPUID/).
+ */
+static int cpumR3CpuIdInitLoadOverrideSet(uint32_t uStart, PCPUMCPUID paLeaves, uint32_t cLeaves, PCFGMNODE pCfgNode)
+{
+    for (uint32_t i = 0; i < cLeaves; i++)
+    {
+        int rc = cpumR3CpuIdFetchLeafOverride(&paLeaves[i], pCfgNode, uStart + i);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Init a set of host CPUID leaves.
+ *
+ * @returns VBox status code.
+ * @param   paLeaves            The leaf array.
+ * @param   cLeaves             The number of leaves.
+ * @param   uStart              The start leaf number.
+ * @param   pCfgNode            The /CPUM/HostCPUID/ node.
+ */
+static int cpumR3CpuIdInitHostSet(uint32_t uStart, PCPUMCPUID paLeaves, uint32_t cLeaves, PCFGMNODE pCfgNode)
+{
+    /* Using the ECX variant for all of them can't hurt... */
+    for (uint32_t i = 0; i < cLeaves; i++)
+        ASMCpuId_Idx_ECX(uStart + i, 0, &paLeaves[i].eax, &paLeaves[i].ebx, &paLeaves[i].ecx, &paLeaves[i].edx);
+
+    /* Load CPUID leaf override; we currently don't care if the user
+       specifies features the host CPU doesn't support. */
+    return cpumR3CpuIdInitLoadOverrideSet(uStart, paLeaves, cLeaves, pCfgNode);
+}
+
+
+/**
+ * Initializes the emulated CPU's cpuid information.
+ *
+ * @returns VBox status code.
+ * @param   pVM          The VM to operate on.
+ */
+static int cpumR3CpuIdInit(PVM pVM)
+{
+    PCPUM       pCPUM    = &pVM->cpum.s;
+    PCFGMNODE   pCpumCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM");
+    uint32_t    i;
+    int         rc;
+
+#define PORTABLE_CLEAR_BITS_WHEN(Lvl, LeafSuffReg, FeatNm, fMask, uValue) \
+    if (pCPUM->u8PortableCpuIdLevel >= (Lvl) && (pCPUM->aGuestCpuId##LeafSuffReg & (fMask)) == (uValue) ) \
+    { \
+        LogRel(("PortableCpuId: " #LeafSuffReg "[" #FeatNm "]: %#x -> 0\n", pCPUM->aGuestCpuId##LeafSuffReg & (fMask))); \
+        pCPUM->aGuestCpuId##LeafSuffReg &= ~(uint32_t)(fMask); \
+    }
+#define PORTABLE_DISABLE_FEATURE_BIT(Lvl, LeafSuffReg, FeatNm, fBitMask) \
+    if (pCPUM->u8PortableCpuIdLevel >= (Lvl) && (pCPUM->aGuestCpuId##LeafSuffReg & (fBitMask)) ) \
+    { \
+        LogRel(("PortableCpuId: " #LeafSuffReg "[" #FeatNm "]: 1 -> 0\n")); \
+        pCPUM->aGuestCpuId##LeafSuffReg &= ~(uint32_t)(fBitMask); \
+    }
+
+    /*
+     * Read the configuration.
+     */
+    /** @cfgm{CPUM/SyntheticCpu, boolean, false}
+     * Enables the Synthetic CPU.  The Vendor ID and Processor Name are
+     * completely overridden by VirtualBox custom strings.  Some
+     * CPUID information is withheld, like the cache info. */
+    rc = CFGMR3QueryBoolDef(pCpumCfg, "SyntheticCpu",  &pCPUM->fSyntheticCpu,  false);
+    AssertRCReturn(rc, rc);
+
+    /** @cfgm{CPUM/PortableCpuIdLevel, 8-bit, 0, 3, 0}
+     * When non-zero CPUID features that could cause portability issues will be
+     * stripped.  The higher the value the more features gets stripped.  Higher
+     * values should only be used when older CPUs are involved since it may
+     * harm performance and maybe also cause problems with specific guests. */
+    rc = CFGMR3QueryU8Def(pCpumCfg, "PortableCpuIdLevel", &pCPUM->u8PortableCpuIdLevel, 0);
+    AssertRCReturn(rc, rc);
+
+    AssertLogRelReturn(!pCPUM->fSyntheticCpu || !pCPUM->u8PortableCpuIdLevel, VERR_INTERNAL_ERROR_2);
+
+    /*
+     * Get the host CPUID leaves and redetect the guest CPU vendor (could've
+     * been overridden).
+     */
+    /** @cfgm{CPUM/HostCPUID/[000000xx|800000xx|c000000x]/[eax|ebx|ecx|edx],32-bit}
+     * Overrides the host CPUID leaf values used for calculating the guest CPUID
+     * leaves.  This can be used to preserve the CPUID values when moving a VM
+     * to a different machine.  Another use is restricting (or extending) the
+     * feature set exposed to the guest. */
+    PCFGMNODE pHostOverrideCfg = CFGMR3GetChild(pCpumCfg, "HostCPUID");
+    rc = cpumR3CpuIdInitHostSet(UINT32_C(0x00000000), &pCPUM->aGuestCpuIdStd[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdStd),     pHostOverrideCfg);
+    AssertRCReturn(rc, rc);
+    rc = cpumR3CpuIdInitHostSet(UINT32_C(0x80000000), &pCPUM->aGuestCpuIdExt[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdExt),     pHostOverrideCfg);
+    AssertRCReturn(rc, rc);
+    rc = cpumR3CpuIdInitHostSet(UINT32_C(0xc0000000), &pCPUM->aGuestCpuIdCentaur[0], RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur), pHostOverrideCfg);
+    AssertRCReturn(rc, rc);
+
+    pCPUM->enmGuestCpuVendor = cpumR3DetectVendor(pCPUM->aGuestCpuIdStd[0].eax, pCPUM->aGuestCpuIdStd[0].ebx,
+                                                  pCPUM->aGuestCpuIdStd[0].ecx, pCPUM->aGuestCpuIdStd[0].edx);
+
+    /*
+     * Determine the default leaf.
+     *
+     * Intel returns values of the highest standard function, while AMD
+     * returns zeros. VIA on the other hand seems to returning nothing or
+     * perhaps some random garbage, we don't try to duplicate this behavior.
+     */
+    ASMCpuId(pCPUM->aGuestCpuIdStd[0].eax + 10, /** @todo r=bird: Use the host value here in case of overrides and more than 10 leaves being stripped already. */
+             &pCPUM->GuestCpuIdDef.eax, &pCPUM->GuestCpuIdDef.ebx,
+             &pCPUM->GuestCpuIdDef.ecx, &pCPUM->GuestCpuIdDef.edx);
+
+
+    /* Cpuid 1 & 0x80000001:
+     * Only report features we can support.
+     *
+     * Note! When enabling new features the Synthetic CPU and Portable CPUID
+     *       options may require adjusting (i.e. stripping what was enabled).
+     */
+    pCPUM->aGuestCpuIdStd[1].edx &= X86_CPUID_FEATURE_EDX_FPU
+                                  | X86_CPUID_FEATURE_EDX_VME
+                                  | X86_CPUID_FEATURE_EDX_DE
+                                  | X86_CPUID_FEATURE_EDX_PSE
+                                  | X86_CPUID_FEATURE_EDX_TSC
+                                  | X86_CPUID_FEATURE_EDX_MSR
+                                  //| X86_CPUID_FEATURE_EDX_PAE   - set later if configured.
+                                  | X86_CPUID_FEATURE_EDX_MCE
+                                  | X86_CPUID_FEATURE_EDX_CX8
+                                  //| X86_CPUID_FEATURE_EDX_APIC  - set by the APIC device if present.
+                                  /* Note! we don't report sysenter/sysexit support due to our inability to keep the IOPL part of eflags in sync while in ring 1 (see #1757) */
+                                  //| X86_CPUID_FEATURE_EDX_SEP
+                                  | X86_CPUID_FEATURE_EDX_MTRR
+                                  | X86_CPUID_FEATURE_EDX_PGE
+                                  | X86_CPUID_FEATURE_EDX_MCA
+                                  | X86_CPUID_FEATURE_EDX_CMOV
+                                  | X86_CPUID_FEATURE_EDX_PAT
+                                  | X86_CPUID_FEATURE_EDX_PSE36
+                                  //| X86_CPUID_FEATURE_EDX_PSN   - no serial number.
+                                  | X86_CPUID_FEATURE_EDX_CLFSH
+                                  //| X86_CPUID_FEATURE_EDX_DS    - no debug store.
+                                  //| X86_CPUID_FEATURE_EDX_ACPI  - not virtualized yet.
+                                  | X86_CPUID_FEATURE_EDX_MMX
+                                  | X86_CPUID_FEATURE_EDX_FXSR
+                                  | X86_CPUID_FEATURE_EDX_SSE
+                                  | X86_CPUID_FEATURE_EDX_SSE2
+                                  //| X86_CPUID_FEATURE_EDX_SS    - no self snoop.
+                                  //| X86_CPUID_FEATURE_EDX_HTT   - no hyperthreading.
+                                  //| X86_CPUID_FEATURE_EDX_TM    - no thermal monitor.
+                                  //| X86_CPUID_FEATURE_EDX_PBE   - no pending break enabled.
+                                  | 0;
+    pCPUM->aGuestCpuIdStd[1].ecx &= 0
+                                  | X86_CPUID_FEATURE_ECX_SSE3
+                                  /* Can't properly emulate monitor & mwait with guest SMP; force the guest to use hlt for idling VCPUs. */
+                                  | ((pVM->cCpus == 1) ? X86_CPUID_FEATURE_ECX_MONITOR : 0)
+                                  //| X86_CPUID_FEATURE_ECX_CPLDS - no CPL qualified debug store.
+                                  //| X86_CPUID_FEATURE_ECX_VMX   - not virtualized.
+                                  //| X86_CPUID_FEATURE_ECX_EST   - no extended speed step.
+                                  //| X86_CPUID_FEATURE_ECX_TM2   - no thermal monitor 2.
+                                    | X86_CPUID_FEATURE_ECX_SSSE3
+                                  //| X86_CPUID_FEATURE_ECX_CNTXID - no L1 context id (MSR++).
+                                  //| X86_CPUID_FEATURE_ECX_CX16  - no cmpxchg16b
+                                  /* ECX Bit 14 - xTPR Update Control. Processor supports changing IA32_MISC_ENABLES[bit 23]. */
+                                  //| X86_CPUID_FEATURE_ECX_TPRUPDATE
+                                  /* ECX Bit 21 - x2APIC support - not yet. */
+                                  // | X86_CPUID_FEATURE_ECX_X2APIC
+                                  /* ECX Bit 23 - POPCNT instruction. */
+                                  //| X86_CPUID_FEATURE_ECX_POPCNT
+                                  | 0;
+    if (pCPUM->u8PortableCpuIdLevel > 0)
+    {
+        PORTABLE_CLEAR_BITS_WHEN(1, Std[1].eax, ProcessorType, (UINT32_C(3) << 12), (UINT32_C(2) << 12));
+        PORTABLE_DISABLE_FEATURE_BIT(1, Std[1].ecx, SSSE3, X86_CPUID_FEATURE_ECX_SSSE3);
+        PORTABLE_DISABLE_FEATURE_BIT(1, Std[1].ecx, SSE3,  X86_CPUID_FEATURE_ECX_SSE3);
+        PORTABLE_DISABLE_FEATURE_BIT(2, Std[1].edx, SSE2,  X86_CPUID_FEATURE_EDX_SSE2);
+        PORTABLE_DISABLE_FEATURE_BIT(3, Std[1].edx, SSE,   X86_CPUID_FEATURE_EDX_SSE);
+        PORTABLE_DISABLE_FEATURE_BIT(3, Std[1].edx, CLFSH, X86_CPUID_FEATURE_EDX_CLFSH);
+        PORTABLE_DISABLE_FEATURE_BIT(3, Std[1].edx, CMOV,  X86_CPUID_FEATURE_EDX_CMOV);
+
+        Assert(!(pCPUM->aGuestCpuIdStd[1].edx & (  X86_CPUID_FEATURE_EDX_SEP
+                                                 | X86_CPUID_FEATURE_EDX_PSN
+                                                 | X86_CPUID_FEATURE_EDX_DS
+                                                 | X86_CPUID_FEATURE_EDX_ACPI
+                                                 | X86_CPUID_FEATURE_EDX_SS
+                                                 | X86_CPUID_FEATURE_EDX_TM
+                                                 | X86_CPUID_FEATURE_EDX_PBE
+                                                 )));
+        Assert(!(pCPUM->aGuestCpuIdStd[1].ecx & (  X86_CPUID_FEATURE_ECX_PCLMUL
+                                                 | X86_CPUID_FEATURE_ECX_DTES64
+                                                 | X86_CPUID_FEATURE_ECX_CPLDS
+                                                 | X86_CPUID_FEATURE_ECX_VMX
+                                                 | X86_CPUID_FEATURE_ECX_SMX
+                                                 | X86_CPUID_FEATURE_ECX_EST
+                                                 | X86_CPUID_FEATURE_ECX_TM2
+                                                 | X86_CPUID_FEATURE_ECX_CNTXID
+                                                 | X86_CPUID_FEATURE_ECX_FMA
+                                                 | X86_CPUID_FEATURE_ECX_CX16
+                                                 | X86_CPUID_FEATURE_ECX_TPRUPDATE
+                                                 | X86_CPUID_FEATURE_ECX_PDCM
+                                                 | X86_CPUID_FEATURE_ECX_DCA
+                                                 | X86_CPUID_FEATURE_ECX_MOVBE
+                                                 | X86_CPUID_FEATURE_ECX_AES
+                                                 | X86_CPUID_FEATURE_ECX_POPCNT
+                                                 | X86_CPUID_FEATURE_ECX_XSAVE
+                                                 | X86_CPUID_FEATURE_ECX_OSXSAVE
+                                                 | X86_CPUID_FEATURE_ECX_AVX
+                                                 )));
+    }
+
+    /* Cpuid 0x80000001:
+     * Only report features we can support.
+     *
+     * Note! When enabling new features the Synthetic CPU and Portable CPUID
+     *       options may require adjusting (i.e. stripping what was enabled).
+     *
+     * ASSUMES that this is ALWAYS the AMD defined feature set if present.
+     */
+    pCPUM->aGuestCpuIdExt[1].edx &= X86_CPUID_AMD_FEATURE_EDX_FPU
+                                  | X86_CPUID_AMD_FEATURE_EDX_VME
+                                  | X86_CPUID_AMD_FEATURE_EDX_DE
+                                  | X86_CPUID_AMD_FEATURE_EDX_PSE
+                                  | X86_CPUID_AMD_FEATURE_EDX_TSC
+                                  | X86_CPUID_AMD_FEATURE_EDX_MSR //?? this means AMD MSRs..
+                                  //| X86_CPUID_AMD_FEATURE_EDX_PAE    - not implemented yet.
+                                  //| X86_CPUID_AMD_FEATURE_EDX_MCE    - not virtualized yet.
+                                  | X86_CPUID_AMD_FEATURE_EDX_CX8
+                                  //| X86_CPUID_AMD_FEATURE_EDX_APIC   - set by the APIC device if present.
+                                  /* Note! we don't report sysenter/sysexit support due to our inability to keep the IOPL part of eflags in sync while in ring 1 (see #1757) */
+                                  //| X86_CPUID_AMD_FEATURE_EDX_SEP
+                                  | X86_CPUID_AMD_FEATURE_EDX_MTRR
+                                  | X86_CPUID_AMD_FEATURE_EDX_PGE
+                                  | X86_CPUID_AMD_FEATURE_EDX_MCA
+                                  | X86_CPUID_AMD_FEATURE_EDX_CMOV
+                                  | X86_CPUID_AMD_FEATURE_EDX_PAT
+                                  | X86_CPUID_AMD_FEATURE_EDX_PSE36
+                                  //| X86_CPUID_AMD_FEATURE_EDX_NX     - not virtualized, requires PAE.
+                                  //| X86_CPUID_AMD_FEATURE_EDX_AXMMX
+                                  | X86_CPUID_AMD_FEATURE_EDX_MMX
+                                  | X86_CPUID_AMD_FEATURE_EDX_FXSR
+                                  | X86_CPUID_AMD_FEATURE_EDX_FFXSR
+                                  //| X86_CPUID_AMD_FEATURE_EDX_PAGE1GB
+                                  //| X86_CPUID_AMD_FEATURE_EDX_RDTSCP - AMD only; turned on when necessary
+                                  //| X86_CPUID_AMD_FEATURE_EDX_LONG_MODE - turned on when necessary
+                                  | X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX
+                                  | X86_CPUID_AMD_FEATURE_EDX_3DNOW
+                                  | 0;
+    pCPUM->aGuestCpuIdExt[1].ecx &= 0
+                                  //| X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF
+                                  //| X86_CPUID_AMD_FEATURE_ECX_CMPL
+                                  //| X86_CPUID_AMD_FEATURE_ECX_SVM    - not virtualized.
+                                  //| X86_CPUID_AMD_FEATURE_ECX_EXT_APIC
+                                  /* Note: This could prevent teleporting from AMD to Intel CPUs! */
+                                  | X86_CPUID_AMD_FEATURE_ECX_CR8L         /* expose lock mov cr0 = mov cr8 hack for guests that can use this feature to access the TPR. */
+                                  //| X86_CPUID_AMD_FEATURE_ECX_ABM
+                                  //| X86_CPUID_AMD_FEATURE_ECX_SSE4A
+                                  //| X86_CPUID_AMD_FEATURE_ECX_MISALNSSE
+                                  //| X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF
+                                  //| X86_CPUID_AMD_FEATURE_ECX_OSVW
+                                  //| X86_CPUID_AMD_FEATURE_ECX_IBS
+                                  //| X86_CPUID_AMD_FEATURE_ECX_SSE5
+                                  //| X86_CPUID_AMD_FEATURE_ECX_SKINIT
+                                  //| X86_CPUID_AMD_FEATURE_ECX_WDT
+                                  | 0;
+    if (pCPUM->u8PortableCpuIdLevel > 0)
+    {
+        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].ecx, CR8L,       X86_CPUID_AMD_FEATURE_ECX_CR8L);
+        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, 3DNOW,      X86_CPUID_AMD_FEATURE_EDX_3DNOW);
+        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, 3DNOW_EX,   X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
+        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, FFXSR,      X86_CPUID_AMD_FEATURE_EDX_FFXSR);
+        PORTABLE_DISABLE_FEATURE_BIT(1, Ext[1].edx, RDTSCP,     X86_CPUID_AMD_FEATURE_EDX_RDTSCP);
+        PORTABLE_DISABLE_FEATURE_BIT(2, Ext[1].ecx, LAHF_SAHF,  X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF);
+        PORTABLE_DISABLE_FEATURE_BIT(3, Ext[1].ecx, CMOV,       X86_CPUID_AMD_FEATURE_EDX_CMOV);
+
+        Assert(!(pCPUM->aGuestCpuIdExt[1].ecx & (  X86_CPUID_AMD_FEATURE_ECX_CMPL
+                                                 | X86_CPUID_AMD_FEATURE_ECX_SVM
+                                                 | X86_CPUID_AMD_FEATURE_ECX_EXT_APIC
+                                                 | X86_CPUID_AMD_FEATURE_ECX_CR8L
+                                                 | X86_CPUID_AMD_FEATURE_ECX_ABM
+                                                 | X86_CPUID_AMD_FEATURE_ECX_SSE4A
+                                                 | X86_CPUID_AMD_FEATURE_ECX_MISALNSSE
+                                                 | X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF
+                                                 | X86_CPUID_AMD_FEATURE_ECX_OSVW
+                                                 | X86_CPUID_AMD_FEATURE_ECX_IBS
+                                                 | X86_CPUID_AMD_FEATURE_ECX_SSE5
+                                                 | X86_CPUID_AMD_FEATURE_ECX_SKINIT
+                                                 | X86_CPUID_AMD_FEATURE_ECX_WDT
+                                                 | UINT32_C(0xffffc000)
+                                                 )));
+        Assert(!(pCPUM->aGuestCpuIdExt[1].edx & (  RT_BIT(10)
+                                                 | X86_CPUID_AMD_FEATURE_EDX_SEP
+                                                 | RT_BIT(18)
+                                                 | RT_BIT(19)
+                                                 | RT_BIT(21)
+                                                 | X86_CPUID_AMD_FEATURE_EDX_AXMMX
+                                                 | X86_CPUID_AMD_FEATURE_EDX_PAGE1GB
+                                                 | RT_BIT(28)
+                                                 )));
+    }
+
+    /*
+     * Apply the Synthetic CPU modifications. (TODO: move this up)
+     */
+    if (pCPUM->fSyntheticCpu)
+    {
+        static const char s_szVendor[13]    = "VirtualBox  ";
+        static const char s_szProcessor[48] = "VirtualBox SPARCx86 Processor v1000            "; /* includes null terminator */
+
+        pCPUM->enmGuestCpuVendor = CPUMCPUVENDOR_SYNTHETIC;
+
+        /* Limit the nr of standard leaves; 5 for monitor/mwait */
+        pCPUM->aGuestCpuIdStd[0].eax = RT_MIN(pCPUM->aGuestCpuIdStd[0].eax, 5);
+
+        /* 0: Vendor */
+        pCPUM->aGuestCpuIdStd[0].ebx = pCPUM->aGuestCpuIdExt[0].ebx = ((uint32_t *)s_szVendor)[0];
+        pCPUM->aGuestCpuIdStd[0].ecx = pCPUM->aGuestCpuIdExt[0].ecx = ((uint32_t *)s_szVendor)[2];
+        pCPUM->aGuestCpuIdStd[0].edx = pCPUM->aGuestCpuIdExt[0].edx = ((uint32_t *)s_szVendor)[1];
+
+        /* 1.eax: Version information.  family : model : stepping */
+        pCPUM->aGuestCpuIdStd[1].eax = (0xf << 8) + (0x1 << 4) + 1;
+
+        /* Leaves 2 - 4 are Intel only - zero them out */
+        memset(&pCPUM->aGuestCpuIdStd[2], 0, sizeof(pCPUM->aGuestCpuIdStd[2]));
+        memset(&pCPUM->aGuestCpuIdStd[3], 0, sizeof(pCPUM->aGuestCpuIdStd[3]));
+        memset(&pCPUM->aGuestCpuIdStd[4], 0, sizeof(pCPUM->aGuestCpuIdStd[4]));
+
+        /* Leaf 5 = monitor/mwait */
+
+        /* Limit the nr of extended leaves: 0x80000008 to include the max virtual and physical address size (64 bits guests). */
+        pCPUM->aGuestCpuIdExt[0].eax = RT_MIN(pCPUM->aGuestCpuIdExt[0].eax, 0x80000008);
+        /* AMD only - set to zero. */
+        pCPUM->aGuestCpuIdExt[0].ebx = pCPUM->aGuestCpuIdExt[0].ecx = pCPUM->aGuestCpuIdExt[0].edx = 0;
+
+        /* 0x800000001: AMD only; shared feature bits are set dynamically. */
+        memset(&pCPUM->aGuestCpuIdExt[1], 0, sizeof(pCPUM->aGuestCpuIdExt[1]));
+
+        /* 0x800000002-4: Processor Name String Identifier. */
+        pCPUM->aGuestCpuIdExt[2].eax = ((uint32_t *)s_szProcessor)[0];
+        pCPUM->aGuestCpuIdExt[2].ebx = ((uint32_t *)s_szProcessor)[1];
+        pCPUM->aGuestCpuIdExt[2].ecx = ((uint32_t *)s_szProcessor)[2];
+        pCPUM->aGuestCpuIdExt[2].edx = ((uint32_t *)s_szProcessor)[3];
+        pCPUM->aGuestCpuIdExt[3].eax = ((uint32_t *)s_szProcessor)[4];
+        pCPUM->aGuestCpuIdExt[3].ebx = ((uint32_t *)s_szProcessor)[5];
+        pCPUM->aGuestCpuIdExt[3].ecx = ((uint32_t *)s_szProcessor)[6];
+        pCPUM->aGuestCpuIdExt[3].edx = ((uint32_t *)s_szProcessor)[7];
+        pCPUM->aGuestCpuIdExt[4].eax = ((uint32_t *)s_szProcessor)[8];
+        pCPUM->aGuestCpuIdExt[4].ebx = ((uint32_t *)s_szProcessor)[9];
+        pCPUM->aGuestCpuIdExt[4].ecx = ((uint32_t *)s_szProcessor)[10];
+        pCPUM->aGuestCpuIdExt[4].edx = ((uint32_t *)s_szProcessor)[11];
+
+        /* 0x800000005-7 - reserved -> zero */
+        memset(&pCPUM->aGuestCpuIdExt[5], 0, sizeof(pCPUM->aGuestCpuIdExt[5]));
+        memset(&pCPUM->aGuestCpuIdExt[6], 0, sizeof(pCPUM->aGuestCpuIdExt[6]));
+        memset(&pCPUM->aGuestCpuIdExt[7], 0, sizeof(pCPUM->aGuestCpuIdExt[7]));
+
+        /* 0x800000008: only the max virtual and physical address size. */
+        pCPUM->aGuestCpuIdExt[8].ecx = pCPUM->aGuestCpuIdExt[8].ebx = pCPUM->aGuestCpuIdExt[8].edx = 0;  /* reserved */
+    }
+
+    /*
+     * Hide HTT, multicode, SMP, whatever.
+     * (APIC-ID := 0 and #LogCpus := 0)
+     */
+    pCPUM->aGuestCpuIdStd[1].ebx &= 0x0000ffff;
+#ifdef VBOX_WITH_MULTI_CORE
+    if (    pCPUM->enmGuestCpuVendor != CPUMCPUVENDOR_SYNTHETIC
+        &&  pVM->cCpus > 1)
+    {
+        /* If CPUID Fn0000_0001_EDX[HTT] = 1 then LogicalProcessorCount is the number of threads per CPU core times the number of CPU cores per processor */
+        pCPUM->aGuestCpuIdStd[1].ebx |= (pVM->cCpus << 16);
+        pCPUM->aGuestCpuIdStd[1].edx |= X86_CPUID_FEATURE_EDX_HTT;  /* necessary for hyper-threading *or* multi-core CPUs */
+    }
+#endif
+
+    /* Cpuid 2:
+     * Intel: Cache and TLB information
+     * AMD:   Reserved
+     * Safe to expose; restrict the number of calls to 1 for the portable case.
+     */
+    if (    pCPUM->u8PortableCpuIdLevel > 0
+        &&  pCPUM->aGuestCpuIdStd[0].eax >= 2
+        && (pCPUM->aGuestCpuIdStd[2].eax & 0xff) > 1)
+    {
+        LogRel(("PortableCpuId: Std[2].al: %d -> 1\n", pCPUM->aGuestCpuIdStd[2].eax & 0xff));
+        pCPUM->aGuestCpuIdStd[2].eax &= UINT32_C(0xfffffffe);
+    }
+
+    /* Cpuid 3:
+     * Intel: EAX, EBX - reserved (transmeta uses these)
+     *        ECX, EDX - Processor Serial Number if available, otherwise reserved
+     * AMD:   Reserved
+     * Safe to expose
+     */
+    if (!(pCPUM->aGuestCpuIdStd[1].edx & X86_CPUID_FEATURE_EDX_PSN))
+    {
+        pCPUM->aGuestCpuIdStd[3].ecx = pCPUM->aGuestCpuIdStd[3].edx = 0;
+        if (pCPUM->u8PortableCpuIdLevel > 0)
+            pCPUM->aGuestCpuIdStd[3].eax = pCPUM->aGuestCpuIdStd[3].ebx = 0;
+    }
+
+    /* Cpuid 4:
+     * Intel: Deterministic Cache Parameters Leaf
+     *        Note: Depends on the ECX input! -> Feeling rather lazy now, so we just return 0
+     * AMD:   Reserved
+     * Safe to expose, except for EAX:
+     *      Bits 25-14: Maximum number of addressable IDs for logical processors sharing this cache (see note)**
+     *      Bits 31-26: Maximum number of processor cores in this physical package**
+     * Note: These SMP values are constant regardless of ECX
+     */
+    pCPUM->aGuestCpuIdStd[4].ecx = pCPUM->aGuestCpuIdStd[4].edx = 0;
+    pCPUM->aGuestCpuIdStd[4].eax = pCPUM->aGuestCpuIdStd[4].ebx = 0;
+#ifdef VBOX_WITH_MULTI_CORE
+    if (    pVM->cCpus > 1
+        &&  pVM->cpum.s.enmGuestCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        AssertReturn(pVM->cCpus <= 64, VERR_TOO_MANY_CPUS);
+        /* One logical processor with possibly multiple cores. */
+        /* See  http://www.intel.com/Assets/PDF/appnote/241618.pdf p. 29 */
+        pCPUM->aGuestCpuIdStd[4].eax |= ((pVM->cCpus - 1) << 26);   /* 6 bits only -> 64 cores! */
+    }
+#endif
+
+    /* Cpuid 5:     Monitor/mwait Leaf
+     * Intel: ECX, EDX - reserved
+     *        EAX, EBX - Smallest and largest monitor line size
+     * AMD:   EDX - reserved
+     *        EAX, EBX - Smallest and largest monitor line size
+     *        ECX - extensions (ignored for now)
+     * Safe to expose
+     */
+    if (!(pCPUM->aGuestCpuIdStd[1].ecx & X86_CPUID_FEATURE_ECX_MONITOR))
+        pCPUM->aGuestCpuIdStd[5].eax = pCPUM->aGuestCpuIdStd[5].ebx = 0;
+
+    pCPUM->aGuestCpuIdStd[5].ecx = pCPUM->aGuestCpuIdStd[5].edx = 0;
+    /** @cfgm{/CPUM/MWaitExtensions, boolean, false}
+     * Expose MWAIT extended features to the guest.  For now we expose
+     * just MWAIT break on interrupt feature (bit 1).
+     */
+    bool fMWaitExtensions;
+    rc = CFGMR3QueryBoolDef(pCpumCfg, "MWaitExtensions", &fMWaitExtensions, false); AssertRCReturn(rc, rc);
+    if (fMWaitExtensions)
+    {
+        pCPUM->aGuestCpuIdStd[5].ecx = X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0;
+        /* @todo: for now we just expose host's MWAIT C-states, although conceptually
+           it shall be part of our power management virtualization model */
+#if 0
+        /* MWAIT sub C-states */
+        pCPUM->aGuestCpuIdStd[5].edx =
+                (0 << 0)  /* 0 in C0 */ |
+                (2 << 4)  /* 2 in C1 */ |
+                (2 << 8)  /* 2 in C2 */ |
+                (2 << 12) /* 2 in C3 */ |
+                (0 << 16) /* 0 in C4 */
+                ;
+#endif
+    }
+    else
+        pCPUM->aGuestCpuIdStd[5].ecx = pCPUM->aGuestCpuIdStd[5].edx = 0;
+
+    /* Cpuid 0x800000005 & 0x800000006 contain information about L1, L2 & L3 cache and TLB identifiers.
+     * Safe to pass on to the guest.
+     *
+     * Intel: 0x800000005 reserved
+     *        0x800000006 L2 cache information
+     * AMD:   0x800000005 L1 cache information
+     *        0x800000006 L2/L3 cache information
+     */
+
+    /* Cpuid 0x800000007:
+     * AMD:               EAX, EBX, ECX - reserved
+     *                    EDX: Advanced Power Management Information
+     * Intel:             Reserved
+     */
+    if (pCPUM->aGuestCpuIdExt[0].eax >= UINT32_C(0x80000007))
+    {
+        Assert(pVM->cpum.s.enmGuestCpuVendor != CPUMCPUVENDOR_INVALID);
+
+        pCPUM->aGuestCpuIdExt[7].eax = pCPUM->aGuestCpuIdExt[7].ebx = pCPUM->aGuestCpuIdExt[7].ecx = 0;
+
+        if (pVM->cpum.s.enmGuestCpuVendor == CPUMCPUVENDOR_AMD)
+        {
+            /* Only expose the TSC invariant capability bit to the guest. */
+            pCPUM->aGuestCpuIdExt[7].edx    &= 0
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_TS
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_FID
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_VID
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_TTP
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_TM
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_STC
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_MC
+                                            //| X86_CPUID_AMD_ADVPOWER_EDX_HWPSTATE
+#if 0 /* We don't expose X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR, because newer
+       * Linux kernels blindly assume that the AMD performance counters work
+       * if this is set for 64 bits guests. (Can't really find a CPUID feature
+       * bit for them though.) */
+                                            | X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR
+#endif
+                                            | 0;
+        }
+        else
+            pCPUM->aGuestCpuIdExt[7].edx    = 0;
+    }
+
+    /* Cpuid 0x800000008:
+     * AMD:               EBX, EDX - reserved
+     *                    EAX: Virtual/Physical/Guest address Size
+     *                    ECX: Number of cores + APICIdCoreIdSize
+     * Intel:             EAX: Virtual/Physical address Size
+     *                    EBX, ECX, EDX - reserved
+     */
+    if (pCPUM->aGuestCpuIdExt[0].eax >= UINT32_C(0x80000008))
+    {
+        /* Only expose the virtual and physical address sizes to the guest. */
+        pCPUM->aGuestCpuIdExt[8].eax &= UINT32_C(0x0000ffff);
+        pCPUM->aGuestCpuIdExt[8].ebx = pCPUM->aGuestCpuIdExt[8].edx = 0;  /* reserved */
+        /* Set APICIdCoreIdSize to zero (use legacy method to determine the number of cores per cpu)
+         * NC (0-7) Number of cores; 0 equals 1 core */
+        pCPUM->aGuestCpuIdExt[8].ecx = 0;
+#ifdef VBOX_WITH_MULTI_CORE
+        if (    pVM->cCpus > 1
+            &&  pVM->cpum.s.enmGuestCpuVendor == CPUMCPUVENDOR_AMD)
+        {
+            /* Legacy method to determine the number of cores. */
+            pCPUM->aGuestCpuIdExt[1].ecx |= X86_CPUID_AMD_FEATURE_ECX_CMPL;
+            pCPUM->aGuestCpuIdExt[8].ecx |= (pVM->cCpus - 1); /* NC: Number of CPU cores - 1; 8 bits */
+        }
+#endif
+    }
+
+    /** @cfgm{/CPUM/NT4LeafLimit, boolean, false}
+     * Limit the number of standard CPUID leaves to 0..3 to prevent NT4 from
+     * bugchecking with MULTIPROCESSOR_CONFIGURATION_NOT_SUPPORTED (0x3e).
+     * This option corresponds somewhat to IA32_MISC_ENABLES.BOOT_NT4[bit 22].
+     */
+    bool fNt4LeafLimit;
+    rc = CFGMR3QueryBoolDef(pCpumCfg, "NT4LeafLimit", &fNt4LeafLimit, false); AssertRCReturn(rc, rc);
+    if (fNt4LeafLimit)
+        pCPUM->aGuestCpuIdStd[0].eax = 3; /** @todo r=bird: shouldn't we check if pCPUM->aGuestCpuIdStd[0].eax > 3 before setting it 3 here? */
+
+    /*
+     * Limit it the number of entries and fill the remaining with the defaults.
+     *
+     * The limits are masking off stuff about power saving and similar, this
+     * is perhaps a bit crudely done as there is probably some relatively harmless
+     * info too in these leaves (like words about having a constant TSC).
+     */
+    if (pCPUM->aGuestCpuIdStd[0].eax > 5)
+        pCPUM->aGuestCpuIdStd[0].eax = 5;
+    for (i = pCPUM->aGuestCpuIdStd[0].eax + 1; i < RT_ELEMENTS(pCPUM->aGuestCpuIdStd); i++)
+        pCPUM->aGuestCpuIdStd[i] = pCPUM->GuestCpuIdDef;
+
+    if (pCPUM->aGuestCpuIdExt[0].eax > UINT32_C(0x80000008))
+        pCPUM->aGuestCpuIdExt[0].eax = UINT32_C(0x80000008);
+    for (i = pCPUM->aGuestCpuIdExt[0].eax >= UINT32_C(0x80000000)
+           ? pCPUM->aGuestCpuIdExt[0].eax - UINT32_C(0x80000000) + 1
+           : 0;
+         i < RT_ELEMENTS(pCPUM->aGuestCpuIdExt);
+         i++)
+        pCPUM->aGuestCpuIdExt[i] = pCPUM->GuestCpuIdDef;
+
+    /*
+     * Centaur stuff (VIA).
+     *
+     * The important part here (we think) is to make sure the 0xc0000000
+     * function returns 0xc0000001. As for the features, we don't currently
+     * let on about any of those... 0xc0000002 seems to be some
+     * temperature/hz/++ stuff, include it as well (static).
+     */
+    if (    pCPUM->aGuestCpuIdCentaur[0].eax >= UINT32_C(0xc0000000)
+        &&  pCPUM->aGuestCpuIdCentaur[0].eax <= UINT32_C(0xc0000004))
+    {
+        pCPUM->aGuestCpuIdCentaur[0].eax = RT_MIN(pCPUM->aGuestCpuIdCentaur[0].eax, UINT32_C(0xc0000002));
+        pCPUM->aGuestCpuIdCentaur[1].edx = 0; /* all features hidden */
+        for (i = pCPUM->aGuestCpuIdCentaur[0].eax - UINT32_C(0xc0000000);
+             i < RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur);
+             i++)
+            pCPUM->aGuestCpuIdCentaur[i] = pCPUM->GuestCpuIdDef;
+    }
+    else
+        for (i = 0; i < RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur); i++)
+            pCPUM->aGuestCpuIdCentaur[i] = pCPUM->GuestCpuIdDef;
+
+
+    /*
+     * Load CPUID overrides from configuration.
+     * Note: Kind of redundant now, but allows unchanged overrides
+     */
+    /** @cfgm{CPUM/CPUID/[000000xx|800000xx|c000000x]/[eax|ebx|ecx|edx],32-bit}
+     * Overrides the CPUID leaf values. */
+    PCFGMNODE pOverrideCfg = CFGMR3GetChild(pCpumCfg, "CPUID");
+    rc = cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x00000000), &pCPUM->aGuestCpuIdStd[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdStd),     pOverrideCfg);
+    AssertRCReturn(rc, rc);
+    rc = cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x80000000), &pCPUM->aGuestCpuIdExt[0],     RT_ELEMENTS(pCPUM->aGuestCpuIdExt),     pOverrideCfg);
+    AssertRCReturn(rc, rc);
+    rc = cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0xc0000000), &pCPUM->aGuestCpuIdCentaur[0], RT_ELEMENTS(pCPUM->aGuestCpuIdCentaur), pOverrideCfg);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Check if PAE was explicitely enabled by the user.
+     */
+    bool fEnable;
+    rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "EnablePAE", &fEnable, false);      AssertRCReturn(rc, rc);
+    if (fEnable)
+        CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
+
+    /*
+     * We don't normally enable NX for raw-mode, so give the user a chance to
+     * force it on.
+     */
+    rc = CFGMR3QueryBoolDef(pCpumCfg, "EnableNX", &fEnable, false);                 AssertRCReturn(rc, rc);
+    if (fEnable)
+        CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
+
+    /*
+     * Log the cpuid and we're good.
+     */
+    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+    RTCPUSET OnlineSet;
+    LogRel(("Logical host processors: %u present, %u max, %u online, online mask: %016RX64\n",
+            (unsigned)RTMpGetPresentCount(), (unsigned)RTMpGetCount(), (unsigned)RTMpGetOnlineCount(),
+            RTCpuSetToU64(RTMpGetOnlineSet(&OnlineSet)) ));
+    LogRel(("************************* CPUID dump ************************\n"));
+    DBGFR3Info(pVM, "cpuid", "verbose", DBGFR3InfoLogRelHlp());
+    LogRel(("\n"));
+    DBGFR3InfoLog(pVM, "cpuid", "verbose"); /* macro */
+    RTLogRelSetBuffering(fOldBuffered);
+    LogRel(("******************** End of CPUID dump **********************\n"));
+
+#undef PORTABLE_DISABLE_FEATURE_BIT
+#undef PORTABLE_CLEAR_BITS_WHEN
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * The CPUM will update the addresses used by the switcher.
+ *
+ * @param   pVM     The VM.
+ */
+VMMR3DECL(void) CPUMR3Relocate(PVM pVM)
+{
+    LogFlow(("CPUMR3Relocate\n"));
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        /*
+         * Switcher pointers.
+         */
+        PVMCPU pVCpu = &pVM->aCpus[i];
+        pVCpu->cpum.s.pHyperCoreRC = MMHyperCCToRC(pVM, pVCpu->cpum.s.pHyperCoreR3);
+        Assert(pVCpu->cpum.s.pHyperCoreRC != NIL_RTRCPTR);
+
+    }
+}
+
+
+/**
+ * Apply late CPUM property changes based on the fHWVirtEx setting
+ *
+ * @param   pVM                 The VM to operate on.
+ * @param   fHWVirtExEnabled    HWVirtEx enabled/disabled
+ */
+VMMR3DECL(void) CPUMR3SetHWVirtEx(PVM pVM, bool fHWVirtExEnabled)
+{
+    /*
+     * Workaround for missing cpuid(0) patches when leaf 4 returns GuestCpuIdDef:
+     * If we miss to patch a cpuid(0).eax then Linux tries to determine the number
+     * of processors from (cpuid(4).eax >> 26) + 1.
+     *
+     * Note: this code is obsolete, but let's keep it here for reference.
+     *       Purpose is valid when we artificially cap the max std id to less than 4.
+     */
+    if (!fHWVirtExEnabled)
+    {
+        Assert(pVM->cpum.s.aGuestCpuIdStd[4].eax == 0);
+        pVM->cpum.s.aGuestCpuIdStd[4].eax = 0;
+    }
+}
+
+/**
+ * Terminates the CPUM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) CPUMR3Term(PVM pVM)
+{
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU   pVCpu = &pVM->aCpus[i];
+        PCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
+
+        memset(pVCpu->cpum.s.aMagic, 0, sizeof(pVCpu->cpum.s.aMagic));
+        pVCpu->cpum.s.uMagic     = 0;
+        pCtx->dr[5]              = 0;
+    }
+#endif
+    return 0;
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by CPUMR3Reset and CPU hot plugging.
+ *
+ * @param   pVCpu               The virtual CPU handle.
+ */
+VMMR3DECL(void) CPUMR3ResetCpu(PVMCPU pVCpu)
+{
+    /** @todo anything different for VCPU > 0? */
+    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+
+    /*
+     * Initialize everything to ZERO first.
+     */
+    uint32_t fUseFlags =  pVCpu->cpum.s.fUseFlags & ~CPUM_USED_FPU_SINCE_REM;
+    memset(pCtx, 0, sizeof(*pCtx));
+    pVCpu->cpum.s.fUseFlags  = fUseFlags;
+
+    pCtx->cr0                       = X86_CR0_CD | X86_CR0_NW | X86_CR0_ET;  //0x60000010
+    pCtx->eip                       = 0x0000fff0;
+    pCtx->edx                       = 0x00000600;   /* P6 processor */
+    pCtx->eflags.Bits.u1Reserved0   = 1;
+
+    pCtx->cs                        = 0xf000;
+    pCtx->csHid.u64Base             = UINT64_C(0xffff0000);
+    pCtx->csHid.u32Limit            = 0x0000ffff;
+    pCtx->csHid.Attr.n.u1DescType   = 1; /* code/data segment */
+    pCtx->csHid.Attr.n.u1Present    = 1;
+    pCtx->csHid.Attr.n.u4Type       = X86_SEL_TYPE_READ | X86_SEL_TYPE_CODE;
+
+    pCtx->dsHid.u32Limit            = 0x0000ffff;
+    pCtx->dsHid.Attr.n.u1DescType   = 1; /* code/data segment */
+    pCtx->dsHid.Attr.n.u1Present    = 1;
+    pCtx->dsHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
+
+    pCtx->esHid.u32Limit            = 0x0000ffff;
+    pCtx->esHid.Attr.n.u1DescType   = 1; /* code/data segment */
+    pCtx->esHid.Attr.n.u1Present    = 1;
+    pCtx->esHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
+
+    pCtx->fsHid.u32Limit            = 0x0000ffff;
+    pCtx->fsHid.Attr.n.u1DescType   = 1; /* code/data segment */
+    pCtx->fsHid.Attr.n.u1Present    = 1;
+    pCtx->fsHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
+
+    pCtx->gsHid.u32Limit            = 0x0000ffff;
+    pCtx->gsHid.Attr.n.u1DescType   = 1; /* code/data segment */
+    pCtx->gsHid.Attr.n.u1Present    = 1;
+    pCtx->gsHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
+
+    pCtx->ssHid.u32Limit            = 0x0000ffff;
+    pCtx->ssHid.Attr.n.u1Present    = 1;
+    pCtx->ssHid.Attr.n.u1DescType   = 1; /* code/data segment */
+    pCtx->ssHid.Attr.n.u4Type       = X86_SEL_TYPE_RW;
+
+    pCtx->idtr.cbIdt                = 0xffff;
+    pCtx->gdtr.cbGdt                = 0xffff;
+
+    pCtx->ldtrHid.u32Limit          = 0xffff;
+    pCtx->ldtrHid.Attr.n.u1Present  = 1;
+    pCtx->ldtrHid.Attr.n.u4Type     = X86_SEL_TYPE_SYS_LDT;
+
+    pCtx->trHid.u32Limit            = 0xffff;
+    pCtx->trHid.Attr.n.u1Present    = 1;
+    pCtx->trHid.Attr.n.u4Type       = X86_SEL_TYPE_SYS_286_TSS_BUSY;
+
+    pCtx->dr[6]                     = X86_DR6_INIT_VAL;
+    pCtx->dr[7]                     = X86_DR7_INIT_VAL;
+
+    pCtx->fpu.FTW                   = 0xff;         /* All tags are set, i.e. the regs are empty. */
+    pCtx->fpu.FCW                   = 0x37f;
+
+    /* Intel 64 and IA-32 Architectures Software Developer's Manual Volume 3A, Table 8-1. IA-32 Processor States Following Power-up, Reset, or INIT */
+    pCtx->fpu.MXCSR                 = 0x1F80;
+
+    /* Init PAT MSR */
+    pCtx->msrPAT                    = UINT64_C(0x0007040600070406); /** @todo correct? */
+
+    /* Reset EFER; see AMD64 Architecture Programmer's Manual Volume 2: Table 14-1. Initial Processor State
+    * The Intel docs don't mention it.
+    */
+    pCtx->msrEFER                   = 0;
+}
+
+
+/**
+ * Resets the CPU.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(void) CPUMR3Reset(PVM pVM)
+{
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        CPUMR3ResetCpu(&pVM->aCpus[i]);
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+        PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(&pVM->aCpus[i]);
+
+        /* Magic marker for searching in crash dumps. */
+        strcpy((char *)pVM->aCpus[i].cpum.s.aMagic, "CPUMCPU Magic");
+        pVM->aCpus[i].cpum.s.uMagic     = UINT64_C(0xDEADBEEFDEADBEEF);
+        pCtx->dr[5]                     = UINT64_C(0xDEADBEEFDEADBEEF);
+#endif
+    }
+}
+
+
+/**
+ * Called both in pass 0 and the final pass.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static void cpumR3SaveCpuId(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Save all the CPU ID leaves here so we can check them for compatibility
+     * upon loading.
+     */
+    SSMR3PutU32(pSSM, RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd));
+    SSMR3PutMem(pSSM, &pVM->cpum.s.aGuestCpuIdStd[0], sizeof(pVM->cpum.s.aGuestCpuIdStd));
+
+    SSMR3PutU32(pSSM, RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt));
+    SSMR3PutMem(pSSM, &pVM->cpum.s.aGuestCpuIdExt[0], sizeof(pVM->cpum.s.aGuestCpuIdExt));
+
+    SSMR3PutU32(pSSM, RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur));
+    SSMR3PutMem(pSSM, &pVM->cpum.s.aGuestCpuIdCentaur[0], sizeof(pVM->cpum.s.aGuestCpuIdCentaur));
+
+    SSMR3PutMem(pSSM, &pVM->cpum.s.GuestCpuIdDef, sizeof(pVM->cpum.s.GuestCpuIdDef));
+
+    /*
+     * Save a good portion of the raw CPU IDs as well as they may come in
+     * handy when validating features for raw mode.
+     */
+    CPUMCPUID   aRawStd[16];
+    for (unsigned i = 0; i < RT_ELEMENTS(aRawStd); i++)
+        ASMCpuId(i, &aRawStd[i].eax, &aRawStd[i].ebx, &aRawStd[i].ecx, &aRawStd[i].edx);
+    SSMR3PutU32(pSSM, RT_ELEMENTS(aRawStd));
+    SSMR3PutMem(pSSM, &aRawStd[0], sizeof(aRawStd));
+
+    CPUMCPUID   aRawExt[32];
+    for (unsigned i = 0; i < RT_ELEMENTS(aRawExt); i++)
+        ASMCpuId(i | UINT32_C(0x80000000), &aRawExt[i].eax, &aRawExt[i].ebx, &aRawExt[i].ecx, &aRawExt[i].edx);
+    SSMR3PutU32(pSSM, RT_ELEMENTS(aRawExt));
+    SSMR3PutMem(pSSM, &aRawExt[0], sizeof(aRawExt));
+}
+
+
+/**
+ * Loads the CPU ID leaves saved by pass 0.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ * @param   uVersion            The format version.
+ */
+static int cpumR3LoadCpuId(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
+{
+    AssertMsgReturn(uVersion >= CPUM_SAVED_STATE_VERSION_VER3_2, ("%u\n", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+
+    /*
+     * Define a bunch of macros for simplifying the code.
+     */
+    /* Generic expression + failure message. */
+#define CPUID_CHECK_RET(expr, fmt) \
+    do { \
+        if (!(expr)) \
+        { \
+            char *pszMsg = RTStrAPrintf2 fmt; /* lack of variadic macros sucks */ \
+            if (fStrictCpuIdChecks) \
+            { \
+                int rcCpuid = SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, "%s", pszMsg); \
+                RTStrFree(pszMsg); \
+                return rcCpuid; \
+            } \
+            LogRel(("CPUM: %s\n", pszMsg)); \
+            RTStrFree(pszMsg); \
+        } \
+    } while (0)
+#define CPUID_CHECK_WRN(expr, fmt) \
+    do { \
+        if (!(expr)) \
+            LogRel(fmt); \
+    } while (0)
+
+    /* For comparing two values and bitch if they differs. */
+#define CPUID_CHECK2_RET(what, host, saved) \
+    do { \
+        if ((host) != (saved)) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#what " mismatch: host=%#x saved=%#x"), (host), (saved)); \
+            LogRel(("CPUM: " #what " differs: host=%#x saved=%#x\n", (host), (saved))); \
+        } \
+    } while (0)
+#define CPUID_CHECK2_WRN(what, host, saved) \
+    do { \
+        if ((host) != (saved)) \
+            LogRel(("CPUM: " #what " differs: host=%#x saved=%#x\n", (host), (saved))); \
+    } while (0)
+
+    /* For checking raw cpu features (raw mode). */
+#define CPUID_RAW_FEATURE_RET(set, reg, bit) \
+    do { \
+        if ((aHostRaw##set [1].reg & bit) != (aRaw##set [1].reg & bit)) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#bit " mismatch: host=%d saved=%d"), \
+                                         !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) ); \
+            LogRel(("CPUM: " #bit" differs: host=%d saved=%d\n", \
+                    !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) )); \
+        } \
+    } while (0)
+#define CPUID_RAW_FEATURE_WRN(set, reg, bit) \
+    do { \
+        if ((aHostRaw##set [1].reg & bit) != (aRaw##set [1].reg & bit)) \
+            LogRel(("CPUM: " #bit" differs: host=%d saved=%d\n", \
+                    !!(aHostRaw##set [1].reg & (bit)), !!(aRaw##set [1].reg & (bit)) )); \
+    } while (0)
+#define CPUID_RAW_FEATURE_IGN(set, reg, bit) do { } while (0)
+
+    /* For checking guest features. */
+#define CPUID_GST_FEATURE_RET(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            && !(aHostRaw##set [1].reg & bit) \
+            && !(aHostOverride##set [1].reg & bit) \
+            && !(aGuestOverride##set [1].reg & bit) \
+           ) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#bit " is not supported by the host but has already exposed to the guest")); \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+        } \
+    } while (0)
+#define CPUID_GST_FEATURE_WRN(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            && !(aHostRaw##set [1].reg & bit) \
+            && !(aHostOverride##set [1].reg & bit) \
+            && !(aGuestOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE_EMU(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            && !(aHostRaw##set [1].reg & bit) \
+            && !(aHostOverride##set [1].reg & bit) \
+            && !(aGuestOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: Warning - " #bit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE_IGN(set, reg, bit) do { } while (0)
+
+    /* For checking guest features if AMD guest CPU. */
+#define CPUID_GST_AMD_FEATURE_RET(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            &&  fGuestAmd \
+            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
+            && !(aHostOverride##set [1].reg & bit) \
+            && !(aGuestOverride##set [1].reg & bit) \
+           ) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#bit " is not supported by the host but has already exposed to the guest")); \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+        } \
+    } while (0)
+#define CPUID_GST_AMD_FEATURE_WRN(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            &&  fGuestAmd \
+            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
+            && !(aHostOverride##set [1].reg & bit) \
+            && !(aGuestOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: " #bit " is not supported by the host but has already exposed to the guest\n")); \
+    } while (0)
+#define CPUID_GST_AMD_FEATURE_EMU(set, reg, bit) \
+    do { \
+        if (    (aGuestCpuId##set [1].reg & bit) \
+            &&  fGuestAmd \
+            && (!fGuestAmd || !(aHostRaw##set [1].reg & bit)) \
+            && !(aHostOverride##set [1].reg & bit) \
+            && !(aGuestOverride##set [1].reg & bit) \
+           ) \
+            LogRel(("CPUM: Warning - " #bit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
+    } while (0)
+#define CPUID_GST_AMD_FEATURE_IGN(set, reg, bit) do { } while (0)
+
+    /* For checking AMD features which have a corresponding bit in the standard
+       range.  (Intel defines very few bits in the extended feature sets.) */
+#define CPUID_GST_FEATURE2_RET(reg, ExtBit, StdBit) \
+    do { \
+        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
+            && !(fHostAmd  \
+                 ? aHostRawExt[1].reg      & (ExtBit) \
+                 : aHostRawStd[1].reg      & (StdBit)) \
+            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
+            && !(aGuestOverrideExt[1].reg  & (ExtBit)) \
+           ) \
+        { \
+            if (fStrictCpuIdChecks) \
+                return SSMR3SetLoadError(pSSM, VERR_SSM_LOAD_CPUID_MISMATCH, RT_SRC_POS, \
+                                         N_(#ExtBit " is not supported by the host but has already exposed to the guest")); \
+            LogRel(("CPUM: " #ExtBit " is not supported by the host but has already exposed to the guest\n")); \
+        } \
+    } while (0)
+#define CPUID_GST_FEATURE2_WRN(reg, ExtBit, StdBit) \
+    do { \
+        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
+            && !(fHostAmd  \
+                 ? aHostRawExt[1].reg      & (ExtBit) \
+                 : aHostRawStd[1].reg      & (StdBit)) \
+            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
+            && !(aGuestOverrideExt[1].reg  & (ExtBit)) \
+           ) \
+            LogRel(("CPUM: " #ExtBit " is not supported by the host but has already exposed to the guest\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE2_EMU(reg, ExtBit, StdBit) \
+    do { \
+        if (    (aGuestCpuIdExt [1].reg    & (ExtBit)) \
+            && !(fHostAmd  \
+                 ? aHostRawExt[1].reg      & (ExtBit) \
+                 : aHostRawStd[1].reg      & (StdBit)) \
+            && !(aHostOverrideExt[1].reg   & (ExtBit)) \
+            && !(aGuestOverrideExt[1].reg  & (ExtBit)) \
+           ) \
+            LogRel(("CPUM: Warning - " #ExtBit " is not supported by the host but already exposed to the guest. This may impact performance.\n")); \
+    } while (0)
+#define CPUID_GST_FEATURE2_IGN(reg, ExtBit, StdBit) do { } while (0)
+
+    /*
+     * Load them into stack buffers first.
+     */
+    CPUMCPUID   aGuestCpuIdStd[RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd)];
+    uint32_t    cGuestCpuIdStd;
+    int rc = SSMR3GetU32(pSSM, &cGuestCpuIdStd); AssertRCReturn(rc, rc);
+    if (cGuestCpuIdStd > RT_ELEMENTS(aGuestCpuIdStd))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &aGuestCpuIdStd[0], cGuestCpuIdStd * sizeof(aGuestCpuIdStd[0]));
+
+    CPUMCPUID   aGuestCpuIdExt[RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt)];
+    uint32_t    cGuestCpuIdExt;
+    rc = SSMR3GetU32(pSSM, &cGuestCpuIdExt); AssertRCReturn(rc, rc);
+    if (cGuestCpuIdExt > RT_ELEMENTS(aGuestCpuIdExt))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &aGuestCpuIdExt[0], cGuestCpuIdExt * sizeof(aGuestCpuIdExt[0]));
+
+    CPUMCPUID   aGuestCpuIdCentaur[RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur)];
+    uint32_t    cGuestCpuIdCentaur;
+    rc = SSMR3GetU32(pSSM, &cGuestCpuIdCentaur); AssertRCReturn(rc, rc);
+    if (cGuestCpuIdCentaur > RT_ELEMENTS(aGuestCpuIdCentaur))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &aGuestCpuIdCentaur[0], cGuestCpuIdCentaur * sizeof(aGuestCpuIdCentaur[0]));
+
+    CPUMCPUID   GuestCpuIdDef;
+    rc = SSMR3GetMem(pSSM, &GuestCpuIdDef, sizeof(GuestCpuIdDef));
+    AssertRCReturn(rc, rc);
+
+    CPUMCPUID   aRawStd[16];
+    uint32_t    cRawStd;
+    rc = SSMR3GetU32(pSSM, &cRawStd); AssertRCReturn(rc, rc);
+    if (cRawStd > RT_ELEMENTS(aRawStd))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &aRawStd[0], cRawStd * sizeof(aRawStd[0]));
+
+    CPUMCPUID   aRawExt[32];
+    uint32_t    cRawExt;
+    rc = SSMR3GetU32(pSSM, &cRawExt); AssertRCReturn(rc, rc);
+    if (cRawExt > RT_ELEMENTS(aRawExt))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    rc = SSMR3GetMem(pSSM, &aRawExt[0], cRawExt * sizeof(aRawExt[0]));
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Note that we support restoring less than the current amount of standard
+     * leaves because we've been allowed more is newer version of VBox.
+     *
+     * So, pad new entries with the default.
+     */
+    for (uint32_t i = cGuestCpuIdStd; i < RT_ELEMENTS(aGuestCpuIdStd); i++)
+        aGuestCpuIdStd[i] = GuestCpuIdDef;
+
+    for (uint32_t i = cGuestCpuIdExt; i < RT_ELEMENTS(aGuestCpuIdExt); i++)
+        aGuestCpuIdExt[i] = GuestCpuIdDef;
+
+    for (uint32_t i = cGuestCpuIdCentaur; i < RT_ELEMENTS(aGuestCpuIdCentaur); i++)
+        aGuestCpuIdCentaur[i] = GuestCpuIdDef;
+
+    for (uint32_t i = cRawStd; i < RT_ELEMENTS(aRawStd); i++)
+        ASMCpuId(i, &aRawStd[i].eax, &aRawStd[i].ebx, &aRawStd[i].ecx, &aRawStd[i].edx);
+
+    for (uint32_t i = cRawExt; i < RT_ELEMENTS(aRawExt); i++)
+        ASMCpuId(i | UINT32_C(0x80000000), &aRawExt[i].eax, &aRawExt[i].ebx, &aRawExt[i].ecx, &aRawExt[i].edx);
+
+    /*
+     * Get the raw CPU IDs for the current host.
+     */
+    CPUMCPUID   aHostRawStd[16];
+    for (unsigned i = 0; i < RT_ELEMENTS(aHostRawStd); i++)
+        ASMCpuId(i, &aHostRawStd[i].eax, &aHostRawStd[i].ebx, &aHostRawStd[i].ecx, &aHostRawStd[i].edx);
+
+    CPUMCPUID   aHostRawExt[32];
+    for (unsigned i = 0; i < RT_ELEMENTS(aHostRawExt); i++)
+        ASMCpuId(i | UINT32_C(0x80000000), &aHostRawExt[i].eax, &aHostRawExt[i].ebx, &aHostRawExt[i].ecx, &aHostRawExt[i].edx);
+
+    /*
+     * Get the host and guest overrides so we don't reject the state because
+     * some feature was enabled thru these interfaces.
+     * Note! We currently only need the feature leaves, so skip rest.
+     */
+    PCFGMNODE   pOverrideCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM/CPUID");
+    CPUMCPUID   aGuestOverrideStd[2];
+    memcpy(&aGuestOverrideStd[0], &aHostRawStd[0], sizeof(aGuestOverrideStd));
+    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x00000000), &aGuestOverrideStd[0], RT_ELEMENTS(aGuestOverrideStd), pOverrideCfg);
+
+    CPUMCPUID   aGuestOverrideExt[2];
+    memcpy(&aGuestOverrideExt[0], &aHostRawExt[0], sizeof(aGuestOverrideExt));
+    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x80000000), &aGuestOverrideExt[0], RT_ELEMENTS(aGuestOverrideExt), pOverrideCfg);
+
+    pOverrideCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM/HostCPUID");
+    CPUMCPUID   aHostOverrideStd[2];
+    memcpy(&aHostOverrideStd[0], &aHostRawStd[0], sizeof(aHostOverrideStd));
+    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x00000000), &aHostOverrideStd[0], RT_ELEMENTS(aHostOverrideStd), pOverrideCfg);
+
+    CPUMCPUID   aHostOverrideExt[2];
+    memcpy(&aHostOverrideExt[0], &aHostRawExt[0], sizeof(aHostOverrideExt));
+    cpumR3CpuIdInitLoadOverrideSet(UINT32_C(0x80000000), &aHostOverrideExt[0], RT_ELEMENTS(aHostOverrideExt), pOverrideCfg);
+
+    /*
+     * This can be skipped.
+     */
+    bool fStrictCpuIdChecks;
+    CFGMR3QueryBoolDef(CFGMR3GetChild(CFGMR3GetRoot(pVM), "CPUM"), "StrictCpuIdChecks", &fStrictCpuIdChecks, true);
+
+
+
+    /*
+     * For raw-mode we'll require that the CPUs are very similar since we don't
+     * intercept CPUID instructions for user mode applications.
+     */
+    if (!HWACCMIsEnabled(pVM))
+    {
+        /* CPUID(0) */
+        CPUID_CHECK_RET(   aHostRawStd[0].ebx == aRawStd[0].ebx
+                        && aHostRawStd[0].ecx == aRawStd[0].ecx
+                        && aHostRawStd[0].edx == aRawStd[0].edx,
+                        (N_("CPU vendor mismatch: host='%.4s%.4s%.4s' saved='%.4s%.4s%.4s'"),
+                         &aHostRawStd[0].ebx, &aHostRawStd[0].edx, &aHostRawStd[0].ecx,
+                         &aRawStd[0].ebx, &aRawStd[0].edx, &aRawStd[0].ecx));
+        CPUID_CHECK2_WRN("Std CPUID max leaf",   aHostRawStd[0].eax, aRawStd[0].eax);
+        CPUID_CHECK2_WRN("Reserved bits 15:14", (aHostRawExt[1].eax >> 14) & 3, (aRawExt[1].eax >> 14) & 3);
+        CPUID_CHECK2_WRN("Reserved bits 31:28",  aHostRawExt[1].eax >> 28,       aRawExt[1].eax >> 28);
+
+        bool const fIntel = ASMIsIntelCpuEx(aRawStd[0].ebx, aRawStd[0].ecx, aRawStd[0].edx);
+
+        /* CPUID(1).eax */
+        CPUID_CHECK2_RET("CPU family",          ASMGetCpuFamily(aHostRawStd[1].eax),        ASMGetCpuFamily(aRawStd[1].eax));
+        CPUID_CHECK2_RET("CPU model",           ASMGetCpuModel(aHostRawStd[1].eax, fIntel), ASMGetCpuModel(aRawStd[1].eax, fIntel));
+        CPUID_CHECK2_WRN("CPU type",            (aHostRawStd[1].eax >> 12) & 3,             (aRawStd[1].eax >> 12) & 3 );
+
+        /* CPUID(1).ebx - completely ignore CPU count and APIC ID. */
+        CPUID_CHECK2_RET("CPU brand ID",         aHostRawStd[1].ebx & 0xff,                 aRawStd[1].ebx & 0xff);
+        CPUID_CHECK2_WRN("CLFLUSH chunk count", (aHostRawStd[1].ebx >> 8) & 0xff,           (aRawStd[1].ebx >> 8) & 0xff);
+
+        /* CPUID(1).ecx */
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE3);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_PCLMUL);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_DTES64);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_MONITOR);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CPLDS);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_VMX);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_SMX);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_EST);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_TM2);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSSE3);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_CNTXID);
+        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(11) /*reserved*/ );
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_FMA);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CX16);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_TPRUPDATE);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_PDCM);
+        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(16) /*reserved*/);
+        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(17) /*reserved*/);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_DCA);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_1);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_2);
+        CPUID_RAW_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_X2APIC);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_MOVBE);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_POPCNT);
+        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(24) /*reserved*/);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AES);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_XSAVE);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_OSXSAVE);
+        CPUID_RAW_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AVX);
+        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(29) /*reserved*/);
+        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(30) /*reserved*/);
+        CPUID_RAW_FEATURE_RET(Std, ecx, RT_BIT_32(31) /*reserved*/);
+
+        /* CPUID(1).edx */
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FPU);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_VME);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_DE);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_TSC);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MSR);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PAE);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCE);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CX8);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_APIC);
+        CPUID_RAW_FEATURE_RET(Std, edx, RT_BIT_32(10) /*reserved*/);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_SEP);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MTRR);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PGE);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCA);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CMOV);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PAT);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE36);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSN);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CLFSH);
+        CPUID_RAW_FEATURE_RET(Std, edx, RT_BIT_32(20) /*reserved*/);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_DS);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_ACPI);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MMX);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FXSR);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE);
+        CPUID_RAW_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE2);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_SS);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_HTT);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_TM);
+        CPUID_RAW_FEATURE_RET(Std, edx, RT_BIT_32(30) /*JMPE/IA64*/);
+        CPUID_RAW_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PBE);
+
+        /* CPUID(2) - config, mostly about caches. ignore. */
+        /* CPUID(3) - processor serial number. ignore. */
+        /* CPUID(4) - config, cache and topology - takes ECX as input. ignore. */
+        /* CPUID(5) - mwait/monitor config. ignore. */
+        /* CPUID(6) - power management. ignore. */
+        /* CPUID(7) - ???. ignore. */
+        /* CPUID(8) - ???. ignore. */
+        /* CPUID(9) - DCA. ignore for now. */
+        /* CPUID(a) - PeMo info. ignore for now. */
+        /* CPUID(b) - topology info - takes ECX as input. ignore. */
+
+        /* CPUID(d) - XCR0 stuff - takes ECX as input. We only warn about the main level (ECX=0) for now. */
+        CPUID_CHECK_WRN(   aRawStd[0].eax     <  UINT32_C(0x0000000d)
+                        || aHostRawStd[0].eax >= UINT32_C(0x0000000d),
+                        ("CPUM: Standard leaf D was present on saved state host, not present on current.\n"));
+        if (   aRawStd[0].eax     >= UINT32_C(0x0000000d)
+            && aHostRawStd[0].eax >= UINT32_C(0x0000000d))
+        {
+            CPUID_CHECK2_WRN("Valid low XCR0 bits",             aHostRawStd[0xd].eax, aRawStd[0xd].eax);
+            CPUID_CHECK2_WRN("Valid high XCR0 bits",            aHostRawStd[0xd].edx, aRawStd[0xd].edx);
+            CPUID_CHECK2_WRN("Current XSAVE/XRSTOR area size",  aHostRawStd[0xd].ebx, aRawStd[0xd].ebx);
+            CPUID_CHECK2_WRN("Max XSAVE/XRSTOR area size",      aHostRawStd[0xd].ecx, aRawStd[0xd].ecx);
+        }
+
+        /* CPUID(0x80000000) - same as CPUID(0) except for eax.
+           Note! Intel have/is marking many of the fields here as reserved. We
+                 will verify them as if it's an AMD CPU. */
+        CPUID_CHECK_RET(   (aHostRawExt[0].eax >= UINT32_C(0x80000001) && aHostRawExt[0].eax <= UINT32_C(0x8000007f))
+                        || !(aRawExt[0].eax    >= UINT32_C(0x80000001) && aRawExt[0].eax     <= UINT32_C(0x8000007f)),
+                        (N_("Extended leaves was present on saved state host, but is missing on the current\n")));
+        if (aRawExt[0].eax >= UINT32_C(0x80000001) && aRawExt[0].eax     <= UINT32_C(0x8000007f))
+        {
+            CPUID_CHECK_RET(   aHostRawExt[0].ebx == aRawExt[0].ebx
+                            && aHostRawExt[0].ecx == aRawExt[0].ecx
+                            && aHostRawExt[0].edx == aRawExt[0].edx,
+                            (N_("CPU vendor mismatch: host='%.4s%.4s%.4s' saved='%.4s%.4s%.4s'"),
+                             &aHostRawExt[0].ebx, &aHostRawExt[0].edx, &aHostRawExt[0].ecx,
+                             &aRawExt[0].ebx,     &aRawExt[0].edx,     &aRawExt[0].ecx));
+            CPUID_CHECK2_WRN("Ext CPUID max leaf",   aHostRawExt[0].eax, aRawExt[0].eax);
+
+            /* CPUID(0x80000001).eax - same as CPUID(0).eax. */
+            CPUID_CHECK2_RET("CPU family",          ASMGetCpuFamily(aHostRawExt[1].eax),        ASMGetCpuFamily(aRawExt[1].eax));
+            CPUID_CHECK2_RET("CPU model",           ASMGetCpuModel(aHostRawExt[1].eax, fIntel), ASMGetCpuModel(aRawExt[1].eax, fIntel));
+            CPUID_CHECK2_WRN("CPU type",            (aHostRawExt[1].eax >> 12) & 3, (aRawExt[1].eax >> 12) & 3 );
+            CPUID_CHECK2_WRN("Reserved bits 15:14", (aHostRawExt[1].eax >> 14) & 3, (aRawExt[1].eax >> 14) & 3 );
+            CPUID_CHECK2_WRN("Reserved bits 31:28",  aHostRawExt[1].eax >> 28, aRawExt[1].eax >> 28);
+
+            /* CPUID(0x80000001).ebx - Brand ID (maybe), just warn if things differs. */
+            CPUID_CHECK2_WRN("CPU BrandID",          aHostRawExt[1].ebx & 0xffff, aRawExt[1].ebx & 0xffff);
+            CPUID_CHECK2_WRN("Reserved bits 16:27", (aHostRawExt[1].ebx >> 16) & 0xfff, (aRawExt[1].ebx >> 16) & 0xfff);
+            CPUID_CHECK2_WRN("PkgType",             (aHostRawExt[1].ebx >> 28) &   0xf, (aRawExt[1].ebx >> 28) &   0xf);
+
+            /* CPUID(0x80000001).ecx */
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF);
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CMPL);
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SVM);
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_EXT_APIC);
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CR8L);
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_ABM);
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE4A);
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_MISALNSSE);
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF);
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_OSVW);
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_IBS);
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE5);
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SKINIT);
+            CPUID_RAW_FEATURE_IGN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_WDT);
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(14));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(15));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(16));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(17));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(18));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(19));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(20));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(21));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(22));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(23));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(24));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(25));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(26));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(27));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(28));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(29));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(30));
+            CPUID_RAW_FEATURE_WRN(Ext, ecx, RT_BIT_32(31));
+
+            /* CPUID(0x80000001).edx */
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FPU);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_VME);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_DE);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PSE);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_TSC);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MSR);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAE);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MCE);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_CX8);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_APIC);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(10) /*reserved*/);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_SEP);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MTRR);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PGE);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MCA);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_CMOV);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAT);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PSE36);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(18) /*reserved*/);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(19) /*reserved*/);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_NX);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(21) /*reserved*/);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_AXMMX);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_MMX);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FXSR);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FFXSR);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAGE1GB);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_RDTSCP);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, RT_BIT_32(28) /*reserved*/);
+            CPUID_RAW_FEATURE_IGN(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_LONG_MODE);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
+            CPUID_RAW_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW);
+
+            /** @todo verify the rest as well. */
+        }
+    }
+
+
+
+    /*
+     * Verify that we can support the features already exposed to the guest on
+     * this host.
+     *
+     * Most of the features we're emulating requires intercepting instruction
+     * and doing it the slow way, so there is no need to warn when they aren't
+     * present in the host CPU.  Thus we use IGN instead of EMU on these.
+     *
+     * Trailing comments:
+     *      "EMU"  - Possible to emulate, could be lots of work and very slow.
+     *      "EMU?" - Can this be emulated?
+     */
+    /* CPUID(1).ecx */
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE3);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_PCLMUL);  // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_DTES64);  // -> EMU?
+    CPUID_GST_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_MONITOR);
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CPLDS);   // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_VMX);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SMX);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_EST);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_TM2);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSSE3);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CNTXID);  // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(11) /*reserved*/ );
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_FMA);     // -> EMU? what's this?
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_CX16);    // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_TPRUPDATE);//-> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_PDCM);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(16) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(17) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_DCA);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_1);  // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_SSE4_2);  // -> EMU
+    CPUID_GST_FEATURE_IGN(Std, ecx, X86_CPUID_FEATURE_ECX_X2APIC);
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_MOVBE);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_POPCNT);  // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(24) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AES);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_XSAVE);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_OSXSAVE); // -> EMU
+    CPUID_GST_FEATURE_RET(Std, ecx, X86_CPUID_FEATURE_ECX_AVX);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(29) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(30) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, ecx, RT_BIT_32(31) /*reserved*/);
+
+    /* CPUID(1).edx */
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FPU);
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_VME);
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_DE);      // -> EMU?
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE);
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_TSC);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MSR);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_PAE);
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCE);
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CX8);     // -> EMU?
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_APIC);
+    CPUID_GST_FEATURE_RET(Std, edx, RT_BIT_32(10) /*reserved*/);
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_SEP);
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MTRR);
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PGE);
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_MCA);
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CMOV);    // -> EMU
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PAT);
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSE36);
+    CPUID_GST_FEATURE_IGN(Std, edx, X86_CPUID_FEATURE_EDX_PSN);
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_CLFSH);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, RT_BIT_32(20) /*reserved*/);
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_DS);      // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_ACPI);    // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_MMX);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_FXSR);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE);     // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SSE2);    // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_SS);      // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_HTT);     // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_TM);      // -> EMU?
+    CPUID_GST_FEATURE_RET(Std, edx, RT_BIT_32(30) /*JMPE/IA64*/);   // -> EMU
+    CPUID_GST_FEATURE_RET(Std, edx, X86_CPUID_FEATURE_EDX_PBE);     // -> EMU?
+
+    /* CPUID(0x80000000). */
+    if (    aGuestCpuIdExt[0].eax >= UINT32_C(0x80000001)
+        &&  aGuestCpuIdExt[0].eax <  UINT32_C(0x8000007f))
+    {
+        /** @todo deal with no 0x80000001 on the host. */
+        bool const fHostAmd  = ASMIsAmdCpuEx(aHostRawStd[0].ebx, aHostRawStd[0].ecx, aHostRawStd[0].edx);
+        bool const fGuestAmd = ASMIsAmdCpuEx(aGuestCpuIdExt[0].ebx, aGuestCpuIdExt[0].ecx, aGuestCpuIdExt[0].edx);
+
+        /* CPUID(0x80000001).ecx */
+        CPUID_GST_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_LAHF_SAHF);   // -> EMU
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CMPL);    // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SVM);     // -> EMU
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_EXT_APIC);// ???
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_CR8L);    // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_ABM);     // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE4A);   // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_MISALNSSE);//-> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF);// -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_OSVW);    // -> EMU?
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_IBS);     // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SSE5);    // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_SKINIT);  // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, ecx, X86_CPUID_AMD_FEATURE_ECX_WDT);     // -> EMU
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(14));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(15));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(16));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(17));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(18));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(19));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(20));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(21));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(22));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(23));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(24));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(25));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(26));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(27));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(28));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(29));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(30));
+        CPUID_GST_AMD_FEATURE_WRN(Ext, ecx, RT_BIT_32(31));
+
+        /* CPUID(0x80000001).edx */
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_FPU,   X86_CPUID_FEATURE_EDX_FPU);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_VME,   X86_CPUID_FEATURE_EDX_VME);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_DE,    X86_CPUID_FEATURE_EDX_DE);      // -> EMU
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PSE,   X86_CPUID_FEATURE_EDX_PSE);
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_TSC,   X86_CPUID_FEATURE_EDX_TSC);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_MSR,   X86_CPUID_FEATURE_EDX_MSR);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_PAE,   X86_CPUID_FEATURE_EDX_PAE);
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_MCE,   X86_CPUID_FEATURE_EDX_MCE);
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_CX8,   X86_CPUID_FEATURE_EDX_CX8);     // -> EMU?
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_APIC,  X86_CPUID_FEATURE_EDX_APIC);
+        CPUID_GST_AMD_FEATURE_WRN(Ext, edx, RT_BIT_32(10) /*reserved*/);
+        CPUID_GST_FEATURE_IGN(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_SEP);                                  // Intel: long mode only.
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_MTRR,  X86_CPUID_FEATURE_EDX_MTRR);
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PGE,   X86_CPUID_FEATURE_EDX_PGE);
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_MCA,   X86_CPUID_FEATURE_EDX_MCA);
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_CMOV,  X86_CPUID_FEATURE_EDX_CMOV);    // -> EMU
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PAT,   X86_CPUID_FEATURE_EDX_PAT);
+        CPUID_GST_FEATURE2_IGN(        edx, X86_CPUID_AMD_FEATURE_EDX_PSE36, X86_CPUID_FEATURE_EDX_PSE36);
+        CPUID_GST_AMD_FEATURE_WRN(Ext, edx, RT_BIT_32(18) /*reserved*/);
+        CPUID_GST_AMD_FEATURE_WRN(Ext, edx, RT_BIT_32(19) /*reserved*/);
+        CPUID_GST_FEATURE_RET(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_NX);
+        CPUID_GST_FEATURE_WRN(    Ext, edx, RT_BIT_32(21) /*reserved*/);
+        CPUID_GST_FEATURE_RET(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_AXMMX);
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_MMX,   X86_CPUID_FEATURE_EDX_MMX);     // -> EMU
+        CPUID_GST_FEATURE2_RET(        edx, X86_CPUID_AMD_FEATURE_EDX_FXSR,  X86_CPUID_FEATURE_EDX_FXSR);    // -> EMU
+        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_FFXSR);
+        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_PAGE1GB);
+        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_RDTSCP);
+        CPUID_GST_FEATURE_IGN(    Ext, edx, RT_BIT_32(28) /*reserved*/);
+        CPUID_GST_FEATURE_RET(    Ext, edx, X86_CPUID_AMD_FEATURE_EDX_LONG_MODE);
+        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX);
+        CPUID_GST_AMD_FEATURE_RET(Ext, edx, X86_CPUID_AMD_FEATURE_EDX_3DNOW);
+    }
+
+    /*
+     * We're good, commit the CPU ID leaves.
+     */
+    memcpy(&pVM->cpum.s.aGuestCpuIdStd[0],     &aGuestCpuIdStd[0],     sizeof(aGuestCpuIdStd));
+    memcpy(&pVM->cpum.s.aGuestCpuIdExt[0],     &aGuestCpuIdExt[0],     sizeof(aGuestCpuIdExt));
+    memcpy(&pVM->cpum.s.aGuestCpuIdCentaur[0], &aGuestCpuIdCentaur[0], sizeof(aGuestCpuIdCentaur));
+    pVM->cpum.s.GuestCpuIdDef = GuestCpuIdDef;
+
+#undef CPUID_CHECK_RET
+#undef CPUID_CHECK_WRN
+#undef CPUID_CHECK2_RET
+#undef CPUID_CHECK2_WRN
+#undef CPUID_RAW_FEATURE_RET
+#undef CPUID_RAW_FEATURE_WRN
+#undef CPUID_RAW_FEATURE_IGN
+#undef CPUID_GST_FEATURE_RET
+#undef CPUID_GST_FEATURE_WRN
+#undef CPUID_GST_FEATURE_EMU
+#undef CPUID_GST_FEATURE_IGN
+#undef CPUID_GST_FEATURE2_RET
+#undef CPUID_GST_FEATURE2_WRN
+#undef CPUID_GST_FEATURE2_EMU
+#undef CPUID_GST_FEATURE2_IGN
+#undef CPUID_GST_AMD_FEATURE_RET
+#undef CPUID_GST_AMD_FEATURE_WRN
+#undef CPUID_GST_AMD_FEATURE_EMU
+#undef CPUID_GST_AMD_FEATURE_IGN
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Pass 0 live exec callback.
+ *
+ * @returns VINF_SSM_DONT_CALL_AGAIN.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ * @param   uPass               The pass (0).
+ */
+static DECLCALLBACK(int) cpumR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    AssertReturn(uPass == 0, VERR_INTERNAL_ERROR_4);
+    cpumR3SaveCpuId(pVM, pSSM);
+    return VINF_SSM_DONT_CALL_AGAIN;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) cpumR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Save.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        SSMR3PutMem(pSSM, &pVCpu->cpum.s.Hyper, sizeof(pVCpu->cpum.s.Hyper));
+    }
+
+    SSMR3PutU32(pSSM, pVM->cCpus);
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        SSMR3PutMem(pSSM, &pVCpu->cpum.s.Guest, sizeof(pVCpu->cpum.s.Guest));
+        SSMR3PutU32(pSSM, pVCpu->cpum.s.fUseFlags);
+        SSMR3PutU32(pSSM, pVCpu->cpum.s.fChanged);
+        SSMR3PutMem(pSSM, &pVCpu->cpum.s.GuestMsr, sizeof(pVCpu->cpum.s.GuestMsr));
+    }
+
+    cpumR3SaveCpuId(pVM, pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Load a version 1.6 CPUMCTX structure.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pCpumctx16      Version 1.6 CPUMCTX
+ */
+static void cpumR3LoadCPUM1_6(PVM pVM, CPUMCTX_VER1_6 *pCpumctx16)
+{
+#define CPUMCTX16_LOADREG(RegName) \
+        pVM->aCpus[0].cpum.s.Guest.RegName = pCpumctx16->RegName;
+
+#define CPUMCTX16_LOADDRXREG(RegName) \
+        pVM->aCpus[0].cpum.s.Guest.dr[RegName] = pCpumctx16->dr##RegName;
+
+#define CPUMCTX16_LOADHIDREG(RegName) \
+        pVM->aCpus[0].cpum.s.Guest.RegName##Hid.u64Base  = pCpumctx16->RegName##Hid.u32Base; \
+        pVM->aCpus[0].cpum.s.Guest.RegName##Hid.u32Limit = pCpumctx16->RegName##Hid.u32Limit; \
+        pVM->aCpus[0].cpum.s.Guest.RegName##Hid.Attr     = pCpumctx16->RegName##Hid.Attr;
+
+#define CPUMCTX16_LOADSEGREG(RegName) \
+        pVM->aCpus[0].cpum.s.Guest.RegName = pCpumctx16->RegName; \
+        CPUMCTX16_LOADHIDREG(RegName);
+
+    pVM->aCpus[0].cpum.s.Guest.fpu = pCpumctx16->fpu;
+
+    CPUMCTX16_LOADREG(rax);
+    CPUMCTX16_LOADREG(rbx);
+    CPUMCTX16_LOADREG(rcx);
+    CPUMCTX16_LOADREG(rdx);
+    CPUMCTX16_LOADREG(rdi);
+    CPUMCTX16_LOADREG(rsi);
+    CPUMCTX16_LOADREG(rbp);
+    CPUMCTX16_LOADREG(esp);
+    CPUMCTX16_LOADREG(rip);
+    CPUMCTX16_LOADREG(rflags);
+
+    CPUMCTX16_LOADSEGREG(cs);
+    CPUMCTX16_LOADSEGREG(ds);
+    CPUMCTX16_LOADSEGREG(es);
+    CPUMCTX16_LOADSEGREG(fs);
+    CPUMCTX16_LOADSEGREG(gs);
+    CPUMCTX16_LOADSEGREG(ss);
+
+    CPUMCTX16_LOADREG(r8);
+    CPUMCTX16_LOADREG(r9);
+    CPUMCTX16_LOADREG(r10);
+    CPUMCTX16_LOADREG(r11);
+    CPUMCTX16_LOADREG(r12);
+    CPUMCTX16_LOADREG(r13);
+    CPUMCTX16_LOADREG(r14);
+    CPUMCTX16_LOADREG(r15);
+
+    CPUMCTX16_LOADREG(cr0);
+    CPUMCTX16_LOADREG(cr2);
+    CPUMCTX16_LOADREG(cr3);
+    CPUMCTX16_LOADREG(cr4);
+
+    CPUMCTX16_LOADDRXREG(0);
+    CPUMCTX16_LOADDRXREG(1);
+    CPUMCTX16_LOADDRXREG(2);
+    CPUMCTX16_LOADDRXREG(3);
+    CPUMCTX16_LOADDRXREG(4);
+    CPUMCTX16_LOADDRXREG(5);
+    CPUMCTX16_LOADDRXREG(6);
+    CPUMCTX16_LOADDRXREG(7);
+
+    pVM->aCpus[0].cpum.s.Guest.gdtr.cbGdt   = pCpumctx16->gdtr.cbGdt;
+    pVM->aCpus[0].cpum.s.Guest.gdtr.pGdt    = pCpumctx16->gdtr.pGdt;
+    pVM->aCpus[0].cpum.s.Guest.idtr.cbIdt   = pCpumctx16->idtr.cbIdt;
+    pVM->aCpus[0].cpum.s.Guest.idtr.pIdt    = pCpumctx16->idtr.pIdt;
+
+    CPUMCTX16_LOADREG(ldtr);
+    CPUMCTX16_LOADREG(tr);
+
+    pVM->aCpus[0].cpum.s.Guest.SysEnter     = pCpumctx16->SysEnter;
+
+    CPUMCTX16_LOADREG(msrEFER);
+    CPUMCTX16_LOADREG(msrSTAR);
+    CPUMCTX16_LOADREG(msrPAT);
+    CPUMCTX16_LOADREG(msrLSTAR);
+    CPUMCTX16_LOADREG(msrCSTAR);
+    CPUMCTX16_LOADREG(msrSFMASK);
+    CPUMCTX16_LOADREG(msrKERNELGSBASE);
+
+    CPUMCTX16_LOADHIDREG(ldtr);
+    CPUMCTX16_LOADHIDREG(tr);
+
+#undef CPUMCTX16_LOADSEGREG
+#undef CPUMCTX16_LOADHIDREG
+#undef CPUMCTX16_LOADDRXREG
+#undef CPUMCTX16_LOADREG
+}
+
+
+/**
+ * @copydoc FNSSMINTLOADPREP
+ */
+static DECLCALLBACK(int) cpumR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    pVM->cpum.s.fPendingRestore = true;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @copydoc FNSSMINTLOADEXEC
+ */
+static DECLCALLBACK(int) cpumR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    /*
+     * Validate version.
+     */
+    if (    uVersion != CPUM_SAVED_STATE_VERSION
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER3_2
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER3_0
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER2_0
+        &&  uVersion != CPUM_SAVED_STATE_VERSION_VER1_6)
+    {
+        AssertMsgFailed(("cpumR3LoadExec: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    if (uPass == SSM_PASS_FINAL)
+    {
+        /*
+         * Set the size of RTGCPTR for SSMR3GetGCPtr. (Only necessary for
+         * really old SSM file versions.)
+         */
+        if (uVersion == CPUM_SAVED_STATE_VERSION_VER1_6)
+            SSMR3HandleSetGCPtrSize(pSSM, sizeof(RTGCPTR32));
+        else if (uVersion <= CPUM_SAVED_STATE_VERSION_VER3_0)
+            SSMR3HandleSetGCPtrSize(pSSM, HC_ARCH_BITS == 32 ? sizeof(RTGCPTR32) : sizeof(RTGCPTR));
+
+        /*
+         * Restore.
+         */
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PVMCPU   pVCpu = &pVM->aCpus[i];
+            uint32_t uCR3  = pVCpu->cpum.s.Hyper.cr3;
+            uint32_t uESP  = pVCpu->cpum.s.Hyper.esp; /* see VMMR3Relocate(). */
+
+            SSMR3GetMem(pSSM, &pVCpu->cpum.s.Hyper, sizeof(pVCpu->cpum.s.Hyper));
+            pVCpu->cpum.s.Hyper.cr3 = uCR3;
+            pVCpu->cpum.s.Hyper.esp = uESP;
+        }
+
+        if (uVersion == CPUM_SAVED_STATE_VERSION_VER1_6)
+        {
+            CPUMCTX_VER1_6 cpumctx16;
+            memset(&pVM->aCpus[0].cpum.s.Guest, 0, sizeof(pVM->aCpus[0].cpum.s.Guest));
+            SSMR3GetMem(pSSM, &cpumctx16, sizeof(cpumctx16));
+
+            /* Save the old cpumctx state into the new one. */
+            cpumR3LoadCPUM1_6(pVM, &cpumctx16);
+
+            SSMR3GetU32(pSSM, &pVM->aCpus[0].cpum.s.fUseFlags);
+            SSMR3GetU32(pSSM, &pVM->aCpus[0].cpum.s.fChanged);
+        }
+        else
+        {
+            if (uVersion >= CPUM_SAVED_STATE_VERSION_VER2_1_NOMSR)
+            {
+                uint32_t cCpus;
+                int rc = SSMR3GetU32(pSSM, &cCpus); AssertRCReturn(rc, rc);
+                AssertLogRelMsgReturn(cCpus == pVM->cCpus, ("Mismatching CPU counts: saved: %u; configured: %u \n", cCpus, pVM->cCpus),
+                                      VERR_SSM_UNEXPECTED_DATA);
+            }
+            AssertLogRelMsgReturn(   uVersion != CPUM_SAVED_STATE_VERSION_VER2_0
+                                  || pVM->cCpus == 1,
+                                  ("cCpus=%u\n", pVM->cCpus),
+                                  VERR_SSM_UNEXPECTED_DATA);
+
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                SSMR3GetMem(pSSM, &pVM->aCpus[i].cpum.s.Guest, sizeof(pVM->aCpus[i].cpum.s.Guest));
+                SSMR3GetU32(pSSM, &pVM->aCpus[i].cpum.s.fUseFlags);
+                SSMR3GetU32(pSSM, &pVM->aCpus[i].cpum.s.fChanged);
+                if (uVersion >= CPUM_SAVED_STATE_VERSION_VER3_0)
+                    SSMR3GetMem(pSSM, &pVM->aCpus[i].cpum.s.GuestMsr, sizeof(pVM->aCpus[i].cpum.s.GuestMsr));
+            }
+        }
+
+        /* Older states does not set CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID for
+           raw-mode guest, so we have to do it ourselves. */
+        if (   uVersion <= CPUM_SAVED_STATE_VERSION_VER3_2
+            && !HWACCMIsEnabled(pVM))
+            for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+                pVM->aCpus[iCpu].cpum.s.fChanged |= CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
+    }
+
+    pVM->cpum.s.fPendingRestore = false;
+
+    /*
+     * Guest CPUIDs.
+     */
+    if (uVersion > CPUM_SAVED_STATE_VERSION_VER3_0)
+        return cpumR3LoadCpuId(pVM, pSSM, uVersion);
+
+    /** @todo Merge the code below into cpumR3LoadCpuId when we've found out what is
+     *        actually required. */
+
+    /*
+     * Restore the CPUID leaves.
+     *
+     * Note that we support restoring less than the current amount of standard
+     * leaves because we've been allowed more is newer version of VBox.
+     */
+    uint32_t cElements;
+    int rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
+    if (cElements > RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdStd[0], cElements*sizeof(pVM->cpum.s.aGuestCpuIdStd[0]));
+
+    rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
+    if (cElements != RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdExt[0], sizeof(pVM->cpum.s.aGuestCpuIdExt));
+
+    rc = SSMR3GetU32(pSSM, &cElements); AssertRCReturn(rc, rc);
+    if (cElements != RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur))
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    SSMR3GetMem(pSSM, &pVM->cpum.s.aGuestCpuIdCentaur[0], sizeof(pVM->cpum.s.aGuestCpuIdCentaur));
+
+    SSMR3GetMem(pSSM, &pVM->cpum.s.GuestCpuIdDef, sizeof(pVM->cpum.s.GuestCpuIdDef));
+
+    /*
+     * Check that the basic cpuid id information is unchanged.
+     */
+    /** @todo we should check the 64 bits capabilities too! */
+    uint32_t au32CpuId[8] = {0,0,0,0, 0,0,0,0};
+    ASMCpuId(0, &au32CpuId[0], &au32CpuId[1], &au32CpuId[2], &au32CpuId[3]);
+    ASMCpuId(1, &au32CpuId[4], &au32CpuId[5], &au32CpuId[6], &au32CpuId[7]);
+    uint32_t au32CpuIdSaved[8];
+    rc = SSMR3GetMem(pSSM, &au32CpuIdSaved[0], sizeof(au32CpuIdSaved));
+    if (RT_SUCCESS(rc))
+    {
+        /* Ignore CPU stepping. */
+        au32CpuId[4]      &=  0xfffffff0;
+        au32CpuIdSaved[4] &=  0xfffffff0;
+
+        /* Ignore APIC ID (AMD specs). */
+        au32CpuId[5]      &= ~0xff000000;
+        au32CpuIdSaved[5] &= ~0xff000000;
+
+        /* Ignore the number of Logical CPUs (AMD specs). */
+        au32CpuId[5]      &= ~0x00ff0000;
+        au32CpuIdSaved[5] &= ~0x00ff0000;
+
+        /* Ignore some advanced capability bits, that we don't expose to the guest. */
+        au32CpuId[6]      &= ~(   X86_CPUID_FEATURE_ECX_DTES64
+                               |  X86_CPUID_FEATURE_ECX_VMX
+                               |  X86_CPUID_FEATURE_ECX_SMX
+                               |  X86_CPUID_FEATURE_ECX_EST
+                               |  X86_CPUID_FEATURE_ECX_TM2
+                               |  X86_CPUID_FEATURE_ECX_CNTXID
+                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
+                               |  X86_CPUID_FEATURE_ECX_PDCM
+                               |  X86_CPUID_FEATURE_ECX_DCA
+                               |  X86_CPUID_FEATURE_ECX_X2APIC
+                              );
+        au32CpuIdSaved[6] &= ~(   X86_CPUID_FEATURE_ECX_DTES64
+                               |  X86_CPUID_FEATURE_ECX_VMX
+                               |  X86_CPUID_FEATURE_ECX_SMX
+                               |  X86_CPUID_FEATURE_ECX_EST
+                               |  X86_CPUID_FEATURE_ECX_TM2
+                               |  X86_CPUID_FEATURE_ECX_CNTXID
+                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
+                               |  X86_CPUID_FEATURE_ECX_PDCM
+                               |  X86_CPUID_FEATURE_ECX_DCA
+                               |  X86_CPUID_FEATURE_ECX_X2APIC
+                              );
+
+        /* Make sure we don't forget to update the masks when enabling
+         * features in the future.
+         */
+        AssertRelease(!(pVM->cpum.s.aGuestCpuIdStd[1].ecx &
+                              (   X86_CPUID_FEATURE_ECX_DTES64
+                               |  X86_CPUID_FEATURE_ECX_VMX
+                               |  X86_CPUID_FEATURE_ECX_SMX
+                               |  X86_CPUID_FEATURE_ECX_EST
+                               |  X86_CPUID_FEATURE_ECX_TM2
+                               |  X86_CPUID_FEATURE_ECX_CNTXID
+                               |  X86_CPUID_FEATURE_ECX_TPRUPDATE
+                               |  X86_CPUID_FEATURE_ECX_PDCM
+                               |  X86_CPUID_FEATURE_ECX_DCA
+                               |  X86_CPUID_FEATURE_ECX_X2APIC
+                              )));
+        /* do the compare */
+        if (memcmp(au32CpuIdSaved, au32CpuId, sizeof(au32CpuIdSaved)))
+        {
+            if (SSMR3HandleGetAfter(pSSM) == SSMAFTER_DEBUG_IT)
+                LogRel(("cpumR3LoadExec: CpuId mismatch! (ignored due to SSMAFTER_DEBUG_IT)\n"
+                        "Saved=%.*Rhxs\n"
+                        "Real =%.*Rhxs\n",
+                        sizeof(au32CpuIdSaved), au32CpuIdSaved,
+                        sizeof(au32CpuId), au32CpuId));
+            else
+            {
+                LogRel(("cpumR3LoadExec: CpuId mismatch!\n"
+                        "Saved=%.*Rhxs\n"
+                        "Real =%.*Rhxs\n",
+                        sizeof(au32CpuIdSaved), au32CpuIdSaved,
+                        sizeof(au32CpuId), au32CpuId));
+                rc = VERR_SSM_LOAD_CPUID_MISMATCH;
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * @copydoc FNSSMINTLOADPREP
+ */
+static DECLCALLBACK(int) cpumR3LoadDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    if (RT_FAILURE(SSMR3HandleGetStatus(pSSM)))
+        return VINF_SUCCESS;
+
+    /* just check this since we can. */ /** @todo Add a SSM unit flag for indicating that it's mandatory during a restore.  */
+    if (pVM->cpum.s.fPendingRestore)
+    {
+        LogRel(("CPUM: Missing state!\n"));
+        return VERR_INTERNAL_ERROR_2;
+    }
+
+    /* Notify PGM of the NXE states in case they've changed. */
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+        PGMNotifyNxeChanged(&pVM->aCpus[iCpu], !!(pVM->aCpus[iCpu].cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the CPUM state restore is still pending.
+ *
+ * @returns true / false.
+ * @param   pVM                 The VM handle.
+ */
+VMMDECL(bool) CPUMR3IsStateRestorePending(PVM pVM)
+{
+    return pVM->cpum.s.fPendingRestore;
+}
+
+
+/**
+ * Formats the EFLAGS value into mnemonics.
+ *
+ * @param   pszEFlags   Where to write the mnemonics. (Assumes sufficient buffer space.)
+ * @param   efl         The EFLAGS value.
+ */
+static void cpumR3InfoFormatFlags(char *pszEFlags, uint32_t efl)
+{
+    /*
+     * Format the flags.
+     */
+    static const struct
+    {
+        const char *pszSet; const char *pszClear; uint32_t fFlag;
+    }   s_aFlags[] =
+    {
+        { "vip",NULL, X86_EFL_VIP },
+        { "vif",NULL, X86_EFL_VIF },
+        { "ac", NULL, X86_EFL_AC },
+        { "vm", NULL, X86_EFL_VM },
+        { "rf", NULL, X86_EFL_RF },
+        { "nt", NULL, X86_EFL_NT },
+        { "ov", "nv", X86_EFL_OF },
+        { "dn", "up", X86_EFL_DF },
+        { "ei", "di", X86_EFL_IF },
+        { "tf", NULL, X86_EFL_TF },
+        { "nt", "pl", X86_EFL_SF },
+        { "nz", "zr", X86_EFL_ZF },
+        { "ac", "na", X86_EFL_AF },
+        { "po", "pe", X86_EFL_PF },
+        { "cy", "nc", X86_EFL_CF },
+    };
+    char *psz = pszEFlags;
+    for (unsigned i = 0; i < RT_ELEMENTS(s_aFlags); i++)
+    {
+        const char *pszAdd = s_aFlags[i].fFlag & efl ? s_aFlags[i].pszSet : s_aFlags[i].pszClear;
+        if (pszAdd)
+        {
+            strcpy(psz, pszAdd);
+            psz += strlen(pszAdd);
+            *psz++ = ' ';
+        }
+    }
+    psz[-1] = '\0';
+}
+
+
+/**
+ * Formats a full register dump.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pCtx        The context to format.
+ * @param   pCtxCore    The context core to format.
+ * @param   pHlp        Output functions.
+ * @param   enmType     The dump type.
+ * @param   pszPrefix   Register name prefix.
+ */
+static void cpumR3InfoOne(PVM pVM, PCPUMCTX pCtx, PCCPUMCTXCORE pCtxCore, PCDBGFINFOHLP pHlp, CPUMDUMPTYPE enmType, const char *pszPrefix)
+{
+    /*
+     * Format the EFLAGS.
+     */
+    uint32_t efl = pCtxCore->eflags.u32;
+    char szEFlags[80];
+    cpumR3InfoFormatFlags(&szEFlags[0], efl);
+
+    /*
+     * Format the registers.
+     */
+    switch (enmType)
+    {
+        case CPUMDUMPTYPE_TERSE:
+            if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                pHlp->pfnPrintf(pHlp,
+                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
+                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
+                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
+                    "%sr14=%016RX64 %sr15=%016RX64\n"
+                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x                %seflags=%08x\n",
+                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
+                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
+                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
+                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
+                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, efl);
+            else
+                pHlp->pfnPrintf(pHlp,
+                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
+                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x                %seflags=%08x\n",
+                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
+                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
+                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, efl);
+            break;
+
+        case CPUMDUMPTYPE_DEFAULT:
+            if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                pHlp->pfnPrintf(pHlp,
+                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
+                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
+                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
+                    "%sr14=%016RX64 %sr15=%016RX64\n"
+                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x %str=%04x      %seflags=%08x\n"
+                    "%scr0=%08RX64 %scr2=%08RX64 %scr3=%08RX64 %scr4=%08RX64 %sgdtr=%016RX64:%04x %sldtr=%04x\n"
+                    ,
+                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
+                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
+                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
+                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
+                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, (RTSEL)pCtx->tr, pszPrefix, efl,
+                    pszPrefix, pCtx->cr0, pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3, pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, (RTSEL)pCtx->ldtr);
+            else
+                pHlp->pfnPrintf(pHlp,
+                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
+                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
+                    "%scs=%04x %sss=%04x %sds=%04x %ses=%04x %sfs=%04x %sgs=%04x %str=%04x      %seflags=%08x\n"
+                    "%scr0=%08RX64 %scr2=%08RX64 %scr3=%08RX64 %scr4=%08RX64 %sgdtr=%08RX64:%04x %sldtr=%04x\n"
+                    ,
+                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
+                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, (RTSEL)pCtxCore->cs, pszPrefix, (RTSEL)pCtxCore->ss, pszPrefix, (RTSEL)pCtxCore->ds, pszPrefix, (RTSEL)pCtxCore->es,
+                    pszPrefix, (RTSEL)pCtxCore->fs, pszPrefix, (RTSEL)pCtxCore->gs, pszPrefix, (RTSEL)pCtx->tr, pszPrefix, efl,
+                    pszPrefix, pCtx->cr0, pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3, pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, (RTSEL)pCtx->ldtr);
+            break;
+
+        case CPUMDUMPTYPE_VERBOSE:
+            if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                pHlp->pfnPrintf(pHlp,
+                    "%srax=%016RX64 %srbx=%016RX64 %srcx=%016RX64 %srdx=%016RX64\n"
+                    "%srsi=%016RX64 %srdi=%016RX64 %sr8 =%016RX64 %sr9 =%016RX64\n"
+                    "%sr10=%016RX64 %sr11=%016RX64 %sr12=%016RX64 %sr13=%016RX64\n"
+                    "%sr14=%016RX64 %sr15=%016RX64\n"
+                    "%srip=%016RX64 %srsp=%016RX64 %srbp=%016RX64 %siopl=%d %*s\n"
+                    "%scs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%ses={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sfs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sgs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%sss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
+                    "%scr0=%016RX64 %scr2=%016RX64 %scr3=%016RX64 %scr4=%016RX64\n"
+                    "%sdr0=%016RX64 %sdr1=%016RX64 %sdr2=%016RX64 %sdr3=%016RX64\n"
+                    "%sdr4=%016RX64 %sdr5=%016RX64 %sdr6=%016RX64 %sdr7=%016RX64\n"
+                    "%sgdtr=%016RX64:%04x  %sidtr=%016RX64:%04x  %seflags=%08x\n"
+                    "%sldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%str  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%sSysEnter={cs=%04llx eip=%016RX64 esp=%016RX64}\n"
+                    ,
+                    pszPrefix, pCtxCore->rax, pszPrefix, pCtxCore->rbx, pszPrefix, pCtxCore->rcx, pszPrefix, pCtxCore->rdx, pszPrefix, pCtxCore->rsi, pszPrefix, pCtxCore->rdi,
+                    pszPrefix, pCtxCore->r8, pszPrefix, pCtxCore->r9, pszPrefix, pCtxCore->r10, pszPrefix, pCtxCore->r11, pszPrefix, pCtxCore->r12, pszPrefix, pCtxCore->r13,
+                    pszPrefix, pCtxCore->r14, pszPrefix, pCtxCore->r15,
+                    pszPrefix, pCtxCore->rip, pszPrefix, pCtxCore->rsp, pszPrefix, pCtxCore->rbp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, (RTSEL)pCtxCore->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u,
+                    pszPrefix, (RTSEL)pCtxCore->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u,
+                    pszPrefix, (RTSEL)pCtxCore->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u,
+                    pszPrefix, (RTSEL)pCtxCore->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u,
+                    pszPrefix, (RTSEL)pCtxCore->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u,
+                    pszPrefix, (RTSEL)pCtxCore->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u,
+                    pszPrefix, pCtx->cr0,  pszPrefix, pCtx->cr2, pszPrefix, pCtx->cr3,  pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->dr[0],  pszPrefix, pCtx->dr[1], pszPrefix, pCtx->dr[2],  pszPrefix, pCtx->dr[3],
+                    pszPrefix, pCtx->dr[4],  pszPrefix, pCtx->dr[5], pszPrefix, pCtx->dr[6],  pszPrefix, pCtx->dr[7],
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, pszPrefix, efl,
+                    pszPrefix, (RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
+                    pszPrefix, (RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
+                    pszPrefix, pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
+            else
+                pHlp->pfnPrintf(pHlp,
+                    "%seax=%08x %sebx=%08x %secx=%08x %sedx=%08x %sesi=%08x %sedi=%08x\n"
+                    "%seip=%08x %sesp=%08x %sebp=%08x %siopl=%d %*s\n"
+                    "%scs={%04x base=%016RX64 limit=%08x flags=%08x} %sdr0=%08RX64 %sdr1=%08RX64\n"
+                    "%sds={%04x base=%016RX64 limit=%08x flags=%08x} %sdr2=%08RX64 %sdr3=%08RX64\n"
+                    "%ses={%04x base=%016RX64 limit=%08x flags=%08x} %sdr4=%08RX64 %sdr5=%08RX64\n"
+                    "%sfs={%04x base=%016RX64 limit=%08x flags=%08x} %sdr6=%08RX64 %sdr7=%08RX64\n"
+                    "%sgs={%04x base=%016RX64 limit=%08x flags=%08x} %scr0=%08RX64 %scr2=%08RX64\n"
+                    "%sss={%04x base=%016RX64 limit=%08x flags=%08x} %scr3=%08RX64 %scr4=%08RX64\n"
+                    "%sgdtr=%016RX64:%04x  %sidtr=%016RX64:%04x  %seflags=%08x\n"
+                    "%sldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%str  ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
+                    "%sSysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
+                    ,
+                    pszPrefix, pCtxCore->eax, pszPrefix, pCtxCore->ebx, pszPrefix, pCtxCore->ecx, pszPrefix, pCtxCore->edx, pszPrefix, pCtxCore->esi, pszPrefix, pCtxCore->edi,
+                    pszPrefix, pCtxCore->eip, pszPrefix, pCtxCore->esp, pszPrefix, pCtxCore->ebp, pszPrefix, X86_EFL_GET_IOPL(efl), *pszPrefix ? 33 : 31, szEFlags,
+                    pszPrefix, (RTSEL)pCtxCore->cs, pCtx->csHid.u64Base, pCtx->csHid.u32Limit, pCtx->csHid.Attr.u, pszPrefix, pCtx->dr[0],  pszPrefix, pCtx->dr[1],
+                    pszPrefix, (RTSEL)pCtxCore->ds, pCtx->dsHid.u64Base, pCtx->dsHid.u32Limit, pCtx->dsHid.Attr.u, pszPrefix, pCtx->dr[2],  pszPrefix, pCtx->dr[3],
+                    pszPrefix, (RTSEL)pCtxCore->es, pCtx->esHid.u64Base, pCtx->esHid.u32Limit, pCtx->esHid.Attr.u, pszPrefix, pCtx->dr[4],  pszPrefix, pCtx->dr[5],
+                    pszPrefix, (RTSEL)pCtxCore->fs, pCtx->fsHid.u64Base, pCtx->fsHid.u32Limit, pCtx->fsHid.Attr.u, pszPrefix, pCtx->dr[6],  pszPrefix, pCtx->dr[7],
+                    pszPrefix, (RTSEL)pCtxCore->gs, pCtx->gsHid.u64Base, pCtx->gsHid.u32Limit, pCtx->gsHid.Attr.u, pszPrefix, pCtx->cr0,  pszPrefix, pCtx->cr2,
+                    pszPrefix, (RTSEL)pCtxCore->ss, pCtx->ssHid.u64Base, pCtx->ssHid.u32Limit, pCtx->ssHid.Attr.u, pszPrefix, pCtx->cr3,  pszPrefix, pCtx->cr4,
+                    pszPrefix, pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pszPrefix, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, pszPrefix, efl,
+                    pszPrefix, (RTSEL)pCtx->ldtr, pCtx->ldtrHid.u64Base, pCtx->ldtrHid.u32Limit, pCtx->ldtrHid.Attr.u,
+                    pszPrefix, (RTSEL)pCtx->tr, pCtx->trHid.u64Base, pCtx->trHid.u32Limit, pCtx->trHid.Attr.u,
+                    pszPrefix, pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
+
+            pHlp->pfnPrintf(pHlp,
+                "%sFCW=%04x %sFSW=%04x %sFTW=%04x %sFOP=%04x %sMXCSR=%08x %sMXCSR_MASK=%08x\n"
+                "%sFPUIP=%08x %sCS=%04x %sRsvrd1=%04x  %sFPUDP=%08x %sDS=%04x %sRsvrd2=%04x\n"
+                ,
+                pszPrefix, pCtx->fpu.FCW,   pszPrefix, pCtx->fpu.FSW, pszPrefix, pCtx->fpu.FTW, pszPrefix, pCtx->fpu.FOP,
+                pszPrefix, pCtx->fpu.MXCSR, pszPrefix, pCtx->fpu.MXCSR_MASK,
+                pszPrefix, pCtx->fpu.FPUIP, pszPrefix, pCtx->fpu.CS,  pszPrefix, pCtx->fpu.Rsvrd1,
+                pszPrefix, pCtx->fpu.FPUDP, pszPrefix, pCtx->fpu.DS,  pszPrefix, pCtx->fpu.Rsrvd2
+                );
+            unsigned iShift = (pCtx->fpu.FSW >> 11) & 7;
+            for (unsigned iST = 0; iST < RT_ELEMENTS(pCtx->fpu.aRegs); iST++)
+            {
+                unsigned iFPR        = (iST + iShift) % RT_ELEMENTS(pCtx->fpu.aRegs);
+                unsigned uTag        = pCtx->fpu.FTW & (1 << iFPR) ? 1 : 0;
+                char     chSign      = pCtx->fpu.aRegs[0].au16[4] & 0x8000 ? '-' : '+';
+                unsigned iInteger    = (unsigned)(pCtx->fpu.aRegs[0].au64[0] >> 63);
+                uint64_t u64Fraction = pCtx->fpu.aRegs[0].au64[0] & UINT64_C(0x7fffffffffffffff);
+                unsigned uExponent   = pCtx->fpu.aRegs[0].au16[4] & 0x7fff;
+                /** @todo This isn't entirenly correct and needs more work! */
+                pHlp->pfnPrintf(pHlp,
+                                "%sST(%u)=%sFPR%u={%04RX16'%08RX32'%08RX32} t%d %c%u.%022llu ^ %u",
+                                pszPrefix, iST, pszPrefix, iFPR,
+                                pCtx->fpu.aRegs[0].au16[4], pCtx->fpu.aRegs[0].au32[1], pCtx->fpu.aRegs[0].au32[0],
+                                uTag, chSign, iInteger, u64Fraction, uExponent);
+                if (pCtx->fpu.aRegs[0].au16[5] || pCtx->fpu.aRegs[0].au16[6] || pCtx->fpu.aRegs[0].au16[7])
+                    pHlp->pfnPrintf(pHlp, " res={%04RX16,%04RX16,%04RX16}\n",
+                                    pCtx->fpu.aRegs[0].au16[5], pCtx->fpu.aRegs[0].au16[6], pCtx->fpu.aRegs[0].au16[7]);
+                else
+                    pHlp->pfnPrintf(pHlp, "\n");
+            }
+            for (unsigned iXMM = 0; iXMM < RT_ELEMENTS(pCtx->fpu.aXMM); iXMM++)
+                pHlp->pfnPrintf(pHlp,
+                                iXMM & 1
+                                ? "%sXMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32\n"
+                                : "%sXMM%u%s=%08RX32'%08RX32'%08RX32'%08RX32  ",
+                                pszPrefix, iXMM, iXMM < 10 ? " " : "",
+                                pCtx->fpu.aXMM[iXMM].au32[3],
+                                pCtx->fpu.aXMM[iXMM].au32[2],
+                                pCtx->fpu.aXMM[iXMM].au32[1],
+                                pCtx->fpu.aXMM[iXMM].au32[0]);
+            for (unsigned i = 0; i < RT_ELEMENTS(pCtx->fpu.au32RsrvdRest); i++)
+                if (pCtx->fpu.au32RsrvdRest[i])
+                    pHlp->pfnPrintf(pHlp, "%sRsrvdRest[i]=%RX32 (offset=%#x)\n",
+                                    pszPrefix, i, pCtx->fpu.au32RsrvdRest[i], RT_OFFSETOF(X86FXSTATE, au32RsrvdRest[i]) );
+
+            pHlp->pfnPrintf(pHlp,
+                "%sEFER         =%016RX64\n"
+                "%sPAT          =%016RX64\n"
+                "%sSTAR         =%016RX64\n"
+                "%sCSTAR        =%016RX64\n"
+                "%sLSTAR        =%016RX64\n"
+                "%sSFMASK       =%016RX64\n"
+                "%sKERNELGSBASE =%016RX64\n",
+                pszPrefix, pCtx->msrEFER,
+                pszPrefix, pCtx->msrPAT,
+                pszPrefix, pCtx->msrSTAR,
+                pszPrefix, pCtx->msrCSTAR,
+                pszPrefix, pCtx->msrLSTAR,
+                pszPrefix, pCtx->msrSFMASK,
+                pszPrefix, pCtx->msrKERNELGSBASE);
+            break;
+    }
+}
+
+
+/**
+ * Display all cpu states and any other cpum info.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoAll(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    cpumR3InfoGuest(pVM, pHlp, pszArgs);
+    cpumR3InfoGuestInstr(pVM, pHlp, pszArgs);
+    cpumR3InfoHyper(pVM, pHlp, pszArgs);
+    cpumR3InfoHost(pVM, pHlp, pszArgs);
+}
+
+
+/**
+ * Parses the info argument.
+ *
+ * The argument starts with 'verbose', 'terse' or 'default' and then
+ * continues with the comment string.
+ *
+ * @param   pszArgs         The pointer to the argument string.
+ * @param   penmType        Where to store the dump type request.
+ * @param   ppszComment     Where to store the pointer to the comment string.
+ */
+static void cpumR3InfoParseArg(const char *pszArgs, CPUMDUMPTYPE *penmType, const char **ppszComment)
+{
+    if (!pszArgs)
+    {
+        *penmType = CPUMDUMPTYPE_DEFAULT;
+        *ppszComment = "";
+    }
+    else
+    {
+        if (!strncmp(pszArgs, "verbose", sizeof("verbose") - 1))
+        {
+            pszArgs += 5;
+            *penmType = CPUMDUMPTYPE_VERBOSE;
+        }
+        else if (!strncmp(pszArgs, "terse", sizeof("terse") - 1))
+        {
+            pszArgs += 5;
+            *penmType = CPUMDUMPTYPE_TERSE;
+        }
+        else if (!strncmp(pszArgs, "default", sizeof("default") - 1))
+        {
+            pszArgs += 7;
+            *penmType = CPUMDUMPTYPE_DEFAULT;
+        }
+        else
+            *penmType = CPUMDUMPTYPE_DEFAULT;
+        *ppszComment = RTStrStripL(pszArgs);
+    }
+}
+
+
+/**
+ * Display the guest cpu state.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    CPUMDUMPTYPE enmType;
+    const char *pszComment;
+    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
+
+    /* @todo SMP support! */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = &pVM->aCpus[0];
+
+    pHlp->pfnPrintf(pHlp, "Guest CPUM (VCPU %d) state: %s\n", pVCpu->idCpu, pszComment);
+
+    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+    cpumR3InfoOne(pVM, pCtx, CPUMCTX2CORE(pCtx), pHlp, enmType, "");
+}
+
+
+/**
+ * Display the current guest instruction
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoGuestInstr(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    char szInstruction[256];
+    /* @todo SMP support! */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = &pVM->aCpus[0];
+
+    int rc = DBGFR3DisasInstrCurrent(pVCpu, szInstruction, sizeof(szInstruction));
+    if (RT_SUCCESS(rc))
+        pHlp->pfnPrintf(pHlp, "\nCPUM: %s\n\n", szInstruction);
+}
+
+
+/**
+ * Display the hypervisor cpu state.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoHyper(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    CPUMDUMPTYPE enmType;
+    const char *pszComment;
+    /* @todo SMP */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
+    pHlp->pfnPrintf(pHlp, "Hypervisor CPUM state: %s\n", pszComment);
+    cpumR3InfoOne(pVM, &pVCpu->cpum.s.Hyper, pVCpu->cpum.s.pHyperCoreR3, pHlp, enmType, ".");
+    pHlp->pfnPrintf(pHlp, "CR4OrMask=%#x CR4AndMask=%#x\n", pVM->cpum.s.CR4.OrMask, pVM->cpum.s.CR4.AndMask);
+}
+
+
+/**
+ * Display the host cpu state.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) cpumR3InfoHost(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    CPUMDUMPTYPE enmType;
+    const char *pszComment;
+    cpumR3InfoParseArg(pszArgs, &enmType, &pszComment);
+    pHlp->pfnPrintf(pHlp, "Host CPUM state: %s\n", pszComment);
+
+    /*
+     * Format the EFLAGS.
+     */
+    /* @todo SMP */
+    PCPUMHOSTCTX pCtx = &pVM->aCpus[0].cpum.s.Host;
+#if HC_ARCH_BITS == 32
+    uint32_t efl = pCtx->eflags.u32;
+#else
+    uint64_t efl = pCtx->rflags;
+#endif
+    char szEFlags[80];
+    cpumR3InfoFormatFlags(&szEFlags[0], efl);
+
+    /*
+     * Format the registers.
+     */
+#if HC_ARCH_BITS == 32
+# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
+    if (!(pCtx->efer & MSR_K6_EFER_LMA))
+# endif
+    {
+        pHlp->pfnPrintf(pHlp,
+            "eax=xxxxxxxx ebx=%08x ecx=xxxxxxxx edx=xxxxxxxx esi=%08x edi=%08x\n"
+            "eip=xxxxxxxx esp=%08x ebp=%08x iopl=%d %31s\n"
+            "cs=%04x ds=%04x es=%04x fs=%04x gs=%04x                       eflags=%08x\n"
+            "cr0=%08RX64 cr2=xxxxxxxx cr3=%08RX64 cr4=%08RX64 gdtr=%08x:%04x ldtr=%04x\n"
+            "dr[0]=%08RX64 dr[1]=%08RX64x dr[2]=%08RX64 dr[3]=%08RX64x dr[6]=%08RX64 dr[7]=%08RX64\n"
+            "SysEnter={cs=%04x eip=%08x esp=%08x}\n"
+            ,
+            /*pCtx->eax,*/ pCtx->ebx, /*pCtx->ecx, pCtx->edx,*/ pCtx->esi, pCtx->edi,
+            /*pCtx->eip,*/ pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(efl), szEFlags,
+            (RTSEL)pCtx->cs, (RTSEL)pCtx->ds, (RTSEL)pCtx->es, (RTSEL)pCtx->fs, (RTSEL)pCtx->gs, efl,
+            pCtx->cr0, /*pCtx->cr2,*/ pCtx->cr3, pCtx->cr4,
+            pCtx->dr0, pCtx->dr1, pCtx->dr2, pCtx->dr3, pCtx->dr6, pCtx->dr7,
+            (uint32_t)pCtx->gdtr.uAddr, pCtx->gdtr.cb, (RTSEL)pCtx->ldtr,
+            pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp);
+    }
+# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
+    else
+# endif
+#endif
+#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    {
+        pHlp->pfnPrintf(pHlp,
+            "rax=xxxxxxxxxxxxxxxx rbx=%016RX64 rcx=xxxxxxxxxxxxxxxx\n"
+            "rdx=xxxxxxxxxxxxxxxx rsi=%016RX64 rdi=%016RX64\n"
+            "rip=xxxxxxxxxxxxxxxx rsp=%016RX64 rbp=%016RX64\n"
+            " r8=xxxxxxxxxxxxxxxx  r9=xxxxxxxxxxxxxxxx r10=%016RX64\n"
+            "r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
+            "r14=%016RX64 r15=%016RX64\n"
+            "iopl=%d  %31s\n"
+            "cs=%04x  ds=%04x  es=%04x  fs=%04x  gs=%04x                   eflags=%08RX64\n"
+            "cr0=%016RX64 cr2=xxxxxxxxxxxxxxxx cr3=%016RX64\n"
+            "cr4=%016RX64 ldtr=%04x tr=%04x\n"
+            "dr[0]=%016RX64 dr[1]=%016RX64 dr[2]=%016RX64\n"
+            "dr[3]=%016RX64 dr[6]=%016RX64 dr[7]=%016RX64\n"
+            "gdtr=%016RX64:%04x  idtr=%016RX64:%04x\n"
+            "SysEnter={cs=%04x eip=%08x esp=%08x}\n"
+            "FSbase=%016RX64 GSbase=%016RX64 efer=%08RX64\n"
+            ,
+            /*pCtx->rax,*/ pCtx->rbx, /*pCtx->rcx,
+            pCtx->rdx,*/ pCtx->rsi, pCtx->rdi,
+            /*pCtx->rip,*/ pCtx->rsp, pCtx->rbp,
+            /*pCtx->r8,  pCtx->r9,*/  pCtx->r10,
+            pCtx->r11, pCtx->r12, pCtx->r13,
+            pCtx->r14, pCtx->r15,
+            X86_EFL_GET_IOPL(efl), szEFlags,
+            (RTSEL)pCtx->cs, (RTSEL)pCtx->ds, (RTSEL)pCtx->es, (RTSEL)pCtx->fs, (RTSEL)pCtx->gs, efl,
+            pCtx->cr0, /*pCtx->cr2,*/ pCtx->cr3,
+            pCtx->cr4, pCtx->ldtr, pCtx->tr,
+            pCtx->dr0, pCtx->dr1, pCtx->dr2,
+            pCtx->dr3, pCtx->dr6, pCtx->dr7,
+            pCtx->gdtr.uAddr, pCtx->gdtr.cb, pCtx->idtr.uAddr, pCtx->idtr.cb,
+            pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp,
+            pCtx->FSbase, pCtx->GSbase, pCtx->efer);
+    }
+#endif
+}
+
+
+/**
+ * Get L1 cache / TLS associativity.
+ */
+static const char *getCacheAss(unsigned u, char *pszBuf)
+{
+    if (u == 0)
+        return "res0  ";
+    if (u == 1)
+        return "direct";
+    if (u == 255)
+        return "fully";
+    if (u >= 256)
+        return "???";
+
+    RTStrPrintf(pszBuf, 16, "%d way", u);
+    return pszBuf;
+}
+
+
+/**
+ * Get L2 cache associativity.
+ */
+const char *getL2CacheAss(unsigned u)
+{
+    switch (u)
+    {
+        case 0:  return "off   ";
+        case 1:  return "direct";
+        case 2:  return "2 way ";
+        case 3:  return "res3  ";
+        case 4:  return "4 way ";
+        case 5:  return "res5  ";
+        case 6:  return "8 way ";
+        case 7:  return "res7  ";
+        case 8:  return "16 way";
+        case 9:  return "res9  ";
+        case 10: return "res10 ";
+        case 11: return "res11 ";
+        case 12: return "res12 ";
+        case 13: return "res13 ";
+        case 14: return "res14 ";
+        case 15: return "fully ";
+        default: return "????";
+    }
+}
+
+
+/**
+ * Display the guest CpuId leaves.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helper functions.
+ * @param   pszArgs     "terse", "default" or "verbose".
+ */
+static DECLCALLBACK(void) cpumR3CpuIdInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /*
+     * Parse the argument.
+     */
+    unsigned iVerbosity = 1;
+    if (pszArgs)
+    {
+        pszArgs = RTStrStripL(pszArgs);
+        if (!strcmp(pszArgs, "terse"))
+            iVerbosity--;
+        else if (!strcmp(pszArgs, "verbose"))
+            iVerbosity++;
+    }
+
+    /*
+     * Start cracking.
+     */
+    CPUMCPUID   Host;
+    CPUMCPUID   Guest;
+    unsigned    cStdMax = pVM->cpum.s.aGuestCpuIdStd[0].eax;
+
+    pHlp->pfnPrintf(pHlp,
+                    "         RAW Standard CPUIDs\n"
+                    "     Function  eax      ebx      ecx      edx\n");
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdStd); i++)
+    {
+        Guest = pVM->cpum.s.aGuestCpuIdStd[i];
+        ASMCpuId_Idx_ECX(i, 0, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
+
+        pHlp->pfnPrintf(pHlp,
+                        "Gst: %08x  %08x %08x %08x %08x%s\n"
+                        "Hst:           %08x %08x %08x %08x\n",
+                        i, Guest.eax, Guest.ebx, Guest.ecx, Guest.edx,
+                        i <= cStdMax ? "" : "*",
+                        Host.eax, Host.ebx, Host.ecx, Host.edx);
+    }
+
+    /*
+     * If verbose, decode it.
+     */
+    if (iVerbosity)
+    {
+        Guest = pVM->cpum.s.aGuestCpuIdStd[0];
+        pHlp->pfnPrintf(pHlp,
+                        "Name:                            %.04s%.04s%.04s\n"
+                        "Supports:                        0-%x\n",
+                        &Guest.ebx, &Guest.edx, &Guest.ecx, Guest.eax);
+    }
+
+    /*
+     * Get Features.
+     */
+    bool const fIntel = ASMIsIntelCpuEx(pVM->cpum.s.aGuestCpuIdStd[0].ebx,
+                                        pVM->cpum.s.aGuestCpuIdStd[0].ecx,
+                                        pVM->cpum.s.aGuestCpuIdStd[0].edx);
+    if (cStdMax >= 1 && iVerbosity)
+    {
+        static const char * const s_apszTypes[4] = { "primary", "overdrive", "MP", "reserved" };
+
+        Guest = pVM->cpum.s.aGuestCpuIdStd[1];
+        uint32_t uEAX = Guest.eax;
+
+        pHlp->pfnPrintf(pHlp,
+                        "Family:                          %d  \tExtended: %d \tEffective: %d\n"
+                        "Model:                           %d  \tExtended: %d \tEffective: %d\n"
+                        "Stepping:                        %d\n"
+                        "Type:                            %d (%s)\n"
+                        "APIC ID:                         %#04x\n"
+                        "Logical CPUs:                    %d\n"
+                        "CLFLUSH Size:                    %d\n"
+                        "Brand ID:                        %#04x\n",
+                        (uEAX >> 8) & 0xf, (uEAX >> 20) & 0x7f, ASMGetCpuFamily(uEAX),
+                        (uEAX >> 4) & 0xf, (uEAX >> 16) & 0x0f, ASMGetCpuModel(uEAX, fIntel),
+                        ASMGetCpuStepping(uEAX),
+                        (uEAX >> 12) & 3, s_apszTypes[(uEAX >> 12) & 3],
+                        (Guest.ebx >> 24) & 0xff,
+                        (Guest.ebx >> 16) & 0xff,
+                        (Guest.ebx >>  8) & 0xff,
+                        (Guest.ebx >>  0) & 0xff);
+        if (iVerbosity == 1)
+        {
+            uint32_t uEDX = Guest.edx;
+            pHlp->pfnPrintf(pHlp, "Features EDX:                   ");
+            if (uEDX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " FPU");
+            if (uEDX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " VME");
+            if (uEDX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " DE");
+            if (uEDX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " PSE");
+            if (uEDX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " TSC");
+            if (uEDX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " MSR");
+            if (uEDX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " PAE");
+            if (uEDX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " MCE");
+            if (uEDX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " CX8");
+            if (uEDX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " APIC");
+            if (uEDX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " 10");
+            if (uEDX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " SEP");
+            if (uEDX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " MTRR");
+            if (uEDX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " PGE");
+            if (uEDX & RT_BIT(14))  pHlp->pfnPrintf(pHlp, " MCA");
+            if (uEDX & RT_BIT(15))  pHlp->pfnPrintf(pHlp, " CMOV");
+            if (uEDX & RT_BIT(16))  pHlp->pfnPrintf(pHlp, " PAT");
+            if (uEDX & RT_BIT(17))  pHlp->pfnPrintf(pHlp, " PSE36");
+            if (uEDX & RT_BIT(18))  pHlp->pfnPrintf(pHlp, " PSN");
+            if (uEDX & RT_BIT(19))  pHlp->pfnPrintf(pHlp, " CLFSH");
+            if (uEDX & RT_BIT(20))  pHlp->pfnPrintf(pHlp, " 20");
+            if (uEDX & RT_BIT(21))  pHlp->pfnPrintf(pHlp, " DS");
+            if (uEDX & RT_BIT(22))  pHlp->pfnPrintf(pHlp, " ACPI");
+            if (uEDX & RT_BIT(23))  pHlp->pfnPrintf(pHlp, " MMX");
+            if (uEDX & RT_BIT(24))  pHlp->pfnPrintf(pHlp, " FXSR");
+            if (uEDX & RT_BIT(25))  pHlp->pfnPrintf(pHlp, " SSE");
+            if (uEDX & RT_BIT(26))  pHlp->pfnPrintf(pHlp, " SSE2");
+            if (uEDX & RT_BIT(27))  pHlp->pfnPrintf(pHlp, " SS");
+            if (uEDX & RT_BIT(28))  pHlp->pfnPrintf(pHlp, " HTT");
+            if (uEDX & RT_BIT(29))  pHlp->pfnPrintf(pHlp, " TM");
+            if (uEDX & RT_BIT(30))  pHlp->pfnPrintf(pHlp, " 30");
+            if (uEDX & RT_BIT(31))  pHlp->pfnPrintf(pHlp, " PBE");
+            pHlp->pfnPrintf(pHlp, "\n");
+
+            uint32_t uECX = Guest.ecx;
+            pHlp->pfnPrintf(pHlp, "Features ECX:                   ");
+            if (uECX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " SSE3");
+            if (uECX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " PCLMUL");
+            if (uECX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " DTES64");
+            if (uECX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " MONITOR");
+            if (uECX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " DS-CPL");
+            if (uECX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " VMX");
+            if (uECX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " SMX");
+            if (uECX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " EST");
+            if (uECX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " TM2");
+            if (uECX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " SSSE3");
+            if (uECX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " CNXT-ID");
+            if (uECX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " 11");
+            if (uECX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " FMA");
+            if (uECX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " CX16");
+            if (uECX & RT_BIT(14))  pHlp->pfnPrintf(pHlp, " TPRUPDATE");
+            if (uECX & RT_BIT(15))  pHlp->pfnPrintf(pHlp, " PDCM");
+            if (uECX & RT_BIT(16))  pHlp->pfnPrintf(pHlp, " 16");
+            if (uECX & RT_BIT(17))  pHlp->pfnPrintf(pHlp, " PCID");
+            if (uECX & RT_BIT(18))  pHlp->pfnPrintf(pHlp, " DCA");
+            if (uECX & RT_BIT(19))  pHlp->pfnPrintf(pHlp, " SSE4.1");
+            if (uECX & RT_BIT(20))  pHlp->pfnPrintf(pHlp, " SSE4.2");
+            if (uECX & RT_BIT(21))  pHlp->pfnPrintf(pHlp, " X2APIC");
+            if (uECX & RT_BIT(22))  pHlp->pfnPrintf(pHlp, " MOVBE");
+            if (uECX & RT_BIT(23))  pHlp->pfnPrintf(pHlp, " POPCNT");
+            if (uECX & RT_BIT(24))  pHlp->pfnPrintf(pHlp, " TSCDEADL");
+            if (uECX & RT_BIT(25))  pHlp->pfnPrintf(pHlp, " AES");
+            if (uECX & RT_BIT(26))  pHlp->pfnPrintf(pHlp, " XSAVE");
+            if (uECX & RT_BIT(27))  pHlp->pfnPrintf(pHlp, " OSXSAVE");
+            if (uECX & RT_BIT(28))  pHlp->pfnPrintf(pHlp, " AVX");
+            if (uECX & RT_BIT(29))  pHlp->pfnPrintf(pHlp, " 29");
+            if (uECX & RT_BIT(30))  pHlp->pfnPrintf(pHlp, " 30");
+            if (uECX & RT_BIT(31))  pHlp->pfnPrintf(pHlp, " 31");
+            pHlp->pfnPrintf(pHlp, "\n");
+        }
+        else
+        {
+            ASMCpuId(1, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
+
+            X86CPUIDFEATEDX EdxHost  = *(PX86CPUIDFEATEDX)&Host.edx;
+            X86CPUIDFEATECX EcxHost  = *(PX86CPUIDFEATECX)&Host.ecx;
+            X86CPUIDFEATEDX EdxGuest = *(PX86CPUIDFEATEDX)&Guest.edx;
+            X86CPUIDFEATECX EcxGuest = *(PX86CPUIDFEATECX)&Guest.ecx;
+
+            pHlp->pfnPrintf(pHlp, "Mnemonic - Description                 = guest (host)\n");
+            pHlp->pfnPrintf(pHlp, "FPU - x87 FPU on Chip                  = %d (%d)\n",  EdxGuest.u1FPU,        EdxHost.u1FPU);
+            pHlp->pfnPrintf(pHlp, "VME - Virtual 8086 Mode Enhancements   = %d (%d)\n",  EdxGuest.u1VME,        EdxHost.u1VME);
+            pHlp->pfnPrintf(pHlp, "DE - Debugging extensions              = %d (%d)\n",  EdxGuest.u1DE,         EdxHost.u1DE);
+            pHlp->pfnPrintf(pHlp, "PSE - Page Size Extension              = %d (%d)\n",  EdxGuest.u1PSE,        EdxHost.u1PSE);
+            pHlp->pfnPrintf(pHlp, "TSC - Time Stamp Counter               = %d (%d)\n",  EdxGuest.u1TSC,        EdxHost.u1TSC);
+            pHlp->pfnPrintf(pHlp, "MSR - Model Specific Registers         = %d (%d)\n",  EdxGuest.u1MSR,        EdxHost.u1MSR);
+            pHlp->pfnPrintf(pHlp, "PAE - Physical Address Extension       = %d (%d)\n",  EdxGuest.u1PAE,        EdxHost.u1PAE);
+            pHlp->pfnPrintf(pHlp, "MCE - Machine Check Exception          = %d (%d)\n",  EdxGuest.u1MCE,        EdxHost.u1MCE);
+            pHlp->pfnPrintf(pHlp, "CX8 - CMPXCHG8B instruction            = %d (%d)\n",  EdxGuest.u1CX8,        EdxHost.u1CX8);
+            pHlp->pfnPrintf(pHlp, "APIC - APIC On-Chip                    = %d (%d)\n",  EdxGuest.u1APIC,       EdxHost.u1APIC);
+            pHlp->pfnPrintf(pHlp, "10 - Reserved                          = %d (%d)\n",  EdxGuest.u1Reserved1,  EdxHost.u1Reserved1);
+            pHlp->pfnPrintf(pHlp, "SEP - SYSENTER and SYSEXIT             = %d (%d)\n",  EdxGuest.u1SEP,        EdxHost.u1SEP);
+            pHlp->pfnPrintf(pHlp, "MTRR - Memory Type Range Registers     = %d (%d)\n",  EdxGuest.u1MTRR,       EdxHost.u1MTRR);
+            pHlp->pfnPrintf(pHlp, "PGE - PTE Global Bit                   = %d (%d)\n",  EdxGuest.u1PGE,        EdxHost.u1PGE);
+            pHlp->pfnPrintf(pHlp, "MCA - Machine Check Architecture       = %d (%d)\n",  EdxGuest.u1MCA,        EdxHost.u1MCA);
+            pHlp->pfnPrintf(pHlp, "CMOV - Conditional Move Instructions   = %d (%d)\n",  EdxGuest.u1CMOV,       EdxHost.u1CMOV);
+            pHlp->pfnPrintf(pHlp, "PAT - Page Attribute Table             = %d (%d)\n",  EdxGuest.u1PAT,        EdxHost.u1PAT);
+            pHlp->pfnPrintf(pHlp, "PSE-36 - 36-bit Page Size Extention    = %d (%d)\n",  EdxGuest.u1PSE36,      EdxHost.u1PSE36);
+            pHlp->pfnPrintf(pHlp, "PSN - Processor Serial Number          = %d (%d)\n",  EdxGuest.u1PSN,        EdxHost.u1PSN);
+            pHlp->pfnPrintf(pHlp, "CLFSH - CLFLUSH Instruction.           = %d (%d)\n",  EdxGuest.u1CLFSH,      EdxHost.u1CLFSH);
+            pHlp->pfnPrintf(pHlp, "20 - Reserved                          = %d (%d)\n",  EdxGuest.u1Reserved2,  EdxHost.u1Reserved2);
+            pHlp->pfnPrintf(pHlp, "DS - Debug Store                       = %d (%d)\n",  EdxGuest.u1DS,         EdxHost.u1DS);
+            pHlp->pfnPrintf(pHlp, "ACPI - Thermal Mon. & Soft. Clock Ctrl.= %d (%d)\n",  EdxGuest.u1ACPI,       EdxHost.u1ACPI);
+            pHlp->pfnPrintf(pHlp, "MMX - Intel MMX Technology             = %d (%d)\n",  EdxGuest.u1MMX,        EdxHost.u1MMX);
+            pHlp->pfnPrintf(pHlp, "FXSR - FXSAVE and FXRSTOR Instructions = %d (%d)\n",  EdxGuest.u1FXSR,       EdxHost.u1FXSR);
+            pHlp->pfnPrintf(pHlp, "SSE - SSE Support                      = %d (%d)\n",  EdxGuest.u1SSE,        EdxHost.u1SSE);
+            pHlp->pfnPrintf(pHlp, "SSE2 - SSE2 Support                    = %d (%d)\n",  EdxGuest.u1SSE2,       EdxHost.u1SSE2);
+            pHlp->pfnPrintf(pHlp, "SS - Self Snoop                        = %d (%d)\n",  EdxGuest.u1SS,         EdxHost.u1SS);
+            pHlp->pfnPrintf(pHlp, "HTT - Hyper-Threading Technology       = %d (%d)\n",  EdxGuest.u1HTT,        EdxHost.u1HTT);
+            pHlp->pfnPrintf(pHlp, "TM - Thermal Monitor                   = %d (%d)\n",  EdxGuest.u1TM,         EdxHost.u1TM);
+            pHlp->pfnPrintf(pHlp, "30 - Reserved                          = %d (%d)\n",  EdxGuest.u1Reserved3,  EdxHost.u1Reserved3);
+            pHlp->pfnPrintf(pHlp, "PBE - Pending Break Enable             = %d (%d)\n",  EdxGuest.u1PBE,        EdxHost.u1PBE);
+
+            pHlp->pfnPrintf(pHlp, "Supports SSE3                          = %d (%d)\n",  EcxGuest.u1SSE3,       EcxHost.u1SSE3);
+            pHlp->pfnPrintf(pHlp, "PCLMULQDQ                              = %d (%d)\n",  EcxGuest.u1PCLMULQDQ,  EcxHost.u1PCLMULQDQ);
+            pHlp->pfnPrintf(pHlp, "DS Area 64-bit layout                  = %d (%d)\n",  EcxGuest.u1DTE64,      EcxHost.u1DTE64);
+            pHlp->pfnPrintf(pHlp, "Supports MONITOR/MWAIT                 = %d (%d)\n",  EcxGuest.u1Monitor,    EcxHost.u1Monitor);
+            pHlp->pfnPrintf(pHlp, "CPL-DS - CPL Qualified Debug Store     = %d (%d)\n",  EcxGuest.u1CPLDS,      EcxHost.u1CPLDS);
+            pHlp->pfnPrintf(pHlp, "VMX - Virtual Machine Technology       = %d (%d)\n",  EcxGuest.u1VMX,        EcxHost.u1VMX);
+            pHlp->pfnPrintf(pHlp, "SMX - Safer Mode Extensions            = %d (%d)\n",  EcxGuest.u1SMX,        EcxHost.u1SMX);
+            pHlp->pfnPrintf(pHlp, "Enhanced SpeedStep Technology          = %d (%d)\n",  EcxGuest.u1EST,        EcxHost.u1EST);
+            pHlp->pfnPrintf(pHlp, "Terminal Monitor 2                     = %d (%d)\n",  EcxGuest.u1TM2,        EcxHost.u1TM2);
+            pHlp->pfnPrintf(pHlp, "Supplemental SSE3 instructions         = %d (%d)\n",  EcxGuest.u1SSSE3,      EcxHost.u1SSSE3);
+            pHlp->pfnPrintf(pHlp, "L1 Context ID                          = %d (%d)\n",  EcxGuest.u1CNTXID,     EcxHost.u1CNTXID);
+            pHlp->pfnPrintf(pHlp, "11 - Reserved                          = %d (%d)\n",  EcxGuest.u1Reserved1,  EcxHost.u1Reserved1);
+            pHlp->pfnPrintf(pHlp, "FMA extensions using YMM state         = %d (%d)\n",  EcxGuest.u1FMA,        EcxHost.u1FMA);
+            pHlp->pfnPrintf(pHlp, "CMPXCHG16B instruction                 = %d (%d)\n",  EcxGuest.u1CX16,       EcxHost.u1CX16);
+            pHlp->pfnPrintf(pHlp, "xTPR Update Control                    = %d (%d)\n",  EcxGuest.u1TPRUpdate,  EcxHost.u1TPRUpdate);
+            pHlp->pfnPrintf(pHlp, "Perf/Debug Capability MSR              = %d (%d)\n",  EcxGuest.u1PDCM,       EcxHost.u1PDCM);
+            pHlp->pfnPrintf(pHlp, "16 - Reserved                          = %d (%d)\n",  EcxGuest.u1Reserved2,  EcxHost.u1Reserved2);
+            pHlp->pfnPrintf(pHlp, "PCID - Process-context identifiers     = %d (%d)\n",  EcxGuest.u1PCID,       EcxHost.u1PCID);
+            pHlp->pfnPrintf(pHlp, "DCA - Direct Cache Access              = %d (%d)\n",  EcxGuest.u1DCA,        EcxHost.u1DCA);
+            pHlp->pfnPrintf(pHlp, "SSE4.1 instruction extensions          = %d (%d)\n",  EcxGuest.u1SSE4_1,     EcxHost.u1SSE4_1);
+            pHlp->pfnPrintf(pHlp, "SSE4.2 instruction extensions          = %d (%d)\n",  EcxGuest.u1SSE4_2,     EcxHost.u1SSE4_2);
+            pHlp->pfnPrintf(pHlp, "Supports the x2APIC extensions         = %d (%d)\n",  EcxGuest.u1x2APIC,     EcxHost.u1x2APIC);
+            pHlp->pfnPrintf(pHlp, "MOVBE instruction                      = %d (%d)\n",  EcxGuest.u1MOVBE,      EcxHost.u1MOVBE);
+            pHlp->pfnPrintf(pHlp, "POPCNT instruction                     = %d (%d)\n",  EcxGuest.u1POPCNT,     EcxHost.u1POPCNT);
+            pHlp->pfnPrintf(pHlp, "TSC-Deadline LAPIC timer mode          = %d (%d)\n",  EcxGuest.u1TSCDEADLINE,EcxHost.u1TSCDEADLINE);
+            pHlp->pfnPrintf(pHlp, "AESNI instruction extensions           = %d (%d)\n",  EcxGuest.u1AES,        EcxHost.u1AES);
+            pHlp->pfnPrintf(pHlp, "XSAVE/XRSTOR extended state feature    = %d (%d)\n",  EcxGuest.u1XSAVE,      EcxHost.u1XSAVE);
+            pHlp->pfnPrintf(pHlp, "Supports OSXSAVE                       = %d (%d)\n",  EcxGuest.u1OSXSAVE,    EcxHost.u1OSXSAVE);
+            pHlp->pfnPrintf(pHlp, "AVX instruction extensions             = %d (%d)\n",  EcxGuest.u1AVX,        EcxHost.u1AVX);
+            pHlp->pfnPrintf(pHlp, "29/30 - Reserved                       = %#x (%#x)\n",EcxGuest.u2Reserved3,  EcxHost.u2Reserved3);
+            pHlp->pfnPrintf(pHlp, "31 - Reserved (always 0)               = %d (%d)\n",  EcxGuest.u1Reserved4,  EcxHost.u1Reserved4);
+        }
+    }
+    if (cStdMax >= 2 && iVerbosity)
+    {
+        /** @todo */
+    }
+
+    /*
+     * Extended.
+     * Implemented after AMD specs.
+     */
+    unsigned    cExtMax = pVM->cpum.s.aGuestCpuIdExt[0].eax & 0xffff;
+
+    pHlp->pfnPrintf(pHlp,
+                    "\n"
+                    "         RAW Extended CPUIDs\n"
+                    "     Function  eax      ebx      ecx      edx\n");
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdExt); i++)
+    {
+        Guest = pVM->cpum.s.aGuestCpuIdExt[i];
+        ASMCpuId(0x80000000 | i, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
+
+        pHlp->pfnPrintf(pHlp,
+                        "Gst: %08x  %08x %08x %08x %08x%s\n"
+                        "Hst:           %08x %08x %08x %08x\n",
+                        0x80000000 | i, Guest.eax, Guest.ebx, Guest.ecx, Guest.edx,
+                        i <= cExtMax ? "" : "*",
+                        Host.eax, Host.ebx, Host.ecx, Host.edx);
+    }
+
+    /*
+     * Understandable output
+     */
+    if (iVerbosity)
+    {
+        Guest = pVM->cpum.s.aGuestCpuIdExt[0];
+        pHlp->pfnPrintf(pHlp,
+                        "Ext Name:                        %.4s%.4s%.4s\n"
+                        "Ext Supports:                    0x80000000-%#010x\n",
+                        &Guest.ebx, &Guest.edx, &Guest.ecx, Guest.eax);
+    }
+
+    if (iVerbosity && cExtMax >= 1)
+    {
+        Guest = pVM->cpum.s.aGuestCpuIdExt[1];
+        uint32_t uEAX = Guest.eax;
+        pHlp->pfnPrintf(pHlp,
+                        "Family:                          %d  \tExtended: %d \tEffective: %d\n"
+                        "Model:                           %d  \tExtended: %d \tEffective: %d\n"
+                        "Stepping:                        %d\n"
+                        "Brand ID:                        %#05x\n",
+                        (uEAX >> 8) & 0xf, (uEAX >> 20) & 0x7f, ASMGetCpuFamily(uEAX),
+                        (uEAX >> 4) & 0xf, (uEAX >> 16) & 0x0f, ASMGetCpuModel(uEAX, fIntel),
+                        ASMGetCpuStepping(uEAX),
+                        Guest.ebx & 0xfff);
+
+        if (iVerbosity == 1)
+        {
+            uint32_t uEDX = Guest.edx;
+            pHlp->pfnPrintf(pHlp, "Features EDX:                   ");
+            if (uEDX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " FPU");
+            if (uEDX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " VME");
+            if (uEDX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " DE");
+            if (uEDX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " PSE");
+            if (uEDX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " TSC");
+            if (uEDX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " MSR");
+            if (uEDX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " PAE");
+            if (uEDX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " MCE");
+            if (uEDX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " CX8");
+            if (uEDX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " APIC");
+            if (uEDX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " 10");
+            if (uEDX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " SCR");
+            if (uEDX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " MTRR");
+            if (uEDX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " PGE");
+            if (uEDX & RT_BIT(14))  pHlp->pfnPrintf(pHlp, " MCA");
+            if (uEDX & RT_BIT(15))  pHlp->pfnPrintf(pHlp, " CMOV");
+            if (uEDX & RT_BIT(16))  pHlp->pfnPrintf(pHlp, " PAT");
+            if (uEDX & RT_BIT(17))  pHlp->pfnPrintf(pHlp, " PSE36");
+            if (uEDX & RT_BIT(18))  pHlp->pfnPrintf(pHlp, " 18");
+            if (uEDX & RT_BIT(19))  pHlp->pfnPrintf(pHlp, " 19");
+            if (uEDX & RT_BIT(20))  pHlp->pfnPrintf(pHlp, " NX");
+            if (uEDX & RT_BIT(21))  pHlp->pfnPrintf(pHlp, " 21");
+            if (uEDX & RT_BIT(22))  pHlp->pfnPrintf(pHlp, " ExtMMX");
+            if (uEDX & RT_BIT(23))  pHlp->pfnPrintf(pHlp, " MMX");
+            if (uEDX & RT_BIT(24))  pHlp->pfnPrintf(pHlp, " FXSR");
+            if (uEDX & RT_BIT(25))  pHlp->pfnPrintf(pHlp, " FastFXSR");
+            if (uEDX & RT_BIT(26))  pHlp->pfnPrintf(pHlp, " Page1GB");
+            if (uEDX & RT_BIT(27))  pHlp->pfnPrintf(pHlp, " RDTSCP");
+            if (uEDX & RT_BIT(28))  pHlp->pfnPrintf(pHlp, " 28");
+            if (uEDX & RT_BIT(29))  pHlp->pfnPrintf(pHlp, " LongMode");
+            if (uEDX & RT_BIT(30))  pHlp->pfnPrintf(pHlp, " Ext3DNow");
+            if (uEDX & RT_BIT(31))  pHlp->pfnPrintf(pHlp, " 3DNow");
+            pHlp->pfnPrintf(pHlp, "\n");
+
+            uint32_t uECX = Guest.ecx;
+            pHlp->pfnPrintf(pHlp, "Features ECX:                   ");
+            if (uECX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " LAHF/SAHF");
+            if (uECX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " CMPL");
+            if (uECX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " SVM");
+            if (uECX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " ExtAPIC");
+            if (uECX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " CR8L");
+            if (uECX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " ABM");
+            if (uECX & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " SSE4A");
+            if (uECX & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " MISALNSSE");
+            if (uECX & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " 3DNOWPRF");
+            if (uECX & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " OSVW");
+            if (uECX & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " IBS");
+            if (uECX & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " SSE5");
+            if (uECX & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " SKINIT");
+            if (uECX & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " WDT");
+            for (unsigned iBit = 5; iBit < 32; iBit++)
+                if (uECX & RT_BIT(iBit))
+                    pHlp->pfnPrintf(pHlp, " %d", iBit);
+            pHlp->pfnPrintf(pHlp, "\n");
+        }
+        else
+        {
+            ASMCpuId(0x80000001, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
+
+            uint32_t uEdxGst = Guest.edx;
+            uint32_t uEdxHst = Host.edx;
+            pHlp->pfnPrintf(pHlp, "Mnemonic - Description                 = guest (host)\n");
+            pHlp->pfnPrintf(pHlp, "FPU - x87 FPU on Chip                  = %d (%d)\n",  !!(uEdxGst & RT_BIT( 0)),  !!(uEdxHst & RT_BIT( 0)));
+            pHlp->pfnPrintf(pHlp, "VME - Virtual 8086 Mode Enhancements   = %d (%d)\n",  !!(uEdxGst & RT_BIT( 1)),  !!(uEdxHst & RT_BIT( 1)));
+            pHlp->pfnPrintf(pHlp, "DE - Debugging extensions              = %d (%d)\n",  !!(uEdxGst & RT_BIT( 2)),  !!(uEdxHst & RT_BIT( 2)));
+            pHlp->pfnPrintf(pHlp, "PSE - Page Size Extension              = %d (%d)\n",  !!(uEdxGst & RT_BIT( 3)),  !!(uEdxHst & RT_BIT( 3)));
+            pHlp->pfnPrintf(pHlp, "TSC - Time Stamp Counter               = %d (%d)\n",  !!(uEdxGst & RT_BIT( 4)),  !!(uEdxHst & RT_BIT( 4)));
+            pHlp->pfnPrintf(pHlp, "MSR - K86 Model Specific Registers     = %d (%d)\n",  !!(uEdxGst & RT_BIT( 5)),  !!(uEdxHst & RT_BIT( 5)));
+            pHlp->pfnPrintf(pHlp, "PAE - Physical Address Extension       = %d (%d)\n",  !!(uEdxGst & RT_BIT( 6)),  !!(uEdxHst & RT_BIT( 6)));
+            pHlp->pfnPrintf(pHlp, "MCE - Machine Check Exception          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 7)),  !!(uEdxHst & RT_BIT( 7)));
+            pHlp->pfnPrintf(pHlp, "CX8 - CMPXCHG8B instruction            = %d (%d)\n",  !!(uEdxGst & RT_BIT( 8)),  !!(uEdxHst & RT_BIT( 8)));
+            pHlp->pfnPrintf(pHlp, "APIC - APIC On-Chip                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 9)),  !!(uEdxHst & RT_BIT( 9)));
+            pHlp->pfnPrintf(pHlp, "10 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(10)),  !!(uEdxHst & RT_BIT(10)));
+            pHlp->pfnPrintf(pHlp, "SEP - SYSCALL and SYSRET               = %d (%d)\n",  !!(uEdxGst & RT_BIT(11)),  !!(uEdxHst & RT_BIT(11)));
+            pHlp->pfnPrintf(pHlp, "MTRR - Memory Type Range Registers     = %d (%d)\n",  !!(uEdxGst & RT_BIT(12)),  !!(uEdxHst & RT_BIT(12)));
+            pHlp->pfnPrintf(pHlp, "PGE - PTE Global Bit                   = %d (%d)\n",  !!(uEdxGst & RT_BIT(13)),  !!(uEdxHst & RT_BIT(13)));
+            pHlp->pfnPrintf(pHlp, "MCA - Machine Check Architecture       = %d (%d)\n",  !!(uEdxGst & RT_BIT(14)),  !!(uEdxHst & RT_BIT(14)));
+            pHlp->pfnPrintf(pHlp, "CMOV - Conditional Move Instructions   = %d (%d)\n",  !!(uEdxGst & RT_BIT(15)),  !!(uEdxHst & RT_BIT(15)));
+            pHlp->pfnPrintf(pHlp, "PAT - Page Attribute Table             = %d (%d)\n",  !!(uEdxGst & RT_BIT(16)),  !!(uEdxHst & RT_BIT(16)));
+            pHlp->pfnPrintf(pHlp, "PSE-36 - 36-bit Page Size Extention    = %d (%d)\n",  !!(uEdxGst & RT_BIT(17)),  !!(uEdxHst & RT_BIT(17)));
+            pHlp->pfnPrintf(pHlp, "18 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(18)),  !!(uEdxHst & RT_BIT(18)));
+            pHlp->pfnPrintf(pHlp, "19 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(19)),  !!(uEdxHst & RT_BIT(19)));
+            pHlp->pfnPrintf(pHlp, "NX - No-Execute Page Protection        = %d (%d)\n",  !!(uEdxGst & RT_BIT(20)),  !!(uEdxHst & RT_BIT(20)));
+            pHlp->pfnPrintf(pHlp, "DS - Debug Store                       = %d (%d)\n",  !!(uEdxGst & RT_BIT(21)),  !!(uEdxHst & RT_BIT(21)));
+            pHlp->pfnPrintf(pHlp, "AXMMX - AMD Extensions to MMX Instr.   = %d (%d)\n",  !!(uEdxGst & RT_BIT(22)),  !!(uEdxHst & RT_BIT(22)));
+            pHlp->pfnPrintf(pHlp, "MMX - Intel MMX Technology             = %d (%d)\n",  !!(uEdxGst & RT_BIT(23)),  !!(uEdxHst & RT_BIT(23)));
+            pHlp->pfnPrintf(pHlp, "FXSR - FXSAVE and FXRSTOR Instructions = %d (%d)\n",  !!(uEdxGst & RT_BIT(24)),  !!(uEdxHst & RT_BIT(24)));
+            pHlp->pfnPrintf(pHlp, "25 - AMD fast FXSAVE and FXRSTOR Instr.= %d (%d)\n",  !!(uEdxGst & RT_BIT(25)),  !!(uEdxHst & RT_BIT(25)));
+            pHlp->pfnPrintf(pHlp, "26 - 1 GB large page support           = %d (%d)\n",  !!(uEdxGst & RT_BIT(26)),  !!(uEdxHst & RT_BIT(26)));
+            pHlp->pfnPrintf(pHlp, "27 - RDTSCP instruction                = %d (%d)\n",  !!(uEdxGst & RT_BIT(27)),  !!(uEdxHst & RT_BIT(27)));
+            pHlp->pfnPrintf(pHlp, "28 - Reserved                          = %d (%d)\n",  !!(uEdxGst & RT_BIT(28)),  !!(uEdxHst & RT_BIT(28)));
+            pHlp->pfnPrintf(pHlp, "29 - AMD Long Mode                     = %d (%d)\n",  !!(uEdxGst & RT_BIT(29)),  !!(uEdxHst & RT_BIT(29)));
+            pHlp->pfnPrintf(pHlp, "30 - AMD Extensions to 3DNow           = %d (%d)\n",  !!(uEdxGst & RT_BIT(30)),  !!(uEdxHst & RT_BIT(30)));
+            pHlp->pfnPrintf(pHlp, "31 - AMD 3DNow                         = %d (%d)\n",  !!(uEdxGst & RT_BIT(31)),  !!(uEdxHst & RT_BIT(31)));
+
+            uint32_t uEcxGst = Guest.ecx;
+            uint32_t uEcxHst = Host.ecx;
+            pHlp->pfnPrintf(pHlp, "LahfSahf - LAHF/SAHF in 64-bit mode    = %d (%d)\n",  !!(uEcxGst & RT_BIT( 0)),  !!(uEcxHst & RT_BIT( 0)));
+            pHlp->pfnPrintf(pHlp, "CmpLegacy - Core MP legacy mode (depr) = %d (%d)\n",  !!(uEcxGst & RT_BIT( 1)),  !!(uEcxHst & RT_BIT( 1)));
+            pHlp->pfnPrintf(pHlp, "SVM - AMD VM Extensions                = %d (%d)\n",  !!(uEcxGst & RT_BIT( 2)),  !!(uEcxHst & RT_BIT( 2)));
+            pHlp->pfnPrintf(pHlp, "APIC registers starting at 0x400       = %d (%d)\n",  !!(uEcxGst & RT_BIT( 3)),  !!(uEcxHst & RT_BIT( 3)));
+            pHlp->pfnPrintf(pHlp, "AltMovCR8 - LOCK MOV CR0 means MOV CR8 = %d (%d)\n",  !!(uEcxGst & RT_BIT( 4)),  !!(uEcxHst & RT_BIT( 4)));
+            pHlp->pfnPrintf(pHlp, "Advanced bit manipulation              = %d (%d)\n",  !!(uEcxGst & RT_BIT( 5)),  !!(uEcxHst & RT_BIT( 5)));
+            pHlp->pfnPrintf(pHlp, "SSE4A instruction support              = %d (%d)\n",  !!(uEcxGst & RT_BIT( 6)),  !!(uEcxHst & RT_BIT( 6)));
+            pHlp->pfnPrintf(pHlp, "Misaligned SSE mode                    = %d (%d)\n",  !!(uEcxGst & RT_BIT( 7)),  !!(uEcxHst & RT_BIT( 7)));
+            pHlp->pfnPrintf(pHlp, "PREFETCH and PREFETCHW instruction     = %d (%d)\n",  !!(uEcxGst & RT_BIT( 8)),  !!(uEcxHst & RT_BIT( 8)));
+            pHlp->pfnPrintf(pHlp, "OS visible workaround                  = %d (%d)\n",  !!(uEcxGst & RT_BIT( 9)),  !!(uEcxHst & RT_BIT( 9)));
+            pHlp->pfnPrintf(pHlp, "Instruction based sampling             = %d (%d)\n",  !!(uEcxGst & RT_BIT(10)),  !!(uEcxHst & RT_BIT(10)));
+            pHlp->pfnPrintf(pHlp, "SSE5 support                           = %d (%d)\n",  !!(uEcxGst & RT_BIT(11)),  !!(uEcxHst & RT_BIT(11)));
+            pHlp->pfnPrintf(pHlp, "SKINIT, STGI, and DEV support          = %d (%d)\n",  !!(uEcxGst & RT_BIT(12)),  !!(uEcxHst & RT_BIT(12)));
+            pHlp->pfnPrintf(pHlp, "Watchdog timer support.                = %d (%d)\n",  !!(uEcxGst & RT_BIT(13)),  !!(uEcxHst & RT_BIT(13)));
+            pHlp->pfnPrintf(pHlp, "31:14 - Reserved                       = %#x (%#x)\n",   uEcxGst >> 14,          uEcxHst >> 14);
+        }
+    }
+
+    if (iVerbosity && cExtMax >= 2)
+    {
+        char szString[4*4*3+1] = {0};
+        uint32_t *pu32 = (uint32_t *)szString;
+        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].eax;
+        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].ebx;
+        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].ecx;
+        *pu32++ = pVM->cpum.s.aGuestCpuIdExt[2].edx;
+        if (cExtMax >= 3)
+        {
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].eax;
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].ebx;
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].ecx;
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[3].edx;
+        }
+        if (cExtMax >= 4)
+        {
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].eax;
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].ebx;
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].ecx;
+            *pu32++ = pVM->cpum.s.aGuestCpuIdExt[4].edx;
+        }
+        pHlp->pfnPrintf(pHlp, "Full Name:                       %s\n", szString);
+    }
+
+    if (iVerbosity && cExtMax >= 5)
+    {
+        uint32_t uEAX = pVM->cpum.s.aGuestCpuIdExt[5].eax;
+        uint32_t uEBX = pVM->cpum.s.aGuestCpuIdExt[5].ebx;
+        uint32_t uECX = pVM->cpum.s.aGuestCpuIdExt[5].ecx;
+        uint32_t uEDX = pVM->cpum.s.aGuestCpuIdExt[5].edx;
+        char sz1[32];
+        char sz2[32];
+
+        pHlp->pfnPrintf(pHlp,
+                        "TLB 2/4M Instr/Uni:              %s %3d entries\n"
+                        "TLB 2/4M Data:                   %s %3d entries\n",
+                        getCacheAss((uEAX >>  8) & 0xff, sz1), (uEAX >>  0) & 0xff,
+                        getCacheAss((uEAX >> 24) & 0xff, sz2), (uEAX >> 16) & 0xff);
+        pHlp->pfnPrintf(pHlp,
+                        "TLB 4K Instr/Uni:                %s %3d entries\n"
+                        "TLB 4K Data:                     %s %3d entries\n",
+                        getCacheAss((uEBX >>  8) & 0xff, sz1), (uEBX >>  0) & 0xff,
+                        getCacheAss((uEBX >> 24) & 0xff, sz2), (uEBX >> 16) & 0xff);
+        pHlp->pfnPrintf(pHlp, "L1 Instr Cache Line Size:        %d bytes\n"
+                        "L1 Instr Cache Lines Per Tag:    %d\n"
+                        "L1 Instr Cache Associativity:    %s\n"
+                        "L1 Instr Cache Size:             %d KB\n",
+                        (uEDX >> 0) & 0xff,
+                        (uEDX >> 8) & 0xff,
+                        getCacheAss((uEDX >> 16) & 0xff, sz1),
+                        (uEDX >> 24) & 0xff);
+        pHlp->pfnPrintf(pHlp,
+                        "L1 Data Cache Line Size:         %d bytes\n"
+                        "L1 Data Cache Lines Per Tag:     %d\n"
+                        "L1 Data Cache Associativity:     %s\n"
+                        "L1 Data Cache Size:              %d KB\n",
+                        (uECX >> 0) & 0xff,
+                        (uECX >> 8) & 0xff,
+                        getCacheAss((uECX >> 16) & 0xff, sz1),
+                        (uECX >> 24) & 0xff);
+    }
+
+    if (iVerbosity && cExtMax >= 6)
+    {
+        uint32_t uEAX = pVM->cpum.s.aGuestCpuIdExt[6].eax;
+        uint32_t uEBX = pVM->cpum.s.aGuestCpuIdExt[6].ebx;
+        uint32_t uEDX = pVM->cpum.s.aGuestCpuIdExt[6].edx;
+
+        pHlp->pfnPrintf(pHlp,
+                        "L2 TLB 2/4M Instr/Uni:           %s %4d entries\n"
+                        "L2 TLB 2/4M Data:                %s %4d entries\n",
+                        getL2CacheAss((uEAX >> 12) & 0xf),  (uEAX >>  0) & 0xfff,
+                        getL2CacheAss((uEAX >> 28) & 0xf),  (uEAX >> 16) & 0xfff);
+        pHlp->pfnPrintf(pHlp,
+                        "L2 TLB 4K Instr/Uni:             %s %4d entries\n"
+                        "L2 TLB 4K Data:                  %s %4d entries\n",
+                        getL2CacheAss((uEBX >> 12) & 0xf),  (uEBX >>  0) & 0xfff,
+                        getL2CacheAss((uEBX >> 28) & 0xf),  (uEBX >> 16) & 0xfff);
+        pHlp->pfnPrintf(pHlp,
+                        "L2 Cache Line Size:              %d bytes\n"
+                        "L2 Cache Lines Per Tag:          %d\n"
+                        "L2 Cache Associativity:          %s\n"
+                        "L2 Cache Size:                   %d KB\n",
+                        (uEDX >> 0) & 0xff,
+                        (uEDX >> 8) & 0xf,
+                        getL2CacheAss((uEDX >> 12) & 0xf),
+                        (uEDX >> 16) & 0xffff);
+    }
+
+    if (iVerbosity && cExtMax >= 7)
+    {
+        uint32_t uEDX = pVM->cpum.s.aGuestCpuIdExt[7].edx;
+
+        pHlp->pfnPrintf(pHlp, "APM Features:                   ");
+        if (uEDX & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " TS");
+        if (uEDX & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " FID");
+        if (uEDX & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " VID");
+        if (uEDX & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " TTP");
+        if (uEDX & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " TM");
+        if (uEDX & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " STC");
+        for (unsigned iBit = 6; iBit < 32; iBit++)
+            if (uEDX & RT_BIT(iBit))
+                pHlp->pfnPrintf(pHlp, " %d", iBit);
+        pHlp->pfnPrintf(pHlp, "\n");
+    }
+
+    if (iVerbosity && cExtMax >= 8)
+    {
+        uint32_t uEAX = pVM->cpum.s.aGuestCpuIdExt[8].eax;
+        uint32_t uECX = pVM->cpum.s.aGuestCpuIdExt[8].ecx;
+
+        pHlp->pfnPrintf(pHlp,
+                        "Physical Address Width:          %d bits\n"
+                        "Virtual Address Width:           %d bits\n"
+                        "Guest Physical Address Width:    %d bits\n",
+                        (uEAX >> 0) & 0xff,
+                        (uEAX >> 8) & 0xff,
+                        (uEAX >> 16) & 0xff);
+        pHlp->pfnPrintf(pHlp,
+                        "Physical Core Count:             %d\n",
+                        (uECX >> 0) & 0xff);
+    }
+
+
+    /*
+     * Centaur.
+     */
+    unsigned cCentaurMax = pVM->cpum.s.aGuestCpuIdCentaur[0].eax & 0xffff;
+
+    pHlp->pfnPrintf(pHlp,
+                    "\n"
+                    "         RAW Centaur CPUIDs\n"
+                    "     Function  eax      ebx      ecx      edx\n");
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->cpum.s.aGuestCpuIdCentaur); i++)
+    {
+        Guest = pVM->cpum.s.aGuestCpuIdCentaur[i];
+        ASMCpuId(0xc0000000 | i, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
+
+        pHlp->pfnPrintf(pHlp,
+                        "Gst: %08x  %08x %08x %08x %08x%s\n"
+                        "Hst:           %08x %08x %08x %08x\n",
+                        0xc0000000 | i, Guest.eax, Guest.ebx, Guest.ecx, Guest.edx,
+                        i <= cCentaurMax ? "" : "*",
+                        Host.eax, Host.ebx, Host.ecx, Host.edx);
+    }
+
+    /*
+     * Understandable output
+     */
+    if (iVerbosity)
+    {
+        Guest = pVM->cpum.s.aGuestCpuIdCentaur[0];
+        pHlp->pfnPrintf(pHlp,
+                        "Centaur Supports:                0xc0000000-%#010x\n",
+                        Guest.eax);
+    }
+
+    if (iVerbosity && cCentaurMax >= 1)
+    {
+        ASMCpuId(0xc0000001, &Host.eax, &Host.ebx, &Host.ecx, &Host.edx);
+        uint32_t uEdxGst = pVM->cpum.s.aGuestCpuIdExt[1].edx;
+        uint32_t uEdxHst = Host.edx;
+
+        if (iVerbosity == 1)
+        {
+            pHlp->pfnPrintf(pHlp, "Centaur Features EDX:           ");
+            if (uEdxGst & RT_BIT(0))   pHlp->pfnPrintf(pHlp, " AIS");
+            if (uEdxGst & RT_BIT(1))   pHlp->pfnPrintf(pHlp, " AIS-E");
+            if (uEdxGst & RT_BIT(2))   pHlp->pfnPrintf(pHlp, " RNG");
+            if (uEdxGst & RT_BIT(3))   pHlp->pfnPrintf(pHlp, " RNG-E");
+            if (uEdxGst & RT_BIT(4))   pHlp->pfnPrintf(pHlp, " LH");
+            if (uEdxGst & RT_BIT(5))   pHlp->pfnPrintf(pHlp, " FEMMS");
+            if (uEdxGst & RT_BIT(6))   pHlp->pfnPrintf(pHlp, " ACE");
+            if (uEdxGst & RT_BIT(7))   pHlp->pfnPrintf(pHlp, " ACE-E");
+            /* possibly indicating MM/HE and MM/HE-E on older chips... */
+            if (uEdxGst & RT_BIT(8))   pHlp->pfnPrintf(pHlp, " ACE2");
+            if (uEdxGst & RT_BIT(9))   pHlp->pfnPrintf(pHlp, " ACE2-E");
+            if (uEdxGst & RT_BIT(10))  pHlp->pfnPrintf(pHlp, " PHE");
+            if (uEdxGst & RT_BIT(11))  pHlp->pfnPrintf(pHlp, " PHE-E");
+            if (uEdxGst & RT_BIT(12))  pHlp->pfnPrintf(pHlp, " PMM");
+            if (uEdxGst & RT_BIT(13))  pHlp->pfnPrintf(pHlp, " PMM-E");
+            for (unsigned iBit = 14; iBit < 32; iBit++)
+                if (uEdxGst & RT_BIT(iBit))
+                    pHlp->pfnPrintf(pHlp, " %d", iBit);
+            pHlp->pfnPrintf(pHlp, "\n");
+        }
+        else
+        {
+            pHlp->pfnPrintf(pHlp, "Mnemonic - Description                 = guest (host)\n");
+            pHlp->pfnPrintf(pHlp, "AIS - Alternate Instruction Set        = %d (%d)\n",  !!(uEdxGst & RT_BIT( 0)),  !!(uEdxHst & RT_BIT( 0)));
+            pHlp->pfnPrintf(pHlp, "AIS-E - AIS enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 1)),  !!(uEdxHst & RT_BIT( 1)));
+            pHlp->pfnPrintf(pHlp, "RNG - Random Number Generator          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 2)),  !!(uEdxHst & RT_BIT( 2)));
+            pHlp->pfnPrintf(pHlp, "RNG-E - RNG enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 3)),  !!(uEdxHst & RT_BIT( 3)));
+            pHlp->pfnPrintf(pHlp, "LH - LongHaul MSR 0000_110Ah           = %d (%d)\n",  !!(uEdxGst & RT_BIT( 4)),  !!(uEdxHst & RT_BIT( 4)));
+            pHlp->pfnPrintf(pHlp, "FEMMS - FEMMS                          = %d (%d)\n",  !!(uEdxGst & RT_BIT( 5)),  !!(uEdxHst & RT_BIT( 5)));
+            pHlp->pfnPrintf(pHlp, "ACE - Advanced Cryptography Engine     = %d (%d)\n",  !!(uEdxGst & RT_BIT( 6)),  !!(uEdxHst & RT_BIT( 6)));
+            pHlp->pfnPrintf(pHlp, "ACE-E - ACE enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT( 7)),  !!(uEdxHst & RT_BIT( 7)));
+            /* possibly indicating MM/HE and MM/HE-E on older chips... */
+            pHlp->pfnPrintf(pHlp, "ACE2 - Advanced Cryptography Engine 2  = %d (%d)\n",  !!(uEdxGst & RT_BIT( 8)),  !!(uEdxHst & RT_BIT( 8)));
+            pHlp->pfnPrintf(pHlp, "ACE2-E - ACE enabled                   = %d (%d)\n",  !!(uEdxGst & RT_BIT( 9)),  !!(uEdxHst & RT_BIT( 9)));
+            pHlp->pfnPrintf(pHlp, "PHE - Hash Engine                      = %d (%d)\n",  !!(uEdxGst & RT_BIT(10)),  !!(uEdxHst & RT_BIT(10)));
+            pHlp->pfnPrintf(pHlp, "PHE-E - PHE enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT(11)),  !!(uEdxHst & RT_BIT(11)));
+            pHlp->pfnPrintf(pHlp, "PMM - Montgomery Multiplier            = %d (%d)\n",  !!(uEdxGst & RT_BIT(12)),  !!(uEdxHst & RT_BIT(12)));
+            pHlp->pfnPrintf(pHlp, "PMM-E - PMM enabled                    = %d (%d)\n",  !!(uEdxGst & RT_BIT(13)),  !!(uEdxHst & RT_BIT(13)));
+            for (unsigned iBit = 14; iBit < 32; iBit++)
+                if ((uEdxGst | uEdxHst) & RT_BIT(iBit))
+                    pHlp->pfnPrintf(pHlp, "Bit %d                                 = %d (%d)\n",  !!(uEdxGst & RT_BIT(iBit)),  !!(uEdxHst & RT_BIT(iBit)));
+            pHlp->pfnPrintf(pHlp, "\n");
+        }
+    }
+}
+
+
+/**
+ * Structure used when disassembling and instructions in DBGF.
+ * This is used so the reader function can get the stuff it needs.
+ */
+typedef struct CPUMDISASSTATE
+{
+    /** Pointer to the CPU structure. */
+    PDISCPUSTATE    pCpu;
+    /** The VM handle. */
+    PVM             pVM;
+    /** The VMCPU handle. */
+    PVMCPU          pVCpu;
+    /** Pointer to the first byte in the segment. */
+    RTGCUINTPTR     GCPtrSegBase;
+    /** Pointer to the byte after the end of the segment. (might have wrapped!) */
+    RTGCUINTPTR     GCPtrSegEnd;
+    /** The size of the segment minus 1. */
+    RTGCUINTPTR     cbSegLimit;
+    /** Pointer to the current page - R3 Ptr. */
+    void const     *pvPageR3;
+    /** Pointer to the current page - GC Ptr. */
+    RTGCPTR         pvPageGC;
+    /** The lock information that PGMPhysReleasePageMappingLock needs. */
+    PGMPAGEMAPLOCK  PageMapLock;
+    /** Whether the PageMapLock is valid or not. */
+    bool            fLocked;
+    /** 64 bits mode or not. */
+    bool            f64Bits;
+} CPUMDISASSTATE, *PCPUMDISASSTATE;
+
+
+/**
+ * Instruction reader.
+ *
+ * @returns VBox status code.
+ * @param   PtrSrc      Address to read from.
+ *                      In our case this is relative to the selector pointed to by the 2nd user argument of uDisCpu.
+ * @param   pu8Dst      Where to store the bytes.
+ * @param   cbRead      Number of bytes to read.
+ * @param   uDisCpu     Pointer to the disassembler cpu state.
+ *                      In this context it's always pointer to the Core of a DBGFDISASSTATE.
+ */
+static DECLCALLBACK(int) cpumR3DisasInstrRead(RTUINTPTR PtrSrc, uint8_t *pu8Dst, unsigned cbRead, void *uDisCpu)
+{
+    PDISCPUSTATE pCpu = (PDISCPUSTATE)uDisCpu;
+    PCPUMDISASSTATE pState = (PCPUMDISASSTATE)pCpu->apvUserData[0];
+    Assert(cbRead > 0);
+    for (;;)
+    {
+        RTGCUINTPTR GCPtr = PtrSrc + pState->GCPtrSegBase;
+
+        /* Need to update the page translation? */
+        if (    !pState->pvPageR3
+            ||  (GCPtr >> PAGE_SHIFT) != (pState->pvPageGC >> PAGE_SHIFT))
+        {
+            int rc = VINF_SUCCESS;
+
+            /* translate the address */
+            pState->pvPageGC = GCPtr & PAGE_BASE_GC_MASK;
+            if (    MMHyperIsInsideArea(pState->pVM, pState->pvPageGC)
+                &&  !HWACCMIsEnabled(pState->pVM))
+            {
+                pState->pvPageR3 = MMHyperRCToR3(pState->pVM, (RTRCPTR)pState->pvPageGC);
+                if (!pState->pvPageR3)
+                    rc = VERR_INVALID_POINTER;
+            }
+            else
+            {
+                /* Release mapping lock previously acquired. */
+                if (pState->fLocked)
+                    PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
+                rc = PGMPhysGCPtr2CCPtrReadOnly(pState->pVCpu, pState->pvPageGC, &pState->pvPageR3, &pState->PageMapLock);
+                pState->fLocked = RT_SUCCESS_NP(rc);
+            }
+            if (RT_FAILURE(rc))
+            {
+                pState->pvPageR3 = NULL;
+                return rc;
+            }
+        }
+
+        /* check the segment limit */
+        if (!pState->f64Bits && PtrSrc > pState->cbSegLimit)
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+
+        /* calc how much we can read */
+        uint32_t cb = PAGE_SIZE - (GCPtr & PAGE_OFFSET_MASK);
+        if (!pState->f64Bits)
+        {
+            RTGCUINTPTR cbSeg = pState->GCPtrSegEnd - GCPtr;
+            if (cb > cbSeg && cbSeg)
+                cb = cbSeg;
+        }
+        if (cb > cbRead)
+            cb = cbRead;
+
+        /* read and advance */
+        memcpy(pu8Dst, (char *)pState->pvPageR3 + (GCPtr & PAGE_OFFSET_MASK), cb);
+        cbRead -= cb;
+        if (!cbRead)
+            return VINF_SUCCESS;
+        pu8Dst += cb;
+        PtrSrc += cb;
+    }
+}
+
+
+/**
+ * Disassemble an instruction and return the information in the provided structure.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM Handle
+ * @param   pVCpu       VMCPU Handle
+ * @param   pCtx        CPU context
+ * @param   GCPtrPC     Program counter (relative to CS) to disassemble from.
+ * @param   pCpu        Disassembly state
+ * @param   pszPrefix   String prefix for logging (debug only)
+ *
+ */
+VMMR3DECL(int) CPUMR3DisasmInstrCPU(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTGCPTR GCPtrPC, PDISCPUSTATE pCpu, const char *pszPrefix)
+{
+    CPUMDISASSTATE  State;
+    int             rc;
+
+    const PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+    State.pCpu            = pCpu;
+    State.pvPageGC        = 0;
+    State.pvPageR3        = NULL;
+    State.pVM             = pVM;
+    State.pVCpu           = pVCpu;
+    State.fLocked         = false;
+    State.f64Bits         = false;
+
+    /*
+     * Get selector information.
+     */
+    if (    (pCtx->cr0 & X86_CR0_PE)
+        &&   pCtx->eflags.Bits.u1VM == 0)
+    {
+        if (CPUMAreHiddenSelRegsValid(pVCpu))
+        {
+            State.f64Bits         = enmMode >= PGMMODE_AMD64 && pCtx->csHid.Attr.n.u1Long;
+            State.GCPtrSegBase    = pCtx->csHid.u64Base;
+            State.GCPtrSegEnd     = pCtx->csHid.u32Limit + 1 + (RTGCUINTPTR)pCtx->csHid.u64Base;
+            State.cbSegLimit      = pCtx->csHid.u32Limit;
+            pCpu->mode            = (State.f64Bits)
+                                    ? CPUMODE_64BIT
+                                    : pCtx->csHid.Attr.n.u1DefBig
+                                    ? CPUMODE_32BIT
+                                    : CPUMODE_16BIT;
+        }
+        else
+        {
+            DBGFSELINFO SelInfo;
+
+            rc = SELMR3GetShadowSelectorInfo(pVM, pCtx->cs, &SelInfo);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("SELMR3GetShadowSelectorInfo failed for %04X:%RGv rc=%d\n", pCtx->cs, GCPtrPC, rc));
+                return rc;
+            }
+
+            /*
+             * Validate the selector.
+             */
+            rc = DBGFR3SelInfoValidateCS(&SelInfo, pCtx->ss);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("SELMSelInfoValidateCS failed for %04X:%RGv rc=%d\n", pCtx->cs, GCPtrPC, rc));
+                return rc;
+            }
+            State.GCPtrSegBase    = SelInfo.GCPtrBase;
+            State.GCPtrSegEnd     = SelInfo.cbLimit + 1 + (RTGCUINTPTR)SelInfo.GCPtrBase;
+            State.cbSegLimit      = SelInfo.cbLimit;
+            pCpu->mode            = SelInfo.u.Raw.Gen.u1DefBig ? CPUMODE_32BIT : CPUMODE_16BIT;
+        }
+    }
+    else
+    {
+        /* real or V86 mode */
+        pCpu->mode            = CPUMODE_16BIT;
+        State.GCPtrSegBase    = pCtx->cs * 16;
+        State.GCPtrSegEnd     = 0xFFFFFFFF;
+        State.cbSegLimit      = 0xFFFFFFFF;
+    }
+
+    /*
+     * Disassemble the instruction.
+     */
+    pCpu->pfnReadBytes    = cpumR3DisasInstrRead;
+    pCpu->apvUserData[0]  = &State;
+
+    uint32_t cbInstr;
+#ifndef LOG_ENABLED
+    rc = DISInstr(pCpu, GCPtrPC, 0, &cbInstr, NULL);
+    if (RT_SUCCESS(rc))
+    {
+#else
+    char szOutput[160];
+    rc = DISInstr(pCpu, GCPtrPC, 0, &cbInstr, &szOutput[0]);
+    if (RT_SUCCESS(rc))
+    {
+        /* log it */
+        if (pszPrefix)
+            Log(("%s-CPU%d: %s", pszPrefix, pVCpu->idCpu, szOutput));
+        else
+            Log(("%s", szOutput));
+#endif
+        rc = VINF_SUCCESS;
+    }
+    else
+        Log(("CPUMR3DisasmInstrCPU: DISInstr failed for %04X:%RGv rc=%Rrc\n", pCtx->cs, GCPtrPC, rc));
+
+    /* Release mapping lock acquired in cpumR3DisasInstrRead. */
+    if (State.fLocked)
+        PGMPhysReleasePageMappingLock(pVM, &State.PageMapLock);
+
+    return rc;
+}
+
+#ifdef DEBUG
+
+/**
+ * Disassemble an instruction and dump it to the log
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM Handle
+ * @param   pVCpu       VMCPU Handle
+ * @param   pCtx        CPU context
+ * @param   pc          GC instruction pointer
+ * @param   pszPrefix   String prefix for logging
+ *
+ * @deprecated  Use DBGFR3DisasInstrCurrentLog().
+ */
+VMMR3DECL(void) CPUMR3DisasmInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTGCPTR pc, const char *pszPrefix)
+{
+    DISCPUSTATE Cpu;
+    CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pc, &Cpu, pszPrefix);
+}
+
+
+/**
+ * Debug helper - Saves guest context on raw mode entry (for fatal dump)
+ *
+ * @internal
+ */
+VMMR3DECL(void) CPUMR3SaveEntryCtx(PVM pVM)
+{
+    /** @todo SMP support!! */
+    pVM->cpum.s.GuestEntry = *CPUMQueryGuestCtxPtr(VMMGetCpu(pVM));
+}
+
+#endif /* DEBUG */
+
+/**
+ * API for controlling a few of the CPU features found in CR4.
+ *
+ * Currently only X86_CR4_TSD is accepted as input.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM     The VM handle.
+ * @param   fOr     The CR4 OR mask.
+ * @param   fAnd    The CR4 AND mask.
+ */
+VMMR3DECL(int) CPUMR3SetCR4Feature(PVM pVM, RTHCUINTREG fOr, RTHCUINTREG fAnd)
+{
+    AssertMsgReturn(!(fOr & ~(X86_CR4_TSD)), ("%#x\n", fOr), VERR_INVALID_PARAMETER);
+    AssertMsgReturn((fAnd & ~(X86_CR4_TSD)) == ~(X86_CR4_TSD), ("%#x\n", fAnd), VERR_INVALID_PARAMETER);
+
+    pVM->cpum.s.CR4.OrMask &= fAnd;
+    pVM->cpum.s.CR4.OrMask |= fOr;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets a pointer to the array of standard CPUID leaves.
+ *
+ * CPUMR3GetGuestCpuIdStdMax() give the size of the array.
+ *
+ * @returns Pointer to the standard CPUID leaves (read-only).
+ * @param   pVM         The VM handle.
+ * @remark  Intended for PATM.
+ */
+VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdStdRCPtr(PVM pVM)
+{
+    return RCPTRTYPE(PCCPUMCPUID)VM_RC_ADDR(pVM, &pVM->cpum.s.aGuestCpuIdStd[0]);
+}
+
+
+/**
+ * Gets a pointer to the array of extended CPUID leaves.
+ *
+ * CPUMGetGuestCpuIdExtMax() give the size of the array.
+ *
+ * @returns Pointer to the extended CPUID leaves (read-only).
+ * @param   pVM         The VM handle.
+ * @remark  Intended for PATM.
+ */
+VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdExtRCPtr(PVM pVM)
+{
+    return (RCPTRTYPE(PCCPUMCPUID))VM_RC_ADDR(pVM, &pVM->cpum.s.aGuestCpuIdExt[0]);
+}
+
+
+/**
+ * Gets a pointer to the array of centaur CPUID leaves.
+ *
+ * CPUMGetGuestCpuIdCentaurMax() give the size of the array.
+ *
+ * @returns Pointer to the centaur CPUID leaves (read-only).
+ * @param   pVM         The VM handle.
+ * @remark  Intended for PATM.
+ */
+VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdCentaurRCPtr(PVM pVM)
+{
+    return (RCPTRTYPE(PCCPUMCPUID))VM_RC_ADDR(pVM, &pVM->cpum.s.aGuestCpuIdCentaur[0]);
+}
+
+
+/**
+ * Gets a pointer to the default CPUID leaf.
+ *
+ * @returns Pointer to the default CPUID leaf (read-only).
+ * @param   pVM         The VM handle.
+ * @remark  Intended for PATM.
+ */
+VMMR3DECL(RCPTRTYPE(PCCPUMCPUID)) CPUMR3GetGuestCpuIdDefRCPtr(PVM pVM)
+{
+    return (RCPTRTYPE(PCCPUMCPUID))VM_RC_ADDR(pVM, &pVM->cpum.s.GuestCpuIdDef);
+}
+
+
+/**
+ * Transforms the guest CPU state to raw-ring mode.
+ *
+ * This function will change the any of the cs and ss register with DPL=0 to DPL=1.
+ *
+ * @returns VBox status. (recompiler failure)
+ * @param   pVCpu       The VMCPU handle.
+ * @param   pCtxCore    The context core (for trap usage).
+ * @see     @ref pg_raw
+ */
+VMMR3DECL(int) CPUMR3RawEnter(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    Assert(!pVCpu->cpum.s.fRawEntered);
+    Assert(!pVCpu->cpum.s.fRemEntered);
+    if (!pCtxCore)
+        pCtxCore = CPUMCTX2CORE(&pVCpu->cpum.s.Guest);
+
+    /*
+     * Are we in Ring-0?
+     */
+    if (    pCtxCore->ss && (pCtxCore->ss & X86_SEL_RPL) == 0
+        &&  !pCtxCore->eflags.Bits.u1VM)
+    {
+        /*
+         * Enter execution mode.
+         */
+        PATMRawEnter(pVM, pCtxCore);
+
+        /*
+         * Set CPL to Ring-1.
+         */
+        pCtxCore->ss |= 1;
+        if (pCtxCore->cs && (pCtxCore->cs & X86_SEL_RPL) == 0)
+            pCtxCore->cs |= 1;
+    }
+    else
+    {
+        AssertMsg((pCtxCore->ss & X86_SEL_RPL) >= 2 || pCtxCore->eflags.Bits.u1VM,
+                  ("ring-1 code not supported\n"));
+        /*
+         * PATM takes care of IOPL and IF flags for Ring-3 and Ring-2 code as well.
+         */
+        PATMRawEnter(pVM, pCtxCore);
+    }
+
+    /*
+     * Invalidate the hidden registers.
+     */
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
+
+    /*
+     * Assert sanity.
+     */
+    AssertMsg((pCtxCore->eflags.u32 & X86_EFL_IF), ("X86_EFL_IF is clear\n"));
+    AssertReleaseMsg(   pCtxCore->eflags.Bits.u2IOPL < (unsigned)(pCtxCore->ss & X86_SEL_RPL)
+                     || pCtxCore->eflags.Bits.u1VM,
+                     ("X86_EFL_IOPL=%d CPL=%d\n", pCtxCore->eflags.Bits.u2IOPL, pCtxCore->ss & X86_SEL_RPL));
+    Assert((pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE)) == (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP));
+
+    pCtxCore->eflags.u32        |= X86_EFL_IF; /* paranoia */
+
+    pVCpu->cpum.s.fRawEntered = true;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Transforms the guest CPU state from raw-ring mode to correct values.
+ *
+ * This function will change any selector registers with DPL=1 to DPL=0.
+ *
+ * @returns Adjusted rc.
+ * @param   pVCpu       The VMCPU handle.
+ * @param   rc          Raw mode return code
+ * @param   pCtxCore    The context core (for trap usage).
+ * @see     @ref pg_raw
+ */
+VMMR3DECL(int) CPUMR3RawLeave(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, int rc)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Don't leave if we've already left (in GC).
+     */
+    Assert(pVCpu->cpum.s.fRawEntered);
+    Assert(!pVCpu->cpum.s.fRemEntered);
+    if (!pVCpu->cpum.s.fRawEntered)
+        return rc;
+    pVCpu->cpum.s.fRawEntered = false;
+
+    PCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (!pCtxCore)
+        pCtxCore = CPUMCTX2CORE(pCtx);
+    Assert(pCtxCore->eflags.Bits.u1VM || (pCtxCore->ss & X86_SEL_RPL));
+    AssertMsg(pCtxCore->eflags.Bits.u1VM || pCtxCore->eflags.Bits.u2IOPL < (unsigned)(pCtxCore->ss & X86_SEL_RPL),
+              ("X86_EFL_IOPL=%d CPL=%d\n", pCtxCore->eflags.Bits.u2IOPL, pCtxCore->ss & X86_SEL_RPL));
+
+    /*
+     * Are we executing in raw ring-1?
+     */
+    if (    (pCtxCore->ss & X86_SEL_RPL) == 1
+        &&  !pCtxCore->eflags.Bits.u1VM)
+    {
+        /*
+         * Leave execution mode.
+         */
+        PATMRawLeave(pVM, pCtxCore, rc);
+        /* Not quite sure if this is really required, but shouldn't harm (too much anyways). */
+        /** @todo See what happens if we remove this. */
+        if ((pCtxCore->ds & X86_SEL_RPL) == 1)
+            pCtxCore->ds &= ~X86_SEL_RPL;
+        if ((pCtxCore->es & X86_SEL_RPL) == 1)
+            pCtxCore->es &= ~X86_SEL_RPL;
+        if ((pCtxCore->fs & X86_SEL_RPL) == 1)
+            pCtxCore->fs &= ~X86_SEL_RPL;
+        if ((pCtxCore->gs & X86_SEL_RPL) == 1)
+            pCtxCore->gs &= ~X86_SEL_RPL;
+
+        /*
+         * Ring-1 selector => Ring-0.
+         */
+        pCtxCore->ss &= ~X86_SEL_RPL;
+        if ((pCtxCore->cs & X86_SEL_RPL) == 1)
+            pCtxCore->cs &= ~X86_SEL_RPL;
+    }
+    else
+    {
+        /*
+         * PATM is taking care of the IOPL and IF flags for us.
+         */
+        PATMRawLeave(pVM, pCtxCore, rc);
+        if (!pCtxCore->eflags.Bits.u1VM)
+        {
+            /** @todo See what happens if we remove this. */
+            if ((pCtxCore->ds & X86_SEL_RPL) == 1)
+                pCtxCore->ds &= ~X86_SEL_RPL;
+            if ((pCtxCore->es & X86_SEL_RPL) == 1)
+                pCtxCore->es &= ~X86_SEL_RPL;
+            if ((pCtxCore->fs & X86_SEL_RPL) == 1)
+                pCtxCore->fs &= ~X86_SEL_RPL;
+            if ((pCtxCore->gs & X86_SEL_RPL) == 1)
+                pCtxCore->gs &= ~X86_SEL_RPL;
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Enters REM, gets and resets the changed flags (CPUM_CHANGED_*).
+ *
+ * Only REM should ever call this function!
+ *
+ * @returns The changed flags.
+ * @param   pVCpu       The VMCPU handle.
+ * @param   puCpl       Where to return the current privilege level (CPL).
+ */
+VMMR3DECL(uint32_t) CPUMR3RemEnter(PVMCPU pVCpu, uint32_t *puCpl)
+{
+    Assert(!pVCpu->cpum.s.fRawEntered);
+    Assert(!pVCpu->cpum.s.fRemEntered);
+
+    /*
+     * Get the CPL first.
+     */
+    *puCpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.s.Guest));
+
+    /*
+     * Get and reset the flags, leaving CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID set.
+     */
+    uint32_t fFlags = pVCpu->cpum.s.fChanged;
+    pVCpu->cpum.s.fChanged &= CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID; /* leave it set */
+
+    /** @todo change the switcher to use the fChanged flags. */
+    if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_SINCE_REM)
+    {
+        fFlags |= CPUM_CHANGED_FPU_REM;
+        pVCpu->cpum.s.fUseFlags &= ~CPUM_USED_FPU_SINCE_REM;
+    }
+
+    pVCpu->cpum.s.fRemEntered = true;
+    return fFlags;
+}
+
+
+/**
+ * Leaves REM and works the CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID flag.
+ *
+ * @param   pVCpu               The virtual CPU handle.
+ * @param   fNoOutOfSyncSels    This is @c false if there are out of sync
+ *                              registers.
+ */
+VMMR3DECL(void) CPUMR3RemLeave(PVMCPU pVCpu, bool fNoOutOfSyncSels)
+{
+    Assert(!pVCpu->cpum.s.fRawEntered);
+    Assert(pVCpu->cpum.s.fRemEntered);
+
+    if (fNoOutOfSyncSels)
+        pVCpu->cpum.s.fChanged &= ~CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
+    else
+        pVCpu->cpum.s.fChanged |= ~CPUM_CHANGED_HIDDEN_SEL_REGS_INVALID;
+
+    pVCpu->cpum.s.fRemEntered = false;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGF.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGF.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGF.cpp	(revision 35333)
@@ -0,0 +1,1118 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page   pg_dbgf     DBGF - The Debugger Facility
+ *
+ * The purpose of the DBGF is to provide an interface for debuggers to
+ * manipulate the VMM without having to mess up the source code for each of
+ * them. The DBGF is always built in and will always work when a debugger
+ * attaches to the VM. The DBGF provides the basic debugger features, such as
+ * halting execution, handling breakpoints, single step execution, instruction
+ * disassembly, info querying, OS specific diggers, symbol and module
+ * management.
+ *
+ * The interface is working in a manner similar to the win32, linux and os2
+ * debugger interfaces. The interface has an asynchronous nature. This comes
+ * from the fact that the VMM and the Debugger are running in different threads.
+ * They are referred to as the "emulation thread" and the "debugger thread", or
+ * as the "ping thread" and the "pong thread, respectivly. (The last set of
+ * names comes from the use of the Ping-Pong synchronization construct from the
+ * RTSem API.)
+ *
+ * @see grp_dbgf
+ *
+ *
+ * @section sec_dbgf_scenario   Usage Scenario
+ *
+ * The debugger starts by attaching to the VM. For practical reasons we limit the
+ * number of concurrently attached debuggers to 1 per VM. The action of
+ * attaching to the VM causes the VM to check and generate debug events.
+ *
+ * The debugger then will wait/poll for debug events and issue commands.
+ *
+ * The waiting and polling is done by the DBGFEventWait() function. It will wait
+ * for the emulation thread to send a ping, thus indicating that there is an
+ * event waiting to be processed.
+ *
+ * An event can be a response to a command issued previously, the hitting of a
+ * breakpoint, or running into a bad/fatal VMM condition. The debugger now has
+ * the ping and must respond to the event at hand - the VMM is waiting. This
+ * usually means that the user of the debugger must do something, but it doesn't
+ * have to. The debugger is free to call any DBGF function (nearly at least)
+ * while processing the event.
+ *
+ * Typically the user will issue a request for the execution to be resumed, so
+ * the debugger calls DBGFResume() and goes back to waiting/polling for events.
+ *
+ * When the user eventually terminates the debugging session or selects another
+ * VM, the debugger detaches from the VM. This means that breakpoints are
+ * disabled and that the emulation thread no longer polls for debugger commands.
+ *
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/selm.h>
+#include <VBox/rem.h>
+#include <VBox/em.h>
+#include <VBox/hwaccm.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/semaphore.h>
+#include <iprt/thread.h>
+#include <iprt/asm.h>
+#include <iprt/time.h>
+#include <iprt/assert.h>
+#include <iprt/stream.h>
+#include <iprt/env.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int dbgfR3VMMWait(PVM pVM);
+static int dbgfR3VMMCmd(PVM pVM, DBGFCMD enmCmd, PDBGFCMDDATA pCmdData, bool *pfResumeExecution);
+static DECLCALLBACK(int) dbgfR3Attach(PVM pVM);
+
+
+/**
+ * Sets the VMM Debug Command variable.
+ *
+ * @returns Previous command.
+ * @param   pVM     VM Handle.
+ * @param   enmCmd  The command.
+ */
+DECLINLINE(DBGFCMD) dbgfR3SetCmd(PVM pVM, DBGFCMD enmCmd)
+{
+    DBGFCMD rc;
+    if (enmCmd == DBGFCMD_NO_COMMAND)
+    {
+        Log2(("DBGF: Setting command to %d (DBGFCMD_NO_COMMAND)\n", enmCmd));
+        rc = (DBGFCMD)ASMAtomicXchgU32((uint32_t volatile *)(void *)&pVM->dbgf.s.enmVMMCmd, enmCmd);
+        VM_FF_CLEAR(pVM, VM_FF_DBGF);
+    }
+    else
+    {
+        Log2(("DBGF: Setting command to %d\n", enmCmd));
+        AssertMsg(pVM->dbgf.s.enmVMMCmd == DBGFCMD_NO_COMMAND, ("enmCmd=%d enmVMMCmd=%d\n", enmCmd, pVM->dbgf.s.enmVMMCmd));
+        rc = (DBGFCMD)ASMAtomicXchgU32((uint32_t volatile *)(void *)&pVM->dbgf.s.enmVMMCmd, enmCmd);
+        VM_FF_SET(pVM, VM_FF_DBGF);
+        VMR3NotifyGlobalFFU(pVM->pUVM, 0 /* didn't notify REM */);
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes the DBGF.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(int) DBGFR3Init(PVM pVM)
+{
+    int rc = dbgfR3InfoInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3AsInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3SymInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3BpInit(pVM);
+    return rc;
+}
+
+
+/**
+ * Terminates and cleans up resources allocated by the DBGF.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(int) DBGFR3Term(PVM pVM)
+{
+    int rc;
+
+    /*
+     * Send a termination event to any attached debugger.
+     */
+    /* wait to become the speaker (we should already be that). */
+    if (    pVM->dbgf.s.fAttached
+        &&  RTSemPingShouldWait(&pVM->dbgf.s.PingPong))
+        RTSemPingWait(&pVM->dbgf.s.PingPong, 5000);
+
+    /* now, send the event if we're the speaker. */
+    if (    pVM->dbgf.s.fAttached
+        &&  RTSemPingIsSpeaker(&pVM->dbgf.s.PingPong))
+    {
+        DBGFCMD enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+        if (enmCmd == DBGFCMD_DETACH_DEBUGGER)
+            /* the debugger beat us to initiating the detaching. */
+            rc = VINF_SUCCESS;
+        else
+        {
+            /* ignore the command (if any). */
+            enmCmd = DBGFCMD_NO_COMMAND;
+            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_TERMINATING;
+            pVM->dbgf.s.DbgEvent.enmCtx  = DBGFEVENTCTX_OTHER;
+            rc = RTSemPing(&pVM->dbgf.s.PingPong);
+        }
+
+        /*
+         * Process commands until we get a detached command.
+         */
+        while (RT_SUCCESS(rc) && enmCmd != DBGFCMD_DETACHED_DEBUGGER)
+        {
+            if (enmCmd != DBGFCMD_NO_COMMAND)
+            {
+                /* process command */
+                bool fResumeExecution;
+                DBGFCMDDATA CmdData = pVM->dbgf.s.VMMCmdData;
+                rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
+                enmCmd = DBGFCMD_NO_COMMAND;
+            }
+            else
+            {
+                /* wait for new command. */
+                rc = RTSemPingWait(&pVM->dbgf.s.PingPong, RT_INDEFINITE_WAIT);
+                if (RT_SUCCESS(rc))
+                    enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+            }
+        }
+    }
+
+    /*
+     * Terminate the other bits.
+     */
+    dbgfR3OSTerm(pVM);
+    dbgfR3AsTerm(pVM);
+    dbgfR3InfoTerm(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM         VM handle.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void) DBGFR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    dbgfR3AsRelocate(pVM, offDelta);
+}
+
+
+/**
+ * Waits a little while for a debuggger to attach.
+ *
+ * @returns True is a debugger have attached.
+ * @param   pVM         VM handle.
+ * @param   enmEvent    Event.
+ */
+bool dbgfR3WaitForAttach(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    /*
+     * First a message.
+     */
+#ifndef RT_OS_L4
+
+# if !defined(DEBUG) || defined(DEBUG_sandervl) || defined(DEBUG_frank)
+    int cWait = 10;
+# else
+    int cWait = HWACCMIsEnabled(pVM)
+             && (   enmEvent == DBGFEVENT_ASSERTION_HYPER
+                 || enmEvent == DBGFEVENT_FATAL_ERROR)
+             && !RTEnvExist("VBOX_DBGF_WAIT_FOR_ATTACH")
+              ? 10
+              : 150;
+# endif
+    RTStrmPrintf(g_pStdErr, "DBGF: No debugger attached, waiting %d second%s for one to attach (event=%d)\n",
+                 cWait / 10, cWait != 10 ? "s" : "", enmEvent);
+    RTStrmFlush(g_pStdErr);
+    while (cWait > 0)
+    {
+        RTThreadSleep(100);
+        if (pVM->dbgf.s.fAttached)
+        {
+            RTStrmPrintf(g_pStdErr, "Attached!\n");
+            RTStrmFlush(g_pStdErr);
+            return true;
+        }
+
+        /* next */
+        if (!(cWait % 10))
+        {
+            RTStrmPrintf(g_pStdErr, "%d.", cWait / 10);
+            RTStrmFlush(g_pStdErr);
+        }
+        cWait--;
+    }
+#endif
+
+    RTStrmPrintf(g_pStdErr, "Stopping the VM!\n");
+    RTStrmFlush(g_pStdErr);
+    return false;
+}
+
+
+/**
+ * Forced action callback.
+ * The VMM will call this from it's main loop when VM_FF_DBGF is set.
+ *
+ * The function checks and executes pending commands from the debugger.
+ *
+ * @returns VINF_SUCCESS normally.
+ * @returns VERR_DBGF_RAISE_FATAL_ERROR to pretend a fatal error happened.
+ * @param   pVM         VM Handle.
+ */
+VMMR3DECL(int) DBGFR3VMMForcedAction(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+
+    if (VM_FF_TESTANDCLEAR(pVM, VM_FF_DBGF))
+    {
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+
+        /*
+         * Commands?
+         */
+        if (pVM->dbgf.s.enmVMMCmd != DBGFCMD_NO_COMMAND)
+        {
+            /** @todo stupid GDT/LDT sync hack. go away! */
+            SELMR3UpdateFromCPUM(pVM, pVCpu);
+
+            /*
+             * Process the command.
+             */
+            bool            fResumeExecution;
+            DBGFCMDDATA     CmdData = pVM->dbgf.s.VMMCmdData;
+            DBGFCMD         enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+            rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
+            if (!fResumeExecution)
+                rc = dbgfR3VMMWait(pVM);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Flag whether the event implies that we're stopped in the hypervisor code
+ * and have to block certain operations.
+ *
+ * @param   pVM         The VM handle.
+ * @param   enmEvent    The event.
+ */
+static void dbgfR3EventSetStoppedInHyperFlag(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    switch (enmEvent)
+    {
+        case DBGFEVENT_STEPPED_HYPER:
+        case DBGFEVENT_ASSERTION_HYPER:
+        case DBGFEVENT_BREAKPOINT_HYPER:
+            pVM->dbgf.s.fStoppedInHyper = true;
+            break;
+        default:
+            pVM->dbgf.s.fStoppedInHyper = false;
+            break;
+    }
+}
+
+
+/**
+ * Try to determine the event context.
+ *
+ * @returns debug event context.
+ * @param   pVM         The VM handle.
+ */
+static DBGFEVENTCTX dbgfR3FigureEventCtx(PVM pVM)
+{
+    /** @todo SMP support! */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    switch (EMGetState(pVCpu))
+    {
+        case EMSTATE_RAW:
+        case EMSTATE_DEBUG_GUEST_RAW:
+            return DBGFEVENTCTX_RAW;
+
+        case EMSTATE_REM:
+        case EMSTATE_DEBUG_GUEST_REM:
+            return DBGFEVENTCTX_REM;
+
+        case EMSTATE_DEBUG_HYPER:
+        case EMSTATE_GURU_MEDITATION:
+            return DBGFEVENTCTX_HYPER;
+
+        default:
+            return DBGFEVENTCTX_OTHER;
+    }
+}
+
+/**
+ * The common event prologue code.
+ * It will set the 'stopped-in-hyper' flag, make sure someone is attached,
+ * and perhaps process any high priority pending actions (none yet).
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   enmEvent    The event to be sent.
+ */
+static int dbgfR3EventPrologue(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    /** @todo SMP */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    /*
+     * Check if a debugger is attached.
+     */
+    if (    !pVM->dbgf.s.fAttached
+        &&  !dbgfR3WaitForAttach(pVM, enmEvent))
+    {
+        Log(("DBGFR3VMMEventSrc: enmEvent=%d - debugger not attached\n", enmEvent));
+        return VERR_DBGF_NOT_ATTACHED;
+    }
+
+    /*
+     * Sync back the state from the REM.
+     */
+    dbgfR3EventSetStoppedInHyperFlag(pVM, enmEvent);
+    if (!pVM->dbgf.s.fStoppedInHyper)
+        REMR3StateUpdate(pVM, pVCpu);
+
+    /*
+     * Look thru pending commands and finish those which make sense now.
+     */
+    /** @todo Process/purge pending commands. */
+    //int rc = DBGFR3VMMForcedAction(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sends the event in the event buffer.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+static int dbgfR3SendEvent(PVM pVM)
+{
+    int rc = RTSemPing(&pVM->dbgf.s.PingPong);
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3VMMWait(pVM);
+
+    pVM->dbgf.s.fStoppedInHyper = false;
+    /** @todo sync VMM -> REM after exitting the debugger. everything may change while in the debugger! */
+    return rc;
+}
+
+
+/**
+ * Send a generic debugger event which takes no data.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   enmEvent    The event to send.
+ */
+VMMR3DECL(int) DBGFR3Event(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Send a debugger event which takes the full source file location.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   enmEvent    The event to send.
+ * @param   pszFile     Source file.
+ * @param   uLine       Line number in source file.
+ * @param   pszFunction Function name.
+ * @param   pszFormat   Message which accompanies the event.
+ * @param   ...         Message arguments.
+ */
+VMMR3DECL(int) DBGFR3EventSrc(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszFile, unsigned uLine, const char *pszFunction, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    int rc = DBGFR3EventSrcV(pVM, enmEvent, pszFile, uLine, pszFunction, pszFormat, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Send a debugger event which takes the full source file location.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   enmEvent    The event to send.
+ * @param   pszFile     Source file.
+ * @param   uLine       Line number in source file.
+ * @param   pszFunction Function name.
+ * @param   pszFormat   Message which accompanies the event.
+ * @param   args        Message arguments.
+ */
+VMMR3DECL(int) DBGFR3EventSrcV(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszFile, unsigned uLine, const char *pszFunction, const char *pszFormat, va_list args)
+{
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Format the message.
+     */
+    char   *pszMessage = NULL;
+    char    szMessage[8192];
+    if (pszFormat && *pszFormat)
+    {
+        pszMessage = &szMessage[0];
+        RTStrPrintfV(szMessage, sizeof(szMessage), pszFormat, args);
+    }
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+    pVM->dbgf.s.DbgEvent.u.Src.pszFile      = pszFile;
+    pVM->dbgf.s.DbgEvent.u.Src.uLine        = uLine;
+    pVM->dbgf.s.DbgEvent.u.Src.pszFunction  = pszFunction;
+    pVM->dbgf.s.DbgEvent.u.Src.pszMessage   = pszMessage;
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Send a debugger event which takes the two assertion messages.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   enmEvent    The event to send.
+ * @param   pszMsg1     First assertion message.
+ * @param   pszMsg2     Second assertion message.
+ */
+VMMR3DECL(int) DBGFR3EventAssertion(PVM pVM, DBGFEVENTTYPE enmEvent, const char *pszMsg1, const char *pszMsg2)
+{
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+    pVM->dbgf.s.DbgEvent.u.Assert.pszMsg1 = pszMsg1;
+    pVM->dbgf.s.DbgEvent.u.Assert.pszMsg2 = pszMsg2;
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Breakpoint was hit somewhere.
+ * Figure out which breakpoint it is and notify the debugger.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   enmEvent    DBGFEVENT_BREAKPOINT_HYPER or DBGFEVENT_BREAKPOINT.
+ */
+VMMR3DECL(int) DBGFR3EventBreakpoint(PVM pVM, DBGFEVENTTYPE enmEvent)
+{
+    int rc = dbgfR3EventPrologue(pVM, enmEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Send the event and process the reply communication.
+     */
+    /** @todo SMP */
+    PVMCPU pVCpu = VMMGetCpu0(pVM);
+
+    pVM->dbgf.s.DbgEvent.enmType = enmEvent;
+    RTUINT iBp = pVM->dbgf.s.DbgEvent.u.Bp.iBp = pVCpu->dbgf.s.iActiveBp;
+    pVCpu->dbgf.s.iActiveBp = ~0U;
+    if (iBp != ~0U)
+        pVM->dbgf.s.DbgEvent.enmCtx = DBGFEVENTCTX_RAW;
+    else
+    {
+        /* REM breakpoints has be been searched for. */
+#if 0   /** @todo get flat PC api! */
+        uint32_t eip = CPUMGetGuestEIP(pVM);
+#else
+        /* @todo SMP support!! */
+        PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(VMMGetCpu(pVM));
+        RTGCPTR  eip = pCtx->rip + pCtx->csHid.u64Base;
+#endif
+        for (iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); iBp++)
+            if (    pVM->dbgf.s.aBreakpoints[iBp].enmType == DBGFBPTYPE_REM
+                &&  pVM->dbgf.s.aBreakpoints[iBp].GCPtr == eip)
+            {
+                pVM->dbgf.s.DbgEvent.u.Bp.iBp = iBp;
+                break;
+            }
+        AssertMsg(pVM->dbgf.s.DbgEvent.u.Bp.iBp != ~0U, ("eip=%08x\n", eip));
+        pVM->dbgf.s.DbgEvent.enmCtx = DBGFEVENTCTX_REM;
+    }
+    return dbgfR3SendEvent(pVM);
+}
+
+
+/**
+ * Waits for the debugger to respond.
+ *
+ * @returns VBox status. (clearify)
+ * @param   pVM     VM handle.
+ */
+static int dbgfR3VMMWait(PVM pVM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    LogFlow(("dbgfR3VMMWait:\n"));
+
+    /** @todo stupid GDT/LDT sync hack. go away! */
+    SELMR3UpdateFromCPUM(pVM, pVCpu);
+    int rcRet = VINF_SUCCESS;
+
+    /*
+     * Waits for the debugger to reply (i.e. issue an command).
+     */
+    for (;;)
+    {
+        /*
+         * Wait.
+         */
+        uint32_t cPollHack = 1; /** @todo this interface is horrible now that we're using lots of VMR3ReqCall stuff all over DBGF. */
+        for (;;)
+        {
+            int rc;
+            if (    !VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_REQUEST)
+                &&  !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
+            {
+                rc = RTSemPingWait(&pVM->dbgf.s.PingPong, cPollHack);
+                if (RT_SUCCESS(rc))
+                    break;
+                if (rc != VERR_TIMEOUT)
+                {
+                    LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
+                    return rc;
+                }
+            }
+
+            if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
+            {
+                rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
+                cPollHack = 1;
+            }
+            else if (   VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
+                     || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
+            {
+                LogFlow(("dbgfR3VMMWait: Processes requests...\n"));
+                rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+                if (rc == VINF_SUCCESS)
+                    rc = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu);
+                LogFlow(("dbgfR3VMMWait: VMR3ReqProcess -> %Rrc rcRet=%Rrc\n", rc, rcRet));
+                cPollHack = 1;
+            }
+            else
+            {
+                rc = VINF_SUCCESS;
+                if (cPollHack < 120)
+                    cPollHack++;
+            }
+
+            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+            {
+                switch (rc)
+                {
+                    case VINF_EM_DBG_BREAKPOINT:
+                    case VINF_EM_DBG_STEPPED:
+                    case VINF_EM_DBG_STEP:
+                    case VINF_EM_DBG_STOP:
+                        AssertMsgFailed(("rc=%Rrc\n", rc));
+                        break;
+
+                    /* return straight away */
+                    case VINF_EM_TERMINATE:
+                    case VINF_EM_OFF:
+                        LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
+                        return rc;
+
+                    /* remember return code. */
+                    default:
+                        AssertReleaseMsgFailed(("rc=%Rrc is not in the switch!\n", rc));
+                    case VINF_EM_RESET:
+                    case VINF_EM_SUSPEND:
+                    case VINF_EM_HALT:
+                    case VINF_EM_RESUME:
+                    case VINF_EM_RESCHEDULE:
+                    case VINF_EM_RESCHEDULE_REM:
+                    case VINF_EM_RESCHEDULE_RAW:
+                        if (rc < rcRet || rcRet == VINF_SUCCESS)
+                            rcRet = rc;
+                        break;
+                }
+            }
+            else if (RT_FAILURE(rc))
+            {
+                LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rc));
+                return rc;
+            }
+        }
+
+        /*
+         * Process the command.
+         */
+        bool            fResumeExecution;
+        DBGFCMDDATA     CmdData = pVM->dbgf.s.VMMCmdData;
+        DBGFCMD         enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_NO_COMMAND);
+        int rc = dbgfR3VMMCmd(pVM, enmCmd, &CmdData, &fResumeExecution);
+        if (fResumeExecution)
+        {
+            if (RT_FAILURE(rc))
+                rcRet = rc;
+            else if (    rc >= VINF_EM_FIRST
+                     &&  rc <= VINF_EM_LAST
+                     &&  (rc < rcRet || rcRet == VINF_SUCCESS))
+                rcRet = rc;
+            LogFlow(("dbgfR3VMMWait: returns %Rrc\n", rcRet));
+            return rcRet;
+        }
+    }
+}
+
+
+/**
+ * Executes command from debugger.
+ * The caller is responsible for waiting or resuming execution based on the
+ * value returned in the *pfResumeExecution indicator.
+ *
+ * @returns VBox status. (clearify!)
+ * @param   pVM                 VM Handle.
+ * @param   enmCmd              The command in question.
+ * @param   pCmdData            Pointer to the command data.
+ * @param   pfResumeExecution   Where to store the resume execution / continue waiting indicator.
+ */
+static int dbgfR3VMMCmd(PVM pVM, DBGFCMD enmCmd, PDBGFCMDDATA pCmdData, bool *pfResumeExecution)
+{
+    bool    fSendEvent;
+    bool    fResume;
+    int     rc = VINF_SUCCESS;
+
+    switch (enmCmd)
+    {
+        /*
+         * Halt is answered by an event say that we've halted.
+         */
+        case DBGFCMD_HALT:
+        {
+            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_HALT_DONE;
+            pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+            fSendEvent = true;
+            fResume = false;
+            break;
+        }
+
+
+        /*
+         * Resume is not answered we'll just resume execution.
+         */
+        case DBGFCMD_GO:
+        {
+            fSendEvent = false;
+            fResume = true;
+            break;
+        }
+
+        /** @todo implement (and define) the rest of the commands. */
+
+        /*
+         * Disable breakpoints and stuff.
+         * Send an everythings cool event to the debugger thread and resume execution.
+         */
+        case DBGFCMD_DETACH_DEBUGGER:
+        {
+            ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, false);
+            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_DETACH_DONE;
+            pVM->dbgf.s.DbgEvent.enmCtx  = DBGFEVENTCTX_OTHER;
+            fSendEvent = true;
+            fResume = true;
+            break;
+        }
+
+        /*
+         * The debugger has detached successfully.
+         * There is no reply to this event.
+         */
+        case DBGFCMD_DETACHED_DEBUGGER:
+        {
+            fSendEvent = false;
+            fResume = true;
+            break;
+        }
+
+        /*
+         * Single step, with trace into.
+         */
+        case DBGFCMD_SINGLE_STEP:
+        {
+            Log2(("Single step\n"));
+            rc = VINF_EM_DBG_STEP;
+            /** @todo SMP */
+            PVMCPU pVCpu = VMMGetCpu0(pVM);
+            pVCpu->dbgf.s.fSingleSteppingRaw = true;
+            fSendEvent = false;
+            fResume = true;
+            break;
+        }
+
+        /*
+         * Default is to send an invalid command event.
+         */
+        default:
+        {
+            pVM->dbgf.s.DbgEvent.enmType = DBGFEVENT_INVALID_COMMAND;
+            pVM->dbgf.s.DbgEvent.enmCtx  = dbgfR3FigureEventCtx(pVM);
+            fSendEvent = true;
+            fResume = false;
+            break;
+        }
+    }
+
+    /*
+     * Send pending event.
+     */
+    if (fSendEvent)
+    {
+        Log2(("DBGF: Emulation thread: sending event %d\n", pVM->dbgf.s.DbgEvent.enmType));
+        int rc2 = RTSemPing(&pVM->dbgf.s.PingPong);
+        if (RT_FAILURE(rc2))
+        {
+            AssertRC(rc2);
+            *pfResumeExecution = true;
+            return rc2;
+        }
+    }
+
+    /*
+     * Return.
+     */
+    *pfResumeExecution = fResume;
+    return rc;
+}
+
+
+/**
+ * Attaches a debugger to the specified VM.
+ *
+ * Only one debugger at a time.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(int) DBGFR3Attach(PVM pVM)
+{
+    /*
+     * Some validations first.
+     */
+    if (!VALID_PTR(pVM))
+    {
+        Log(("DBGFR3Attach: bad VM handle: %p\n", pVM));
+        return VERR_INVALID_HANDLE;
+    }
+    VMSTATE enmVMState = pVM->enmVMState;
+    if (    enmVMState >= VMSTATE_DESTROYING
+        ||  enmVMState <  VMSTATE_CREATING)
+    {
+        Log(("DBGFR3Attach: Invalid VM state: %s\n", VMGetStateName(enmVMState)));
+        return VERR_INVALID_HANDLE;
+    }
+
+    /*
+     * Call the VM, use EMT for serialization.
+     */
+    /** @todo SMP */
+    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3Attach, 1, pVM);
+}
+
+
+/**
+ * EMT worker for DBGFR3Attach.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+static DECLCALLBACK(int) dbgfR3Attach(PVM pVM)
+{
+    if (pVM->dbgf.s.fAttached)
+    {
+        Log(("dbgR3Attach: Debugger already attached\n"));
+        return VERR_DBGF_ALREADY_ATTACHED;
+    }
+
+    /*
+     * Create the Ping-Pong structure.
+     */
+    int rc = RTSemPingPongInit(&pVM->dbgf.s.PingPong);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Set the attached flag.
+     */
+    ASMAtomicWriteBool(&pVM->dbgf.s.fAttached, true);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Detaches a debugger from the specified VM.
+ *
+ * Caller must be attached to the VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(int) DBGFR3Detach(PVM pVM)
+{
+    LogFlow(("DBGFR3Detach:\n"));
+    int rc;
+
+    /*
+     * Check if attached.
+     */
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+
+    /*
+     * Try send the detach command.
+     * Keep in mind that we might be racing EMT, so, be extra careful.
+     */
+    DBGFCMD enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_DETACH_DEBUGGER);
+    if (RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong))
+    {
+        rc = RTSemPong(&pVM->dbgf.s.PingPong);
+        AssertMsgRCReturn(rc, ("Failed to signal emulation thread. rc=%Rrc\n", rc), rc);
+        LogRel(("DBGFR3Detach: enmCmd=%d (pong -> ping)\n", enmCmd));
+    }
+
+    /*
+     * Wait for the OK event.
+     */
+    rc = RTSemPongWait(&pVM->dbgf.s.PingPong, RT_INDEFINITE_WAIT);
+    AssertLogRelMsgRCReturn(rc, ("Wait on detach command failed, rc=%Rrc\n", rc), rc);
+
+    /*
+     * Send the notification command indicating that we're really done.
+     */
+    enmCmd = dbgfR3SetCmd(pVM, DBGFCMD_DETACHED_DEBUGGER);
+    rc = RTSemPong(&pVM->dbgf.s.PingPong);
+    AssertMsgRCReturn(rc, ("Failed to signal emulation thread. rc=%Rrc\n", rc), rc);
+
+    LogFlowFunc(("returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Wait for a debug event.
+ *
+ * @returns VBox status. Will not return VBOX_INTERRUPTED.
+ * @param   pVM         VM handle.
+ * @param   cMillies    Number of millis to wait.
+ * @param   ppEvent     Where to store the event pointer.
+ */
+VMMR3DECL(int) DBGFR3EventWait(PVM pVM, RTMSINTERVAL cMillies, PCDBGFEVENT *ppEvent)
+{
+    /*
+     * Check state.
+     */
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    *ppEvent = NULL;
+
+    /*
+     * Wait.
+     */
+    int rc = RTSemPongWait(&pVM->dbgf.s.PingPong, cMillies);
+    if (RT_SUCCESS(rc))
+    {
+        *ppEvent = &pVM->dbgf.s.DbgEvent;
+        Log2(("DBGF: Debugger thread: receiving event %d\n", (*ppEvent)->enmType));
+        return VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Halts VM execution.
+ *
+ * After calling this the VM isn't actually halted till an DBGFEVENT_HALT_DONE
+ * arrives. Until that time it's not possible to issue any new commands.
+ *
+ * @returns VBox status.
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(int) DBGFR3Halt(PVM pVM)
+{
+    /*
+     * Check state.
+     */
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    RTPINGPONGSPEAKER enmSpeaker = pVM->dbgf.s.PingPong.enmSpeaker;
+    if (   enmSpeaker == RTPINGPONGSPEAKER_PONG
+        || enmSpeaker == RTPINGPONGSPEAKER_PONG_SIGNALED)
+        return VWRN_DBGF_ALREADY_HALTED;
+
+    /*
+     * Send command.
+     */
+    dbgfR3SetCmd(pVM, DBGFCMD_HALT);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the VM is halted by the debugger.
+ *
+ * @returns True if halted.
+ * @returns False if not halted.
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(bool) DBGFR3IsHalted(PVM pVM)
+{
+    AssertReturn(pVM->dbgf.s.fAttached, false);
+    RTPINGPONGSPEAKER enmSpeaker = pVM->dbgf.s.PingPong.enmSpeaker;
+    return enmSpeaker == RTPINGPONGSPEAKER_PONG_SIGNALED
+        || enmSpeaker == RTPINGPONGSPEAKER_PONG;
+}
+
+
+/**
+ * Checks if the debugger can wait for events or not.
+ *
+ * This function is only used by lazy, multiplexing debuggers. :-)
+ *
+ * @returns True if waitable.
+ * @returns False if not waitable.
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(bool) DBGFR3CanWait(PVM pVM)
+{
+    AssertReturn(pVM->dbgf.s.fAttached, false);
+    return RTSemPongShouldWait(&pVM->dbgf.s.PingPong);
+}
+
+
+/**
+ * Resumes VM execution.
+ *
+ * There is no receipt event on this command.
+ *
+ * @returns VBox status.
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(int) DBGFR3Resume(PVM pVM)
+{
+    /*
+     * Check state.
+     */
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    AssertReturn(RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong), VERR_SEM_OUT_OF_TURN);
+
+    /*
+     * Send the ping back to the emulation thread telling it to run.
+     */
+    dbgfR3SetCmd(pVM, DBGFCMD_GO);
+    int rc = RTSemPong(&pVM->dbgf.s.PingPong);
+    AssertRC(rc);
+
+    return rc;
+}
+
+
+/**
+ * Step Into.
+ *
+ * A single step event is generated from this command.
+ * The current implementation is not reliable, so don't rely on the event coming.
+ *
+ * @returns VBox status.
+ * @param   pVM     VM handle.
+ * @param   idCpu   The ID of the CPU to single step on.
+ */
+VMMR3DECL(int) DBGFR3Step(PVM pVM, VMCPUID idCpu)
+{
+    /*
+     * Check state.
+     */
+    AssertReturn(pVM->dbgf.s.fAttached, VERR_DBGF_NOT_ATTACHED);
+    AssertReturn(RTSemPongIsSpeaker(&pVM->dbgf.s.PingPong), VERR_SEM_OUT_OF_TURN);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
+
+    /*
+     * Send the ping back to the emulation thread telling it to run.
+     */
+/** @todo SMP (idCpu) */
+    dbgfR3SetCmd(pVM, DBGFCMD_SINGLE_STEP);
+    int rc = RTSemPong(&pVM->dbgf.s.PingPong);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Call this to single step programmatically.
+ *
+ * You must pass down the return code to the EM loop! That's
+ * where the actual single stepping take place (at least in the
+ * current implementation).
+ *
+ * @returns VINF_EM_DBG_STEP
+ *
+ * @param   pVCpu       The virtual CPU handle.
+ *
+ * @thread  VCpu EMT
+ */
+VMMR3DECL(int) DBGFR3PrgStep(PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    pVCpu->dbgf.s.fSingleSteppingRaw = true;
+    return VINF_EM_DBG_STEP;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFAddr.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFAddr.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFAddr.cpp	(revision 35333)
@@ -0,0 +1,492 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Mixed Address Methods.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/selm.h>
+#include <VBox/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include "internal/pgm.h"
+
+
+
+/**
+ * Checks if an address is in the HMA or not.
+ * @returns true if it's inside the HMA.
+ * @returns flase if it's not inside the HMA.
+ * @param   pVM         The VM handle.
+ * @param   FlatPtr     The address in question.
+ */
+DECLINLINE(bool) dbgfR3IsHMA(PVM pVM, RTGCUINTPTR FlatPtr)
+{
+    return MMHyperIsInsideArea(pVM, FlatPtr);
+}
+
+
+/**
+ * Common worker for DBGFR3AddrFromSelOff and DBGFR3AddrFromSelInfoOff.
+ */
+static int dbgfR3AddrFromSelInfoOffWorker(PDBGFADDRESS pAddress, PCDBGFSELINFO pSelInfo, RTUINTPTR off)
+{
+    if (pSelInfo->fFlags & (DBGFSELINFO_FLAGS_INVALID | DBGFSELINFO_FLAGS_NOT_PRESENT))
+        return pSelInfo->fFlags & DBGFSELINFO_FLAGS_NOT_PRESENT
+             ? VERR_SELECTOR_NOT_PRESENT
+             : VERR_INVALID_SELECTOR;
+
+    /** @todo This all goes voodoo in long mode. */
+    /* check limit. */
+    if (DBGFSelInfoIsExpandDown(pSelInfo))
+    {
+        if (    !pSelInfo->u.Raw.Gen.u1Granularity
+            &&  off > UINT32_C(0xffff))
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+        if (off <= pSelInfo->cbLimit)
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+    }
+    else if (off > pSelInfo->cbLimit)
+        return VERR_OUT_OF_SELECTOR_BOUNDS;
+
+    pAddress->FlatPtr = pSelInfo->GCPtrBase + off;
+
+    /** @todo fix all these selector tests! */
+    if (    !pSelInfo->GCPtrBase
+        &&  pSelInfo->u.Raw.Gen.u1Granularity
+        &&  pSelInfo->u.Raw.Gen.u1DefBig)
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FLAT;
+    else if (pSelInfo->cbLimit <= UINT32_C(0xffff))
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR16;
+    else if (pSelInfo->cbLimit <= UINT32_C(0xffffffff))
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR32;
+    else
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FAR64;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a mixed address from a Sel:off pair.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   idCpu       The CPU ID.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   Sel         The selector part.
+ * @param   off         The offset part.
+ */
+VMMR3DECL(int) DBGFR3AddrFromSelOff(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, RTSEL Sel, RTUINTPTR off)
+{
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
+
+    pAddress->Sel = Sel;
+    pAddress->off = off;
+    if (Sel != DBGF_SEL_FLAT)
+    {
+        DBGFSELINFO SelInfo;
+        int rc = DBGFR3SelQueryInfo(pVM, idCpu, Sel, DBGFSELQI_FLAGS_DT_GUEST | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE, &SelInfo);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = dbgfR3AddrFromSelInfoOffWorker(pAddress, &SelInfo, off);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    else
+    {
+        pAddress->FlatPtr = off;
+        pAddress->fFlags = DBGFADDRESS_FLAGS_FLAT;
+    }
+    pAddress->fFlags |= DBGFADDRESS_FLAGS_VALID;
+    if (dbgfR3IsHMA(pVM, pAddress->FlatPtr))
+        pAddress->fFlags |= DBGFADDRESS_FLAGS_HMA;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a mixed address from selector info and an offset into the segment
+ * described by it.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   idCpu       The CPU ID.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   pSelInfo    The selector info.
+ * @param   off         The offset part.
+ */
+VMMR3DECL(int) DBGFR3AddrFromSelInfoOff(PVM pVM, PDBGFADDRESS pAddress, PCDBGFSELINFO pSelInfo, RTUINTPTR off)
+{
+    pAddress->Sel = pSelInfo->Sel;
+    pAddress->off = off;
+    int rc = dbgfR3AddrFromSelInfoOffWorker(pAddress, pSelInfo, off);
+    if (RT_FAILURE(rc))
+        return rc;
+    pAddress->fFlags |= DBGFADDRESS_FLAGS_VALID;
+    if (dbgfR3IsHMA(pVM, pAddress->FlatPtr))
+        pAddress->fFlags |= DBGFADDRESS_FLAGS_HMA;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a mixed address from a flat address.
+ *
+ * @returns pAddress.
+ * @param   pVM         The VM handle.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   FlatPtr     The flat pointer.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrFromFlat(PVM pVM, PDBGFADDRESS pAddress, RTGCUINTPTR FlatPtr)
+{
+    pAddress->Sel     = DBGF_SEL_FLAT;
+    pAddress->off     = FlatPtr;
+    pAddress->FlatPtr = FlatPtr;
+    pAddress->fFlags  = DBGFADDRESS_FLAGS_FLAT | DBGFADDRESS_FLAGS_VALID;
+    if (dbgfR3IsHMA(pVM, pAddress->FlatPtr))
+        pAddress->fFlags |= DBGFADDRESS_FLAGS_HMA;
+    return pAddress;
+}
+
+
+/**
+ * Creates a mixed address from a guest physical address.
+ *
+ * @returns pAddress.
+ * @param   pVM         The VM handle.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   PhysAddr    The guest physical address.
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrFromPhys(PVM pVM, PDBGFADDRESS pAddress, RTGCPHYS PhysAddr)
+{
+    pAddress->Sel     = DBGF_SEL_FLAT;
+    pAddress->off     = PhysAddr;
+    pAddress->FlatPtr = PhysAddr;
+    pAddress->fFlags  = DBGFADDRESS_FLAGS_PHYS | DBGFADDRESS_FLAGS_VALID;
+    return pAddress;
+}
+
+
+/**
+ * Checks if the specified address is valid (checks the structure pointer too).
+ *
+ * @returns true if valid.
+ * @returns false if invalid.
+ * @param   pVM         The VM handle.
+ * @param   pAddress    The address to validate.
+ */
+VMMR3DECL(bool) DBGFR3AddrIsValid(PVM pVM, PCDBGFADDRESS pAddress)
+{
+    if (!VALID_PTR(pAddress))
+        return false;
+    if (!DBGFADDRESS_IS_VALID(pAddress))
+        return false;
+    /* more? */
+    return true;
+}
+
+
+/**
+ * Called on the EMT for the VCpu.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The virtual CPU handle.
+ * @param   pAddress        The address.
+ * @param   pGCPhys         Where to return the physical address.
+ */
+static DECLCALLBACK(int) dbgfR3AddrToPhysOnVCpu(PVMCPU pVCpu, PDBGFADDRESS pAddress, PRTGCPHYS pGCPhys)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    /* This is just a wrapper because we cannot pass FlatPtr thru VMR3ReqCall directly. */
+    return PGMGstGetPage(pVCpu, pAddress->FlatPtr, NULL, pGCPhys);
+}
+
+
+/**
+ * Converts an address to a guest physical address.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_PARAMETER if the address is invalid.
+ * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
+ *          CPU handle is invalid.
+ * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
+ * @retval  VERR_PAGE_NOT_PRESENT
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT
+ * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+ * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
+ *
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The ID of the CPU context to convert virtual
+ *                          addresses.
+ * @param   pAddress        The address.
+ * @param   pGCPhys         Where to return the physical address.
+ */
+VMMR3DECL(int)  DBGFR3AddrToPhys(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, PRTGCPHYS pGCPhys)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtr(pGCPhys);
+    *pGCPhys = NIL_RTGCPHYS;
+    AssertPtr(pAddress);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
+
+    /*
+     * Convert by address type.
+     */
+    int rc;
+    if (pAddress->fFlags & DBGFADDRESS_FLAGS_HMA)
+        rc = VERR_NOT_SUPPORTED;
+    else if (pAddress->fFlags & DBGFADDRESS_FLAGS_PHYS)
+    {
+        *pGCPhys = pAddress->FlatPtr;
+        rc = VINF_SUCCESS;
+    }
+    else
+    {
+        PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+        if (VMCPU_IS_EMT(pVCpu))
+            rc = dbgfR3AddrToPhysOnVCpu(pVCpu, pAddress, pGCPhys);
+        else
+            rc = VMR3ReqCallWait(pVCpu->pVMR3, pVCpu->idCpu,
+                                 (PFNRT)dbgfR3AddrToPhysOnVCpu, 3, pVCpu, pAddress, pGCPhys);
+    }
+    return rc;
+}
+
+
+/**
+ * Converts an address to a host physical address.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_PARAMETER if the address is invalid.
+ * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
+ *          CPU handle is invalid.
+ * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
+ * @retval  VERR_PAGE_NOT_PRESENT
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT
+ * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+ * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS
+ *
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The ID of the CPU context to convert virtual
+ *                          addresses.
+ * @param   pAddress        The address.
+ * @param   pHCPhys         Where to return the physical address.
+ */
+VMMR3DECL(int)  DBGFR3AddrToHostPhys(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, PRTHCPHYS pHCPhys)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtr(pHCPhys);
+    *pHCPhys = NIL_RTHCPHYS;
+    AssertPtr(pAddress);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
+
+    /*
+     * Convert it if we can.
+     */
+    int rc;
+    if (pAddress->fFlags & DBGFADDRESS_FLAGS_HMA)
+        rc = VERR_NOT_SUPPORTED; /** @todo implement this */
+    else
+    {
+        RTGCPHYS GCPhys;
+        rc = DBGFR3AddrToPhys(pVM, idCpu, pAddress, &GCPhys);
+        if (RT_SUCCESS(rc))
+            rc = PGMPhysGCPhys2HCPhys(pVM, pAddress->FlatPtr, pHCPhys);
+    }
+    return rc;
+}
+
+
+/**
+ * Called on the EMT for the VCpu.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The ID of the CPU context.
+ * @param   pAddress        The address.
+ * @param   fReadOnly       Whether returning a read-only page is fine or not.
+ * @param   ppvR3Ptr        Where to return the address.
+ */
+static DECLCALLBACK(int) dbgfR3AddrToVolatileR3PtrOnVCpu(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, bool fReadOnly, void **ppvR3Ptr)
+{
+    Assert(idCpu == VMMGetCpuId(pVM));
+
+    int rc;
+    if (pAddress->fFlags & DBGFADDRESS_FLAGS_HMA)
+    {
+        rc = VERR_NOT_SUPPORTED; /** @todo create some dedicated errors for this stuff. */
+        /** @todo this may assert, create a debug version of this which doesn't. */
+        if (MMHyperIsInsideArea(pVM, pAddress->FlatPtr))
+        {
+            void *pv = MMHyperRCToCC(pVM, (RTRCPTR)pAddress->FlatPtr);
+            if (pv)
+            {
+                *ppvR3Ptr = pv;
+                rc = VINF_SUCCESS;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * This is a tad ugly, but it gets the job done.
+         */
+        PGMPAGEMAPLOCK Lock;
+        if (pAddress->fFlags & DBGFADDRESS_FLAGS_PHYS)
+        {
+            if (fReadOnly)
+                rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, pAddress->FlatPtr, (void const **)ppvR3Ptr, &Lock);
+            else
+                rc = PGMPhysGCPhys2CCPtr(pVM, pAddress->FlatPtr, ppvR3Ptr, &Lock);
+        }
+        else
+        {
+            PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+            if (fReadOnly)
+                rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, pAddress->FlatPtr, (void const **)ppvR3Ptr, &Lock);
+            else
+                rc = PGMPhysGCPtr2CCPtr(pVCpu, pAddress->FlatPtr, ppvR3Ptr, &Lock);
+        }
+        if (RT_SUCCESS(rc))
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+    }
+    return rc;
+}
+
+
+
+
+/**
+ * Converts an address to a volatile host virtual address.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_PARAMETER if the address is invalid.
+ * @retval  VERR_INVALID_STATE if the VM is being terminated or if the virtual
+ *          CPU handle is invalid.
+ * @retval  VERR_NOT_SUPPORTED is the type of address cannot be converted.
+ * @retval  VERR_PAGE_NOT_PRESENT
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT
+ * @retval  VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+ * @retval  VERR_PAGE_MAP_LEVEL4_NOT_PRESENT
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS
+ *
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The ID of the CPU context to convert virtual
+ *                          addresses.
+ * @param   pAddress        The address.
+ * @param   fReadOnly       Whether returning a read-only page is fine or not.
+ *                          If set to thru the page may have to be made writable
+ *                          before we return.
+ * @param   ppvR3Ptr        Where to return the address.
+ */
+VMMR3DECL(int)  DBGFR3AddrToVolatileR3Ptr(PVM pVM, VMCPUID idCpu, PDBGFADDRESS pAddress, bool fReadOnly, void **ppvR3Ptr)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtr(ppvR3Ptr);
+    *ppvR3Ptr = NULL;
+    AssertPtr(pAddress);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
+
+    /*
+     * Convert it.
+     */
+    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3AddrToVolatileR3PtrOnVCpu, 5, pVM, idCpu, pAddress, fReadOnly, ppvR3Ptr);
+}
+
+
+/**
+ * Adds an offset to an address.
+ *
+ * @returns pAddress.
+ *
+ * @param   pAddress        The address.
+ * @param   uAddend         How much to add.
+ *
+ * @remarks No address space or segment limit checks are performed,
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrAdd(PDBGFADDRESS pAddress, RTGCUINTPTR uAddend)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtrReturn(pAddress, NULL);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), NULL);
+
+    /*
+     * Add the stuff.
+     */
+    pAddress->off     += uAddend;
+    pAddress->FlatPtr += uAddend;
+
+    return pAddress;
+}
+
+
+/**
+ * Subtracts an offset from an address.
+ *
+ * @returns VINF_SUCCESS on success.
+ *
+ * @param   pAddress        The address.
+ * @param   uSubtrahend     How much to subtract.
+ *
+ * @remarks No address space or segment limit checks are performed,
+ */
+VMMR3DECL(PDBGFADDRESS) DBGFR3AddrSub(PDBGFADDRESS pAddress, RTGCUINTPTR uSubtrahend)
+{
+    /*
+     * Parameter validation.
+     */
+    AssertPtrReturn(pAddress, NULL);
+    AssertReturn(DBGFADDRESS_IS_VALID(pAddress), NULL);
+
+    /*
+     * Add the stuff.
+     */
+    pAddress->off     -= uSubtrahend;
+    pAddress->FlatPtr -= uSubtrahend;
+
+    return pAddress;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFAddrSpace.cpp	(revision 35333)
@@ -0,0 +1,1114 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Address Space Management.
+ */
+
+/*
+ * Copyright (C) 2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_dbgf_addr_space     DBGFAddrSpace - Address Space Management
+ *
+ * What's an address space? It's mainly a convenient way of stuffing
+ * module segments and ad-hoc symbols together. It will also help out
+ * when the debugger gets extended to deal with user processes later.
+ *
+ * There are two standard address spaces that will always be present:
+ *   - The physical address space.
+ *   - The global virtual address space.
+ *
+ * Additional address spaces will be added and removed at runtime for
+ * guest processes. The global virtual address space will be used to
+ * track the kernel parts of the OS, or at least the bits of the kernel
+ * that is part of all address spaces (mac os x and 4G/4G patched linux).
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/pdmapi.h>
+#include <VBox/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/env.h>
+#include <iprt/path.h>
+#include <iprt/param.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Address space database node.
+ */
+typedef struct DBGFASDBNODE
+{
+    /** The node core for DBGF::AsHandleTree, the key is the address space handle. */
+    AVLPVNODECORE   HandleCore;
+    /** The node core for DBGF::AsPidTree, the key is the process id. */
+    AVLU32NODECORE  PidCore;
+    /** The node core for DBGF::AsNameSpace, the string is the address space name. */
+    RTSTRSPACECORE  NameCore;
+
+} DBGFASDBNODE;
+/** Pointer to an address space database node. */
+typedef DBGFASDBNODE *PDBGFASDBNODE;
+
+
+/**
+ * For dbgfR3AsLoadImageOpenData and dbgfR3AsLoadMapOpenData.
+ */
+typedef struct DBGFR3ASLOADOPENDATA
+{
+    const char *pszModName;
+    RTGCUINTPTR uSubtrahend;
+    uint32_t fFlags;
+    RTDBGMOD hMod;
+} DBGFR3ASLOADOPENDATA;
+
+/**
+ * Callback for dbgfR3AsSearchPath and dbgfR3AsSearchEnvPath.
+ *
+ * @returns VBox status code. If success, then the search is completed.
+ * @param   pszFilename     The file name under evaluation.
+ * @param   pvUser          The user argument.
+ */
+typedef int FNDBGFR3ASSEARCHOPEN(const char *pszFilename, void *pvUser);
+/** Pointer to a FNDBGFR3ASSEARCHOPEN. */
+typedef FNDBGFR3ASSEARCHOPEN *PFNDBGFR3ASSEARCHOPEN;
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** Locks the address space database for writing. */
+#define DBGF_AS_DB_LOCK_WRITE(pVM) \
+    do { \
+        int rcSem = RTSemRWRequestWrite((pVM)->dbgf.s.hAsDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the address space database after writing. */
+#define DBGF_AS_DB_UNLOCK_WRITE(pVM) \
+    do { \
+        int rcSem = RTSemRWReleaseWrite((pVM)->dbgf.s.hAsDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Locks the address space database for reading. */
+#define DBGF_AS_DB_LOCK_READ(pVM) \
+    do { \
+        int rcSem = RTSemRWRequestRead((pVM)->dbgf.s.hAsDbLock, RT_INDEFINITE_WAIT); \
+        AssertRC(rcSem); \
+    } while (0)
+
+/** Unlocks the address space database after reading. */
+#define DBGF_AS_DB_UNLOCK_READ(pVM) \
+    do { \
+        int rcSem = RTSemRWReleaseRead((pVM)->dbgf.s.hAsDbLock); \
+        AssertRC(rcSem); \
+    } while (0)
+
+
+
+/**
+ * Initializes the address space parts of DBGF.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ */
+int dbgfR3AsInit(PVM pVM)
+{
+    /*
+     * Create the semaphore.
+     */
+    int rc = RTSemRWCreate(&pVM->dbgf.s.hAsDbLock);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Create the standard address spaces.
+     */
+    RTDBGAS hDbgAs;
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTGCPTR_MAX, "Global");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    RTDbgAsRetain(hDbgAs);
+    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_GLOBAL)] = hDbgAs;
+
+    RTDbgAsRetain(hDbgAs);
+    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_KERNEL)] = hDbgAs;
+
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTGCPHYS_MAX, "Physical");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    RTDbgAsRetain(hDbgAs);
+    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_PHYS)] = hDbgAs;
+
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTRCPTR_MAX, "HyperRawMode");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    RTDbgAsRetain(hDbgAs);
+    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC)] = hDbgAs;
+    RTDbgAsRetain(hDbgAs);
+    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_RC_AND_GC_GLOBAL)] = hDbgAs;
+
+    rc = RTDbgAsCreate(&hDbgAs, 0, RTR0PTR_MAX, "HyperRing0");
+    AssertRCReturn(rc, rc);
+    rc = DBGFR3AsAdd(pVM, hDbgAs, NIL_RTPROCESS);
+    AssertRCReturn(rc, rc);
+    RTDbgAsRetain(hDbgAs);
+    pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(DBGF_AS_R0)] = hDbgAs;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback used by dbgfR3AsTerm / RTAvlPVDestroy to release an address space.
+ *
+ * @returns 0.
+ * @param   pNode           The address space database node.
+ * @param   pvIgnore        NULL.
+ */
+static DECLCALLBACK(int) dbgfR3AsTermDestroyNode(PAVLPVNODECORE pNode, void *pvIgnore)
+{
+    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)pNode;
+    RTDbgAsRelease((RTDBGAS)pDbNode->HandleCore.Key);
+    pDbNode->HandleCore.Key = NIL_RTDBGAS;
+    /* Don't bother freeing it here as MM will free it soon and MM is much at
+       it when doing it wholesale instead of piecemeal. */
+    NOREF(pvIgnore);
+    return 0;
+}
+
+
+/**
+ * Terminates the address space parts of DBGF.
+ *
+ * @param   pVM             The VM handle.
+ */
+void dbgfR3AsTerm(PVM pVM)
+{
+    /*
+     * Create the semaphore.
+     */
+    int rc = RTSemRWDestroy(pVM->dbgf.s.hAsDbLock);
+    AssertRC(rc);
+    pVM->dbgf.s.hAsDbLock = NIL_RTSEMRW;
+
+    /*
+     * Release all the address spaces.
+     */
+    RTAvlPVDestroy(&pVM->dbgf.s.AsHandleTree, dbgfR3AsTermDestroyNode, NULL);
+    for (size_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.ahAsAliases); i++)
+    {
+        RTDbgAsRelease(pVM->dbgf.s.ahAsAliases[i]);
+        pVM->dbgf.s.ahAsAliases[i] = NIL_RTDBGAS;
+    }
+}
+
+
+/**
+ * Relocates the RC address space.
+ *
+ * @param   pVM             The VM handle.
+ * @param   offDelta        The relocation delta.
+ */
+void dbgfR3AsRelocate(PVM pVM, RTGCUINTPTR offDelta)
+{
+    /** @todo */
+}
+
+
+/**
+ * Adds the address space to the database.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   hDbgAs          The address space handle. The reference of the
+ *                          caller will NOT be consumed.
+ * @param   ProcId          The process id or NIL_RTPROCESS.
+ */
+VMMR3DECL(int) DBGFR3AsAdd(PVM pVM, RTDBGAS hDbgAs, RTPROCESS ProcId)
+{
+    /*
+     * Input validation.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    const char *pszName = RTDbgAsName(hDbgAs);
+    if (!pszName)
+        return VERR_INVALID_HANDLE;
+    uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+    if (cRefs == UINT32_MAX)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Allocate a tracking node.
+     */
+    int rc = VERR_NO_MEMORY;
+    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)MMR3HeapAlloc(pVM, MM_TAG_DBGF_AS, sizeof(*pDbNode));
+    if (pDbNode)
+    {
+        pDbNode->HandleCore.Key = hDbgAs;
+        pDbNode->PidCore.Key = NIL_RTPROCESS;
+        pDbNode->NameCore.pszString = pszName;
+        pDbNode->NameCore.cchString = strlen(pszName);
+        DBGF_AS_DB_LOCK_WRITE(pVM);
+        if (RTStrSpaceInsert(&pVM->dbgf.s.AsNameSpace, &pDbNode->NameCore))
+        {
+            if (RTAvlPVInsert(&pVM->dbgf.s.AsHandleTree, &pDbNode->HandleCore))
+            {
+                DBGF_AS_DB_UNLOCK_WRITE(pVM);
+                return VINF_SUCCESS;
+            }
+
+            /* bail out */
+            RTStrSpaceRemove(&pVM->dbgf.s.AsNameSpace, pszName);
+        }
+        DBGF_AS_DB_UNLOCK_WRITE(pVM);
+        MMR3HeapFree(pDbNode);
+    }
+    RTDbgAsRelease(hDbgAs);
+    return rc;
+}
+
+
+/**
+ * Delete an address space from the database.
+ *
+ * The address space must not be engaged as any of the standard aliases.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SHARING_VIOLATION if in use as an alias.
+ * @retval  VERR_NOT_FOUND if not found in the address space database.
+ *
+ * @param   pVM             The VM handle.
+ * @param   hDbgAs          The address space handle. Aliases are not allowed.
+ */
+VMMR3DECL(int) DBGFR3AsDelete(PVM pVM, RTDBGAS hDbgAs)
+{
+    /*
+     * Input validation. Retain the address space so it can be released outside
+     * the lock as well as validated.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    if (hDbgAs == NIL_RTDBGAS)
+        return VINF_SUCCESS;
+    uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+    if (cRefs == UINT32_MAX)
+        return VERR_INVALID_HANDLE;
+    RTDbgAsRelease(hDbgAs);
+
+    DBGF_AS_DB_LOCK_WRITE(pVM);
+
+    /*
+     * You cannot delete any of the aliases.
+     */
+    for (size_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.ahAsAliases); i++)
+        if (pVM->dbgf.s.ahAsAliases[i] == hDbgAs)
+        {
+            DBGF_AS_DB_UNLOCK_WRITE(pVM);
+            return VERR_SHARING_VIOLATION;
+        }
+
+    /*
+     * Ok, try remove it from the database.
+     */
+    PDBGFASDBNODE pDbNode = (PDBGFASDBNODE)RTAvlPVRemove(&pVM->dbgf.s.AsHandleTree, hDbgAs);
+    if (!pDbNode)
+    {
+        DBGF_AS_DB_UNLOCK_WRITE(pVM);
+        return VERR_NOT_FOUND;
+    }
+    RTStrSpaceRemove(&pVM->dbgf.s.AsNameSpace, pDbNode->NameCore.pszString);
+    if (pDbNode->PidCore.Key != NIL_RTPROCESS)
+        RTAvlU32Remove(&pVM->dbgf.s.AsPidTree, pDbNode->PidCore.Key);
+
+    DBGF_AS_DB_UNLOCK_WRITE(pVM);
+
+    /*
+     * Free the resources.
+     */
+    RTDbgAsRelease(hDbgAs);
+    MMR3HeapFree(pDbNode);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Changes an alias to point to a new address space.
+ *
+ * Not all the aliases can be changed, currently it's only DBGF_AS_GLOBAL
+ * and DBGF_AS_KERNEL.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   hAlias          The alias to change.
+ * @param   hAliasFor       The address space hAlias should be an alias for.
+ *                          This can be an alias. The caller's reference to
+ *                          this address space will NOT be consumed.
+ */
+VMMR3DECL(int) DBGFR3AsSetAlias(PVM pVM, RTDBGAS hAlias, RTDBGAS hAliasFor)
+{
+    /*
+     * Input validation.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertMsgReturn(DBGF_AS_IS_ALIAS(hAlias), ("%p\n", hAlias), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!DBGF_AS_IS_FIXED_ALIAS(hAlias), ("%p\n", hAlias), VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAliasFor = DBGFR3AsResolveAndRetain(pVM, hAlias);
+    if (hRealAliasFor == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Make sure the handle has is already in the database.
+     */
+    int rc = VERR_NOT_FOUND;
+    DBGF_AS_DB_LOCK_WRITE(pVM);
+    if (RTAvlPVGet(&pVM->dbgf.s.AsHandleTree, hRealAliasFor))
+    {
+        /*
+         * Update the alias table and release the current address space.
+         */
+        RTDBGAS hAsOld;
+        ASMAtomicXchgHandle(&pVM->dbgf.s.ahAsAliases[DBGF_AS_ALIAS_2_INDEX(hAlias)], hRealAliasFor, &hAsOld);
+        uint32_t cRefs = RTDbgAsRelease(hAsOld);
+        Assert(cRefs > 0);
+        Assert(cRefs != UINT32_MAX);
+        rc = VINF_SUCCESS;
+    }
+    DBGF_AS_DB_UNLOCK_WRITE(pVM);
+
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNPDMR3ENUM}
+ */
+static DECLCALLBACK(int) dbgfR3AsLazyPopulateR0Callback(PVM pVM, const char *pszFilename, const char *pszName,
+                                                        RTUINTPTR ImageBase, size_t cbImage, bool fRC, void *pvArg)
+{
+    /* Only ring-0 modules. */
+    if (!fRC)
+    {
+        RTDBGMOD hDbgMod;
+        int rc = RTDbgModCreateFromImage(&hDbgMod, pszFilename, pszName, 0 /*fFlags*/);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTDbgAsModuleLink((RTDBGAS)pvArg, hDbgMod, ImageBase, 0 /*fFlags*/);
+            if (RT_FAILURE(rc))
+                LogRel(("DBGF: Failed to link module \"%s\" into DBGF_AS_R0 at %RTptr: %Rrc\n",
+                        pszName, ImageBase, rc));
+        }
+        else
+            LogRel(("DBGF: RTDbgModCreateFromImage failed with rc=%Rrc for module \"%s\" (%s)\n",
+                    rc, pszName, pszFilename));
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Lazily populates the specified address space.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   hAlias              The alias.
+ */
+static void dbgfR3AsLazyPopulate(PVM pVM, RTDBGAS hAlias)
+{
+    DBGF_AS_DB_LOCK_WRITE(pVM);
+    uintptr_t iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
+    if (!pVM->dbgf.s.afAsAliasPopuplated[iAlias])
+    {
+        RTDBGAS hAs = pVM->dbgf.s.ahAsAliases[iAlias];
+        if (hAlias == DBGF_AS_R0)
+            PDMR3LdrEnumModules(pVM, dbgfR3AsLazyPopulateR0Callback, hAs);
+        /** @todo what do we do about DBGF_AS_RC?  */
+
+        pVM->dbgf.s.afAsAliasPopuplated[iAlias] = true;
+    }
+    DBGF_AS_DB_UNLOCK_WRITE(pVM);
+}
+
+
+/**
+ * Resolves the address space handle into a real handle if it's an alias.
+ *
+ * @returns Real address space handle. NIL_RTDBGAS if invalid handle.
+ *
+ * @param   pVM             The VM handle.
+ * @param   hAlias          The possibly address space alias.
+ *
+ * @remarks Doesn't take any locks.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsResolve(PVM pVM, RTDBGAS hAlias)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NULL);
+    AssertCompileNS(NIL_RTDBGAS == (RTDBGAS)0);
+
+    uintptr_t   iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
+    if (iAlias < DBGF_AS_COUNT)
+        ASMAtomicReadHandle(&pVM->dbgf.s.ahAsAliases[iAlias], &hAlias);
+    return hAlias;
+}
+
+
+/**
+ * Resolves the address space handle into a real handle if it's an alias,
+ * and retains whatever it is.
+ *
+ * @returns Real address space handle. NIL_RTDBGAS if invalid handle.
+ *
+ * @param   pVM             The VM handle.
+ * @param   hAlias          The possibly address space alias.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsResolveAndRetain(PVM pVM, RTDBGAS hAlias)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NULL);
+    AssertCompileNS(NIL_RTDBGAS == (RTDBGAS)0);
+
+    uint32_t    cRefs;
+    uintptr_t   iAlias = DBGF_AS_ALIAS_2_INDEX(hAlias);
+    if (iAlias < DBGF_AS_COUNT)
+    {
+        if (DBGF_AS_IS_FIXED_ALIAS(hAlias))
+        {
+            /* Perform lazy address space population. */
+            if (!pVM->dbgf.s.afAsAliasPopuplated[iAlias])
+                dbgfR3AsLazyPopulate(pVM, hAlias);
+
+            /* Won't ever change, no need to grab the lock. */
+            hAlias = pVM->dbgf.s.ahAsAliases[iAlias];
+            cRefs = RTDbgAsRetain(hAlias);
+        }
+        else
+        {
+            /* May change, grab the lock so we can read it safely. */
+            DBGF_AS_DB_LOCK_READ(pVM);
+            hAlias = pVM->dbgf.s.ahAsAliases[iAlias];
+            cRefs = RTDbgAsRetain(hAlias);
+            DBGF_AS_DB_UNLOCK_READ(pVM);
+        }
+    }
+    else
+        /* Not an alias, just retain it. */
+        cRefs = RTDbgAsRetain(hAlias);
+
+    return cRefs != UINT32_MAX ? hAlias : NIL_RTDBGAS;
+}
+
+
+/**
+ * Query an address space by name.
+ *
+ * @returns Retained address space handle if found, NIL_RTDBGAS if not.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pszName     The name.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsQueryByName(PVM pVM, const char *pszName)
+{
+    /*
+     * Validate the input.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NIL_RTDBGAS);
+    AssertPtrReturn(pszName, NIL_RTDBGAS);
+    AssertReturn(*pszName, NIL_RTDBGAS);
+
+    /*
+     * Look it up in the string space and retain the result.
+     */
+    RTDBGAS hDbgAs = NIL_RTDBGAS;
+    DBGF_AS_DB_LOCK_READ(pVM);
+
+    PRTSTRSPACECORE pNode = RTStrSpaceGet(&pVM->dbgf.s.AsNameSpace, pszName);
+    if (pNode)
+    {
+        PDBGFASDBNODE pDbNode = RT_FROM_MEMBER(pNode, DBGFASDBNODE, NameCore);
+        hDbgAs = (RTDBGAS)pDbNode->HandleCore.Key;
+        uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+        if (RT_UNLIKELY(cRefs == UINT32_MAX))
+            hDbgAs = NIL_RTDBGAS;
+    }
+    DBGF_AS_DB_UNLOCK_READ(pVM);
+
+    return hDbgAs;
+}
+
+
+/**
+ * Query an address space by process ID.
+ *
+ * @returns Retained address space handle if found, NIL_RTDBGAS if not.
+ *
+ * @param   pVM         The VM handle.
+ * @param   ProcId      The process ID.
+ */
+VMMR3DECL(RTDBGAS) DBGFR3AsQueryByPid(PVM pVM, RTPROCESS ProcId)
+{
+    /*
+     * Validate the input.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, NIL_RTDBGAS);
+    AssertReturn(ProcId != NIL_RTPROCESS, NIL_RTDBGAS);
+
+    /*
+     * Look it up in the PID tree and retain the result.
+     */
+    RTDBGAS hDbgAs = NIL_RTDBGAS;
+    DBGF_AS_DB_LOCK_READ(pVM);
+
+    PAVLU32NODECORE pNode = RTAvlU32Get(&pVM->dbgf.s.AsPidTree, ProcId);
+    if (pNode)
+    {
+        PDBGFASDBNODE pDbNode = RT_FROM_MEMBER(pNode, DBGFASDBNODE, PidCore);
+        hDbgAs = (RTDBGAS)pDbNode->HandleCore.Key;
+        uint32_t cRefs = RTDbgAsRetain(hDbgAs);
+        if (RT_UNLIKELY(cRefs == UINT32_MAX))
+            hDbgAs = NIL_RTDBGAS;
+    }
+    DBGF_AS_DB_UNLOCK_READ(pVM);
+
+    return hDbgAs;
+}
+
+
+/**
+ * Searches for the file in the path.
+ *
+ * The file is first tested without any path modification, then we walk the path
+ * looking in each directory.
+ *
+ * @returns VBox status code.
+ * @param   pszFilename     The file to search for.
+ * @param   pszPath         The search path.
+ * @param   pfnOpen         The open callback function.
+ * @param   pvUser          User argument for the callback.
+ */
+static int dbgfR3AsSearchPath(const char *pszFilename, const char *pszPath, PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
+{
+    char szFound[RTPATH_MAX];
+
+    /* Check the filename length. */
+    size_t const    cchFilename = strlen(pszFilename);
+    if (cchFilename >= sizeof(szFound))
+        return VERR_FILENAME_TOO_LONG;
+    const char     *pszName = RTPathFilename(pszFilename);
+    if (!pszName)
+        return VERR_IS_A_DIRECTORY;
+    size_t const    cchName = strlen(pszName);
+
+    /*
+     * Try default location first.
+     */
+    memcpy(szFound, pszFilename, cchFilename + 1);
+    int rc = pfnOpen(szFound, pvUser);
+    if (RT_SUCCESS(rc))
+        return rc;
+
+    /*
+     * Walk the search path.
+     */
+    const char *psz = pszPath;
+    while (*psz)
+    {
+        /* Skip leading blanks - no directories with leading spaces, thank you. */
+        while (RT_C_IS_BLANK(*psz))
+            psz++;
+
+        /* Find the end of this element. */
+        const char *pszNext;
+        const char *pszEnd = strchr(psz, ';');
+        if (!pszEnd)
+            pszEnd = pszNext = strchr(psz, '\0');
+        else
+            pszNext = pszEnd + 1;
+        if (pszEnd != psz)
+        {
+            size_t const cch = pszEnd - psz;
+            if (cch + 1 + cchName < sizeof(szFound))
+            {
+                /** @todo RTPathCompose, RTPathComposeN(). This code isn't right
+                 * for 'E:' on DOS systems. It may also create unwanted double slashes. */
+                memcpy(szFound, psz, cch);
+                szFound[cch] = '/';
+                memcpy(szFound + cch + 1, pszName, cchName + 1);
+                int rc2 = pfnOpen(szFound, pvUser);
+                if (RT_SUCCESS(rc2))
+                    return rc2;
+                if (    rc2 != rc
+                    &&  (   rc == VERR_FILE_NOT_FOUND
+                         || rc == VERR_OPEN_FAILED))
+                    rc = rc2;
+            }
+        }
+
+        /* advance */
+        psz = pszNext;
+    }
+
+    /*
+     * Walk the path once again, this time do a depth search.
+     */
+    /** @todo do a depth search using the specified path. */
+
+    /* failed */
+    return rc;
+}
+
+
+/**
+ * Same as dbgfR3AsSearchEnv, except that the path is taken from the environment.
+ *
+ * It the environment variable doesn't exist, the current directory is searched instead.
+ *
+ * @returns VBox status code.
+ * @param   pszFilename     The filename.
+ * @param   pszEnvVar       The environment variable name.
+ * @param   pfnOpen         The open callback function.
+ * @param   pvUser          User argument for the callback.
+ */
+static int dbgfR3AsSearchEnvPath(const char *pszFilename, const char *pszEnvVar, PFNDBGFR3ASSEARCHOPEN pfnOpen, void *pvUser)
+{
+    int     rc;
+    char   *pszPath = RTEnvDupEx(RTENV_DEFAULT, pszEnvVar);
+    if (pszPath)
+    {
+        rc = dbgfR3AsSearchPath(pszFilename, pszPath, pfnOpen, pvUser);
+        RTStrFree(pszPath);
+    }
+    else
+        rc = dbgfR3AsSearchPath(pszFilename, ".", pfnOpen, pvUser);
+    return rc;
+}
+
+
+/**
+ * Callback function used by DBGFR3AsLoadImage.
+ *
+ * @returns VBox status code.
+ * @param   pszFilename     The filename under evaluation.
+ * @param   pvUser          Use arguments (DBGFR3ASLOADOPENDATA).
+ */
+static DECLCALLBACK(int) dbgfR3AsLoadImageOpen(const char *pszFilename, void *pvUser)
+{
+    DBGFR3ASLOADOPENDATA *pData = (DBGFR3ASLOADOPENDATA *)pvUser;
+    return RTDbgModCreateFromImage(&pData->hMod, pszFilename, pData->pszModName, pData->fFlags);
+}
+
+
+/**
+ * Load symbols from an executable module into the specified address space.
+ *
+ * If an module exist at the specified address it will be replaced by this
+ * call, otherwise a new module is created.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The VM handle.
+ * @param   hDbgAs          The address space.
+ * @param   pszFilename     The filename of the executable module.
+ * @param   pszModName      The module name. If NULL, then then the file name
+ *                          base is used (no extension or nothing).
+ * @param   pModAddress     The load address of the module.
+ * @param   iModSeg         The segment to load, pass NIL_RTDBGSEGIDX to load
+ *                          the whole image.
+ * @param   fFlags          Flags reserved for future extensions, must be 0.
+ */
+VMMR3DECL(int) DBGFR3AsLoadImage(PVM pVM, RTDBGAS hDbgAs, const char *pszFilename, const char *pszModName, PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, uint32_t fFlags)
+{
+    /*
+     * Validate input
+     */
+    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
+    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
+    AssertReturn(DBGFR3AddrIsValid(pVM, pModAddress), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags == 0, VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the work.
+     */
+    DBGFR3ASLOADOPENDATA Data;
+    Data.pszModName = pszModName;
+    Data.uSubtrahend = 0;
+    Data.fFlags = 0;
+    Data.hMod = NIL_RTDBGMOD;
+    int rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_IMAGE_PATH", dbgfR3AsLoadImageOpen, &Data);
+    if (RT_FAILURE(rc))
+        rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_PATH", dbgfR3AsLoadImageOpen, &Data);
+    if (RT_SUCCESS(rc))
+    {
+        rc = DBGFR3AsLinkModule(pVM, hRealAS, Data.hMod, pModAddress, iModSeg, 0);
+        if (RT_FAILURE(rc))
+            RTDbgModRelease(Data.hMod);
+    }
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Callback function used by DBGFR3AsLoadMap.
+ *
+ * @returns VBox status code.
+ * @param   pszFilename     The filename under evaluation.
+ * @param   pvUser          Use arguments (DBGFR3ASLOADOPENDATA).
+ */
+static DECLCALLBACK(int) dbgfR3AsLoadMapOpen(const char *pszFilename, void *pvUser)
+{
+    DBGFR3ASLOADOPENDATA *pData = (DBGFR3ASLOADOPENDATA *)pvUser;
+    return RTDbgModCreateFromMap(&pData->hMod, pszFilename, pData->pszModName, pData->uSubtrahend, pData->fFlags);
+}
+
+
+/**
+ * Load symbols from a map file into a module at the specified address space.
+ *
+ * If an module exist at the specified address it will be replaced by this
+ * call, otherwise a new module is created.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The VM handle.
+ * @param   hDbgAs          The address space.
+ * @param   pszFilename     The map file.
+ * @param   pszModName      The module name. If NULL, then then the file name
+ *                          base is used (no extension or nothing).
+ * @param   pModAddress     The load address of the module.
+ * @param   iModSeg         The segment to load, pass NIL_RTDBGSEGIDX to load
+ *                          the whole image.
+ * @param   uSubtrahend     Value to to subtract from the symbols in the map
+ *                          file. This is useful for the linux System.map and
+ *                          /proc/kallsyms.
+ * @param   fFlags          Flags reserved for future extensions, must be 0.
+ */
+VMMR3DECL(int) DBGFR3AsLoadMap(PVM pVM, RTDBGAS hDbgAs, const char *pszFilename, const char *pszModName,
+                               PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, RTGCUINTPTR uSubtrahend, uint32_t fFlags)
+{
+    /*
+     * Validate input
+     */
+    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
+    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
+    AssertReturn(DBGFR3AddrIsValid(pVM, pModAddress), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags == 0, VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the work.
+     */
+    DBGFR3ASLOADOPENDATA Data;
+    Data.pszModName = pszModName;
+    Data.uSubtrahend = uSubtrahend;
+    Data.fFlags = 0;
+    Data.hMod = NIL_RTDBGMOD;
+    int rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_MAP_PATH", dbgfR3AsLoadMapOpen, &Data);
+    if (RT_FAILURE(rc))
+        rc = dbgfR3AsSearchEnvPath(pszFilename, "VBOXDBG_PATH", dbgfR3AsLoadMapOpen, &Data);
+    if (RT_SUCCESS(rc))
+    {
+        rc = DBGFR3AsLinkModule(pVM, hRealAS, Data.hMod, pModAddress, iModSeg, 0);
+        if (RT_FAILURE(rc))
+            RTDbgModRelease(Data.hMod);
+    }
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Wrapper around RTDbgAsModuleLink, RTDbgAsModuleLinkSeg and DBGFR3AsResolve.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   hDbgAs          The address space handle.
+ * @param   hMod            The module handle.
+ * @param   pModAddress     The link address.
+ * @param   iModSeg         The segment to link, NIL_RTDBGSEGIDX for the entire image.
+ * @param   fFlags          Flags to pass to the link functions, see RTDBGASLINK_FLAGS_*.
+ */
+VMMR3DECL(int) DBGFR3AsLinkModule(PVM pVM, RTDBGAS hDbgAs, RTDBGMOD hMod, PCDBGFADDRESS pModAddress, RTDBGSEGIDX iModSeg, uint32_t fFlags)
+{
+    /*
+     * Input validation.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(DBGFR3AddrIsValid(pVM, pModAddress), VERR_INVALID_PARAMETER);
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the job.
+     */
+    int rc;
+    if (iModSeg == NIL_RTDBGSEGIDX)
+        rc = RTDbgAsModuleLink(hRealAS, hMod, pModAddress->FlatPtr, fFlags);
+    else
+        rc = RTDbgAsModuleLinkSeg(hRealAS, hMod, iModSeg, pModAddress->FlatPtr, fFlags);
+
+    RTDbgAsRelease(hRealAS);
+    return rc;
+}
+
+
+/**
+ * Adds the module name to the symbol name.
+ *
+ * @param   pSymbol         The symbol info (in/out).
+ * @param   hMod            The module handle.
+ */
+static void dbgfR3AsSymbolJoinNames(PRTDBGSYMBOL pSymbol, RTDBGMOD hMod)
+{
+    /* Figure the lengths, adjust them if the result is too long. */
+    const char *pszModName = RTDbgModName(hMod);
+    size_t      cchModName = strlen(pszModName);
+    size_t      cchSymbol  = strlen(pSymbol->szName);
+    if (cchModName + 1 + cchSymbol >= sizeof(pSymbol->szName))
+    {
+        if (cchModName >= sizeof(pSymbol->szName) / 4)
+            cchModName = sizeof(pSymbol->szName) / 4;
+        if (cchModName + 1 + cchSymbol >= sizeof(pSymbol->szName))
+            cchSymbol = sizeof(pSymbol->szName) - cchModName - 2;
+        Assert(cchModName + 1 + cchSymbol < sizeof(pSymbol->szName));
+    }
+
+    /* Do the moving and copying. */
+    memmove(&pSymbol->szName[cchModName + 1], &pSymbol->szName[0], cchSymbol + 1);
+    memcpy(&pSymbol->szName[0], pszModName, cchModName);
+    pSymbol->szName[cchModName] = '!';
+}
+
+
+/** Temporary symbol conversion function. */
+static void dbgfR3AsSymbolConvert(PRTDBGSYMBOL pSymbol, PCDBGFSYMBOL pDbgfSym)
+{
+    pSymbol->offSeg = pSymbol->Value = pDbgfSym->Value;
+    pSymbol->cb = pDbgfSym->cb;
+    pSymbol->iSeg = 0;
+    pSymbol->fFlags = 0;
+    pSymbol->iOrdinal = UINT32_MAX;
+    strcpy(pSymbol->szName, pDbgfSym->szName);
+}
+
+
+/**
+ * Query a symbol by address.
+ *
+ * The returned symbol is the one we consider closes to the specified address.
+ *
+ * @returns VBox status code. See RTDbgAsSymbolByAddr.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   hDbgAs              The address space handle.
+ * @param   pAddress            The address to lookup.
+ * @param   poffDisp            Where to return the distance between the
+ *                              returned symbol and pAddress. Optional.
+ * @param   pSymbol             Where to return the symbol information.
+ *                              The returned symbol name will be prefixed by
+ *                              the module name as far as space allows.
+ * @param   phMod               Where to return the module handle. Optional.
+ */
+VMMR3DECL(int) DBGFR3AsSymbolByAddr(PVM pVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress,
+                                    PRTGCINTPTR poffDisp, PRTDBGSYMBOL pSymbol, PRTDBGMOD phMod)
+{
+    /*
+     * Implement the special address space aliases the lazy way.
+     */
+    if (hDbgAs == DBGF_AS_RC_AND_GC_GLOBAL)
+    {
+        int rc = DBGFR3AsSymbolByAddr(pVM, DBGF_AS_RC, pAddress, poffDisp, pSymbol, phMod);
+        if (RT_FAILURE(rc))
+            rc = DBGFR3AsSymbolByAddr(pVM, DBGF_AS_GLOBAL, pAddress, poffDisp, pSymbol, phMod);
+        return rc;
+    }
+
+    /*
+     * Input validation.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(DBGFR3AddrIsValid(pVM, pAddress), VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(poffDisp, VERR_INVALID_POINTER);
+    AssertPtrReturn(pSymbol, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(phMod, VERR_INVALID_POINTER);
+    if (poffDisp)
+        *poffDisp = 0;
+    if (phMod)
+        *phMod = NIL_RTDBGMOD;
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+    /*
+     * Do the lookup.
+     */
+    RTDBGMOD hMod;
+    int rc = RTDbgAsSymbolByAddr(hRealAS, pAddress->FlatPtr, poffDisp, pSymbol, &hMod);
+    if (RT_SUCCESS(rc))
+    {
+        dbgfR3AsSymbolJoinNames(pSymbol, hMod);
+        if (!phMod)
+            RTDbgModRelease(hMod);
+    }
+    /* Temporary conversions. */
+    else if (hDbgAs == DBGF_AS_GLOBAL)
+    {
+        DBGFSYMBOL DbgfSym;
+        rc = DBGFR3SymbolByAddr(pVM, pAddress->FlatPtr, poffDisp, &DbgfSym);
+        if (RT_SUCCESS(rc))
+            dbgfR3AsSymbolConvert(pSymbol, &DbgfSym);
+    }
+    else if (hDbgAs == DBGF_AS_R0)
+    {
+        RTR0PTR     R0PtrMod;
+        char        szNearSym[260];
+        RTR0PTR     R0PtrNearSym;
+        RTR0PTR     R0PtrNearSym2;
+        rc = PDMR3LdrQueryR0ModFromPC(pVM, pAddress->FlatPtr,
+                                      pSymbol->szName, sizeof(pSymbol->szName) / 2, &R0PtrMod,
+                                      &szNearSym[0],   sizeof(szNearSym),           &R0PtrNearSym,
+                                      NULL,            0,                           &R0PtrNearSym2);
+        if (RT_SUCCESS(rc))
+        {
+            pSymbol->offSeg     = pSymbol->Value = R0PtrNearSym;
+            pSymbol->cb         = R0PtrNearSym2 > R0PtrNearSym ? R0PtrNearSym2 - R0PtrNearSym : 0;
+            pSymbol->iSeg       = 0;
+            pSymbol->fFlags     = 0;
+            pSymbol->iOrdinal   = UINT32_MAX;
+            size_t offName = strlen(pSymbol->szName);
+            pSymbol->szName[offName++] = '!';
+            size_t cchNearSym = strlen(szNearSym);
+            if (cchNearSym + offName >= sizeof(pSymbol->szName))
+                cchNearSym = sizeof(pSymbol->szName) - offName - 1;
+            strncpy(&pSymbol->szName[offName], szNearSym, cchNearSym);
+            pSymbol->szName[offName + cchNearSym] = '\0';
+            if (poffDisp)
+                *poffDisp = pAddress->FlatPtr - pSymbol->Value;
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Convenience function that combines RTDbgSymbolDup and DBGFR3AsSymbolByAddr.
+ *
+ * @returns Pointer to the symbol on success. This must be free using
+ *          RTDbgSymbolFree(). NULL is returned if not found or any error
+ *          occurs.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   hDbgAs              See DBGFR3AsSymbolByAddr.
+ * @param   pAddress            See DBGFR3AsSymbolByAddr.
+ * @param   poffDisp            See DBGFR3AsSymbolByAddr.
+ * @param   phMod               See DBGFR3AsSymbolByAddr.
+ */
+VMMR3DECL(PRTDBGSYMBOL) DBGFR3AsSymbolByAddrA(PVM pVM, RTDBGAS hDbgAs, PCDBGFADDRESS pAddress, PRTGCINTPTR poffDisp, PRTDBGMOD phMod)
+{
+    RTDBGSYMBOL SymInfo;
+    int rc = DBGFR3AsSymbolByAddr(pVM, hDbgAs, pAddress, poffDisp, &SymInfo, phMod);
+    if (RT_SUCCESS(rc))
+        return RTDbgSymbolDup(&SymInfo);
+    return NULL;
+}
+
+
+/**
+ * Query a symbol by name.
+ *
+ * The symbol can be prefixed by a module name pattern to scope the search. The
+ * pattern is a simple string pattern with '*' and '?' as wild chars. See
+ * RTStrSimplePatternMatch().
+ *
+ * @returns VBox status code. See RTDbgAsSymbolByAddr.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   hDbgAs              The address space handle.
+ * @param   pszSymbol           The symbol to search for, maybe prefixed by a
+ *                              module pattern.
+ * @param   pSymbol             Where to return the symbol information.
+ *                              The returned symbol name will be prefixed by
+ *                              the module name as far as space allows.
+ * @param   phMod               Where to return the module handle. Optional.
+ */
+VMMR3DECL(int) DBGFR3AsSymbolByName(PVM pVM, RTDBGAS hDbgAs, const char *pszSymbol,
+                                    PRTDBGSYMBOL pSymbol, PRTDBGMOD phMod)
+{
+    /*
+     * Implement the special address space aliases the lazy way.
+     */
+    if (hDbgAs == DBGF_AS_RC_AND_GC_GLOBAL)
+    {
+        int rc = DBGFR3AsSymbolByName(pVM, DBGF_AS_RC, pszSymbol, pSymbol, phMod);
+        if (RT_FAILURE(rc))
+            rc = DBGFR3AsSymbolByName(pVM, DBGF_AS_GLOBAL, pszSymbol, pSymbol, phMod);
+        return rc;
+    }
+
+    /*
+     * Input validation.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pSymbol, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(phMod, VERR_INVALID_POINTER);
+    if (phMod)
+        *phMod = NIL_RTDBGMOD;
+    RTDBGAS hRealAS = DBGFR3AsResolveAndRetain(pVM, hDbgAs);
+    if (hRealAS == NIL_RTDBGAS)
+        return VERR_INVALID_HANDLE;
+
+
+    /*
+     * Do the lookup.
+     */
+    RTDBGMOD hMod;
+    int rc = RTDbgAsSymbolByName(hRealAS, pszSymbol, pSymbol, &hMod);
+    if (RT_SUCCESS(rc))
+    {
+        dbgfR3AsSymbolJoinNames(pSymbol, hMod);
+        if (!phMod)
+            RTDbgModRelease(hMod);
+    }
+    /* Temporary conversion. */
+    else if (hDbgAs == DBGF_AS_GLOBAL)
+    {
+        DBGFSYMBOL DbgfSym;
+        rc = DBGFR3SymbolByName(pVM, pszSymbol, &DbgfSym);
+        if (RT_SUCCESS(rc))
+            dbgfR3AsSymbolConvert(pSymbol, &DbgfSym);
+    }
+
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFBp.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFBp.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFBp.cpp	(revision 35333)
@@ -0,0 +1,968 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Breakpoint Management.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/selm.h>
+#include <VBox/rem.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/mm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+RT_C_DECLS_BEGIN
+static DECLCALLBACK(int) dbgfR3BpSetReg(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable,
+                                        uint8_t u8Type, uint8_t cb, PRTUINT piBp);
+static DECLCALLBACK(int) dbgfR3BpSetInt3(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp);
+static DECLCALLBACK(int) dbgfR3BpSetREM(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp);
+static DECLCALLBACK(int) dbgfR3BpClear(PVM pVM, RTUINT iBp);
+static DECLCALLBACK(int) dbgfR3BpEnable(PVM pVM, RTUINT iBp);
+static DECLCALLBACK(int) dbgfR3BpDisable(PVM pVM, RTUINT iBp);
+static DECLCALLBACK(int) dbgfR3BpEnum(PVM pVM, PFNDBGFBPENUM pfnCallback, void *pvUser);
+static int dbgfR3BpRegArm(PVM pVM, PDBGFBP pBp);
+static int dbgfR3BpRegDisarm(PVM pVM, PDBGFBP pBp);
+static int dbgfR3BpInt3Arm(PVM pVM, PDBGFBP pBp);
+static int dbgfR3BpInt3Disarm(PVM pVM, PDBGFBP pBp);
+RT_C_DECLS_END
+
+
+
+/**
+ * Initialize the breakpoint stuff.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pVM     The VM handle.
+ */
+int dbgfR3BpInit(PVM pVM)
+{
+    /*
+     * Init structures.
+     */
+    unsigned i;
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++)
+    {
+        pVM->dbgf.s.aHwBreakpoints[i].iBp = i;
+        pVM->dbgf.s.aHwBreakpoints[i].enmType = DBGFBPTYPE_FREE;
+        pVM->dbgf.s.aHwBreakpoints[i].u.Reg.iReg = i;
+    }
+
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
+    {
+        pVM->dbgf.s.aBreakpoints[i].iBp = i + RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+        pVM->dbgf.s.aBreakpoints[i].enmType = DBGFBPTYPE_FREE;
+    }
+
+    /*
+     * Register saved state.
+     */
+    /** @todo */
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Allocate a breakpoint.
+ *
+ * @returns Pointer to the allocated breakpoint.
+ * @returns NULL if we're out of breakpoints.
+ * @param   pVM     The VM handle.
+ * @param   enmType The type to allocate.
+ */
+static PDBGFBP dbgfR3BpAlloc(PVM pVM, DBGFBPTYPE enmType)
+{
+    /*
+     * Determine which array to search.
+     */
+    unsigned cBps;
+    PRTUINT  pcBpsCur;
+    PDBGFBP  paBps;
+    switch (enmType)
+    {
+        case DBGFBPTYPE_REG:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+            paBps = &pVM->dbgf.s.aHwBreakpoints[0];
+            pcBpsCur = &pVM->dbgf.s.cHwBreakpoints;
+            break;
+
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aBreakpoints);
+            paBps = &pVM->dbgf.s.aBreakpoints[0];
+            pcBpsCur = &pVM->dbgf.s.cBreakpoints;
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            return NULL;
+    }
+
+    /*
+     * Search.
+     */
+    for (unsigned iBp = 0; iBp < cBps; iBp++)
+        if (paBps[iBp].enmType == DBGFBPTYPE_FREE)
+        {
+            ++*pcBpsCur;
+            paBps[iBp].cHits   = 0;
+            paBps[iBp].enmType = enmType;
+            return &paBps[iBp];
+        }
+
+    LogFlow(("dbgfR3BpAlloc: returns NULL - we're out of breakpoint slots! %u/%u\n", *pcBpsCur, cBps));
+    return NULL;
+}
+
+
+/**
+ * Get a breakpoint give by breakpoint id.
+ *
+ * @returns Pointer to the allocated breakpoint.
+ * @returns NULL if the breakpoint is invalid.
+ * @param   pVM     The VM handle.
+ * @param   iBp     The breakpoint id.
+ */
+static PDBGFBP dbgfR3BpGet(PVM pVM, RTUINT iBp)
+{
+    /* Find it. */
+    PDBGFBP pBp;
+    if (iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints))
+        pBp = &pVM->dbgf.s.aHwBreakpoints[iBp];
+    else
+    {
+        iBp -= RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+        if (iBp >= RT_ELEMENTS(pVM->dbgf.s.aBreakpoints))
+            return NULL;
+        pBp = &pVM->dbgf.s.aBreakpoints[iBp];
+    }
+
+    /* check if it's valid. */
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_FREE:
+            return NULL;
+
+        case DBGFBPTYPE_REG:
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
+            return NULL;
+    }
+
+    return pBp;
+}
+
+
+/**
+ * Get a breakpoint give by address.
+ *
+ * @returns Pointer to the allocated breakpoint.
+ * @returns NULL if the breakpoint is invalid.
+ * @param   pVM     The VM handle.
+ * @param   enmType The breakpoint type.
+ * @param   GCPtr   The breakpoint address.
+ */
+static PDBGFBP dbgfR3BpGetByAddr(PVM pVM, DBGFBPTYPE enmType, RTGCUINTPTR GCPtr)
+{
+    /*
+     * Determine which array to search.
+     */
+    unsigned cBps;
+    PDBGFBP  paBps;
+    switch (enmType)
+    {
+        case DBGFBPTYPE_REG:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints);
+            paBps = &pVM->dbgf.s.aHwBreakpoints[0];
+            break;
+
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+            cBps = RT_ELEMENTS(pVM->dbgf.s.aBreakpoints);
+            paBps = &pVM->dbgf.s.aBreakpoints[0];
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            return NULL;
+    }
+
+    /*
+     * Search.
+     */
+    for (unsigned iBp = 0; iBp < cBps; iBp++)
+    {
+        if (    paBps[iBp].enmType == enmType
+            &&  paBps[iBp].GCPtr == GCPtr)
+            return &paBps[iBp];
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Frees a breakpoint.
+ *
+ * @param   pVM     The VM handle.
+ * @param   pBp     The breakpoint to free.
+ */
+static void dbgfR3BpFree(PVM pVM, PDBGFBP pBp)
+{
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_FREE:
+            AssertMsgFailed(("Already freed!\n"));
+            return;
+
+        case DBGFBPTYPE_REG:
+            Assert(pVM->dbgf.s.cHwBreakpoints > 0);
+            pVM->dbgf.s.cHwBreakpoints--;
+            break;
+
+        case DBGFBPTYPE_INT3:
+        case DBGFBPTYPE_REM:
+            Assert(pVM->dbgf.s.cBreakpoints > 0);
+            pVM->dbgf.s.cBreakpoints--;
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
+            return;
+
+    }
+    pBp->enmType = DBGFBPTYPE_FREE;
+}
+
+
+/**
+ * Sets a breakpoint (int 3 based).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pAddress    The address of the breakpoint.
+ * @param   iHitTrigger The hit count at which the breakpoint start triggering.
+ *                      Use 0 (or 1) if it's gonna trigger at once.
+ * @param   iHitDisable The hit count which disables the breakpoint.
+ *                      Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp        Where to store the breakpoint id. (optional)
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpSet(PVM pVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable, PRTUINT piBp)
+{
+    /*
+     * This must be done in EMT.
+     */
+    /** @todo SMP? */
+    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpSetInt3, 5, pVM, pAddress, &iHitTrigger, &iHitDisable, piBp);
+    LogFlow(("DBGFR3BpSet: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Sets a breakpoint (int 3 based).
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pAddress        The address of the breakpoint.
+ * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
+ *                          Use 0 (or 1) if it's gonna trigger at once.
+ * @param   piHitDisable    The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp            Where to store the breakpoint id. (optional)
+ * @thread  Any thread.
+ */
+static DECLCALLBACK(int) dbgfR3BpSetInt3(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp)
+{
+    /*
+     * Validate input.
+     */
+    if (!DBGFR3AddrIsValid(pVM, pAddress))
+        return VERR_INVALID_PARAMETER;
+    if (*piHitTrigger > *piHitDisable)
+        return VERR_INVALID_PARAMETER;
+    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
+    if (piBp)
+        *piBp = ~0;
+
+    /*
+     * Check if the breakpoint already exists.
+     */
+    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_INT3, pAddress->FlatPtr);
+    if (pBp)
+    {
+        int rc = VINF_SUCCESS;
+        if (!pBp->fEnabled)
+            rc = dbgfR3BpInt3Arm(pVM, pBp);
+        if (RT_SUCCESS(rc))
+        {
+            rc = VINF_DBGF_BP_ALREADY_EXIST;
+            if (piBp)
+                *piBp = pBp->iBp;
+        }
+        return rc;
+    }
+
+    /*
+     * Allocate and initialize the bp.
+     */
+    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_INT3);
+    if (!pBp)
+        return VERR_DBGF_NO_MORE_BP_SLOTS;
+    pBp->GCPtr       = pAddress->FlatPtr;
+    pBp->iHitTrigger = *piHitTrigger;
+    pBp->iHitDisable = *piHitDisable;
+    pBp->fEnabled    = true;
+
+    /*
+     * Now ask REM to set the breakpoint.
+     */
+    int rc = dbgfR3BpInt3Arm(pVM, pBp);
+    if (RT_SUCCESS(rc))
+    {
+        if (piBp)
+            *piBp = pBp->iBp;
+    }
+    else
+        dbgfR3BpFree(pVM, pBp);
+
+    return rc;
+}
+
+
+/**
+ * Arms an int 3 breakpoint.
+ * This is used to implement both DBGFR3BpSetReg() and DBGFR3BpEnable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pBp         The breakpoint.
+ */
+static int dbgfR3BpInt3Arm(PVM pVM, PDBGFBP pBp)
+{
+    /** @todo should actually use physical address here! */
+
+    /* @todo SMP support! */
+    VMCPUID idCpu = 0;
+
+    /*
+     * Save current byte and write int3 instruction.
+     */
+    DBGFADDRESS Addr;
+    DBGFR3AddrFromFlat(pVM, &Addr, pBp->GCPtr);
+    int rc = DBGFR3MemRead(pVM, idCpu, &Addr, &pBp->u.Int3.bOrg, 1);
+    if (RT_SUCCESS(rc))
+    {
+        static const uint8_t s_bInt3 = 0xcc;
+        rc = DBGFR3MemWrite(pVM, idCpu, &Addr, &s_bInt3, 1);
+    }
+    return rc;
+}
+
+
+/**
+ * Disarms an int 3 breakpoint.
+ * This is used to implement both DBGFR3BpClear() and DBGFR3BpDisable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pBp         The breakpoint.
+ */
+static int dbgfR3BpInt3Disarm(PVM pVM, PDBGFBP pBp)
+{
+    /* @todo SMP support! */
+    VMCPUID idCpu = 0;
+
+    /*
+     * Check that the current byte is the int3 instruction, and restore the original one.
+     * We currently ignore invalid bytes.
+     */
+    DBGFADDRESS     Addr;
+    DBGFR3AddrFromFlat(pVM, &Addr, pBp->GCPtr);
+    uint8_t         bCurrent;
+    int rc = DBGFR3MemRead(pVM, idCpu, &Addr, &bCurrent, 1);
+    if (bCurrent == 0xcc)
+        rc = DBGFR3MemWrite(pVM, idCpu, &Addr, &pBp->u.Int3.bOrg, 1);
+    return rc;
+}
+
+
+/**
+ * Sets a register breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pAddress    The address of the breakpoint.
+ * @param   iHitTrigger The hit count at which the breakpoint start triggering.
+ *                      Use 0 (or 1) if it's gonna trigger at once.
+ * @param   iHitDisable The hit count which disables the breakpoint.
+ *                      Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   fType       The access type (one of the X86_DR7_RW_* defines).
+ * @param   cb          The access size - 1,2,4 or 8 (the latter is AMD64 long mode only.
+ *                      Must be 1 if fType is X86_DR7_RW_EO.
+ * @param   piBp        Where to store the breakpoint id. (optional)
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpSetReg(PVM pVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable,
+                              uint8_t fType, uint8_t cb, PRTUINT piBp)
+{
+    /** @todo SMP - broadcast, VT-x/AMD-V. */
+    /*
+     * This must be done in EMT.
+     */
+    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpSetReg, 7, pVM, pAddress, &iHitTrigger, &iHitDisable, fType, cb, piBp);
+    LogFlow(("DBGFR3BpSetReg: returns %Rrc\n", rc));
+    return rc;
+
+}
+
+
+/**
+ * Sets a register breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pAddress        The address of the breakpoint.
+ * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
+ *                          Use 0 (or 1) if it's gonna trigger at once.
+ * @param   piHitDisable    The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   fType           The access type (one of the X86_DR7_RW_* defines).
+ * @param   cb              The access size - 1,2,4 or 8 (the latter is AMD64 long mode only.
+ *                          Must be 1 if fType is X86_DR7_RW_EO.
+ * @param   piBp            Where to store the breakpoint id. (optional)
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpSetReg(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable,
+                                        uint8_t fType, uint8_t cb, PRTUINT piBp)
+{
+    /*
+     * Validate input.
+     */
+    if (!DBGFR3AddrIsValid(pVM, pAddress))
+        return VERR_INVALID_PARAMETER;
+    if (*piHitTrigger > *piHitDisable)
+        return VERR_INVALID_PARAMETER;
+    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
+    if (piBp)
+        *piBp = ~0;
+    switch (fType)
+    {
+        case X86_DR7_RW_EO:
+            if (cb == 1)
+                break;
+            AssertMsgFailed(("fType=%#x cb=%d != 1\n", fType, cb));
+            return VERR_INVALID_PARAMETER;
+        case X86_DR7_RW_IO:
+        case X86_DR7_RW_RW:
+        case X86_DR7_RW_WO:
+            break;
+        default:
+            AssertMsgFailed(("fType=%#x\n", fType));
+            return VERR_INVALID_PARAMETER;
+    }
+    switch (cb)
+    {
+        case 1:
+        case 2:
+        case 4:
+            break;
+        default:
+            AssertMsgFailed(("cb=%#x\n", cb));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Check if the breakpoint already exists.
+     */
+    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_REG, pAddress->FlatPtr);
+    if (    pBp
+        &&  pBp->u.Reg.cb == cb
+        &&  pBp->u.Reg.fType == fType)
+    {
+        int rc = VINF_SUCCESS;
+        if (!pBp->fEnabled)
+            rc = dbgfR3BpRegArm(pVM, pBp);
+        if (RT_SUCCESS(rc))
+        {
+            rc = VINF_DBGF_BP_ALREADY_EXIST;
+            if (piBp)
+                *piBp = pBp->iBp;
+        }
+        return rc;
+    }
+
+    /*
+     * Allocate and initialize the bp.
+     */
+    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_REG);
+    if (!pBp)
+        return VERR_DBGF_NO_MORE_BP_SLOTS;
+    pBp->GCPtr       = pAddress->FlatPtr;
+    pBp->iHitTrigger = *piHitTrigger;
+    pBp->iHitDisable = *piHitDisable;
+    pBp->fEnabled    = true;
+    Assert(pBp->iBp == pBp->u.Reg.iReg);
+    pBp->u.Reg.fType = fType;
+    pBp->u.Reg.cb    = cb;
+
+    /*
+     * Arm the breakpoint.
+     */
+    int rc = dbgfR3BpRegArm(pVM, pBp);
+    if (RT_SUCCESS(rc))
+    {
+        if (piBp)
+            *piBp = pBp->iBp;
+    }
+    else
+        dbgfR3BpFree(pVM, pBp);
+
+    return rc;
+}
+
+
+/**
+ * Arms a debug register breakpoint.
+ * This is used to implement both DBGFR3BpSetReg() and DBGFR3BpEnable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pBp         The breakpoint.
+ */
+static int dbgfR3BpRegArm(PVM pVM, PDBGFBP pBp)
+{
+    /* @todo SMP support! */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    Assert(pBp->fEnabled);
+    return CPUMRecalcHyperDRx(pVCpu);
+}
+
+
+/**
+ * Disarms a debug register breakpoint.
+ * This is used to implement both DBGFR3BpClear() and DBGFR3BpDisable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pBp         The breakpoint.
+ */
+static int dbgfR3BpRegDisarm(PVM pVM, PDBGFBP pBp)
+{
+    /** @todo SMP support! */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    Assert(!pBp->fEnabled);
+    return CPUMRecalcHyperDRx(pVCpu);
+}
+
+
+/**
+ * Sets a recompiler breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pAddress    The address of the breakpoint.
+ * @param   iHitTrigger The hit count at which the breakpoint start triggering.
+ *                      Use 0 (or 1) if it's gonna trigger at once.
+ * @param   iHitDisable The hit count which disables the breakpoint.
+ *                      Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp        Where to store the breakpoint id. (optional)
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpSetREM(PVM pVM, PCDBGFADDRESS pAddress, uint64_t iHitTrigger, uint64_t iHitDisable, PRTUINT piBp)
+{
+    /*
+     * This must be done in EMT.
+     */
+    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpSetREM, 5, pVM, pAddress, &iHitTrigger, &iHitDisable, piBp);
+    LogFlow(("DBGFR3BpSetREM: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpSetREM().
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pAddress        The address of the breakpoint.
+ * @param   piHitTrigger    The hit count at which the breakpoint start triggering.
+ *                          Use 0 (or 1) if it's gonna trigger at once.
+ * @param   piHitDisable    The hit count which disables the breakpoint.
+ *                          Use ~(uint64_t) if it's never gonna be disabled.
+ * @param   piBp            Where to store the breakpoint id. (optional)
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpSetREM(PVM pVM, PCDBGFADDRESS pAddress, uint64_t *piHitTrigger, uint64_t *piHitDisable, PRTUINT piBp)
+{
+    /*
+     * Validate input.
+     */
+    if (!DBGFR3AddrIsValid(pVM, pAddress))
+        return VERR_INVALID_PARAMETER;
+    if (*piHitTrigger > *piHitDisable)
+        return VERR_INVALID_PARAMETER;
+    AssertMsgReturn(!piBp || VALID_PTR(piBp), ("piBp=%p\n", piBp), VERR_INVALID_POINTER);
+    if (piBp)
+        *piBp = ~0;
+
+
+    /*
+     * Check if the breakpoint already exists.
+     */
+    PDBGFBP pBp = dbgfR3BpGetByAddr(pVM, DBGFBPTYPE_REM, pAddress->FlatPtr);
+    if (pBp)
+    {
+        int rc = VINF_SUCCESS;
+        if (!pBp->fEnabled)
+            rc = REMR3BreakpointSet(pVM, pBp->GCPtr);
+        if (RT_SUCCESS(rc))
+        {
+            rc = VINF_DBGF_BP_ALREADY_EXIST;
+            if (piBp)
+                *piBp = pBp->iBp;
+        }
+        return rc;
+    }
+
+    /*
+     * Allocate and initialize the bp.
+     */
+    pBp = dbgfR3BpAlloc(pVM, DBGFBPTYPE_REM);
+    if (!pBp)
+        return VERR_DBGF_NO_MORE_BP_SLOTS;
+    pBp->GCPtr       = pAddress->FlatPtr;
+    pBp->iHitTrigger = *piHitTrigger;
+    pBp->iHitDisable = *piHitDisable;
+    pBp->fEnabled    = true;
+
+    /*
+     * Now ask REM to set the breakpoint.
+     */
+    int rc = REMR3BreakpointSet(pVM, pAddress->FlatPtr);
+    if (RT_SUCCESS(rc))
+    {
+        if (piBp)
+            *piBp = pBp->iBp;
+    }
+    else
+        dbgfR3BpFree(pVM, pBp);
+
+    return rc;
+}
+
+
+/**
+ * Clears a breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   iBp         The id of the breakpoint which should be removed (cleared).
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpClear(PVM pVM, RTUINT iBp)
+{
+    /*
+     * This must be done in EMT.
+     */
+    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpClear, 2, pVM, iBp);
+    LogFlow(("DBGFR3BpClear: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpClear().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   iBp         The id of the breakpoint which should be removed (cleared).
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpClear(PVM pVM, RTUINT iBp)
+{
+    /*
+     * Validate input.
+     */
+    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
+    if (!pBp)
+        return VERR_DBGF_BP_NOT_FOUND;
+
+    /*
+     * Disarm the breakpoint if it's enabled.
+     */
+    if (pBp->fEnabled)
+    {
+        pBp->fEnabled = false;
+        int rc;
+        switch (pBp->enmType)
+        {
+            case DBGFBPTYPE_REG:
+                rc = dbgfR3BpRegDisarm(pVM, pBp);
+                break;
+
+            case DBGFBPTYPE_INT3:
+                rc = dbgfR3BpInt3Disarm(pVM, pBp);
+                break;
+
+            case DBGFBPTYPE_REM:
+                rc = REMR3BreakpointClear(pVM, pBp->GCPtr);
+                break;
+
+            default:
+                AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
+                return VERR_INTERNAL_ERROR;
+        }
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Free the breakpoint.
+     */
+    dbgfR3BpFree(pVM, pBp);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enables a breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   iBp         The id of the breakpoint which should be enabled.
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpEnable(PVM pVM, RTUINT iBp)
+{
+    /*
+     * This must be done in EMT.
+     */
+    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpEnable, 2, pVM, iBp);
+    LogFlow(("DBGFR3BpEnable: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpEnable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   iBp         The id of the breakpoint which should be enabled.
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpEnable(PVM pVM, RTUINT iBp)
+{
+    /*
+     * Validate input.
+     */
+    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
+    if (!pBp)
+        return VERR_DBGF_BP_NOT_FOUND;
+
+    /*
+     * Already enabled?
+     */
+    if (pBp->fEnabled)
+        return VINF_DBGF_BP_ALREADY_ENABLED;
+
+    /*
+     * Remove the breakpoint.
+     */
+    int rc;
+    pBp->fEnabled = true;
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_REG:
+            rc = dbgfR3BpRegArm(pVM, pBp);
+            break;
+
+        case DBGFBPTYPE_INT3:
+            rc = dbgfR3BpInt3Arm(pVM, pBp);
+            break;
+
+        case DBGFBPTYPE_REM:
+            rc = REMR3BreakpointSet(pVM, pBp->GCPtr);
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
+            return VERR_INTERNAL_ERROR;
+    }
+    if (RT_FAILURE(rc))
+        pBp->fEnabled = false;
+
+    return rc;
+}
+
+
+/**
+ * Disables a breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   iBp         The id of the breakpoint which should be disabled.
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3BpDisable(PVM pVM, RTUINT iBp)
+{
+    /*
+     * This must be done in EMT.
+     */
+    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpDisable, 2, pVM, iBp);
+    LogFlow(("DBGFR3BpDisable: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpDisable().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   iBp         The id of the breakpoint which should be disabled.
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpDisable(PVM pVM, RTUINT iBp)
+{
+    /*
+     * Validate input.
+     */
+    PDBGFBP pBp = dbgfR3BpGet(pVM, iBp);
+    if (!pBp)
+        return VERR_DBGF_BP_NOT_FOUND;
+
+    /*
+     * Already enabled?
+     */
+    if (!pBp->fEnabled)
+        return VINF_DBGF_BP_ALREADY_DISABLED;
+
+    /*
+     * Remove the breakpoint.
+     */
+    pBp->fEnabled = false;
+    int rc;
+    switch (pBp->enmType)
+    {
+        case DBGFBPTYPE_REG:
+            rc = dbgfR3BpRegDisarm(pVM, pBp);
+            break;
+
+        case DBGFBPTYPE_INT3:
+            rc = dbgfR3BpInt3Disarm(pVM, pBp);
+            break;
+
+        case DBGFBPTYPE_REM:
+            rc = REMR3BreakpointClear(pVM, pBp->GCPtr);
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid enmType=%d!\n", pBp->enmType));
+            return VERR_INTERNAL_ERROR;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Enumerate the breakpoints.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pfnCallback The callback function.
+ * @param   pvUser      The user argument to pass to the callback.
+ * @thread  Any thread but the callback will be called from EMT.
+ */
+VMMR3DECL(int) DBGFR3BpEnum(PVM pVM, PFNDBGFBPENUM pfnCallback, void *pvUser)
+{
+    /*
+     * This must be done in EMT.
+     */
+    int rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3BpEnum, 3, pVM, pfnCallback, pvUser);
+    LogFlow(("DBGFR3BpClear: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT worker for DBGFR3BpEnum().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pfnCallback The callback function.
+ * @param   pvUser      The user argument to pass to the callback.
+ * @thread  EMT
+ * @internal
+ */
+static DECLCALLBACK(int) dbgfR3BpEnum(PVM pVM, PFNDBGFBPENUM pfnCallback, void *pvUser)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(VALID_PTR(pfnCallback), ("pfnCallback=%p\n", pfnCallback), VERR_INVALID_POINTER);
+
+    /*
+     * Enumerate the hardware breakpoints.
+     */
+    unsigned i;
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++)
+        if (pVM->dbgf.s.aHwBreakpoints[i].enmType != DBGFBPTYPE_FREE)
+        {
+            int rc = pfnCallback(pVM, pvUser, &pVM->dbgf.s.aHwBreakpoints[i]);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+    /*
+     * Enumerate the other breakpoints.
+     */
+    for (i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aBreakpoints); i++)
+        if (pVM->dbgf.s.aBreakpoints[i].enmType != DBGFBPTYPE_FREE)
+        {
+            int rc = pfnCallback(pVM, pvUser, &pVM->dbgf.s.aBreakpoints[i]);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFCoreWrite.cpp	(revision 35333)
@@ -0,0 +1,570 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Guest Core Dump.
+ */
+
+/*
+ * Copyright (C) 2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_dbgf_vmcore    VMCore Format
+ *
+ * The VirtualBox VMCore Format:
+ * [ ELF 64 Header]  -- Only 1
+ *
+ * [ PT_NOTE ]       -- Only 1
+ *    - Offset into CoreDescriptor followed by list of Notes (Note Hdr + data) of VBox CPUs.
+ *    - (Any Additional custom Note sections).
+ *
+ * [ PT_LOAD ]       -- One for each contiguous memory chunk
+ *    - Memory offset (physical).
+ *    - File offset.
+ *
+ * CoreDescriptor
+ *    - Magic, VBox version.
+ *    - Number of CPus.
+ *
+ * Per-CPU register dump
+ *    - CPU 1 Note Hdr + Data.
+ *    - CPU 2 Note Hdr + Data.
+ *    ...
+ * (Additional custom notes Hdr+data)
+ *    - VBox 1 Note Hdr + Data.
+ *    - VBox 2 Note Hdr + Data.
+ *    ...
+ * Memory dump
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <iprt/param.h>
+#include <iprt/file.h>
+
+#include "DBGFInternal.h"
+
+#include <VBox/cpum.h>
+#include "CPUMInternal.h"
+#include <VBox/dbgf.h>
+#include <VBox/dbgfcorefmt.h>
+#include <VBox/vm.h>
+#include <VBox/pgm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/mm.h>
+#include <VBox/version.h>
+
+#include "../../Runtime/include/internal/ldrELF64.h"
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#ifdef DEBUG_ramshankar
+# undef Log
+# define Log LogRel
+#endif
+#define DBGFLOG_NAME           "DBGFCoreWrite"
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+static const int s_NoteAlign  = 8;
+static const int s_cbNoteName = 16;
+
+/* These strings *HAVE* to be 8-byte aligned */
+static const char *s_pcszCoreVBoxCore = "VBCORE";
+static const char *s_pcszCoreVBoxCpu  = "VBCPU";
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Guest core writer data.
+ *
+ * Used to pass parameters from DBGFR3CoreWrite to dbgfR3CoreWriteRendezvous.
+ */
+typedef struct DBGFCOREDATA
+{
+    /** The name of the file to write the file to. */
+    const char *pszFilename;
+    /** Whether to replace (/overwrite) any existing file. */
+    bool        fReplaceFile;
+} DBGFCOREDATA;
+/** Pointer to the guest core writer data.  */
+typedef DBGFCOREDATA *PDBGFCOREDATA;
+
+
+
+/**
+ * ELF function to write 64-bit ELF header.
+ *
+ * @param hFile             The file to write to.
+ * @param cProgHdrs         Number of program headers.
+ * @param cSecHdrs          Number of section headers.
+ *
+ * @return IPRT status code.
+ */
+static int Elf64WriteElfHdr(RTFILE hFile, uint16_t cProgHdrs, uint16_t cSecHdrs)
+{
+    Elf64_Ehdr ElfHdr;
+    RT_ZERO(ElfHdr);
+    ElfHdr.e_ident[EI_MAG0]  = ELFMAG0;
+    ElfHdr.e_ident[EI_MAG1]  = ELFMAG1;
+    ElfHdr.e_ident[EI_MAG2]  = ELFMAG2;
+    ElfHdr.e_ident[EI_MAG3]  = ELFMAG3;
+    ElfHdr.e_ident[EI_DATA]  = ELFDATA2LSB;
+    ElfHdr.e_type            = ET_CORE;
+    ElfHdr.e_version         = EV_CURRENT;
+    ElfHdr.e_ident[EI_CLASS] = ELFCLASS64;
+    /* 32-bit builds will produce cores with e_machine EM_386. */
+#ifdef RT_ARCH_AMD64
+    ElfHdr.e_machine         = EM_X86_64;
+#else
+    ElfHdr.e_machine         = EM_386;
+#endif
+    ElfHdr.e_phnum           = cProgHdrs;
+    ElfHdr.e_shnum           = cSecHdrs;
+    ElfHdr.e_ehsize          = sizeof(ElfHdr);
+    ElfHdr.e_phoff           = sizeof(ElfHdr);
+    ElfHdr.e_phentsize       = sizeof(Elf64_Phdr);
+    ElfHdr.e_shentsize       = sizeof(Elf64_Shdr);
+
+    return RTFileWrite(hFile, &ElfHdr, sizeof(ElfHdr), NULL /* all */);
+}
+
+
+/**
+ * ELF function to write 64-bit program header.
+ *
+ * @param hFile             The file to write to.
+ * @param Type              Type of program header (PT_*).
+ * @param fFlags            Flags (access permissions, PF_*).
+ * @param offFileData       File offset of contents.
+ * @param cbFileData        Size of contents in the file.
+ * @param cbMemData         Size of contents in memory.
+ * @param Phys              Physical address, pass zero if not applicable.
+ *
+ * @return IPRT status code.
+ */
+static int Elf64WriteProgHdr(RTFILE hFile, uint32_t Type, uint32_t fFlags, uint64_t offFileData, uint64_t cbFileData,
+                             uint64_t cbMemData, RTGCPHYS Phys)
+{
+    Elf64_Phdr ProgHdr;
+    RT_ZERO(ProgHdr);
+    ProgHdr.p_type          = Type;
+    ProgHdr.p_flags         = fFlags;
+    ProgHdr.p_offset        = offFileData;
+    ProgHdr.p_filesz        = cbFileData;
+    ProgHdr.p_memsz         = cbMemData;
+    ProgHdr.p_paddr         = Phys;
+
+    return RTFileWrite(hFile, &ProgHdr, sizeof(ProgHdr), NULL /* all */);
+}
+
+
+/**
+ * Returns the size of the NOTE section given the name and size of the data.
+ *
+ * @param pszName           Name of the note section.
+ * @param cb                Size of the data portion of the note section.
+ *
+ * @return The size of the NOTE section as rounded to the file alignment.
+ */
+static uint64_t Elf64NoteSectionSize(const char *pszName, uint64_t cbData)
+{
+    uint64_t cbNote = sizeof(Elf64_Nhdr);
+
+    size_t cchName      = strlen(pszName) + 1;
+    size_t cchNameAlign = RT_ALIGN_Z(cchName, s_NoteAlign);
+
+    cbNote += cchNameAlign;
+    cbNote += RT_ALIGN_64(cbData, s_NoteAlign);
+    return cbNote;
+}
+
+
+/**
+ * Elf function to write 64-bit note header.
+ *
+ * @param hFile             The file to write to.
+ * @param Type              Type of this section.
+ * @param pszName           Name of this section.
+ * @param pcv               Opaque pointer to the data, if NULL only computes size.
+ * @param cbData            Size of the data.
+ *
+ * @return IPRT status code.
+ */
+static int Elf64WriteNoteHdr(RTFILE hFile, uint16_t Type, const char *pszName, const void *pcvData, uint64_t cbData)
+{
+    AssertReturn(pcvData, VERR_INVALID_POINTER);
+    AssertReturn(cbData > 0, VERR_NO_DATA);
+
+    char szNoteName[s_cbNoteName];
+    RT_ZERO(szNoteName);
+    RTStrCopy(szNoteName, sizeof(szNoteName), pszName);
+
+    size_t cchName       = strlen(szNoteName) + 1;
+    size_t cchNameAlign  = RT_ALIGN_Z(cchName, s_NoteAlign);
+    uint64_t cbDataAlign = RT_ALIGN_64(cbData, s_NoteAlign);
+
+    /*
+     * Yell loudly and bail if we are going to be writing a core file that is not compatible with
+     * both Solaris and the 64-bit ELF spec. which dictates 8-byte alignment. See #5211 comment 3.
+     */
+    if (cchNameAlign - cchName > 3)
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr pszName=%s cchName=%u cchNameAlign=%u, cchName aligns to 4 not 8-bytes!\n", pszName, cchName,
+                cchNameAlign));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (cbDataAlign - cbData > 3)
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr pszName=%s cbData=%u cbDataAlign=%u, cbData aligns to 4 not 8-bytes!\n", pszName, cbData,
+                cbDataAlign));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    static const char s_achPad[7] = { 0, 0, 0, 0, 0, 0, 0 };
+    AssertCompile(sizeof(s_achPad) >= s_NoteAlign - 1);
+
+    Elf64_Nhdr ElfNoteHdr;
+    RT_ZERO(ElfNoteHdr);
+    ElfNoteHdr.n_namesz = (Elf64_Word)cchName - 1; /* Again a discrepancy between ELF-64 and Solaris (#5211 comment 3), we will follow ELF-64 */
+    ElfNoteHdr.n_type   = Type;
+    ElfNoteHdr.n_descsz = (Elf64_Word)cbDataAlign;
+
+    /*
+     * Write note header.
+     */
+    int rc = RTFileWrite(hFile, &ElfNoteHdr, sizeof(ElfNoteHdr), NULL /* all */);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Write note name.
+         */
+        rc = RTFileWrite(hFile, szNoteName, cchName, NULL /* all */);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Write note name padding if required.
+             */
+            if (cchNameAlign > cchName)
+                rc = RTFileWrite(hFile, s_achPad, cchNameAlign - cchName, NULL);
+
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Write note data.
+                 */
+                rc = RTFileWrite(hFile, pcvData, cbData, NULL /* all */);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Write note data padding if required.
+                     */
+                    if (cbDataAlign > cbData)
+                        rc = RTFileWrite(hFile, s_achPad, cbDataAlign - cbData, NULL /* all*/);
+                }
+            }
+        }
+    }
+
+    if (RT_FAILURE(rc))
+        LogRel((DBGFLOG_NAME ": RTFileWrite failed. rc=%Rrc pszName=%s cchName=%u cchNameAlign=%u cbData=%u cbDataAlign=%u\n",
+                rc, pszName, cchName, cchNameAlign, cbData, cbDataAlign));
+
+    return rc;
+}
+
+
+/**
+ * Count the number of memory ranges that go into the core file.
+ *
+ * We cannot do a page-by-page dump of the entire guest memory as there will be
+ * way too many program header entries. Also we don't want to dump MMIO regions
+ * which means we cannot have a 1:1 mapping between core file offset and memory
+ * offset. Instead we dump the memory in ranges. A memory range is a contiguous
+ * memory area suitable for dumping to a core file.
+ *
+ * @param pVM               The VM handle.
+ *
+ * @return Number of memory ranges
+ */
+static uint32_t dbgfR3GetRamRangeCount(PVM pVM)
+{
+    return PGMR3PhysGetRamRangeCount(pVM);
+}
+
+
+/**
+ * Worker function for dbgfR3CoreWrite which does the writing.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The VM handle.
+ * @param   hFile               The file to write to.  Caller closes this.
+ */
+static int dbgfR3CoreWriteWorker(PVM pVM, RTFILE hFile)
+{
+    /*
+     * Collect core information.
+     */
+    uint32_t const cu32MemRanges = dbgfR3GetRamRangeCount(pVM);
+    uint16_t const cMemRanges    = cu32MemRanges < UINT16_MAX - 1 ? cu32MemRanges : UINT16_MAX - 1; /* One PT_NOTE Program header */
+    uint16_t const cProgHdrs     = cMemRanges + 1;
+
+    DBGFCOREDESCRIPTOR CoreDescriptor;
+    RT_ZERO(CoreDescriptor);
+    CoreDescriptor.u32Magic         = DBGFCORE_MAGIC;
+    CoreDescriptor.u32FmtVersion    = DBGFCORE_FMT_VERSION;
+    CoreDescriptor.cbSelf           = sizeof(CoreDescriptor);
+    CoreDescriptor.u32VBoxVersion   = VBOX_FULL_VERSION;
+    CoreDescriptor.u32VBoxRevision  = VMMGetSvnRev();
+    CoreDescriptor.cCpus            = pVM->cCpus;
+
+    Log((DBGFLOG_NAME ": CoreDescriptor Version=%u Revision=%u\n", CoreDescriptor.u32VBoxVersion, CoreDescriptor.u32VBoxRevision));
+
+    /*
+     * Compute the file layout (see pg_dbgf_vmcore).
+     */
+    uint64_t const offElfHdr        = RTFileTell(hFile);
+    uint64_t const offNoteSection   = offElfHdr         + sizeof(Elf64_Ehdr);
+    uint64_t const offLoadSections  = offNoteSection    + sizeof(Elf64_Phdr);
+    uint64_t const cbLoadSections   = cMemRanges * sizeof(Elf64_Phdr);
+    uint64_t const offCoreDescriptor= offLoadSections   + cbLoadSections;
+    uint64_t const cbCoreDescriptor = Elf64NoteSectionSize(s_pcszCoreVBoxCore, sizeof(CoreDescriptor));
+    uint64_t const offCpuDumps      = offCoreDescriptor + cbCoreDescriptor;
+    uint64_t const cbCpuDumps       = pVM->cCpus * Elf64NoteSectionSize(s_pcszCoreVBoxCpu, sizeof(CPUMCTX));
+    uint64_t const offMemory        = offCpuDumps       + cbCpuDumps;
+
+    uint64_t const offNoteSectionData = offCoreDescriptor;
+    uint64_t const cbNoteSectionData  = cbCoreDescriptor + cbCpuDumps;
+
+    /*
+     * Write ELF header.
+     */
+    int rc = Elf64WriteElfHdr(hFile, cProgHdrs, 0 /* cSecHdrs */);
+    if (RT_FAILURE(rc))
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteElfHdr failed. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Write PT_NOTE program header.
+     */
+    Assert(RTFileTell(hFile) == offNoteSection);
+    rc = Elf64WriteProgHdr(hFile, PT_NOTE, PF_R,
+                           offNoteSectionData,  /* file offset to contents */
+                           cbNoteSectionData,   /* size in core file */
+                           cbNoteSectionData,   /* size in memory */
+                           0);                  /* physical address */
+    if (RT_FAILURE(rc))
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WritreProgHdr failed for PT_NOTE. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Write PT_LOAD program header for each memory range.
+     */
+    Assert(RTFileTell(hFile) == offLoadSections);
+    uint64_t offMemRange = offMemory;
+    for (uint16_t iRange = 0; iRange < cMemRanges; iRange++)
+    {
+        RTGCPHYS    GCPhysStart;
+        RTGCPHYS    GCPhysEnd;
+        bool        fIsMmio;
+        rc = PGMR3PhysGetRange(pVM, iRange, &GCPhysStart, &GCPhysEnd, NULL /* pszDesc */, &fIsMmio);
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": PGMR3PhysGetRange failed for iRange(%u) rc=%Rrc\n", iRange, rc));
+            return rc;
+        }
+
+        uint64_t cbMemRange  = GCPhysEnd - GCPhysStart + 1;
+        uint64_t cbFileRange = fIsMmio ? 0 : cbMemRange;
+
+        Log((DBGFLOG_NAME ": PGMR3PhysGetRange iRange=%u GCPhysStart=%#x GCPhysEnd=%#x cbMemRange=%u\n",
+             iRange, GCPhysStart, GCPhysEnd, cbMemRange));
+
+        rc = Elf64WriteProgHdr(hFile, PT_LOAD, PF_R,
+                               offMemRange,                         /* file offset to contents */
+                               cbFileRange,                         /* size in core file */
+                               cbMemRange,                          /* size in memory */
+                               GCPhysStart);                        /* physical address */
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": Elf64WriteProgHdr failed for memory range(%u) cbFileRange=%u cbMemRange=%u rc=%Rrc\n",
+                    iRange, cbFileRange, cbMemRange, rc));
+            return rc;
+        }
+
+        offMemRange += cbFileRange;
+    }
+
+    /*
+     * Write the Core descriptor note header and data.
+     */
+    Assert(RTFileTell(hFile) == offCoreDescriptor);
+    rc = Elf64WriteNoteHdr(hFile, NT_VBOXCORE, s_pcszCoreVBoxCore, &CoreDescriptor, sizeof(CoreDescriptor));
+    if (RT_FAILURE(rc))
+    {
+        LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr failed for Note '%s' rc=%Rrc\n", s_pcszCoreVBoxCore, rc));
+        return rc;
+    }
+
+    /*
+     * Write the CPU context note headers and data.
+     */
+    Assert(RTFileTell(hFile) == offCpuDumps);
+    for (uint32_t iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+    {
+        PCPUMCTX pCpuCtx = &pVM->aCpus[iCpu].cpum.s.Guest;
+        rc = Elf64WriteNoteHdr(hFile, NT_VBOXCPU, s_pcszCoreVBoxCpu, pCpuCtx, sizeof(CPUMCTX));
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": Elf64WriteNoteHdr failed for vCPU[%u] rc=%Rrc\n", iCpu, rc));
+            return rc;
+        }
+    }
+
+    /*
+     * Write memory ranges.
+     */
+    Assert(RTFileTell(hFile) == offMemory);
+    for (uint16_t iRange = 0; iRange < cMemRanges; iRange++)
+    {
+        RTGCPHYS GCPhysStart;
+        RTGCPHYS GCPhysEnd;
+        bool     fIsMmio;
+        rc = PGMR3PhysGetRange(pVM, iRange, &GCPhysStart, &GCPhysEnd, NULL /* pszDesc */, &fIsMmio);
+        if (RT_FAILURE(rc))
+        {
+            LogRel((DBGFLOG_NAME ": PGMR3PhysGetRange(2) failed for iRange(%u) rc=%Rrc\n", iRange, rc));
+            return rc;
+        }
+
+        if (fIsMmio)
+            continue;
+
+        /*
+         * Write page-by-page of this memory range.
+         *
+         * The read function may fail on MMIO ranges, we write these as zero
+         * pages for now (would be nice to have the VGA bits there though).
+         */
+        uint64_t cbMemRange  = GCPhysEnd - GCPhysStart + 1;
+        uint64_t cPages      = cbMemRange >> PAGE_SHIFT;
+        for (uint64_t iPage = 0; iPage < cPages; iPage++)
+        {
+            uint8_t abPage[PAGE_SIZE];
+            rc = PGMPhysSimpleReadGCPhys(pVM, abPage, GCPhysStart + (iPage << PAGE_SHIFT),  sizeof(abPage));
+            if (RT_FAILURE(rc))
+            {
+                if (rc != VERR_PGM_PHYS_PAGE_RESERVED)
+                    LogRel((DBGFLOG_NAME ": PGMPhysRead failed for iRange=%u iPage=%u. rc=%Rrc. Ignoring...\n", iRange, iPage, rc));
+                RT_ZERO(abPage);
+            }
+
+            rc = RTFileWrite(hFile, abPage, sizeof(abPage), NULL /* all */);
+            if (RT_FAILURE(rc))
+            {
+                LogRel((DBGFLOG_NAME ": RTFileWrite failed. iRange=%u iPage=%u rc=%Rrc\n", iRange, iPage, rc));
+                return rc;
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * EMT Rendezvous worker function for DBGFR3CoreWrite.
+ *
+ * @param   pVM              The VM handle.
+ * @param   pVCpu            The handle of the calling VCPU.
+ * @param   pvData           Opaque data.
+ *
+ * @return VBox status code.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) dbgfR3CoreWriteRendezvous(PVM pVM, PVMCPU pVCpu, void *pvData)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pVCpu, VERR_INVALID_VMCPU_HANDLE);
+    AssertReturn(pvData, VERR_INVALID_POINTER);
+
+    PDBGFCOREDATA pDbgfData = (PDBGFCOREDATA)pvData;
+
+    /*
+     * Create the core file.
+     */
+    uint32_t fFlags = (pDbgfData->fReplaceFile ? RTFILE_O_CREATE_REPLACE : RTFILE_O_CREATE)
+                    | RTFILE_O_WRITE
+                    | RTFILE_O_DENY_ALL
+                    | (0600 << RTFILE_O_CREATE_MODE_SHIFT);
+    RTFILE   hFile;
+    int rc = RTFileOpen(&hFile, pDbgfData->pszFilename, fFlags);
+    if (RT_SUCCESS(rc))
+    {
+        rc = dbgfR3CoreWriteWorker(pVM, hFile);
+        RTFileClose(hFile);
+    }
+    else
+        LogRel((DBGFLOG_NAME ": RTFileOpen failed for '%s' rc=%Rrc\n", pDbgfData->pszFilename, rc));
+    return rc;
+}
+
+
+/**
+ * Write core dump of the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszFilename         The name of the file to which the guest core
+ *                              dump should be written.
+ * @param   fReplaceFile        Whether to replace the file or not.
+ *
+ * @remarks The VM should be suspended before calling this function or DMA may
+ *          interfer with the state.
+ */
+VMMR3DECL(int) DBGFR3CoreWrite(PVM pVM, const char *pszFilename, bool fReplaceFile)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pszFilename, VERR_INVALID_HANDLE);
+
+    /*
+     * Pass the core write request down to EMT rendezvous which makes sure
+     * other EMTs, if any, are not running. IO threads could still be running
+     * but we don't care about them.
+     */
+    DBGFCOREDATA CoreData;
+    RT_ZERO(CoreData);
+    CoreData.pszFilename  = pszFilename;
+    CoreData.fReplaceFile = fReplaceFile;
+
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, dbgfR3CoreWriteRendezvous, &CoreData);
+    if (RT_SUCCESS(rc))
+        LogRel((DBGFLOG_NAME ": Successfully wrote guest core dump '%s'\n", pszFilename));
+    else
+        LogRel((DBGFLOG_NAME ": Failed to write guest core dump '%s'. rc=%Rrc\n", pszFilename, rc));
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFCpu.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFCpu.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFCpu.cpp	(revision 35333)
@@ -0,0 +1,68 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, CPU State Accessors.
+ */
+
+/*
+ * Copyright (C) 2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/cpum.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+
+
+/**
+ * Wrapper around CPUMGetGuestMode.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The current CPU ID.
+ * @param   penmMode            Where to return the mode.
+ */
+static DECLCALLBACK(int) dbgfR3CpuGetMode(PVM pVM, VMCPUID idCpu, CPUMMODE *penmMode)
+{
+    Assert(idCpu == VMMGetCpuId(pVM));
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    *penmMode = CPUMGetGuestMode(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get the current CPU mode.
+ *
+ * @returns The CPU mode on success, CPUMMODE_INVALID on failure.
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ */
+VMMR3DECL(CPUMMODE) DBGFR3CpuGetMode(PVM pVM, VMCPUID idCpu)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, CPUMMODE_INVALID);
+    AssertReturn(idCpu < pVM->cCpus, CPUMMODE_INVALID);
+
+    CPUMMODE enmMode;
+    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3CpuGetMode, 3, pVM, idCpu, &enmMode);
+    if (RT_FAILURE(rc))
+        return CPUMMODE_INVALID;
+    return enmMode;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFDisas.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFDisas.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFDisas.cpp	(revision 35333)
@@ -0,0 +1,667 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Disassembler.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/selm.h>
+#include <VBox/mm.h>
+#include <VBox/pgm.h>
+#include <VBox/cpum.h>
+#include "DBGFInternal.h"
+#include <VBox/dis.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/vm.h>
+#include "internal/pgm.h"
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/alloca.h>
+#include <iprt/ctype.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Structure used when disassembling and instructions in DBGF.
+ * This is used so the reader function can get the stuff it needs.
+ */
+typedef struct
+{
+    /** The core structure. */
+    DISCPUSTATE     Cpu;
+    /** The VM handle. */
+    PVM             pVM;
+    /** The VMCPU handle. */
+    PVMCPU          pVCpu;
+    /** The address space for resolving symbol. */
+    RTDBGAS         hAs;
+    /** Pointer to the first byte in the segment. */
+    RTGCUINTPTR     GCPtrSegBase;
+    /** Pointer to the byte after the end of the segment. (might have wrapped!) */
+    RTGCUINTPTR     GCPtrSegEnd;
+    /** The size of the segment minus 1. */
+    RTGCUINTPTR     cbSegLimit;
+    /** The guest paging mode. */
+    PGMMODE         enmMode;
+    /** Pointer to the current page - R3 Ptr. */
+    void const     *pvPageR3;
+    /** Pointer to the current page - GC Ptr. */
+    RTGCPTR         pvPageGC;
+    /** Pointer to the next instruction (relative to GCPtrSegBase). */
+    RTGCUINTPTR     GCPtrNext;
+    /** The lock information that PGMPhysReleasePageMappingLock needs. */
+    PGMPAGEMAPLOCK  PageMapLock;
+    /** Whether the PageMapLock is valid or not. */
+    bool            fLocked;
+    /** 64 bits mode or not. */
+    bool            f64Bits;
+} DBGFDISASSTATE, *PDBGFDISASSTATE;
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) dbgfR3DisasInstrRead(RTUINTPTR pSrc, uint8_t *pDest, uint32_t size, void *pvUserdata);
+
+
+
+/**
+ * Calls the disassembler with the proper reader functions and such for disa
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle
+ * @param   pVCpu       VMCPU handle
+ * @param   pSelInfo    The selector info.
+ * @param   enmMode     The guest paging mode.
+ * @param   fFlags      DBGF_DISAS_FLAGS_XXX.
+ * @param   GCPtr       The GC pointer (selector offset).
+ * @param   pState      The disas CPU state.
+ */
+static int dbgfR3DisasInstrFirst(PVM pVM, PVMCPU pVCpu, PDBGFSELINFO pSelInfo, PGMMODE enmMode,
+                                 RTGCPTR GCPtr, uint32_t fFlags, PDBGFDISASSTATE pState)
+{
+    pState->GCPtrSegBase    = pSelInfo->GCPtrBase;
+    pState->GCPtrSegEnd     = pSelInfo->cbLimit + 1 + (RTGCUINTPTR)pSelInfo->GCPtrBase;
+    pState->cbSegLimit      = pSelInfo->cbLimit;
+    pState->enmMode         = enmMode;
+    pState->pvPageGC        = 0;
+    pState->pvPageR3        = NULL;
+    pState->hAs             = pSelInfo->fFlags & DBGFSELINFO_FLAGS_HYPER /** @todo Deal more explicitly with RC in DBGFR3Disas*. */
+                            ? DBGF_AS_RC_AND_GC_GLOBAL
+                            : DBGF_AS_GLOBAL;
+    pState->pVM             = pVM;
+    pState->pVCpu           = pVCpu;
+    pState->fLocked         = false;
+    pState->f64Bits         = enmMode >= PGMMODE_AMD64 && pSelInfo->u.Raw.Gen.u1Long;
+
+    DISCPUMODE enmCpuMode;
+    switch (fFlags & DBGF_DISAS_FLAGS_MODE_MASK)
+    {
+        default:
+            AssertFailed();
+        case DBGF_DISAS_FLAGS_DEFAULT_MODE:
+            enmCpuMode   = pState->f64Bits
+                         ? CPUMODE_64BIT
+                         : pSelInfo->u.Raw.Gen.u1DefBig
+                         ? CPUMODE_32BIT
+                         : CPUMODE_16BIT;
+            break;
+        case DBGF_DISAS_FLAGS_16BIT_MODE:
+        case DBGF_DISAS_FLAGS_16BIT_REAL_MODE:
+            enmCpuMode = CPUMODE_16BIT;
+            break;
+        case DBGF_DISAS_FLAGS_32BIT_MODE:
+            enmCpuMode = CPUMODE_32BIT;
+            break;
+        case DBGF_DISAS_FLAGS_64BIT_MODE:
+            enmCpuMode = CPUMODE_64BIT;
+            break;
+    }
+
+    uint32_t cbInstr;
+    int rc = DISCoreOneEx(GCPtr,
+                          enmCpuMode,
+                          dbgfR3DisasInstrRead,
+                          &pState->Cpu,
+                          &pState->Cpu,
+                          &cbInstr);
+    if (RT_SUCCESS(rc))
+    {
+        pState->GCPtrNext = GCPtr + cbInstr;
+        return VINF_SUCCESS;
+    }
+
+    /* cleanup */
+    if (pState->fLocked)
+    {
+        PGMPhysReleasePageMappingLock(pVM, &pState->PageMapLock);
+        pState->fLocked = false;
+    }
+    return rc;
+}
+
+
+#if 0
+/**
+ * Calls the disassembler for disassembling the next instruction.
+ *
+ * @returns VBox status code.
+ * @param   pState      The disas CPU state.
+ */
+static int dbgfR3DisasInstrNext(PDBGFDISASSTATE pState)
+{
+    uint32_t cbInstr;
+    int rc = DISInstr(&pState->Cpu, (void *)pState->GCPtrNext, 0, &cbInstr, NULL);
+    if (RT_SUCCESS(rc))
+    {
+        pState->GCPtrNext = GCPtr + cbInstr;
+        return VINF_SUCCESS;
+    }
+    return rc;
+}
+#endif
+
+
+/**
+ * Done with the disassembler state, free associated resources.
+ *
+ * @param   pState      The disas CPU state ++.
+ */
+static void dbgfR3DisasInstrDone(PDBGFDISASSTATE pState)
+{
+    if (pState->fLocked)
+    {
+        PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
+        pState->fLocked = false;
+    }
+}
+
+
+/**
+ * Instruction reader.
+ *
+ * @returns VBox status code. (Why this is a int32_t and not just an int is also beyond me.)
+ * @param   PtrSrc      Address to read from.
+ *                      In our case this is relative to the selector pointed to by the 2nd user argument of uDisCpu.
+ * @param   pu8Dst      Where to store the bytes.
+ * @param   cbRead      Number of bytes to read.
+ * @param   uDisCpu     Pointer to the disassembler cpu state. (Why this is a VBOXHUINTPTR is beyond me...)
+ *                      In this context it's always pointer to the Core of a DBGFDISASSTATE.
+ */
+static DECLCALLBACK(int) dbgfR3DisasInstrRead(RTUINTPTR PtrSrc, uint8_t *pu8Dst, uint32_t cbRead, void *pvDisCpu)
+{
+    PDBGFDISASSTATE pState = (PDBGFDISASSTATE)pvDisCpu;
+    Assert(cbRead > 0);
+    for (;;)
+    {
+        RTGCUINTPTR GCPtr = PtrSrc + pState->GCPtrSegBase;
+
+        /* Need to update the page translation? */
+        if (    !pState->pvPageR3
+            ||  (GCPtr >> PAGE_SHIFT) != (pState->pvPageGC >> PAGE_SHIFT))
+        {
+            int rc = VINF_SUCCESS;
+
+            /* translate the address */
+            pState->pvPageGC = GCPtr & PAGE_BASE_GC_MASK;
+            if (MMHyperIsInsideArea(pState->pVM, pState->pvPageGC))
+            {
+                pState->pvPageR3 = MMHyperRCToR3(pState->pVM, (RTRCPTR)pState->pvPageGC);
+                if (!pState->pvPageR3)
+                    rc = VERR_INVALID_POINTER;
+            }
+            else
+            {
+                if (pState->fLocked)
+                    PGMPhysReleasePageMappingLock(pState->pVM, &pState->PageMapLock);
+
+                if (pState->enmMode <= PGMMODE_PROTECTED)
+                    rc = PGMPhysGCPhys2CCPtrReadOnly(pState->pVM, pState->pvPageGC, &pState->pvPageR3, &pState->PageMapLock);
+                else
+                    rc = PGMPhysGCPtr2CCPtrReadOnly(pState->pVCpu, pState->pvPageGC, &pState->pvPageR3, &pState->PageMapLock);
+                pState->fLocked = RT_SUCCESS_NP(rc);
+            }
+            if (RT_FAILURE(rc))
+            {
+                pState->pvPageR3 = NULL;
+                return rc;
+            }
+        }
+
+        /* check the segment limit */
+        if (!pState->f64Bits && PtrSrc > pState->cbSegLimit)
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+
+        /* calc how much we can read */
+        uint32_t cb = PAGE_SIZE - (GCPtr & PAGE_OFFSET_MASK);
+        if (!pState->f64Bits)
+        {
+            RTGCUINTPTR cbSeg = pState->GCPtrSegEnd - GCPtr;
+            if (cb > cbSeg && cbSeg)
+                cb = cbSeg;
+        }
+        if (cb > cbRead)
+            cb = cbRead;
+
+        /* read and advance */
+        memcpy(pu8Dst, (char *)pState->pvPageR3 + (GCPtr & PAGE_OFFSET_MASK), cb);
+        cbRead -= cb;
+        if (!cbRead)
+            return VINF_SUCCESS;
+        pu8Dst += cb;
+        PtrSrc += cb;
+    }
+}
+
+
+/**
+ * @copydoc FNDISGETSYMBOL
+ */
+static DECLCALLBACK(int) dbgfR3DisasGetSymbol(PCDISCPUSTATE pCpu, uint32_t u32Sel, RTUINTPTR uAddress, char *pszBuf, size_t cchBuf, RTINTPTR *poff, void *pvUser)
+{
+    PDBGFDISASSTATE pState   = (PDBGFDISASSTATE)pCpu;
+    PCDBGFSELINFO   pSelInfo = (PCDBGFSELINFO)pvUser;
+    DBGFADDRESS     Addr;
+    RTDBGSYMBOL     Sym;
+    RTGCINTPTR      off;
+    int             rc;
+
+    if (   DIS_FMT_SEL_IS_REG(u32Sel)
+        ?  DIS_FMT_SEL_GET_REG(u32Sel) == DIS_SELREG_CS
+        :  pSelInfo->Sel == DIS_FMT_SEL_GET_VALUE(u32Sel))
+    {
+        rc = DBGFR3AddrFromSelInfoOff(pState->pVM, &Addr, pSelInfo, uAddress);
+        if (RT_SUCCESS(rc))
+            rc = DBGFR3AsSymbolByAddr(pState->pVM, pState->hAs, &Addr, &off, &Sym, NULL /*phMod*/);
+    }
+    else
+        rc = VERR_SYMBOL_NOT_FOUND; /** @todo implement this */
+    if (RT_SUCCESS(rc))
+    {
+        size_t cchName = strlen(Sym.szName);
+        if (cchName >= cchBuf)
+            cchName = cchBuf - 1;
+        memcpy(pszBuf, Sym.szName, cchName);
+        pszBuf[cchName] = '\0';
+
+        *poff = off;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Disassembles the one instruction according to the specified flags and
+ * address, internal worker executing on the EMT of the specified virtual CPU.
+ *
+ * @returns VBox status code.
+ * @param       pVM             The VM handle.
+ * @param       pVCpu           The virtual CPU handle.
+ * @param       Sel             The code selector. This used to determine the 32/16 bit ness and
+ *                              calculation of the actual instruction address.
+ * @param       pGCPtr          Pointer to the variable holding the code address
+ *                              relative to the base of Sel.
+ * @param       fFlags          Flags controlling where to start and how to format.
+ *                              A combination of the DBGF_DISAS_FLAGS_* \#defines.
+ * @param       pszOutput       Output buffer.
+ * @param       cbOutput        Size of the output buffer.
+ * @param       pcbInstr        Where to return the size of the instruction.
+ */
+static DECLCALLBACK(int)
+dbgfR3DisasInstrExOnVCpu(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PRTGCPTR pGCPtr, uint32_t fFlags,
+                         char *pszOutput, uint32_t cbOutput, uint32_t *pcbInstr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    RTGCPTR GCPtr = *pGCPtr;
+
+    /*
+     * Get the Sel and GCPtr if fFlags requests that.
+     */
+    PCCPUMCTXCORE  pCtxCore   = NULL;
+    PCPUMSELREGHID pHiddenSel = NULL;
+    int rc;
+    if (fFlags & (DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_CURRENT_HYPER))
+    {
+        if (fFlags & DBGF_DISAS_FLAGS_CURRENT_GUEST)
+            pCtxCore = CPUMGetGuestCtxCore(pVCpu);
+        else
+            pCtxCore = CPUMGetHyperCtxCore(pVCpu);
+        Sel        = pCtxCore->cs;
+        pHiddenSel = (CPUMSELREGHID *)&pCtxCore->csHid;
+        GCPtr      = pCtxCore->rip;
+    }
+
+    /*
+     * Read the selector info - assume no stale selectors and nasty stuff like that.
+     * Since the selector flags in the CPUMCTX structures aren't up to date unless
+     * we recently visited REM, we'll not search for the selector there.
+     */
+    DBGFSELINFO     SelInfo;
+    const PGMMODE   enmMode          = PGMGetGuestMode(pVCpu);
+    bool            fRealModeAddress = false;
+
+    if (    pHiddenSel
+        &&  (   (fFlags & DBGF_DISAS_FLAGS_HID_SEL_REGS_VALID)
+             || CPUMAreHiddenSelRegsValid(pVCpu)))
+    {
+        SelInfo.Sel                     = Sel;
+        SelInfo.SelGate                 = 0;
+        SelInfo.GCPtrBase               = pHiddenSel->u64Base;
+        SelInfo.cbLimit                 = pHiddenSel->u32Limit;
+        SelInfo.fFlags                  = PGMMODE_IS_LONG_MODE(enmMode)
+                                        ? DBGFSELINFO_FLAGS_LONG_MODE
+                                        : enmMode != PGMMODE_REAL && (!pCtxCore || !pCtxCore->eflags.Bits.u1VM)
+                                        ? DBGFSELINFO_FLAGS_PROT_MODE
+                                        : DBGFSELINFO_FLAGS_REAL_MODE;
+
+        SelInfo.u.Raw.au32[0]           = 0;
+        SelInfo.u.Raw.au32[1]           = 0;
+        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
+        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
+        SelInfo.u.Raw.Gen.u1Present     = pHiddenSel->Attr.n.u1Present;
+        SelInfo.u.Raw.Gen.u1Granularity = pHiddenSel->Attr.n.u1Granularity;;
+        SelInfo.u.Raw.Gen.u1DefBig      = pHiddenSel->Attr.n.u1DefBig;
+        SelInfo.u.Raw.Gen.u1Long        = pHiddenSel->Attr.n.u1Long;
+        SelInfo.u.Raw.Gen.u1DescType    = pHiddenSel->Attr.n.u1DescType;
+        SelInfo.u.Raw.Gen.u4Type        = pHiddenSel->Attr.n.u4Type;
+        fRealModeAddress                = !!(SelInfo.fFlags & DBGFSELINFO_FLAGS_REAL_MODE);
+    }
+    else if (Sel == DBGF_SEL_FLAT)
+    {
+        SelInfo.Sel                     = Sel;
+        SelInfo.SelGate                 = 0;
+        SelInfo.GCPtrBase               = 0;
+        SelInfo.cbLimit                 = ~0;
+        SelInfo.fFlags                  = PGMMODE_IS_LONG_MODE(enmMode)
+                                        ? DBGFSELINFO_FLAGS_LONG_MODE
+                                        : enmMode != PGMMODE_REAL
+                                        ? DBGFSELINFO_FLAGS_PROT_MODE
+                                        : DBGFSELINFO_FLAGS_REAL_MODE;
+        SelInfo.u.Raw.au32[0]           = 0;
+        SelInfo.u.Raw.au32[1]           = 0;
+        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
+        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
+
+        if (   (fFlags & DBGF_DISAS_FLAGS_HID_SEL_REGS_VALID)
+            || CPUMAreHiddenSelRegsValid(pVCpu))
+        {   /* Assume the current CS defines the execution mode. */
+            pCtxCore   = CPUMGetGuestCtxCore(pVCpu);
+            pHiddenSel = (CPUMSELREGHID *)&pCtxCore->csHid;
+
+            SelInfo.u.Raw.Gen.u1Present     = pHiddenSel->Attr.n.u1Present;
+            SelInfo.u.Raw.Gen.u1Granularity = pHiddenSel->Attr.n.u1Granularity;;
+            SelInfo.u.Raw.Gen.u1DefBig      = pHiddenSel->Attr.n.u1DefBig;
+            SelInfo.u.Raw.Gen.u1Long        = pHiddenSel->Attr.n.u1Long;
+            SelInfo.u.Raw.Gen.u1DescType    = pHiddenSel->Attr.n.u1DescType;
+            SelInfo.u.Raw.Gen.u4Type        = pHiddenSel->Attr.n.u4Type;
+        }
+        else
+        {
+            SelInfo.u.Raw.Gen.u1Present     = 1;
+            SelInfo.u.Raw.Gen.u1Granularity = 1;
+            SelInfo.u.Raw.Gen.u1DefBig      = 1;
+            SelInfo.u.Raw.Gen.u1DescType    = 1;
+            SelInfo.u.Raw.Gen.u4Type        = X86_SEL_TYPE_EO;
+        }
+    }
+    else if (   !(fFlags & DBGF_DISAS_FLAGS_CURRENT_HYPER)
+             && (   (pCtxCore && pCtxCore->eflags.Bits.u1VM)
+                 || enmMode == PGMMODE_REAL
+                 || (fFlags & DBGF_DISAS_FLAGS_MODE_MASK) == DBGF_DISAS_FLAGS_16BIT_REAL_MODE
+                )
+            )
+    {   /* V86 mode or real mode - real mode addressing */
+        SelInfo.Sel                     = Sel;
+        SelInfo.SelGate                 = 0;
+        SelInfo.GCPtrBase               = Sel * 16;
+        SelInfo.cbLimit                 = ~0;
+        SelInfo.fFlags                  = DBGFSELINFO_FLAGS_REAL_MODE;
+        SelInfo.u.Raw.au32[0]           = 0;
+        SelInfo.u.Raw.au32[1]           = 0;
+        SelInfo.u.Raw.Gen.u16LimitLow   = 0xffff;
+        SelInfo.u.Raw.Gen.u4LimitHigh   = 0xf;
+        SelInfo.u.Raw.Gen.u1Present     = 1;
+        SelInfo.u.Raw.Gen.u1Granularity = 1;
+        SelInfo.u.Raw.Gen.u1DefBig      = 0; /* 16 bits */
+        SelInfo.u.Raw.Gen.u1DescType    = 1;
+        SelInfo.u.Raw.Gen.u4Type        = X86_SEL_TYPE_EO;
+        fRealModeAddress                = true;
+    }
+    else
+    {
+        rc = SELMR3GetSelectorInfo(pVM, pVCpu, Sel, &SelInfo);
+        if (RT_FAILURE(rc))
+        {
+            RTStrPrintf(pszOutput, cbOutput, "Sel=%04x -> %Rrc\n", Sel, rc);
+            return rc;
+        }
+    }
+
+    /*
+     * Disassemble it.
+     */
+    DBGFDISASSTATE State;
+    rc = dbgfR3DisasInstrFirst(pVM, pVCpu, &SelInfo, enmMode, GCPtr, fFlags, &State);
+    if (RT_FAILURE(rc))
+    {
+        RTStrPrintf(pszOutput, cbOutput, "Disas -> %Rrc\n", rc);
+        return rc;
+    }
+
+    /*
+     * Format it.
+     */
+    char szBuf[512];
+    DISFormatYasmEx(&State.Cpu, szBuf, sizeof(szBuf),
+                    DIS_FMT_FLAGS_RELATIVE_BRANCH,
+                    fFlags & DBGF_DISAS_FLAGS_NO_SYMBOLS ? NULL : dbgfR3DisasGetSymbol,
+                    &SelInfo);
+
+    /*
+     * Print it to the user specified buffer.
+     */
+    if (fFlags & DBGF_DISAS_FLAGS_NO_BYTES)
+    {
+        if (fFlags & DBGF_DISAS_FLAGS_NO_ADDRESS)
+            RTStrPrintf(pszOutput, cbOutput, "%s", szBuf);
+        else if (fRealModeAddress)
+            RTStrPrintf(pszOutput, cbOutput, "%04x:%04x  %s", Sel, (unsigned)GCPtr, szBuf);
+        else if (Sel == DBGF_SEL_FLAT)
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                RTStrPrintf(pszOutput, cbOutput, "%RGv  %s", GCPtr, szBuf);
+            else
+                RTStrPrintf(pszOutput, cbOutput, "%08RX32  %s", (uint32_t)GCPtr, szBuf);
+        }
+        else
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                RTStrPrintf(pszOutput, cbOutput, "%04x:%RGv  %s", Sel, GCPtr, szBuf);
+            else
+                RTStrPrintf(pszOutput, cbOutput, "%04x:%08RX32  %s", Sel, (uint32_t)GCPtr, szBuf);
+        }
+    }
+    else
+    {
+        uint32_t cbBits = State.Cpu.opsize;
+        uint8_t *pau8Bits = (uint8_t *)alloca(cbBits);
+        rc = dbgfR3DisasInstrRead(GCPtr, pau8Bits, cbBits, &State);
+        AssertRC(rc);
+        if (fFlags & DBGF_DISAS_FLAGS_NO_ADDRESS)
+            RTStrPrintf(pszOutput, cbOutput, "%.*Rhxs%*s %s",
+                        cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
+                        szBuf);
+        else if (fRealModeAddress)
+            RTStrPrintf(pszOutput, cbOutput, "%04x:%04x %.*Rhxs%*s %s",
+                        Sel, (unsigned)GCPtr,
+                        cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
+                        szBuf);
+        else if (Sel == DBGF_SEL_FLAT)
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                RTStrPrintf(pszOutput, cbOutput, "%RGv %.*Rhxs%*s %s",
+                            GCPtr,
+                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
+                            szBuf);
+            else
+                RTStrPrintf(pszOutput, cbOutput, "%08RX32 %.*Rhxs%*s %s",
+                            (uint32_t)GCPtr,
+                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
+                            szBuf);
+        }
+        else
+        {
+            if (enmMode >= PGMMODE_AMD64)
+                RTStrPrintf(pszOutput, cbOutput, "%04x:%RGv %.*Rhxs%*s %s",
+                            Sel, GCPtr,
+                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
+                            szBuf);
+            else
+                RTStrPrintf(pszOutput, cbOutput, "%04x:%08RX32 %.*Rhxs%*s %s",
+                            Sel, (uint32_t)GCPtr,
+                            cbBits, pau8Bits, cbBits < 8 ? (8 - cbBits) * 3 : 0, "",
+                            szBuf);
+        }
+    }
+
+    if (pcbInstr)
+        *pcbInstr = State.Cpu.opsize;
+
+    dbgfR3DisasInstrDone(&State);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Disassembles the one instruction according to the specified flags and address.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   idCpu           The ID of virtual CPU.
+ * @param   Sel             The code selector. This used to determine the 32/16 bit ness and
+ *                          calculation of the actual instruction address.
+ * @param   GCPtr           The code address relative to the base of Sel.
+ * @param   fFlags          Flags controlling where to start and how to format.
+ *                          A combination of the DBGF_DISAS_FLAGS_* \#defines.
+ * @param   pszOutput       Output buffer.  This will always be properly
+ *                          terminated if @a cbOutput is greater than zero.
+ * @param   cbOutput        Size of the output buffer.
+ * @param   pcbInstr        Where to return the size of the instruction.
+ *
+ * @remarks May have to switch to the EMT of the virtual CPU in order to do
+ *          address conversion.
+ */
+VMMR3DECL(int) DBGFR3DisasInstrEx(PVM pVM, VMCPUID idCpu, RTSEL Sel, RTGCPTR GCPtr, uint32_t fFlags,
+                                  char *pszOutput, uint32_t cbOutput, uint32_t *pcbInstr)
+{
+    AssertReturn(cbOutput > 0, VERR_INVALID_PARAMETER);
+    *pszOutput = '\0';
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(!(fFlags & ~DBGF_DISAS_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn((fFlags & DBGF_DISAS_FLAGS_MODE_MASK) <= DBGF_DISAS_FLAGS_64BIT_MODE, VERR_INVALID_PARAMETER);
+
+    /*
+     * Optimize the common case where we're called on the EMT of idCpu since
+     * we're using this all the time when logging.
+     */
+    int     rc;
+    PVMCPU  pVCpu = VMMGetCpu(pVM);
+    if (    pVCpu
+        &&  pVCpu->idCpu == idCpu)
+        rc = dbgfR3DisasInstrExOnVCpu(pVM, pVCpu, Sel, &GCPtr, fFlags, pszOutput, cbOutput, pcbInstr);
+    else
+        rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3DisasInstrExOnVCpu, 8,
+                             pVM, VMMGetCpuById(pVM, idCpu), Sel, &GCPtr, fFlags, pszOutput, cbOutput, pcbInstr);
+    return rc;
+}
+
+
+/**
+ * Disassembles the current guest context instruction.
+ * All registers and data will be displayed. Addresses will be attempted resolved to symbols.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           VMCPU handle.
+ * @param   pszOutput       Output buffer.  This will always be properly
+ *                          terminated if @a cbOutput is greater than zero.
+ * @param   cbOutput        Size of the output buffer.
+ */
+VMMR3DECL(int) DBGFR3DisasInstrCurrent(PVMCPU pVCpu, char *pszOutput, uint32_t cbOutput)
+{
+    AssertReturn(cbOutput > 0, VERR_INVALID_PARAMETER);
+    *pszOutput = '\0';
+    AssertReturn(pVCpu, VERR_INVALID_CONTEXT);
+    return DBGFR3DisasInstrEx(pVCpu->pVMR3, pVCpu->idCpu, 0, 0,
+                              DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                              pszOutput, cbOutput, NULL);
+}
+
+
+/**
+ * Disassembles the current guest context instruction and writes it to the log.
+ * All registers and data will be displayed. Addresses will be attempted resolved to symbols.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           VMCPU handle.
+ * @param   pszPrefix       Short prefix string to the disassembly string. (optional)
+ */
+VMMR3DECL(int) DBGFR3DisasInstrCurrentLogInternal(PVMCPU pVCpu, const char *pszPrefix)
+{
+    char szBuf[256];
+    szBuf[0] = '\0';
+    int rc = DBGFR3DisasInstrCurrent(pVCpu, &szBuf[0], sizeof(szBuf));
+    if (RT_FAILURE(rc))
+        RTStrPrintf(szBuf, sizeof(szBuf), "DBGFR3DisasInstrCurrentLog failed with rc=%Rrc\n", rc);
+    if (pszPrefix && *pszPrefix)
+        RTLogPrintf("%s-CPU%d: %s\n", pszPrefix, pVCpu->idCpu, szBuf);
+    else
+        RTLogPrintf("%s\n", szBuf);
+    return rc;
+}
+
+
+
+/**
+ * Disassembles the specified guest context instruction and writes it to the log.
+ * Addresses will be attempted resolved to symbols.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pVCpu           The virtual CPU handle, defaults to CPU 0 if NULL.
+ * @param   Sel             The code selector. This used to determine the 32/16 bit-ness and
+ *                          calculation of the actual instruction address.
+ * @param   GCPtr           The code address relative to the base of Sel.
+ */
+VMMR3DECL(int) DBGFR3DisasInstrLogInternal(PVMCPU pVCpu, RTSEL Sel, RTGCPTR GCPtr)
+{
+    char szBuf[256];
+    int rc = DBGFR3DisasInstrEx(pVCpu->pVMR3, pVCpu->idCpu, Sel, GCPtr, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                &szBuf[0], sizeof(szBuf), NULL);
+    if (RT_FAILURE(rc))
+        RTStrPrintf(szBuf, sizeof(szBuf), "DBGFR3DisasInstrLog(, %RTsel, %RGv) failed with rc=%Rrc\n", Sel, GCPtr, rc);
+    RTLogPrintf("%s\n", szBuf);
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFInfo.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFInfo.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFInfo.cpp	(revision 35333)
@@ -0,0 +1,994 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Info.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF_INFO
+#include <VBox/dbgf.h>
+
+#include <VBox/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/semaphore.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(void) dbgfR3InfoLog_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
+static DECLCALLBACK(void) dbgfR3InfoLog_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
+static DECLCALLBACK(void) dbgfR3InfoLogRel_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
+static DECLCALLBACK(void) dbgfR3InfoLogRel_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
+static DECLCALLBACK(void) dbgfR3InfoStdErr_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...);
+static DECLCALLBACK(void) dbgfR3InfoStdErr_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args);
+static DECLCALLBACK(void) dbgfR3InfoHelp(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/** Logger output. */
+static const DBGFINFOHLP g_dbgfR3InfoLogHlp =
+{
+    dbgfR3InfoLog_Printf,
+    dbgfR3InfoLog_PrintfV
+};
+
+/** Release logger output. */
+static const DBGFINFOHLP g_dbgfR3InfoLogRelHlp =
+{
+    dbgfR3InfoLogRel_Printf,
+    dbgfR3InfoLogRel_PrintfV
+};
+
+/** Standard error output. */
+static const DBGFINFOHLP g_dbgfR3InfoStdErrHlp =
+{
+    dbgfR3InfoStdErr_Printf,
+    dbgfR3InfoStdErr_PrintfV
+};
+
+
+/**
+ * Initialize the info handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM handle.
+ */
+int dbgfR3InfoInit(PVM pVM)
+{
+    /*
+     * Make sure we already didn't initialized in the lazy manner.
+     */
+    if (RTCritSectIsInitialized(&pVM->dbgf.s.InfoCritSect))
+        return VINF_SUCCESS;
+
+    /*
+     * Initialize the crit sect.
+     */
+    int rc = RTCritSectInit(&pVM->dbgf.s.InfoCritSect);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register the 'info help' item.
+     */
+    rc = DBGFR3InfoRegisterInternal(pVM, "help", "List of info items.", dbgfR3InfoHelp);
+    AssertRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminate the info handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM handle.
+ */
+int dbgfR3InfoTerm(PVM pVM)
+{
+    /*
+     * Delete the crit sect.
+     */
+    int rc = RTCritSectDelete(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/** Logger output.
+ * @copydoc DBGFINFOHLP::pfnPrintf */
+static DECLCALLBACK(void) dbgfR3InfoLog_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    RTLogPrintfV(pszFormat, args);
+    va_end(args);
+}
+
+/** Logger output.
+ * @copydoc DBGFINFOHLP::pfnPrintfV */
+static DECLCALLBACK(void) dbgfR3InfoLog_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    RTLogPrintfV(pszFormat, args);
+}
+
+
+/**
+ * Gets the logger info helper.
+ * The returned info helper will unconditionally write all output to the log.
+ *
+ * @returns Pointer to the logger info helper.
+ */
+VMMR3DECL(PCDBGFINFOHLP) DBGFR3InfoLogHlp(void)
+{
+    return &g_dbgfR3InfoLogHlp;
+}
+
+
+/** Release logger output.
+ * @copydoc DBGFINFOHLP::pfnPrintf */
+static DECLCALLBACK(void) dbgfR3InfoLogRel_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    RTLogRelPrintfV(pszFormat, args);
+    va_end(args);
+}
+
+/** Release logger output.
+ * @copydoc DBGFINFOHLP::pfnPrintfV */
+static DECLCALLBACK(void) dbgfR3InfoLogRel_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    RTLogRelPrintfV(pszFormat, args);
+}
+
+
+/** Standard error output.
+ * @copydoc DBGFINFOHLP::pfnPrintf */
+static DECLCALLBACK(void) dbgfR3InfoStdErr_Printf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    RTStrmPrintfV(g_pStdErr, pszFormat, args);
+    va_end(args);
+}
+
+/** Standard error output.
+ * @copydoc DBGFINFOHLP::pfnPrintfV */
+static DECLCALLBACK(void) dbgfR3InfoStdErr_PrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    RTStrmPrintfV(g_pStdErr, pszFormat, args);
+}
+
+
+/**
+ * Gets the release logger info helper.
+ * The returned info helper will unconditionally write all output to the release log.
+ *
+ * @returns Pointer to the release logger info helper.
+ */
+VMMR3DECL(PCDBGFINFOHLP) DBGFR3InfoLogRelHlp(void)
+{
+    return &g_dbgfR3InfoLogRelHlp;
+}
+
+
+/**
+ * Handle registration worker.
+ * This allocates the structure, initializes the common fields and inserts into the list.
+ * Upon successful return the we're inside the crit sect and the caller must leave it.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   fFlags      The flags.
+ * @param   ppInfo      Where to store the created
+ */
+static int dbgfR3InfoRegister(PVM pVM, const char *pszName, const char *pszDesc, uint32_t fFlags, PDBGFINFO *ppInfo)
+{
+    /*
+     * Validate.
+     */
+    AssertPtrReturn(pszName, VERR_INVALID_POINTER);
+    AssertReturn(*pszName, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertMsgReturn(!(fFlags & ~(DBGFINFO_FLAGS_RUN_ON_EMT)), ("fFlags=%#x\n", fFlags), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate and initialize.
+     */
+    int rc;
+    size_t cchName = strlen(pszName) + 1;
+    PDBGFINFO pInfo = (PDBGFINFO)MMR3HeapAlloc(pVM, MM_TAG_DBGF_INFO, RT_OFFSETOF(DBGFINFO, szName[cchName]));
+    if (pInfo)
+    {
+        pInfo->enmType = DBGFINFOTYPE_INVALID;
+        pInfo->fFlags = fFlags;
+        pInfo->pszDesc = pszDesc;
+        pInfo->cchName = cchName - 1;
+        memcpy(pInfo->szName, pszName, cchName);
+
+        /* lazy init */
+        rc = VINF_SUCCESS;
+        if (!RTCritSectIsInitialized(&pVM->dbgf.s.InfoCritSect))
+            rc = dbgfR3InfoInit(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Insert in alphabetical order.
+             */
+            rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+            AssertRC(rc);
+            PDBGFINFO pPrev = NULL;
+            PDBGFINFO pCur;
+            for (pCur = pVM->dbgf.s.pInfoFirst; pCur; pPrev = pCur, pCur = pCur->pNext)
+                if (strcmp(pszName, pCur->szName) < 0)
+                    break;
+            pInfo->pNext = pCur;
+            if (pPrev)
+                pPrev->pNext = pInfo;
+            else
+                pVM->dbgf.s.pInfoFirst = pInfo;
+
+            *ppInfo = pInfo;
+            return VINF_SUCCESS;
+        }
+        MMR3HeapFree(pInfo);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pDevIns     The device instance owning the info.
+ */
+VMMR3DECL(int) DBGFR3InfoRegisterDevice(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDEV pfnHandler, PPDMDEVINS pDevIns)
+{
+    LogFlow(("DBGFR3InfoRegisterDevice: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDevIns=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDevIns));
+
+    /*
+     * Validate the specific stuff.
+     */
+    if (!pfnHandler)
+    {
+        AssertMsgFailed(("No handler\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!pDevIns)
+    {
+        AssertMsgFailed(("No pDevIns\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_DEV;
+        pInfo->u.Dev.pfnHandler = pfnHandler;
+        pInfo->u.Dev.pDevIns = pDevIns;
+        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pDrvIns     The driver instance owning the info.
+ */
+VMMR3DECL(int) DBGFR3InfoRegisterDriver(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDRV pfnHandler, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("DBGFR3InfoRegisterDriver: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pDrvIns=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pDrvIns));
+
+    /*
+     * Validate the specific stuff.
+     */
+    if (!pfnHandler)
+    {
+        AssertMsgFailed(("No handler\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!pDrvIns)
+    {
+        AssertMsgFailed(("No pDrvIns\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_DRV;
+        pInfo->u.Drv.pfnHandler = pfnHandler;
+        pInfo->u.Drv.pDrvIns = pDrvIns;
+        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by an internal component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ */
+VMMR3DECL(int) DBGFR3InfoRegisterInternal(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERINT pfnHandler)
+{
+    return DBGFR3InfoRegisterInternalEx(pVM, pszName, pszDesc, pfnHandler, 0);
+}
+
+
+/**
+ * Register a info handler owned by an internal component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   fFlags      Flags, see the DBGFINFO_FLAGS_*.
+ */
+VMMR3DECL(int) DBGFR3InfoRegisterInternalEx(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLERINT pfnHandler, uint32_t fFlags)
+{
+    LogFlow(("DBGFR3InfoRegisterInternal: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p fFlags=%x\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, fFlags));
+
+    /*
+     * Validate the specific stuff.
+     */
+    if (!pfnHandler)
+    {
+        AssertMsgFailed(("No handler\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, fFlags, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_INT;
+        pInfo->u.Int.pfnHandler = pfnHandler;
+        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Register a info handler owned by an external component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pszName     The identifier of the info.
+ * @param   pszDesc     The description of the info and any arguments the handler may take.
+ * @param   pfnHandler  The handler function to be called to display the info.
+ * @param   pvUser      User argument to be passed to the handler.
+ */
+VMMR3DECL(int) DBGFR3InfoRegisterExternal(PVM pVM, const char *pszName, const char *pszDesc, PFNDBGFHANDLEREXT pfnHandler, void *pvUser)
+{
+    LogFlow(("DBGFR3InfoRegisterExternal: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p pvUser=%p\n",
+             pszName, pszName, pszDesc, pszDesc, pfnHandler, pvUser));
+
+    /*
+     * Validate the specific stuff.
+     */
+    if (!pfnHandler)
+    {
+        AssertMsgFailed(("No handler\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Register
+     */
+    PDBGFINFO pInfo;
+    int rc = dbgfR3InfoRegister(pVM, pszName, pszDesc, 0, &pInfo);
+    if (RT_SUCCESS(rc))
+    {
+        pInfo->enmType = DBGFINFOTYPE_EXT;
+        pInfo->u.Ext.pfnHandler = pfnHandler;
+        pInfo->u.Ext.pvUser = pvUser;
+        RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister one(/all) info handler(s) owned by a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM Handle.
+ * @param   pDevIns     Device instance.
+ * @param   pszName     The identifier of the info. If NULL all owned by the device.
+ */
+VMMR3DECL(int) DBGFR3InfoDeregisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterDevice: pDevIns=%p pszName=%p:{%s}\n", pDevIns, pszName, pszName));
+
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+    {
+        AssertMsgFailed(("!pDevIns\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+    size_t cchName = pszName ? strlen(pszName) : 0;
+
+    /*
+     * Enumerate the info handlers and free the requested entries.
+     */
+    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+    rc = VERR_FILE_NOT_FOUND;
+    PDBGFINFO pPrev = NULL;
+    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
+    if (pszName)
+    {
+        /*
+         * Free a specific one.
+         */
+        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+            if (    pInfo->enmType == DBGFINFOTYPE_DEV
+                &&  pInfo->u.Dev.pDevIns == pDevIns
+                &&  pInfo->cchName == cchName
+                &&  !strcmp(pInfo->szName, pszName))
+            {
+                if (pPrev)
+                    pPrev->pNext = pInfo->pNext;
+                else
+                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
+                MMR3HeapFree(pInfo);
+                rc = VINF_SUCCESS;
+                break;
+            }
+    }
+    else
+    {
+        /*
+         * Free all owned by the driver.
+         */
+        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+            if (    pInfo->enmType == DBGFINFOTYPE_DEV
+                &&  pInfo->u.Dev.pDevIns == pDevIns)
+            {
+                if (pPrev)
+                    pPrev->pNext = pInfo->pNext;
+                else
+                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
+                MMR3HeapFree(pInfo);
+                pInfo = pPrev;
+            }
+        rc = VINF_SUCCESS;
+    }
+    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc2);
+    AssertRC(rc);
+    LogFlow(("DBGFR3InfoDeregisterDevice: returns %Rrc\n", rc));
+    return rc;
+}
+
+/**
+ * Deregister one(/all) info handler(s) owned by a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM Handle.
+ * @param   pDrvIns     Driver instance.
+ * @param   pszName     The identifier of the info. If NULL all owned by the driver.
+ */
+VMMR3DECL(int) DBGFR3InfoDeregisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterDriver: pDrvIns=%p pszName=%p:{%s}\n", pDrvIns, pszName, pszName));
+
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+    {
+        AssertMsgFailed(("!pDrvIns\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+    size_t cchName = pszName ? strlen(pszName) : 0;
+
+    /*
+     * Enumerate the info handlers and free the requested entries.
+     */
+    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+    rc = VERR_FILE_NOT_FOUND;
+    PDBGFINFO pPrev = NULL;
+    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
+    if (pszName)
+    {
+        /*
+         * Free a specific one.
+         */
+        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+            if (    pInfo->enmType == DBGFINFOTYPE_DRV
+                &&  pInfo->u.Drv.pDrvIns == pDrvIns
+                &&  pInfo->cchName == cchName
+                &&  !strcmp(pInfo->szName, pszName))
+            {
+                if (pPrev)
+                    pPrev->pNext = pInfo->pNext;
+                else
+                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
+                MMR3HeapFree(pInfo);
+                rc = VINF_SUCCESS;
+                break;
+            }
+    }
+    else
+    {
+        /*
+         * Free all owned by the driver.
+         */
+        for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+            if (    pInfo->enmType == DBGFINFOTYPE_DRV
+                &&  pInfo->u.Drv.pDrvIns == pDrvIns)
+            {
+                if (pPrev)
+                    pPrev->pNext = pInfo->pNext;
+                else
+                    pVM->dbgf.s.pInfoFirst = pInfo->pNext;
+                MMR3HeapFree(pInfo);
+                pInfo = pPrev;
+            }
+        rc = VINF_SUCCESS;
+    }
+    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc2);
+    AssertRC(rc);
+    LogFlow(("DBGFR3InfoDeregisterDriver: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Internal deregistration helper.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM Handle.
+ * @param   pszName     The identifier of the info.
+ * @param   enmType     The info owner type.
+ */
+static int dbgfR3InfoDeregister(PVM pVM, const char *pszName, DBGFINFOTYPE enmType)
+{
+    /*
+     * Validate input.
+     */
+    if (!pszName)
+    {
+        AssertMsgFailed(("!pszName\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Find the info handler.
+     */
+    size_t cchName = strlen(pszName);
+    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+    rc = VERR_FILE_NOT_FOUND;
+    PDBGFINFO pPrev = NULL;
+    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
+    for (; pInfo; pPrev = pInfo, pInfo = pInfo->pNext)
+        if (    pInfo->cchName == cchName
+            &&  !strcmp(pInfo->szName, pszName)
+            &&  pInfo->enmType == enmType)
+        {
+            if (pPrev)
+                pPrev->pNext = pInfo->pNext;
+            else
+                pVM->dbgf.s.pInfoFirst = pInfo->pNext;
+            MMR3HeapFree(pInfo);
+            rc = VINF_SUCCESS;
+            break;
+        }
+    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc2);
+    AssertRC(rc);
+    LogFlow(("dbgfR3InfoDeregister: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Deregister a info handler owned by an internal component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM Handle.
+ * @param   pszName     The identifier of the info. If NULL all owned by the device.
+ */
+VMMR3DECL(int) DBGFR3InfoDeregisterInternal(PVM pVM, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterInternal: pszName=%p:{%s}\n", pszName, pszName));
+    return dbgfR3InfoDeregister(pVM, pszName, DBGFINFOTYPE_INT);
+}
+
+
+/**
+ * Deregister a info handler owned by an external component.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM Handle.
+ * @param   pszName     The identifier of the info. If NULL all owned by the device.
+ */
+VMMR3DECL(int) DBGFR3InfoDeregisterExternal(PVM pVM, const char *pszName)
+{
+    LogFlow(("DBGFR3InfoDeregisterExternal: pszName=%p:{%s}\n", pszName, pszName));
+    return dbgfR3InfoDeregister(pVM, pszName, DBGFINFOTYPE_EXT);
+}
+
+
+/**
+ * Display a piece of info writing to the supplied handler.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pszName     The identifier of the info to display.
+ * @param   pszArgs     Arguments to the info handler.
+ * @param   pHlp        The output helper functions. If NULL the logger will be used.
+ */
+VMMR3DECL(int) DBGFR3Info(PVM pVM, const char *pszName, const char *pszArgs, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Validate input.
+     */
+    if (!pszName)
+    {
+        AssertMsgFailed(("!pszName\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (pHlp)
+    {
+        if (    !pHlp->pfnPrintf
+            ||  !pHlp->pfnPrintfV)
+        {
+            AssertMsgFailed(("A pHlp member is missing!\n"));
+            return VERR_INVALID_PARAMETER;
+        }
+    }
+    else
+        pHlp = &g_dbgfR3InfoLogHlp;
+
+    /*
+     * Find the info handler.
+     */
+    size_t cchName = strlen(pszName);
+    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+    PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst;
+    for (; pInfo; pInfo = pInfo->pNext)
+        if (    pInfo->cchName == cchName
+            &&  !memcmp(pInfo->szName, pszName, cchName))
+            break;
+    if (pInfo)
+    {
+        /*
+         * Found it.
+         *      Make a copy of it on the stack so we can leave the crit sect.
+         *      Switch on the type and invoke the handler.
+         */
+        DBGFINFO Info = *pInfo;
+        rc = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+        AssertRC(rc);
+        rc = VINF_SUCCESS;
+        switch (Info.enmType)
+        {
+            case DBGFINFOTYPE_DEV:
+                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Dev.pfnHandler, 3, Info.u.Dev.pDevIns, pHlp, pszArgs);
+                else
+                    Info.u.Dev.pfnHandler(Info.u.Dev.pDevIns, pHlp, pszArgs);
+                break;
+
+            case DBGFINFOTYPE_DRV:
+                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Drv.pfnHandler, 3, Info.u.Drv.pDrvIns, pHlp, pszArgs);
+                else
+                    Info.u.Drv.pfnHandler(Info.u.Drv.pDrvIns, pHlp, pszArgs);
+                break;
+
+            case DBGFINFOTYPE_INT:
+                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Int.pfnHandler, 3, pVM, pHlp, pszArgs);
+                else
+                    Info.u.Int.pfnHandler(pVM, pHlp, pszArgs);
+                break;
+
+            case DBGFINFOTYPE_EXT:
+                if (Info.fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                    rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)Info.u.Ext.pfnHandler, 3, Info.u.Ext.pvUser, pHlp, pszArgs);
+                else
+                    Info.u.Ext.pfnHandler(Info.u.Ext.pvUser, pHlp, pszArgs);
+                break;
+
+            default:
+                AssertMsgFailed(("Invalid info type enmType=%d\n", Info.enmType));
+                rc = VERR_INTERNAL_ERROR;
+                break;
+        }
+    }
+    else
+    {
+        rc = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+        AssertRC(rc);
+        rc = VERR_FILE_NOT_FOUND;
+    }
+    return rc;
+}
+
+
+/**
+ * Wrapper for DBGFR3Info that outputs to the release log.
+ *
+ * @returns See DBGFR3Info.
+ * @param   pVM                 The VM handle.
+ * @param   pszName             See DBGFR3Info.
+ * @param   pszArgs             See DBGFR3Info.
+ */
+VMMR3DECL(int) DBGFR3InfoLogRel(PVM pVM, const char *pszName, const char *pszArgs)
+{
+    return DBGFR3Info(pVM, pszName, pszArgs, &g_dbgfR3InfoLogRelHlp);
+}
+
+
+/**
+ * Wrapper for DBGFR3Info that outputs to standard error.
+ *
+ * @returns See DBGFR3Info.
+ * @param   pVM                 The VM handle.
+ * @param   pszName             See DBGFR3Info.
+ * @param   pszArgs             See DBGFR3Info.
+ */
+VMMR3DECL(int) DBGFR3InfoStdErr(PVM pVM, const char *pszName, const char *pszArgs)
+{
+    return DBGFR3Info(pVM, pszName, pszArgs, &g_dbgfR3InfoStdErrHlp);
+}
+
+
+/**
+ * Display several info items.
+ *
+ * This is intended used by the fatal error dump only.
+ *
+ * @returns
+ * @param   pVM             The VM handle.
+ * @param   pszIncludePat   Simple string pattern of info items to include.
+ * @param   pszExcludePat   Simple string pattern of info items to exclude.
+ * @param   pszSepFmt       Item separator format string.  The item name will be
+ *                          given as parameter.
+ * @param   pHlp            The output helper functions.  If NULL the logger
+ *                          will be used.
+ *
+ * @threads EMT
+ */
+VMMR3DECL(int) DBGFR3InfoMulti(PVM pVM, const char *pszIncludePat, const char *pszExcludePat, const char *pszSepFmt,
+                               PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pszIncludePat, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszExcludePat, VERR_INVALID_POINTER);
+    if (pHlp)
+    {
+        AssertPtrReturn(pHlp->pfnPrintf, VERR_INVALID_POINTER);
+        AssertPtrReturn(pHlp->pfnPrintfV, VERR_INVALID_POINTER);
+    }
+    else
+        pHlp = &g_dbgfR3InfoLogHlp;
+
+    size_t const cchIncludePat = strlen(pszIncludePat);
+    size_t const cchExcludePat = strlen(pszExcludePat);
+    const char *pszArgs = "";
+
+    /*
+     * Enumerate the info handlers and call the ones matching.
+     * Note! We won't leave the critical section here...
+     */
+    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+    rc = VWRN_NOT_FOUND;
+    for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
+    {
+        if (   RTStrSimplePatternMultiMatch(pszIncludePat, cchIncludePat, pInfo->szName, pInfo->cchName, NULL)
+            && !RTStrSimplePatternMultiMatch(pszExcludePat, cchExcludePat, pInfo->szName, pInfo->cchName, NULL))
+        {
+            pHlp->pfnPrintf(pHlp, pszSepFmt, pInfo->szName);
+            rc = VINF_SUCCESS;
+            switch (pInfo->enmType)
+            {
+                case DBGFINFOTYPE_DEV:
+                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Dev.pfnHandler, 3, pInfo->u.Dev.pDevIns, pHlp, pszArgs);
+                    else
+                        pInfo->u.Dev.pfnHandler(pInfo->u.Dev.pDevIns, pHlp, pszArgs);
+                    break;
+
+                case DBGFINFOTYPE_DRV:
+                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Drv.pfnHandler, 3, pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
+                    else
+                        pInfo->u.Drv.pfnHandler(pInfo->u.Drv.pDrvIns, pHlp, pszArgs);
+                    break;
+
+                case DBGFINFOTYPE_INT:
+                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Int.pfnHandler, 3, pVM, pHlp, pszArgs);
+                    else
+                        pInfo->u.Int.pfnHandler(pVM, pHlp, pszArgs);
+                    break;
+
+                case DBGFINFOTYPE_EXT:
+                    if (pInfo->fFlags & DBGFINFO_FLAGS_RUN_ON_EMT)
+                        rc = VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)pInfo->u.Ext.pfnHandler, 3, pInfo->u.Ext.pvUser, pHlp, pszArgs);
+                    else
+                        pInfo->u.Ext.pfnHandler(pInfo->u.Ext.pvUser, pHlp, pszArgs);
+                    break;
+
+                default:
+                    AssertMsgFailed(("Invalid info type enmType=%d\n", pInfo->enmType));
+                    rc = VERR_INTERNAL_ERROR;
+                    break;
+            }
+        }
+    }
+    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc2);
+
+    return rc;
+}
+
+
+/**
+ * Enumerate all the register info handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pfnCallback     Pointer to callback function.
+ * @param   pvUser          User argument to pass to the callback.
+ */
+VMMR3DECL(int) DBGFR3InfoEnum(PVM pVM, PFNDBGFINFOENUM pfnCallback, void *pvUser)
+{
+    LogFlow(("DBGFR3InfoLog: pfnCallback=%p pvUser=%p\n", pfnCallback, pvUser));
+
+    /*
+     * Validate input.
+     */
+    if (!pfnCallback)
+    {
+        AssertMsgFailed(("!pfnCallback\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Enter and enumerate.
+     */
+    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+
+    rc = VINF_SUCCESS;
+    for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; RT_SUCCESS(rc) && pInfo; pInfo = pInfo->pNext)
+        rc = pfnCallback(pVM, pInfo->szName, pInfo->pszDesc, pvUser);
+
+    /*
+     * Leave and exit.
+     */
+    int rc2 = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc2);
+
+    LogFlow(("DBGFR3InfoLog: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Info handler, internal version.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pHlp        Callback functions for doing output.
+ * @param   pszArgs     Argument string. Optional and specific to the handler.
+ */
+static DECLCALLBACK(void) dbgfR3InfoHelp(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    LogFlow(("dbgfR3InfoHelp: pszArgs=%s\n", pszArgs));
+
+    /*
+     * Enter and enumerate.
+     */
+    int rc = RTCritSectEnter(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+
+    if (pszArgs && *pszArgs)
+    {
+        for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
+        {
+            const char *psz = strstr(pszArgs, pInfo->szName);
+            if (    psz
+                &&  (   psz == pszArgs
+                     || RT_C_IS_SPACE(psz[-1]))
+                &&  (   !psz[pInfo->cchName]
+                     || RT_C_IS_SPACE(psz[pInfo->cchName])))
+                pHlp->pfnPrintf(pHlp, "%-16s  %s\n",
+                                pInfo->szName, pInfo->pszDesc);
+        }
+    }
+    else
+    {
+        for (PDBGFINFO pInfo = pVM->dbgf.s.pInfoFirst; pInfo; pInfo = pInfo->pNext)
+            pHlp->pfnPrintf(pHlp, "%-16s  %s\n",
+                            pInfo->szName, pInfo->pszDesc);
+    }
+
+    /*
+     * Leave and exit.
+     */
+    rc = RTCritSectLeave(&pVM->dbgf.s.InfoCritSect);
+    AssertRC(rc);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFLog.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFLog.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFLog.cpp	(revision 35333)
@@ -0,0 +1,132 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Log Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#include <VBox/vmapi.h>
+#include <VBox/vmm.h>
+#include <VBox/dbgf.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) dbgfR3LogModifyGroups(PVM pVM, const char *pszGroupSettings);
+static DECLCALLBACK(int) dbgfR3LogModifyFlags(PVM pVM, const char *pszFlagSettings);
+static DECLCALLBACK(int) dbgfR3LogModifyDestinations(PVM pVM, const char *pszDestSettings);
+
+
+/**
+ * Changes the logger group settings.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszGroupSettings    The group settings string. (VBOX_LOG)
+ */
+VMMR3DECL(int) DBGFR3LogModifyGroups(PVM pVM, const char *pszGroupSettings)
+{
+    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszGroupSettings, VERR_INVALID_POINTER);
+
+    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyGroups, 2, pVM, pszGroupSettings);
+}
+
+
+/**
+ * EMT worker for DBGFR3LogModifyGroups.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszGroupSettings    The group settings string. (VBOX_LOG)
+ */
+static DECLCALLBACK(int) dbgfR3LogModifyGroups(PVM pVM, const char *pszGroupSettings)
+{
+    int rc = RTLogGroupSettings(NULL, pszGroupSettings);
+    if (RT_SUCCESS(rc))
+        rc = VMMR3UpdateLoggers(pVM);
+    return rc;
+}
+
+
+/**
+ * Changes the logger flag settings.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszFlagSettings     The group settings string. (VBOX_LOG_FLAGS)
+ */
+VMMR3DECL(int) DBGFR3LogModifyFlags(PVM pVM, const char *pszFlagSettings)
+{
+    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszFlagSettings, VERR_INVALID_POINTER);
+
+    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyFlags, 2, pVM, pszFlagSettings);
+}
+
+
+/**
+ * EMT worker for DBGFR3LogModifyFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszFlagSettings     The group settings string. (VBOX_LOG_FLAGS)
+ */
+static DECLCALLBACK(int) dbgfR3LogModifyFlags(PVM pVM, const char *pszFlagSettings)
+{
+    int rc = RTLogFlags(NULL, pszFlagSettings);
+    if (RT_SUCCESS(rc))
+        rc = VMMR3UpdateLoggers(pVM);
+    return rc;
+}
+
+
+/**
+ * Changes the logger destination settings.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszDestSettings     The destination settings string. (VBOX_LOG_DEST)
+ */
+VMMR3DECL(int) DBGFR3LogModifyDestinations(PVM pVM, const char *pszDestSettings)
+{
+    AssertReturn(VALID_PTR(pVM), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(pszDestSettings), VERR_INVALID_POINTER);
+
+    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3LogModifyDestinations, 2, pVM, pszDestSettings);
+}
+
+
+/**
+ * EMT worker for DBGFR3LogModifyFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszDestSettings     The destination settings string. (VBOX_LOG_DEST)
+ */
+static DECLCALLBACK(int) dbgfR3LogModifyDestinations(PVM pVM, const char *pszDestSettings)
+{
+    int rc = RTLogDestinations(NULL, pszDestSettings);
+    if (RT_SUCCESS(rc))
+        rc = VMMR3UpdateLoggers(pVM);
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFMem.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFMem.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFMem.cpp	(revision 35333)
@@ -0,0 +1,662 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Memory Methods.
+ */
+
+/*
+ * Copyright (C) 2007-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/selm.h>
+#include <VBox/hwaccm.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/mm.h>
+
+
+
+/**
+ * Scan guest memory for an exact byte string.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   idCpu       The ID of the CPU context to search in.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   puAlign     The alignment restriction imposed on the search result.
+ * @param   pcbRange    The number of bytes to scan. Passed as a pointer because
+ *                      it may be 64-bit.
+ * @param   pabNeedle   What to search for - exact search.
+ * @param   cbNeedle    Size of the search byte string.
+ * @param   pHitAddress Where to put the address of the first hit.
+ */
+static DECLCALLBACK(int) dbgfR3MemScan(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, PCRTGCUINTPTR pcbRange, RTGCUINTPTR *puAlign,
+                                       const uint8_t *pabNeedle, size_t cbNeedle, PDBGFADDRESS pHitAddress)
+{
+    Assert(idCpu == VMMGetCpuId(pVM));
+
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    RTGCUINTPTR cbRange = *pcbRange;
+    if (!DBGFR3AddrIsValid(pVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pHitAddress))
+        return VERR_INVALID_POINTER;
+    if (DBGFADDRESS_IS_HMA(pAddress))
+        return VERR_INVALID_POINTER;
+
+    /*
+     * Select DBGF worker by addressing mode.
+     */
+    int     rc;
+    PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
+    PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+    if (    enmMode == PGMMODE_REAL
+        ||  enmMode == PGMMODE_PROTECTED
+        ||  DBGFADDRESS_IS_PHYS(pAddress)
+        )
+    {
+        RTGCPHYS GCPhysAlign = *puAlign;
+        if (GCPhysAlign != *puAlign)
+            return VERR_OUT_OF_RANGE;
+        RTGCPHYS PhysHit;
+        rc = PGMR3DbgScanPhysical(pVM, pAddress->FlatPtr, cbRange, GCPhysAlign, pabNeedle, cbNeedle, &PhysHit);
+        if (RT_SUCCESS(rc))
+            DBGFR3AddrFromPhys(pVM, pHitAddress, PhysHit);
+    }
+    else
+    {
+#if GC_ARCH_BITS > 32
+        if (    (   pAddress->FlatPtr >= _4G
+                 || pAddress->FlatPtr + cbRange > _4G)
+            &&  enmMode != PGMMODE_AMD64
+            &&  enmMode != PGMMODE_AMD64_NX)
+            return VERR_DBGF_MEM_NOT_FOUND;
+#endif
+        RTGCUINTPTR GCPtrHit;
+        rc = PGMR3DbgScanVirtual(pVM, pVCpu, pAddress->FlatPtr, cbRange, *puAlign, pabNeedle, cbNeedle, &GCPtrHit);
+        if (RT_SUCCESS(rc))
+            DBGFR3AddrFromFlat(pVM, pHitAddress, GCPtrHit);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Scan guest memory for an exact byte string.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS and *pGCPtrHit on success.
+ * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
+ * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
+ * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
+ *
+ * @param   pVM         The VM handle.
+ * @param   idCpu       The ID of the CPU context to search in.
+ * @param   pAddress    Where to store the mixed address.
+ * @param   cbRange     The number of bytes to scan.
+ * @param   uAlign      The alignment restriction imposed on the result.
+ *                      Usually set to 1.
+ * @param   pvNeedle    What to search for - exact search.
+ * @param   cbNeedle    Size of the search byte string.
+ * @param   pHitAddress Where to put the address of the first hit.
+ *
+ * @thread  Any thread.
+ */
+VMMR3DECL(int) DBGFR3MemScan(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, RTGCUINTPTR cbRange, RTGCUINTPTR uAlign,
+                             const void *pvNeedle, size_t cbNeedle, PDBGFADDRESS pHitAddress)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3MemScan, 8,
+                           pVM, idCpu, pAddress, &cbRange, &uAlign, pvNeedle, cbNeedle, pHitAddress);
+
+}
+
+
+/**
+ * Read guest memory.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pAddress        Where to start reading.
+ * @param   pvBuf           Where to store the data we've read.
+ * @param   cbRead          The number of bytes to read.
+ */
+static DECLCALLBACK(int) dbgfR3MemRead(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void *pvBuf, size_t cbRead)
+{
+    Assert(idCpu == VMMGetCpuId(pVM));
+
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    if (!DBGFR3AddrIsValid(pVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pvBuf))
+        return VERR_INVALID_POINTER;
+
+    /*
+     * HMA is special
+     */
+    int rc;
+    if (DBGFADDRESS_IS_HMA(pAddress))
+    {
+        if (DBGFADDRESS_IS_PHYS(pAddress))
+            rc = VERR_INVALID_POINTER;
+        else
+            rc = MMR3HyperReadGCVirt(pVM, pvBuf, pAddress->FlatPtr, cbRead);
+    }
+    else
+    {
+        /*
+         * Select DBGF worker by addressing mode.
+         */
+        PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
+        PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+        if (    enmMode == PGMMODE_REAL
+            ||  enmMode == PGMMODE_PROTECTED
+            ||  DBGFADDRESS_IS_PHYS(pAddress) )
+            rc = PGMPhysSimpleReadGCPhys(pVM, pvBuf, pAddress->FlatPtr, cbRead);
+        else
+        {
+#if GC_ARCH_BITS > 32
+            if (    (   pAddress->FlatPtr >= _4G
+                     || pAddress->FlatPtr + cbRead > _4G)
+                &&  enmMode != PGMMODE_AMD64
+                &&  enmMode != PGMMODE_AMD64_NX)
+                return VERR_PAGE_TABLE_NOT_PRESENT;
+#endif
+            rc = PGMPhysSimpleReadGCPtr(pVCpu, pvBuf, pAddress->FlatPtr, cbRead);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Read guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idCpu           The ID of the source CPU context (for the address).
+ * @param   pAddress        Where to start reading.
+ * @param   pvBuf           Where to store the data we've read.
+ * @param   cbRead          The number of bytes to read.
+ */
+VMMR3DECL(int) DBGFR3MemRead(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void *pvBuf, size_t cbRead)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    if ((pAddress->fFlags & DBGFADDRESS_FLAGS_TYPE_MASK) == DBGFADDRESS_FLAGS_RING0)
+    {
+        AssertCompile(sizeof(RTHCUINTPTR) <= sizeof(pAddress->FlatPtr));
+        return VMMR3ReadR0Stack(pVM, idCpu, (RTHCUINTPTR)pAddress->FlatPtr, pvBuf, cbRead);
+    }
+    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3MemRead, 5, pVM, idCpu, pAddress, pvBuf, cbRead);
+}
+
+
+/**
+ * Read a zero terminated string from guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idCpu           The ID of the source CPU context (for the address).
+ * @param   pAddress        Where to start reading.
+ * @param   pszBuf          Where to store the string.
+ * @param   cchBuf          The size of the buffer.
+ */
+static DECLCALLBACK(int) dbgfR3MemReadString(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, char *pszBuf, size_t cchBuf)
+{
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    if (!DBGFR3AddrIsValid(pVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pszBuf))
+        return VERR_INVALID_POINTER;
+
+    /*
+     * Let dbgfR3MemRead do the job.
+     */
+    int rc = dbgfR3MemRead(pVM, idCpu, pAddress, pszBuf, cchBuf);
+
+    /*
+     * Make sure the result is terminated and that overflow is signaled.
+     * This may look a bit reckless with the rc but, it should be fine.
+     */
+    if (!RTStrEnd(pszBuf, cchBuf))
+    {
+        pszBuf[cchBuf - 1] = '\0';
+        rc = VINF_BUFFER_OVERFLOW;
+    }
+    /*
+     * Handle partial reads (not perfect).
+     */
+    else if (RT_FAILURE(rc))
+    {
+        if (pszBuf[0])
+            rc = VINF_SUCCESS;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Read a zero terminated string from guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idCpu           The ID of the source CPU context (for the address).
+ * @param   pAddress        Where to start reading.
+ * @param   pszBuf          Where to store the string.
+ * @param   cchBuf          The size of the buffer.
+ */
+VMMR3DECL(int) DBGFR3MemReadString(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, char *pszBuf, size_t cchBuf)
+{
+    /*
+     * Validate and zero output.
+     */
+    if (!VALID_PTR(pszBuf))
+        return VERR_INVALID_POINTER;
+    if (cchBuf <= 0)
+        return VERR_INVALID_PARAMETER;
+    memset(pszBuf, 0, cchBuf);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /*
+     * Pass it on to the EMT.
+     */
+    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3MemReadString, 5, pVM, idCpu, pAddress, pszBuf, cchBuf);
+}
+
+
+/**
+ * Writes guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idCpu           The ID of the target CPU context (for the address).
+ * @param   pAddress        Where to start writing.
+ * @param   pvBuf           The data to write.
+ * @param   cbWrite         The number of bytes to write.
+ */
+static DECLCALLBACK(int) dbgfR3MemWrite(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void const *pvBuf, size_t cbWrite)
+{
+    /*
+     * Validate the input we use, PGM does the rest.
+     */
+    if (!DBGFR3AddrIsValid(pVM, pAddress))
+        return VERR_INVALID_POINTER;
+    if (!VALID_PTR(pvBuf))
+        return VERR_INVALID_POINTER;
+
+    /*
+     * HMA is always special.
+     */
+    int rc;
+    if (DBGFADDRESS_IS_HMA(pAddress))
+    {
+        /** @todo write to HMA. */
+        rc = VERR_ACCESS_DENIED;
+    }
+    else
+    {
+        /*
+         * Select PGM function by addressing mode.
+         */
+        PVMCPU  pVCpu   = VMMGetCpuById(pVM, idCpu);
+        PGMMODE enmMode = PGMGetGuestMode(pVCpu);
+        if (    enmMode == PGMMODE_REAL
+            ||  enmMode == PGMMODE_PROTECTED
+            ||  DBGFADDRESS_IS_PHYS(pAddress) )
+            rc = PGMPhysSimpleWriteGCPhys(pVM, pAddress->FlatPtr, pvBuf, cbWrite);
+        else
+        {
+#if GC_ARCH_BITS > 32
+            if (    (   pAddress->FlatPtr >= _4G
+                     || pAddress->FlatPtr + cbWrite > _4G)
+                &&  enmMode != PGMMODE_AMD64
+                &&  enmMode != PGMMODE_AMD64_NX)
+                return VERR_PAGE_TABLE_NOT_PRESENT;
+#endif
+            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pAddress->FlatPtr, pvBuf, cbWrite);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Read guest memory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idCpu           The ID of the target CPU context (for the address).
+ * @param   pAddress        Where to start writing.
+ * @param   pvBuf           The data to write.
+ * @param   cbRead          The number of bytes to write.
+ */
+VMMR3DECL(int) DBGFR3MemWrite(PVM pVM, VMCPUID idCpu, PCDBGFADDRESS pAddress, void const *pvBuf, size_t cbWrite)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3MemWrite, 5, pVM, idCpu, pAddress, pvBuf, cbWrite);
+}
+
+
+/**
+ * Worker for DBGFR3SelQueryInfo that calls into SELM.
+ */
+static DECLCALLBACK(int) dbgfR3SelQueryInfo(PVM pVM, VMCPUID idCpu, RTSEL Sel, uint32_t fFlags, PDBGFSELINFO pSelInfo)
+{
+    /*
+     * Make the query.
+     */
+    int rc;
+    if (!(fFlags & DBGFSELQI_FLAGS_DT_SHADOW))
+    {
+        PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+        VMCPU_ASSERT_EMT(pVCpu);
+        rc = SELMR3GetSelectorInfo(pVM, pVCpu, Sel, pSelInfo);
+
+        /*
+         * 64-bit mode HACKS for making data and stack selectors wide open when
+         * queried. This is voodoo magic.
+         */
+        if (fFlags & DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE)
+        {
+            /* Expand 64-bit data and stack selectors. The check is a bit bogus... */
+            if (    RT_SUCCESS(rc)
+                &&  (pSelInfo->fFlags & (  DBGFSELINFO_FLAGS_LONG_MODE | DBGFSELINFO_FLAGS_REAL_MODE | DBGFSELINFO_FLAGS_PROT_MODE
+                                         | DBGFSELINFO_FLAGS_GATE      | DBGFSELINFO_FLAGS_HYPER
+                                         | DBGFSELINFO_FLAGS_INVALID   | DBGFSELINFO_FLAGS_NOT_PRESENT))
+                     == DBGFSELINFO_FLAGS_LONG_MODE
+                &&  pSelInfo->cbLimit != ~(RTGCPTR)0
+                &&  CPUMIsGuestIn64BitCode(pVCpu, CPUMGetGuestCtxCore(pVCpu)) )
+            {
+                pSelInfo->GCPtrBase = 0;
+                pSelInfo->cbLimit   = ~(RTGCPTR)0;
+            }
+            else if (   Sel == 0
+                     && CPUMIsGuestIn64BitCode(pVCpu, CPUMGetGuestCtxCore(pVCpu)))
+            {
+                pSelInfo->GCPtrBase = 0;
+                pSelInfo->cbLimit   = ~(RTGCPTR)0;
+                pSelInfo->Sel       = 0;
+                pSelInfo->SelGate   = 0;
+                pSelInfo->fFlags    = DBGFSELINFO_FLAGS_LONG_MODE;
+                pSelInfo->u.Raw64.Gen.u1Present  = 1;
+                pSelInfo->u.Raw64.Gen.u1Long     = 1;
+                pSelInfo->u.Raw64.Gen.u1DescType = 1;
+                rc = VINF_SUCCESS;
+            }
+        }
+    }
+    else
+    {
+        if (HWACCMIsEnabled(pVM))
+            rc = VERR_INVALID_STATE;
+        else
+            rc = SELMR3GetShadowSelectorInfo(pVM, Sel, pSelInfo);
+    }
+    return rc;
+}
+
+
+/**
+ * Gets information about a selector.
+ *
+ * Intended for the debugger mostly and will prefer the guest
+ * descriptor tables over the shadow ones.
+ *
+ * @returns VBox status code, the following are the common ones.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
+ *          descriptor table.
+ * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
+ *          is not returned if the selector itself isn't present, you have to
+ *          check that for yourself (see DBGFSELINFO::fFlags).
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
+ *          pagetable or page backing the selector table wasn't present.
+ *
+ * @param   pVM         VM handle.
+ * @param   idCpu       The ID of the virtual CPU context.
+ * @param   Sel         The selector to get info about.
+ * @param   fFlags      Flags, see DBGFQSEL_FLAGS_*.
+ * @param   pSelInfo    Where to store the information. This will always be
+ *                      updated.
+ *
+ * @remarks This is a wrapper around SELMR3GetSelectorInfo and
+ *          SELMR3GetShadowSelectorInfo.
+ */
+VMMR3DECL(int) DBGFR3SelQueryInfo(PVM pVM, VMCPUID idCpu, RTSEL Sel, uint32_t fFlags, PDBGFSELINFO pSelInfo)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(!(fFlags & ~(DBGFSELQI_FLAGS_DT_GUEST | DBGFSELQI_FLAGS_DT_SHADOW | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE)), VERR_INVALID_PARAMETER);
+    AssertReturn(    (fFlags & (DBGFSELQI_FLAGS_DT_SHADOW | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE))
+                  !=           (DBGFSELQI_FLAGS_DT_SHADOW | DBGFSELQI_FLAGS_DT_ADJ_64BIT_MODE), VERR_INVALID_PARAMETER);
+
+    /* Clear the return data here on this thread. */
+    memset(pSelInfo, 0, sizeof(*pSelInfo));
+
+    /*
+     * Dispatch the request to a worker running on the target CPU.
+     */
+    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3SelQueryInfo, 5, pVM, idCpu, Sel, fFlags, pSelInfo);
+}
+
+
+/**
+ * Validates a CS selector.
+ *
+ * @returns VBox status code.
+ * @param   pSelInfo    Pointer to the selector information for the CS selector.
+ * @param   SelCPL      The selector defining the CPL (SS).
+ */
+VMMDECL(int) DBGFR3SelInfoValidateCS(PCDBGFSELINFO pSelInfo, RTSEL SelCPL)
+{
+    /*
+     * Check if present.
+     */
+    if (pSelInfo->u.Raw.Gen.u1Present)
+    {
+        /*
+         * Type check.
+         */
+        if (    pSelInfo->u.Raw.Gen.u1DescType == 1
+            &&  (pSelInfo->u.Raw.Gen.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /*
+             * Check level.
+             */
+            unsigned uLevel = RT_MAX(SelCPL & X86_SEL_RPL, pSelInfo->Sel & X86_SEL_RPL);
+            if (    !(pSelInfo->u.Raw.Gen.u4Type & X86_SEL_TYPE_CONF)
+                ?   uLevel <= pSelInfo->u.Raw.Gen.u2Dpl
+                :   uLevel >= pSelInfo->u.Raw.Gen.u2Dpl /* hope I got this right now... */
+                    )
+                return VINF_SUCCESS;
+            return VERR_INVALID_RPL;
+        }
+        return VERR_NOT_CODE_SELECTOR;
+    }
+    return VERR_SELECTOR_NOT_PRESENT;
+}
+
+
+/**
+ * Converts a PGM paging mode to a set of DBGFPGDMP_XXX flags.
+ *
+ * @returns Flags. UINT32_MAX if the mode is invalid (asserted).
+ * @param   enmMode             The mode.
+ */
+static uint32_t dbgfR3PagingDumpModeToFlags(PGMMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case PGMMODE_32_BIT:
+            return DBGFPGDMP_FLAGS_PSE;
+        case PGMMODE_PAE:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE;
+        case PGMMODE_PAE_NX:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_NXE;
+        case PGMMODE_AMD64:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME;
+        case PGMMODE_AMD64_NX:
+            return DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME | DBGFPGDMP_FLAGS_NXE;
+        case PGMMODE_NESTED:
+            return DBGFPGDMP_FLAGS_NP;
+        case PGMMODE_EPT:
+            return DBGFPGDMP_FLAGS_EPT;
+        default:
+            AssertFailedReturn(UINT32_MAX);
+    }
+}
+
+
+/**
+ * EMT worker for DBGFR3PagingDumpEx.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The current CPU ID.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.  Valid.
+ * @param   pcr3            The CR3 to use (unless we're getting the current
+ *                          state, see @a fFlags).
+ * @param   pu64FirstAddr   The first address.
+ * @param   pu64LastAddr    The last address.
+ * @param   cMaxDepth       The depth.
+ * @param   pHlp            The output callbacks.
+ */
+static DECLCALLBACK(int) dbgfR3PagingDumpEx(PVM pVM, VMCPUID idCpu, uint32_t fFlags, uint64_t *pcr3,
+                                            uint64_t *pu64FirstAddr, uint64_t *pu64LastAddr,
+                                            uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Implement dumping both context by means of recursion.
+     */
+    if ((fFlags & (DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_SHADOW)) == (DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_SHADOW))
+    {
+        int rc1 = dbgfR3PagingDumpEx(pVM, idCpu, fFlags & ~DBGFPGDMP_FLAGS_GUEST,
+                                     pcr3, pu64FirstAddr, pu64LastAddr, cMaxDepth, pHlp);
+        int rc2 = dbgfR3PagingDumpEx(pVM, idCpu, fFlags & ~DBGFPGDMP_FLAGS_SHADOW,
+                                     pcr3, pu64FirstAddr, pu64LastAddr, cMaxDepth, pHlp);
+        return RT_FAILURE(rc1) ? rc1 : rc2;
+    }
+
+    /*
+     * Get the current CR3/mode if required.
+     */
+    uint64_t cr3 = *pcr3;
+    if (fFlags & (DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE))
+    {
+        PVMCPU pVCpu = &pVM->aCpus[idCpu];
+        if (fFlags & DBGFPGDMP_FLAGS_SHADOW)
+        {
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_CR3)
+                cr3 = PGMGetHyperCR3(pVCpu);
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE)
+            {
+                fFlags |= dbgfR3PagingDumpModeToFlags(PGMGetShadowMode(pVCpu));
+                if (fFlags & DBGFPGDMP_FLAGS_NP)
+                {
+                    fFlags |= dbgfR3PagingDumpModeToFlags(PGMGetHostMode(pVM));
+                    if (HC_ARCH_BITS == 32 && CPUMIsGuestInLongMode(pVCpu))
+                        fFlags |= DBGFPGDMP_FLAGS_LME;
+                }
+            }
+        }
+        else
+        {
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_CR3)
+                cr3 = CPUMGetGuestCR3(pVCpu);
+            if (fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE)
+            {
+                AssertCompile(DBGFPGDMP_FLAGS_PSE == X86_CR4_PSE);      AssertCompile(DBGFPGDMP_FLAGS_PAE == X86_CR4_PAE);
+                fFlags |= CPUMGetGuestCR4(pVCpu)  & (X86_CR4_PSE | X86_CR4_PAE);
+                AssertCompile(DBGFPGDMP_FLAGS_LME == MSR_K6_EFER_LME);  AssertCompile(DBGFPGDMP_FLAGS_NXE == MSR_K6_EFER_NXE);
+                fFlags |= CPUMGetGuestEFER(pVCpu) & (MSR_K6_EFER_LME | MSR_K6_EFER_NXE);
+            }
+        }
+    }
+    fFlags &= ~(DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3);
+
+    /*
+     * Call PGM to do the real work.
+     */
+    int rc;
+    if (fFlags & DBGFPGDMP_FLAGS_SHADOW)
+        rc = PGMR3DumpHierarchyShw(pVM, cr3, fFlags, *pu64FirstAddr, *pu64LastAddr, cMaxDepth, pHlp);
+    else
+        rc = PGMR3DumpHierarchyGst(pVM, cr3, fFlags, *pu64FirstAddr, *pu64LastAddr, cMaxDepth, pHlp);
+    return rc;
+}
+
+
+/**
+ * Dump paging structures.
+ *
+ * This API can be used to dump both guest and shadow structures.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The current CPU ID.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
+ * @param   cr3             The CR3 to use (unless we're getting the current
+ *                          state, see @a fFlags).
+ * @param   u64FirstAddr    The address to start dumping at.
+ * @param   u64LastAddr     The address to end dumping after.
+ * @param   cMaxDepth       The depth.
+ * @param   pHlp            The output callbacks.  Defaults to the debug log if
+ *                          NULL.
+ */
+VMMDECL(int) DBGFR3PagingDumpEx(PVM pVM, VMCPUID idCpu, uint32_t fFlags, uint64_t cr3, uint64_t u64FirstAddr,
+                                uint64_t u64LastAddr, uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /*
+     * Input validation.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    AssertReturn(!(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags & (DBGFPGDMP_FLAGS_SHADOW | DBGFPGDMP_FLAGS_GUEST), VERR_INVALID_PARAMETER);
+    AssertReturn((fFlags & DBGFPGDMP_FLAGS_CURRENT_MODE) || !(fFlags & DBGFPGDMP_FLAGS_MODE_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(   !(fFlags & DBGFPGDMP_FLAGS_EPT)
+                 || !(fFlags & (DBGFPGDMP_FLAGS_LME | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_NXE))
+                 , VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pHlp, VERR_INVALID_POINTER);
+    AssertReturn(cMaxDepth, VERR_INVALID_PARAMETER);
+
+    /*
+     * Forward the request to the target CPU.
+     */
+    return VMR3ReqCallWaitU(pVM->pUVM, idCpu, (PFNRT)dbgfR3PagingDumpEx, 8,
+                            pVM, idCpu, fFlags, &cr3, &u64FirstAddr, &u64LastAddr, cMaxDepth, pHlp);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFModule.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFModule.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFModule.cpp	(revision 35333)
@@ -0,0 +1,290 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Module & Segment Management.
+ */
+
+/*
+ * Copyright (C) 2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_dbgf_module    DBGFModule - Module & Segment Management
+ *
+ * A module is our representation of an executable binary. It's main purpose
+ * is to provide segments that can be mapped into address spaces and thereby
+ * provide debug info for those parts for the guest code or data.
+ *
+ * This module will not deal directly with debug info, it will only serve
+ * as an interface between the debugger / symbol lookup and the debug info
+ * readers.
+ *
+ * An executable binary doesn't need to have a file, or that is, we don't
+ * need the file to create a module for it. There will be interfaces for
+ * ROMs to register themselves so we can get to their symbols, and there
+ * will be interfaces for the guest OS plugins (@see pg_dbgf_os) to
+ * register kernel, drivers and other global modules.
+ */
+
+#if 0
+#include <VBox/dbgf.h>
+
+
+/** Special segment number that indicates that the offset is a relative
+ * virtual address (RVA). I.e. an offset from the start of the module. */
+#define DBGF_SEG_RVA UINT32_C(0xfffffff0)
+
+/** @defgroup grp_dbgf_dbginfo Debug Info Types
+ * @{ */
+/** Other format. */
+#define DBGF_DBGINFO_OTHER      RT_BIT_32(0)
+/** Stabs. */
+#define DBGF_DBGINFO_STABS      RT_BIT_32(1)
+/** Debug With Arbitrary Record Format (DWARF). */
+#define DBGF_DBGINFO_DWARF      RT_BIT_32(2)
+/** Microsoft Codeview debug info. */
+#define DBGF_DBGINFO_CODEVIEW   RT_BIT_32(3)
+/** Watcom debug info. */
+#define DBGF_DBGINFO_WATCOM     RT_BIT_32(4)
+/** IBM High Level Language debug info. */
+#define DBGF_DBGINFO_HLL        RT_BIT_32(5)
+/** Old OS/2 and Windows symbol file. */
+#define DBGF_DBGINFO_SYM        RT_BIT_32(6)
+/** Map file. */
+#define DBGF_DBGINFO_MAP        RT_BIT_32(7)
+/** @} */
+
+/** @defgroup grp_dbgf_exeimg Executable Image Types
+ * @{ */
+/** Some other format. */
+#define DBGF_EXEIMG_OTHER       RT_BIT_32(0)
+/** Portable Executable. */
+#define DBGF_EXEIMG_PE          RT_BIT_32(1)
+/** Linear eXecutable. */
+#define DBGF_EXEIMG_LX          RT_BIT_32(2)
+/** Linear Executable. */
+#define DBGF_EXEIMG_LE          RT_BIT_32(3)
+/** New Executable. */
+#define DBGF_EXEIMG_NE          RT_BIT_32(4)
+/** DOS Executable (Mark Zbikowski). */
+#define DBGF_EXEIMG_MZ          RT_BIT_32(5)
+/** COM Executable. */
+#define DBGF_EXEIMG_COM         RT_BIT_32(6)
+/** a.out Executable. */
+#define DBGF_EXEIMG_AOUT        RT_BIT_32(7)
+/** Executable and Linkable Format. */
+#define DBGF_EXEIMG_ELF         RT_BIT_32(8)
+/** Mach-O Executable (including FAT ones). */
+#define DBGF_EXEIMG_MACHO       RT_BIT_32(9)
+/** @} */
+
+/** Pointer to a module. */
+typedef struct DBGFMOD *PDBGFMOD;
+
+
+/**
+ * Virtual method table for executable image interpreters.
+ */
+typedef struct DBGFMODVTIMG
+{
+    /** Magic number (DBGFMODVTIMG_MAGIC). */
+    uint32_t    u32Magic;
+    /** Mask of supported debug info types, see grp_dbgf_exeimg.
+     * Used to speed up the search for a suitable interpreter. */
+    uint32_t    fSupports;
+    /** The name of the interpreter. */
+    const char *pszName;
+
+    /**
+     * Try open the image.
+     *
+     * This combines probing and opening.
+     *
+     * @returns VBox status code. No informational returns defined.
+     *
+     * @param   pMod        Pointer to the module that is being opened.
+     *
+     *                      The DBGFMOD::pszDbgFile member will point to
+     *                      the filename of any debug info we're aware of
+     *                      on input. Also, or alternatively, it is expected
+     *                      that the interpreter will look for debug info in
+     *                      the executable image file when present and that it
+     *                      may ask the image interpreter for this when it's
+     *                      around.
+     *
+     *                      Upon successful return the method is expected to
+     *                      initialize pDbgOps and pvDbgPriv.
+     */
+    DECLCALLBACKMEMBER(int, pfnTryOpen)(PDBGFMOD pMod);
+
+    /**
+     * Close the interpreter, freeing all associated resources.
+     *
+     * The caller sets the pDbgOps and pvDbgPriv DBGFMOD members
+     * to NULL upon return.
+     *
+     * @param   pMod        Pointer to the module structure.
+     */
+    DECLCALLBACKMEMBER(int, pfnClose)(PDBGFMOD pMod);
+
+} DBGFMODVTIMG
+
+/**
+ * Virtual method table for debug info interpreters.
+ */
+typedef struct DBGFMODVTDBG
+{
+    /** Magic number (DBGFMODVTDBG_MAGIC). */
+    uint32_t    u32Magic;
+    /** Mask of supported debug info types, see grp_dbgf_dbginfo.
+     * Used to speed up the search for a suitable interpreter. */
+    uint32_t    fSupports;
+    /** The name of the interpreter. */
+    const char *pszName;
+
+    /**
+     * Try open the image.
+     *
+     * This combines probing and opening.
+     *
+     * @returns VBox status code. No informational returns defined.
+     *
+     * @param   pMod        Pointer to the module that is being opened.
+     *
+     *                      The DBGFMOD::pszDbgFile member will point to
+     *                      the filename of any debug info we're aware of
+     *                      on input. Also, or alternatively, it is expected
+     *                      that the interpreter will look for debug info in
+     *                      the executable image file when present and that it
+     *                      may ask the image interpreter for this when it's
+     *                      around.
+     *
+     *                      Upon successful return the method is expected to
+     *                      initialize pDbgOps and pvDbgPriv.
+     */
+    DECLCALLBACKMEMBER(int, pfnTryOpen)(PDBGFMOD pMod);
+
+    /**
+     * Close the interpreter, freeing all associated resources.
+     *
+     * The caller sets the pDbgOps and pvDbgPriv DBGFMOD members
+     * to NULL upon return.
+     *
+     * @param   pMod        Pointer to the module structure.
+     */
+    DECLCALLBACKMEMBER(int, pfnClose)(PDBGFMOD pMod);
+
+    /**
+     * Queries symbol information by symbol name.
+     *
+     * @returns VBox status code.
+     * @retval  VINF_SUCCESS on success, no informational status code.
+     * @retval  VERR_DBGF_NO_SYMBOLS if there aren't any symbols.
+     * @retval  VERR_SYMBOL_NOT_FOUND if no suitable symbol was found.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   pszSymbol   The symbol name.
+     * @para    pSymbol     Where to store the symbol information.
+     */
+    DECLCALLBACKMEMBER(int, pfnSymbolByName)(PDBGFMOD pMod, const char *pszSymbol, PDBGFSYMBOL pSymbol);
+
+    /**
+     * Queries symbol information by address.
+     *
+     * The returned symbol is what the debug info interpreter considers the symbol
+     * most applicable to the specified address. This usually means a symbol with an
+     * address equal or lower than the requested.
+     *
+     * @returns VBox status code.
+     * @retval  VINF_SUCCESS on success, no informational status code.
+     * @retval  VERR_DBGF_NO_SYMBOLS if there aren't any symbols.
+     * @retval  VERR_SYMBOL_NOT_FOUND if no suitable symbol was found.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
+     * @param   off         The offset into the segment.
+     * @param   poffDisp    Where to store the distance between the specified address
+     *                      and the returned symbol. Optional.
+     * @param   pSymbol     Where to store the symbol information.
+     */
+    DECLCALLBACKMEMBER(int, pfnSymbolByAddr)(PDBGFMOD pMod, uint32_t iSeg, RTGCUINTPTR off, PRTGCINTPTR poffDisp, PDBGFSYMBOL pSymbol);
+
+    /**
+     * Queries line number information by address.
+     *
+     * @returns VBox status code.
+     * @retval  VINF_SUCCESS on success, no informational status code.
+     * @retval  VERR_DBGF_NO_LINE_NUMBERS if there aren't any line numbers.
+     * @retval  VERR_DBGF_LINE_NOT_FOUND if no suitable line number was found.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
+     * @param   off         The offset into the segment.
+     * @param   poffDisp    Where to store the distance between the specified address
+     *                      and the returned line number. Optional.
+     * @param   pLine       Where to store the information about the closest line number.
+     */
+    DECLCALLBACKMEMBER(int, pfnLineByAddr)(PDBGFMOD pMod, uint32_t iSeg, RTGCUINTPTR off, PRTGCINTPTR poffDisp, PDBGFLINE pLine);
+
+    /**
+     * Adds a symbol to the module (optional).
+     *
+     * This method is used to implement DBGFR3SymbolAdd.
+     *
+     * @returns VBox status code.
+     * @retval  VERR_NOT_SUPPORTED if the interpreter doesn't support this feature.
+     *
+     * @param   pMod        Pointer to the module structure.
+     * @param   pszSymbol   The symbol name.
+     * @param   iSeg        The segment number (0-based). DBGF_SEG_RVA can be used.
+     * @param   off         The offset into the segment.
+     * @param   cbSymbol    The area covered by the symbol. 0 is fine.
+     */
+    DECLCALLBACKMEMBER(int, pfnSymbolAdd)(PDBGFMOD pMod, const char *pszSymbol, uint32_t iSeg, RTGCUINTPTR off, RTUINT cbSymbol);
+
+    /** For catching initialization errors (DBGFMODVTDBG_MAGIC). */
+    uint32_t    u32EndMagic;
+} DBGFMODVTDBG;
+
+#define DBGFMODVTDBG_MAGIC 123
+
+/**
+ * Module.
+ */
+typedef struct DBGFMOD
+{
+    /** Magic value (DBGFMOD_MAGIC). */
+    uint32_t        u32Magic;
+    /** The number of address spaces this module is currently linked into.
+     * This is used to perform automatic cleanup and sharing. */
+    uint32_t        cLinks;
+    /** The module name (short). */
+    const char     *pszName;
+    /** The module filename. Can be NULL. */
+    const char     *pszImgFile;
+    /** The debug info file (if external). Can be NULL. */
+    const char     *pszDbgFile;
+
+    /** The method table for the executable image interpreter. */
+    PCDBGFMODVTIMG  pImgVt;
+    /** Pointer to the private data of the executable image interpreter. */
+    void           *pvImgPriv;
+
+    /** The method table for the debug info interpreter. */
+    PCDBGFMODVTDBG  pDbgVt;
+    /** Pointer to the private data of the debug info interpreter. */
+    void           *pvDbgPriv;
+
+} DBGFMOD;
+
+#define DBGFMOD_MAGIC   0x12345678
+
+#endif
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFOS.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFOS.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFOS.cpp	(revision 35333)
@@ -0,0 +1,447 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Guest OS Diggers.
+ */
+
+/*
+ * Copyright (C) 2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/param.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#define DBGF_OS_READ_LOCK(pVM)      do { } while (0)
+#define DBGF_OS_READ_UNLOCK(pVM)    do { } while (0)
+
+#define DBGF_OS_WRITE_LOCK(pVM)     do { } while (0)
+#define DBGF_OS_WRITE_UNLOCK(pVM)   do { } while (0)
+
+
+/**
+ * Internal cleanup routine called by DBGFR3Term().
+ *
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+void dbgfR3OSTerm(PVM pVM)
+{
+    /*
+     * Terminate the current one.
+     */
+    if (pVM->dbgf.s.pCurOS)
+    {
+        pVM->dbgf.s.pCurOS->pReg->pfnTerm(pVM, pVM->dbgf.s.pCurOS->abData);
+        pVM->dbgf.s.pCurOS = NULL;
+    }
+
+    /*
+     * Destroy all the instances.
+     */
+    while (pVM->dbgf.s.pOSHead)
+    {
+        PDBGFOS pOS = pVM->dbgf.s.pOSHead;
+        pVM->dbgf.s.pOSHead = pOS->pNext;
+        if (pOS->pReg->pfnDestruct)
+            pOS->pReg->pfnDestruct(pVM, pOS->abData);
+        MMR3HeapFree(pOS);
+    }
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSRegister.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ * @param   pReg    The registration structure.
+ */
+static DECLCALLBACK(int) dbgfR3OSRegister(PVM pVM, PDBGFOSREG pReg)
+{
+    /* more validations. */
+    DBGF_OS_READ_LOCK(pVM);
+    PDBGFOS pOS;
+    for (pOS = pVM->dbgf.s.pOSHead; pOS; pOS = pOS->pNext)
+        if (!strcmp(pOS->pReg->szName, pReg->szName))
+        {
+            DBGF_OS_READ_UNLOCK(pVM);
+            Log(("dbgfR3OSRegister: %s -> VERR_ALREADY_LOADED\n", pReg->szName));
+            return VERR_ALREADY_LOADED;
+        }
+
+    /*
+     * Allocate a new structure, call the constructor and link it into the list.
+     */
+    pOS = (PDBGFOS)MMR3HeapAllocZ(pVM, MM_TAG_DBGF_OS, RT_OFFSETOF(DBGFOS, abData[pReg->cbData]));
+    AssertReturn(pOS, VERR_NO_MEMORY);
+    pOS->pReg = pReg;
+
+    int rc = pOS->pReg->pfnConstruct(pVM, pOS->abData);
+    if (RT_SUCCESS(rc))
+    {
+        DBGF_OS_WRITE_LOCK(pVM);
+        pOS->pNext = pVM->dbgf.s.pOSHead;
+        pVM->dbgf.s.pOSHead = pOS;
+        DBGF_OS_WRITE_UNLOCK(pVM);
+    }
+    else
+    {
+        if (pOS->pReg->pfnDestruct)
+            pOS->pReg->pfnDestruct(pVM, pOS->abData);
+        MMR3HeapFree(pOS);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers a guest OS digger.
+ *
+ * This will instantiate an instance of the digger and add it
+ * to the list for us in the next call to DBGFR3OSDetect().
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ * @param   pReg    The registration structure.
+ * @thread  Any.
+ */
+VMMR3DECL(int) DBGFR3OSRegister(PVM pVM, PCDBGFOSREG pReg)
+{
+    /*
+     * Validate intput.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
+    AssertReturn(pReg->u32Magic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(pReg->u32EndMagic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(!pReg->fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pReg->cbData < _2G, VERR_INVALID_PARAMETER);
+    AssertReturn(pReg->szName[0], VERR_INVALID_NAME);
+    AssertReturn(RTStrEnd(&pReg->szName[0], sizeof(pReg->szName)), VERR_INVALID_NAME);
+    AssertPtrReturn(pReg->pfnConstruct, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pReg->pfnDestruct, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnProbe, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnInit, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnRefresh, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnTerm, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnQueryVersion, VERR_INVALID_POINTER);
+    AssertPtrReturn(pReg->pfnQueryInterface, VERR_INVALID_POINTER);
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSRegister, 2, pVM, pReg);
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSDeregister.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ * @param   pReg    The registration structure.
+ */
+static DECLCALLBACK(int) dbgfR3OSDeregister(PVM pVM, PDBGFOSREG pReg)
+{
+    /*
+     * Unlink it.
+     */
+    bool    fWasCurOS = false;
+    PDBGFOS pOSPrev   = NULL;
+    PDBGFOS pOS;
+    DBGF_OS_WRITE_LOCK(pVM);
+    for (pOS = pVM->dbgf.s.pOSHead; pOS; pOSPrev = pOS, pOS = pOS->pNext)
+        if (pOS->pReg == pReg)
+        {
+            if (pOSPrev)
+                pOSPrev->pNext = pOS->pNext;
+            else
+                pVM->dbgf.s.pOSHead = pOS->pNext;
+            if (pVM->dbgf.s.pCurOS == pOS)
+            {
+                pVM->dbgf.s.pCurOS = NULL;
+                fWasCurOS = true;
+            }
+            break;
+        }
+    DBGF_OS_WRITE_UNLOCK(pVM);
+    if (!pOS)
+    {
+        Log(("DBGFR3OSDeregister: %s -> VERR_NOT_FOUND\n", pReg->szName));
+        return VERR_NOT_FOUND;
+    }
+
+    /*
+     * Terminate it if it was the current OS, then invoke the
+     * destructor and clean up.
+     */
+    if (fWasCurOS)
+        pOS->pReg->pfnTerm(pVM, pOS->abData);
+    if (pOS->pReg->pfnDestruct)
+        pOS->pReg->pfnDestruct(pVM, pOS->abData);
+    MMR3HeapFree(pOS);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a guest OS digger previously registered by DBGFR3OSRegister.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM     Pointer to the shared VM structure.
+ * @param   pReg    The registration structure.
+ * @thread  Any.
+ */
+VMMR3DECL(int)  DBGFR3OSDeregister(PVM pVM, PCDBGFOSREG pReg)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
+    AssertReturn(pReg->u32Magic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(pReg->u32EndMagic == DBGFOSREG_MAGIC, VERR_INVALID_MAGIC);
+    AssertReturn(RTStrEnd(&pReg->szName[0], sizeof(pReg->szName)), VERR_INVALID_NAME);
+
+    DBGF_OS_READ_LOCK(pVM);
+    PDBGFOS pOS;
+    for (pOS = pVM->dbgf.s.pOSHead; pOS; pOS = pOS->pNext)
+        if (pOS->pReg == pReg)
+            break;
+    DBGF_OS_READ_LOCK(pVM);
+
+    if (!pOS)
+    {
+        Log(("DBGFR3OSDeregister: %s -> VERR_NOT_FOUND\n", pReg->szName));
+        return VERR_NOT_FOUND;
+    }
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSDeregister, 2, pVM, pReg);
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSDetect.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully detected.
+ * @retval  VINF_DBGF_OS_NOT_DETCTED if we cannot figure it out.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pszName     Where to store the OS name. Empty string if not detected.
+ * @param   cchName     Size of the buffer.
+ */
+static DECLCALLBACK(int) dbgfR3OSDetect(PVM pVM, char *pszName, size_t cchName)
+{
+    /*
+     * Cycle thru the detection routines.
+     */
+    PDBGFOS const pOldOS = pVM->dbgf.s.pCurOS;
+    pVM->dbgf.s.pCurOS = NULL;
+
+    for (PDBGFOS pNewOS = pVM->dbgf.s.pOSHead; pNewOS; pNewOS = pNewOS->pNext)
+        if (pNewOS->pReg->pfnProbe(pVM, pNewOS->abData))
+        {
+            int rc;
+            pVM->dbgf.s.pCurOS = pNewOS;
+            if (pOldOS == pNewOS)
+                rc = pNewOS->pReg->pfnRefresh(pVM, pNewOS->abData);
+            else
+            {
+                if (pOldOS)
+                    pOldOS->pReg->pfnTerm(pVM, pNewOS->abData);
+                rc = pNewOS->pReg->pfnInit(pVM, pNewOS->abData);
+            }
+            if (pszName && cchName)
+                strncat(pszName, pNewOS->pReg->szName, cchName);
+            return rc;
+        }
+
+    /* not found */
+    if (pOldOS)
+        pOldOS->pReg->pfnTerm(pVM, pOldOS->abData);
+    return VINF_DBGF_OS_NOT_DETCTED;
+}
+
+
+/**
+ * Detects the guest OS and try dig out symbols and useful stuff.
+ *
+ * When called the 2nd time, symbols will be updated that if the OS
+ * is the same.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully detected.
+ * @retval  VINF_DBGF_OS_NOT_DETCTED if we cannot figure it out.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pszName     Where to store the OS name. Empty string if not detected.
+ * @param   cchName     Size of the buffer.
+ * @thread  Any.
+ */
+VMMR3DECL(int) DBGFR3OSDetect(PVM pVM, char *pszName, size_t cchName)
+{
+    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
+    if (pszName && cchName)
+        *pszName = '\0';
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)dbgfR3OSDetect, 3, pVM, pszName, cchName);
+}
+
+
+/**
+ * EMT worker function for DBGFR3OSQueryNameAndVersion
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pszName         Where to store the OS name. Optional.
+ * @param   cchName         The size of the name buffer.
+ * @param   pszVersion      Where to store the version string. Optional.
+ * @param   cchVersion      The size of the version buffer.
+ */
+static DECLCALLBACK(int) dbgfR3OSQueryNameAndVersion(PVM pVM, char *pszName, size_t cchName, char *pszVersion, size_t cchVersion)
+{
+    /*
+     * Any known OS?
+     */
+    if (pVM->dbgf.s.pCurOS)
+    {
+        int rc = VINF_SUCCESS;
+        if (pszName && cchName)
+        {
+            size_t cch = strlen(pVM->dbgf.s.pCurOS->pReg->szName);
+            if (cchName > cch)
+                memcpy(pszName, pVM->dbgf.s.pCurOS->pReg->szName, cch + 1);
+            else
+            {
+                memcpy(pszName, pVM->dbgf.s.pCurOS->pReg->szName, cchName - 1);
+                pszName[cchName - 1] = '\0';
+                rc = VINF_BUFFER_OVERFLOW;
+            }
+        }
+
+        if (pszVersion && cchVersion)
+        {
+            int rc2 = pVM->dbgf.s.pCurOS->pReg->pfnQueryVersion(pVM, pVM->dbgf.s.pCurOS->abData, pszVersion, cchVersion);
+            if (RT_FAILURE(rc2) || rc == VINF_SUCCESS)
+                rc = rc2;
+        }
+        return rc;
+    }
+
+    return VERR_DBGF_OS_NOT_DETCTED;
+}
+
+
+/**
+ * Queries the name and/or version string for the guest OS.
+ *
+ * It goes without saying that this querying is done using the current
+ * guest OS digger and not additions or user configuration.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pszName         Where to store the OS name. Optional.
+ * @param   cchName         The size of the name buffer.
+ * @param   pszVersion      Where to store the version string. Optional.
+ * @param   cchVersion      The size of the version buffer.
+ * @thread  Any.
+ */
+VMMR3DECL(int) DBGFR3OSQueryNameAndVersion(PVM pVM, char *pszName, size_t cchName, char *pszVersion, size_t cchVersion)
+{
+    AssertPtrNullReturn(pszName, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pszVersion, VERR_INVALID_POINTER);
+
+    /*
+     * Initialize the output up front.
+     */
+    if (pszName && cchName)
+        *pszName = '\0';
+    if (pszVersion && cchVersion)
+        *pszVersion = '\0';
+
+    /*
+     * Pass it on to EMT(0).
+     */
+    return VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/,
+                            (PFNRT)dbgfR3OSQueryNameAndVersion, 5, pVM, pszName, cchName, pszVersion, cchVersion);
+}
+
+
+/**
+ * EMT worker for DBGFR3OSQueryInterface.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   enmIf       The interface identifier.
+ * @param   ppvIf       Where to store the interface pointer on success.
+ */
+static DECLCALLBACK(void) dbgfR3OSQueryInterface(PVM pVM, DBGFOSINTERFACE enmIf, void **ppvIf)
+{
+    if (pVM->dbgf.s.pCurOS)
+    {
+        *ppvIf = pVM->dbgf.s.pCurOS->pReg->pfnQueryInterface(pVM, pVM->dbgf.s.pCurOS->abData, enmIf);
+        if (*ppvIf)
+        {
+            /** @todo Create EMT wrapper for the returned interface once we've defined one...
+             * Just keep a list of wrapper together with the OS instance. */
+        }
+    }
+    else
+        *ppvIf = NULL;
+}
+
+
+/**
+ * Query an optional digger interface.
+ *
+ * @returns Pointer to the digger interface on success, NULL if the interfaces isn't
+ *          available or no active guest OS digger.
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   enmIf       The interface identifier.
+ * @thread  Any.
+ */
+VMMR3DECL(void *) DBGFR3OSQueryInterface(PVM pVM, DBGFOSINTERFACE enmIf)
+{
+    AssertMsgReturn(enmIf > DBGFOSINTERFACE_INVALID && enmIf < DBGFOSINTERFACE_END, ("%d\n", enmIf), NULL);
+
+    /*
+     * Pass it on to an EMT.
+     */
+    void *pvIf = NULL;
+    VMR3ReqCallVoidWait(pVM, VMCPUID_ANY, (PFNRT)dbgfR3OSQueryInterface, 3, pVM, enmIf, &pvIf);
+    return pvIf;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFReg.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFReg.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFReg.cpp	(revision 35333)
@@ -0,0 +1,1198 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Register Methods.
+ */
+
+/*
+ * Copyright (C) 2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** @name Register and value sizes used by dbgfR3RegQueryWorker and
+ *        dbgfR3RegSetWorker.
+ * @{ */
+#define R_SZ_8          RT_BIT(0)
+#define R_SZ_16         RT_BIT(1)
+#define R_SZ_32         RT_BIT(2)
+#define R_SZ_64         RT_BIT(3)
+#define R_SZ_64_16      RT_BIT(4)
+#define R_SZ_80         RT_BIT(5)
+#define R_SZ_128        RT_BIT(6)
+#define R_SZ_8_TO_64    (R_SZ_8 | R_SZ_16 | R_SZ_32 | R_SZ_64)
+#define R_SZ_16_TO_64   (R_SZ_16 | R_SZ_32 | R_SZ_64)
+#define R_SZ_32_OR_64   (R_SZ_32 | R_SZ_64)
+/** @}  */
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Register sub-field descriptor.
+ */
+typedef struct DBGFREGSUBFIELD
+{
+    /** The name of the sub-field.  NULL is used to terminate the array. */
+    const char     *pszName;
+    /** The index of the first bit. */
+    uint8_t         iFirstBit;
+    /** The number of bits. */
+    uint8_t         cBits;
+    /** The shift count. */
+    int8_t          cShift;
+    /** @todo getter/setter callbacks? */
+} DBGFREGSUBFIELD;
+/** Pointer to a const register sub-field descriptor. */
+typedef DBGFREGSUBFIELD const *PCDBGFREGSUBFIELD;
+
+/**
+ * Register alias descriptor.
+ */
+typedef struct DBGFREGALIAS
+{
+    /** The alias name.  NULL is used to terminate the array. */
+    const char     *pszName;
+    /** Set to a valid type if the alias has a different type. */
+    DBGFREGVALTYPE  enmType;
+} DBGFREGALIAS;
+/** Pointer to a const register alias descriptor. */
+typedef DBGFREGALIAS const *PCDBGFREGALIAS;
+
+/**
+ * Register descriptor.
+ */
+typedef struct DBGFREGDESC
+{
+    /** The normal register name. */
+    const char             *pszName;
+    /** The register identifier (same as the table index). */
+    DBGFREG                 enmReg;
+    /** The default register type. */
+    DBGFREGVALTYPE          enmType;
+    /** The offset info the CPUMCTX structure. ~(size_t)0 if not applicable. */
+    size_t                  offCtx;
+    /** Getter (optional). */
+    DECLCALLBACKMEMBER(int, pfnGet)(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+    /** Setter (optional). */
+    DECLCALLBACKMEMBER(int, pfnSet)(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+    /** Aliases (optional). */
+    PCDBGFREGALIAS          paAliases;
+    /** Sub fields (optional). */
+    PCDBGFREGSUBFIELD       paSubFields;
+} DBGFREGDESC;
+/** Pointer to a const register descriptor. */
+typedef DBGFREGDESC const *PCDBGFREGDESC;
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) dbgfR3RegSet_seg(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+static DECLCALLBACK(int) dbgfR3RegGet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+static DECLCALLBACK(int) dbgfR3RegSet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+static DECLCALLBACK(int) dbgfR3RegGet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+static DECLCALLBACK(int) dbgfR3RegSet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+static DECLCALLBACK(int) dbgfR3RegGet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+static DECLCALLBACK(int) dbgfR3RegSet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+static DECLCALLBACK(int) dbgfR3RegGet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+static DECLCALLBACK(int) dbgfR3RegSet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+static DECLCALLBACK(int) dbgfR3RegGet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+static DECLCALLBACK(int) dbgfR3RegSet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+static DECLCALLBACK(int) dbgfR3RegGet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+static DECLCALLBACK(int) dbgfR3RegSet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+static DECLCALLBACK(int) dbgfR3RegGet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue);
+static DECLCALLBACK(int) dbgfR3RegSet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask);
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/*
+ * Set up aliases.
+ */
+#define DBGFREGALIAS_STD(Name, psz32, psz16, psz8)  \
+    static DBGFREGALIAS const g_aDbgfRegAliases_##Name[] = \
+    { \
+        { psz32, DBGFREGVALTYPE_U32    }, \
+        { psz16, DBGFREGVALTYPE_U16    }, \
+        { psz8,  DBGFREGVALTYPE_U8     }, \
+        { NULL, DBGFREGVALTYPE_INVALID } \
+    }
+DBGFREGALIAS_STD(rax,  "eax",   "ax",   "al");
+DBGFREGALIAS_STD(rcx,  "ecx",   "cx",   "cl");
+DBGFREGALIAS_STD(rdx,  "edx",   "dx",   "dl");
+DBGFREGALIAS_STD(rbx,  "ebx",   "bx",   "bl");
+DBGFREGALIAS_STD(rsp,  "esp",   "sp",   NULL);
+DBGFREGALIAS_STD(rbp,  "ebp",   "bp",   NULL);
+DBGFREGALIAS_STD(rsi,  "esi",   "si",  "sil");
+DBGFREGALIAS_STD(rdi,  "edi",   "di",  "dil");
+DBGFREGALIAS_STD(r8,   "r8d",  "r8w",  "r8b");
+DBGFREGALIAS_STD(r9,   "r9d",  "r9w",  "r9b");
+DBGFREGALIAS_STD(r10, "r10d", "r10w", "r10b");
+DBGFREGALIAS_STD(r11, "r11d", "r11w", "r11b");
+DBGFREGALIAS_STD(r12, "r12d", "r12w", "r12b");
+DBGFREGALIAS_STD(r13, "r13d", "r13w", "r13b");
+DBGFREGALIAS_STD(r14, "r14d", "r14w", "r14b");
+DBGFREGALIAS_STD(r15, "r15d", "r15w", "r15b");
+DBGFREGALIAS_STD(rip, "eip",   "ip",    NULL);
+DBGFREGALIAS_STD(rflags, "eflags", "flags", NULL);
+#undef DBGFREGALIAS_STD
+
+static DBGFREGALIAS const g_aDbgfRegAliases_fpuip[] =
+{
+    { "fpuip", DBGFREGVALTYPE_U16  },
+    { NULL, DBGFREGVALTYPE_INVALID }
+};
+
+static DBGFREGALIAS const g_aDbgfRegAliases_fpudp[] =
+{
+    { "fpudp", DBGFREGVALTYPE_U16  },
+    { NULL, DBGFREGVALTYPE_INVALID }
+};
+
+static DBGFREGALIAS const g_aDbgfRegAliases_cr0[] =
+{
+    { "msw", DBGFREGVALTYPE_U16  },
+    { NULL, DBGFREGVALTYPE_INVALID }
+};
+
+/*
+ * Sub fields.
+ */
+/** Sub-fields for the (hidden) segment attribute register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_seg[] =
+{
+    { "type",   0,   4,  0 },
+    { "s",      4,   1,  0 },
+    { "dpl",    5,   2,  0 },
+    { "p",      7,   1,  0 },
+    { "avl",   12,   1,  0 },
+    { "l",     13,   1,  0 },
+    { "d",     14,   1,  0 },
+    { "g",     15,   1,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the flags register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_rflags[] =
+{
+    { "cf",     0,   1,  0 },
+    { "pf",     2,   1,  0 },
+    { "af",     4,   1,  0 },
+    { "zf",     6,   1,  0 },
+    { "sf",     7,   1,  0 },
+    { "tf",     8,   1,  0 },
+    { "if",     9,   1,  0 },
+    { "df",    10,   1,  0 },
+    { "of",    11,   1,  0 },
+    { "iopl",  12,   2,  0 },
+    { "nt",    14,   1,  0 },
+    { "rf",    16,   1,  0 },
+    { "vm",    17,   1,  0 },
+    { "ac",    18,   1,  0 },
+    { "vif",   19,   1,  0 },
+    { "vip",   20,   1,  0 },
+    { "id",    21,   1,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the FPU control word register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_fcw[] =
+{
+    { "im",     1,   1,  0 },
+    { "dm",     2,   1,  0 },
+    { "zm",     3,   1,  0 },
+    { "om",     4,   1,  0 },
+    { "um",     5,   1,  0 },
+    { "pm",     6,   1,  0 },
+    { "pc",     8,   2,  0 },
+    { "rc",    10,   2,  0 },
+    { "x",     12,   1,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the FPU status word register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_fsw[] =
+{
+    { "ie",     0,   1,  0 },
+    { "de",     1,   1,  0 },
+    { "ze",     2,   1,  0 },
+    { "oe",     3,   1,  0 },
+    { "ue",     4,   1,  0 },
+    { "pe",     5,   1,  0 },
+    { "se",     6,   1,  0 },
+    { "es",     7,   1,  0 },
+    { "c0",     8,   1,  0 },
+    { "c1",     9,   1,  0 },
+    { "c2",    10,   1,  0 },
+    { "top",   11,   3,  0 },
+    { "c3",    14,   1,  0 },
+    { "b",     15,   1,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the FPU tag word register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_ftw[] =
+{
+    { "tag0",   0,   2,  0 },
+    { "tag1",   2,   2,  0 },
+    { "tag2",   4,   2,  0 },
+    { "tag3",   6,   2,  0 },
+    { "tag4",   8,   2,  0 },
+    { "tag5",  10,   2,  0 },
+    { "tag6",  12,   2,  0 },
+    { "tag7",  14,   2,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the Multimedia Extensions Control and Status Register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_mxcsr[] =
+{
+    { "ie",     0,   1,  0 },
+    { "de",     1,   1,  0 },
+    { "ze",     2,   1,  0 },
+    { "oe",     3,   1,  0 },
+    { "ue",     4,   1,  0 },
+    { "pe",     5,   1,  0 },
+    { "daz",    6,   1,  0 },
+    { "im",     7,   1,  0 },
+    { "dm",     8,   1,  0 },
+    { "zm",     9,   1,  0 },
+    { "om",    10,   1,  0 },
+    { "um",    11,   1,  0 },
+    { "pm",    12,   1,  0 },
+    { "rc",    13,   2,  0 },
+    { "fz",    14,   1,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the FPU tag word register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_stN[] =
+{
+    { "man",    0,  64,  0 },
+    { "exp",   64,  15,  0 },
+    { "sig",   79,   1,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the MMX registers. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_mmN[] =
+{
+    { "dw0",    0,  32,  0 },
+    { "dw1",   32,  32,  0 },
+    { "w0",     0,  16,  0 },
+    { "w1",    16,  16,  0 },
+    { "w2",    32,  16,  0 },
+    { "w3",    48,  16,  0 },
+    { "b0",     0,   8,  0 },
+    { "b1",     8,   8,  0 },
+    { "b2",    16,   8,  0 },
+    { "b3",    24,   8,  0 },
+    { "b4",    32,   8,  0 },
+    { "b5",    40,   8,  0 },
+    { "b6",    48,   8,  0 },
+    { "b7",    56,   8,  0 },
+    { NULL,     0,   0,  0 }
+};
+
+/** Sub-fields for the XMM registers. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_xmmN[] =
+{
+    { "r0",      0,     32,  0 },
+    { "r0.man",  0+ 0,  23,  0 },
+    { "r0.exp",  0+23,   8,  0 },
+    { "r0.sig",  0+31,   1,  0 },
+    { "r1",     32,     32,  0 },
+    { "r1.man", 32+ 0,  23,  0 },
+    { "r1.exp", 32+23,   8,  0 },
+    { "r1.sig", 32+31,   1,  0 },
+    { "r2",     64,     32,  0 },
+    { "r2.man", 64+ 0,  23,  0 },
+    { "r2.exp", 64+23,   8,  0 },
+    { "r2.sig", 64+31,   1,  0 },
+    { "r3",     96,     32,  0 },
+    { "r3.man", 96+ 0,  23,  0 },
+    { "r3.exp", 96+23,   8,  0 },
+    { "r3.sig", 96+31,   1,  0 },
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the CR0 register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_cr0[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the CR3 register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_cr3[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the CR4 register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_cr4[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the DR6 register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_dr6[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the DR7 register. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_dr7[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the CR_PAT MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_apic_base[] =
+{
+    { "bsp",     8,      1,  0 },
+    { "ge",      9,      1,  0 },
+    { "base",    12,    20, 12 },
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the CR_PAT MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_cr_pat[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the PERF_STATUS MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_perf_status[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the EFER MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_efer[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the STAR MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_star[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the CSTAR MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_cstar[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the LSTAR MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_lstar[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+/** Sub-fields for the SF_MASK MSR. */
+static DBGFREGSUBFIELD const g_aDbgfRegFields_sf_mask[] =
+{
+    /** @todo  */
+    { NULL,      0,      0,  0 }
+};
+
+
+
+/**
+ * The register descriptors.
+ */
+static DBGFREGDESC const g_aDbgfRegDescs[] =
+{
+#define DBGFREGDESC_REG(UName, LName) \
+    { #LName, DBGFREG_##UName, DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, LName), NULL, NULL, g_aDbgfRegAliases_##LName, NULL }
+    DBGFREGDESC_REG(RAX, rax),
+    DBGFREGDESC_REG(RCX, rcx),
+    DBGFREGDESC_REG(RDX, rdx),
+    DBGFREGDESC_REG(RSP, rsp),
+    DBGFREGDESC_REG(RBP, rbp),
+    DBGFREGDESC_REG(RSI, rsi),
+    DBGFREGDESC_REG(RDI, rdi),
+    DBGFREGDESC_REG(R8,   r8),
+    DBGFREGDESC_REG(R9,   r9),
+    DBGFREGDESC_REG(R10, r10),
+    DBGFREGDESC_REG(R11, r11),
+    DBGFREGDESC_REG(R12, r12),
+    DBGFREGDESC_REG(R13, r13),
+    DBGFREGDESC_REG(R14, r14),
+    DBGFREGDESC_REG(R15, r15),
+#define DBGFREGDESC_SEG(UName, LName) \
+    { #LName,         DBGFREG_##UName,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, LName),               NULL, dbgfR3RegSet_seg, NULL, NULL },  \
+    { #LName "_attr", DBGFREG_##UName##_ATTR, DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, LName##Hid.Attr.u),   NULL, NULL, NULL, g_aDbgfRegFields_seg }, \
+    { #LName "_base", DBGFREG_##UName##_ATTR, DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, LName##Hid.u64Base),  NULL, NULL, NULL, NULL }, \
+    { #LName "_lim",  DBGFREG_##UName##_ATTR, DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, LName##Hid.u32Limit), NULL, NULL, NULL, NULL }
+    DBGFREGDESC_SEG(CS, cs),
+    DBGFREGDESC_SEG(DS, ds),
+    DBGFREGDESC_SEG(ES, es),
+    DBGFREGDESC_SEG(FS, fs),
+    DBGFREGDESC_SEG(GS, gs),
+    DBGFREGDESC_SEG(SS, ss),
+    DBGFREGDESC_REG(RIP, rip),
+    { "rflags",     DBGFREG_RFLAGS,     DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, rflags),           NULL, NULL, g_aDbgfRegAliases_rflags, g_aDbgfRegFields_rflags },
+    { "fcw",        DBGFREG_FCW,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FCW),          NULL, NULL, NULL, g_aDbgfRegFields_fcw },
+    { "fsw",        DBGFREG_FSW,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FSW),          NULL, NULL, NULL, g_aDbgfRegFields_fsw },
+    { "ftw",        DBGFREG_FTW,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FTW), dbgfR3RegGet_ftw, dbgfR3RegSet_ftw, NULL, g_aDbgfRegFields_ftw },
+    { "fop",        DBGFREG_FOP,        DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.FOP),          NULL, NULL, NULL, NULL },
+    { "fpuip",      DBGFREG_FPUIP,      DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.FPUIP),        NULL, NULL, g_aDbgfRegAliases_fpuip, NULL },
+    { "fpucs",      DBGFREG_FPUCS,      DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.CS),           NULL, NULL, NULL, NULL },
+    { "fpudp",      DBGFREG_FPUDP,      DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.FPUDP),        NULL, NULL, g_aDbgfRegAliases_fpudp, NULL },
+    { "fpuds",      DBGFREG_FPUDS,      DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, fpu.DS),           NULL, NULL, NULL, NULL },
+    { "mxcsr",      DBGFREG_MXCSR,      DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.MXCSR),        NULL, NULL, NULL, g_aDbgfRegFields_mxcsr },
+    { "mxcsr_mask", DBGFREG_MXCSR_MASK, DBGFREGVALTYPE_U32, RT_OFFSETOF(CPUMCTX, fpu.MXCSR_MASK),   NULL, NULL, NULL, g_aDbgfRegFields_mxcsr },
+#define DBGFREGDESC_ST(n) \
+    { "st" #n,      DBGFREG_ST##n,      DBGFREGVALTYPE_80, ~(size_t)0, dbgfR3RegGet_stN, dbgfR3RegSet_stN, NULL, g_aDbgfRegFields_stN }
+    DBGFREGDESC_ST(0),
+    DBGFREGDESC_ST(1),
+    DBGFREGDESC_ST(2),
+    DBGFREGDESC_ST(3),
+    DBGFREGDESC_ST(4),
+    DBGFREGDESC_ST(5),
+    DBGFREGDESC_ST(6),
+    DBGFREGDESC_ST(7),
+#define DBGFREGDESC_MM(n) \
+    { "mm" #n,      DBGFREG_MM##n,      DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, fpu.aRegs[n].mmx), NULL, NULL, NULL, g_aDbgfRegFields_mmN }
+    DBGFREGDESC_MM(0),
+    DBGFREGDESC_MM(1),
+    DBGFREGDESC_MM(2),
+    DBGFREGDESC_MM(3),
+    DBGFREGDESC_MM(4),
+    DBGFREGDESC_MM(5),
+    DBGFREGDESC_MM(6),
+    DBGFREGDESC_MM(7),
+#define DBGFREGDESC_XMM(n) \
+    { "xmm" #n,     DBGFREG_XMM##n,     DBGFREGVALTYPE_U128, RT_OFFSETOF(CPUMCTX, fpu.aXMM[n].xmm), NULL, NULL, NULL, g_aDbgfRegFields_xmmN }
+    DBGFREGDESC_XMM(0),
+    DBGFREGDESC_XMM(1),
+    DBGFREGDESC_XMM(2),
+    DBGFREGDESC_XMM(3),
+    DBGFREGDESC_XMM(4),
+    DBGFREGDESC_XMM(5),
+    DBGFREGDESC_XMM(6),
+    DBGFREGDESC_XMM(7),
+    DBGFREGDESC_XMM(8),
+    DBGFREGDESC_XMM(9),
+    DBGFREGDESC_XMM(10),
+    DBGFREGDESC_XMM(11),
+    DBGFREGDESC_XMM(12),
+    DBGFREGDESC_XMM(13),
+    DBGFREGDESC_XMM(14),
+    DBGFREGDESC_XMM(15),
+    { "gdtr_base",  DBGFREG_GDTR_BASE,      DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, gdtr.pGdt),           NULL, NULL, NULL, NULL },
+    { "gdtr_limit", DBGFREG_GDTR_LIMIT,     DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, gdtr.cbGdt),          NULL, NULL, NULL, NULL },
+    { "idtr_base",  DBGFREG_IDTR_BASE,      DBGFREGVALTYPE_U64, RT_OFFSETOF(CPUMCTX, idtr.pIdt),           NULL, NULL, NULL, NULL },
+    { "idtr_limit", DBGFREG_IDTR_LIMIT,     DBGFREGVALTYPE_U16, RT_OFFSETOF(CPUMCTX, idtr.cbIdt),          NULL, NULL, NULL, NULL },
+    DBGFREGDESC_SEG(LDTR, ldtr),
+    DBGFREGDESC_SEG(TR, tr),
+    { "cr0",        DBGFREG_CR0,       DBGFREGVALTYPE_U32, 0, dbgfR3RegGet_crX, dbgfR3RegSet_crX, g_aDbgfRegAliases_cr0, g_aDbgfRegFields_cr0 },
+    { "cr2",        DBGFREG_CR2,       DBGFREGVALTYPE_U64, 2, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, NULL },
+    { "cr3",        DBGFREG_CR3,       DBGFREGVALTYPE_U64, 3, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, g_aDbgfRegFields_cr3 },
+    { "cr4",        DBGFREG_CR4,       DBGFREGVALTYPE_U32, 4, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, g_aDbgfRegFields_cr4 },
+    { "cr8",        DBGFREG_CR8,       DBGFREGVALTYPE_U32, 8, dbgfR3RegGet_crX, dbgfR3RegSet_crX, NULL, NULL },
+    { "dr0",        DBGFREG_DR0,       DBGFREGVALTYPE_U64, 0, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
+    { "dr1",        DBGFREG_DR1,       DBGFREGVALTYPE_U64, 1, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
+    { "dr2",        DBGFREG_DR2,       DBGFREGVALTYPE_U64, 2, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
+    { "dr3",        DBGFREG_DR3,       DBGFREGVALTYPE_U64, 3, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, NULL },
+    { "dr6",        DBGFREG_DR6,       DBGFREGVALTYPE_U32, 6, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, g_aDbgfRegFields_dr6 },
+    { "dr7",        DBGFREG_DR7,       DBGFREGVALTYPE_U32, 7, dbgfR3RegGet_drX, dbgfR3RegSet_drX, NULL, g_aDbgfRegFields_dr7 },
+    { "apic_base",    DBGFREG_MSR_IA32_APICBASE,      DBGFREGVALTYPE_U32, MSR_IA32_APICBASE,      dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_apic_base },
+    { "pat",          DBGFREG_MSR_IA32_CR_PAT,        DBGFREGVALTYPE_U64, MSR_IA32_CR_PAT,        dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_cr_pat },
+    { "perf_status",  DBGFREG_MSR_IA32_PERF_STATUS,   DBGFREGVALTYPE_U64, MSR_IA32_PERF_STATUS,   dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_perf_status },
+    { "sysenter_cs",  DBGFREG_MSR_IA32_SYSENTER_CS,   DBGFREGVALTYPE_U16, MSR_IA32_SYSENTER_CS,   dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "sysenter_eip", DBGFREG_MSR_IA32_SYSENTER_EIP,  DBGFREGVALTYPE_U32, MSR_IA32_SYSENTER_EIP,  dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "sysenter_esp", DBGFREG_MSR_IA32_SYSENTER_ESP,  DBGFREGVALTYPE_U32, MSR_IA32_SYSENTER_ESP,  dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "tsc",          DBGFREG_MSR_IA32_TSC,           DBGFREGVALTYPE_U32, MSR_IA32_TSC,           dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "efer",         DBGFREG_MSR_K6_EFER,            DBGFREGVALTYPE_U32, MSR_K6_EFER,            dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_efer },
+    { "star",         DBGFREG_MSR_K6_STAR,            DBGFREGVALTYPE_U64, MSR_K6_STAR,            dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_star },
+    { "cstar",        DBGFREG_MSR_K8_CSTAR,           DBGFREGVALTYPE_U64, MSR_K8_CSTAR,           dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_cstar },
+    { "msr_fs_base",  DBGFREG_MSR_K8_FS_BASE,         DBGFREGVALTYPE_U64, MSR_K8_FS_BASE,         dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "msr_gs_base",  DBGFREG_MSR_K8_GS_BASE,         DBGFREGVALTYPE_U64, MSR_K8_GS_BASE,         dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "krnl_gs_base", DBGFREG_MSR_K8_KERNEL_GS_BASE,  DBGFREGVALTYPE_U64, MSR_K8_KERNEL_GS_BASE,  dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "lstar",        DBGFREG_MSR_K8_LSTAR,           DBGFREGVALTYPE_U64, MSR_K8_LSTAR,           dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, g_aDbgfRegFields_lstar },
+    { "tsc_aux",      DBGFREG_MSR_K8_TSC_AUX,         DBGFREGVALTYPE_U64, MSR_K8_TSC_AUX,         dbgfR3RegGet_msr, dbgfR3RegSet_msr, NULL, NULL },
+    { "ah",           DBGFREG_AH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rax) + 1,  NULL, NULL, NULL, NULL },
+    { "ch",           DBGFREG_CH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rcx) + 1,  NULL, NULL, NULL, NULL },
+    { "dh",           DBGFREG_DH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rdx) + 1,  NULL, NULL, NULL, NULL },
+    { "bh",           DBGFREG_BH,       DBGFREGVALTYPE_U8,  RT_OFFSETOF(CPUMCTX, rbx) + 1,  NULL, NULL, NULL, NULL },
+    { "gdtr",         DBGFREG_GDTR,     DBGFREGVALTYPE_DTR, ~(size_t)0, dbgfR3RegGet_gdtr, dbgfR3RegSet_gdtr, NULL, NULL },
+    { "idtr",         DBGFREG_IDTR,     DBGFREGVALTYPE_DTR, ~(size_t)0, dbgfR3RegGet_idtr, dbgfR3RegSet_idtr, NULL, NULL },
+#undef DBGFREGDESC_REG
+#undef DBGFREGDESC_SEG
+#undef DBGFREGDESC_ST
+#undef DBGFREGDESC_MM
+#undef DBGFREGDESC_XMM
+};
+
+
+static DECLCALLBACK(int) dbgfR3RegSet_seg(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegGet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegSet_crX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegGet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegSet_drX(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegGet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegSet_msr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegGet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegSet_gdtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegGet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegSet_idtr(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegGet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegSet_ftw(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegGet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCCPUMCTX pCtx, PRTUINT128U puValue)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+static DECLCALLBACK(int) dbgfR3RegSet_stN(PVMCPU pVCpu, struct DBGFREGDESC const *pDesc, PCPUMCTX pCtx, RTUINT128U uValue, RTUINT128U fMask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+
+/**
+ * Wrapper around CPUMQueryGuestMsr for dbgfR3RegQueryWorker.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_DBGF_INVALID_REGISTER
+ *
+ * @param   pVCpu               The current CPU.
+ * @param   pu64                Where to store the register value.
+ * @param   pfRegSizes          Where to store the register sizes.
+ * @param   idMsr               The MSR to get.
+ */
+static uint64_t dbgfR3RegGetMsr(PVMCPU pVCpu, uint64_t *pu64, uint32_t *pfRegSizes, uint32_t idMsr)
+{
+    *pfRegSizes = R_SZ_64;
+    int rc = CPUMQueryGuestMsr(pVCpu, idMsr, pu64);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", rc));
+        *pu64 = 0;
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * Worker for DBGFR3RegQueryU8, DBGFR3RegQueryU16, DBGFR3RegQueryU32 and
+ * DBGFR3RegQueryU64.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu64                Where to store the register value.
+ * @param   pfRegSizes          Where to store the register sizes.
+ */
+static DECLCALLBACK(int) dbgfR3RegQueryWorker(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint64_t *pu64, uint32_t *pfRegSizes)
+{
+/** @todo rewrite this to use the register descriptors. */
+    PVMCPU    pVCpu = &pVM->aCpus[idCpu];
+    PCCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
+    switch (enmReg)
+    {
+        case DBGFREG_RAX:       *pu64 = pCtx->rax;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_RCX:       *pu64 = pCtx->rcx;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_RDX:       *pu64 = pCtx->rdx;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_RBX:       *pu64 = pCtx->rbx;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_RSP:       *pu64 = pCtx->rsp;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_RBP:       *pu64 = pCtx->rbp;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_RSI:       *pu64 = pCtx->rsi;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_RDI:       *pu64 = pCtx->rdi;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R8:        *pu64 = pCtx->r8;   *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R9:        *pu64 = pCtx->r9;   *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R10:       *pu64 = pCtx->r10;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R11:       *pu64 = pCtx->r11;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R12:       *pu64 = pCtx->r12;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R13:       *pu64 = pCtx->r13;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R14:       *pu64 = pCtx->r14;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+        case DBGFREG_R15:       *pu64 = pCtx->r15;  *pfRegSizes = R_SZ_8_TO_64;         return VINF_SUCCESS;
+
+        case DBGFREG_CS:        *pu64 = pCtx->cs;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
+        case DBGFREG_DS:        *pu64 = pCtx->ds;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
+        case DBGFREG_ES:        *pu64 = pCtx->es;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
+        case DBGFREG_FS:        *pu64 = pCtx->fs;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
+        case DBGFREG_GS:        *pu64 = pCtx->gs;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
+        case DBGFREG_SS:        *pu64 = pCtx->ss;   *pfRegSizes = R_SZ_16;              return VINF_SUCCESS;
+
+        case DBGFREG_CS_ATTR:   *pu64 = pCtx->csHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
+        case DBGFREG_DS_ATTR:   *pu64 = pCtx->dsHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
+        case DBGFREG_ES_ATTR:   *pu64 = pCtx->esHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
+        case DBGFREG_FS_ATTR:   *pu64 = pCtx->fsHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
+        case DBGFREG_GS_ATTR:   *pu64 = pCtx->gsHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
+        case DBGFREG_SS_ATTR:   *pu64 = pCtx->ssHid.Attr.u;  *pfRegSizes = R_SZ_32;     return VINF_SUCCESS;
+
+        case DBGFREG_CS_BASE:   *pu64 = pCtx->csHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
+        case DBGFREG_DS_BASE:   *pu64 = pCtx->dsHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
+        case DBGFREG_ES_BASE:   *pu64 = pCtx->esHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
+        case DBGFREG_FS_BASE:   *pu64 = pCtx->fsHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
+        case DBGFREG_GS_BASE:   *pu64 = pCtx->gsHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
+        case DBGFREG_SS_BASE:   *pu64 = pCtx->ssHid.u64Base; *pfRegSizes = R_SZ_64;     return VINF_SUCCESS;
+
+        case DBGFREG_CS_LIMIT:  *pu64 = pCtx->csHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
+        case DBGFREG_DS_LIMIT:  *pu64 = pCtx->dsHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
+        case DBGFREG_ES_LIMIT:  *pu64 = pCtx->esHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
+        case DBGFREG_FS_LIMIT:  *pu64 = pCtx->fsHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
+        case DBGFREG_GS_LIMIT:  *pu64 = pCtx->gsHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
+        case DBGFREG_SS_LIMIT:  *pu64 = pCtx->ssHid.u32Limit; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
+
+        case DBGFREG_RIP:       *pu64 = pCtx->rip;      *pfRegSizes = R_SZ_16_TO_64;    return VINF_SUCCESS;
+        case DBGFREG_FLAGS:     *pu64 = pCtx->rflags.u; *pfRegSizes = R_SZ_16_TO_64;    return VINF_SUCCESS;
+
+        /** @todo check fxsave/fsave format? */
+        case DBGFREG_FCW:       *pu64 = pCtx->fpu.FCW;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_FSW:       *pu64 = pCtx->fpu.FSW;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_FTW:       *pu64 = pCtx->fpu.FTW;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_FOP:       *pu64 = pCtx->fpu.FOP;      *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_FPUIP:     *pu64 = pCtx->fpu.FPUIP;    *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_FPUCS:     *pu64 = pCtx->fpu.CS;       *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_FPUDP:     *pu64 = pCtx->fpu.FPUDP;    *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_FPUDS:     *pu64 = pCtx->fpu.DS;       *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_MXCSR:     *pu64 = pCtx->fpu.MXCSR;    *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_MXCSR_MASK:*pu64 = pCtx->fpu.MXCSR_MASK; *pfRegSizes = R_SZ_32;    return VINF_SUCCESS;
+
+        case DBGFREG_ST0:
+        case DBGFREG_ST1:
+        case DBGFREG_ST2:
+        case DBGFREG_ST3:
+        case DBGFREG_ST4:
+        case DBGFREG_ST5:
+        case DBGFREG_ST6:
+        case DBGFREG_ST7:
+        {
+            /** @todo check tags and fxsave/fsave format. */
+            unsigned iReg = (pCtx->fpu.FSW >> 11) & 7;
+            iReg += enmReg - DBGFREG_ST0;
+            iReg &= 7;
+
+            *pu64 = pCtx->fpu.aRegs[iReg].au64[0];
+            *pfRegSizes = R_SZ_80;
+            return VINF_SUCCESS;
+        }
+
+        case DBGFREG_MM0:
+        case DBGFREG_MM1:
+        case DBGFREG_MM2:
+        case DBGFREG_MM3:
+        case DBGFREG_MM4:
+        case DBGFREG_MM5:
+        case DBGFREG_MM6:
+        case DBGFREG_MM7:
+            *pu64 = pCtx->fpu.aRegs[enmReg - DBGFREG_MM0].mmx;
+            *pfRegSizes = R_SZ_64;
+            return VINF_SUCCESS;
+
+        case DBGFREG_XMM0:
+        case DBGFREG_XMM1:
+        case DBGFREG_XMM2:
+        case DBGFREG_XMM3:
+        case DBGFREG_XMM4:
+        case DBGFREG_XMM5:
+        case DBGFREG_XMM6:
+        case DBGFREG_XMM7:
+        case DBGFREG_XMM8:
+        case DBGFREG_XMM9:
+        case DBGFREG_XMM10:
+        case DBGFREG_XMM11:
+        case DBGFREG_XMM12:
+        case DBGFREG_XMM13:
+        case DBGFREG_XMM14:
+        case DBGFREG_XMM15:
+            *pu64 = pCtx->fpu.aRegs[enmReg - DBGFREG_XMM0].au64[0];
+            *pfRegSizes = R_SZ_128;
+            return VINF_SUCCESS;
+
+        case DBGFREG_GDTR_BASE:     *pu64 = pCtx->gdtr.pGdt;        *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
+        case DBGFREG_GDTR_LIMIT:    *pu64 = pCtx->gdtr.cbGdt;       *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_IDTR_BASE:     *pu64 = pCtx->idtr.pIdt;        *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
+        case DBGFREG_IDTR_LIMIT:    *pu64 = pCtx->idtr.cbIdt;       *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_LDTR:          *pu64 = pCtx->ldtr;             *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_LDTR_ATTR:     *pu64 = pCtx->ldtrHid.Attr.u;   *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_LDTR_BASE:     *pu64 = pCtx->ldtrHid.u64Base;  *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
+        case DBGFREG_LDTR_LIMIT:    *pu64 = pCtx->ldtrHid.u32Limit; *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_TR:            *pu64 = pCtx->tr;               *pfRegSizes = R_SZ_16;      return VINF_SUCCESS;
+        case DBGFREG_TR_ATTR:       *pu64 = pCtx->trHid.Attr.u;     *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+        case DBGFREG_TR_BASE:       *pu64 = pCtx->trHid.u64Base;    *pfRegSizes = R_SZ_64;      return VINF_SUCCESS;
+        case DBGFREG_TR_LIMIT:      *pu64 = pCtx->trHid.u32Limit;   *pfRegSizes = R_SZ_32;      return VINF_SUCCESS;
+
+        case DBGFREG_CR0:           *pu64 = CPUMGetGuestCR0(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_CR2:           *pu64 = CPUMGetGuestCR2(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_CR3:           *pu64 = CPUMGetGuestCR3(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_CR4:           *pu64 = CPUMGetGuestCR4(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_CR8:           *pu64 = CPUMGetGuestCR8(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+
+        case DBGFREG_DR0:           *pu64 = CPUMGetGuestDR0(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_DR1:           *pu64 = CPUMGetGuestDR1(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_DR2:           *pu64 = CPUMGetGuestDR2(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_DR3:           *pu64 = CPUMGetGuestDR3(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_DR6:           *pu64 = CPUMGetGuestDR6(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+        case DBGFREG_DR7:           *pu64 = CPUMGetGuestDR7(pVCpu); *pfRegSizes = R_SZ_32_OR_64; return VINF_SUCCESS;
+
+        case DBGFREG_MSR_IA32_APICBASE:     return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_APICBASE);
+        case DBGFREG_MSR_IA32_CR_PAT:       return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_CR_PAT);
+        case DBGFREG_MSR_IA32_PERF_STATUS:  return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_PERF_STATUS);
+        case DBGFREG_MSR_IA32_SYSENTER_CS:  return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_SYSENTER_CS);
+        case DBGFREG_MSR_IA32_SYSENTER_EIP: return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_SYSENTER_EIP);
+        case DBGFREG_MSR_IA32_SYSENTER_ESP: return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_SYSENTER_ESP);
+        case DBGFREG_MSR_IA32_TSC:          return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_IA32_TSC);
+        case DBGFREG_MSR_K6_EFER:           return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K6_EFER);
+        case DBGFREG_MSR_K6_STAR:           return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K6_STAR);
+        case DBGFREG_MSR_K8_CSTAR:          return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_CSTAR);
+        case DBGFREG_MSR_K8_FS_BASE:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_FS_BASE);
+        case DBGFREG_MSR_K8_GS_BASE:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_GS_BASE);
+        case DBGFREG_MSR_K8_KERNEL_GS_BASE: return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_KERNEL_GS_BASE);
+        case DBGFREG_MSR_K8_LSTAR:          return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_LSTAR);
+        case DBGFREG_MSR_K8_SF_MASK:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_SF_MASK);
+        case DBGFREG_MSR_K8_TSC_AUX:        return dbgfR3RegGetMsr(pVCpu, pu64, pfRegSizes, MSR_K8_TSC_AUX);
+
+        case DBGFREG_AH:        *pu64 = RT_BYTE2(pCtx->ax); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
+        case DBGFREG_CH:        *pu64 = RT_BYTE2(pCtx->cx); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
+        case DBGFREG_DH:        *pu64 = RT_BYTE2(pCtx->dx); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
+        case DBGFREG_BH:        *pu64 = RT_BYTE2(pCtx->bx); *pfRegSizes = R_SZ_8;       return VINF_SUCCESS;
+        case DBGFREG_GDTR:      *pu64 = pCtx->gdtr.pGdt;    *pfRegSizes = R_SZ_64_16;   return VINF_SUCCESS;
+        case DBGFREG_IDTR:      *pu64 = pCtx->idtr.pIdt;    *pfRegSizes = R_SZ_64_16;   return VINF_SUCCESS;
+
+        case DBGFREG_END:
+        case DBGFREG_32BIT_HACK:
+            /* no default! We want GCC warnings. */
+            break;
+    }
+
+    AssertMsgFailed(("%d (%#x)\n", enmReg, enmReg));
+    return VERR_DBGF_INVALID_REGISTER;
+}
+
+
+/**
+ * Queries a 8-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_INVALID_REGISTER
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ *
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu8                 Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegQueryU8(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint8_t *pu8)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    uint64_t u64Value;
+    uint32_t fRegSizes;
+    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
+    if (RT_SUCCESS(rc))
+    {
+        *pu8 = (uint8_t)u64Value;
+        if (R_SZ_8 & fRegSizes)
+            rc = VINF_SUCCESS;
+        else
+            rc = VINF_DBGF_TRUNCATED_REGISTER;
+    }
+    else
+        *pu8 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 16-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_INVALID_REGISTER
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu16                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegQueryU16(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint16_t *pu16)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    uint64_t u64Value;
+    uint32_t fRegSizes;
+    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
+    if (RT_SUCCESS(rc))
+    {
+        *pu16 = (uint16_t)u64Value;
+        if (R_SZ_16 & fRegSizes)
+            rc = VINF_SUCCESS;
+        else if (~(R_SZ_8 | R_SZ_16) & fRegSizes)
+            rc = VINF_DBGF_TRUNCATED_REGISTER;
+        else
+            rc = VINF_DBGF_ZERO_EXTENDED_REGISTER;
+    }
+    else
+        *pu16 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 32-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_INVALID_REGISTER
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu32                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegQueryU32(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint32_t *pu32)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    uint64_t u64Value;
+    uint32_t fRegSizes;
+    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
+    if (RT_SUCCESS(rc))
+    {
+        *pu32 = (uint32_t)u64Value;
+        if (R_SZ_32 & fRegSizes)
+            rc = VINF_SUCCESS;
+        else if (~(R_SZ_8 | R_SZ_16 | R_SZ_32) & fRegSizes)
+            rc = VINF_DBGF_TRUNCATED_REGISTER;
+        else
+            rc = VINF_DBGF_ZERO_EXTENDED_REGISTER;
+    }
+    else
+        *pu32 = 0;
+    return rc;
+}
+
+
+/**
+ * Queries a 64-bit register value.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_INVALID_REGISTER
+ * @retval  VINF_DBGF_TRUNCATED_REGISTER
+ * @retval  VINF_DBGF_ZERO_EXTENDED_REGISTER
+ *
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ * @param   enmReg              The register that's being queried.
+ * @param   pu64                Where to store the register value.
+ */
+VMMR3DECL(int) DBGFR3RegQueryU64(PVM pVM, VMCPUID idCpu, DBGFREG enmReg, uint64_t *pu64)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    uint64_t u64Value;
+    uint32_t fRegSizes;
+    int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryWorker, 5, pVM, idCpu, enmReg, &u64Value, &fRegSizes);
+    if (RT_SUCCESS(rc))
+    {
+        *pu64 = u64Value;
+        if (R_SZ_64 & fRegSizes)
+            rc = VINF_SUCCESS;
+        else if (~(R_SZ_8 | R_SZ_16 | R_SZ_32 | R_SZ_64) & fRegSizes)
+            rc = VINF_DBGF_TRUNCATED_REGISTER;
+        else
+            rc = VINF_DBGF_ZERO_EXTENDED_REGISTER;
+    }
+    else
+        *pu64 = 0;
+    return rc;
+}
+
+
+/**
+ * Wrapper around CPUMQueryGuestMsr for dbgfR3RegQueryBatchWorker.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_DBGF_INVALID_REGISTER
+ *
+ * @param   pVCpu               The current CPU.
+ * @param   pReg                The where to store the register value and
+ *                              size.
+ * @param   idMsr               The MSR to get.
+ */
+static void dbgfR3RegGetMsrBatch(PVMCPU pVCpu, PDBGFREGENTRY pReg, uint32_t idMsr)
+{
+    pReg->enmType = DBGFREGVALTYPE_U64;
+    int rc = CPUMQueryGuestMsr(pVCpu, idMsr, &pReg->Val.u64);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", rc));
+        pReg->Val.u64 = 0;
+    }
+}
+
+
+static DECLCALLBACK(int) dbgfR3RegQueryBatchWorker(PVM pVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
+{
+    PVMCPU    pVCpu = &pVM->aCpus[idCpu];
+    PCCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
+
+    PDBGFREGENTRY pReg = paRegs - 1;
+    while (cRegs-- > 0)
+    {
+        pReg++;
+        pReg->Val.au64[0] = 0;
+        pReg->Val.au64[1] = 0;
+
+        DBGFREG const enmReg = pReg->enmReg;
+        AssertMsgReturn(enmReg >= 0 && enmReg <= DBGFREG_END, ("%d (%#x)\n", enmReg, enmReg), VERR_DBGF_INVALID_REGISTER);
+        if (enmReg != DBGFREG_END)
+        {
+            PCDBGFREGDESC pDesc = &g_aDbgfRegDescs[enmReg];
+            if (!pDesc->pfnGet)
+            {
+                PCRTUINT128U pu = (PCRTUINT128U)((uintptr_t)pCtx + pDesc->offCtx);
+                pReg->enmType = pDesc->enmType;
+                switch (pDesc->enmType)
+                {
+                    case DBGFREGVALTYPE_U8:     pReg->Val.u8   = pu->au8[0];   break;
+                    case DBGFREGVALTYPE_U16:    pReg->Val.u16  = pu->au16[0];  break;
+                    case DBGFREGVALTYPE_U32:    pReg->Val.u32  = pu->au32[0];  break;
+                    case DBGFREGVALTYPE_U64:    pReg->Val.u64  = pu->au64[0];  break;
+                    case DBGFREGVALTYPE_U128:
+                        pReg->Val.au64[0] = pu->au64[0];
+                        pReg->Val.au64[1] = pu->au64[1];
+                        break;
+                    case DBGFREGVALTYPE_LRD:
+                        pReg->Val.au64[0] = pu->au64[0];
+                        pReg->Val.au16[5] = pu->au16[5];
+                        break;
+                    default:
+                        AssertMsgFailedReturn(("%s %d\n", pDesc->pszName, pDesc->enmType), VERR_INTERNAL_ERROR_3);
+                }
+            }
+            else
+            {
+                int rc = pDesc->pfnGet(pVCpu, pDesc, pCtx, &pReg->Val.u);
+                if (RT_FAILURE(rc))
+                    return rc;
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Query a batch of registers.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ * @retval  VERR_DBGF_INVALID_REGISTER
+ *
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ * @param   paRegs              Pointer to an array of @a cRegs elements.  On
+ *                              input the enmReg members indicates which
+ *                              registers to query.  On successful return the
+ *                              other members are set.  DBGFREG_END can be used
+ *                              as a filler.
+ * @param   cRegs               The number of entries in @a paRegs.
+ */
+VMMR3DECL(int) DBGFR3RegQueryBatch(PVM pVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    if (!cRegs)
+        return VINF_SUCCESS;
+    AssertReturn(cRegs < _1M, VERR_OUT_OF_RANGE);
+    AssertPtrReturn(paRegs, VERR_INVALID_POINTER);
+    size_t iReg = cRegs;
+    while (iReg-- > 0)
+    {
+        DBGFREG enmReg = paRegs[iReg].enmReg;
+        AssertMsgReturn(enmReg < DBGFREG_END && enmReg >= DBGFREG_AL, ("%d (%#x)", enmReg, enmReg), VERR_DBGF_INVALID_REGISTER);
+    }
+
+    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryBatchWorker, 4, pVM, idCpu, paRegs, cRegs);
+}
+
+
+/**
+ * Query a all registers for a Virtual CPU.
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_INVALID_VM_HANDLE
+ * @retval  VERR_INVALID_CPU_ID
+ *
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The target CPU ID.
+ * @param   paRegs              Pointer to an array of @a cRegs elements.
+ *                              These will be filled with the CPU register
+ *                              values. Overflowing entries will be set to
+ *                              DBGFREG_END.  The returned registers can be
+ *                              accessed by using the DBGFREG values as index.
+ * @param   cRegs               The number of entries in @a paRegs.  The
+ *                              recommended value is DBGFREG_ALL_COUNT.
+ */
+VMMR3DECL(int) DBGFR3RegQueryAll(PVM pVM, VMCPUID idCpu, PDBGFREGENTRY paRegs, size_t cRegs)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    if (!cRegs)
+        return VINF_SUCCESS;
+    AssertReturn(cRegs < _1M, VERR_OUT_OF_RANGE);
+    AssertPtrReturn(paRegs, VERR_INVALID_POINTER);
+
+    /*
+     * Convert it into a batch query (lazy bird).
+     */
+    unsigned iReg = 0;
+    while (iReg < cRegs && iReg < DBGFREG_ALL_COUNT)
+    {
+        paRegs[iReg].enmReg = (DBGFREG)iReg;
+        iReg++;
+    }
+    while (iReg < cRegs)
+        paRegs[iReg++].enmReg = DBGFREG_END;
+
+    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3RegQueryBatchWorker, 4, pVM, idCpu, paRegs, cRegs);
+}
+
+
+
+/**
+ * Gets the name of a register.
+ *
+ * @returns Pointer to read-only register name (lower case).  NULL if the
+ *          parameters are invalid.
+ * @param   enmReg              The register identifier.
+ * @param   enmType             The register type.  This is for sort out
+ *                              aliases.  Pass DBGFREGVALTYPE_INVALID to get
+ *                              the standard name.
+ */
+VMMR3DECL(const char *) DBGFR3RegName(DBGFREG enmReg, DBGFREGVALTYPE enmType)
+{
+    AssertReturn(enmReg >= DBGFREG_AL && enmReg < DBGFREG_END, NULL);
+    AssertReturn(enmType >= DBGFREGVALTYPE_INVALID && enmType < DBGFREGVALTYPE_END, NULL);
+
+    PCDBGFREGDESC   pDesc  = &g_aDbgfRegDescs[enmReg];
+    PCDBGFREGALIAS  pAlias = pDesc->paAliases;
+    if (   pAlias
+        && pDesc->enmType != enmType
+        && enmType != DBGFREGVALTYPE_INVALID)
+    {
+        while (pAlias->pszName)
+        {
+            if (pAlias->enmType == enmType)
+                return pAlias->pszName;
+            pAlias++;
+        }
+    }
+
+    return pDesc->pszName;
+}
+
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFStack.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFStack.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFStack.cpp	(revision 35333)
@@ -0,0 +1,585 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Call Stack Analyser.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include <VBox/dbgf.h>
+#include <VBox/selm.h>
+#include <VBox/mm.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/param.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/alloca.h>
+
+
+
+/**
+ * Read stack memory.
+ */
+DECLINLINE(int) dbgfR3Read(PVM pVM, VMCPUID idCpu, void *pvBuf, PCDBGFADDRESS pSrcAddr, size_t cb, size_t *pcbRead)
+{
+    int rc = DBGFR3MemRead(pVM, idCpu, pSrcAddr, pvBuf, cb);
+    if (RT_FAILURE(rc))
+    {
+        /* fallback: byte by byte and zero the ones we fail to read. */
+        size_t cbRead;
+        for (cbRead = 0; cbRead < cb; cbRead++)
+        {
+            DBGFADDRESS Addr = *pSrcAddr;
+            rc = DBGFR3MemRead(pVM, idCpu, DBGFR3AddrAdd(&Addr, cbRead), (uint8_t *)pvBuf + cbRead, 1);
+            if (RT_FAILURE(rc))
+                break;
+        }
+        if (cbRead)
+            rc = VINF_SUCCESS;
+        memset((char *)pvBuf + cbRead, 0, cb - cbRead);
+        *pcbRead = cbRead;
+    }
+    else
+        *pcbRead = cb;
+    return rc;
+}
+
+
+/**
+ * Internal worker routine.
+ *
+ * On x86 the typical stack frame layout is like this:
+ *     ..  ..
+ *     16  parameter 2
+ *     12  parameter 1
+ *      8  parameter 0
+ *      4  return address
+ *      0  old ebp; current ebp points here
+ *
+ * @todo Add AMD64 support (needs teaming up with the module management for
+ *       unwind tables).
+ */
+static int dbgfR3StackWalk(PVM pVM, VMCPUID idCpu, RTDBGAS hAs, PDBGFSTACKFRAME pFrame)
+{
+    /*
+     * Stop if we got a read error in the previous run.
+     */
+    if (pFrame->fFlags & DBGFSTACKFRAME_FLAGS_LAST)
+        return VERR_NO_MORE_FILES;
+
+    /*
+     * Read the raw frame data.
+     */
+    const DBGFADDRESS AddrOldPC = pFrame->AddrPC;
+    const unsigned cbRetAddr = DBGFReturnTypeSize(pFrame->enmReturnType);
+    unsigned cbStackItem;
+    switch (AddrOldPC.fFlags & DBGFADDRESS_FLAGS_TYPE_MASK)
+    {
+        case DBGFADDRESS_FLAGS_FAR16: cbStackItem = 2; break;
+        case DBGFADDRESS_FLAGS_FAR32: cbStackItem = 4; break;
+        case DBGFADDRESS_FLAGS_FAR64: cbStackItem = 8; break;
+        case DBGFADDRESS_FLAGS_RING0: cbStackItem = sizeof(RTHCUINTPTR); break;
+        default:
+            switch (pFrame->enmReturnType)
+            {
+                case DBGFRETURNTYPE_FAR16:
+                case DBGFRETURNTYPE_IRET16:
+                case DBGFRETURNTYPE_IRET32_V86:
+                case DBGFRETURNTYPE_NEAR16: cbStackItem = 2; break;
+
+                case DBGFRETURNTYPE_FAR32:
+                case DBGFRETURNTYPE_IRET32:
+                case DBGFRETURNTYPE_IRET32_PRIV:
+                case DBGFRETURNTYPE_NEAR32: cbStackItem = 4; break;
+
+                case DBGFRETURNTYPE_FAR64:
+                case DBGFRETURNTYPE_IRET64:
+                case DBGFRETURNTYPE_NEAR64: cbStackItem = 8; break;
+
+                default:
+                    AssertMsgFailed(("%d\n", pFrame->enmReturnType));
+                    cbStackItem = 4;
+                    break;
+            }
+    }
+
+    union
+    {
+        uint64_t *pu64;
+        uint32_t *pu32;
+        uint16_t *pu16;
+        uint8_t  *pb;
+        void     *pv;
+    } u, uRet, uArgs, uBp;
+    size_t cbRead = cbRetAddr + cbStackItem + sizeof(pFrame->Args);
+    u.pv = alloca(cbRead);
+    uBp = u;
+    uRet.pb = u.pb + cbStackItem;
+    uArgs.pb = u.pb + cbStackItem + cbRetAddr;
+
+    Assert(DBGFADDRESS_IS_VALID(&pFrame->AddrFrame));
+    int rc = dbgfR3Read(pVM, idCpu, u.pv,
+                        pFrame->fFlags & DBGFSTACKFRAME_FLAGS_ALL_VALID
+                        ? &pFrame->AddrReturnFrame
+                        : &pFrame->AddrFrame,
+                        cbRead, &cbRead);
+    if (    RT_FAILURE(rc)
+        ||  cbRead < cbRetAddr + cbStackItem)
+        pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_LAST;
+
+    /*
+     * The first step is taken in a different way than the others.
+     */
+    if (!(pFrame->fFlags & DBGFSTACKFRAME_FLAGS_ALL_VALID))
+    {
+        pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_ALL_VALID;
+        pFrame->iFrame = 0;
+
+        /* Current PC - set by caller, just find symbol & line. */
+        if (DBGFADDRESS_IS_VALID(&pFrame->AddrPC))
+        {
+            pFrame->pSymPC  = DBGFR3AsSymbolByAddrA(pVM, hAs, &pFrame->AddrPC, NULL /*offDisp*/, NULL /*phMod*/);
+            pFrame->pLinePC = DBGFR3LineByAddrAlloc(pVM, pFrame->AddrPC.FlatPtr, NULL);
+        }
+    }
+    else /* 2nd and subsequent steps */
+    {
+        /* frame, pc and stack is taken from the existing frames return members. */
+        pFrame->AddrFrame = pFrame->AddrReturnFrame;
+        pFrame->AddrPC    = pFrame->AddrReturnPC;
+        pFrame->pSymPC    = pFrame->pSymReturnPC;
+        pFrame->pLinePC   = pFrame->pLineReturnPC;
+
+        /* increment the frame number. */
+        pFrame->iFrame++;
+    }
+
+    /*
+     * Return Frame address.
+     */
+    pFrame->AddrReturnFrame = pFrame->AddrFrame;
+    switch (cbStackItem)
+    {
+        case 2:    pFrame->AddrReturnFrame.off = *uBp.pu16; break;
+        case 4:    pFrame->AddrReturnFrame.off = *uBp.pu32; break;
+        case 8:    pFrame->AddrReturnFrame.off = *uBp.pu64; break;
+        default:    AssertMsgFailed(("cbStackItem=%d\n", cbStackItem)); return VERR_INTERNAL_ERROR;
+    }
+    pFrame->AddrReturnFrame.FlatPtr += pFrame->AddrReturnFrame.off - pFrame->AddrFrame.off;
+
+    /*
+     * Return PC and Stack Addresses.
+     */
+    /** @todo AddrReturnStack is not correct for stdcall and pascal. (requires scope info) */
+    pFrame->AddrReturnStack          = pFrame->AddrFrame;
+    pFrame->AddrReturnStack.off     += cbStackItem + cbRetAddr;
+    pFrame->AddrReturnStack.FlatPtr += cbStackItem + cbRetAddr;
+
+    pFrame->AddrReturnPC             = pFrame->AddrPC;
+    switch (pFrame->enmReturnType)
+    {
+        case DBGFRETURNTYPE_NEAR16:
+            if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
+            {
+                pFrame->AddrReturnPC.FlatPtr += *uRet.pu16 - pFrame->AddrReturnPC.off;
+                pFrame->AddrReturnPC.off      = *uRet.pu16;
+            }
+            else
+                DBGFR3AddrFromFlat(pVM, &pFrame->AddrReturnPC, *uRet.pu16);
+            break;
+        case DBGFRETURNTYPE_NEAR32:
+            if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
+            {
+                pFrame->AddrReturnPC.FlatPtr += *uRet.pu32 - pFrame->AddrReturnPC.off;
+                pFrame->AddrReturnPC.off      = *uRet.pu32;
+            }
+            else
+                DBGFR3AddrFromFlat(pVM, &pFrame->AddrReturnPC, *uRet.pu32);
+            break;
+        case DBGFRETURNTYPE_NEAR64:
+            if (DBGFADDRESS_IS_VALID(&pFrame->AddrReturnPC))
+            {
+                pFrame->AddrReturnPC.FlatPtr += *uRet.pu64 - pFrame->AddrReturnPC.off;
+                pFrame->AddrReturnPC.off      = *uRet.pu64;
+            }
+            else
+                DBGFR3AddrFromFlat(pVM, &pFrame->AddrReturnPC, *uRet.pu64);
+            break;
+        case DBGFRETURNTYPE_FAR16:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[1], uRet.pu16[0]);
+            break;
+        case DBGFRETURNTYPE_FAR32:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+            break;
+        case DBGFRETURNTYPE_FAR64:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[4], uRet.pu64[0]);
+            break;
+        case DBGFRETURNTYPE_IRET16:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[1], uRet.pu16[0]);
+            break;
+        case DBGFRETURNTYPE_IRET32:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+            break;
+        case DBGFRETURNTYPE_IRET32_PRIV:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+            break;
+        case DBGFRETURNTYPE_IRET32_V86:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[2], uRet.pu32[0]);
+            break;
+        case DBGFRETURNTYPE_IRET64:
+            DBGFR3AddrFromSelOff(pVM, idCpu, &pFrame->AddrReturnPC, uRet.pu16[4], uRet.pu64[0]);
+            break;
+        default:
+            AssertMsgFailed(("enmReturnType=%d\n", pFrame->enmReturnType));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    pFrame->pSymReturnPC  = DBGFR3AsSymbolByAddrA(pVM, hAs, &pFrame->AddrReturnPC, NULL /*offDisp*/, NULL /*phMod*/);
+    pFrame->pLineReturnPC = DBGFR3LineByAddrAlloc(pVM, pFrame->AddrReturnPC.FlatPtr, NULL);
+
+    /*
+     * Frame bitness flag.
+     */
+    switch (cbStackItem)
+    {
+        case 2: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_16BIT; break;
+        case 4: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_32BIT; break;
+        case 8: pFrame->fFlags |= DBGFSTACKFRAME_FLAGS_64BIT; break;
+        default:    AssertMsgFailed(("cbStackItem=%d\n", cbStackItem)); return VERR_INTERNAL_ERROR;
+    }
+
+    /*
+     * The arguments.
+     */
+    memcpy(&pFrame->Args, uArgs.pv, sizeof(pFrame->Args));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Walks the entire stack allocating memory as we walk.
+ */
+static DECLCALLBACK(int) dbgfR3StackWalkCtxFull(PVM pVM, VMCPUID idCpu, PCCPUMCTXCORE pCtxCore, RTDBGAS hAs,
+                                                DBGFCODETYPE enmCodeType,
+                                                PCDBGFADDRESS pAddrFrame,
+                                                PCDBGFADDRESS pAddrStack,
+                                                PCDBGFADDRESS pAddrPC,
+                                                DBGFRETURNTYPE enmReturnType,
+                                                PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    /* alloc first frame. */
+    PDBGFSTACKFRAME pCur = (PDBGFSTACKFRAME)MMR3HeapAllocZ(pVM, MM_TAG_DBGF_STACK, sizeof(*pCur));
+    if (!pCur)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Initialize the frame.
+     */
+    pCur->pNextInternal = NULL;
+    pCur->pFirstInternal = pCur;
+
+    int rc = VINF_SUCCESS;
+    if (pAddrPC)
+        pCur->AddrPC = *pAddrPC;
+    else
+        rc = DBGFR3AddrFromSelOff(pVM, idCpu, &pCur->AddrPC, pCtxCore->cs, pCtxCore->rip);
+    if (RT_SUCCESS(rc))
+    {
+        if (enmReturnType == DBGFRETURNTYPE_INVALID)
+            switch (pCur->AddrPC.fFlags & DBGFADDRESS_FLAGS_TYPE_MASK)
+            {
+                case DBGFADDRESS_FLAGS_FAR16: pCur->enmReturnType = DBGFRETURNTYPE_NEAR16; break;
+                case DBGFADDRESS_FLAGS_FAR32: pCur->enmReturnType = DBGFRETURNTYPE_NEAR32; break;
+                case DBGFADDRESS_FLAGS_FAR64: pCur->enmReturnType = DBGFRETURNTYPE_NEAR64; break;
+                case DBGFADDRESS_FLAGS_RING0: pCur->enmReturnType = HC_ARCH_BITS == 64 ? DBGFRETURNTYPE_NEAR64 : DBGFRETURNTYPE_NEAR32; break;
+                default:                      pCur->enmReturnType = DBGFRETURNTYPE_NEAR32; break; /// @todo 64-bit guests
+            }
+
+        uint64_t fAddrMask;
+        if (enmCodeType == DBGFCODETYPE_RING0)
+            fAddrMask = HC_ARCH_BITS == 64 ? UINT64_MAX : UINT32_MAX;
+        else if (enmCodeType == DBGFCODETYPE_HYPER)
+            fAddrMask = UINT32_MAX;
+        else if (DBGFADDRESS_IS_FAR16(&pCur->AddrPC))
+            fAddrMask = UINT16_MAX;
+        else if (DBGFADDRESS_IS_FAR32(&pCur->AddrPC))
+            fAddrMask = UINT32_MAX;
+        else if (DBGFADDRESS_IS_FAR64(&pCur->AddrPC))
+            fAddrMask = UINT64_MAX;
+        else
+        {
+            PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+            CPUMMODE CpuMode = CPUMGetGuestMode(pVCpu);
+            if (CpuMode == CPUMMODE_REAL)
+                fAddrMask = UINT16_MAX;
+            else if (   CpuMode == CPUMMODE_PROTECTED
+                     || !CPUMIsGuestIn64BitCode(pVCpu, pCtxCore))
+                fAddrMask = UINT32_MAX;
+            else
+                fAddrMask = UINT64_MAX;
+        }
+
+        if (pAddrStack)
+            pCur->AddrStack = *pAddrStack;
+        else
+            rc = DBGFR3AddrFromSelOff(pVM, idCpu, &pCur->AddrStack, pCtxCore->ss, pCtxCore->rsp & fAddrMask);
+
+        if (pAddrFrame)
+            pCur->AddrFrame = *pAddrFrame;
+        else if (RT_SUCCESS(rc))
+            rc = DBGFR3AddrFromSelOff(pVM, idCpu, &pCur->AddrFrame, pCtxCore->ss, pCtxCore->rbp & fAddrMask);
+    }
+    else
+        pCur->enmReturnType = enmReturnType;
+
+    /*
+     * The first frame.
+     */
+    if (RT_SUCCESS(rc))
+        rc = dbgfR3StackWalk(pVM, idCpu, hAs, pCur);
+    if (RT_FAILURE(rc))
+    {
+        DBGFR3StackWalkEnd(pCur);
+        return rc;
+    }
+
+    /*
+     * The other frames.
+     */
+    DBGFSTACKFRAME Next = *pCur;
+    while (!(pCur->fFlags & (DBGFSTACKFRAME_FLAGS_LAST | DBGFSTACKFRAME_FLAGS_MAX_DEPTH | DBGFSTACKFRAME_FLAGS_LOOP)))
+    {
+        /* try walk. */
+        rc = dbgfR3StackWalk(pVM, idCpu, hAs, &Next);
+        if (RT_FAILURE(rc))
+            break;
+
+        /* add the next frame to the chain. */
+        PDBGFSTACKFRAME pNext = (PDBGFSTACKFRAME)MMR3HeapAlloc(pVM, MM_TAG_DBGF_STACK, sizeof(*pNext));
+        if (!pNext)
+        {
+            DBGFR3StackWalkEnd(pCur);
+            return VERR_NO_MEMORY;
+        }
+        *pNext = Next;
+        pCur->pNextInternal = pNext;
+        pCur = pNext;
+        Assert(pCur->pNextInternal == NULL);
+
+        /* check for loop */
+        for (PCDBGFSTACKFRAME pLoop = pCur->pFirstInternal;
+             pLoop && pLoop != pCur;
+             pLoop = pLoop->pNextInternal)
+            if (pLoop->AddrFrame.FlatPtr == pCur->AddrFrame.FlatPtr)
+            {
+                pCur->fFlags |= DBGFSTACKFRAME_FLAGS_LOOP;
+                break;
+            }
+
+        /* check for insane recursion */
+        if (pCur->iFrame >= 2048)
+            pCur->fFlags |= DBGFSTACKFRAME_FLAGS_MAX_DEPTH;
+    }
+
+    *ppFirstFrame = pCur->pFirstInternal;
+    return rc;
+}
+
+
+/**
+ * Common worker for DBGFR3StackWalkBeginGuestEx, DBGFR3StackWalkBeginHyperEx,
+ * DBGFR3StackWalkBeginGuest and DBGFR3StackWalkBeginHyper.
+ */
+static int dbgfR3StackWalkBeginCommon(PVM pVM,
+                                      VMCPUID idCpu,
+                                      DBGFCODETYPE enmCodeType,
+                                      PCDBGFADDRESS pAddrFrame,
+                                      PCDBGFADDRESS pAddrStack,
+                                      PCDBGFADDRESS pAddrPC,
+                                      DBGFRETURNTYPE enmReturnType,
+                                      PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    /*
+     * Validate parameters.
+     */
+    *ppFirstFrame = NULL;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+    if (pAddrFrame)
+        AssertReturn(DBGFR3AddrIsValid(pVM, pAddrFrame), VERR_INVALID_PARAMETER);
+    if (pAddrStack)
+        AssertReturn(DBGFR3AddrIsValid(pVM, pAddrStack), VERR_INVALID_PARAMETER);
+    if (pAddrPC)
+        AssertReturn(DBGFR3AddrIsValid(pVM, pAddrPC), VERR_INVALID_PARAMETER);
+    AssertReturn(enmReturnType >= DBGFRETURNTYPE_INVALID && enmReturnType < DBGFRETURNTYPE_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Get the CPUM context pointer and pass it on the specified EMT.
+     */
+    RTDBGAS         hAs;
+    PCCPUMCTXCORE   pCtxCore;
+    switch (enmCodeType)
+    {
+        case DBGFCODETYPE_GUEST:
+            pCtxCore = CPUMGetGuestCtxCore(VMMGetCpuById(pVM, idCpu));
+            hAs = DBGF_AS_GLOBAL;
+            break;
+        case DBGFCODETYPE_HYPER:
+            pCtxCore = CPUMGetHyperCtxCore(VMMGetCpuById(pVM, idCpu));
+            hAs = DBGF_AS_RC_AND_GC_GLOBAL;
+            break;
+        case DBGFCODETYPE_RING0:
+            pCtxCore = NULL;    /* No valid context present. */
+            hAs = DBGF_AS_R0;
+            break;
+        default:
+            AssertFailedReturn(VERR_INVALID_PARAMETER);
+    }
+    return VMR3ReqCallWait(pVM, idCpu, (PFNRT)dbgfR3StackWalkCtxFull, 10,
+                           pVM, idCpu, pCtxCore, hAs, enmCodeType,
+                           pAddrFrame, pAddrStack, pAddrPC, enmReturnType, ppFirstFrame);
+}
+
+
+/**
+ * Begins a guest stack walk, extended version.
+ *
+ * This will walk the current stack, constructing a list of info frames which is
+ * returned to the caller. The caller uses DBGFR3StackWalkNext to traverse the
+ * list and DBGFR3StackWalkEnd to release it.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_NO_MEMORY if we're out of memory.
+ *
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The ID of the virtual CPU which stack we want to walk.
+ * @param   enmCodeType     Code type
+ * @param   pAddrFrame      Frame address to start at. (Optional)
+ * @param   pAddrStack      Stack address to start at. (Optional)
+ * @param   pAddrPC         Program counter to start at. (Optional)
+ * @param   enmReturnType   The return address type. (Optional)
+ * @param   ppFirstFrame    Where to return the pointer to the first info frame.
+ */
+VMMR3DECL(int) DBGFR3StackWalkBeginEx(PVM pVM,
+                                      VMCPUID idCpu,
+                                      DBGFCODETYPE enmCodeType,
+                                      PCDBGFADDRESS pAddrFrame,
+                                      PCDBGFADDRESS pAddrStack,
+                                      PCDBGFADDRESS pAddrPC,
+                                      DBGFRETURNTYPE enmReturnType,
+                                      PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    return dbgfR3StackWalkBeginCommon(pVM, idCpu, enmCodeType, pAddrFrame, pAddrStack, pAddrPC, enmReturnType, ppFirstFrame);
+}
+
+
+/**
+ * Begins a guest stack walk.
+ *
+ * This will walk the current stack, constructing a list of info frames which is
+ * returned to the caller. The caller uses DBGFR3StackWalkNext to traverse the
+ * list and DBGFR3StackWalkEnd to release it.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_NO_MEMORY if we're out of memory.
+ *
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The ID of the virtual CPU which stack we want to walk.
+ * @param   enmCodeType     Code type
+ * @param   ppFirstFrame    Where to return the pointer to the first info frame.
+ */
+VMMR3DECL(int) DBGFR3StackWalkBegin(PVM pVM, VMCPUID idCpu, DBGFCODETYPE enmCodeType, PCDBGFSTACKFRAME *ppFirstFrame)
+{
+    return dbgfR3StackWalkBeginCommon(pVM, idCpu, enmCodeType, NULL, NULL, NULL, DBGFRETURNTYPE_INVALID, ppFirstFrame);
+}
+
+/**
+ * Gets the next stack frame.
+ *
+ * @returns Pointer to the info for the next stack frame.
+ *          NULL if no more frames.
+ *
+ * @param   pCurrent    Pointer to the current stack frame.
+ *
+ */
+VMMR3DECL(PCDBGFSTACKFRAME) DBGFR3StackWalkNext(PCDBGFSTACKFRAME pCurrent)
+{
+    return pCurrent
+         ? pCurrent->pNextInternal
+         : NULL;
+}
+
+
+/**
+ * Ends a stack walk process.
+ *
+ * This *must* be called after a successful first call to any of the stack
+ * walker functions. If not called we will leak memory or other resources.
+ *
+ * @param   pFirstFrame     The frame returned by one of the the begin
+ *                          functions.
+ */
+VMMR3DECL(void) DBGFR3StackWalkEnd(PCDBGFSTACKFRAME pFirstFrame)
+{
+    if (    !pFirstFrame
+        ||  !pFirstFrame->pFirstInternal)
+        return;
+
+    PDBGFSTACKFRAME pFrame = (PDBGFSTACKFRAME)pFirstFrame->pFirstInternal;
+    while (pFrame)
+    {
+        PDBGFSTACKFRAME pCur = pFrame;
+        pFrame = (PDBGFSTACKFRAME)pCur->pNextInternal;
+        if (pFrame)
+        {
+            if (pCur->pSymReturnPC == pFrame->pSymPC)
+                pFrame->pSymPC = NULL;
+            if (pCur->pSymReturnPC == pFrame->pSymReturnPC)
+                pFrame->pSymReturnPC = NULL;
+
+            if (pCur->pSymPC == pFrame->pSymPC)
+                pFrame->pSymPC = NULL;
+            if (pCur->pSymPC == pFrame->pSymReturnPC)
+                pFrame->pSymReturnPC = NULL;
+
+            if (pCur->pLineReturnPC == pFrame->pLinePC)
+                pFrame->pLinePC = NULL;
+            if (pCur->pLineReturnPC == pFrame->pLineReturnPC)
+                pFrame->pLineReturnPC = NULL;
+
+            if (pCur->pLinePC == pFrame->pLinePC)
+                pFrame->pLinePC = NULL;
+            if (pCur->pLinePC == pFrame->pLineReturnPC)
+                pFrame->pLineReturnPC = NULL;
+        }
+
+        RTDbgSymbolFree(pCur->pSymPC);
+        RTDbgSymbolFree(pCur->pSymReturnPC);
+        DBGFR3LineFree(pCur->pLinePC);
+        DBGFR3LineFree(pCur->pLineReturnPC);
+
+        pCur->pNextInternal = NULL;
+        pCur->pFirstInternal = NULL;
+        pCur->fFlags = 0;
+        MMR3HeapFree(pCur);
+    }
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/DBGFSym.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/DBGFSym.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/DBGFSym.cpp	(revision 35333)
@@ -0,0 +1,1066 @@
+/* $Id$ */
+/** @file
+ * DBGF - Debugger Facility, Symbol Management.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#if defined(RT_OS_WINDOWS) && 0 //defined(DEBUG_bird) // enabled this is you want to debug win32 guests, the hypervisor of EFI.
+# include <Windows.h>
+# define _IMAGEHLP64
+# include <DbgHelp.h>
+# define HAVE_DBGHELP /* if doing guest stuff, this can be nice. */
+#endif
+/** @todo Only use DBGHELP for reading modules since it doesn't do all we want (relocations), or is way to slow in some cases (add symbol)! */
+#include <VBox/dbgf.h>
+#include <VBox/mm.h>
+#include <VBox/pdmapi.h>
+#include "DBGFInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/path.h>
+#include <iprt/ctype.h>
+#include <iprt/env.h>
+#include <iprt/param.h>
+#ifndef HAVE_DBGHELP
+# include <iprt/avl.h>
+# include <iprt/string.h>
+#endif
+
+#include <stdio.h> /* for fopen(). */ /** @todo use iprt/stream.h! */
+#include <stdlib.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+#ifdef HAVE_DBGHELP
+static DECLCALLBACK(int) dbgfR3EnumModules(PVM pVM, const char *pszFilename, const char *pszName,
+                                           RTUINTPTR ImageBase, size_t cbImage, bool fRC, void *pvArg);
+static int win32Error(PVM pVM);
+#endif
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+#ifndef HAVE_DBGHELP
+/* later */
+typedef struct DBGFMOD *PDBGFMOD;
+
+/**
+ * Internal representation of a symbol.
+ */
+typedef struct DBGFSYM
+{
+    /** Node core with the symbol address range. */
+    AVLRGCPTRNODECORE       Core;
+    /** Pointer to the module this symbol is associated with. */
+    PDBGFMOD                pModule;
+    /** Pointer to the next symbol in with this name. */
+    struct DBGFSYM         *pNext;
+    /** Symbol name. */
+    char                    szName[1];
+} DBGFSYM, *PDBGFSYM;
+
+/**
+ * Symbol name space node.
+ */
+typedef struct DBGFSYMSPACE
+{
+    /** Node core with the symbol name.
+     * (it's allocated in the same block as this struct) */
+    RTSTRSPACECORE          Core;
+    /** Pointer to the first symbol with this name (LIFO). */
+    PDBGFSYM                pSym;
+} DBGFSYMSPACE, *PDBGFSYMSPACE;
+
+#endif
+
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+#ifndef HAVE_DBGHELP
+
+/**
+ * Initializes the symbol tree.
+ */
+static int dbgfR3SymbolInit(PVM pVM)
+{
+    PDBGFSYM pSym = (PDBGFSYM)MMR3HeapAlloc(pVM, MM_TAG_DBGF_SYMBOL, sizeof(*pSym));
+    if (pSym)
+    {
+        pSym->Core.Key = 0;
+        pSym->Core.KeyLast = ~0;
+        pSym->pModule = NULL;
+        pSym->szName[0] = '\0';
+        if (RTAvlrGCPtrInsert(&pVM->dbgf.s.SymbolTree, &pSym->Core))
+            return VINF_SUCCESS;
+        AssertReleaseMsgFailed(("Failed to insert %RGv-%RGv!\n", pSym->Core.Key, pSym->Core.KeyLast));
+        return VERR_INTERNAL_ERROR;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Insert a record into the symbol tree.
+ */
+static int dbgfR3SymbolInsert(PVM pVM, const char *pszName, RTGCPTR Address, size_t cb, PDBGFMOD pModule)
+{
+    /*
+     * Make the address space node.
+     */
+    size_t      cchName = strlen(pszName) + 1;
+    PDBGFSYM pSym = (PDBGFSYM)MMR3HeapAlloc(pVM, MM_TAG_DBGF_SYMBOL, RT_OFFSETOF(DBGFSYM, szName[cchName]));
+    if (pSym)
+    {
+        pSym->Core.Key = Address;
+        pSym->Core.KeyLast = Address + cb;
+        pSym->pModule = pModule;
+        memcpy(pSym->szName, pszName, cchName);
+
+        PDBGFSYM pOld = (PDBGFSYM)RTAvlrGCPtrRangeGet(&pVM->dbgf.s.SymbolTree, (RTGCPTR)Address);
+        if (pOld)
+        {
+            pSym->Core.KeyLast = pOld->Core.KeyLast;
+            if (pOld->Core.Key == pSym->Core.Key)
+            {
+                pOld = (PDBGFSYM)RTAvlrGCPtrRemove(&pVM->dbgf.s.SymbolTree, (RTGCPTR)Address);
+                AssertRelease(pOld);
+                MMR3HeapFree(pOld);
+            }
+            else
+                pOld->Core.KeyLast = Address - 1;
+            if (RTAvlrGCPtrInsert(&pVM->dbgf.s.SymbolTree, &pSym->Core))
+            {
+                /*
+                 * Make the name space node.
+                 */
+                PDBGFSYMSPACE pName = (PDBGFSYMSPACE)RTStrSpaceGet(pVM->dbgf.s.pSymbolSpace, pszName);
+                if (!pName)
+                {
+                    /* make new symbol space node. */
+                    pName = (PDBGFSYMSPACE)MMR3HeapAlloc(pVM, MM_TAG_DBGF_SYMBOL, sizeof(*pName) + cchName);
+                    if (pName)
+                    {
+                        pName->Core.pszString = (char *)memcpy(pName + 1, pszName, cchName);
+                        pName->pSym = pSym;
+                        if (RTStrSpaceInsert(pVM->dbgf.s.pSymbolSpace, &pName->Core))
+                            return VINF_SUCCESS;
+                    }
+                    else
+                        return VINF_SUCCESS;
+                }
+                else
+                {
+                    /* Add to existing symbol name. */
+                    pSym->pNext = pName->pSym;
+                    pName->pSym = pSym;
+                    return VINF_SUCCESS;
+                }
+            }
+            AssertReleaseMsgFailed(("Failed to insert %RGv-%RGv!\n", pSym->Core.Key, pSym->Core.KeyLast));
+        }
+        else
+            AssertMsgFailed(("pOld! %RGv %s\n", pSym->Core.Key, pszName));
+        return VERR_INTERNAL_ERROR;
+
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Get nearest symbol.
+ * @returns NULL if no symbol was the for that address.
+ */
+static PDBGFSYM dbgfR3SymbolGetAddr(PVM pVM, RTGCPTR Address)
+{
+    PDBGFSYM pSym = (PDBGFSYM)RTAvlrGCPtrRangeGet(&pVM->dbgf.s.SymbolTree, Address);
+    Assert(pSym);
+    if (pSym && pSym->szName[0])
+        return pSym;
+    return NULL;
+}
+
+
+/**
+ * Get first symbol.
+ * @returns NULL if no symbol by that name.
+ */
+static PDBGFSYM dbgfR3SymbolGetName(PVM pVM, const char *pszSymbol)
+{
+    PDBGFSYMSPACE pName = (PDBGFSYMSPACE)RTStrSpaceGet(pVM->dbgf.s.pSymbolSpace, pszSymbol);
+    if (pName)
+        return pName->pSym;
+    return NULL;
+}
+
+#endif
+
+
+/**
+ * Strips all kind of spaces from head and tail of a string.
+ */
+static char *dbgfR3Strip(char *psz)
+{
+    while (*psz && RT_C_IS_SPACE(*psz))
+        psz++;
+    char *psz2 = strchr(psz, '\0') - 1;
+    while (psz2 >= psz && RT_C_IS_SPACE(*psz2))
+        *psz2-- = '\0';
+    return psz;
+}
+
+
+/**
+ * Initialize the debug info for a VM.
+ *
+ * This will check the CFGM for any symbols or symbol files
+ * which needs loading.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+int dbgfR3SymInit(PVM pVM)
+{
+    int rc;
+
+    /*
+     * Initialize the symbol table.
+     */
+    pVM->dbgf.s.pSymbolSpace = (PRTSTRSPACE)MMR3HeapAllocZ(pVM, MM_TAG_DBGF_SYMBOL, sizeof(*pVM->dbgf.s.pSymbolSpace));
+    AssertReturn(pVM->dbgf.s.pSymbolSpace, VERR_NO_MEMORY);
+
+#ifndef HAVE_DBGHELP
+    /* modules & lines later */
+    rc = dbgfR3SymbolInit(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+    pVM->dbgf.s.fSymInited = true;
+#endif
+
+    /*
+     * Check if there are 'loadsyms' commands in the configuration.
+     */
+    PCFGMNODE pNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/DBGF/loadsyms/");
+    if (pNode)
+    {
+        /*
+         * Enumerate the commands.
+         */
+        for (PCFGMNODE pCmdNode = CFGMR3GetFirstChild(pNode);
+             pCmdNode;
+             pCmdNode = CFGMR3GetNextChild(pCmdNode))
+        {
+            char szCmdName[128];
+            CFGMR3GetName(pCmdNode, &szCmdName[0], sizeof(szCmdName));
+
+            /* File */
+            char *pszFilename;
+            rc = CFGMR3QueryStringAlloc(pCmdNode, "Filename", &pszFilename);
+            AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'File' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
+
+            /* Delta (optional) */
+            RTGCINTPTR offDelta;
+            rc = CFGMR3QueryGCPtrS(pNode, "Delta", &offDelta);
+            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+                offDelta = 0;
+            else
+                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'Delta' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
+
+            /* Module (optional) */
+            char *pszModule;
+            rc = CFGMR3QueryStringAlloc(pCmdNode, "Module", &pszModule);
+            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+                pszModule = NULL;
+            else
+                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'Module' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
+
+            /* Module (optional) */
+            RTGCUINTPTR ModuleAddress;
+            rc = CFGMR3QueryGCPtrU(pNode, "ModuleAddress", &ModuleAddress);
+            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+                ModuleAddress = 0;
+            else
+                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'ModuleAddress' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
+
+            /* Image size (optional) */
+            RTGCUINTPTR cbModule;
+            rc = CFGMR3QueryGCPtrU(pNode, "ModuleSize", &cbModule);
+            if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+                cbModule = 0;
+            else
+                AssertMsgRCReturn(rc, ("rc=%Rrc querying the 'ModuleAddress' attribute of '/DBGF/loadsyms/%s'!\n", rc, szCmdName), rc);
+
+
+            /*
+             * Execute the command.
+             */
+            rc = DBGFR3ModuleLoad(pVM, pszFilename, offDelta, pszModule, ModuleAddress, cbModule);
+            AssertMsgRCReturn(rc, ("pszFilename=%s offDelta=%RGv pszModule=%s ModuleAddress=%RGv cbModule=%RGv\n",
+                                   pszFilename, offDelta, pszModule, ModuleAddress, cbModule), rc);
+
+            MMR3HeapFree(pszModule);
+            MMR3HeapFree(pszFilename);
+        }
+    }
+
+    /*
+     * Check if there are any 'symadd' commands in the configuration.
+     */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * We delay certain
+ * Initialize the debug info for a VM.
+ */
+int dbgfR3SymLazyInit(PVM pVM)
+{
+    if (pVM->dbgf.s.fSymInited)
+        return VINF_SUCCESS;
+#ifdef HAVE_DBGHELP
+    if (SymInitialize(pVM, NULL, FALSE))
+    {
+        pVM->dbgf.s.fSymInited = true;
+        SymSetOptions(SYMOPT_LOAD_LINES | SYMOPT_AUTO_PUBLICS | SYMOPT_ALLOW_ABSOLUTE_SYMBOLS);
+
+        /*
+         * Enumerate all modules loaded by PDM and add them to the symbol database.
+         */
+        PDMR3LdrEnumModules(pVM, dbgfR3EnumModules, NULL);
+        return VINF_SUCCESS;
+    }
+    return win32Error(pVM);
+#else
+    return VINF_SUCCESS;
+#endif
+}
+
+
+#ifdef HAVE_DBGHELP
+/**
+ * Module enumeration callback function.
+ *
+ * @returns VBox status.
+ *          Failure will stop the search and return the return code.
+ *          Warnings will be ignored and not returned.
+ * @param   pVM             VM Handle.
+ * @param   pszFilename     Module filename.
+ * @param   pszName         Module name. (short and unique)
+ * @param   ImageBase       Address where to executable image is loaded.
+ * @param   cbImage         Size of the executable image.
+ * @param   fRC             Set if guest context, clear if host context.
+ * @param   pvArg           User argument.
+ */
+static DECLCALLBACK(int) dbgfR3EnumModules(PVM pVM, const char *pszFilename, const char *pszName,
+                                           RTUINTPTR ImageBase, size_t cbImage, bool fRC, void *pvArg)
+{
+    DWORD64 LoadedImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)pszFilename,
+                                                (char *)(void *)pszName, ImageBase, (DWORD)cbImage);
+    if (!LoadedImageBase)
+        Log(("SymLoadModule64(,,%s,,) -> lasterr=%d\n", pszFilename, GetLastError()));
+    else
+        Log(("Loaded debuginfo for %s - %s %llx\n", pszName, pszFilename, LoadedImageBase));
+
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Terminate the debug info repository for the specified VM.
+ *
+ * @returns VBox status.
+ * @param   pVM             VM Handle.
+ */
+int dbgfR3SymTerm(PVM pVM)
+{
+#ifdef HAVE_DBGHELP
+    if (pVM->dbgf.s.fSymInited)
+        SymCleanup(pVM);
+    pVM->dbgf.s.fSymInited = false;
+    return VINF_SUCCESS;
+#else
+    pVM->dbgf.s.SymbolTree = 0; /* MM cleans up allocations */
+    pVM->dbgf.s.fSymInited = false;
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/** Symbol file type.. */
+typedef enum SYMFILETYPE
+{
+    SYMFILETYPE_UNKNOWN,
+    SYMFILETYPE_LD_MAP,
+    SYMFILETYPE_MS_MAP,
+    SYMFILETYPE_OBJDUMP,
+    SYMFILETYPE_LINUX_SYSTEM_MAP,
+    SYMFILETYPE_PDB,
+    SYMFILETYPE_DBG,
+    SYMFILETYPE_MZ,
+    SYMFILETYPE_ELF
+} SYMFILETYPE, *PSYMFILETYPE;
+
+
+
+/**
+ * Probe the type of a symbol information file.
+ *
+ * @returns The file type.
+ * @param   pFile   File handle.
+ */
+SYMFILETYPE dbgfR3ModuleProbe(FILE *pFile)
+{
+    char szHead[4096];
+    size_t cchHead = fread(szHead, 1, sizeof(szHead) - 1, pFile);
+    if (cchHead > 0)
+    {
+        szHead[cchHead] = '\0';
+        if (strstr(szHead, "Preferred load address is"))
+            return SYMFILETYPE_MS_MAP;
+
+        if (    strstr(szHead, "Archive member included because of")
+            ||  strstr(szHead, "Memory Configuration")
+            ||  strstr(szHead, "Linker script and memory map"))
+            return SYMFILETYPE_LD_MAP;
+
+        if (   RT_C_IS_XDIGIT(szHead[0])
+            && RT_C_IS_XDIGIT(szHead[1])
+            && RT_C_IS_XDIGIT(szHead[2])
+            && RT_C_IS_XDIGIT(szHead[3])
+            && RT_C_IS_XDIGIT(szHead[4])
+            && RT_C_IS_XDIGIT(szHead[5])
+            && RT_C_IS_XDIGIT(szHead[6])
+            && RT_C_IS_XDIGIT(szHead[7])
+            && szHead[8] == ' '
+            && RT_C_IS_ALPHA(szHead[9])
+            && szHead[10] == ' '
+            && (RT_C_IS_ALPHA(szHead[11]) || szHead[11] == '_' || szHead[11] == '$')
+            )
+            return SYMFILETYPE_LINUX_SYSTEM_MAP;
+
+        if (   RT_C_IS_XDIGIT(szHead[0])
+            && RT_C_IS_XDIGIT(szHead[1])
+            && RT_C_IS_XDIGIT(szHead[2])
+            && RT_C_IS_XDIGIT(szHead[3])
+            && RT_C_IS_XDIGIT(szHead[4])
+            && RT_C_IS_XDIGIT(szHead[5])
+            && RT_C_IS_XDIGIT(szHead[6])
+            && RT_C_IS_XDIGIT(szHead[7])
+            && RT_C_IS_XDIGIT(szHead[8])
+            && RT_C_IS_XDIGIT(szHead[9])
+            && RT_C_IS_XDIGIT(szHead[10])
+            && RT_C_IS_XDIGIT(szHead[11])
+            && RT_C_IS_XDIGIT(szHead[12])
+            && RT_C_IS_XDIGIT(szHead[13])
+            && RT_C_IS_XDIGIT(szHead[14])
+            && RT_C_IS_XDIGIT(szHead[15])
+            && szHead[16] == ' '
+            && RT_C_IS_ALPHA(szHead[17])
+            && szHead[18] == ' '
+            && (RT_C_IS_ALPHA(szHead[19]) || szHead[19] == '_' || szHead[19] == '$')
+            )
+            return SYMFILETYPE_LINUX_SYSTEM_MAP;
+
+        if (strstr(szHead, "Microsoft C/C++ MSF") == szHead)
+            return SYMFILETYPE_PDB;
+
+        if (strstr(szHead, "ELF") == szHead + 1)
+            return SYMFILETYPE_ELF;
+
+        if (   strstr(szHead, "MZ") == szHead
+            || strstr(szHead, "PE") == szHead
+            || strstr(szHead, "LE") == szHead
+            || strstr(szHead, "LX") == szHead
+            || strstr(szHead, "NE") == szHead)
+            return SYMFILETYPE_MZ;
+
+
+        if (strstr(szHead, "file format"))
+            return SYMFILETYPE_OBJDUMP;
+    }
+
+    return SYMFILETYPE_UNKNOWN;
+}
+
+
+static int dbgfR3LoadLinuxSystemMap(PVM pVM, FILE *pFile, RTGCUINTPTR ModuleAddress, RTGCUINTPTR AddressDelta)
+{
+    char szLine[4096];
+    while (fgets(szLine, sizeof(szLine), pFile))
+    {
+        /* parse the line: <address> <type> <name> */
+        const char *psz = dbgfR3Strip(szLine);
+        char *pszEnd = NULL;
+        uint64_t u64Address;
+        int rc = RTStrToUInt64Ex(psz, &pszEnd, 16, &u64Address);
+        RTGCUINTPTR Address = u64Address;
+        if (    RT_SUCCESS(rc)
+            &&  (*pszEnd == ' ' || *pszEnd == '\t')
+            &&  Address == u64Address
+            &&  u64Address != 0
+            &&  u64Address != (RTGCUINTPTR)~0)
+        {
+            pszEnd++;
+            if (    RT_C_IS_ALPHA(*pszEnd)
+                &&  (pszEnd[1] == ' ' || pszEnd[1] == '\t'))
+            {
+                psz = dbgfR3Strip(pszEnd + 2);
+                if (*psz)
+                {
+                    int rc2 = DBGFR3SymbolAdd(pVM, ModuleAddress, Address + AddressDelta, 0, psz);
+                    if (RT_FAILURE(rc2))
+                        Log2(("DBGFR3SymbolAdd(,, %RGv, 0, '%s') -> %Rrc\n", Address, psz, rc2));
+                }
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * Tries to open the file using the image search paths.
+ *
+ * This is currently a quick hack and the only way to specifying the path is by setting
+ * VBOXDBG_IMAGE_PATH in the environment. It uses semicolon as separator everywhere.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pszFilename     The name of the file to locate and open.
+ * @param   pszFound        Where to return the actual filename.
+ * @param   cchFound        The buffer size.
+ * @param   ppFile          Where to return the opened file.
+ */
+int dbgfR3ModuleLocateAndOpen(PVM pVM, const char *pszFilename, char *pszFound, size_t cchFound, FILE **ppFile)
+{
+    /* Check the filename length. */
+    size_t const    cchFilename = strlen(pszFilename);
+    if (cchFilename >= cchFound)
+        return VERR_FILENAME_TOO_LONG;
+    const char     *pszName = RTPathFilename(pszFilename);
+    if (!pszName)
+        return VERR_IS_A_DIRECTORY;
+    size_t const    cchName = strlen(pszName);
+
+    /*
+     * Try default location first.
+     */
+    memcpy(pszFound, pszFilename, cchFilename + 1);
+    FILE *pFile = *ppFile = fopen(pszFound, "rb");
+    if (pFile)
+        return VINF_SUCCESS;
+
+    /*
+     * Walk the search path.
+     */
+    char *pszFreeMe = RTEnvDupEx(RTENV_DEFAULT, "VBOXDBG_IMAGE_PATH");
+    const char *psz = pszFreeMe ? pszFreeMe : ".";
+    while (*psz)
+    {
+        /* Skip leading blanks - no directories with leading spaces, thank you. */
+        while (RT_C_IS_BLANK(*psz))
+            psz++;
+
+        /* Fine the end of this element. */
+        const char *pszNext;
+        const char *pszEnd = strchr(psz, ';');
+        if (!pszEnd)
+            pszEnd = pszNext = strchr(psz, '\0');
+        else
+            pszNext = pszEnd + 1;
+        if (pszEnd != psz)
+        {
+            size_t const cch = pszEnd - psz;
+            if (cch + 1 + cchName < cchFound)
+            {
+                /** @todo RTPathCompose, RTPathComposeN(). This code isn't right
+                 * for 'E:' on DOS systems. It may also create unwanted double slashes. */
+                memcpy(pszFound, psz, cch);
+                pszFound[cch] = '/';
+                memcpy(pszFound + cch + 1, pszName, cchName + 1);
+                *ppFile = pFile = fopen(pszFound, "rb");
+                if (pFile)
+                {
+                    RTStrFree(pszFreeMe);
+                    return VINF_SUCCESS;
+                }
+            }
+
+            /** @todo do a depth search using the specified path. */
+        }
+
+        /* advance */
+        psz = pszNext;
+    }
+
+    /* not found */
+    RTStrFree(pszFreeMe);
+    return VERR_OPEN_FAILED;
+}
+
+
+/**
+ * Load debug info, optionally related to a specific module.
+ *
+ * @returns VBox status.
+ * @param   pVM             VM Handle.
+ * @param   pszFilename     Path to the file containing the symbol information.
+ *                          This can be the executable image, a flat symbol file of some kind or stripped debug info.
+ * @param   AddressDelta    The value to add to the loaded symbols.
+ * @param   pszName         Short hand name for the module. If not related to a module specify NULL.
+ * @param   ModuleAddress   Address which the image is loaded at. This will be used to reference the module other places in the api.
+ *                          Ignored when pszName is NULL.
+ * @param   cbImage         Size of the image.
+ *                          Ignored when pszName is NULL.
+ */
+VMMR3DECL(int) DBGFR3ModuleLoad(PVM pVM, const char *pszFilename, RTGCUINTPTR AddressDelta, const char *pszName, RTGCUINTPTR ModuleAddress, unsigned cbImage)
+{
+    /*
+     * Lazy init.
+     */
+    if (!pVM->dbgf.s.fSymInited)
+    {
+        int rc = dbgfR3SymLazyInit(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Open the load file.
+     */
+    FILE *pFile = NULL;
+    char szFoundFile[RTPATH_MAX];
+    int rc = dbgfR3ModuleLocateAndOpen(pVM, pszFilename, szFoundFile, sizeof(szFoundFile), &pFile);
+    if (pFile)
+    {
+        /*
+         * Probe the file type.
+         */
+        SYMFILETYPE enmType = dbgfR3ModuleProbe(pFile);
+        if (enmType != SYMFILETYPE_UNKNOWN)
+        {
+            /*
+             * Add the module.
+             */
+            if (pszName)
+            {
+                #ifdef HAVE_DBGHELP
+                /** @todo arg! checkout the inserting of modules and then loading them again.... Or just the module representation.... */
+                DWORD64 ImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)szFoundFile, (char *)(void *)pszName, ModuleAddress, cbImage);
+                if (!ImageBase)
+                    ImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)pszName, (char *)(void *)pszName, ModuleAddress, cbImage);
+                if (ImageBase)
+                {
+                    AssertMsg(ModuleAddress == 0 || ModuleAddress == ImageBase, ("ModuleAddres=%RGv ImageBase=%llx\n", ModuleAddress, ImageBase));
+                    ModuleAddress = ImageBase;
+                }
+                else
+                    rc = win32Error(pVM);
+                #else
+                rc = VERR_NOT_IMPLEMENTED;
+                #endif
+            }
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Seek to the start of the file.
+                 */
+                rc = fseek(pFile, 0, SEEK_SET);
+                Assert(!rc);
+
+                /*
+                 * Process the specific.
+                 */
+                switch (enmType)
+                {
+                    case SYMFILETYPE_LINUX_SYSTEM_MAP:
+                        rc = dbgfR3LoadLinuxSystemMap(pVM, pFile, ModuleAddress, AddressDelta);
+                        break;
+
+                    case SYMFILETYPE_PDB:
+                    case SYMFILETYPE_DBG:
+                    case SYMFILETYPE_MZ:
+                #ifdef HAVE_DBGHELP
+                        /* done it all above! */
+                        break;
+                #endif
+                    case SYMFILETYPE_LD_MAP:
+                    case SYMFILETYPE_MS_MAP:
+                    case SYMFILETYPE_OBJDUMP:
+                    case SYMFILETYPE_ELF:
+                        rc = VERR_NOT_SUPPORTED;
+                        break;
+
+                    default:
+                        AssertFailed();
+                        rc = VERR_INTERNAL_ERROR;
+                        break;
+                } /* file switch. */
+            } /* module added successfully. */
+        } /* format identified */
+        else
+            rc = VERR_NOT_SUPPORTED;
+        /** @todo check for read errors */
+        fclose(pFile);
+    }
+    return rc;
+}
+
+
+/**
+ * Interface used by PDMR3LdrRelocate for telling us that a GC module has been relocated.
+ *
+ * @param   pVM             The VM handle.
+ * @param   OldImageBase    The old image base.
+ * @param   NewImageBase    The new image base.
+ * @param   cbImage         The image size.
+ * @param   pszFilename     The image filename.
+ * @param   pszName         The module name.
+ */
+VMMR3DECL(void) DBGFR3ModuleRelocate(PVM pVM, RTGCUINTPTR OldImageBase, RTGCUINTPTR NewImageBase, RTGCUINTPTR cbImage,
+                                      const char *pszFilename, const char *pszName)
+{
+#ifdef HAVE_DBGHELP
+    if (pVM->dbgf.s.fSymInited)
+    {
+        if (!SymUnloadModule64(pVM, OldImageBase))
+            Log(("SymUnloadModule64(,%RGv) failed, lasterr=%d\n", OldImageBase, GetLastError()));
+
+        DWORD ImageSize = (DWORD)cbImage; Assert(ImageSize == cbImage);
+        DWORD64 LoadedImageBase = SymLoadModule64(pVM, NULL, (char *)(void *)pszFilename, (char *)(void *)pszName, NewImageBase, ImageSize);
+        if (!LoadedImageBase)
+            Log(("SymLoadModule64(,,%s,,) -> lasterr=%d (relocate)\n", pszFilename, GetLastError()));
+        else
+            Log(("Reloaded debuginfo for %s - %s %llx\n", pszName, pszFilename, LoadedImageBase));
+    }
+#else
+
+#endif
+}
+
+
+/**
+ * Adds a symbol to the debug info manager.
+ *
+ * @returns VBox status.
+ * @param   pVM             VM Handle.
+ * @param   ModuleAddress   Module address. Use 0 if no module.
+ * @param   SymbolAddress   Symbol address
+ * @param   cbSymbol        Size of the symbol. Use 0 if info not available.
+ * @param   pszSymbol       Symbol name.
+ */
+VMMR3DECL(int) DBGFR3SymbolAdd(PVM pVM, RTGCUINTPTR ModuleAddress, RTGCUINTPTR SymbolAddress, RTUINT cbSymbol, const char *pszSymbol)
+{
+    /*
+     * Validate.
+     */
+    if (!pszSymbol || !*pszSymbol)
+    {
+        AssertMsgFailed(("No symbol name!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Lazy init.
+     */
+    if (!pVM->dbgf.s.fSymInited)
+    {
+        int rc = dbgfR3SymLazyInit(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+#ifdef HAVE_DBGHELP
+    if (SymAddSymbol(pVM, ModuleAddress, (char *)(void *)pszSymbol, SymbolAddress, cbSymbol, 0))
+        return VINF_SUCCESS;
+    return win32Error(pVM);
+#else
+    /** @todo module lookup. */
+    return dbgfR3SymbolInsert(pVM, pszSymbol, SymbolAddress, cbSymbol, NULL);
+#endif
+}
+
+
+/**
+ * Find symbol by address (nearest).
+ *
+ * @returns VBox status.
+ * @param   pVM                 VM handle.
+ * @param   Address             Address.
+ * @param   poffDisplacement    Where to store the symbol displacement from Address.
+ * @param   pSymbol             Where to store the symbol info.
+ */
+VMMR3DECL(int) DBGFR3SymbolByAddr(PVM pVM, RTGCUINTPTR Address, PRTGCINTPTR poffDisplacement, PDBGFSYMBOL pSymbol)
+{
+    /*
+     * Lazy init.
+     */
+    if (!pVM->dbgf.s.fSymInited)
+    {
+        int rc = dbgfR3SymLazyInit(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Look it up.
+     */
+#ifdef HAVE_DBGHELP
+    char                achBuffer[sizeof(IMAGEHLP_SYMBOL64) + DBGF_SYMBOL_NAME_LENGTH * sizeof(TCHAR) + sizeof(ULONG64)];
+    PIMAGEHLP_SYMBOL64  pSym = (PIMAGEHLP_SYMBOL64)&achBuffer[0];
+    pSym->SizeOfStruct      = sizeof(IMAGEHLP_SYMBOL64);
+    pSym->MaxNameLength     = DBGF_SYMBOL_NAME_LENGTH;
+
+    if (SymGetSymFromAddr64(pVM, Address, (PDWORD64)poffDisplacement, pSym))
+    {
+        pSymbol->Value  = (RTGCUINTPTR)pSym->Address;
+        pSymbol->cb     = pSym->Size;
+        pSymbol->fFlags = pSym->Flags;
+        strcpy(pSymbol->szName, pSym->Name);
+        return VINF_SUCCESS;
+    }
+    //return win32Error(pVM);
+
+#else
+
+    PDBGFSYM pSym = dbgfR3SymbolGetAddr(pVM, Address);
+    if (pSym)
+    {
+        pSymbol->Value = pSym->Core.Key;
+        pSymbol->cb = pSym->Core.KeyLast - pSym->Core.Key + 1;
+        pSymbol->fFlags = 0;
+        pSymbol->szName[0] = '\0';
+        strncat(pSymbol->szName, pSym->szName,  sizeof(pSymbol->szName) - 1);
+        if (poffDisplacement)
+            *poffDisplacement = Address - pSymbol->Value;
+        return VINF_SUCCESS;
+    }
+
+#endif
+
+    /*
+     * Try PDM.
+     */
+    if (MMHyperIsInsideArea(pVM, Address))
+    {
+        char        szModName[64];
+        RTRCPTR     RCPtrMod;
+        char        szNearSym1[260];
+        RTRCPTR     RCPtrNearSym1;
+        char        szNearSym2[260];
+        RTRCPTR     RCPtrNearSym2;
+        int rc = PDMR3LdrQueryRCModFromPC(pVM, Address,
+                                          &szModName[0],  sizeof(szModName),  &RCPtrMod,
+                                          &szNearSym1[0], sizeof(szNearSym1), &RCPtrNearSym1,
+                                          &szNearSym2[0], sizeof(szNearSym2), &RCPtrNearSym2);
+        if (RT_SUCCESS(rc) && szNearSym1[0])
+        {
+            pSymbol->Value = RCPtrNearSym1;
+            pSymbol->cb = RCPtrNearSym2 > RCPtrNearSym1 ? RCPtrNearSym2 - RCPtrNearSym1 : 0;
+            pSymbol->fFlags = 0;
+            pSymbol->szName[0] = '\0';
+            strncat(pSymbol->szName, szNearSym1,  sizeof(pSymbol->szName) - 1);
+            if (poffDisplacement)
+                *poffDisplacement = Address - pSymbol->Value;
+            return VINF_SUCCESS;
+        }
+    }
+
+    return VERR_SYMBOL_NOT_FOUND;
+}
+
+
+/**
+ * Find symbol by name (first).
+ *
+ * @returns VBox status.
+ * @param   pVM                 VM handle.
+ * @param   pszSymbol           Symbol name.
+ * @param   pSymbol             Where to store the symbol info.
+ */
+VMMR3DECL(int) DBGFR3SymbolByName(PVM pVM, const char *pszSymbol, PDBGFSYMBOL pSymbol)
+{
+    /*
+     * Lazy init.
+     */
+    if (!pVM->dbgf.s.fSymInited)
+    {
+        int rc = dbgfR3SymLazyInit(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Look it up.
+     */
+#ifdef HAVE_DBGHELP
+    char            achBuffer[sizeof(IMAGEHLP_SYMBOL64) + DBGF_SYMBOL_NAME_LENGTH * sizeof(TCHAR) + sizeof(ULONG64)];
+    PIMAGEHLP_SYMBOL64 pSym = (PIMAGEHLP_SYMBOL64)&achBuffer[0];
+    pSym->SizeOfStruct      = sizeof(IMAGEHLP_SYMBOL64);
+    pSym->MaxNameLength     = DBGF_SYMBOL_NAME_LENGTH;
+
+    if (SymGetSymFromName64(pVM, (char *)(void *)pszSymbol, pSym))
+    {
+        pSymbol->Value  = (RTGCUINTPTR)pSym->Address;
+        pSymbol->cb     = pSym->Size;
+        pSymbol->fFlags = pSym->Flags;
+        strcpy(pSymbol->szName, pSym->Name);
+        return VINF_SUCCESS;
+    }
+    return win32Error(pVM);
+#else
+
+    PDBGFSYM pSym = dbgfR3SymbolGetName(pVM, pszSymbol);
+    if (pSym)
+    {
+        pSymbol->Value = pSym->Core.Key;
+        pSymbol->cb = pSym->Core.KeyLast - pSym->Core.Key + 1;
+        pSymbol->fFlags = 0;
+        pSymbol->szName[0] = '\0';
+        strncat(pSymbol->szName, pSym->szName, sizeof(pSymbol->szName) - 1);
+        return VINF_SUCCESS;
+    }
+
+    return VERR_SYMBOL_NOT_FOUND;
+#endif
+}
+
+
+/**
+ * Find line by address (nearest).
+ *
+ * @returns VBox status.
+ * @param   pVM                 VM handle.
+ * @param   Address             Address.
+ * @param   poffDisplacement    Where to store the line displacement from Address.
+ * @param   pLine               Where to store the line info.
+ */
+VMMR3DECL(int) DBGFR3LineByAddr(PVM pVM, RTGCUINTPTR Address, PRTGCINTPTR poffDisplacement, PDBGFLINE pLine)
+{
+    /*
+     * Lazy init.
+     */
+    if (!pVM->dbgf.s.fSymInited)
+    {
+        int rc = dbgfR3SymLazyInit(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Look it up.
+     */
+#ifdef HAVE_DBGHELP
+    IMAGEHLP_LINE64     Line = {0};
+    DWORD               off = 0;
+    Line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
+    if (SymGetLineFromAddr64(pVM, Address, &off, &Line))
+    {
+        if (poffDisplacement)
+            *poffDisplacement = (long)off;
+        pLine->Address      = (RTGCUINTPTR)Line.Address;
+        pLine->uLineNo      = Line.LineNumber;
+        pLine->szFilename[0] = '\0';
+        strncat(pLine->szFilename, Line.FileName, sizeof(pLine->szFilename));
+        return VINF_SUCCESS;
+    }
+    return win32Error(pVM);
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * Duplicates a line.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pLine           The line to duplicate.
+ */
+static PDBGFLINE dbgfR3LineDup(PVM pVM, PCDBGFLINE pLine)
+{
+    size_t cb = strlen(pLine->szFilename) + RT_OFFSETOF(DBGFLINE, szFilename[1]);
+    PDBGFLINE pDup = (PDBGFLINE)MMR3HeapAlloc(pVM, MM_TAG_DBGF_LINE_DUP, cb);
+    if (pDup)
+        memcpy(pDup, pLine, cb);
+    return pDup;
+}
+
+
+/**
+ * Find line by address (nearest), allocate return buffer.
+ *
+ * @returns Pointer to the line. Must be freed using DBGFR3LineFree().
+ * @returns NULL if the line was not found or if we're out of memory.
+ * @param   pVM                 VM handle.
+ * @param   Address             Address.
+ * @param   poffDisplacement    Where to store the line displacement from Address.
+ */
+VMMR3DECL(PDBGFLINE) DBGFR3LineByAddrAlloc(PVM pVM, RTGCUINTPTR Address, PRTGCINTPTR poffDisplacement)
+{
+    DBGFLINE Line;
+    int rc = DBGFR3LineByAddr(pVM, Address, poffDisplacement, &Line);
+    if (RT_FAILURE(rc))
+        return NULL;
+    return dbgfR3LineDup(pVM, &Line);
+}
+
+
+/**
+ * Frees a line returned by DBGFR3LineByAddressAlloc().
+ *
+ * @param   pLine           Pointer to the line.
+ */
+VMMR3DECL(void) DBGFR3LineFree(PDBGFLINE pLine)
+{
+    if (pLine)
+        MMR3HeapFree(pLine);
+}
+
+
+#ifdef HAVE_DBGHELP
+
+//static BOOL CALLBACK win32EnumModulesCallback(PSTR ModuleName, DWORD64 BaseOfDll, PVOID UserContext)
+//{
+//    Log(("dbg: module: %08llx %s\n", ModuleName, BaseOfDll));
+//    return TRUE;
+//}
+
+static int win32Error(PVM pVM)
+{
+    int rc = GetLastError();
+    Log(("Lasterror=%d\n", rc));
+
+    //SymEnumerateModules64(pVM, win32EnumModulesCallback, NULL);
+
+    return VERR_GENERAL_FAILURE;
+}
+#endif
+
Index: /trunk/src/VBox/VMM/VMMR3/EM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/EM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/EM.cpp	(revision 35333)
@@ -0,0 +1,2149 @@
+/* $Id$ */
+/** @file
+ * EM - Execution Monitor / Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_em         EM - The Execution Monitor / Manager
+ *
+ * The Execution Monitor/Manager is responsible for running the VM, scheduling
+ * the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
+ * Interpreted), and keeping the CPU states in sync. The function
+ * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
+ * modes has different inner loops (emR3RawExecute, emR3HwAccExecute, and
+ * emR3RemExecute).
+ *
+ * The interpreted execution is only used to avoid switching between
+ * raw-mode/hwaccm and the recompiler when fielding virtualization traps/faults.
+ * The interpretation is thus implemented as part of EM.
+ *
+ * @see grp_em
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/em.h>
+#include <VBox/vmm.h>
+#include <VBox/patm.h>
+#include <VBox/csam.h>
+#include <VBox/selm.h>
+#include <VBox/trpm.h>
+#include <VBox/iom.h>
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/rem.h>
+#include <VBox/tm.h>
+#include <VBox/mm.h>
+#include <VBox/ssm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmqueue.h>
+#include <VBox/hwaccm.h>
+#include <VBox/patm.h>
+#include "EMInternal.h"
+#include "internal/em.h"
+#include <VBox/vm.h>
+#include <VBox/cpumdis.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/dbgf.h>
+
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/stream.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#if 0 /* Disabled till after 2.1.0 when we've time to test it. */
+#define EM_NOTIFY_HWACCM
+#endif
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static const char *emR3GetStateName(EMSTATE enmState);
+static int emR3Debug(PVM pVM, PVMCPU pVCpu, int rc);
+static int emR3RemStep(PVM pVM, PVMCPU pVCpu);
+static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
+int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
+
+
+/**
+ * Initializes the EM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) EMR3Init(PVM pVM)
+{
+    LogFlow(("EMR3Init\n"));
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, em.s, 32);
+    AssertCompile(sizeof(pVM->em.s) <= sizeof(pVM->em.padding));
+    AssertCompile(sizeof(pVM->aCpus[0].em.s.u.FatalLongJump) <= sizeof(pVM->aCpus[0].em.s.u.achPaddingFatalLongJump));
+    AssertCompileMemberAlignment(EM, CritSectREM, sizeof(uintptr_t));
+
+    /*
+     * Init the structure.
+     */
+    pVM->em.s.offVM = RT_OFFSETOF(VM, em.s);
+    int rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "RawR3Enabled", &pVM->fRawR3Enabled);
+    if (RT_FAILURE(rc))
+        pVM->fRawR3Enabled = true;
+    rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "RawR0Enabled", &pVM->fRawR0Enabled);
+    if (RT_FAILURE(rc))
+        pVM->fRawR0Enabled = true;
+    Log(("EMR3Init: fRawR3Enabled=%d fRawR0Enabled=%d\n", pVM->fRawR3Enabled, pVM->fRawR0Enabled));
+
+    /*
+     * Initialize the REM critical section.
+     */
+    rc = PDMR3CritSectInit(pVM, &pVM->em.s.CritSectREM, RT_SRC_POS, "EM-REM");
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Saved state.
+     */
+    rc = SSMR3RegisterInternal(pVM, "em", 0, EM_SAVED_STATE_VERSION, 16,
+                               NULL, NULL, NULL,
+                               NULL, emR3Save, NULL,
+                               NULL, emR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        pVCpu->em.s.offVMCPU = RT_OFFSETOF(VMCPU, em.s);
+
+        pVCpu->em.s.enmState     = (i == 0) ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
+        pVCpu->em.s.enmPrevState = EMSTATE_NONE;
+        pVCpu->em.s.fForceRAW    = false;
+
+        pVCpu->em.s.pCtx         = CPUMQueryGuestCtxPtr(pVCpu);
+        pVCpu->em.s.pPatmGCState = PATMR3QueryGCStateHC(pVM);
+        AssertMsg(pVCpu->em.s.pPatmGCState, ("PATMR3QueryGCStateHC failed!\n"));
+
+        /* Force reset of the time slice. */
+        pVCpu->em.s.u64TimeSliceStart = 0;
+
+# define EM_REG_COUNTER(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, i); \
+        AssertRC(rc);
+
+# define EM_REG_COUNTER_USED(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, c, b, i); \
+        AssertRC(rc);
+
+# define EM_REG_PROFILE(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
+        AssertRC(rc);
+
+# define EM_REG_PROFILE_ADV(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, i); \
+        AssertRC(rc);
+
+        /*
+         * Statistics.
+         */
+#ifdef VBOX_WITH_STATISTICS
+        PEMSTATS pStats;
+        rc = MMHyperAlloc(pVM, sizeof(*pStats), 0, MM_TAG_EM, (void **)&pStats);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        pVCpu->em.s.pStatsR3 = pStats;
+        pVCpu->em.s.pStatsR0 = MMHyperR3ToR0(pVM, pStats);
+        pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pStats);
+
+        EM_REG_PROFILE(&pStats->StatRZEmulate,               "/EM/CPU%d/RZ/Interpret",                   "Profiling of EMInterpretInstruction.");
+        EM_REG_PROFILE(&pStats->StatR3Emulate,               "/EM/CPU%d/R3/Interpret",                   "Profiling of EMInterpretInstruction.");
+
+        EM_REG_PROFILE(&pStats->StatRZInterpretSucceeded,    "/EM/CPU%d/RZ/Interpret/Success",           "The number of times an instruction was successfully interpreted.");
+        EM_REG_PROFILE(&pStats->StatR3InterpretSucceeded,    "/EM/CPU%d/R3/Interpret/Success",           "The number of times an instruction was successfully interpreted.");
+
+        EM_REG_COUNTER_USED(&pStats->StatRZAnd,                  "/EM/CPU%d/RZ/Interpret/Success/And",       "The number of times AND was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3And,                  "/EM/CPU%d/R3/Interpret/Success/And",       "The number of times AND was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZAdd,                  "/EM/CPU%d/RZ/Interpret/Success/Add",       "The number of times ADD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Add,                  "/EM/CPU%d/R3/Interpret/Success/Add",       "The number of times ADD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZAdc,                  "/EM/CPU%d/RZ/Interpret/Success/Adc",       "The number of times ADC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Adc,                  "/EM/CPU%d/R3/Interpret/Success/Adc",       "The number of times ADC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZSub,                  "/EM/CPU%d/RZ/Interpret/Success/Sub",       "The number of times SUB was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Sub,                  "/EM/CPU%d/R3/Interpret/Success/Sub",       "The number of times SUB was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZCpuId,                "/EM/CPU%d/RZ/Interpret/Success/CpuId",     "The number of times CPUID was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3CpuId,                "/EM/CPU%d/R3/Interpret/Success/CpuId",     "The number of times CPUID was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZDec,                  "/EM/CPU%d/RZ/Interpret/Success/Dec",       "The number of times DEC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Dec,                  "/EM/CPU%d/R3/Interpret/Success/Dec",       "The number of times DEC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZHlt,                  "/EM/CPU%d/RZ/Interpret/Success/Hlt",       "The number of times HLT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Hlt,                  "/EM/CPU%d/R3/Interpret/Success/Hlt",       "The number of times HLT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZInc,                  "/EM/CPU%d/RZ/Interpret/Success/Inc",       "The number of times INC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Inc,                  "/EM/CPU%d/R3/Interpret/Success/Inc",       "The number of times INC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZInvlPg,               "/EM/CPU%d/RZ/Interpret/Success/Invlpg",    "The number of times INVLPG was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3InvlPg,               "/EM/CPU%d/R3/Interpret/Success/Invlpg",    "The number of times INVLPG was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZIret,                 "/EM/CPU%d/RZ/Interpret/Success/Iret",      "The number of times IRET was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Iret,                 "/EM/CPU%d/R3/Interpret/Success/Iret",      "The number of times IRET was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZLLdt,                 "/EM/CPU%d/RZ/Interpret/Success/LLdt",      "The number of times LLDT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3LLdt,                 "/EM/CPU%d/R3/Interpret/Success/LLdt",      "The number of times LLDT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZLIdt,                 "/EM/CPU%d/RZ/Interpret/Success/LIdt",      "The number of times LIDT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3LIdt,                 "/EM/CPU%d/R3/Interpret/Success/LIdt",      "The number of times LIDT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZLGdt,                 "/EM/CPU%d/RZ/Interpret/Success/LGdt",      "The number of times LGDT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3LGdt,                 "/EM/CPU%d/R3/Interpret/Success/LGdt",      "The number of times LGDT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZMov,                  "/EM/CPU%d/RZ/Interpret/Success/Mov",       "The number of times MOV was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Mov,                  "/EM/CPU%d/R3/Interpret/Success/Mov",       "The number of times MOV was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZMovCRx,               "/EM/CPU%d/RZ/Interpret/Success/MovCRx",    "The number of times MOV CRx was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3MovCRx,               "/EM/CPU%d/R3/Interpret/Success/MovCRx",    "The number of times MOV CRx was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZMovDRx,               "/EM/CPU%d/RZ/Interpret/Success/MovDRx",    "The number of times MOV DRx was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3MovDRx,               "/EM/CPU%d/R3/Interpret/Success/MovDRx",    "The number of times MOV DRx was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZOr,                   "/EM/CPU%d/RZ/Interpret/Success/Or",        "The number of times OR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Or,                   "/EM/CPU%d/R3/Interpret/Success/Or",        "The number of times OR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZPop,                  "/EM/CPU%d/RZ/Interpret/Success/Pop",       "The number of times POP was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Pop,                  "/EM/CPU%d/R3/Interpret/Success/Pop",       "The number of times POP was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZRdtsc,                "/EM/CPU%d/RZ/Interpret/Success/Rdtsc",     "The number of times RDTSC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Rdtsc,                "/EM/CPU%d/R3/Interpret/Success/Rdtsc",     "The number of times RDTSC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZRdpmc,                "/EM/CPU%d/RZ/Interpret/Success/Rdpmc",     "The number of times RDPMC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Rdpmc,                "/EM/CPU%d/R3/Interpret/Success/Rdpmc",     "The number of times RDPMC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZSti,                  "/EM/CPU%d/RZ/Interpret/Success/Sti",       "The number of times STI was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Sti,                  "/EM/CPU%d/R3/Interpret/Success/Sti",       "The number of times STI was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZXchg,                 "/EM/CPU%d/RZ/Interpret/Success/Xchg",      "The number of times XCHG was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Xchg,                 "/EM/CPU%d/R3/Interpret/Success/Xchg",      "The number of times XCHG was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZXor,                  "/EM/CPU%d/RZ/Interpret/Success/Xor",       "The number of times XOR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Xor,                  "/EM/CPU%d/R3/Interpret/Success/Xor",       "The number of times XOR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZMonitor,              "/EM/CPU%d/RZ/Interpret/Success/Monitor",   "The number of times MONITOR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Monitor,              "/EM/CPU%d/R3/Interpret/Success/Monitor",   "The number of times MONITOR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZMWait,                "/EM/CPU%d/RZ/Interpret/Success/MWait",     "The number of times MWAIT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3MWait,                "/EM/CPU%d/R3/Interpret/Success/MWait",     "The number of times MWAIT was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZBtr,                  "/EM/CPU%d/RZ/Interpret/Success/Btr",       "The number of times BTR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Btr,                  "/EM/CPU%d/R3/Interpret/Success/Btr",       "The number of times BTR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZBts,                  "/EM/CPU%d/RZ/Interpret/Success/Bts",       "The number of times BTS was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Bts,                  "/EM/CPU%d/R3/Interpret/Success/Bts",       "The number of times BTS was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZBtc,                  "/EM/CPU%d/RZ/Interpret/Success/Btc",       "The number of times BTC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Btc,                  "/EM/CPU%d/R3/Interpret/Success/Btc",       "The number of times BTC was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg,              "/EM/CPU%d/RZ/Interpret/Success/CmpXchg",   "The number of times CMPXCHG was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg,              "/EM/CPU%d/R3/Interpret/Success/CmpXchg",   "The number of times CMPXCHG was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZCmpXchg8b,            "/EM/CPU%d/RZ/Interpret/Success/CmpXchg8b",   "The number of times CMPXCHG8B was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3CmpXchg8b,            "/EM/CPU%d/R3/Interpret/Success/CmpXchg8b",   "The number of times CMPXCHG8B was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZXAdd,                 "/EM/CPU%d/RZ/Interpret/Success/XAdd",      "The number of times XADD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3XAdd,                 "/EM/CPU%d/R3/Interpret/Success/XAdd",      "The number of times XADD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Rdmsr,                "/EM/CPU%d/R3/Interpret/Success/Rdmsr",      "The number of times RDMSR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZRdmsr,                "/EM/CPU%d/RZ/Interpret/Success/Rdmsr",      "The number of times RDMSR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Wrmsr,                "/EM/CPU%d/R3/Interpret/Success/Wrmsr",      "The number of times WRMSR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZWrmsr,                "/EM/CPU%d/RZ/Interpret/Success/Wrmsr",      "The number of times WRMSR was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3StosWD,               "/EM/CPU%d/R3/Interpret/Success/Stoswd",     "The number of times STOSWD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZStosWD,               "/EM/CPU%d/RZ/Interpret/Success/Stoswd",     "The number of times STOSWD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZWbInvd,               "/EM/CPU%d/RZ/Interpret/Success/WbInvd",     "The number of times WBINVD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3WbInvd,               "/EM/CPU%d/R3/Interpret/Success/WbInvd",     "The number of times WBINVD was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZLmsw,                 "/EM/CPU%d/RZ/Interpret/Success/Lmsw",       "The number of times LMSW was successfully interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3Lmsw,                 "/EM/CPU%d/R3/Interpret/Success/Lmsw",       "The number of times LMSW was successfully interpreted.");
+
+        EM_REG_COUNTER(&pStats->StatRZInterpretFailed,           "/EM/CPU%d/RZ/Interpret/Failed",            "The number of times an instruction was not interpreted.");
+        EM_REG_COUNTER(&pStats->StatR3InterpretFailed,           "/EM/CPU%d/R3/Interpret/Failed",            "The number of times an instruction was not interpreted.");
+
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedAnd,            "/EM/CPU%d/RZ/Interpret/Failed/And",        "The number of times AND was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedAnd,            "/EM/CPU%d/R3/Interpret/Failed/And",        "The number of times AND was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedCpuId,          "/EM/CPU%d/RZ/Interpret/Failed/CpuId",      "The number of times CPUID was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedCpuId,          "/EM/CPU%d/R3/Interpret/Failed/CpuId",      "The number of times CPUID was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedDec,            "/EM/CPU%d/RZ/Interpret/Failed/Dec",        "The number of times DEC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedDec,            "/EM/CPU%d/R3/Interpret/Failed/Dec",        "The number of times DEC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedHlt,            "/EM/CPU%d/RZ/Interpret/Failed/Hlt",        "The number of times HLT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedHlt,            "/EM/CPU%d/R3/Interpret/Failed/Hlt",        "The number of times HLT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedInc,            "/EM/CPU%d/RZ/Interpret/Failed/Inc",        "The number of times INC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedInc,            "/EM/CPU%d/R3/Interpret/Failed/Inc",        "The number of times INC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedInvlPg,         "/EM/CPU%d/RZ/Interpret/Failed/InvlPg",     "The number of times INVLPG was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedInvlPg,         "/EM/CPU%d/R3/Interpret/Failed/InvlPg",     "The number of times INVLPG was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedIret,           "/EM/CPU%d/RZ/Interpret/Failed/Iret",       "The number of times IRET was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedIret,           "/EM/CPU%d/R3/Interpret/Failed/Iret",       "The number of times IRET was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedLLdt,           "/EM/CPU%d/RZ/Interpret/Failed/LLdt",       "The number of times LLDT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedLLdt,           "/EM/CPU%d/R3/Interpret/Failed/LLdt",       "The number of times LLDT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedLIdt,           "/EM/CPU%d/RZ/Interpret/Failed/LIdt",       "The number of times LIDT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedLIdt,           "/EM/CPU%d/R3/Interpret/Failed/LIdt",       "The number of times LIDT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedLGdt,           "/EM/CPU%d/RZ/Interpret/Failed/LGdt",       "The number of times LGDT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedLGdt,           "/EM/CPU%d/R3/Interpret/Failed/LGdt",       "The number of times LGDT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedMov,            "/EM/CPU%d/RZ/Interpret/Failed/Mov",        "The number of times MOV was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedMov,            "/EM/CPU%d/R3/Interpret/Failed/Mov",        "The number of times MOV was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedMovCRx,         "/EM/CPU%d/RZ/Interpret/Failed/MovCRx",     "The number of times MOV CRx was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedMovCRx,         "/EM/CPU%d/R3/Interpret/Failed/MovCRx",     "The number of times MOV CRx was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedMovDRx,         "/EM/CPU%d/RZ/Interpret/Failed/MovDRx",     "The number of times MOV DRx was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedMovDRx,         "/EM/CPU%d/R3/Interpret/Failed/MovDRx",     "The number of times MOV DRx was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedOr,             "/EM/CPU%d/RZ/Interpret/Failed/Or",         "The number of times OR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedOr,             "/EM/CPU%d/R3/Interpret/Failed/Or",         "The number of times OR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedPop,            "/EM/CPU%d/RZ/Interpret/Failed/Pop",        "The number of times POP was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedPop,            "/EM/CPU%d/R3/Interpret/Failed/Pop",        "The number of times POP was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedSti,            "/EM/CPU%d/RZ/Interpret/Failed/Sti",        "The number of times STI was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedSti,            "/EM/CPU%d/R3/Interpret/Failed/Sti",        "The number of times STI was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedXchg,           "/EM/CPU%d/RZ/Interpret/Failed/Xchg",       "The number of times XCHG was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedXchg,           "/EM/CPU%d/R3/Interpret/Failed/Xchg",       "The number of times XCHG was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedXor,            "/EM/CPU%d/RZ/Interpret/Failed/Xor",        "The number of times XOR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedXor,            "/EM/CPU%d/R3/Interpret/Failed/Xor",        "The number of times XOR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedMonitor,        "/EM/CPU%d/RZ/Interpret/Failed/Monitor",    "The number of times MONITOR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedMonitor,        "/EM/CPU%d/R3/Interpret/Failed/Monitor",    "The number of times MONITOR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedMWait,          "/EM/CPU%d/RZ/Interpret/Failed/MWait",      "The number of times MWAIT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedMWait,          "/EM/CPU%d/R3/Interpret/Failed/MWait",      "The number of times MWAIT was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedRdtsc,          "/EM/CPU%d/RZ/Interpret/Failed/Rdtsc",      "The number of times RDTSC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedRdtsc,          "/EM/CPU%d/R3/Interpret/Failed/Rdtsc",      "The number of times RDTSC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedRdpmc,          "/EM/CPU%d/RZ/Interpret/Failed/Rdpmc",      "The number of times RDPMC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedRdpmc,          "/EM/CPU%d/R3/Interpret/Failed/Rdpmc",      "The number of times RDPMC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedRdmsr,          "/EM/CPU%d/RZ/Interpret/Failed/Rdmsr",      "The number of times RDMSR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedRdmsr,          "/EM/CPU%d/R3/Interpret/Failed/Rdmsr",      "The number of times RDMSR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedWrmsr,          "/EM/CPU%d/RZ/Interpret/Failed/Wrmsr",      "The number of times WRMSR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedWrmsr,          "/EM/CPU%d/R3/Interpret/Failed/Wrmsr",      "The number of times WRMSR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedLmsw,           "/EM/CPU%d/RZ/Interpret/Failed/Lmsw",       "The number of times LMSW was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedLmsw,           "/EM/CPU%d/R3/Interpret/Failed/Lmsw",       "The number of times LMSW was not interpreted.");
+
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedMisc,           "/EM/CPU%d/RZ/Interpret/Failed/Misc",       "The number of times some misc instruction was encountered.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedMisc,           "/EM/CPU%d/R3/Interpret/Failed/Misc",       "The number of times some misc instruction was encountered.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedAdd,            "/EM/CPU%d/RZ/Interpret/Failed/Add",        "The number of times ADD was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedAdd,            "/EM/CPU%d/R3/Interpret/Failed/Add",        "The number of times ADD was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedAdc,            "/EM/CPU%d/RZ/Interpret/Failed/Adc",        "The number of times ADC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedAdc,            "/EM/CPU%d/R3/Interpret/Failed/Adc",        "The number of times ADC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedBtr,            "/EM/CPU%d/RZ/Interpret/Failed/Btr",        "The number of times BTR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedBtr,            "/EM/CPU%d/R3/Interpret/Failed/Btr",        "The number of times BTR was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedBts,            "/EM/CPU%d/RZ/Interpret/Failed/Bts",        "The number of times BTS was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedBts,            "/EM/CPU%d/R3/Interpret/Failed/Bts",        "The number of times BTS was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedBtc,            "/EM/CPU%d/RZ/Interpret/Failed/Btc",        "The number of times BTC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedBtc,            "/EM/CPU%d/R3/Interpret/Failed/Btc",        "The number of times BTC was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedCli,            "/EM/CPU%d/RZ/Interpret/Failed/Cli",        "The number of times CLI was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedCli,            "/EM/CPU%d/R3/Interpret/Failed/Cli",        "The number of times CLI was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg,        "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg",    "The number of times CMPXCHG was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg,        "/EM/CPU%d/R3/Interpret/Failed/CmpXchg",    "The number of times CMPXCHG was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedCmpXchg8b,      "/EM/CPU%d/RZ/Interpret/Failed/CmpXchg8b",  "The number of times CMPXCHG8B was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedCmpXchg8b,      "/EM/CPU%d/R3/Interpret/Failed/CmpXchg8b",  "The number of times CMPXCHG8B was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedXAdd,           "/EM/CPU%d/RZ/Interpret/Failed/XAdd",       "The number of times XADD was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedXAdd,           "/EM/CPU%d/R3/Interpret/Failed/XAdd",       "The number of times XADD was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedMovNTPS,        "/EM/CPU%d/RZ/Interpret/Failed/MovNTPS",    "The number of times MOVNTPS was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedMovNTPS,        "/EM/CPU%d/R3/Interpret/Failed/MovNTPS",    "The number of times MOVNTPS was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedStosWD,         "/EM/CPU%d/RZ/Interpret/Failed/StosWD",     "The number of times STOSWD was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedStosWD,         "/EM/CPU%d/R3/Interpret/Failed/StosWD",     "The number of times STOSWD was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedSub,            "/EM/CPU%d/RZ/Interpret/Failed/Sub",        "The number of times SUB was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedSub,            "/EM/CPU%d/R3/Interpret/Failed/Sub",        "The number of times SUB was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedWbInvd,         "/EM/CPU%d/RZ/Interpret/Failed/WbInvd",     "The number of times WBINVD was not interpreted.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedWbInvd,         "/EM/CPU%d/R3/Interpret/Failed/WbInvd",     "The number of times WBINVD was not interpreted.");
+
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedUserMode,       "/EM/CPU%d/RZ/Interpret/Failed/UserMode",   "The number of rejections because of CPL.");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedUserMode,       "/EM/CPU%d/R3/Interpret/Failed/UserMode",   "The number of rejections because of CPL.");
+        EM_REG_COUNTER_USED(&pStats->StatRZFailedPrefix,         "/EM/CPU%d/RZ/Interpret/Failed/Prefix",     "The number of rejections because of prefix .");
+        EM_REG_COUNTER_USED(&pStats->StatR3FailedPrefix,         "/EM/CPU%d/R3/Interpret/Failed/Prefix",     "The number of rejections because of prefix .");
+
+        EM_REG_COUNTER_USED(&pStats->StatCli,                    "/EM/CPU%d/R3/PrivInst/Cli",                "Number of cli instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSti,                    "/EM/CPU%d/R3/PrivInst/Sti",                "Number of sli instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatIn,                     "/EM/CPU%d/R3/PrivInst/In",                 "Number of in instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatOut,                    "/EM/CPU%d/R3/PrivInst/Out",                "Number of out instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatIoRestarted,            "/EM/CPU%d/R3/PrivInst/IoRestarted",        "Number of restarted i/o instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatHlt,                    "/EM/CPU%d/R3/PrivInst/Hlt",                "Number of hlt instructions not handled in GC because of PATM.");
+        EM_REG_COUNTER_USED(&pStats->StatInvlpg,                 "/EM/CPU%d/R3/PrivInst/Invlpg",             "Number of invlpg instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMisc,                   "/EM/CPU%d/R3/PrivInst/Misc",               "Number of misc. instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[0],          "/EM/CPU%d/R3/PrivInst/Mov CR0, X",         "Number of mov CR0 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[1],          "/EM/CPU%d/R3/PrivInst/Mov CR1, X",         "Number of mov CR1 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[2],          "/EM/CPU%d/R3/PrivInst/Mov CR2, X",         "Number of mov CR2 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[3],          "/EM/CPU%d/R3/PrivInst/Mov CR3, X",         "Number of mov CR3 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovWriteCR[4],          "/EM/CPU%d/R3/PrivInst/Mov CR4, X",         "Number of mov CR4 read instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[0],           "/EM/CPU%d/R3/PrivInst/Mov X, CR0",         "Number of mov CR0 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[1],           "/EM/CPU%d/R3/PrivInst/Mov X, CR1",         "Number of mov CR1 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[2],           "/EM/CPU%d/R3/PrivInst/Mov X, CR2",         "Number of mov CR2 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[3],           "/EM/CPU%d/R3/PrivInst/Mov X, CR3",         "Number of mov CR3 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovReadCR[4],           "/EM/CPU%d/R3/PrivInst/Mov X, CR4",         "Number of mov CR4 write instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovDRx,                 "/EM/CPU%d/R3/PrivInst/MovDRx",             "Number of mov DRx instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatIret,                   "/EM/CPU%d/R3/PrivInst/Iret",               "Number of iret instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovLgdt,                "/EM/CPU%d/R3/PrivInst/Lgdt",               "Number of lgdt instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovLidt,                "/EM/CPU%d/R3/PrivInst/Lidt",               "Number of lidt instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatMovLldt,                "/EM/CPU%d/R3/PrivInst/Lldt",               "Number of lldt instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysEnter,               "/EM/CPU%d/R3/PrivInst/Sysenter",           "Number of sysenter instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysExit,                "/EM/CPU%d/R3/PrivInst/Sysexit",            "Number of sysexit instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysCall,                "/EM/CPU%d/R3/PrivInst/Syscall",            "Number of syscall instructions.");
+        EM_REG_COUNTER_USED(&pStats->StatSysRet,                 "/EM/CPU%d/R3/PrivInst/Sysret",             "Number of sysret instructions.");
+
+        EM_REG_COUNTER(&pVCpu->em.s.StatTotalClis,               "/EM/CPU%d/Cli/Total",                      "Total number of cli instructions executed.");
+        pVCpu->em.s.pCliStatTree = 0;
+
+        /* these should be considered for release statistics. */
+        EM_REG_COUNTER(&pVCpu->em.s.StatIOEmu,                 "/PROF/CPU%d/EM/Emulation/IO",      "Profiling of emR3RawExecuteIOInstruction.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatPrivEmu,               "/PROF/CPU%d/EM/Emulation/Priv",    "Profiling of emR3RawPrivileged.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatHwAccEntry,           "/PROF/CPU%d/EM/HwAccEnter",        "Profiling Hardware Accelerated Mode entry overhead.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatHwAccExec,            "/PROF/CPU%d/EM/HwAccExec",         "Profiling Hardware Accelerated Mode execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatREMEmu,               "/PROF/CPU%d/EM/REMEmuSingle",      "Profiling single instruction REM execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatREMExec,              "/PROF/CPU%d/EM/REMExec",           "Profiling REM execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatREMSync,              "/PROF/CPU%d/EM/REMSync",           "Profiling REM context syncing.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatRAWEntry,             "/PROF/CPU%d/EM/RAWEnter",          "Profiling Raw Mode entry overhead.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatRAWExec,              "/PROF/CPU%d/EM/RAWExec",           "Profiling Raw Mode execution.");
+        EM_REG_PROFILE(&pVCpu->em.s.StatRAWTail,              "/PROF/CPU%d/EM/RAWTail",           "Profiling Raw Mode tail overhead.");
+
+#endif /* VBOX_WITH_STATISTICS */
+
+        EM_REG_COUNTER(&pVCpu->em.s.StatForcedActions,     "/PROF/CPU%d/EM/ForcedActions",     "Profiling forced action execution.");
+        EM_REG_COUNTER(&pVCpu->em.s.StatHalted,            "/PROF/CPU%d/EM/Halted",            "Profiling halted state (VMR3WaitHalted).");
+        EM_REG_PROFILE_ADV(&pVCpu->em.s.StatCapped,        "/PROF/CPU%d/EM/Capped",            "Profiling capped state (sleep).");
+        EM_REG_COUNTER(&pVCpu->em.s.StatREMTotal,          "/PROF/CPU%d/EM/REMTotal",          "Profiling emR3RemExecute (excluding FFs).");
+        EM_REG_COUNTER(&pVCpu->em.s.StatRAWTotal,          "/PROF/CPU%d/EM/RAWTotal",          "Profiling emR3RawExecute (excluding FFs).");
+
+        EM_REG_PROFILE_ADV(&pVCpu->em.s.StatTotal,         "/PROF/CPU%d/EM/Total",             "Profiling EMR3ExecuteVM.");
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The VM.
+ */
+VMMR3DECL(void) EMR3Relocate(PVM pVM)
+{
+    LogFlow(("EMR3Relocate\n"));
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+        if (pVCpu->em.s.pStatsR3)
+            pVCpu->em.s.pStatsRC = MMHyperR3ToRC(pVM, pVCpu->em.s.pStatsR3);
+    }
+}
+
+
+/**
+ * Reset the EM state for a CPU.
+ *
+ * Called by EMR3Reset and hot plugging.
+ *
+ * @param   pVCpu               The virtual CPU.
+ */
+VMMR3DECL(void) EMR3ResetCpu(PVMCPU pVCpu)
+{
+    pVCpu->em.s.fForceRAW = false;
+
+    /* VMR3Reset may return VINF_EM_RESET or VINF_EM_SUSPEND, so transition
+       out of the HALTED state here so that enmPrevState doesn't end up as
+       HALTED when EMR3Execute returns. */
+    if (pVCpu->em.s.enmState == EMSTATE_HALTED)
+    {
+        Log(("EMR3ResetCpu: Cpu#%u %s -> %s\n", pVCpu->idCpu, emR3GetStateName(pVCpu->em.s.enmState), pVCpu->idCpu == 0 ? "EMSTATE_NONE" : "EMSTATE_WAIT_SIPI"));
+        pVCpu->em.s.enmState = pVCpu->idCpu == 0 ? EMSTATE_NONE : EMSTATE_WAIT_SIPI;
+    }
+}
+
+
+/**
+ * Reset notification.
+ *
+ * @param   pVM                 The VM handle.
+ */
+VMMR3DECL(void) EMR3Reset(PVM pVM)
+{
+    Log(("EMR3Reset: \n"));
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        EMR3ResetCpu(&pVM->aCpus[i]);
+}
+
+
+/**
+ * Terminates the EM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) EMR3Term(PVM pVM)
+{
+    AssertMsg(pVM->em.s.offVM, ("bad init order!\n"));
+
+    PDMR3CritSectDelete(&pVM->em.s.CritSectREM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) emR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        int rc = SSMR3PutBool(pSSM, pVCpu->em.s.fForceRAW);
+        AssertRCReturn(rc, rc);
+
+        Assert(pVCpu->em.s.enmState     == EMSTATE_SUSPENDED);
+        Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
+        rc = SSMR3PutU32(pSSM, pVCpu->em.s.enmPrevState);
+        AssertRCReturn(rc, rc);
+
+        /* Save mwait state. */
+        rc = SSMR3PutU32(pSSM, pVCpu->em.s.mwait.fWait);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMWaitEAX);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMWaitECX);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMonitorEAX);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMonitorECX);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutGCPtr(pSSM, pVCpu->em.s.mwait.uMonitorEDX);
+        AssertRCReturn(rc, rc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) emR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    /*
+     * Validate version.
+     */
+    if (    uVersion != EM_SAVED_STATE_VERSION
+        &&  uVersion != EM_SAVED_STATE_VERSION_PRE_MWAIT
+        &&  uVersion != EM_SAVED_STATE_VERSION_PRE_SMP)
+    {
+        AssertMsgFailed(("emR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, EM_SAVED_STATE_VERSION));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Load the saved state.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        int rc = SSMR3GetBool(pSSM, &pVCpu->em.s.fForceRAW);
+        if (RT_FAILURE(rc))
+            pVCpu->em.s.fForceRAW = false;
+        AssertRCReturn(rc, rc);
+
+        if (uVersion > EM_SAVED_STATE_VERSION_PRE_SMP)
+        {
+            AssertCompile(sizeof(pVCpu->em.s.enmPrevState) == sizeof(uint32_t));
+            rc = SSMR3GetU32(pSSM, (uint32_t *)&pVCpu->em.s.enmPrevState);
+            AssertRCReturn(rc, rc);
+            Assert(pVCpu->em.s.enmPrevState != EMSTATE_SUSPENDED);
+
+            pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
+        }
+        if (uVersion > EM_SAVED_STATE_VERSION_PRE_MWAIT)
+        {
+            /* Load mwait state. */
+            rc = SSMR3GetU32(pSSM, &pVCpu->em.s.mwait.fWait);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMWaitEAX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMWaitECX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMonitorEAX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMonitorECX);
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetGCPtr(pSSM, &pVCpu->em.s.mwait.uMonitorEDX);
+            AssertRCReturn(rc, rc);
+        }
+
+        Assert(!pVCpu->em.s.pCliStatTree);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Raise a fatal error.
+ *
+ * Safely terminate the VM with full state report and stuff. This function
+ * will naturally never return.
+ *
+ * @param   pVCpu       VMCPU handle.
+ * @param   rc          VBox status code.
+ */
+VMMR3DECL(void) EMR3FatalError(PVMCPU pVCpu, int rc)
+{
+    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+    longjmp(pVCpu->em.s.u.FatalLongJump, rc);
+    AssertReleaseMsgFailed(("longjmp returned!\n"));
+}
+
+
+/**
+ * Gets the EM state name.
+ *
+ * @returns pointer to read only state name,
+ * @param   enmState    The state.
+ */
+static const char *emR3GetStateName(EMSTATE enmState)
+{
+    switch (enmState)
+    {
+        case EMSTATE_NONE:              return "EMSTATE_NONE";
+        case EMSTATE_RAW:               return "EMSTATE_RAW";
+        case EMSTATE_HWACC:             return "EMSTATE_HWACC";
+        case EMSTATE_REM:               return "EMSTATE_REM";
+        case EMSTATE_PARAV:             return "EMSTATE_PARAV";
+        case EMSTATE_HALTED:            return "EMSTATE_HALTED";
+        case EMSTATE_WAIT_SIPI:         return "EMSTATE_WAIT_SIPI";
+        case EMSTATE_SUSPENDED:         return "EMSTATE_SUSPENDED";
+        case EMSTATE_TERMINATING:       return "EMSTATE_TERMINATING";
+        case EMSTATE_DEBUG_GUEST_RAW:   return "EMSTATE_DEBUG_GUEST_RAW";
+        case EMSTATE_DEBUG_GUEST_REM:   return "EMSTATE_DEBUG_GUEST_REM";
+        case EMSTATE_DEBUG_HYPER:       return "EMSTATE_DEBUG_HYPER";
+        case EMSTATE_GURU_MEDITATION:   return "EMSTATE_GURU_MEDITATION";
+        default:                        return "Unknown!";
+    }
+}
+
+
+/**
+ * Debug loop.
+ *
+ * @returns VBox status code for EM.
+ * @param   pVM     VM handle.
+ * @param   pVCpu   VMCPU handle.
+ * @param   rc      Current EM VBox status code..
+ */
+static int emR3Debug(PVM pVM, PVMCPU pVCpu, int rc)
+{
+    for (;;)
+    {
+        Log(("emR3Debug: rc=%Rrc\n", rc));
+        const int rcLast = rc;
+
+        /*
+         * Debug related RC.
+         */
+        switch (rc)
+        {
+            /*
+             * Single step an instruction.
+             */
+            case VINF_EM_DBG_STEP:
+                if (    pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
+                    ||  pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER
+                    ||  pVCpu->em.s.fForceRAW /* paranoia */)
+                    rc = emR3RawStep(pVM, pVCpu);
+                else
+                {
+                    Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM);
+                    rc = emR3RemStep(pVM, pVCpu);
+                }
+                break;
+
+            /*
+             * Simple events: stepped, breakpoint, stop/assertion.
+             */
+            case VINF_EM_DBG_STEPPED:
+                rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED);
+                break;
+
+            case VINF_EM_DBG_BREAKPOINT:
+                rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT);
+                break;
+
+            case VINF_EM_DBG_STOP:
+                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, NULL, 0, NULL, NULL);
+                break;
+
+            case VINF_EM_DBG_HYPER_STEPPED:
+                rc = DBGFR3Event(pVM, DBGFEVENT_STEPPED_HYPER);
+                break;
+
+            case VINF_EM_DBG_HYPER_BREAKPOINT:
+                rc = DBGFR3EventBreakpoint(pVM, DBGFEVENT_BREAKPOINT_HYPER);
+                break;
+
+            case VINF_EM_DBG_HYPER_ASSERTION:
+                RTPrintf("\nVINF_EM_DBG_HYPER_ASSERTION:\n%s%s\n", VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
+                rc = DBGFR3EventAssertion(pVM, DBGFEVENT_ASSERTION_HYPER, VMMR3GetRZAssertMsg1(pVM), VMMR3GetRZAssertMsg2(pVM));
+                break;
+
+            /*
+             * Guru meditation.
+             */
+            case VERR_VMM_RING0_ASSERTION: /** @todo Make a guru meditation event! */
+                rc = DBGFR3EventSrc(pVM, DBGFEVENT_FATAL_ERROR, "VERR_VMM_RING0_ASSERTION", 0, NULL, NULL);
+                break;
+            case VERR_REM_TOO_MANY_TRAPS: /** @todo Make a guru meditation event! */
+                rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, "VERR_REM_TOO_MANY_TRAPS", 0, NULL, NULL);
+                break;
+
+            default: /** @todo don't use default for guru, but make special errors code! */
+                rc = DBGFR3Event(pVM, DBGFEVENT_FATAL_ERROR);
+                break;
+        }
+
+        /*
+         * Process the result.
+         */
+        do
+        {
+            switch (rc)
+            {
+                /*
+                 * Continue the debugging loop.
+                 */
+                case VINF_EM_DBG_STEP:
+                case VINF_EM_DBG_STOP:
+                case VINF_EM_DBG_STEPPED:
+                case VINF_EM_DBG_BREAKPOINT:
+                case VINF_EM_DBG_HYPER_STEPPED:
+                case VINF_EM_DBG_HYPER_BREAKPOINT:
+                case VINF_EM_DBG_HYPER_ASSERTION:
+                    break;
+
+                /*
+                 * Resuming execution (in some form) has to be done here if we got
+                 * a hypervisor debug event.
+                 */
+                case VINF_SUCCESS:
+                case VINF_EM_RESUME:
+                case VINF_EM_SUSPEND:
+                case VINF_EM_RESCHEDULE:
+                case VINF_EM_RESCHEDULE_RAW:
+                case VINF_EM_RESCHEDULE_REM:
+                case VINF_EM_HALT:
+                    if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
+                    {
+                        rc = emR3RawResumeHyper(pVM, pVCpu);
+                        if (rc != VINF_SUCCESS && RT_SUCCESS(rc))
+                            continue;
+                    }
+                    if (rc == VINF_SUCCESS)
+                        rc = VINF_EM_RESCHEDULE;
+                    return rc;
+
+                /*
+                 * The debugger isn't attached.
+                 * We'll simply turn the thing off since that's the easiest thing to do.
+                 */
+                case VERR_DBGF_NOT_ATTACHED:
+                    switch (rcLast)
+                    {
+                        case VINF_EM_DBG_HYPER_STEPPED:
+                        case VINF_EM_DBG_HYPER_BREAKPOINT:
+                        case VINF_EM_DBG_HYPER_ASSERTION:
+                        case VERR_TRPM_PANIC:
+                        case VERR_TRPM_DONT_PANIC:
+                        case VERR_VMM_RING0_ASSERTION:
+                        case VERR_VMM_HYPER_CR3_MISMATCH:
+                        case VERR_VMM_RING3_CALL_DISABLED:
+                            return rcLast;
+                    }
+                    return VINF_EM_OFF;
+
+                /*
+                 * Status codes terminating the VM in one or another sense.
+                 */
+                case VINF_EM_TERMINATE:
+                case VINF_EM_OFF:
+                case VINF_EM_RESET:
+                case VINF_EM_NO_MEMORY:
+                case VINF_EM_RAW_STALE_SELECTOR:
+                case VINF_EM_RAW_IRET_TRAP:
+                case VERR_TRPM_PANIC:
+                case VERR_TRPM_DONT_PANIC:
+                case VERR_VMM_RING0_ASSERTION:
+                case VERR_VMM_HYPER_CR3_MISMATCH:
+                case VERR_VMM_RING3_CALL_DISABLED:
+                case VERR_INTERNAL_ERROR:
+                case VERR_INTERNAL_ERROR_2:
+                case VERR_INTERNAL_ERROR_3:
+                case VERR_INTERNAL_ERROR_4:
+                case VERR_INTERNAL_ERROR_5:
+                case VERR_IPE_UNEXPECTED_STATUS:
+                case VERR_IPE_UNEXPECTED_INFO_STATUS:
+                case VERR_IPE_UNEXPECTED_ERROR_STATUS:
+                    return rc;
+
+                /*
+                 * The rest is unexpected, and will keep us here.
+                 */
+                default:
+                    AssertMsgFailed(("Unexpected rc %Rrc!\n", rc));
+                    break;
+            }
+        } while (false);
+    } /* debug for ever */
+}
+
+/**
+ * Steps recompiled code.
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_STEP_EVENT,
+ *          VINF_EM_RESCHEDULE, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ */
+static int emR3RemStep(PVM pVM, PVMCPU pVCpu)
+{
+    LogFlow(("emR3RemStep: cs:eip=%04x:%08x\n", CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
+
+    EMRemLock(pVM);
+
+    /*
+     * Switch to REM, step instruction, switch back.
+     */
+    int rc = REMR3State(pVM, pVCpu);
+    if (RT_SUCCESS(rc))
+    {
+        rc = REMR3Step(pVM, pVCpu);
+        REMR3StateBack(pVM, pVCpu);
+    }
+    EMRemUnlock(pVM);
+
+    LogFlow(("emR3RemStep: returns %Rrc cs:eip=%04x:%08x\n", rc, CPUMGetGuestCS(pVCpu),  CPUMGetGuestEIP(pVCpu)));
+    return rc;
+}
+
+
+/**
+ * emR3RemExecute helper that syncs the state back from REM and leave the REM
+ * critical section.
+ *
+ * @returns false - new fInREMState value.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The virtual CPU handle.
+ */
+DECLINLINE(bool) emR3RemExecuteSyncBack(PVM pVM, PVMCPU pVCpu)
+{
+    STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, a);
+    REMR3StateBack(pVM, pVCpu);
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, a);
+
+    EMRemUnlock(pVM);
+    return false;
+}
+
+
+/**
+ * Executes recompiled code.
+ *
+ * This function contains the recompiler version of the inner
+ * execution loop (the outer loop being in EMR3ExecuteVM()).
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
+ *          VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   pfFFDone    Where to store an indicator telling whether or not
+ *                      FFs were done before returning.
+ *
+ */
+static int emR3RemExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+#ifdef LOG_ENABLED
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+    uint32_t cpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
+
+    if (pCtx->eflags.Bits.u1VM)
+        Log(("EMV86: %04X:%08X IF=%d\n", pCtx->cs, pCtx->eip, pCtx->eflags.Bits.u1IF));
+    else
+        Log(("EMR%d: %04X:%08X ESP=%08X IF=%d CR0=%x eflags=%x\n", cpl, pCtx->cs, pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, (uint32_t)pCtx->cr0, pCtx->eflags.u));
+#endif
+    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatREMTotal, a);
+
+#if defined(VBOX_STRICT) && defined(DEBUG_bird)
+    AssertMsg(   VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
+              || !MMHyperIsInsideArea(pVM, CPUMGetGuestEIP(pVCpu)),  /** @todo #1419 - get flat address. */
+              ("cs:eip=%RX16:%RX32\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
+#endif
+
+    /*
+     * Spin till we get a forced action which returns anything but VINF_SUCCESS
+     * or the REM suggests raw-mode execution.
+     */
+    *pfFFDone = false;
+    bool    fInREMState = false;
+    int     rc          = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Lock REM and update the state if not already in sync.
+         *
+         * Note! Big lock, but you are not supposed to own any lock when
+         *       coming in here.
+         */
+        if (!fInREMState)
+        {
+            EMRemLock(pVM);
+            STAM_PROFILE_START(&pVCpu->em.s.StatREMSync, b);
+
+            /* Flush the recompiler translation blocks if the VCPU has changed,
+               also force a full CPU state resync. */
+            if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
+            {
+                REMFlushTBs(pVM);
+                CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
+            }
+            pVM->em.s.idLastRemCpu = pVCpu->idCpu;
+
+            rc = REMR3State(pVM, pVCpu);
+
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatREMSync, b);
+            if (RT_FAILURE(rc))
+                break;
+            fInREMState = true;
+
+            /*
+             * We might have missed the raising of VMREQ, TIMER and some other
+             * important FFs while we were busy switching the state. So, check again.
+             */
+            if (    VM_FF_ISPENDING(pVM, VM_FF_REQUEST | VM_FF_PDM_QUEUES | VM_FF_DBGF | VM_FF_CHECK_VM_STATE | VM_FF_RESET)
+                ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TIMER | VMCPU_FF_REQUEST))
+            {
+                LogFlow(("emR3RemExecute: Skipping run, because FF is set. %#x\n", pVM->fGlobalForcedActions));
+                goto l_REMDoForcedActions;
+            }
+        }
+
+
+        /*
+         * Execute REM.
+         */
+        if (RT_LIKELY(EMR3IsExecutionAllowed(pVM, pVCpu)))
+        {
+            STAM_PROFILE_START(&pVCpu->em.s.StatREMExec, c);
+            rc = REMR3Run(pVM, pVCpu);
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatREMExec, c);
+        }
+        else
+        {
+            /* Give up this time slice; virtual time continues */
+            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
+            RTThreadSleep(5);
+            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
+            rc = VINF_SUCCESS;
+        }
+
+        /*
+         * Deal with high priority post execution FFs before doing anything
+         * else.  Sync back the state and leave the lock to be on the safe side.
+         */
+        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+        {
+            fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
+            rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
+        }
+
+        /*
+         * Process the returned status code.
+         */
+        if (rc != VINF_SUCCESS)
+        {
+            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+                break;
+            if (rc != VINF_REM_INTERRUPED_FF)
+            {
+                /*
+                 * Anything which is not known to us means an internal error
+                 * and the termination of the VM!
+                 */
+                AssertMsg(rc == VERR_REM_TOO_MANY_TRAPS, ("Unknown GC return code: %Rra\n", rc));
+                break;
+            }
+        }
+
+
+        /*
+         * Check and execute forced actions.
+         *
+         * Sync back the VM state and leave the lock  before calling any of
+         * these, you never know what's going to happen here.
+         */
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+        TMTimerPollVoid(pVM, pVCpu);
+#endif
+        AssertCompile((VMCPU_FF_ALL_BUT_RAW_MASK & ~(VMCPU_FF_CSAM_PENDING_ACTION | VMCPU_FF_CSAM_SCAN_PAGE)) & VMCPU_FF_TIMER);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_ALL_BUT_RAW_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_BUT_RAW_MASK & ~(VMCPU_FF_CSAM_PENDING_ACTION | VMCPU_FF_CSAM_SCAN_PAGE)))
+        {
+l_REMDoForcedActions:
+            if (fInREMState)
+                fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
+            STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatREMTotal, a);
+            rc = emR3ForcedActions(pVM, pVCpu, rc);
+            STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatREMTotal, a);
+            if (    rc != VINF_SUCCESS
+                &&  rc != VINF_EM_RESCHEDULE_REM)
+            {
+                *pfFFDone = true;
+                break;
+            }
+        }
+
+    } /* The Inner Loop, recompiled execution mode version. */
+
+
+    /*
+     * Returning. Sync back the VM state if required.
+     */
+    if (fInREMState)
+        fInREMState = emR3RemExecuteSyncBack(pVM, pVCpu);
+
+    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatREMTotal, a);
+    return rc;
+}
+
+
+#ifdef DEBUG
+
+int emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
+{
+    EMSTATE  enmOldState = pVCpu->em.s.enmState;
+
+    pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
+
+    Log(("Single step BEGIN:\n"));
+    for (uint32_t i = 0; i < cIterations; i++)
+    {
+        DBGFR3PrgStep(pVCpu);
+        DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
+        emR3RemStep(pVM, pVCpu);
+        if (emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx) != EMSTATE_REM)
+            break;
+    }
+    Log(("Single step END:\n"));
+    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
+    pVCpu->em.s.enmState = enmOldState;
+    return VINF_EM_RESCHEDULE;
+}
+
+#endif /* DEBUG */
+
+
+/**
+ * Decides whether to execute RAW, HWACC or REM.
+ *
+ * @returns new EM state
+ * @param   pVM     The VM.
+ * @param   pVCpu   The VMCPU handle.
+ * @param   pCtx    The CPU context.
+ */
+EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    /*
+     * When forcing raw-mode execution, things are simple.
+     */
+    if (pVCpu->em.s.fForceRAW)
+        return EMSTATE_RAW;
+
+    /*
+     * We stay in the wait for SIPI state unless explicitly told otherwise.
+     */
+    if (pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI)
+        return EMSTATE_WAIT_SIPI;
+
+    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
+    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
+    /* !!! THIS MUST BE IN SYNC WITH remR3CanExecuteRaw !!! */
+
+    X86EFLAGS EFlags = pCtx->eflags;
+    if (HWACCMIsEnabled(pVM))
+    {
+        /* Hardware accelerated raw-mode:
+         *
+         * Typically only 32-bits protected mode, with paging enabled, code is allowed here.
+         */
+        if (HWACCMR3CanExecuteGuest(pVM, pCtx) == true)
+            return EMSTATE_HWACC;
+
+        /* Note: Raw mode and hw accelerated mode are incompatible. The latter turns
+         * off monitoring features essential for raw mode! */
+        return EMSTATE_REM;
+    }
+
+    /*
+     * Standard raw-mode:
+     *
+     * Here we only support 16 & 32 bits protected mode ring 3 code that has no IO privileges
+     * or 32 bits protected mode ring 0 code
+     *
+     * The tests are ordered by the likelihood of being true during normal execution.
+     */
+    if (EFlags.u32 & (X86_EFL_TF /* | HF_INHIBIT_IRQ_MASK*/))
+    {
+        Log2(("raw mode refused: EFlags=%#x\n", EFlags.u32));
+        return EMSTATE_REM;
+    }
+
+#ifndef VBOX_RAW_V86
+    if (EFlags.u32 & X86_EFL_VM) {
+        Log2(("raw mode refused: VM_MASK\n"));
+        return EMSTATE_REM;
+    }
+#endif
+
+    /** @todo check up the X86_CR0_AM flag in respect to raw mode!!! We're probably not emulating it right! */
+    uint32_t u32CR0 = pCtx->cr0;
+    if ((u32CR0 & (X86_CR0_PG | X86_CR0_PE)) != (X86_CR0_PG | X86_CR0_PE))
+    {
+        //Log2(("raw mode refused: %s%s%s\n", (u32CR0 & X86_CR0_PG) ? "" : " !PG", (u32CR0 & X86_CR0_PE) ? "" : " !PE", (u32CR0 & X86_CR0_AM) ? "" : " !AM"));
+        return EMSTATE_REM;
+    }
+
+    if (pCtx->cr4 & X86_CR4_PAE)
+    {
+        uint32_t u32Dummy, u32Features;
+
+        CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
+        if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
+            return EMSTATE_REM;
+    }
+
+    unsigned uSS = pCtx->ss;
+    if (    pCtx->eflags.Bits.u1VM
+        ||  (uSS & X86_SEL_RPL) == 3)
+    {
+        if (!EMIsRawRing3Enabled(pVM))
+            return EMSTATE_REM;
+
+        if (!(EFlags.u32 & X86_EFL_IF))
+        {
+            Log2(("raw mode refused: IF (RawR3)\n"));
+            return EMSTATE_REM;
+        }
+
+        if (!(u32CR0 & X86_CR0_WP) && EMIsRawRing0Enabled(pVM))
+        {
+            Log2(("raw mode refused: CR0.WP + RawR0\n"));
+            return EMSTATE_REM;
+        }
+    }
+    else
+    {
+        if (!EMIsRawRing0Enabled(pVM))
+            return EMSTATE_REM;
+
+        /* Only ring 0 supervisor code. */
+        if ((uSS & X86_SEL_RPL) != 0)
+        {
+            Log2(("raw r0 mode refused: CPL %d\n", uSS & X86_SEL_RPL));
+            return EMSTATE_REM;
+        }
+
+        // Let's start with pure 32 bits ring 0 code first
+        /** @todo What's pure 32-bit mode? flat? */
+        if (    !(pCtx->ssHid.Attr.n.u1DefBig)
+            ||  !(pCtx->csHid.Attr.n.u1DefBig))
+        {
+            Log2(("raw r0 mode refused: SS/CS not 32bit\n"));
+            return EMSTATE_REM;
+        }
+
+        /* Write protection must be turned on, or else the guest can overwrite our hypervisor code and data. */
+        if (!(u32CR0 & X86_CR0_WP))
+        {
+            Log2(("raw r0 mode refused: CR0.WP=0!\n"));
+            return EMSTATE_REM;
+        }
+
+        if (PATMShouldUseRawMode(pVM, (RTGCPTR)pCtx->eip))
+        {
+            Log2(("raw r0 mode forced: patch code\n"));
+            return EMSTATE_RAW;
+        }
+
+#if !defined(VBOX_ALLOW_IF0) && !defined(VBOX_RUN_INTERRUPT_GATE_HANDLERS)
+        if (!(EFlags.u32 & X86_EFL_IF))
+        {
+            ////Log2(("R0: IF=0 VIF=%d %08X\n", eip, pVMeflags));
+            //Log2(("RR0: Interrupts turned off; fall back to emulation\n"));
+            return EMSTATE_REM;
+        }
+#endif
+
+        /** @todo still necessary??? */
+        if (EFlags.Bits.u2IOPL != 0)
+        {
+            Log2(("raw r0 mode refused: IOPL %d\n", EFlags.Bits.u2IOPL));
+            return EMSTATE_REM;
+        }
+    }
+
+    Assert(PGMPhysIsA20Enabled(pVCpu));
+    return EMSTATE_RAW;
+}
+
+
+/**
+ * Executes all high priority post execution force actions.
+ *
+ * @returns rc or a fatal status code.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   rc          The current rc.
+ */
+int emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
+{
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
+        PDMCritSectFF(pVCpu);
+
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_CSAM_PENDING_ACTION))
+        CSAMR3DoPendingAction(pVM, pVCpu);
+
+    if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
+    {
+        if (    rc > VINF_EM_NO_MEMORY
+            &&  rc <= VINF_EM_LAST)
+            rc = VINF_EM_NO_MEMORY;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Executes all pending forced actions.
+ *
+ * Forced actions can cause execution delays and execution
+ * rescheduling. The first we deal with using action priority, so
+ * that for instance pending timers aren't scheduled and ran until
+ * right before execution. The rescheduling we deal with using
+ * return codes. The same goes for VM termination, only in that case
+ * we exit everything.
+ *
+ * @returns VBox status code of equal or greater importance/severity than rc.
+ *          The most important ones are: VINF_EM_RESCHEDULE,
+ *          VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   rc          The current rc.
+ *
+ */
+int emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc)
+{
+    STAM_REL_PROFILE_START(&pVCpu->em.s.StatForcedActions, a);
+#ifdef VBOX_STRICT
+    int rcIrq = VINF_SUCCESS;
+#endif
+    int rc2;
+#define UPDATE_RC() \
+        do { \
+            AssertMsg(rc2 <= 0 || (rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST), ("Invalid FF return code: %Rra\n", rc2)); \
+            if (rc2 == VINF_SUCCESS || rc < VINF_SUCCESS) \
+                break; \
+            if (!rc || rc2 < rc) \
+                rc = rc2; \
+        } while (0)
+
+    /*
+     * Post execution chunk first.
+     */
+    if (    VM_FF_ISPENDING(pVM, VM_FF_NORMAL_PRIORITY_POST_MASK)
+        ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_POST_MASK))
+    {
+        /*
+         * EMT Rendezvous (must be serviced before termination).
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
+        {
+            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
+             * stopped/reset before the next VM state change is made. We need a better
+             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
+             * && rc >= VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /*
+         * State change request (cleared by vmR3SetStateLocked).
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_CHECK_VM_STATE))
+        {
+            VMSTATE enmState = VMR3GetState(pVM);
+            switch (enmState)
+            {
+                case VMSTATE_FATAL_ERROR:
+                case VMSTATE_FATAL_ERROR_LS:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_SUSPEND;
+
+                case VMSTATE_DESTROYING:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_TERMINATE;
+
+                default:
+                    AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
+            }
+        }
+
+        /*
+         * Debugger Facility polling.
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_DBGF))
+        {
+            rc2 = DBGFR3VMMForcedAction(pVM);
+            UPDATE_RC();
+        }
+
+        /*
+         * Postponed reset request.
+         */
+        if (VM_FF_TESTANDCLEAR(pVM, VM_FF_RESET))
+        {
+            rc2 = VMR3Reset(pVM);
+            UPDATE_RC();
+        }
+
+        /*
+         * CSAM page scanning.
+         */
+        if (    !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
+            &&  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE))
+        {
+            PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+
+            /** @todo: check for 16 or 32 bits code! (D bit in the code selector) */
+            Log(("Forced action VMCPU_FF_CSAM_SCAN_PAGE\n"));
+
+            CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_CSAM_SCAN_PAGE);
+        }
+
+        /*
+         * Out of memory? Putting this after CSAM as it may in theory cause us to run out of memory.
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = PGMR3PhysAllocateHandyPages(pVM);
+            UPDATE_RC();
+            if (rc == VINF_EM_NO_MEMORY)
+                return rc;
+        }
+
+        /* check that we got them all  */
+        AssertCompile(VM_FF_NORMAL_PRIORITY_POST_MASK == (VM_FF_CHECK_VM_STATE | VM_FF_DBGF | VM_FF_RESET | VM_FF_PGM_NO_MEMORY | VM_FF_EMT_RENDEZVOUS));
+        AssertCompile(VMCPU_FF_NORMAL_PRIORITY_POST_MASK == VMCPU_FF_CSAM_SCAN_PAGE);
+    }
+
+    /*
+     * Normal priority then.
+     * (Executed in no particular order.)
+     */
+    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_NORMAL_PRIORITY_MASK, VM_FF_PGM_NO_MEMORY))
+    {
+        /*
+         * PDM Queues are pending.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_QUEUES, VM_FF_PGM_NO_MEMORY))
+            PDMR3QueueFlushAll(pVM);
+
+        /*
+         * PDM DMA transfers are pending.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PDM_DMA, VM_FF_PGM_NO_MEMORY))
+            PDMR3DmaRun(pVM);
+
+        /*
+         * EMT Rendezvous (make sure they are handled before the requests).
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
+        {
+            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
+             * stopped/reset before the next VM state change is made. We need a better
+             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
+             * && rc >= VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /*
+         * Requests from other threads.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REQUEST, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+            if (rc2 == VINF_EM_OFF || rc2 == VINF_EM_TERMINATE) /** @todo this shouldn't be necessary */
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc2;
+            }
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
+             * stopped/reset before the next VM state change is made. We need a better
+             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
+             * && rc >= VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /* Replay the handler notification changes. */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_REM_HANDLER_NOTIFY, VM_FF_PGM_NO_MEMORY))
+        {
+            /* Try not to cause deadlocks. */
+            if (    pVM->cCpus == 1
+                ||  (   !PGMIsLockOwner(pVM)
+                     && !IOMIsLockOwner(pVM))
+               )
+            {
+                EMRemLock(pVM);
+                REMR3ReplayHandlerNotifications(pVM);
+                EMRemUnlock(pVM);
+            }
+        }
+
+        /* check that we got them all  */
+        AssertCompile(VM_FF_NORMAL_PRIORITY_MASK == (VM_FF_REQUEST | VM_FF_PDM_QUEUES | VM_FF_PDM_DMA | VM_FF_REM_HANDLER_NOTIFY | VM_FF_EMT_RENDEZVOUS));
+    }
+
+    /*
+     * Normal priority then. (per-VCPU)
+     * (Executed in no particular order.)
+     */
+    if (    !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
+        &&  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_NORMAL_PRIORITY_MASK))
+    {
+        /*
+         * Requests from other threads.
+         */
+        if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
+        {
+            rc2 = VMR3ReqProcessU(pVM->pUVM, pVCpu->idCpu);
+            if (rc2 == VINF_EM_OFF || rc2 == VINF_EM_TERMINATE || rc2 == VINF_EM_RESET)
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc2));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc2;
+            }
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM things the VM is
+             * stopped/reset before the next VM state change is made. We need a better
+             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
+             * && rc >= VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /* check that we got them all  */
+        Assert(!(VMCPU_FF_NORMAL_PRIORITY_MASK & ~(VMCPU_FF_REQUEST)));
+    }
+
+    /*
+     * High priority pre execution chunk last.
+     * (Executed in ascending priority order.)
+     */
+    if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_MASK)
+        ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_MASK))
+    {
+        /*
+         * Timers before interrupts.
+         */
+        if (    VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TIMER)
+            &&  !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
+            TMR3TimerQueuesDo(pVM);
+
+        /*
+         * The instruction following an emulated STI should *always* be executed!
+         */
+        if (    VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+            &&  !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
+        {
+            Log(("VM_FF_EMULATED_STI at %RGv successor %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu), EMGetInhibitInterruptsPC(pVCpu)));
+            if (CPUMGetGuestEIP(pVCpu) != EMGetInhibitInterruptsPC(pVCpu))
+            {
+                /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here if the eip is the same as the inhibited instr address.
+                 *  Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
+                 *  force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
+                 *  break the guest. Sounds very unlikely, but such timing sensitive problem are not as rare as you might think.
+                 */
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
+            }
+            if (HWACCMR3IsActive(pVCpu))
+                rc2 = VINF_EM_RESCHEDULE_HWACC;
+            else
+                rc2 = PATMAreInterruptsEnabled(pVM) ? VINF_EM_RESCHEDULE_RAW : VINF_EM_RESCHEDULE_REM;
+
+            UPDATE_RC();
+        }
+
+        /*
+         * Interrupts.
+         */
+        if (    !VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)
+            &&  !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
+            &&  (!rc || rc >= VINF_EM_RESCHEDULE_HWACC)
+            &&  !TRPMHasTrap(pVCpu) /* an interrupt could already be scheduled for dispatching in the recompiler. */
+            &&  PATMAreInterruptsEnabled(pVM)
+            &&  !HWACCMR3IsEventPending(pVCpu))
+        {
+            Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
+            if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
+            {
+                /* Note: it's important to make sure the return code from TRPMR3InjectEvent isn't ignored! */
+                /** @todo this really isn't nice, should properly handle this */
+                rc2 = TRPMR3InjectEvent(pVM, pVCpu, TRPM_HARDWARE_INT);
+#ifdef VBOX_STRICT
+                rcIrq = rc2;
+#endif
+                UPDATE_RC();
+            }
+            /** @todo really ugly; if we entered the hlt state when exiting the recompiler and an interrupt was pending, we previously got stuck in the halted state. */
+            else if (REMR3QueryPendingInterrupt(pVM, pVCpu) != REM_NO_PENDING_IRQ)
+            {
+                rc2 = VINF_EM_RESCHEDULE_REM;
+                UPDATE_RC();
+            }
+        }
+
+        /*
+         * Allocate handy pages.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = PGMR3PhysAllocateHandyPages(pVM);
+            UPDATE_RC();
+        }
+
+        /*
+         * Debugger Facility request.
+         */
+        if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_DBGF, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = DBGFR3VMMForcedAction(pVM);
+            UPDATE_RC();
+        }
+
+        /*
+         * EMT Rendezvous (must be serviced before termination).
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
+        {
+            rc2 = VMMR3EmtRendezvousFF(pVM, pVCpu);
+            UPDATE_RC();
+            /** @todo HACK ALERT! The following test is to make sure EM+TM thinks the VM is
+             * stopped/reset before the next VM state change is made. We need a better
+             * solution for this, or at least make it possible to do: (rc >= VINF_EM_FIRST
+             * && rc >= VINF_EM_SUSPEND). */
+            if (RT_UNLIKELY(rc == VINF_EM_SUSPEND || rc == VINF_EM_RESET || rc == VINF_EM_OFF))
+            {
+                Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+                STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                return rc;
+            }
+        }
+
+        /*
+         * State change request (cleared by vmR3SetStateLocked).
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_CHECK_VM_STATE))
+        {
+            VMSTATE enmState = VMR3GetState(pVM);
+            switch (enmState)
+            {
+                case VMSTATE_FATAL_ERROR:
+                case VMSTATE_FATAL_ERROR_LS:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_SUSPEND\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_SUSPEND;
+
+                case VMSTATE_DESTROYING:
+                    Log2(("emR3ForcedActions: %s -> VINF_EM_TERMINATE\n", VMGetStateName(enmState) ));
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+                    return VINF_EM_TERMINATE;
+
+                default:
+                    AssertMsgFailed(("%s\n", VMGetStateName(enmState)));
+            }
+        }
+
+        /*
+         * Out of memory? Since most of our fellow high priority actions may cause us
+         * to run out of memory, we're employing VM_FF_IS_PENDING_EXCEPT and putting this
+         * at the end rather than the start. Also, VM_FF_TERMINATE has higher priority
+         * than us since we can terminate without allocating more memory.
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
+        {
+            rc2 = PGMR3PhysAllocateHandyPages(pVM);
+            UPDATE_RC();
+            if (rc == VINF_EM_NO_MEMORY)
+                return rc;
+        }
+
+        /*
+         * If the virtual sync clock is still stopped, make TM restart it.
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_TM_VIRTUAL_SYNC))
+            TMR3VirtualSyncFF(pVM, pVCpu);
+
+#ifdef DEBUG
+        /*
+         * Debug, pause the VM.
+         */
+        if (VM_FF_ISPENDING(pVM, VM_FF_DEBUG_SUSPEND))
+        {
+            VM_FF_CLEAR(pVM, VM_FF_DEBUG_SUSPEND);
+            Log(("emR3ForcedActions: returns VINF_EM_SUSPEND\n"));
+            return VINF_EM_SUSPEND;
+        }
+#endif
+
+        /* check that we got them all  */
+        AssertCompile(VM_FF_HIGH_PRIORITY_PRE_MASK == (VM_FF_TM_VIRTUAL_SYNC | VM_FF_DBGF | VM_FF_CHECK_VM_STATE | VM_FF_DEBUG_SUSPEND | VM_FF_PGM_NEED_HANDY_PAGES | VM_FF_PGM_NO_MEMORY | VM_FF_EMT_RENDEZVOUS));
+        AssertCompile(VMCPU_FF_HIGH_PRIORITY_PRE_MASK == (VMCPU_FF_TIMER | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_SELM_SYNC_TSS | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_INHIBIT_INTERRUPTS));
+    }
+
+#undef UPDATE_RC
+    Log2(("emR3ForcedActions: returns %Rrc\n", rc));
+    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatForcedActions, a);
+    Assert(rcIrq == VINF_SUCCESS || rcIrq == rc);
+    return rc;
+}
+
+
+/**
+ * Check if the preset execution time cap restricts guest execution scheduling.
+ *
+ * @returns true if allowed, false otherwise
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VMCPU to operate on.
+ *
+ */
+VMMR3DECL(bool) EMR3IsExecutionAllowed(PVM pVM, PVMCPU pVCpu)
+{
+    uint64_t u64UserTime, u64KernelTime;
+
+    if (    pVM->uCpuExecutionCap != 100
+        &&  RT_SUCCESS(RTThreadGetExecutionTimeMilli(&u64KernelTime, &u64UserTime)))
+    {
+        uint64_t u64TimeNow = RTTimeMilliTS();
+        if (pVCpu->em.s.u64TimeSliceStart + EM_TIME_SLICE < u64TimeNow)
+        {
+            /* New time slice. */
+            pVCpu->em.s.u64TimeSliceStart     = u64TimeNow;
+            pVCpu->em.s.u64TimeSliceStartExec = u64KernelTime + u64UserTime;
+            pVCpu->em.s.u64TimeSliceExec      = 0;
+        }
+        pVCpu->em.s.u64TimeSliceExec = u64KernelTime + u64UserTime - pVCpu->em.s.u64TimeSliceStartExec;
+
+        Log2(("emR3IsExecutionAllowed: start=%RX64 startexec=%RX64 exec=%RX64 (cap=%x)\n", pVCpu->em.s.u64TimeSliceStart, pVCpu->em.s.u64TimeSliceStartExec, pVCpu->em.s.u64TimeSliceExec, (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100));
+        if (pVCpu->em.s.u64TimeSliceExec >= (EM_TIME_SLICE * pVM->uCpuExecutionCap) / 100)
+            return false;
+    }
+    return true;
+}
+
+
+/**
+ * Execute VM.
+ *
+ * This function is the main loop of the VM. The emulation thread
+ * calls this function when the VM has been successfully constructed
+ * and we're ready for executing the VM.
+ *
+ * Returning from this function means that the VM is turned off or
+ * suspended (state already saved) and deconstruction in next in line.
+ *
+ * All interaction from other thread are done using forced actions
+ * and signaling of the wait object.
+ *
+ * @returns VBox status code, informational status codes may indicate failure.
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VMCPU to operate on.
+ */
+VMMR3DECL(int) EMR3ExecuteVM(PVM pVM, PVMCPU pVCpu)
+{
+    Log(("EMR3ExecuteVM: pVM=%p enmVMState=%d (%s)  enmState=%d (%s) enmPrevState=%d (%s) fForceRAW=%RTbool\n",
+         pVM,
+         pVM->enmVMState,          VMR3GetStateName(pVM->enmVMState),
+         pVCpu->em.s.enmState,     emR3GetStateName(pVCpu->em.s.enmState),
+         pVCpu->em.s.enmPrevState, emR3GetStateName(pVCpu->em.s.enmPrevState),
+         pVCpu->em.s.fForceRAW));
+    VM_ASSERT_EMT(pVM);
+    AssertMsg(   pVCpu->em.s.enmState == EMSTATE_NONE
+              || pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI
+              || pVCpu->em.s.enmState == EMSTATE_SUSPENDED,
+              ("%s\n", emR3GetStateName(pVCpu->em.s.enmState)));
+
+    int rc = setjmp(pVCpu->em.s.u.FatalLongJump);
+    if (rc == 0)
+    {
+        /*
+         * Start the virtual time.
+         */
+        TMR3NotifyResume(pVM, pVCpu);
+
+        /*
+         * The Outer Main Loop.
+         */
+        bool fFFDone = false;
+
+        /* Reschedule right away to start in the right state. */
+        rc = VINF_SUCCESS;
+
+        /* If resuming after a pause or a state load, restore the previous
+           state or else we'll start executing code. Else, just reschedule. */
+        if (    pVCpu->em.s.enmState == EMSTATE_SUSPENDED
+            &&  (   pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
+                 || pVCpu->em.s.enmPrevState == EMSTATE_HALTED))
+            pVCpu->em.s.enmState = pVCpu->em.s.enmPrevState;
+        else
+            pVCpu->em.s.enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
+
+        STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
+        for (;;)
+        {
+            /*
+             * Before we can schedule anything (we're here because
+             * scheduling is required) we must service any pending
+             * forced actions to avoid any pending action causing
+             * immediate rescheduling upon entering an inner loop
+             *
+             * Do forced actions.
+             */
+            if (   !fFFDone
+                && rc != VINF_EM_TERMINATE
+                && rc != VINF_EM_OFF
+                && (   VM_FF_ISPENDING(pVM, VM_FF_ALL_BUT_RAW_MASK)
+                    || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_BUT_RAW_MASK)))
+            {
+                rc = emR3ForcedActions(pVM, pVCpu, rc);
+                if (    (   rc == VINF_EM_RESCHEDULE_REM
+                         || rc == VINF_EM_RESCHEDULE_HWACC)
+                    &&  pVCpu->em.s.fForceRAW)
+                    rc = VINF_EM_RESCHEDULE_RAW;
+            }
+            else if (fFFDone)
+                fFFDone = false;
+
+            /*
+             * Now what to do?
+             */
+            Log2(("EMR3ExecuteVM: rc=%Rrc\n", rc));
+            switch (rc)
+            {
+                /*
+                 * Keep doing what we're currently doing.
+                 */
+                case VINF_SUCCESS:
+                    break;
+
+                /*
+                 * Reschedule - to raw-mode execution.
+                 */
+                case VINF_EM_RESCHEDULE_RAW:
+                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_RAW: %d -> %d (EMSTATE_RAW)\n", pVCpu->em.s.enmState, EMSTATE_RAW));
+                    pVCpu->em.s.enmState = EMSTATE_RAW;
+                    break;
+
+                /*
+                 * Reschedule - to hardware accelerated raw-mode execution.
+                 */
+                case VINF_EM_RESCHEDULE_HWACC:
+                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_HWACC: %d -> %d (EMSTATE_HWACC)\n", pVCpu->em.s.enmState, EMSTATE_HWACC));
+                    Assert(!pVCpu->em.s.fForceRAW);
+                    pVCpu->em.s.enmState = EMSTATE_HWACC;
+                    break;
+
+                /*
+                 * Reschedule - to recompiled execution.
+                 */
+                case VINF_EM_RESCHEDULE_REM:
+                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE_REM: %d -> %d (EMSTATE_REM)\n", pVCpu->em.s.enmState, EMSTATE_REM));
+                    pVCpu->em.s.enmState = EMSTATE_REM;
+                    break;
+
+                /*
+                 * Resume.
+                 */
+                case VINF_EM_RESUME:
+                    Log2(("EMR3ExecuteVM: VINF_EM_RESUME: %d -> VINF_EM_RESCHEDULE\n", pVCpu->em.s.enmState));
+                    /* Don't reschedule in the halted or wait for SIPI case. */
+                    if (    pVCpu->em.s.enmPrevState == EMSTATE_WAIT_SIPI
+                        ||  pVCpu->em.s.enmPrevState == EMSTATE_HALTED)
+                        break;
+                    /* fall through and get scheduled. */
+
+                /*
+                 * Reschedule.
+                 */
+                case VINF_EM_RESCHEDULE:
+                {
+                    EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
+                    Log2(("EMR3ExecuteVM: VINF_EM_RESCHEDULE: %d -> %d (%s)\n", pVCpu->em.s.enmState, enmState, emR3GetStateName(enmState)));
+                    pVCpu->em.s.enmState = enmState;
+                    break;
+                }
+
+                /*
+                 * Halted.
+                 */
+                case VINF_EM_HALT:
+                    Log2(("EMR3ExecuteVM: VINF_EM_HALT: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_HALTED));
+                    pVCpu->em.s.enmState = EMSTATE_HALTED;
+                    break;
+
+                /*
+                 * Switch to the wait for SIPI state (application processor only)
+                 */
+                case VINF_EM_WAIT_SIPI:
+                    Assert(pVCpu->idCpu != 0);
+                    Log2(("EMR3ExecuteVM: VINF_EM_WAIT_SIPI: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_WAIT_SIPI));
+                    pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
+                    break;
+
+
+                /*
+                 * Suspend.
+                 */
+                case VINF_EM_SUSPEND:
+                    Log2(("EMR3ExecuteVM: VINF_EM_SUSPEND: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_SUSPENDED));
+                    pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
+                    pVCpu->em.s.enmState     = EMSTATE_SUSPENDED;
+                    break;
+
+                /*
+                 * Reset.
+                 * We might end up doing a double reset for now, we'll have to clean up the mess later.
+                 */
+                case VINF_EM_RESET:
+                {
+                    if (pVCpu->idCpu == 0)
+                    {
+                        EMSTATE enmState = emR3Reschedule(pVM, pVCpu, pVCpu->em.s.pCtx);
+                        Log2(("EMR3ExecuteVM: VINF_EM_RESET: %d -> %d (%s)\n", pVCpu->em.s.enmState, enmState, emR3GetStateName(enmState)));
+                        pVCpu->em.s.enmState = enmState;
+                    }
+                    else
+                    {
+                        /* All other VCPUs go into the wait for SIPI state. */
+                        pVCpu->em.s.enmState = EMSTATE_WAIT_SIPI;
+                    }
+                    break;
+                }
+
+                /*
+                 * Power Off.
+                 */
+                case VINF_EM_OFF:
+                    pVCpu->em.s.enmState = EMSTATE_TERMINATING;
+                    Log2(("EMR3ExecuteVM: returns VINF_EM_OFF (%d -> %d)\n", pVCpu->em.s.enmState, EMSTATE_TERMINATING));
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    return rc;
+
+                /*
+                 * Terminate the VM.
+                 */
+                case VINF_EM_TERMINATE:
+                    pVCpu->em.s.enmState = EMSTATE_TERMINATING;
+                    Log(("EMR3ExecuteVM returns VINF_EM_TERMINATE (%d -> %d)\n", pVCpu->em.s.enmState, EMSTATE_TERMINATING));
+                    if (pVM->enmVMState < VMSTATE_DESTROYING) /* ugly */
+                        TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    return rc;
+
+
+                /*
+                 * Out of memory, suspend the VM and stuff.
+                 */
+                case VINF_EM_NO_MEMORY:
+                    Log2(("EMR3ExecuteVM: VINF_EM_NO_MEMORY: %d -> %d\n", pVCpu->em.s.enmState, EMSTATE_SUSPENDED));
+                    pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
+                    pVCpu->em.s.enmState = EMSTATE_SUSPENDED;
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+
+                    rc = VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_SUSPEND, "HostMemoryLow",
+                                           N_("Unable to allocate and lock memory. The virtual machine will be paused. Please close applications to free up memory or close the VM"));
+                    if (rc != VINF_EM_SUSPEND)
+                    {
+                        if (RT_SUCCESS_NP(rc))
+                        {
+                            AssertLogRelMsgFailed(("%Rrc\n", rc));
+                            rc = VERR_EM_INTERNAL_ERROR;
+                        }
+                        pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    }
+                    return rc;
+
+                /*
+                 * Guest debug events.
+                 */
+                case VINF_EM_DBG_STEPPED:
+                    AssertMsgFailed(("VINF_EM_DBG_STEPPED cannot be here!"));
+                case VINF_EM_DBG_STOP:
+                case VINF_EM_DBG_BREAKPOINT:
+                case VINF_EM_DBG_STEP:
+                    if (pVCpu->em.s.enmState == EMSTATE_RAW)
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_GUEST_RAW));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_RAW;
+                    }
+                    else
+                    {
+                        Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_GUEST_REM));
+                        pVCpu->em.s.enmState = EMSTATE_DEBUG_GUEST_REM;
+                    }
+                    break;
+
+                /*
+                 * Hypervisor debug events.
+                 */
+                case VINF_EM_DBG_HYPER_STEPPED:
+                case VINF_EM_DBG_HYPER_BREAKPOINT:
+                case VINF_EM_DBG_HYPER_ASSERTION:
+                    Log2(("EMR3ExecuteVM: %Rrc: %d -> %d\n", rc, pVCpu->em.s.enmState, EMSTATE_DEBUG_HYPER));
+                    pVCpu->em.s.enmState = EMSTATE_DEBUG_HYPER;
+                    break;
+
+                /*
+                 * Guru mediations.
+                 */
+                case VERR_VMM_RING0_ASSERTION:
+                    Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, pVCpu->em.s.enmState, EMSTATE_GURU_MEDITATION));
+                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    break;
+
+                /*
+                 * Any error code showing up here other than the ones we
+                 * know and process above are considered to be FATAL.
+                 *
+                 * Unknown warnings and informational status codes are also
+                 * included in this.
+                 */
+                default:
+                    if (RT_SUCCESS_NP(rc))
+                    {
+                        AssertMsgFailed(("Unexpected warning or informational status code %Rra!\n", rc));
+                        rc = VERR_EM_INTERNAL_ERROR;
+                    }
+                    Log(("EMR3ExecuteVM: %Rrc: %d -> %d (EMSTATE_GURU_MEDITATION)\n", rc, pVCpu->em.s.enmState, EMSTATE_GURU_MEDITATION));
+                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    break;
+            }
+
+            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x); /* (skip this in release) */
+            STAM_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
+
+            /*
+             * Act on the state.
+             */
+            switch (pVCpu->em.s.enmState)
+            {
+                /*
+                 * Execute raw.
+                 */
+                case EMSTATE_RAW:
+                    rc = emR3RawExecute(pVM, pVCpu, &fFFDone);
+                    break;
+
+                /*
+                 * Execute hardware accelerated raw.
+                 */
+                case EMSTATE_HWACC:
+                    rc = emR3HwAccExecute(pVM, pVCpu, &fFFDone);
+                    break;
+
+                /*
+                 * Execute recompiled.
+                 */
+                case EMSTATE_REM:
+                    rc = emR3RemExecute(pVM, pVCpu, &fFFDone);
+                    Log2(("EMR3ExecuteVM: emR3RemExecute -> %Rrc\n", rc));
+                    break;
+
+                /*
+                 * Application processor execution halted until SIPI.
+                 */
+                case EMSTATE_WAIT_SIPI:
+                    /* no break */
+                /*
+                 * hlt - execution halted until interrupt.
+                 */
+                case EMSTATE_HALTED:
+                {
+                    STAM_REL_PROFILE_START(&pVCpu->em.s.StatHalted, y);
+                    if (pVCpu->em.s.mwait.fWait & EMMWAIT_FLAG_ACTIVE)
+                    {
+                        /* mwait has a special extension where it's woken up when an interrupt is pending even when IF=0. */
+                        rc = VMR3WaitHalted(pVM, pVCpu, !(pVCpu->em.s.mwait.fWait & EMMWAIT_FLAG_BREAKIRQIF0) && !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF));
+                        pVCpu->em.s.mwait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0);
+                    }
+                    else
+                        rc = VMR3WaitHalted(pVM, pVCpu, !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF));
+
+                    STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHalted, y);
+                    break;
+                }
+
+                /*
+                 * Suspended - return to VM.cpp.
+                 */
+                case EMSTATE_SUSPENDED:
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
+                    return VINF_EM_SUSPEND;
+
+                /*
+                 * Debugging in the guest.
+                 */
+                case EMSTATE_DEBUG_GUEST_REM:
+                case EMSTATE_DEBUG_GUEST_RAW:
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    rc = emR3Debug(pVM, pVCpu, rc);
+                    TMR3NotifyResume(pVM, pVCpu);
+                    Log2(("EMR3ExecuteVM: enmr3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
+                    break;
+
+                /*
+                 * Debugging in the hypervisor.
+                 */
+                case EMSTATE_DEBUG_HYPER:
+                {
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+
+                    rc = emR3Debug(pVM, pVCpu, rc);
+                    Log2(("EMR3ExecuteVM: enmr3Debug -> %Rrc (state %d)\n", rc, pVCpu->em.s.enmState));
+                    if (rc != VINF_SUCCESS)
+                    {
+                        /* switch to guru meditation mode */
+                        pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                        VMMR3FatalDump(pVM, pVCpu, rc);
+                        Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
+                        return rc;
+                    }
+
+                    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatTotal, x);
+                    TMR3NotifyResume(pVM, pVCpu);
+                    break;
+                }
+
+                /*
+                 * Guru meditation takes place in the debugger.
+                 */
+                case EMSTATE_GURU_MEDITATION:
+                {
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    VMMR3FatalDump(pVM, pVCpu, rc);
+                    emR3Debug(pVM, pVCpu, rc);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
+                    return rc;
+                }
+
+                /*
+                 * The states we don't expect here.
+                 */
+                case EMSTATE_NONE:
+                case EMSTATE_TERMINATING:
+                default:
+                    AssertMsgFailed(("EMR3ExecuteVM: Invalid state %d!\n", pVCpu->em.s.enmState));
+                    pVCpu->em.s.enmState = EMSTATE_GURU_MEDITATION;
+                    TMR3NotifySuspend(pVM, pVCpu);
+                    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+                    Log(("EMR3ExecuteVM: actually returns %Rrc (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
+                    return VERR_EM_INTERNAL_ERROR;
+            }
+        } /* The Outer Main Loop */
+    }
+    else
+    {
+        /*
+         * Fatal error.
+         */
+        Log(("EMR3ExecuteVM: returns %Rrc because of longjmp / fatal error; (state %s / %s)\n", rc, emR3GetStateName(pVCpu->em.s.enmState), emR3GetStateName(pVCpu->em.s.enmPrevState)));
+        TMR3NotifySuspend(pVM, pVCpu);
+        VMMR3FatalDump(pVM, pVCpu, rc);
+        emR3Debug(pVM, pVCpu, rc);
+        STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatTotal, x);
+        /** @todo change the VM state! */
+        return rc;
+    }
+
+    /* (won't ever get here). */
+    AssertFailed();
+}
+
+/**
+ * Notify EM of a state change (used by FTM)
+ *
+ * @param   pVM             VM Handle.
+ */
+VMMR3DECL(int) EMR3NotifySuspend(PVM pVM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    TMR3NotifySuspend(pVM, pVCpu);  /* Stop the virtual time. */
+    pVCpu->em.s.enmPrevState = pVCpu->em.s.enmState;
+    pVCpu->em.s.enmState     = EMSTATE_SUSPENDED;
+    return VINF_SUCCESS;
+}
+
+/**
+ * Notify EM of a state change (used by FTM)
+ *
+ * @param   pVM             VM Handle.
+ */
+VMMR3DECL(int) EMR3NotifyResume(PVM pVM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    EMSTATE enmCurState = pVCpu->em.s.enmState;
+
+    TMR3NotifyResume(pVM, pVCpu);  /* Resume the virtual time. */
+    pVCpu->em.s.enmState     = pVCpu->em.s.enmPrevState;
+    pVCpu->em.s.enmPrevState = enmCurState;
+    return VINF_SUCCESS;
+}
Index: /trunk/src/VBox/VMM/VMMR3/EMHwaccm.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/EMHwaccm.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/EMHwaccm.cpp	(revision 35333)
@@ -0,0 +1,609 @@
+/* $Id$ */
+/** @file
+ * EM - Execution Monitor / Manager - hardware virtualization
+ */
+
+/*
+ * Copyright (C) 2006-2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_em         EM - The Execution Monitor / Manager
+ *
+ * The Execution Monitor/Manager is responsible for running the VM, scheduling
+ * the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
+ * Interpreted), and keeping the CPU states in sync. The function
+ * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
+ * modes has different inner loops (emR3RawExecute, emR3HwAccExecute, and
+ * emR3RemExecute).
+ *
+ * The interpreted execution is only used to avoid switching between
+ * raw-mode/hwaccm and the recompiler when fielding virtualization traps/faults.
+ * The interpretation is thus implemented as part of EM.
+ *
+ * @see grp_em
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/em.h>
+#include <VBox/vmm.h>
+#include <VBox/csam.h>
+#include <VBox/selm.h>
+#include <VBox/trpm.h>
+#include <VBox/iom.h>
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/rem.h>
+#include <VBox/tm.h>
+#include <VBox/mm.h>
+#include <VBox/ssm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmqueue.h>
+#include <VBox/hwaccm.h>
+#include "EMInternal.h"
+#include "internal/em.h"
+#include <VBox/vm.h>
+#include <VBox/cpumdis.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/dbgf.h>
+
+#include <iprt/asm.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#if 0 /* Disabled till after 2.1.0 when we've time to test it. */
+#define EM_NOTIFY_HWACCM
+#endif
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC = VINF_SUCCESS);
+static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu);
+static int emR3HwaccmForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
+
+#define EMHANDLERC_WITH_HWACCM
+#include "EMHandleRCTmpl.h"
+
+
+#ifdef DEBUG
+
+/**
+ * Steps hardware accelerated mode.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU handle.
+ */
+static int emR3HwAccStep(PVM pVM, PVMCPU pVCpu)
+{
+    Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC);
+
+    int         rc;
+    PCPUMCTX    pCtx   = pVCpu->em.s.pCtx;
+    VMCPU_FF_CLEAR(pVCpu, (VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_TSS));
+
+    /*
+     * Check vital forced actions, but ignore pending interrupts and timers.
+     */
+    if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+        ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+    {
+        rc = emR3HwaccmForcedActions(pVM, pVCpu, pCtx);
+        if (rc != VINF_SUCCESS)
+            return rc;
+    }
+    /*
+     * Set flags for single stepping.
+     */
+    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
+
+    /*
+     * Single step.
+     * We do not start time or anything, if anything we should just do a few nanoseconds.
+     */
+    do
+    {
+        rc = VMMR3HwAccRunGC(pVM, pVCpu);
+    } while (   rc == VINF_SUCCESS
+             || rc == VINF_EM_RAW_INTERRUPT);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+
+    /*
+     * Make sure the trap flag is cleared.
+     * (Too bad if the guest is trying to single step too.)
+     */
+    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
+
+    /*
+     * Deal with the return codes.
+     */
+    rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
+    rc = emR3HwaccmHandleRC(pVM, pVCpu, pCtx, rc);
+    return rc;
+}
+
+
+static int emR3SingleStepExecHwAcc(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
+{
+    int     rc          = VINF_SUCCESS;
+    EMSTATE enmOldState = pVCpu->em.s.enmState;
+    pVCpu->em.s.enmState  = EMSTATE_DEBUG_GUEST_HWACC;
+
+    Log(("Single step BEGIN:\n"));
+    for (uint32_t i = 0; i < cIterations; i++)
+    {
+        DBGFR3PrgStep(pVCpu);
+        DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
+        rc = emR3HwAccStep(pVM, pVCpu);
+        if (    rc != VINF_SUCCESS
+            ||  !HWACCMR3CanExecuteGuest(pVM, pVCpu->em.s.pCtx))
+            break;
+    }
+    Log(("Single step END: rc=%Rrc\n", rc));
+    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
+    pVCpu->em.s.enmState = enmOldState;
+    return rc == VINF_SUCCESS ? VINF_EM_RESCHEDULE_REM : rc;
+}
+
+#endif /* DEBUG */
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle
+ * @param   rcGC        GC return code
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ */
+#ifdef LOG_ENABLED
+static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC, const char *pszPrefix)
+#else
+static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC)
+#endif
+{
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+    int      rc;
+
+    /*
+     *
+     * The simple solution is to use the recompiler.
+     * The better solution is to disassemble the current instruction and
+     * try handle as many as possible without using REM.
+     *
+     */
+
+#ifdef LOG_ENABLED
+    /*
+     * Disassemble the instruction if requested.
+     */
+    if (pszPrefix)
+    {
+        DBGFR3InfoLog(pVM, "cpumguest", pszPrefix);
+        DBGFR3DisasInstrCurrentLog(pVCpu, pszPrefix);
+    }
+#endif /* LOG_ENABLED */
+
+#if 0
+    /* Try our own instruction emulator before falling back to the recompiler. */
+    DISCPUSTATE Cpu;
+    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "GEN EMU");
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t size;
+
+        switch (Cpu.pCurInstr->opcode)
+        {
+        /* @todo we can do more now */
+        case OP_MOV:
+        case OP_AND:
+        case OP_OR:
+        case OP_XOR:
+        case OP_POP:
+        case OP_INC:
+        case OP_DEC:
+        case OP_XCHG:
+            STAM_PROFILE_START(&pVCpu->em.s.StatMiscEmu, a);
+            rc = EMInterpretInstructionCPU(pVM, pVCpu, &Cpu, CPUMCTX2CORE(pCtx), 0, &size);
+            if (RT_SUCCESS(rc))
+            {
+                pCtx->rip += Cpu.opsize;
+#ifdef EM_NOTIFY_HWACCM
+                if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
+                    HWACCMR3NotifyEmulated(pVCpu);
+#endif
+                STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
+                return rc;
+            }
+            if (rc != VERR_EM_INTERPRETER)
+                AssertMsgFailedReturn(("rc=%Rrc\n", rc), rc);
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
+            break;
+        }
+    }
+#endif /* 0 */
+    STAM_PROFILE_START(&pVCpu->em.s.StatREMEmu, a);
+    Log(("EMINS: %04x:%RGv RSP=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip, (RTGCPTR)pCtx->rsp));
+    EMRemLock(pVM);
+    /* Flush the recompiler TLB if the VCPU has changed. */
+    if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
+    pVM->em.s.idLastRemCpu = pVCpu->idCpu;
+
+    rc = REMR3EmulateInstruction(pVM, pVCpu);
+    EMRemUnlock(pVM);
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatREMEmu, a);
+
+#ifdef EM_NOTIFY_HWACCM
+    if (pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_HWACC)
+        HWACCMR3NotifyEmulated(pVCpu);
+#endif
+    return rc;
+}
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ * This is just a wrapper for discarding pszPrefix in non-logging builds.
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ * @param   rcGC        GC return code
+ */
+DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC)
+{
+#ifdef LOG_ENABLED
+    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC, pszPrefix);
+#else
+    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC);
+#endif
+}
+
+/**
+ * Executes one (or perhaps a few more) IO instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ */
+static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu)
+{
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+
+    STAM_PROFILE_START(&pVCpu->em.s.StatIOEmu, a);
+
+    /* Try to restart the io instruction that was refused in ring-0. */
+    VBOXSTRICTRC rcStrict = HWACCMR3RestartPendingIOInstr(pVM, pVCpu, pCtx);
+    if (IOM_SUCCESS(rcStrict))
+    {
+        STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIoRestarted);
+        STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+        return VBOXSTRICTRC_TODO(rcStrict);     /* rip already updated. */
+    }
+    AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
+                    RT_SUCCESS_NP(rcStrict) ? VERR_INTERNAL_ERROR_5 : VBOXSTRICTRC_TODO(rcStrict));
+
+    /** @todo probably we should fall back to the recompiler; otherwise we'll go back and forth between HC & GC
+     *   as io instructions tend to come in packages of more than one
+     */
+    DISCPUSTATE Cpu;
+    int rc2 = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "IO EMU");
+    if (RT_SUCCESS(rc2))
+    {
+        rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+
+        if (!(Cpu.prefix & (PREFIX_REP | PREFIX_REPNE)))
+        {
+            switch (Cpu.pCurInstr->opcode)
+            {
+                case OP_IN:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
+                    rcStrict = IOMInterpretIN(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+
+                case OP_OUT:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
+                    rcStrict = IOMInterpretOUT(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+            }
+        }
+        else if (Cpu.prefix & PREFIX_REP)
+        {
+            switch (Cpu.pCurInstr->opcode)
+            {
+                case OP_INSB:
+                case OP_INSWD:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
+                    rcStrict = IOMInterpretINS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+
+                case OP_OUTSB:
+                case OP_OUTSWD:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
+                    rcStrict = IOMInterpretOUTS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+            }
+        }
+
+        /*
+         * Handled the I/O return codes.
+         * (The unhandled cases end up with rcStrict == VINF_EM_RAW_EMULATE_INSTR.)
+         */
+        if (IOM_SUCCESS(rcStrict))
+        {
+            pCtx->rip += Cpu.opsize;
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+            return VBOXSTRICTRC_TODO(rcStrict);
+        }
+
+        if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
+        {
+            /* The active trap will be dispatched. */
+            Assert(TRPMHasTrap(pVCpu));
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+            return VINF_SUCCESS;
+        }
+        AssertMsg(rcStrict != VINF_TRPM_XCPT_DISPATCHED, ("Handle VINF_TRPM_XCPT_DISPATCHED\n"));
+
+        if (RT_FAILURE(rcStrict))
+        {
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+            return VBOXSTRICTRC_TODO(rcStrict);
+        }
+        AssertMsg(rcStrict == VINF_EM_RAW_EMULATE_INSTR || rcStrict == VINF_EM_RESCHEDULE_REM, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+    return emR3ExecuteInstruction(pVM, pVCpu, "IO: ");
+}
+
+
+/**
+ * Process raw-mode specific forced actions.
+ *
+ * This function is called when any FFs in the VM_FF_HIGH_PRIORITY_PRE_RAW_MASK is pending.
+ *
+ * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
+ *          EM statuses.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU handle.
+ * @param   pCtx        The guest CPUM register context.
+ */
+static int emR3HwaccmForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    /*
+     * Sync page directory.
+     */
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+    {
+        Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
+        int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+        if (RT_FAILURE(rc))
+            return rc;
+
+        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
+
+        /* Prefetch pages for EIP and ESP. */
+        /** @todo This is rather expensive. Should investigate if it really helps at all. */
+        rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->rip));
+        if (rc == VINF_SUCCESS)
+            rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->rsp));
+        if (rc != VINF_SUCCESS)
+        {
+            if (rc != VINF_PGM_SYNC_CR3)
+            {
+                AssertLogRelMsgReturn(RT_FAILURE(rc), ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                return rc;
+            }
+            rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        /** @todo maybe prefetch the supervisor stack page as well */
+        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
+    }
+
+    /*
+     * Allocate handy pages (just in case the above actions have consumed some pages).
+     */
+    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
+    {
+        int rc = PGMR3PhysAllocateHandyPages(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Check whether we're out of memory now.
+     *
+     * This may stem from some of the above actions or operations that has been executed
+     * since we ran FFs. The allocate handy pages must for instance always be followed by
+     * this check.
+     */
+    if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
+        return VINF_EM_NO_MEMORY;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Executes hardware accelerated raw code. (Intel VT-x & AMD-V)
+ *
+ * This function contains the raw-mode version of the inner
+ * execution loop (the outer loop being in EMR3ExecuteVM()).
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE, VINF_EM_RESCHEDULE_RAW,
+ *          VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   pfFFDone    Where to store an indicator telling whether or not
+ *                      FFs were done before returning.
+ */
+int emR3HwAccExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+    int      rc = VERR_INTERNAL_ERROR;
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+
+    LogFlow(("emR3HwAccExecute%d: (cs:eip=%04x:%RGv)\n", pVCpu->idCpu, pCtx->cs, (RTGCPTR)pCtx->rip));
+    *pfFFDone = false;
+
+    STAM_COUNTER_INC(&pVCpu->em.s.StatHwAccExecuteEntry);
+
+#ifdef EM_NOTIFY_HWACCM
+    HWACCMR3NotifyScheduled(pVCpu);
+#endif
+
+    /*
+     * Spin till we get a forced action which returns anything but VINF_SUCCESS.
+     */
+    for (;;)
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatHwAccEntry, a);
+
+        /* Check if a forced reschedule is pending. */
+        if (HWACCMR3IsRescheduleRequired(pVM, pCtx))
+        {
+            rc = VINF_EM_RESCHEDULE;
+            break;
+        }
+
+        /*
+         * Process high priority pre-execution raw-mode FFs.
+         */
+        VMCPU_FF_CLEAR(pVCpu, (VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_TSS)); /* not relevant in HWACCM mode; shouldn't be set really. */
+        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            rc = emR3HwaccmForcedActions(pVM, pVCpu, pCtx);
+            if (rc != VINF_SUCCESS)
+                break;
+        }
+
+#ifdef LOG_ENABLED
+        /*
+         * Log important stuff before entering GC.
+         */
+        if (TRPMHasTrap(pVCpu))
+            Log(("CPU%d: Pending hardware interrupt=0x%x cs:rip=%04X:%RGv\n", pVCpu->idCpu, TRPMGetTrapNo(pVCpu), pCtx->cs, (RTGCPTR)pCtx->rip));
+
+        uint32_t cpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
+
+        if (pVM->cCpus == 1)
+        {
+            if (pCtx->eflags.Bits.u1VM)
+                Log(("HWV86: %08X IF=%d\n", pCtx->eip, pCtx->eflags.Bits.u1IF));
+            else if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                Log(("HWR%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->rsp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
+            else
+                Log(("HWR%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pCtx->cs,          pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
+        }
+        else
+        {
+            if (pCtx->eflags.Bits.u1VM)
+                Log(("HWV86-CPU%d: %08X IF=%d\n", pVCpu->idCpu, pCtx->eip, pCtx->eflags.Bits.u1IF));
+            else if (CPUMIsGuestIn64BitCodeEx(pCtx))
+                Log(("HWR%d-CPU%d: %04X:%RGv ESP=%RGv IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->rsp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
+            else
+                Log(("HWR%d-CPU%d: %04X:%08X ESP=%08X IF=%d IOPL=%d CR0=%x CR4=%x EFER=%x\n", cpl, pVCpu->idCpu, pCtx->cs,          pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pCtx->eflags.Bits.u2IOPL, (uint32_t)pCtx->cr0, (uint32_t)pCtx->cr4, (uint32_t)pCtx->msrEFER));
+        }
+#endif /* LOG_ENABLED */
+
+        /*
+         * Execute the code.
+         */
+        STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatHwAccEntry, a);
+
+        if (RT_LIKELY(EMR3IsExecutionAllowed(pVM, pVCpu)))
+        {
+            STAM_PROFILE_START(&pVCpu->em.s.StatHwAccExec, x);
+            rc = VMMR3HwAccRunGC(pVM, pVCpu);
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatHwAccExec, x);
+        }
+        else
+        {
+            /* Give up this time slice; virtual time continues */
+            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
+            RTThreadSleep(5);
+            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
+            rc = VINF_SUCCESS;
+        }
+
+
+        /*
+         * Deal with high priority post execution FFs before doing anything else.
+         */
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+            rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
+
+        /*
+         * Process the returned status code.
+         */
+        if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+            break;
+
+        rc = emR3HwaccmHandleRC(pVM, pVCpu, pCtx, rc);
+        if (rc != VINF_SUCCESS)
+            break;
+
+        /*
+         * Check and execute forced actions.
+         */
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+        TMTimerPollVoid(pVM, pVCpu);
+#endif
+        if (    VM_FF_ISPENDING(pVM, VM_FF_ALL_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_ALL_MASK))
+        {
+            rc = emR3ForcedActions(pVM, pVCpu, rc);
+            if (    rc != VINF_SUCCESS
+                &&  rc != VINF_EM_RESCHEDULE_HWACC)
+            {
+                *pfFFDone = true;
+                break;
+            }
+        }
+    }
+
+    /*
+     * Return to outer loop.
+     */
+#if defined(LOG_ENABLED) && defined(DEBUG)
+    RTLogFlush(NULL);
+#endif
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/EMRaw.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/EMRaw.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/EMRaw.cpp	(revision 35333)
@@ -0,0 +1,1656 @@
+/* $Id$ */
+
+/** @file
+ * EM - Execution Monitor / Manager - software virtualization
+ */
+
+/*
+ * Copyright (C) 2006-2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_em         EM - The Execution Monitor / Manager
+ *
+ * The Execution Monitor/Manager is responsible for running the VM, scheduling
+ * the right kind of execution (Raw-mode, Hardware Assisted, Recompiled or
+ * Interpreted), and keeping the CPU states in sync. The function
+ * EMR3ExecuteVM() is the 'main-loop' of the VM, while each of the execution
+ * modes has different inner loops (emR3RawExecute, emR3HwAccExecute, and
+ * emR3RemExecute).
+ *
+ * The interpreted execution is only used to avoid switching between
+ * raw-mode/hwaccm and the recompiler when fielding virtualization traps/faults.
+ * The interpretation is thus implemented as part of EM.
+ *
+ * @see grp_em
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/em.h>
+#include <VBox/vmm.h>
+#include <VBox/patm.h>
+#include <VBox/csam.h>
+#include <VBox/selm.h>
+#include <VBox/trpm.h>
+#include <VBox/iom.h>
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/rem.h>
+#include <VBox/tm.h>
+#include <VBox/mm.h>
+#include <VBox/ssm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmqueue.h>
+#include <VBox/patm.h>
+#include "EMInternal.h"
+#include "internal/em.h"
+#include <VBox/vm.h>
+#include <VBox/cpumdis.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/dbgf.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/stream.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int emR3RawForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
+DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC = VINF_SUCCESS);
+static int emR3RawGuestTrap(PVM pVM, PVMCPU pVCpu);
+static int emR3PatchTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int gcret);
+static int emR3SingleStepExecRem(PVM pVM, uint32_t cIterations);
+static int emR3RawPrivileged(PVM pVM, PVMCPU pVCpu);
+static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu);
+static int emR3RawRingSwitch(PVM pVM, PVMCPU pVCpu);
+
+#define EMHANDLERC_WITH_PATM
+#include "EMHandleRCTmpl.h"
+
+/**
+ * Enables or disables a set of raw-mode execution modes.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VINF_RESCHEDULE if a rescheduling might be required.
+ * @returns VERR_INVALID_PARAMETER on an invalid enmMode value.
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   enmMode     The execution mode change.
+ * @thread  The emulation thread.
+ */
+VMMR3DECL(int) EMR3RawSetMode(PVM pVM, EMRAWMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case EMRAW_NONE:
+            pVM->fRawR3Enabled = false;
+            pVM->fRawR0Enabled = false;
+            break;
+        case EMRAW_RING3_ENABLE:
+            pVM->fRawR3Enabled = true;
+            break;
+        case EMRAW_RING3_DISABLE:
+            pVM->fRawR3Enabled = false;
+            break;
+        case EMRAW_RING0_ENABLE:
+            pVM->fRawR0Enabled = true;
+            break;
+        case EMRAW_RING0_DISABLE:
+            pVM->fRawR0Enabled = false;
+            break;
+        default:
+            AssertMsgFailed(("Invalid enmMode=%d\n", enmMode));
+            return VERR_INVALID_PARAMETER;
+    }
+    Log(("EMR3SetRawMode: fRawR3Enabled=%RTbool fRawR0Enabled=%RTbool\n",
+          pVM->fRawR3Enabled, pVM->fRawR0Enabled));
+    return pVM->aCpus[0].em.s.enmState == EMSTATE_RAW ? VINF_EM_RESCHEDULE : VINF_SUCCESS;
+}
+
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Just a braindead function to keep track of cli addresses.
+ * @param   pVM         VM handle.
+ * @param   pVMCPU      VMCPU handle.
+ * @param   GCPtrInstr  The EIP of the cli instruction.
+ */
+static void emR3RecordCli(PVM pVM, PVMCPU pVCpu, RTGCPTR GCPtrInstr)
+{
+    PCLISTAT pRec;
+
+    pRec = (PCLISTAT)RTAvlGCPtrGet(&pVCpu->em.s.pCliStatTree, GCPtrInstr);
+    if (!pRec)
+    {
+        /* New cli instruction; insert into the tree. */
+        pRec = (PCLISTAT)MMR3HeapAllocZ(pVM, MM_TAG_EM, sizeof(*pRec));
+        Assert(pRec);
+        if (!pRec)
+            return;
+        pRec->Core.Key = GCPtrInstr;
+
+        char szCliStatName[32];
+        RTStrPrintf(szCliStatName, sizeof(szCliStatName), "/EM/Cli/0x%RGv", GCPtrInstr);
+        STAM_REG(pVM, &pRec->Counter, STAMTYPE_COUNTER, szCliStatName, STAMUNIT_OCCURENCES, "Number of times cli was executed.");
+
+        bool fRc = RTAvlGCPtrInsert(&pVCpu->em.s.pCliStatTree, &pRec->Core);
+        Assert(fRc); NOREF(fRc);
+    }
+    STAM_COUNTER_INC(&pRec->Counter);
+    STAM_COUNTER_INC(&pVCpu->em.s.StatTotalClis);
+}
+#endif /* VBOX_WITH_STATISTICS */
+
+
+
+/**
+ * Resumes executing hypervisor after a debug event.
+ *
+ * This is kind of special since our current guest state is
+ * potentially out of sync.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU handle.
+ */
+int emR3RawResumeHyper(PVM pVM, PVMCPU pVCpu)
+{
+    int         rc;
+    PCPUMCTX    pCtx = pVCpu->em.s.pCtx;
+    Assert(pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER);
+    Log(("emR3RawResumeHyper: cs:eip=%RTsel:%RGr efl=%RGr\n", pCtx->cs, pCtx->eip, pCtx->eflags));
+
+    /*
+     * Resume execution.
+     */
+    CPUMR3RawEnter(pVCpu, NULL);
+    CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_RF);
+    rc = VMMR3ResumeHyper(pVM, pVCpu);
+    Log(("emR3RawResumeHyper: cs:eip=%RTsel:%RGr efl=%RGr - returned from GC with rc=%Rrc\n", pCtx->cs, pCtx->eip, pCtx->eflags, rc));
+    rc = CPUMR3RawLeave(pVCpu, NULL, rc);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+
+    /*
+     * Deal with the return code.
+     */
+    rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
+    rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
+    rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
+    return rc;
+}
+
+
+/**
+ * Steps rawmode.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU handle.
+ */
+int emR3RawStep(PVM pVM, PVMCPU pVCpu)
+{
+    Assert(   pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER
+           || pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_RAW
+           || pVCpu->em.s.enmState == EMSTATE_DEBUG_GUEST_REM);
+    int         rc;
+    PCPUMCTX    pCtx   = pVCpu->em.s.pCtx;
+    bool        fGuest = pVCpu->em.s.enmState != EMSTATE_DEBUG_HYPER;
+#ifndef DEBUG_sandervl
+    Log(("emR3RawStep: cs:eip=%RTsel:%RGr efl=%RGr\n", fGuest ? CPUMGetGuestCS(pVCpu) : CPUMGetHyperCS(pVCpu),
+         fGuest ? CPUMGetGuestEIP(pVCpu) : CPUMGetHyperEIP(pVCpu), fGuest ? CPUMGetGuestEFlags(pVCpu) : CPUMGetHyperEFlags(pVCpu)));
+#endif
+    if (fGuest)
+    {
+        /*
+         * Check vital forced actions, but ignore pending interrupts and timers.
+         */
+        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
+            if (rc != VINF_SUCCESS)
+                return rc;
+        }
+
+        /*
+         * Set flags for single stepping.
+         */
+        CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
+    }
+    else
+        CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
+
+    /*
+     * Single step.
+     * We do not start time or anything, if anything we should just do a few nanoseconds.
+     */
+    CPUMR3RawEnter(pVCpu, NULL);
+    do
+    {
+        if (pVCpu->em.s.enmState == EMSTATE_DEBUG_HYPER)
+            rc = VMMR3ResumeHyper(pVM, pVCpu);
+        else
+            rc = VMMR3RawRunGC(pVM, pVCpu);
+#ifndef DEBUG_sandervl
+        Log(("emR3RawStep: cs:eip=%RTsel:%RGr efl=%RGr - GC rc %Rrc\n", fGuest ? CPUMGetGuestCS(pVCpu) : CPUMGetHyperCS(pVCpu),
+             fGuest ? CPUMGetGuestEIP(pVCpu) : CPUMGetHyperEIP(pVCpu), fGuest ? CPUMGetGuestEFlags(pVCpu) : CPUMGetHyperEFlags(pVCpu), rc));
+#endif
+    } while (   rc == VINF_SUCCESS
+             || rc == VINF_EM_RAW_INTERRUPT);
+    rc = CPUMR3RawLeave(pVCpu, NULL, rc);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+
+    /*
+     * Make sure the trap flag is cleared.
+     * (Too bad if the guest is trying to single step too.)
+     */
+    if (fGuest)
+        CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
+    else
+        CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) & ~X86_EFL_TF);
+
+    /*
+     * Deal with the return codes.
+     */
+    rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
+    rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
+    rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
+    return rc;
+}
+
+
+#ifdef DEBUG
+
+
+int emR3SingleStepExecRaw(PVM pVM, PVMCPU pVCpu, uint32_t cIterations)
+{
+    int     rc          = VINF_SUCCESS;
+    EMSTATE enmOldState = pVCpu->em.s.enmState;
+    pVCpu->em.s.enmState  = EMSTATE_DEBUG_GUEST_RAW;
+
+    Log(("Single step BEGIN:\n"));
+    for (uint32_t i = 0; i < cIterations; i++)
+    {
+        DBGFR3PrgStep(pVCpu);
+        DBGFR3DisasInstrCurrentLog(pVCpu, "RSS: ");
+        rc = emR3RawStep(pVM, pVCpu);
+        if (rc != VINF_SUCCESS)
+            break;
+    }
+    Log(("Single step END: rc=%Rrc\n", rc));
+    CPUMSetGuestEFlags(pVCpu, CPUMGetGuestEFlags(pVCpu) & ~X86_EFL_TF);
+    pVCpu->em.s.enmState = enmOldState;
+    return rc;
+}
+
+#endif /* DEBUG */
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle
+ * @param   rcGC        GC return code
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ */
+#ifdef LOG_ENABLED
+static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC, const char *pszPrefix)
+#else
+static int emR3ExecuteInstructionWorker(PVM pVM, PVMCPU pVCpu, int rcGC)
+#endif
+{
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+    int      rc;
+
+    /*
+     *
+     * The simple solution is to use the recompiler.
+     * The better solution is to disassemble the current instruction and
+     * try handle as many as possible without using REM.
+     *
+     */
+
+#ifdef LOG_ENABLED
+    /*
+     * Disassemble the instruction if requested.
+     */
+    if (pszPrefix)
+    {
+        DBGFR3InfoLog(pVM, "cpumguest", pszPrefix);
+        DBGFR3DisasInstrCurrentLog(pVCpu, pszPrefix);
+    }
+#endif /* LOG_ENABLED */
+
+    /*
+     * PATM is making life more interesting.
+     * We cannot hand anything to REM which has an EIP inside patch code. So, we'll
+     * tell PATM there is a trap in this code and have it take the appropriate actions
+     * to allow us execute the code in REM.
+     */
+    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
+    {
+        Log(("emR3ExecuteInstruction: In patch block. eip=%RRv\n", (RTRCPTR)pCtx->eip));
+
+        RTGCPTR pNewEip;
+        rc = PATMR3HandleTrap(pVM, pCtx, pCtx->eip, &pNewEip);
+        switch (rc)
+        {
+            /*
+             * It's not very useful to emulate a single instruction and then go back to raw
+             * mode; just execute the whole block until IF is set again.
+             */
+            case VINF_SUCCESS:
+                Log(("emR3ExecuteInstruction: Executing instruction starting at new address %RGv IF=%d VMIF=%x\n",
+                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
+                pCtx->eip = pNewEip;
+                Assert(pCtx->eip);
+
+                if (pCtx->eflags.Bits.u1IF)
+                {
+                    /*
+                     * The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
+                     */
+                    Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
+                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
+                }
+                else if (rcGC == VINF_PATM_PENDING_IRQ_AFTER_IRET)
+                {
+                    /* special case: iret, that sets IF,  detected a pending irq/event */
+                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIRET");
+                }
+                return VINF_EM_RESCHEDULE_REM;
+
+            /*
+             * One instruction.
+             */
+            case VINF_PATCH_EMULATE_INSTR:
+                Log(("emR3ExecuteInstruction: Emulate patched instruction at %RGv IF=%d VMIF=%x\n",
+                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
+                pCtx->eip = pNewEip;
+                return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
+
+            /*
+             * The patch was disabled, hand it to the REM.
+             */
+            case VERR_PATCH_DISABLED:
+                Log(("emR3ExecuteInstruction: Disabled patch -> new eip %RGv IF=%d VMIF=%x\n",
+                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
+                pCtx->eip = pNewEip;
+                if (pCtx->eflags.Bits.u1IF)
+                {
+                    /*
+                     * The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
+                     */
+                    Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
+                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
+                }
+                return VINF_EM_RESCHEDULE_REM;
+
+            /* Force continued patch exection; usually due to write monitored stack. */
+            case VINF_PATCH_CONTINUE:
+                return VINF_SUCCESS;
+
+            default:
+                AssertReleaseMsgFailed(("Unknown return code %Rrc from PATMR3HandleTrap\n", rc));
+                return VERR_IPE_UNEXPECTED_STATUS;
+        }
+    }
+
+#if 0
+    /* Try our own instruction emulator before falling back to the recompiler. */
+    DISCPUSTATE Cpu;
+    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "GEN EMU");
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t size;
+
+        switch (Cpu.pCurInstr->opcode)
+        {
+        /* @todo we can do more now */
+        case OP_MOV:
+        case OP_AND:
+        case OP_OR:
+        case OP_XOR:
+        case OP_POP:
+        case OP_INC:
+        case OP_DEC:
+        case OP_XCHG:
+            STAM_PROFILE_START(&pVCpu->em.s.StatMiscEmu, a);
+            rc = EMInterpretInstructionCPU(pVM, &Cpu, CPUMCTX2CORE(pCtx), 0, &size);
+            if (RT_SUCCESS(rc))
+            {
+                pCtx->rip += Cpu.opsize;
+                STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
+                return rc;
+            }
+            if (rc != VERR_EM_INTERPRETER)
+                AssertMsgFailedReturn(("rc=%Rrc\n", rc), rc);
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatMiscEmu, a);
+            break;
+        }
+    }
+#endif /* 0 */
+    STAM_PROFILE_START(&pVCpu->em.s.StatREMEmu, a);
+    Log(("EMINS: %04x:%RGv RSP=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip, (RTGCPTR)pCtx->rsp));
+    EMRemLock(pVM);
+    /* Flush the recompiler TLB if the VCPU has changed. */
+    if (pVM->em.s.idLastRemCpu != pVCpu->idCpu)
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
+    pVM->em.s.idLastRemCpu = pVCpu->idCpu;
+
+    rc = REMR3EmulateInstruction(pVM, pVCpu);
+    EMRemUnlock(pVM);
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatREMEmu, a);
+
+    return rc;
+}
+
+
+/**
+ * Executes one (or perhaps a few more) instruction(s).
+ * This is just a wrapper for discarding pszPrefix in non-logging builds.
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   pszPrefix   Disassembly prefix. If not NULL we'll disassemble the
+ *                      instruction and prefix the log output with this text.
+ * @param   rcGC        GC return code
+ */
+DECLINLINE(int) emR3ExecuteInstruction(PVM pVM, PVMCPU pVCpu, const char *pszPrefix, int rcGC)
+{
+#ifdef LOG_ENABLED
+    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC, pszPrefix);
+#else
+    return emR3ExecuteInstructionWorker(pVM, pVCpu, rcGC);
+#endif
+}
+
+/**
+ * Executes one (or perhaps a few more) IO instruction(s).
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ */
+static int emR3ExecuteIOInstruction(PVM pVM, PVMCPU pVCpu)
+{
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+
+    STAM_PROFILE_START(&pVCpu->em.s.StatIOEmu, a);
+
+    /** @todo probably we should fall back to the recompiler; otherwise we'll go back and forth between HC & GC
+     *   as io instructions tend to come in packages of more than one
+     */
+    DISCPUSTATE Cpu;
+    int rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "IO EMU");
+    if (RT_SUCCESS(rc))
+    {
+        VBOXSTRICTRC rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+
+        if (!(Cpu.prefix & (PREFIX_REP | PREFIX_REPNE)))
+        {
+            switch (Cpu.pCurInstr->opcode)
+            {
+                case OP_IN:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
+                    rcStrict = IOMInterpretIN(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+
+                case OP_OUT:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
+                    rcStrict = IOMInterpretOUT(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+            }
+        }
+        else if (Cpu.prefix & PREFIX_REP)
+        {
+            switch (Cpu.pCurInstr->opcode)
+            {
+                case OP_INSB:
+                case OP_INSWD:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatIn);
+                    rcStrict = IOMInterpretINS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+
+                case OP_OUTSB:
+                case OP_OUTSWD:
+                {
+                    STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatOut);
+                    rcStrict = IOMInterpretOUTS(pVM, CPUMCTX2CORE(pCtx), &Cpu);
+                    break;
+                }
+            }
+        }
+
+        /*
+         * Handled the I/O return codes.
+         * (The unhandled cases end up with rcStrict == VINF_EM_RAW_EMULATE_INSTR.)
+         */
+        if (IOM_SUCCESS(rcStrict))
+        {
+            pCtx->rip += Cpu.opsize;
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+            return VBOXSTRICTRC_TODO(rcStrict);
+        }
+
+        if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
+        {
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+            rcStrict = emR3RawGuestTrap(pVM, pVCpu);
+            return VBOXSTRICTRC_TODO(rcStrict);
+        }
+        AssertMsg(rcStrict != VINF_TRPM_XCPT_DISPATCHED, ("Handle VINF_TRPM_XCPT_DISPATCHED\n"));
+
+        if (RT_FAILURE(rcStrict))
+        {
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+            return VBOXSTRICTRC_TODO(rcStrict);
+        }
+        AssertMsg(rcStrict == VINF_EM_RAW_EMULATE_INSTR || rcStrict == VINF_EM_RESCHEDULE_REM, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    }
+    STAM_PROFILE_STOP(&pVCpu->em.s.StatIOEmu, a);
+    return emR3ExecuteInstruction(pVM, pVCpu, "IO: ");
+}
+
+
+/**
+ * Handle a guest context trap.
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ */
+static int emR3RawGuestTrap(PVM pVM, PVMCPU pVCpu)
+{
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+
+    /*
+     * Get the trap info.
+     */
+    uint8_t         u8TrapNo;
+    TRPMEVENT       enmType;
+    RTGCUINT        uErrorCode;
+    RTGCUINTPTR     uCR2;
+    int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
+    if (RT_FAILURE(rc))
+    {
+        AssertReleaseMsgFailed(("No trap! (rc=%Rrc)\n", rc));
+        return rc;
+    }
+
+
+#if 1 /* Experimental: Review, disable if it causes trouble. */
+    /*
+     * Handle traps in patch code first.
+     *
+     * We catch a few of these cases in RC before returning to R3 (#PF, #GP, #BP)
+     * but several traps isn't handled specially by TRPM in RC and we end up here
+     * instead. One example is #DE.
+     */
+    uint32_t uCpl = CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx));
+    if (    uCpl == 0
+        &&  PATMIsPatchGCAddr(pVM, pCtx->eip))
+    {
+        LogFlow(("emR3RawGuestTrap: trap %#x in patch code; eip=%08x\n", u8TrapNo, pCtx->eip));
+        return emR3PatchTrap(pVM, pVCpu, pCtx, rc);
+    }
+#endif
+
+    /*
+     * If the guest gate is marked unpatched, then we will check again if we can patch it.
+     * (This assumes that we've already tried and failed to dispatch the trap in
+     * RC for the gates that already has been patched. Which is true for most high
+     * volume traps, because these are handled specially, but not for odd ones like #DE.)
+     */
+    if (TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) == TRPM_INVALID_HANDLER)
+    {
+        CSAMR3CheckGates(pVM, u8TrapNo, 1);
+        Log(("emR3RawHandleRC: recheck gate %x -> valid=%d\n", u8TrapNo, TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) != TRPM_INVALID_HANDLER));
+
+        /* If it was successful, then we could go back to raw mode. */
+        if (TRPMR3GetGuestTrapHandler(pVM, u8TrapNo) != TRPM_INVALID_HANDLER)
+        {
+            /* Must check pending forced actions as our IDT or GDT might be out of sync. */
+            rc = EMR3CheckRawForcedActions(pVM, pVCpu);
+            AssertRCReturn(rc, rc);
+
+            TRPMERRORCODE enmError = uErrorCode != ~0U
+                                   ? TRPM_TRAP_HAS_ERRORCODE
+                                   : TRPM_TRAP_NO_ERRORCODE;
+            rc = TRPMForwardTrap(pVCpu, CPUMCTX2CORE(pCtx), u8TrapNo, uErrorCode, enmError, TRPM_TRAP, -1);
+            if (rc == VINF_SUCCESS /* Don't use RT_SUCCESS */)
+            {
+                TRPMResetTrap(pVCpu);
+                return VINF_EM_RESCHEDULE_RAW;
+            }
+            AssertMsg(rc == VINF_EM_RAW_GUEST_TRAP, ("%Rrc\n", rc));
+        }
+    }
+
+    /*
+     * Scan kernel code that traps; we might not get another chance.
+     */
+    /** @todo move this up before the dispatching? */
+    if (    (pCtx->ss & X86_SEL_RPL) <= 1
+        &&  !pCtx->eflags.Bits.u1VM)
+    {
+        Assert(!PATMIsPatchGCAddr(pVM, pCtx->eip));
+        CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
+    }
+
+    /*
+     * Trap specific handling.
+     */
+    if (u8TrapNo == 6) /* (#UD) Invalid opcode. */
+    {
+        /*
+         * If MONITOR & MWAIT are supported, then interpret them here.
+         */
+        DISCPUSTATE cpu;
+        rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &cpu, "Guest Trap (#UD): ");
+        if (    RT_SUCCESS(rc)
+            && (cpu.pCurInstr->opcode == OP_MONITOR || cpu.pCurInstr->opcode == OP_MWAIT))
+        {
+            uint32_t u32Dummy, u32Features, u32ExtFeatures;
+            CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32ExtFeatures, &u32Features);
+            if (u32ExtFeatures & X86_CPUID_FEATURE_ECX_MONITOR)
+            {
+                rc = TRPMResetTrap(pVCpu);
+                AssertRC(rc);
+
+                uint32_t opsize;
+                rc = VBOXSTRICTRC_TODO(EMInterpretInstructionCPU(pVM, pVCpu, &cpu, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &opsize));
+                if (RT_SUCCESS(rc))
+                {
+                    pCtx->rip += cpu.opsize;
+                    return rc;
+                }
+                return emR3ExecuteInstruction(pVM, pVCpu, "Monitor: ");
+            }
+        }
+    }
+    else if (u8TrapNo == 13) /* (#GP) Privileged exception */
+    {
+        /*
+         * Handle I/O bitmap?
+         */
+        /** @todo We're not supposed to be here with a false guest trap concerning
+         *        I/O access. We can easily handle those in RC.  */
+        DISCPUSTATE cpu;
+        rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &cpu, "Guest Trap: ");
+        if (    RT_SUCCESS(rc)
+            &&  (cpu.pCurInstr->optype & OPTYPE_PORTIO))
+        {
+            /*
+             * We should really check the TSS for the IO bitmap, but it's not like this
+             * lazy approach really makes things worse.
+             */
+            rc = TRPMResetTrap(pVCpu);
+            AssertRC(rc);
+            return emR3ExecuteInstruction(pVM, pVCpu, "IO Guest Trap: ");
+        }
+    }
+
+#ifdef LOG_ENABLED
+    DBGFR3InfoLog(pVM, "cpumguest", "Guest trap");
+    DBGFR3DisasInstrCurrentLog(pVCpu, "Guest trap");
+
+    /* Get guest page information. */
+    uint64_t    fFlags = 0;
+    RTGCPHYS    GCPhys = 0;
+    int rc2 = PGMGstGetPage(pVCpu, uCR2, &fFlags, &GCPhys);
+    Log(("emR3RawGuestTrap: cs:eip=%04x:%08x: trap=%02x err=%08x cr2=%08x cr0=%08x%s: Phys=%RGp fFlags=%08llx %s %s %s%s rc2=%d\n",
+         pCtx->cs, pCtx->eip, u8TrapNo, uErrorCode, uCR2, (uint32_t)pCtx->cr0, (enmType == TRPM_SOFTWARE_INT) ? " software" : "",  GCPhys, fFlags,
+         fFlags & X86_PTE_P  ? "P " : "NP", fFlags & X86_PTE_US ? "U"  : "S",
+         fFlags & X86_PTE_RW ? "RW" : "R0", fFlags & X86_PTE_G  ? " G" : "", rc2));
+#endif
+
+    /*
+     * #PG has CR2.
+     * (Because of stuff like above we must set CR2 in a delayed fashion.)
+     */
+    if (u8TrapNo == 14 /* #PG */)
+        pCtx->cr2 = uCR2;
+
+    return VINF_EM_RESCHEDULE_REM;
+}
+
+
+/**
+ * Handle a ring switch trap.
+ * Need to do statistics and to install patches. The result is going to REM.
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM     VM handle.
+ * @param   pVCpu       VMCPU handle.
+ */
+static int emR3RawRingSwitch(PVM pVM, PVMCPU pVCpu)
+{
+    int         rc;
+    DISCPUSTATE Cpu;
+    PCPUMCTX    pCtx = pVCpu->em.s.pCtx;
+
+    /*
+     * sysenter, syscall & callgate
+     */
+    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "RSWITCH: ");
+    if (RT_SUCCESS(rc))
+    {
+        if (Cpu.pCurInstr->opcode == OP_SYSENTER)
+        {
+            if (pCtx->SysEnter.cs != 0)
+            {
+                rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
+                                        (SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0);
+                if (RT_SUCCESS(rc))
+                {
+                    DBGFR3DisasInstrCurrentLog(pVCpu, "Patched sysenter instruction");
+                    return VINF_EM_RESCHEDULE_RAW;
+                }
+            }
+        }
+
+#ifdef VBOX_WITH_STATISTICS
+        switch (Cpu.pCurInstr->opcode)
+        {
+            case OP_SYSENTER:
+                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysEnter);
+                break;
+            case OP_SYSEXIT:
+                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysExit);
+                break;
+            case OP_SYSCALL:
+                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysCall);
+                break;
+            case OP_SYSRET:
+                STAM_COUNTER_INC(&pVCpu->em.s.CTX_SUFF(pStats)->StatSysRet);
+                break;
+        }
+#endif
+    }
+    else
+        AssertRC(rc);
+
+    /* go to the REM to emulate a single instruction */
+    return emR3ExecuteInstruction(pVM, pVCpu, "RSWITCH: ");
+}
+
+
+/**
+ * Handle a trap (\#PF or \#GP) in patch code
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM     VM handle.
+ * @param   pVCpu   VMCPU handle.
+ * @param   pCtx    CPU context
+ * @param   gcret   GC return code
+ */
+static int emR3PatchTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int gcret)
+{
+    uint8_t         u8TrapNo;
+    int             rc;
+    TRPMEVENT       enmType;
+    RTGCUINT        uErrorCode;
+    RTGCUINTPTR     uCR2;
+
+    Assert(PATMIsPatchGCAddr(pVM, pCtx->eip));
+
+    if (gcret == VINF_PATM_PATCH_INT3)
+    {
+        u8TrapNo   = 3;
+        uCR2       = 0;
+        uErrorCode = 0;
+    }
+    else if (gcret == VINF_PATM_PATCH_TRAP_GP)
+    {
+        /* No active trap in this case. Kind of ugly. */
+        u8TrapNo   = X86_XCPT_GP;
+        uCR2       = 0;
+        uErrorCode = 0;
+    }
+    else
+    {
+        rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
+        if (RT_FAILURE(rc))
+        {
+            AssertReleaseMsgFailed(("emR3PatchTrap: no trap! (rc=%Rrc) gcret=%Rrc\n", rc, gcret));
+            return rc;
+        }
+        /* Reset the trap as we'll execute the original instruction again. */
+        TRPMResetTrap(pVCpu);
+    }
+
+    /*
+     * Deal with traps inside patch code.
+     * (This code won't run outside GC.)
+     */
+    if (u8TrapNo != 1)
+    {
+#ifdef LOG_ENABLED
+        DBGFR3InfoLog(pVM, "cpumguest", "Trap in patch code");
+        DBGFR3DisasInstrCurrentLog(pVCpu, "Patch code");
+
+        DISCPUSTATE Cpu;
+        rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->eip, &Cpu, "Patch code: ");
+        if (    RT_SUCCESS(rc)
+            &&  Cpu.pCurInstr->opcode == OP_IRET)
+        {
+            uint32_t eip, selCS, uEFlags;
+
+            /* Iret crashes are bad as we have already changed the flags on the stack */
+            rc  = PGMPhysSimpleReadGCPtr(pVCpu, &eip,     pCtx->esp, 4);
+            rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selCS,   pCtx->esp+4, 4);
+            rc |= PGMPhysSimpleReadGCPtr(pVCpu, &uEFlags, pCtx->esp+8, 4);
+            if (rc == VINF_SUCCESS)
+            {
+                if (    (uEFlags & X86_EFL_VM)
+                    ||  (selCS & X86_SEL_RPL) == 3)
+                {
+                    uint32_t selSS, esp;
+
+                    rc |= PGMPhysSimpleReadGCPtr(pVCpu, &esp,     pCtx->esp + 12, 4);
+                    rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selSS,   pCtx->esp + 16, 4);
+
+                    if (uEFlags & X86_EFL_VM)
+                    {
+                        uint32_t selDS, selES, selFS, selGS;
+                        rc  = PGMPhysSimpleReadGCPtr(pVCpu, &selES,   pCtx->esp + 20, 4);
+                        rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selDS,   pCtx->esp + 24, 4);
+                        rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selFS,   pCtx->esp + 28, 4);
+                        rc |= PGMPhysSimpleReadGCPtr(pVCpu, &selGS,   pCtx->esp + 32, 4);
+                        if (rc == VINF_SUCCESS)
+                        {
+                            Log(("Patch code: IRET->VM stack frame: return address %04X:%08RX32 eflags=%08x ss:esp=%04X:%08RX32\n", selCS, eip, uEFlags, selSS, esp));
+                            Log(("Patch code: IRET->VM stack frame: DS=%04X ES=%04X FS=%04X GS=%04X\n", selDS, selES, selFS, selGS));
+                        }
+                    }
+                    else
+                        Log(("Patch code: IRET stack frame: return address %04X:%08RX32 eflags=%08x ss:esp=%04X:%08RX32\n", selCS, eip, uEFlags, selSS, esp));
+                }
+                else
+                    Log(("Patch code: IRET stack frame: return address %04X:%08RX32 eflags=%08x\n", selCS, eip, uEFlags));
+            }
+        }
+#endif /* LOG_ENABLED */
+        Log(("emR3PatchTrap: in patch: eip=%08x: trap=%02x err=%08x cr2=%08x cr0=%08x\n",
+             pCtx->eip, u8TrapNo, uErrorCode, uCR2, (uint32_t)pCtx->cr0));
+
+        RTGCPTR pNewEip;
+        rc = PATMR3HandleTrap(pVM, pCtx, pCtx->eip, &pNewEip);
+        switch (rc)
+        {
+            /*
+             * Execute the faulting instruction.
+             */
+            case VINF_SUCCESS:
+            {
+                /** @todo execute a whole block */
+                Log(("emR3PatchTrap: Executing faulting instruction at new address %RGv\n", pNewEip));
+                if (!(pVCpu->em.s.pPatmGCState->uVMFlags & X86_EFL_IF))
+                    Log(("emR3PatchTrap: Virtual IF flag disabled!!\n"));
+
+                pCtx->eip = pNewEip;
+                AssertRelease(pCtx->eip);
+
+                if (pCtx->eflags.Bits.u1IF)
+                {
+                    /* Windows XP lets irets fault intentionally and then takes action based on the opcode; an
+                     * int3 patch overwrites it and leads to blue screens. Remove the patch in this case.
+                     */
+                    if (    u8TrapNo == X86_XCPT_GP
+                        &&  PATMIsInt3Patch(pVM, pCtx->eip, NULL, NULL))
+                    {
+                        /** @todo move to PATMR3HandleTrap */
+                        Log(("Possible Windows XP iret fault at %08RX32\n", pCtx->eip));
+                        PATMR3RemovePatch(pVM, pCtx->eip);
+                    }
+
+                    /** @todo Knoppix 5 regression when returning VINF_SUCCESS here and going back to raw mode. */
+                    /* Note: possibly because a reschedule is required (e.g. iret to V86 code) */
+
+                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
+                    /* Interrupts are enabled; just go back to the original instruction.
+                    return VINF_SUCCESS; */
+                }
+                return VINF_EM_RESCHEDULE_REM;
+            }
+
+            /*
+             * One instruction.
+             */
+            case VINF_PATCH_EMULATE_INSTR:
+                Log(("emR3PatchTrap: Emulate patched instruction at %RGv IF=%d VMIF=%x\n",
+                     pNewEip, pCtx->eflags.Bits.u1IF, pVCpu->em.s.pPatmGCState->uVMFlags));
+                pCtx->eip = pNewEip;
+                AssertRelease(pCtx->eip);
+                return emR3ExecuteInstruction(pVM, pVCpu, "PATCHEMUL: ");
+
+            /*
+             * The patch was disabled, hand it to the REM.
+             */
+            case VERR_PATCH_DISABLED:
+                if (!(pVCpu->em.s.pPatmGCState->uVMFlags & X86_EFL_IF))
+                    Log(("emR3PatchTrap: Virtual IF flag disabled!!\n"));
+                pCtx->eip = pNewEip;
+                AssertRelease(pCtx->eip);
+
+                if (pCtx->eflags.Bits.u1IF)
+                {
+                    /*
+                     * The last instruction in the patch block needs to be executed!! (sti/sysexit for example)
+                     */
+                    Log(("PATCH: IF=1 -> emulate last instruction as it can't be interrupted!!\n"));
+                    return emR3ExecuteInstruction(pVM, pVCpu, "PATCHIR");
+                }
+                return VINF_EM_RESCHEDULE_REM;
+
+            /* Force continued patch exection; usually due to write monitored stack. */
+            case VINF_PATCH_CONTINUE:
+                return VINF_SUCCESS;
+
+            /*
+             * Anything else is *fatal*.
+             */
+            default:
+                AssertReleaseMsgFailed(("Unknown return code %Rrc from PATMR3HandleTrap!\n", rc));
+                return VERR_IPE_UNEXPECTED_STATUS;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Handle a privileged instruction.
+ *
+ * @returns VBox status code suitable for EM.
+ * @param   pVM     VM handle.
+ * @param   pVCpu   VMCPU handle;
+ */
+static int emR3RawPrivileged(PVM pVM, PVMCPU pVCpu)
+{
+    PCPUMCTX    pCtx = pVCpu->em.s.pCtx;
+
+    Assert(!pCtx->eflags.Bits.u1VM);
+
+    if (PATMIsEnabled(pVM))
+    {
+        /*
+         * Check if in patch code.
+         */
+        if (PATMR3IsInsidePatchJump(pVM, pCtx->eip, NULL))
+        {
+#ifdef LOG_ENABLED
+            DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
+#endif
+            AssertMsgFailed(("FATAL ERROR: executing random instruction inside generated patch jump %08X\n", pCtx->eip));
+            return VERR_EM_RAW_PATCH_CONFLICT;
+        }
+        if (   (pCtx->ss & X86_SEL_RPL) == 0
+            && !pCtx->eflags.Bits.u1VM
+            && !PATMIsPatchGCAddr(pVM, pCtx->eip))
+        {
+            int rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
+                                        (SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0);
+            if (RT_SUCCESS(rc))
+            {
+#ifdef LOG_ENABLED
+                DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
+#endif
+                DBGFR3DisasInstrCurrentLog(pVCpu, "Patched privileged instruction");
+                return VINF_SUCCESS;
+            }
+        }
+    }
+
+#ifdef LOG_ENABLED
+    if (!PATMIsPatchGCAddr(pVM, pCtx->eip))
+    {
+        DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
+        DBGFR3DisasInstrCurrentLog(pVCpu, "Privileged instr: ");
+    }
+#endif
+
+    /*
+     * Instruction statistics and logging.
+     */
+    DISCPUSTATE Cpu;
+    int         rc;
+
+    rc = CPUMR3DisasmInstrCPU(pVM, pVCpu, pCtx, pCtx->rip, &Cpu, "PRIV: ");
+    if (RT_SUCCESS(rc))
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PEMSTATS pStats = pVCpu->em.s.CTX_SUFF(pStats);
+        switch (Cpu.pCurInstr->opcode)
+        {
+            case OP_INVLPG:
+                STAM_COUNTER_INC(&pStats->StatInvlpg);
+                break;
+            case OP_IRET:
+                STAM_COUNTER_INC(&pStats->StatIret);
+                break;
+            case OP_CLI:
+                STAM_COUNTER_INC(&pStats->StatCli);
+                emR3RecordCli(pVM, pVCpu, pCtx->rip);
+                break;
+            case OP_STI:
+                STAM_COUNTER_INC(&pStats->StatSti);
+                break;
+            case OP_INSB:
+            case OP_INSWD:
+            case OP_IN:
+            case OP_OUTSB:
+            case OP_OUTSWD:
+            case OP_OUT:
+                AssertMsgFailed(("Unexpected privileged exception due to port IO\n"));
+                break;
+
+            case OP_MOV_CR:
+                if (Cpu.param1.flags & USE_REG_GEN32)
+                {
+                    //read
+                    Assert(Cpu.param2.flags & USE_REG_CR);
+                    Assert(Cpu.param2.base.reg_ctrl <= USE_REG_CR4);
+                    STAM_COUNTER_INC(&pStats->StatMovReadCR[Cpu.param2.base.reg_ctrl]);
+                }
+                else
+                {
+                    //write
+                    Assert(Cpu.param1.flags & USE_REG_CR);
+                    Assert(Cpu.param1.base.reg_ctrl <= USE_REG_CR4);
+                    STAM_COUNTER_INC(&pStats->StatMovWriteCR[Cpu.param1.base.reg_ctrl]);
+                }
+                break;
+
+            case OP_MOV_DR:
+                STAM_COUNTER_INC(&pStats->StatMovDRx);
+                break;
+            case OP_LLDT:
+                STAM_COUNTER_INC(&pStats->StatMovLldt);
+                break;
+            case OP_LIDT:
+                STAM_COUNTER_INC(&pStats->StatMovLidt);
+                break;
+            case OP_LGDT:
+                STAM_COUNTER_INC(&pStats->StatMovLgdt);
+                break;
+            case OP_SYSENTER:
+                STAM_COUNTER_INC(&pStats->StatSysEnter);
+                break;
+            case OP_SYSEXIT:
+                STAM_COUNTER_INC(&pStats->StatSysExit);
+                break;
+            case OP_SYSCALL:
+                STAM_COUNTER_INC(&pStats->StatSysCall);
+                break;
+            case OP_SYSRET:
+                STAM_COUNTER_INC(&pStats->StatSysRet);
+                break;
+            case OP_HLT:
+                STAM_COUNTER_INC(&pStats->StatHlt);
+                break;
+            default:
+                STAM_COUNTER_INC(&pStats->StatMisc);
+                Log4(("emR3RawPrivileged: opcode=%d\n", Cpu.pCurInstr->opcode));
+                break;
+        }
+#endif /* VBOX_WITH_STATISTICS */
+        if (    (pCtx->ss & X86_SEL_RPL) == 0
+            &&  !pCtx->eflags.Bits.u1VM
+            &&  SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT)
+        {
+            uint32_t size;
+
+            STAM_PROFILE_START(&pVCpu->em.s.StatPrivEmu, a);
+            switch (Cpu.pCurInstr->opcode)
+            {
+                case OP_CLI:
+                    pCtx->eflags.u32 &= ~X86_EFL_IF;
+                    Assert(Cpu.opsize == 1);
+                    pCtx->rip += Cpu.opsize;
+                    STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
+                    return VINF_EM_RESCHEDULE_REM; /* must go to the recompiler now! */
+
+                case OP_STI:
+                    pCtx->eflags.u32 |= X86_EFL_IF;
+                    EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + Cpu.opsize);
+                    Assert(Cpu.opsize == 1);
+                    pCtx->rip += Cpu.opsize;
+                    STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
+                    return VINF_SUCCESS;
+
+                case OP_HLT:
+                    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
+                    {
+                        PATMTRANSSTATE  enmState;
+                        RTGCPTR         pOrgInstrGC = PATMR3PatchToGCPtr(pVM, pCtx->eip, &enmState);
+
+                        if (enmState == PATMTRANS_OVERWRITTEN)
+                        {
+                            rc = PATMR3DetectConflict(pVM, pOrgInstrGC, pOrgInstrGC);
+                            Assert(rc == VERR_PATCH_DISABLED);
+                            /* Conflict detected, patch disabled */
+                            Log(("emR3RawPrivileged: detected conflict -> disabled patch at %08RX32\n", pCtx->eip));
+
+                            enmState = PATMTRANS_SAFE;
+                        }
+
+                        /* The translation had better be successful. Otherwise we can't recover. */
+                        AssertReleaseMsg(pOrgInstrGC && enmState != PATMTRANS_OVERWRITTEN, ("Unable to translate instruction address at %08RX32\n", pCtx->eip));
+                        if (enmState != PATMTRANS_OVERWRITTEN)
+                            pCtx->eip = pOrgInstrGC;
+                    }
+                    /* no break; we could just return VINF_EM_HALT here */
+
+                case OP_MOV_CR:
+                case OP_MOV_DR:
+#ifdef LOG_ENABLED
+                    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
+                    {
+                        DBGFR3InfoLog(pVM, "cpumguest", "PRIV");
+                        DBGFR3DisasInstrCurrentLog(pVCpu, "Privileged instr: ");
+                    }
+#endif
+
+                    rc = VBOXSTRICTRC_TODO(EMInterpretInstructionCPU(pVM, pVCpu, &Cpu, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &size));
+                    if (RT_SUCCESS(rc))
+                    {
+                        pCtx->rip += Cpu.opsize;
+                        STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
+
+                        if (    Cpu.pCurInstr->opcode == OP_MOV_CR
+                            &&  Cpu.param1.flags == USE_REG_CR /* write */
+                           )
+                        {
+                            /* Deal with CR0 updates inside patch code that force
+                             * us to go to the recompiler.
+                             */
+                            if (   PATMIsPatchGCAddr(pVM, pCtx->rip)
+                                && (pCtx->cr0 & (X86_CR0_WP|X86_CR0_PG|X86_CR0_PE)) != (X86_CR0_WP|X86_CR0_PG|X86_CR0_PE))
+                            {
+                                PATMTRANSSTATE  enmState;
+                                RTGCPTR         pOrgInstrGC = PATMR3PatchToGCPtr(pVM, pCtx->rip, &enmState);
+
+                                Log(("Force recompiler switch due to cr0 (%RGp) update rip=%RGv -> %RGv (enmState=%d)\n", pCtx->cr0, pCtx->rip, pOrgInstrGC, enmState));
+                                if (enmState == PATMTRANS_OVERWRITTEN)
+                                {
+                                    rc = PATMR3DetectConflict(pVM, pOrgInstrGC, pOrgInstrGC);
+                                    Assert(rc == VERR_PATCH_DISABLED);
+                                    /* Conflict detected, patch disabled */
+                                    Log(("emR3RawPrivileged: detected conflict -> disabled patch at %RGv\n", (RTGCPTR)pCtx->rip));
+                                    enmState = PATMTRANS_SAFE;
+                                }
+                                /* The translation had better be successful. Otherwise we can't recover. */
+                                AssertReleaseMsg(pOrgInstrGC && enmState != PATMTRANS_OVERWRITTEN, ("Unable to translate instruction address at %RGv\n", (RTGCPTR)pCtx->rip));
+                                if (enmState != PATMTRANS_OVERWRITTEN)
+                                    pCtx->rip = pOrgInstrGC;
+                            }
+
+                            /* Reschedule is necessary as the execution/paging mode might have changed. */
+                            return VINF_EM_RESCHEDULE;
+                        }
+                        return rc; /* can return VINF_EM_HALT as well. */
+                    }
+                    AssertMsgReturn(rc == VERR_EM_INTERPRETER, ("%Rrc\n", rc), rc);
+                    break; /* fall back to the recompiler */
+            }
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatPrivEmu, a);
+        }
+    }
+
+    if (PATMIsPatchGCAddr(pVM, pCtx->eip))
+        return emR3PatchTrap(pVM, pVCpu, pCtx, VINF_PATM_PATCH_TRAP_GP);
+
+    return emR3ExecuteInstruction(pVM, pVCpu, "PRIV");
+}
+
+
+/**
+ * Update the forced rawmode execution modifier.
+ *
+ * This function is called when we're returning from the raw-mode loop(s). If we're
+ * in patch code, it will set a flag forcing execution to be resumed in raw-mode,
+ * if not in patch code, the flag will be cleared.
+ *
+ * We should never interrupt patch code while it's being executed. Cli patches can
+ * contain big code blocks, but they are always executed with IF=0. Other patches
+ * replace single instructions and should be atomic.
+ *
+ * @returns Updated rc.
+ *
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU handle.
+ * @param   pCtx    The guest CPU context.
+ * @param   rc      The result code.
+ */
+int emR3RawUpdateForceFlag(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
+{
+    if (PATMIsPatchGCAddr(pVM, pCtx->eip)) /** @todo check cs selector base/type */
+    {
+        /* ignore reschedule attempts. */
+        switch (rc)
+        {
+            case VINF_EM_RESCHEDULE:
+            case VINF_EM_RESCHEDULE_REM:
+                LogFlow(("emR3RawUpdateForceFlag: patch address -> force raw reschedule\n"));
+                rc = VINF_SUCCESS;
+                break;
+        }
+        pVCpu->em.s.fForceRAW = true;
+    }
+    else
+        pVCpu->em.s.fForceRAW = false;
+    return rc;
+}
+
+
+/**
+ * Check for pending raw actions
+ *
+ * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
+ *          EM statuses.
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VMCPU handle.
+ */
+VMMR3DECL(int) EMR3CheckRawForcedActions(PVM pVM, PVMCPU pVCpu)
+{
+    return emR3RawForcedActions(pVM, pVCpu, pVCpu->em.s.pCtx);
+}
+
+
+/**
+ * Process raw-mode specific forced actions.
+ *
+ * This function is called when any FFs in the VM_FF_HIGH_PRIORITY_PRE_RAW_MASK is pending.
+ *
+ * @returns VBox status code. May return VINF_EM_NO_MEMORY but none of the other
+ *          EM statuses.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU handle.
+ * @param   pCtx        The guest CPUM register context.
+ */
+static int emR3RawForcedActions(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    /*
+     * Note that the order is *vitally* important!
+     * Also note that SELMR3UpdateFromCPUM may trigger VM_FF_SELM_SYNC_TSS.
+     */
+
+
+    /*
+     * Sync selector tables.
+     */
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT))
+    {
+        int rc = SELMR3UpdateFromCPUM(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Sync IDT.
+     *
+     * The CSAMR3CheckGates call in TRPMR3SyncIDT may call PGMPrefetchPage
+     * and PGMShwModifyPage, so we're in for trouble if for instance a
+     * PGMSyncCR3+pgmR3PoolClearAll is pending.
+     */
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TRPM_SYNC_IDT))
+    {
+        if (   VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
+            && EMIsRawRing0Enabled(pVM)
+            && CSAMIsEnabled(pVM))
+        {
+            int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        int rc = TRPMR3SyncIDT(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Sync TSS.
+     */
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_TSS))
+    {
+        int rc = SELMR3SyncTSS(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Sync page directory.
+     */
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+    {
+        Assert(pVCpu->em.s.enmState != EMSTATE_WAIT_SIPI);
+        int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+        if (RT_FAILURE(rc))
+            return rc;
+
+        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
+
+        /* Prefetch pages for EIP and ESP. */
+        /** @todo This is rather expensive. Should investigate if it really helps at all. */
+        rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->rip));
+        if (rc == VINF_SUCCESS)
+            rc = PGMPrefetchPage(pVCpu, SELMToFlat(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->rsp));
+        if (rc != VINF_SUCCESS)
+        {
+            if (rc != VINF_PGM_SYNC_CR3)
+            {
+                AssertLogRelMsgReturn(RT_FAILURE(rc), ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                return rc;
+            }
+            rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        /** @todo maybe prefetch the supervisor stack page as well */
+        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT));
+    }
+
+    /*
+     * Allocate handy pages (just in case the above actions have consumed some pages).
+     */
+    if (VM_FF_IS_PENDING_EXCEPT(pVM, VM_FF_PGM_NEED_HANDY_PAGES, VM_FF_PGM_NO_MEMORY))
+    {
+        int rc = PGMR3PhysAllocateHandyPages(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Check whether we're out of memory now.
+     *
+     * This may stem from some of the above actions or operations that has been executed
+     * since we ran FFs. The allocate handy pages must for instance always be followed by
+     * this check.
+     */
+    if (VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY))
+        return VINF_EM_NO_MEMORY;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Executes raw code.
+ *
+ * This function contains the raw-mode version of the inner
+ * execution loop (the outer loop being in EMR3ExecuteVM()).
+ *
+ * @returns VBox status code. The most important ones are: VINF_EM_RESCHEDULE,
+ *          VINF_EM_RESCHEDULE_REM, VINF_EM_SUSPEND, VINF_EM_RESET and VINF_EM_TERMINATE.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   pfFFDone    Where to store an indicator telling whether or not
+ *                      FFs were done before returning.
+ */
+int emR3RawExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone)
+{
+    STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatRAWTotal, a);
+
+    int      rc = VERR_INTERNAL_ERROR;
+    PCPUMCTX pCtx = pVCpu->em.s.pCtx;
+    LogFlow(("emR3RawExecute: (cs:eip=%04x:%08x)\n", pCtx->cs, pCtx->eip));
+    pVCpu->em.s.fForceRAW = false;
+    *pfFFDone = false;
+
+
+    /*
+     *
+     * Spin till we get a forced action or raw mode status code resulting in
+     * in anything but VINF_SUCCESS or VINF_EM_RESCHEDULE_RAW.
+     *
+     */
+    for (;;)
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatRAWEntry, b);
+
+        /*
+         * Check various preconditions.
+         */
+#ifdef VBOX_STRICT
+        Assert(REMR3QueryPendingInterrupt(pVM, pVCpu) == REM_NO_PENDING_IRQ);
+        Assert(pCtx->eflags.Bits.u1VM || (pCtx->ss & X86_SEL_RPL) == 3 || (pCtx->ss & X86_SEL_RPL) == 0);
+        AssertMsg(   (pCtx->eflags.u32 & X86_EFL_IF)
+                  || PATMShouldUseRawMode(pVM, (RTGCPTR)pCtx->eip),
+                  ("Tried to execute code with IF at EIP=%08x!\n", pCtx->eip));
+        if (    !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
+            &&  PGMMapHasConflicts(pVM))
+        {
+            PGMMapCheck(pVM);
+            AssertMsgFailed(("We should not get conflicts any longer!!!\n"));
+            return VERR_INTERNAL_ERROR;
+        }
+#endif /* VBOX_STRICT */
+
+        /*
+         * Process high priority pre-execution raw-mode FFs.
+         */
+        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
+            if (rc != VINF_SUCCESS)
+                break;
+        }
+
+        /*
+         * If we're going to execute ring-0 code, the guest state needs to
+         * be modified a bit and some of the state components (IF, SS/CS RPL,
+         * and perhaps EIP) needs to be stored with PATM.
+         */
+        rc = CPUMR3RawEnter(pVCpu, NULL);
+        if (rc != VINF_SUCCESS)
+        {
+            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWEntry, b);
+            break;
+        }
+
+        /*
+         * Scan code before executing it. Don't bother with user mode or V86 code
+         */
+        if (    (pCtx->ss & X86_SEL_RPL) <= 1
+            &&  !pCtx->eflags.Bits.u1VM
+            && !PATMIsPatchGCAddr(pVM, pCtx->eip))
+        {
+            STAM_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatRAWEntry, b);
+            CSAMR3CheckCodeEx(pVM, CPUMCTX2CORE(pCtx), pCtx->eip);
+            STAM_PROFILE_ADV_RESUME(&pVCpu->em.s.StatRAWEntry, b);
+            if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_PRE_RAW_MASK)
+                ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+            {
+                rc = emR3RawForcedActions(pVM, pVCpu, pCtx);
+                if (rc != VINF_SUCCESS)
+                {
+                    rc = CPUMR3RawLeave(pVCpu, NULL, rc);
+                    break;
+                }
+            }
+        }
+
+#ifdef LOG_ENABLED
+        /*
+         * Log important stuff before entering GC.
+         */
+        PPATMGCSTATE pGCState = PATMR3QueryGCStateHC(pVM);
+        if (pCtx->eflags.Bits.u1VM)
+            Log(("RV86: %04X:%08X IF=%d VMFlags=%x\n", pCtx->cs, pCtx->eip, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags));
+        else if ((pCtx->ss & X86_SEL_RPL) == 1)
+        {
+            bool fCSAMScanned = CSAMIsPageScanned(pVM, (RTGCPTR)pCtx->eip);
+            Log(("RR0: %08X ESP=%08X IF=%d VMFlags=%x PIF=%d CPL=%d (Scanned=%d)\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags, pGCState->fPIF, (pCtx->ss & X86_SEL_RPL), fCSAMScanned));
+        }
+        else if ((pCtx->ss & X86_SEL_RPL) == 3)
+            Log(("RR3: %08X ESP=%08X IF=%d VMFlags=%x\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags));
+#endif /* LOG_ENABLED */
+
+
+
+        /*
+         * Execute the code.
+         */
+        STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWEntry, b);
+        if (RT_LIKELY(EMR3IsExecutionAllowed(pVM, pVCpu)))
+        {
+            STAM_PROFILE_START(&pVCpu->em.s.StatRAWExec, c);
+            rc = VMMR3RawRunGC(pVM, pVCpu);
+            STAM_PROFILE_STOP(&pVCpu->em.s.StatRAWExec, c);
+        }
+        else
+        {
+            /* Give up this time slice; virtual time continues */
+            STAM_REL_PROFILE_ADV_START(&pVCpu->em.s.StatCapped, u);
+            RTThreadSleep(5);
+            STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatCapped, u);
+            rc = VINF_SUCCESS;
+        }
+        STAM_PROFILE_ADV_START(&pVCpu->em.s.StatRAWTail, d);
+
+        LogFlow(("RR0-E: %08X ESP=%08X IF=%d VMFlags=%x PIF=%d CPL=%d\n", pCtx->eip, pCtx->esp, pCtx->eflags.Bits.u1IF, pGCState->uVMFlags, pGCState->fPIF, (pCtx->ss & X86_SEL_RPL)));
+        LogFlow(("VMMR3RawRunGC returned %Rrc\n", rc));
+
+
+
+        /*
+         * Restore the real CPU state and deal with high priority post
+         * execution FFs before doing anything else.
+         */
+        rc = CPUMR3RawLeave(pVCpu, NULL, rc);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_RESUME_GUEST_MASK);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_HIGH_PRIORITY_POST_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_MASK))
+            rc = emR3HighPriorityPostForcedActions(pVM, pVCpu, rc);
+
+#ifdef VBOX_STRICT
+        /*
+         * Assert TSS consistency & rc vs patch code.
+         */
+        if (   !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_TSS | VMCPU_FF_SELM_SYNC_GDT) /* GDT implies TSS at the moment. */
+            &&  EMIsRawRing0Enabled(pVM))
+            SELMR3CheckTSS(pVM);
+        switch (rc)
+        {
+            case VINF_SUCCESS:
+            case VINF_EM_RAW_INTERRUPT:
+            case VINF_PATM_PATCH_TRAP_PF:
+            case VINF_PATM_PATCH_TRAP_GP:
+            case VINF_PATM_PATCH_INT3:
+            case VINF_PATM_CHECK_PATCH_PAGE:
+            case VINF_EM_RAW_EXCEPTION_PRIVILEGED:
+            case VINF_EM_RAW_GUEST_TRAP:
+            case VINF_EM_RESCHEDULE_RAW:
+                break;
+
+            default:
+                if (PATMIsPatchGCAddr(pVM, pCtx->eip) && !(pCtx->eflags.u32 & X86_EFL_TF))
+                    LogIt(NULL, 0, LOG_GROUP_PATM, ("Patch code interrupted at %RRv for reason %Rrc\n", (RTRCPTR)CPUMGetGuestEIP(pVCpu), rc));
+                break;
+        }
+        /*
+         * Let's go paranoid!
+         */
+        if (    !VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
+            &&  PGMMapHasConflicts(pVM))
+        {
+            PGMMapCheck(pVM);
+            AssertMsgFailed(("We should not get conflicts any longer!!! rc=%Rrc\n", rc));
+            return VERR_INTERNAL_ERROR;
+        }
+#endif /* VBOX_STRICT */
+
+        /*
+         * Process the returned status code.
+         */
+        if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+        {
+            STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
+            break;
+        }
+        rc = emR3RawHandleRC(pVM, pVCpu, pCtx, rc);
+        if (rc != VINF_SUCCESS)
+        {
+            rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
+            if (rc != VINF_SUCCESS)
+            {
+                STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
+                break;
+            }
+        }
+
+        /*
+         * Check and execute forced actions.
+         */
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+        TMTimerPollVoid(pVM, pVCpu);
+#endif
+        STAM_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTail, d);
+        if (    VM_FF_ISPENDING(pVM, ~VM_FF_HIGH_PRIORITY_PRE_RAW_MASK | VM_FF_PGM_NO_MEMORY)
+            ||  VMCPU_FF_ISPENDING(pVCpu, ~VMCPU_FF_HIGH_PRIORITY_PRE_RAW_MASK))
+        {
+            Assert(pCtx->eflags.Bits.u1VM || (pCtx->ss & X86_SEL_RPL) != 1);
+
+            STAM_REL_PROFILE_ADV_SUSPEND(&pVCpu->em.s.StatRAWTotal, a);
+            rc = emR3ForcedActions(pVM, pVCpu, rc);
+            STAM_REL_PROFILE_ADV_RESUME(&pVCpu->em.s.StatRAWTotal, a);
+            if (    rc != VINF_SUCCESS
+                &&  rc != VINF_EM_RESCHEDULE_RAW)
+            {
+                rc = emR3RawUpdateForceFlag(pVM, pVCpu, pCtx, rc);
+                if (rc != VINF_SUCCESS)
+                {
+                    *pfFFDone = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    /*
+     * Return to outer loop.
+     */
+#if defined(LOG_ENABLED) && defined(DEBUG)
+    RTLogFlush(NULL);
+#endif
+    STAM_REL_PROFILE_ADV_STOP(&pVCpu->em.s.StatRAWTotal, a);
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/FTM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/FTM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/FTM.cpp	(revision 35333)
@@ -0,0 +1,1351 @@
+/* $Id$ */
+/** @file
+ * FTM - Fault Tolerance Manager
+ */
+
+/*
+ * Copyright (C) 2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_FTM
+#include "FTMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/vmm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/ssm.h>
+#include <VBox/log.h>
+#include <VBox/pgm.h>
+#include <VBox/pdm.h>
+
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <iprt/mem.h>
+#include <iprt/tcp.h>
+#include <iprt/socket.h>
+#include <iprt/semaphore.h>
+#include <iprt/asm.h>
+
+#include "internal/vm.h"
+#include "internal/em.h"
+#include "internal/pgm.h"
+
+/*******************************************************************************
+ * Structures and Typedefs                                                     *
+ *******************************************************************************/
+
+/**
+ * TCP stream header.
+ *
+ * This is an extra layer for fixing the problem with figuring out when the SSM
+ * stream ends.
+ */
+typedef struct FTMTCPHDR
+{
+    /** Magic value. */
+    uint32_t    u32Magic;
+    /** The size of the data block following this header.
+     * 0 indicates the end of the stream, while UINT32_MAX indicates
+     * cancelation. */
+    uint32_t    cb;
+} FTMTCPHDR;
+/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
+#define FTMTCPHDR_MAGIC       UINT32_C(0x19471205)
+/** The max block size. */
+#define FTMTCPHDR_MAX_SIZE    UINT32_C(0x00fffff8)
+
+/**
+ * TCP stream header.
+ *
+ * This is an extra layer for fixing the problem with figuring out when the SSM
+ * stream ends.
+ */
+typedef struct FTMTCPHDRMEM
+{
+    /** Magic value. */
+    uint32_t    u32Magic;
+    /** Size (Uncompressed) of the pages following the header. */
+    uint32_t    cbPageRange;
+    /** GC Physical address of the page(s) to sync. */
+    RTGCPHYS    GCPhys;
+    /** The size of the data block following this header.
+     * 0 indicates the end of the stream, while UINT32_MAX indicates
+     * cancelation. */
+    uint32_t    cb;
+} FTMTCPHDRMEM;
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
+
+static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser);
+
+/**
+ * Initializes the FTM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) FTMR3Init(PVM pVM)
+{
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
+    AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
+
+    /** @todo saved state for master nodes! */
+    pVM->ftm.s.pszAddress               = NULL;
+    pVM->ftm.s.pszPassword              = NULL;
+    pVM->fFaultTolerantMaster           = false;
+    pVM->ftm.s.fIsStandbyNode           = false;
+    pVM->ftm.s.standby.hServer          = NIL_RTTCPSERVER;
+    pVM->ftm.s.hShutdownEvent           = NIL_RTSEMEVENT;
+    pVM->ftm.s.hSocket                  = NIL_RTSOCKET;
+
+    /*
+     * Initialize the PGM critical section.
+     */
+    int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register statistics.
+     */
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem,               STAMTYPE_COUNTER, "/FT/Received/Mem",                  STAMUNIT_BYTES, "The amount of memory pages that was received.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState,             STAMTYPE_COUNTER, "/FT/Received/State",                STAMUNIT_BYTES, "The amount of state information that was received.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem,                   STAMTYPE_COUNTER, "/FT/Sent/Mem",                      STAMUNIT_BYTES, "The amount of memory pages that was sent.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState,                 STAMTYPE_COUNTER, "/FT/Sent/State",                    STAMUNIT_BYTES, "The amount of state information that was sent.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM,                   STAMTYPE_COUNTER, "/FT/Sync/DeltaVM",                  STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync,                  STAMTYPE_COUNTER, "/FT/Sync/Full",                     STAMUNIT_OCCURENCES, "Number of full vm syncs.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem,                  STAMTYPE_COUNTER, "/FT/Sync/DeltaMem",                 STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage,         STAMTYPE_COUNTER, "/FT/Checkpoint/Storage",            STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
+    STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork,         STAMTYPE_COUNTER, "/FT/Checkpoint/Network",            STAMUNIT_OCCURENCES, "Number of network checkpoints.");
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG(pVM,     &pVM->ftm.s.StatCheckpoint,                STAMTYPE_PROFILE, "/FT/Checkpoint",                    STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
+    STAM_REG(pVM,     &pVM->ftm.s.StatCheckpointPause,           STAMTYPE_PROFILE, "/FT/Checkpoint/Pause",              STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
+    STAM_REG(pVM,     &pVM->ftm.s.StatCheckpointResume,          STAMTYPE_PROFILE, "/FT/Checkpoint/Resume",             STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
+    STAM_REG(pVM,     &pVM->ftm.s.StatSentMemRAM,                STAMTYPE_COUNTER, "/FT/Sent/Mem/RAM",                  STAMUNIT_BYTES, "The amount of memory pages that was sent.");
+    STAM_REG(pVM,     &pVM->ftm.s.StatSentMemMMIO2,              STAMTYPE_COUNTER, "/FT/Sent/Mem/MMIO2",                STAMUNIT_BYTES, "The amount of memory pages that was sent.");
+    STAM_REG(pVM,     &pVM->ftm.s.StatSentMemShwROM,             STAMTYPE_COUNTER, "/FT/Sent/Mem/ShwROM",               STAMUNIT_BYTES, "The amount of memory pages that was sent.");
+    STAM_REG(pVM,     &pVM->ftm.s.StatSentStateWrite,            STAMTYPE_COUNTER, "/FT/Sent/State/Writes",             STAMUNIT_BYTES, "The nr of write calls.");
+#endif
+    return VINF_SUCCESS;
+}
+
+/**
+ * Terminates the FTM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM itself is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) FTMR3Term(PVM pVM)
+{
+    if (pVM->ftm.s.hShutdownEvent != NIL_RTSEMEVENT)
+    {
+        RTSemEventDestroy(pVM->ftm.s.hShutdownEvent);
+        pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
+    }
+    if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
+    {
+        RTTcpClientClose(pVM->ftm.s.hSocket);
+        pVM->ftm.s.hSocket = NIL_RTSOCKET;
+    }
+    if (pVM->ftm.s.standby.hServer)
+    {
+        RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
+        pVM->ftm.s.standby.hServer = NULL;
+    }
+    if (pVM->ftm.s.pszAddress)
+        RTMemFree(pVM->ftm.s.pszAddress);
+    if (pVM->ftm.s.pszPassword)
+        RTMemFree(pVM->ftm.s.pszPassword);
+
+    /* Remove all pending memory updates. */
+    if (pVM->ftm.s.standby.pPhysPageTree)
+    {
+        RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, NULL);
+        pVM->ftm.s.standby.pPhysPageTree = NULL;
+    }
+
+    pVM->ftm.s.pszAddress  = NULL;
+    pVM->ftm.s.pszPassword = NULL;
+
+    PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
+    return VINF_SUCCESS;
+}
+
+
+static int ftmR3TcpWriteACK(PVM pVM)
+{
+    int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
+    }
+    return rc;
+}
+
+
+static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
+{
+    char    szMsg[256];
+    size_t  cch;
+    if (pszMsgText && *pszMsgText)
+    {
+        cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
+        for (size_t off = 6; off + 1 < cch; off++)
+            if (szMsg[off] == '\n')
+                szMsg[off] = '\r';
+    }
+    else
+        cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
+    int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
+    if (RT_FAILURE(rc))
+        LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
+    return rc;
+}
+
+/**
+ * Reads a string from the socket.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pState      The teleporter state structure.
+ * @param   pszBuf      The output buffer.
+ * @param   cchBuf      The size of the output buffer.
+ *
+ */
+static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
+{
+    char       *pszStart = pszBuf;
+    RTSOCKET    Sock     = pVM->ftm.s.hSocket;
+
+    AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
+    *pszBuf = '\0';
+
+    /* dead simple approach. */
+    for (;;)
+    {
+        char ch;
+        int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
+            return rc;
+        }
+        if (    ch == '\n'
+            ||  ch == '\0')
+            return VINF_SUCCESS;
+        if (cchBuf <= 1)
+        {
+            LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
+            return VERR_BUFFER_OVERFLOW;
+        }
+        *pszBuf++ = ch;
+        *pszBuf = '\0';
+        cchBuf--;
+    }
+}
+
+/**
+ * Reads an ACK or NACK.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM to operate on.
+ * @param   pszWhich            Which ACK is this this?
+ * @param   pszNAckMsg          Optional NACK message.
+ */
+static int ftmR3TcpReadACK(PVM pVM, const char *pszWhich, const char *pszNAckMsg = NULL)
+{
+    char szMsg[256];
+    int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
+    if (RT_FAILURE(rc))
+        return rc;
+
+    if (!strcmp(szMsg, "ACK"))
+        return VINF_SUCCESS;
+
+    if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
+    {
+        char *pszMsgText = strchr(szMsg, ';');
+        if (pszMsgText)
+            *pszMsgText++ = '\0';
+
+        int32_t vrc2;
+        rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
+        if (rc == VINF_SUCCESS)
+        {
+            /*
+             * Well formed NACK, transform it into an error.
+             */
+            if (pszNAckMsg)
+            {
+                LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
+                return VERR_INTERNAL_ERROR;
+            }
+
+            if (pszMsgText)
+            {
+                pszMsgText = RTStrStrip(pszMsgText);
+                for (size_t off = 0; pszMsgText[off]; off++)
+                    if (pszMsgText[off] == '\r')
+                        pszMsgText[off] = '\n';
+
+                LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
+            }
+            return VERR_INTERNAL_ERROR_2;
+        }
+
+        if (pszMsgText)
+            pszMsgText[-1] = ';';
+    }
+    return VERR_INTERNAL_ERROR_3;
+}
+
+/**
+ * Submitts a command to the destination and waits for the ACK.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM to operate on.
+ * @param   pszCommand          The command.
+ * @param   fWaitForAck         Whether to wait for the ACK.
+ */
+static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
+{
+    int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (!fWaitForAck)
+        return VINF_SUCCESS;
+    return ftmR3TcpReadACK(pVM, pszCommand);
+}
+
+/**
+ * @copydoc SSMSTRMOPS::pfnWrite
+ */
+static DECLCALLBACK(int) ftmR3TcpOpWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
+{
+    PVM pVM = (PVM)pvUser;
+
+    AssertReturn(cbToWrite > 0, VINF_SUCCESS);
+    AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
+    AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
+
+    STAM_COUNTER_INC(&pVM->ftm.s.StatSentStateWrite);
+    for (;;)
+    {
+        FTMTCPHDR Hdr;
+        Hdr.u32Magic = FTMTCPHDR_MAGIC;
+        Hdr.cb       = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
+        int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
+            return rc;
+        }
+        pVM->ftm.s.StatSentState.c      += Hdr.cb + sizeof(Hdr);
+        pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
+        if (Hdr.cb == cbToWrite)
+            return VINF_SUCCESS;
+
+        /* advance */
+        cbToWrite -= Hdr.cb;
+        pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
+    }
+}
+
+
+/**
+ * Selects and poll for close condition.
+ *
+ * We can use a relatively high poll timeout here since it's only used to get
+ * us out of error paths.  In the normal cause of events, we'll get a
+ * end-of-stream header.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pState          The teleporter state data.
+ */
+static int ftmR3TcpReadSelect(PVM pVM)
+{
+    int rc;
+    do
+    {
+        rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
+        if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
+        {
+            pVM->ftm.s.syncstate.fIOError = true;
+            LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
+            break;
+        }
+        if (pVM->ftm.s.syncstate.fStopReading)
+        {
+            rc = VERR_EOF;
+            break;
+        }
+    } while (rc == VERR_TIMEOUT);
+    return rc;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnRead
+ */
+static DECLCALLBACK(int) ftmR3TcpOpRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
+{
+    PVM pVM = (PVM)pvUser;
+    AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
+
+    for (;;)
+    {
+        int rc;
+
+        /*
+         * Check for various conditions and may have been signalled.
+         */
+        if (pVM->ftm.s.syncstate.fEndOfStream)
+            return VERR_EOF;
+        if (pVM->ftm.s.syncstate.fStopReading)
+            return VERR_EOF;
+        if (pVM->ftm.s.syncstate.fIOError)
+            return VERR_IO_GEN_FAILURE;
+
+        /*
+         * If there is no more data in the current block, read the next
+         * block header.
+         */
+        if (!pVM->ftm.s.syncstate.cbReadBlock)
+        {
+            rc = ftmR3TcpReadSelect(pVM);
+            if (RT_FAILURE(rc))
+                return rc;
+            FTMTCPHDR Hdr;
+            rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
+            if (RT_FAILURE(rc))
+            {
+                pVM->ftm.s.syncstate.fIOError = true;
+                LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
+                return rc;
+            }
+            pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
+
+            if (RT_UNLIKELY(   Hdr.u32Magic != FTMTCPHDR_MAGIC
+                            || Hdr.cb > FTMTCPHDR_MAX_SIZE
+                            || Hdr.cb == 0))
+            {
+                if (    Hdr.u32Magic == FTMTCPHDR_MAGIC
+                    &&  (   Hdr.cb == 0
+                         || Hdr.cb == UINT32_MAX)
+                   )
+                {
+                    pVM->ftm.s.syncstate.fEndOfStream = true;
+                    pVM->ftm.s.syncstate.cbReadBlock  = 0;
+                    return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
+                }
+                pVM->ftm.s.syncstate.fIOError = true;
+                LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
+                return VERR_IO_GEN_FAILURE;
+            }
+
+            pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
+            if (pVM->ftm.s.syncstate.fStopReading)
+                return VERR_EOF;
+        }
+
+        /*
+         * Read more data.
+         */
+        rc = ftmR3TcpReadSelect(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
+        rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
+        if (RT_FAILURE(rc))
+        {
+            pVM->ftm.s.syncstate.fIOError = true;
+            LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
+            return rc;
+        }
+        if (pcbRead)
+        {
+            cb = (uint32_t)*pcbRead;
+            pVM->ftm.s.StatReceivedState.c   += cb;
+            pVM->ftm.s.syncstate.uOffStream  += cb;
+            pVM->ftm.s.syncstate.cbReadBlock -= cb;
+            return VINF_SUCCESS;
+        }
+        pVM->ftm.s.StatReceivedState.c   += cb;
+        pVM->ftm.s.syncstate.uOffStream  += cb;
+        pVM->ftm.s.syncstate.cbReadBlock -= cb;
+        if (cbToRead == cb)
+            return VINF_SUCCESS;
+
+        /* Advance to the next block. */
+        cbToRead -= cb;
+        pvBuf = (uint8_t *)pvBuf + cb;
+    }
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnSeek
+ */
+static DECLCALLBACK(int) ftmR3TcpOpSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
+{
+    return VERR_NOT_SUPPORTED;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnTell
+ */
+static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
+{
+    PVM pVM = (PVM)pvUser;
+    return pVM->ftm.s.syncstate.uOffStream;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnSize
+ */
+static DECLCALLBACK(int) ftmR3TcpOpSize(void *pvUser, uint64_t *pcb)
+{
+    return VERR_NOT_SUPPORTED;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnIsOk
+ */
+static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
+{
+    PVM pVM = (PVM)pvUser;
+
+    if (pVM->fFaultTolerantMaster)
+    {
+        /* Poll for incoming NACKs and errors from the other side */
+        int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
+        if (rc != VERR_TIMEOUT)
+        {
+            if (RT_SUCCESS(rc))
+            {
+                LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
+                rc = VERR_SSM_CANCELLED;
+            }
+            else
+                LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
+            return rc;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnClose
+ */
+static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
+{
+    PVM pVM = (PVM)pvUser;
+
+    if (pVM->fFaultTolerantMaster)
+    {
+        FTMTCPHDR EofHdr;
+        EofHdr.u32Magic = FTMTCPHDR_MAGIC;
+        EofHdr.cb       = fCanceled ? UINT32_MAX : 0;
+        int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
+            return rc;
+        }
+    }
+    else
+    {
+        ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Method table for a TCP based stream.
+ */
+static SSMSTRMOPS const g_ftmR3TcpOps =
+{
+    SSMSTRMOPS_VERSION,
+    ftmR3TcpOpWrite,
+    ftmR3TcpOpRead,
+    ftmR3TcpOpSeek,
+    ftmR3TcpOpTell,
+    ftmR3TcpOpSize,
+    ftmR3TcpOpIsOk,
+    ftmR3TcpOpClose,
+    SSMSTRMOPS_VERSION
+};
+
+
+/**
+ * VMR3ReqCallWait callback
+ *
+ * @param   pVM         The VM handle.
+ *
+ */
+static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
+{
+    int rc = PGMR3PhysWriteProtectRAM(pVM);
+    AssertRC(rc);
+}
+
+
+/**
+ * Sync the VM state
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+static int ftmR3PerformFullSync(PVM pVM)
+{
+    bool fSuspended = false;
+
+    int rc = VMR3Suspend(pVM);
+    AssertRCReturn(rc, rc);
+
+    STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
+
+    RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
+
+    /* Reset the sync state. */
+    pVM->ftm.s.syncstate.uOffStream   = 0;
+    pVM->ftm.s.syncstate.cbReadBlock  = 0;
+    pVM->ftm.s.syncstate.fStopReading = false;
+    pVM->ftm.s.syncstate.fIOError     = false;
+    pVM->ftm.s.syncstate.fEndOfStream = false;
+
+    rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
+    AssertRC(rc);
+
+    pVM->ftm.s.fDeltaLoadSaveActive = false;
+    rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
+    AssertRC(rc);
+
+    rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
+    AssertRC(rc);
+
+    RTSocketRelease(pVM->ftm.s.hSocket);
+
+    /* Write protect all memory. */
+    rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
+    AssertRCReturn(rc, rc);
+
+    rc = VMR3Resume(pVM);
+    AssertRC(rc);
+
+    return rc;
+}
+
+
+/**
+ * PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
+ *
+ * @param   pVM             VM Handle.
+ * @param   GCPhys          GC physical address
+ * @param   pRange          HC virtual address of the page(s)
+ * @param   cbRange         Size of the dirty range in bytes.
+ * @param   pvUser          User argument
+ */
+static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t *pRange, unsigned cbRange, void *pvUser)
+{
+    FTMTCPHDRMEM Hdr;
+    Hdr.u32Magic    = FTMTCPHDR_MAGIC;
+    Hdr.GCPhys      = GCPhys;
+    Hdr.cbPageRange = cbRange;
+    Hdr.cb          = cbRange;
+    /** @todo compress page(s). */
+    int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
+        return rc;
+    }
+    pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
+
+#ifdef VBOX_WITH_STATISTICS
+    switch (PGMPhysGetPageType(pVM, GCPhys))
+    {
+    case PGMPAGETYPE_RAM:
+        pVM->ftm.s.StatSentMemRAM.c    += Hdr.cb + sizeof(Hdr);
+        break;
+
+    case PGMPAGETYPE_MMIO2:
+        pVM->ftm.s.StatSentMemMMIO2.c  += Hdr.cb + sizeof(Hdr);
+        break;
+
+    case PGMPAGETYPE_ROM_SHADOW:
+        pVM->ftm.s.StatSentMemShwROM.c += Hdr.cb + sizeof(Hdr);
+        break;
+
+    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+        AssertFailed();
+        break;
+
+    default:
+        AssertFailed();
+        break;
+    }
+#endif
+
+    return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
+}
+
+/**
+ * Thread function which starts syncing process for this master VM
+ *
+ * @param   Thread      The thread id.
+ * @param   pvUser      Not used
+ * @return  VINF_SUCCESS (ignored).
+ *
+ */
+static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
+{
+    int rc  = VINF_SUCCESS;
+    PVM pVM = (PVM)pvUser;
+
+    for (;;)
+    {
+        /*
+         * Try connect to the standby machine.
+         */
+        Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
+        rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("ftmR3MasterThread: CONNECTED\n"));
+
+            /* Disable Nagle. */
+            rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /*fEnable*/);
+            AssertRC(rc);
+
+            /* Read and check the welcome message. */
+            char szLine[RT_MAX(128, sizeof(g_szWelcome))];
+            RT_ZERO(szLine);
+            rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
+            if (    RT_SUCCESS(rc)
+                &&  !strcmp(szLine, g_szWelcome))
+            {
+                /* password */
+                if (pVM->ftm.s.pszPassword)
+                    rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
+
+                if (RT_SUCCESS(rc))
+                {
+                    /* ACK */
+                    rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
+                    if (RT_SUCCESS(rc))
+                    {
+                        /** todo: verify VM config. */
+                        break;
+                    }
+                }
+            }
+            /* Failed, so don't bother anymore. */
+            return VINF_SUCCESS;
+        }
+        rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, 1000 /* 1 second */);
+        if (rc != VERR_TIMEOUT)
+            return VINF_SUCCESS;    /* told to quit */
+    }
+
+    /* Successfully initialized the connection to the standby node.
+     * Start the sync process.
+     */
+
+    /* First sync all memory and write protect everything so
+     * we can send changed pages later on.
+     */
+
+    rc = ftmR3PerformFullSync(pVM);
+
+    for (;;)
+    {
+        rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
+        if (rc != VERR_TIMEOUT)
+            break;    /* told to quit */
+
+        if (!pVM->ftm.s.fCheckpointingActive)
+        {
+            rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
+            AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+
+            rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
+            AssertRC(rc);
+
+            /* sync the changed memory with the standby node. */
+            /* Write protect all memory. */
+            if (!pVM->ftm.s.fCheckpointingActive)
+            {
+                rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
+                AssertRC(rc);
+            }
+
+            /* Enumerate all dirty pages and send them to the standby VM. */
+            if (!pVM->ftm.s.fCheckpointingActive)
+            {
+                rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
+                Assert(rc == VINF_SUCCESS || rc == VERR_INTERRUPTED);
+            }
+
+            /* Send last memory header to signal the end. */
+            FTMTCPHDRMEM Hdr;
+            Hdr.u32Magic    = FTMTCPHDR_MAGIC;
+            Hdr.GCPhys      = 0;
+            Hdr.cbPageRange = 0;
+            Hdr.cb          = 0;
+            rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
+            if (RT_FAILURE(rc))
+                LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
+
+            rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
+            AssertRC(rc);
+
+            PDMCritSectLeave(&pVM->ftm.s.CritSect);
+        }
+    }
+    return rc;
+}
+
+/**
+ * Syncs memory from the master VM
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ */
+static int ftmR3SyncMem(PVM pVM)
+{
+    while (true)
+    {
+        FTMTCPHDRMEM Hdr;
+        RTGCPHYS GCPhys;
+
+        /* Read memory header. */
+        int rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
+        if (RT_FAILURE(rc))
+        {
+            Log(("RTTcpRead failed with %Rrc\n", rc));
+            break;
+        }
+        pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
+
+        if (Hdr.cb == 0)
+            break;  /* end of sync. */
+
+        Assert(Hdr.cb == Hdr.cbPageRange);  /** @todo uncompress */
+        GCPhys = Hdr.GCPhys;
+
+        /* Must be a multiple of PAGE_SIZE. */
+        Assert((Hdr.cbPageRange & 0xfff) == 0);
+
+        while (Hdr.cbPageRange)
+        {
+            PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)RTAvlGCPhysGet(&pVM->ftm.s.standby.pPhysPageTree, GCPhys);
+            if (!pNode)
+            {
+                /* Allocate memory for the node and page. */
+                pNode = (PFTMPHYSPAGETREENODE)RTMemAllocZ(sizeof(*pNode) + PAGE_SIZE);
+                AssertBreak(pNode);
+
+                /* Insert the node into the tree. */
+                pNode->Core.Key = GCPhys;
+                pNode->pPage = (void *)(pNode + 1);
+                bool fRet = RTAvlGCPhysInsert(&pVM->ftm.s.standby.pPhysPageTree, &pNode->Core);
+                Assert(fRet);
+            }
+
+            /* Fetch the page. */
+            rc = RTTcpRead(pVM->ftm.s.hSocket, pNode->pPage, PAGE_SIZE, NULL);
+            if (RT_FAILURE(rc))
+            {
+                Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
+                break;
+            }
+            pVM->ftm.s.StatReceivedMem.c += PAGE_SIZE;
+            Hdr.cbPageRange              -= PAGE_SIZE;
+            GCPhys                       += PAGE_SIZE;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback handler for RTAvlGCPhysDestroy
+ *
+ * @returns 0 to continue, otherwise stop
+ * @param   pBaseNode       Node to destroy
+ * @param   pvUser          User parameter
+ */
+static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser)
+{
+    PVM pVM = (PVM)pvUser;
+    PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)pBaseNode;
+
+    if (pVM)    /* NULL when the VM is destroyed. */
+    {
+        /* Update the guest memory of the standby VM. */
+        int rc = PGMR3PhysWriteExternal(pVM, pNode->Core.Key, pNode->pPage, PAGE_SIZE, "FTMemSync");
+        AssertRC(rc);
+    }
+    RTMemFree(pNode);
+    return 0;
+}
+
+/**
+ * Thread function which monitors the health of the master VM
+ *
+ * @param   Thread      The thread id.
+ * @param   pvUser      Not used
+ * @return  VINF_SUCCESS (ignored).
+ *
+ */
+static DECLCALLBACK(int) ftmR3StandbyThread(RTTHREAD Thread, void *pvUser)
+{
+    PVM pVM = (PVM)pvUser;
+
+    for (;;)
+    {
+        uint64_t u64TimeNow;
+
+        int rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
+        if (rc != VERR_TIMEOUT)
+            break;    /* told to quit */
+
+        if (pVM->ftm.s.standby.u64LastHeartbeat)
+        {
+            u64TimeNow = RTTimeMilliTS();
+
+            if (u64TimeNow > pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 4)
+            {
+                /* Timeout; prepare to fallover. */
+                LogRel(("FTSync: TIMEOUT (%RX64 vs %RX64 ms): activate standby VM!\n", u64TimeNow, pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 2));
+
+                pVM->ftm.s.fActivateStandby = true;
+                /** todo: prevent split-brain. */
+                break;
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Listen for incoming traffic destined for the standby VM.
+ *
+ * @copydoc FNRTTCPSERVE
+ *
+ * @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
+ */
+static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
+{
+    PVM pVM = (PVM)pvUser;
+
+    pVM->ftm.s.hSocket = Sock;
+
+    /*
+     * Disable Nagle.
+     */
+    int rc = RTTcpSetSendCoalescing(Sock, false /*fEnable*/);
+    AssertRC(rc);
+
+    /* Send the welcome message to the master node. */
+    rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Password.
+     */
+    const char *pszPassword = pVM->ftm.s.pszPassword;
+    if (pszPassword)
+    {
+        unsigned    off = 0;
+        while (pszPassword[off])
+        {
+            char ch;
+            rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
+            if (    RT_FAILURE(rc)
+                ||  pszPassword[off] != ch)
+            {
+                if (RT_FAILURE(rc))
+                    LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
+                else
+                    LogRel(("FTSync: Invalid password (off=%u)\n", off));
+                ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
+                return VINF_SUCCESS;
+            }
+            off++;
+        }
+    }
+    rc = ftmR3TcpWriteACK(pVM);
+    if (RT_FAILURE(rc))
+        return VINF_SUCCESS;
+
+    /** todo: verify VM config. */
+
+    /*
+     * Stop the server.
+     *
+     * Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
+     *       to it we must not return that value!
+     */
+    RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
+
+    /*
+     * Command processing loop.
+     */
+    bool fDone = false;
+    for (;;)
+    {
+        bool fFullSync = false;
+        char szCmd[128];
+
+        rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
+        if (RT_FAILURE(rc))
+            break;
+
+        pVM->ftm.s.standby.u64LastHeartbeat = RTTimeMilliTS();
+        if (!strcmp(szCmd, "mem-sync"))
+        {
+            rc = ftmR3TcpWriteACK(pVM);
+            AssertRC(rc);
+            if (RT_FAILURE(rc))
+                continue;
+
+            rc = ftmR3SyncMem(pVM);
+            AssertRC(rc);
+
+            rc = ftmR3TcpWriteACK(pVM);
+            AssertRC(rc);
+        }
+        else
+        if (    !strcmp(szCmd, "checkpoint")
+            ||  !strcmp(szCmd, "full-sync")
+            ||  (fFullSync = true))  /* intended assignment */
+        {
+            rc = ftmR3TcpWriteACK(pVM);
+            AssertRC(rc);
+            if (RT_FAILURE(rc))
+                continue;
+
+            /* Flush all pending memory updates. */
+            if (pVM->ftm.s.standby.pPhysPageTree)
+            {
+                RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, pVM);
+                pVM->ftm.s.standby.pPhysPageTree = NULL;
+            }
+
+            RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
+
+            /* Reset the sync state. */
+            pVM->ftm.s.syncstate.uOffStream   = 0;
+            pVM->ftm.s.syncstate.cbReadBlock  = 0;
+            pVM->ftm.s.syncstate.fStopReading = false;
+            pVM->ftm.s.syncstate.fIOError     = false;
+            pVM->ftm.s.syncstate.fEndOfStream = false;
+
+            pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
+            rc = VMR3LoadFromStreamFT(pVM, &g_ftmR3TcpOps, pVM);
+            pVM->ftm.s.fDeltaLoadSaveActive = false;
+            RTSocketRelease(pVM->ftm.s.hSocket);
+            AssertRC(rc);
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
+                ftmR3TcpWriteNACK(pVM, rc);
+                continue;
+            }
+
+            /* The EOS might not have been read, make sure it is. */
+            pVM->ftm.s.syncstate.fStopReading = false;
+            size_t cbRead;
+            rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
+            if (rc != VERR_EOF)
+            {
+                LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
+                ftmR3TcpWriteNACK(pVM, rc);
+                continue;
+            }
+
+            rc = ftmR3TcpWriteACK(pVM);
+            AssertRC(rc);
+        }
+    }
+    LogFlowFunc(("returns mRc=%Rrc\n", rc));
+    return VERR_TCP_SERVER_STOP;
+}
+
+/**
+ * Powers on the fault tolerant virtual machine.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   fMaster     FT master or standby
+ * @param   uInterval   FT sync interval
+ * @param   pszAddress  Standby VM address
+ * @param   uPort       Standby VM port
+ * @param   pszPassword FT password (NULL for none)
+ *
+ * @thread      Any thread.
+ * @vmstate     Created
+ * @vmstateto   PoweringOn+Running (master), PoweringOn+Running_FT (standby)
+ */
+VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char *pszAddress, unsigned uPort, const char *pszPassword)
+{
+    int rc = VINF_SUCCESS;
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(enmVMState == VMSTATE_CREATED,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_INTERNAL_ERROR_4);
+    AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
+
+    if (pVM->ftm.s.uInterval)
+        pVM->ftm.s.uInterval    = uInterval;
+    else
+        pVM->ftm.s.uInterval    = 50;   /* standard sync interval of 50ms */
+
+    pVM->ftm.s.uPort            = uPort;
+    pVM->ftm.s.pszAddress       = RTStrDup(pszAddress);
+    if (pszPassword)
+        pVM->ftm.s.pszPassword  = RTStrDup(pszPassword);
+
+    rc = RTSemEventCreate(&pVM->ftm.s.hShutdownEvent);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    if (fMaster)
+    {
+        rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
+                            0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
+        if (RT_FAILURE(rc))
+            return rc;
+
+        pVM->fFaultTolerantMaster = true;
+        if (PGMIsUsingLargePages(pVM))
+        {
+            /* Must disable large page usage as 2 MB pages are too big to write monitor. */
+            LogRel(("FTSync: disabling large page usage.\n"));
+            PGMSetLargePageUsage(pVM, false);
+        }
+        /** @todo might need to disable page fusion as well */
+
+        return VMR3PowerOn(pVM);
+    }
+    else
+    {
+        /* standby */
+        rc = RTThreadCreate(NULL, ftmR3StandbyThread, pVM,
+                            0, RTTHREADTYPE_DEFAULT, 0, "ftmStandby");
+        if (RT_FAILURE(rc))
+            return rc;
+
+        rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
+        if (RT_FAILURE(rc))
+            return rc;
+        pVM->ftm.s.fIsStandbyNode = true;
+
+        rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
+        /** @todo deal with the exit code to check if we should activate this standby VM. */
+        if (pVM->ftm.s.fActivateStandby)
+        {
+            /** @todo fallover. */
+        }
+
+        if (pVM->ftm.s.standby.hServer)
+        {
+            RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
+            pVM->ftm.s.standby.hServer = NULL;
+        }
+        if (rc == VERR_TCP_SERVER_SHUTDOWN)
+            rc = VINF_SUCCESS;  /* ignore this error; the standby process was cancelled. */
+    }
+    return rc;
+}
+
+/**
+ * Powers off the fault tolerant virtual machine (standby).
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
+{
+    AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
+    Assert(pVM->ftm.s.standby.hServer);
+
+    return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
+}
+
+/**
+ * Rendezvous callback used by FTMR3SetCheckpoint
+ * Sync state + changed memory with the standby node.
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
+ * @param   pvUser      User parameter
+ */
+static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    bool fSuspended = false;
+
+    /** We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
+     *  is only a short suspend.
+     */
+    STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointPause, a);
+    PDMR3Suspend(pVM);
+
+    /** Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
+    EMR3NotifySuspend(pVM);
+    STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointPause, a);
+
+    STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
+
+    RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
+
+    /* Reset the sync state. */
+    pVM->ftm.s.syncstate.uOffStream   = 0;
+    pVM->ftm.s.syncstate.cbReadBlock  = 0;
+    pVM->ftm.s.syncstate.fStopReading = false;
+    pVM->ftm.s.syncstate.fIOError     = false;
+    pVM->ftm.s.syncstate.fEndOfStream = false;
+
+    rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
+    AssertRC(rc);
+
+    pVM->ftm.s.fDeltaLoadSaveActive = true;
+    rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
+    pVM->ftm.s.fDeltaLoadSaveActive = false;
+    AssertRC(rc);
+
+    rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
+    AssertRC(rc);
+
+    RTSocketRelease(pVM->ftm.s.hSocket);
+
+    /* Write protect all memory. */
+    rc = PGMR3PhysWriteProtectRAM(pVM);
+    AssertRC(rc);
+
+    /** We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
+     *  is only a short suspend.
+     */
+    STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointResume, b);
+    PGMR3ResetNoMorePhysWritesFlag(pVM);
+    PDMR3Resume(pVM);
+
+    /** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
+    EMR3NotifyResume(pVM);
+    STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointResume, b);
+
+    return rc;
+}
+
+/**
+ * Performs a full sync to the standby node
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The VM to operate on.
+ * @param   enmCheckpoint   Checkpoint type
+ */
+VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
+{
+    int rc;
+
+    if (!pVM->fFaultTolerantMaster)
+        return VINF_SUCCESS;
+
+    switch (enmCheckpoint)
+    {
+    case FTMCHECKPOINTTYPE_NETWORK:
+        STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
+        break;
+
+    case FTMCHECKPOINTTYPE_STORAGE:
+        STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
+        break;
+
+    default:
+        break;
+    }
+    pVM->ftm.s.fCheckpointingActive = true;
+    if (VM_IS_EMT(pVM))
+    {
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+
+        /* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
+        while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
+        {
+            if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
+            {
+                rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
+                AssertRC(rc);
+            }
+
+            if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
+            {
+                rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+                AssertRC(rc);
+            }
+        }
+    }
+    else
+        rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
+
+    AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+
+    STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
+
+    rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
+
+    STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
+
+    PDMCritSectLeave(&pVM->ftm.s.CritSect);
+    pVM->ftm.s.fCheckpointingActive = false;
+
+    return rc;
+}
Index: /trunk/src/VBox/VMM/VMMR3/GMM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/GMM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/GMM.cpp	(revision 35333)
@@ -0,0 +1,442 @@
+/* $Id$ */
+/** @file
+ * GMM - Global Memory Manager, ring-3 request wrappers.
+ */
+
+/*
+ * Copyright (C) 2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_GMM
+#include <VBox/gmm.h>
+#include <VBox/vmm.h>
+#include <VBox/vm.h>
+#include <VBox/sup.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+
+/**
+ * @see GMMR0InitialReservation
+ */
+GMMR3DECL(int)  GMMR3InitialReservation(PVM pVM, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages,
+                                        GMMOCPOLICY enmPolicy, GMMPRIORITY enmPriority)
+{
+    GMMINITIALRESERVATIONREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.cBasePages = cBasePages;
+    Req.cShadowPages = cShadowPages;
+    Req.cFixedPages = cFixedPages;
+    Req.enmPolicy = enmPolicy;
+    Req.enmPriority = enmPriority;
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_INITIAL_RESERVATION, 0, &Req.Hdr);
+}
+
+
+/**
+ * @see GMMR0UpdateReservation
+ */
+GMMR3DECL(int)  GMMR3UpdateReservation(PVM pVM, uint64_t cBasePages, uint32_t cShadowPages, uint32_t cFixedPages)
+{
+    GMMUPDATERESERVATIONREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.cBasePages = cBasePages;
+    Req.cShadowPages = cShadowPages;
+    Req.cFixedPages = cFixedPages;
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_UPDATE_RESERVATION, 0, &Req.Hdr);
+}
+
+
+/**
+ * Prepares a GMMR0AllocatePages request.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
+ * @param       pVM         Pointer to the shared VM structure.
+ * @param[out]  ppReq       Where to store the pointer to the request packet.
+ * @param       cPages      The number of pages that's to be allocated.
+ * @param       enmAccount  The account to charge.
+ */
+GMMR3DECL(int) GMMR3AllocatePagesPrepare(PVM pVM, PGMMALLOCATEPAGESREQ *ppReq, uint32_t cPages, GMMACCOUNT enmAccount)
+{
+    uint32_t cb = RT_OFFSETOF(GMMALLOCATEPAGESREQ, aPages[cPages]);
+    PGMMALLOCATEPAGESREQ pReq = (PGMMALLOCATEPAGESREQ)RTMemTmpAllocZ(cb);
+    if (!pReq)
+        return VERR_NO_TMP_MEMORY;
+
+    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq = cb;
+    pReq->enmAccount = enmAccount;
+    pReq->cPages = cPages;
+    NOREF(pVM);
+    *ppReq = pReq;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs a GMMR0AllocatePages request.
+ * This will call VMSetError on failure.
+ *
+ * @returns VBox status code.
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pReq        Pointer to the request (returned by GMMR3AllocatePagesPrepare).
+ */
+GMMR3DECL(int) GMMR3AllocatePagesPerform(PVM pVM, PGMMALLOCATEPAGESREQ pReq)
+{
+    for (unsigned i = 0; ; i++)
+    {
+        int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_ALLOCATE_PAGES, 0, &pReq->Hdr);
+        if (RT_SUCCESS(rc))
+        {
+#ifdef LOG_ENABLED
+            for (uint32_t iPage = 0; iPage < pReq->cPages; iPage++)
+                Log3(("GMMR3AllocatePagesPerform: idPage=%#x HCPhys=%RHp\n",
+                      pReq->aPages[iPage].idPage, pReq->aPages[iPage].HCPhysGCPhys));
+#endif
+            return rc;
+        }
+        if (rc != VERR_GMM_SEED_ME)
+            return VMSetError(pVM, rc, RT_SRC_POS,
+                              N_("GMMR0AllocatePages failed to allocate %u pages"),
+                              pReq->cPages);
+        Assert(i < pReq->cPages);
+
+        /*
+         * Seed another chunk.
+         */
+        void *pvChunk;
+        rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS,
+                              N_("Out of memory (SUPR3PageAlloc) seeding a %u pages allocation request"),
+                              pReq->cPages);
+
+        rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
+        if (RT_FAILURE(rc))
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("GMM seeding failed"));
+    }
+}
+
+
+/**
+ * Cleans up a GMMR0AllocatePages request.
+ * @param   pReq        Pointer to the request (returned by GMMR3AllocatePagesPrepare).
+ */
+GMMR3DECL(void) GMMR3AllocatePagesCleanup(PGMMALLOCATEPAGESREQ pReq)
+{
+    RTMemTmpFree(pReq);
+}
+
+
+/**
+ * Prepares a GMMR0FreePages request.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
+ * @param       pVM         Pointer to the shared VM structure.
+ * @param[out]  ppReq       Where to store the pointer to the request packet.
+ * @param       cPages      The number of pages that's to be freed.
+ * @param       enmAccount  The account to charge.
+ */
+GMMR3DECL(int) GMMR3FreePagesPrepare(PVM pVM, PGMMFREEPAGESREQ *ppReq, uint32_t cPages, GMMACCOUNT enmAccount)
+{
+    uint32_t cb = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[cPages]);
+    PGMMFREEPAGESREQ pReq = (PGMMFREEPAGESREQ)RTMemTmpAllocZ(cb);
+    if (!pReq)
+        return VERR_NO_TMP_MEMORY;
+
+    pReq->Hdr.u32Magic  = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq     = cb;
+    pReq->enmAccount    = enmAccount;
+    pReq->cPages        = cPages;
+    NOREF(pVM);
+    *ppReq = pReq;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Re-prepares a GMMR0FreePages request.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_TMP_MEMORY.
+ * @param       pVM         Pointer to the shared VM structure.
+ * @param       pReq        A request buffer previously returned by
+ *                          GMMR3FreePagesPrepare().
+ * @param       cPages      The number of pages originally passed to
+ *                          GMMR3FreePagesPrepare().
+ * @param       enmAccount  The account to charge.
+ */
+GMMR3DECL(void) GMMR3FreePagesRePrep(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t cPages, GMMACCOUNT enmAccount)
+{
+    Assert(pReq->Hdr.u32Magic == SUPVMMR0REQHDR_MAGIC);
+    pReq->Hdr.cbReq     = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[cPages]);
+    pReq->enmAccount    = enmAccount;
+    pReq->cPages        = cPages;
+    NOREF(pVM);
+}
+
+
+/**
+ * Performs a GMMR0FreePages request.
+ * This will call VMSetError on failure.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pReq            Pointer to the request (returned by GMMR3FreePagesPrepare).
+ * @param   cActualPages    The number of pages actually freed.
+ */
+GMMR3DECL(int) GMMR3FreePagesPerform(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t cActualPages)
+{
+    /*
+     * Adjust the request if we ended up with fewer pages than anticipated.
+     */
+    if (cActualPages != pReq->cPages)
+    {
+        AssertReturn(cActualPages < pReq->cPages, VERR_INTERNAL_ERROR);
+        if (!cActualPages)
+            return VINF_SUCCESS;
+        pReq->cPages = cActualPages;
+        pReq->Hdr.cbReq = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[cActualPages]);
+    }
+
+    /*
+     * Do the job.
+     */
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_PAGES, 0, &pReq->Hdr);
+    if (RT_SUCCESS(rc))
+        return rc;
+    AssertRC(rc);
+    return VMSetError(pVM, rc, RT_SRC_POS,
+                      N_("GMMR0FreePages failed to free %u pages"),
+                      pReq->cPages);
+}
+
+
+/**
+ * Cleans up a GMMR0FreePages request.
+ * @param   pReq        Pointer to the request (returned by GMMR3FreePagesPrepare).
+ */
+GMMR3DECL(void) GMMR3FreePagesCleanup(PGMMFREEPAGESREQ pReq)
+{
+    RTMemTmpFree(pReq);
+}
+
+
+/**
+ * Frees allocated pages, for bailing out on failure.
+ *
+ * This will not call VMSetError on failure but will use AssertLogRel instead.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pAllocReq   The allocation request to undo.
+ */
+GMMR3DECL(void) GMMR3FreeAllocatedPages(PVM pVM, GMMALLOCATEPAGESREQ const *pAllocReq)
+{
+    uint32_t cb = RT_OFFSETOF(GMMFREEPAGESREQ, aPages[pAllocReq->cPages]);
+    PGMMFREEPAGESREQ pReq = (PGMMFREEPAGESREQ)RTMemTmpAllocZ(cb);
+    AssertLogRelReturnVoid(pReq);
+
+    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq = cb;
+    pReq->enmAccount = pAllocReq->enmAccount;
+    pReq->cPages = pAllocReq->cPages;
+    uint32_t iPage = pAllocReq->cPages;
+    while (iPage-- > 0)
+    {
+        Assert(pAllocReq->aPages[iPage].idPage != NIL_GMM_PAGEID);
+        pReq->aPages[iPage].idPage = pAllocReq->aPages[iPage].idPage;
+    }
+
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_PAGES, 0, &pReq->Hdr);
+    AssertLogRelRC(rc);
+
+    RTMemTmpFree(pReq);
+}
+
+
+/**
+ * @see GMMR0BalloonedPages
+ */
+GMMR3DECL(int)  GMMR3BalloonedPages(PVM pVM, GMMBALLOONACTION enmAction, uint32_t cBalloonedPages)
+{
+    GMMBALLOONEDPAGESREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.enmAction = enmAction;
+    Req.cBalloonedPages = cBalloonedPages;
+
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_BALLOONED_PAGES, 0, &Req.Hdr);
+}
+
+/**
+ * @see GMMR0QueryVMMMemoryStatsReq
+ */
+GMMR3DECL(int)  GMMR3QueryHypervisorMemoryStats(PVM pVM, uint64_t *pcTotalAllocPages, uint64_t *pcTotalFreePages, uint64_t *pcTotalBalloonPages, uint64_t *puTotalBalloonSize)
+{
+    GMMMEMSTATSREQ Req;
+    Req.Hdr.u32Magic     = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq        = sizeof(Req);
+    Req.cAllocPages      = 0;
+    Req.cFreePages       = 0;
+    Req.cBalloonedPages  = 0;
+    Req.cSharedPages     = 0;
+
+    *pcTotalAllocPages   = 0;
+    *pcTotalFreePages    = 0;
+    *pcTotalBalloonPages = 0;
+    *puTotalBalloonSize  = 0;
+
+    /* Must be callable from any thread, so can't use VMMR3CallR0. */
+    int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_QUERY_HYPERVISOR_MEM_STATS, 0, &Req.Hdr);
+    if (rc == VINF_SUCCESS)
+    {
+        *pcTotalAllocPages   = Req.cAllocPages;
+        *pcTotalFreePages    = Req.cFreePages;
+        *pcTotalBalloonPages = Req.cBalloonedPages;
+        *puTotalBalloonSize  = Req.cSharedPages;
+    }
+    return rc;
+}
+
+/**
+ * @see GMMR0QueryMemoryStatsReq
+ */
+GMMR3DECL(int)  GMMR3QueryMemoryStats(PVM pVM, uint64_t *pcAllocPages, uint64_t *pcMaxPages, uint64_t *pcBalloonPages)
+{
+    GMMMEMSTATSREQ Req;
+    Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq       = sizeof(Req);
+    Req.cAllocPages     = 0;
+    Req.cFreePages      = 0;
+    Req.cBalloonedPages = 0;
+
+    *pcAllocPages      = 0;
+    *pcMaxPages         = 0;
+    *pcBalloonPages     = 0;
+
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_QUERY_MEM_STATS, 0, &Req.Hdr);
+    if (rc == VINF_SUCCESS)
+    {
+        *pcAllocPages   = Req.cAllocPages;
+        *pcMaxPages     = Req.cMaxPages;
+        *pcBalloonPages = Req.cBalloonedPages;
+    }
+    return rc;
+}
+
+/**
+ * @see GMMR0MapUnmapChunk
+ */
+GMMR3DECL(int)  GMMR3MapUnmapChunk(PVM pVM, uint32_t idChunkMap, uint32_t idChunkUnmap, PRTR3PTR ppvR3)
+{
+    GMMMAPUNMAPCHUNKREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.idChunkMap = idChunkMap;
+    Req.idChunkUnmap = idChunkUnmap;
+    Req.pvR3 = NULL;
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
+    if (RT_SUCCESS(rc) && ppvR3)
+        *ppvR3 = Req.pvR3;
+    return rc;
+}
+
+/**
+ * @see GMMR0FreeLargePage
+ */
+GMMR3DECL(int)  GMMR3FreeLargePage(PVM pVM,  uint32_t idPage)
+{
+    GMMFREELARGEPAGEREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.idPage = idPage;
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_FREE_LARGE_PAGE, 0, &Req.Hdr);
+}
+
+/**
+ * @see GMMR0SeedChunk
+ */
+GMMR3DECL(int)  GMMR3SeedChunk(PVM pVM, RTR3PTR pvR3)
+{
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvR3, NULL);
+}
+
+
+/**
+ * @see GMMR0RegisterSharedModule
+ */
+GMMR3DECL(int) GMMR3RegisterSharedModule(PVM pVM, PGMMREGISTERSHAREDMODULEREQ pReq)
+{
+    pReq->Hdr.u32Magic  = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq     = RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[pReq->cRegions]);
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_REGISTER_SHARED_MODULE, 0, &pReq->Hdr);
+    if (rc == VINF_SUCCESS)
+        rc = pReq->rc;
+    return rc;
+}
+
+/**
+ * @see GMMR0RegisterSharedModule
+ */
+GMMR3DECL(int) GMMR3UnregisterSharedModule(PVM pVM, PGMMUNREGISTERSHAREDMODULEREQ pReq)
+{
+    pReq->Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    pReq->Hdr.cbReq = sizeof(*pReq);
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_UNREGISTER_SHARED_MODULE, 0, &pReq->Hdr);
+}
+
+/**
+ * @see GMMR0ResetSharedModules
+ */
+GMMR3DECL(int) GMMR3ResetSharedModules(PVM pVM)
+{
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_RESET_SHARED_MODULES, 0, NULL);
+}
+
+/**
+ * @see GMMR0CheckSharedModules
+ */
+GMMR3DECL(int)  GMMR3CheckSharedModules(PVM pVM)
+{
+    return VMMR3CallR0(pVM, VMMR0_DO_GMM_CHECK_SHARED_MODULES, 0, NULL);
+}
+
+#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+/**
+ * @see GMMR0FindDuplicatePage
+ */
+GMMR3DECL(bool) GMMR3IsDuplicatePage(PVM pVM, uint32_t idPage)
+{
+    GMMFINDDUPLICATEPAGEREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq    = sizeof(Req);
+    Req.idPage       = idPage;
+    Req.fDuplicate   = false;
+
+    /* Must be callable from any thread, so can't use VMMR3CallR0. */
+    int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_FIND_DUPLICATE_PAGE, 0, &Req.Hdr);
+    if (rc == VINF_SUCCESS)
+        return Req.fDuplicate;
+    else
+        return false;
+}
+#endif /* VBOX_STRICT && HC_ARCH_BITS == 64 */
+
Index: /trunk/src/VBox/VMM/VMMR3/HWACCM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/HWACCM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/HWACCM.cpp	(revision 35333)
@@ -0,0 +1,2821 @@
+/* $Id$ */
+/** @file
+ * HWACCM - Intel/AMD VM Hardware Support Manager
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_HWACCM
+#include <VBox/cpum.h>
+#include <VBox/stam.h>
+#include <VBox/mm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pgm.h>
+#include <VBox/ssm.h>
+#include <VBox/trpm.h>
+#include <VBox/dbgf.h>
+#include <VBox/iom.h>
+#include <VBox/patm.h>
+#include <VBox/csam.h>
+#include <VBox/selm.h>
+#include <VBox/rem.h>
+#include <VBox/hwacc_vmx.h>
+#include <VBox/hwacc_svm.h>
+#include "HWACCMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+#include <iprt/env.h>
+#include <iprt/thread.h>
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#ifdef VBOX_WITH_STATISTICS
+# define EXIT_REASON(def, val, str) #def " - " #val " - " str
+# define EXIT_REASON_NIL() NULL
+/** Exit reason descriptions for VT-x, used to describe statistics. */
+static const char * const g_apszVTxExitReasons[MAX_EXITREASON_STAT] =
+{
+    EXIT_REASON(VMX_EXIT_EXCEPTION          ,  0, "Exception or non-maskable interrupt (NMI)."),
+    EXIT_REASON(VMX_EXIT_EXTERNAL_IRQ       ,  1, "External interrupt."),
+    EXIT_REASON(VMX_EXIT_TRIPLE_FAULT       ,  2, "Triple fault."),
+    EXIT_REASON(VMX_EXIT_INIT_SIGNAL        ,  3, "INIT signal."),
+    EXIT_REASON(VMX_EXIT_SIPI               ,  4, "Start-up IPI (SIPI)."),
+    EXIT_REASON(VMX_EXIT_IO_SMI_IRQ         ,  5, "I/O system-management interrupt (SMI)."),
+    EXIT_REASON(VMX_EXIT_SMI_IRQ            ,  6, "Other SMI."),
+    EXIT_REASON(VMX_EXIT_IRQ_WINDOW         ,  7, "Interrupt window."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_TASK_SWITCH        ,  9, "Task switch."),
+    EXIT_REASON(VMX_EXIT_CPUID              , 10, "Guest software attempted to execute CPUID."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_HLT                , 12, "Guest software attempted to execute HLT."),
+    EXIT_REASON(VMX_EXIT_INVD               , 13, "Guest software attempted to execute INVD."),
+    EXIT_REASON(VMX_EXIT_INVPG              , 14, "Guest software attempted to execute INVPG."),
+    EXIT_REASON(VMX_EXIT_RDPMC              , 15, "Guest software attempted to execute RDPMC."),
+    EXIT_REASON(VMX_EXIT_RDTSC              , 16, "Guest software attempted to execute RDTSC."),
+    EXIT_REASON(VMX_EXIT_RSM                , 17, "Guest software attempted to execute RSM in SMM."),
+    EXIT_REASON(VMX_EXIT_VMCALL             , 18, "Guest software executed VMCALL."),
+    EXIT_REASON(VMX_EXIT_VMCLEAR            , 19, "Guest software executed VMCLEAR."),
+    EXIT_REASON(VMX_EXIT_VMLAUNCH           , 20, "Guest software executed VMLAUNCH."),
+    EXIT_REASON(VMX_EXIT_VMPTRLD            , 21, "Guest software executed VMPTRLD."),
+    EXIT_REASON(VMX_EXIT_VMPTRST            , 22, "Guest software executed VMPTRST."),
+    EXIT_REASON(VMX_EXIT_VMREAD             , 23, "Guest software executed VMREAD."),
+    EXIT_REASON(VMX_EXIT_VMRESUME           , 24, "Guest software executed VMRESUME."),
+    EXIT_REASON(VMX_EXIT_VMWRITE            , 25, "Guest software executed VMWRITE."),
+    EXIT_REASON(VMX_EXIT_VMXOFF             , 26, "Guest software executed VMXOFF."),
+    EXIT_REASON(VMX_EXIT_VMXON              , 27, "Guest software executed VMXON."),
+    EXIT_REASON(VMX_EXIT_CRX_MOVE           , 28, "Control-register accesses."),
+    EXIT_REASON(VMX_EXIT_DRX_MOVE           , 29, "Debug-register accesses."),
+    EXIT_REASON(VMX_EXIT_PORT_IO            , 30, "I/O instruction."),
+    EXIT_REASON(VMX_EXIT_RDMSR              , 31, "RDMSR. Guest software attempted to execute RDMSR."),
+    EXIT_REASON(VMX_EXIT_WRMSR              , 32, "WRMSR. Guest software attempted to execute WRMSR."),
+    EXIT_REASON(VMX_EXIT_ERR_INVALID_GUEST_STATE,  33, "VM-entry failure due to invalid guest state."),
+    EXIT_REASON(VMX_EXIT_ERR_MSR_LOAD       , 34, "VM-entry failure due to MSR loading."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_MWAIT              , 36, "Guest software executed MWAIT."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_MONITOR            , 39, "Guest software attempted to execute MONITOR."),
+    EXIT_REASON(VMX_EXIT_PAUSE              , 40, "Guest software attempted to execute PAUSE."),
+    EXIT_REASON(VMX_EXIT_ERR_MACHINE_CHECK  , 41, "VM-entry failure due to machine-check."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_TPR                , 43, "TPR below threshold. Guest software executed MOV to CR8."),
+    EXIT_REASON(VMX_EXIT_APIC_ACCESS        , 44, "APIC access. Guest software attempted to access memory at a physical address on the APIC-access page."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_XDTR_ACCESS        , 46, "Access to GDTR or IDTR. Guest software attempted to execute LGDT, LIDT, SGDT, or SIDT."),
+    EXIT_REASON(VMX_EXIT_TR_ACCESS          , 47, "Access to LDTR or TR. Guest software attempted to execute LLDT, LTR, SLDT, or STR."),
+    EXIT_REASON(VMX_EXIT_EPT_VIOLATION      , 48, "EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures."),
+    EXIT_REASON(VMX_EXIT_EPT_MISCONFIG      , 49, "EPT misconfiguration. An attempt to access memory with a guest-physical address encountered a misconfigured EPT paging-structure entry."),
+    EXIT_REASON(VMX_EXIT_INVEPT             , 50, "INVEPT. Guest software attempted to execute INVEPT."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_PREEMPTION_TIMER   , 52, "VMX-preemption timer expired. The preemption timer counted down to zero."),
+    EXIT_REASON(VMX_EXIT_INVVPID            , 53, "INVVPID. Guest software attempted to execute INVVPID."),
+    EXIT_REASON(VMX_EXIT_WBINVD             , 54, "WBINVD. Guest software attempted to execute WBINVD."),
+    EXIT_REASON(VMX_EXIT_XSETBV             , 55, "XSETBV. Guest software attempted to execute XSETBV."),
+    EXIT_REASON_NIL()
+};
+/** Exit reason descriptions for AMD-V, used to describe statistics. */
+static const char * const g_apszAmdVExitReasons[MAX_EXITREASON_STAT] =
+{
+    EXIT_REASON(SVM_EXIT_READ_CR0                   ,  0, "Read CR0."),
+    EXIT_REASON(SVM_EXIT_READ_CR1                   ,  1, "Read CR1."),
+    EXIT_REASON(SVM_EXIT_READ_CR2                   ,  2, "Read CR2."),
+    EXIT_REASON(SVM_EXIT_READ_CR3                   ,  3, "Read CR3."),
+    EXIT_REASON(SVM_EXIT_READ_CR4                   ,  4, "Read CR4."),
+    EXIT_REASON(SVM_EXIT_READ_CR5                   ,  5, "Read CR5."),
+    EXIT_REASON(SVM_EXIT_READ_CR6                   ,  6, "Read CR6."),
+    EXIT_REASON(SVM_EXIT_READ_CR7                   ,  7, "Read CR7."),
+    EXIT_REASON(SVM_EXIT_READ_CR8                   ,  8, "Read CR8."),
+    EXIT_REASON(SVM_EXIT_READ_CR9                   ,  9, "Read CR9."),
+    EXIT_REASON(SVM_EXIT_READ_CR10                  , 10, "Read CR10."),
+    EXIT_REASON(SVM_EXIT_READ_CR11                  , 11, "Read CR11."),
+    EXIT_REASON(SVM_EXIT_READ_CR12                  , 12, "Read CR12."),
+    EXIT_REASON(SVM_EXIT_READ_CR13                  , 13, "Read CR13."),
+    EXIT_REASON(SVM_EXIT_READ_CR14                  , 14, "Read CR14."),
+    EXIT_REASON(SVM_EXIT_READ_CR15                  , 15, "Read CR15."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR0                  , 16, "Write CR0."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR1                  , 17, "Write CR1."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR2                  , 18, "Write CR2."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR3                  , 19, "Write CR3."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR4                  , 20, "Write CR4."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR5                  , 21, "Write CR5."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR6                  , 22, "Write CR6."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR7                  , 23, "Write CR7."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR8                  , 24, "Write CR8."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR9                  , 25, "Write CR9."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR10                 , 26, "Write CR10."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR11                 , 27, "Write CR11."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR12                 , 28, "Write CR12."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR13                 , 29, "Write CR13."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR14                 , 30, "Write CR14."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR15                 , 31, "Write CR15."),
+    EXIT_REASON(SVM_EXIT_READ_DR0                   , 32, "Read DR0."),
+    EXIT_REASON(SVM_EXIT_READ_DR1                   , 33, "Read DR1."),
+    EXIT_REASON(SVM_EXIT_READ_DR2                   , 34, "Read DR2."),
+    EXIT_REASON(SVM_EXIT_READ_DR3                   , 35, "Read DR3."),
+    EXIT_REASON(SVM_EXIT_READ_DR4                   , 36, "Read DR4."),
+    EXIT_REASON(SVM_EXIT_READ_DR5                   , 37, "Read DR5."),
+    EXIT_REASON(SVM_EXIT_READ_DR6                   , 38, "Read DR6."),
+    EXIT_REASON(SVM_EXIT_READ_DR7                   , 39, "Read DR7."),
+    EXIT_REASON(SVM_EXIT_READ_DR8                   , 40, "Read DR8."),
+    EXIT_REASON(SVM_EXIT_READ_DR9                   , 41, "Read DR9."),
+    EXIT_REASON(SVM_EXIT_READ_DR10                  , 42, "Read DR10."),
+    EXIT_REASON(SVM_EXIT_READ_DR11                  , 43, "Read DR11"),
+    EXIT_REASON(SVM_EXIT_READ_DR12                  , 44, "Read DR12."),
+    EXIT_REASON(SVM_EXIT_READ_DR13                  , 45, "Read DR13."),
+    EXIT_REASON(SVM_EXIT_READ_DR14                  , 46, "Read DR14."),
+    EXIT_REASON(SVM_EXIT_READ_DR15                  , 47, "Read DR15."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR0                  , 48, "Write DR0."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR1                  , 49, "Write DR1."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR2                  , 50, "Write DR2."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR3                  , 51, "Write DR3."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR4                  , 52, "Write DR4."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR5                  , 53, "Write DR5."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR6                  , 54, "Write DR6."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR7                  , 55, "Write DR7."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR8                  , 56, "Write DR8."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR9                  , 57, "Write DR9."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR10                 , 58, "Write DR10."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR11                 , 59, "Write DR11."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR12                 , 60, "Write DR12."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR13                 , 61, "Write DR13."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR14                 , 62, "Write DR14."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR15                 , 63, "Write DR15."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_0                , 64, "Exception Vector 0  (0x0)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_1                , 65, "Exception Vector 1  (0x1)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_2                , 66, "Exception Vector 2  (0x2)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_3                , 67, "Exception Vector 3  (0x3)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_4                , 68, "Exception Vector 4  (0x4)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_5                , 69, "Exception Vector 5  (0x5)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_6                , 70, "Exception Vector 6  (0x6)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_7                , 71, "Exception Vector 7  (0x7)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_8                , 72, "Exception Vector 8  (0x8)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_9                , 73, "Exception Vector 9  (0x9)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_A                , 74, "Exception Vector 10 (0xA)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_B                , 75, "Exception Vector 11 (0xB)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_C                , 76, "Exception Vector 12 (0xC)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_D                , 77, "Exception Vector 13 (0xD)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_E                , 78, "Exception Vector 14 (0xE)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_F                , 79, "Exception Vector 15 (0xF)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_10               , 80, "Exception Vector 16 (0x10)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_11               , 81, "Exception Vector 17 (0x11)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_12               , 82, "Exception Vector 18 (0x12)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_13               , 83, "Exception Vector 19 (0x13)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_14               , 84, "Exception Vector 20 (0x14)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_15               , 85, "Exception Vector 22 (0x15)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_16               , 86, "Exception Vector 22 (0x16)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_17               , 87, "Exception Vector 23 (0x17)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_18               , 88, "Exception Vector 24 (0x18)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_19               , 89, "Exception Vector 25 (0x19)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_1A               , 90, "Exception Vector 26 (0x1A)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_1B               , 91, "Exception Vector 27 (0x1B)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_1C               , 92, "Exception Vector 28 (0x1C)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_1D               , 93, "Exception Vector 29 (0x1D)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_1E               , 94, "Exception Vector 30 (0x1E)."),
+    EXIT_REASON(SVM_EXIT_EXCEPTION_1F               , 95, "Exception Vector 31 (0x1F)."),
+    EXIT_REASON(SVM_EXIT_INTR                       , 96, "Physical maskable interrupt."),
+    EXIT_REASON(SVM_EXIT_NMI                        , 97, "Physical non-maskable interrupt."),
+    EXIT_REASON(SVM_EXIT_SMI                        , 98, "System management interrupt."),
+    EXIT_REASON(SVM_EXIT_INIT                       , 99, "Physical INIT signal."),
+    EXIT_REASON(SVM_EXIT_VINTR                      ,100, "Virtual interrupt."),
+    EXIT_REASON(SVM_EXIT_CR0_SEL_WRITE              ,101, "Write to CR0 that changed any bits other than CR0.TS or CR0.MP."),
+    EXIT_REASON(SVM_EXIT_IDTR_READ                  ,102, "Read IDTR"),
+    EXIT_REASON(SVM_EXIT_GDTR_READ                  ,103, "Read GDTR"),
+    EXIT_REASON(SVM_EXIT_LDTR_READ                  ,104, "Read LDTR."),
+    EXIT_REASON(SVM_EXIT_TR_READ                    ,105, "Read TR."),
+    EXIT_REASON(SVM_EXIT_TR_READ                    ,106, "Write IDTR."),
+    EXIT_REASON(SVM_EXIT_TR_READ                    ,107, "Write GDTR."),
+    EXIT_REASON(SVM_EXIT_TR_READ                    ,108, "Write LDTR."),
+    EXIT_REASON(SVM_EXIT_TR_READ                    ,109, "Write TR."),
+    EXIT_REASON(SVM_EXIT_RDTSC                      ,110, "RDTSC instruction."),
+    EXIT_REASON(SVM_EXIT_RDPMC                      ,111, "RDPMC instruction."),
+    EXIT_REASON(SVM_EXIT_PUSHF                      ,112, "PUSHF instruction."),
+    EXIT_REASON(SVM_EXIT_POPF                       ,113, "POPF instruction."),
+    EXIT_REASON(SVM_EXIT_CPUID                      ,114, "CPUID instruction."),
+    EXIT_REASON(SVM_EXIT_RSM                        ,115, "RSM instruction."),
+    EXIT_REASON(SVM_EXIT_IRET                       ,116, "IRET instruction."),
+    EXIT_REASON(SVM_EXIT_SWINT                      ,117, "Software interrupt (INTn instructions)."),
+    EXIT_REASON(SVM_EXIT_INVD                       ,118, "INVD instruction."),
+    EXIT_REASON(SVM_EXIT_PAUSE                      ,119, "PAUSE instruction."),
+    EXIT_REASON(SVM_EXIT_HLT                        ,120, "HLT instruction."),
+    EXIT_REASON(SVM_EXIT_INVLPG                     ,121, "INVLPG instruction."),
+    EXIT_REASON(SVM_EXIT_INVLPGA                    ,122, "INVLPGA instruction."),
+    EXIT_REASON(SVM_EXIT_IOIO                       ,123, "IN/OUT accessing protected port (EXITINFO1 field provides more information)."),
+    EXIT_REASON(SVM_EXIT_MSR                        ,124, "RDMSR or WRMSR access to protected MSR."),
+    EXIT_REASON(SVM_EXIT_TASK_SWITCH                ,125, "Task switch."),
+    EXIT_REASON(SVM_EXIT_FERR_FREEZE                ,126, "FP legacy handling enabled, and processor is frozen in an x87/mmx instruction waiting for an interrupt"),
+    EXIT_REASON(SVM_EXIT_SHUTDOWN                   ,127, "Shutdown."),
+    EXIT_REASON(SVM_EXIT_VMRUN                      ,128, "VMRUN instruction."),
+    EXIT_REASON(SVM_EXIT_VMMCALL                    ,129, "VMCALL instruction."),
+    EXIT_REASON(SVM_EXIT_VMLOAD                     ,130, "VMLOAD instruction."),
+    EXIT_REASON(SVM_EXIT_VMSAVE                     ,131, "VMSAVE instruction."),
+    EXIT_REASON(SVM_EXIT_STGI                       ,132, "STGI instruction."),
+    EXIT_REASON(SVM_EXIT_CLGI                       ,133, "CLGI instruction."),
+    EXIT_REASON(SVM_EXIT_SKINIT                     ,134, "SKINIT instruction."),
+    EXIT_REASON(SVM_EXIT_RDTSCP                     ,135, "RDTSCP instruction."),
+    EXIT_REASON(SVM_EXIT_ICEBP                      ,136, "ICEBP instruction."),
+    EXIT_REASON(SVM_EXIT_WBINVD                     ,137, "WBINVD instruction."),
+    EXIT_REASON(SVM_EXIT_MONITOR                    ,138, "MONITOR instruction."),
+    EXIT_REASON(SVM_EXIT_MWAIT_UNCOND               ,139, "MWAIT instruction unconditional."),
+    EXIT_REASON(SVM_EXIT_MWAIT_ARMED                ,140, "MWAIT instruction when armed."),
+    EXIT_REASON(SVM_EXIT_NPF                        ,1024, "Nested paging: host-level page fault occurred (EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault)."),
+    EXIT_REASON_NIL()
+};
+# undef EXIT_REASON
+# undef EXIT_REASON_NIL
+#endif /* VBOX_WITH_STATISTICS */
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) hwaccmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) hwaccmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static int hwaccmR3InitCPU(PVM pVM);
+static int hwaccmR3TermCPU(PVM pVM);
+
+
+/**
+ * Initializes the HWACCM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) HWACCMR3Init(PVM pVM)
+{
+    LogFlow(("HWACCMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, hwaccm.s, 32);
+    AssertCompile(sizeof(pVM->hwaccm.s) <= sizeof(pVM->hwaccm.padding));
+
+    /* Some structure checks. */
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, ctrl.EventInject) == 0xA8, ("ctrl.EventInject offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.EventInject)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, ctrl.ExitIntInfo) == 0x88, ("ctrl.ExitIntInfo offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.ExitIntInfo)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, ctrl.TLBCtrl) == 0x58, ("ctrl.TLBCtrl offset = %x\n", RT_OFFSETOF(SVM_VMCB, ctrl.TLBCtrl)));
+
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest) == 0x400, ("guest offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.TR) == 0x490, ("guest.TR offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.TR)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u8CPL) == 0x4CB, ("guest.u8CPL offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u8CPL)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64EFER) == 0x4D0, ("guest.u64EFER offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64EFER)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64CR4) == 0x548, ("guest.u64CR4 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64CR4)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64RIP) == 0x578, ("guest.u64RIP offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64RIP)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64RSP) == 0x5D8, ("guest.u64RSP offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64RSP)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64CR2) == 0x640, ("guest.u64CR2 offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64CR2)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64GPAT) == 0x668, ("guest.u64GPAT offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64GPAT)));
+    AssertReleaseMsg(RT_OFFSETOF(SVM_VMCB, guest.u64LASTEXCPTO) == 0x690, ("guest.u64LASTEXCPTO offset = %x\n", RT_OFFSETOF(SVM_VMCB, guest.u64LASTEXCPTO)));
+    AssertReleaseMsg(sizeof(SVM_VMCB) == 0x1000, ("SVM_VMCB size = %x\n", sizeof(SVM_VMCB)));
+
+
+    /*
+     * Register the saved state data unit.
+     */
+    int rc = SSMR3RegisterInternal(pVM, "HWACCM", 0, HWACCM_SSM_VERSION, sizeof(HWACCM),
+                                   NULL, NULL, NULL,
+                                   NULL, hwaccmR3Save, NULL,
+                                   NULL, hwaccmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* Misc initialisation. */
+    pVM->hwaccm.s.vmx.fSupported = false;
+    pVM->hwaccm.s.svm.fSupported = false;
+    pVM->hwaccm.s.vmx.fEnabled   = false;
+    pVM->hwaccm.s.svm.fEnabled   = false;
+
+    pVM->hwaccm.s.fNestedPaging  = false;
+    pVM->hwaccm.s.fLargePages    = false;
+
+    /* Disabled by default. */
+    pVM->fHWACCMEnabled = false;
+
+    /*
+     * Check CFGM options.
+     */
+    PCFGMNODE pRoot      = CFGMR3GetRoot(pVM);
+    PCFGMNODE pHWVirtExt = CFGMR3GetChild(pRoot, "HWVirtExt/");
+    /* Nested paging: disabled by default. */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "EnableNestedPaging", &pVM->hwaccm.s.fAllowNestedPaging, false);
+    AssertRC(rc);
+
+    /* Large pages: disabled by default. */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "EnableLargePages", &pVM->hwaccm.s.fLargePages, false);
+    AssertRC(rc);
+
+    /* VT-x VPID: disabled by default. */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "EnableVPID", &pVM->hwaccm.s.vmx.fAllowVPID, false);
+    AssertRC(rc);
+
+    /* HWACCM support must be explicitely enabled in the configuration file. */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "Enabled", &pVM->hwaccm.s.fAllowed, false);
+    AssertRC(rc);
+
+    /* TPR patching for 32 bits (Windows) guests with IO-APIC: disabled by default. */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "TPRPatchingEnabled", &pVM->hwaccm.s.fTRPPatchingAllowed, false);
+    AssertRC(rc);
+
+#ifdef RT_OS_DARWIN
+    if (VMMIsHwVirtExtForced(pVM) != pVM->hwaccm.s.fAllowed)
+#else
+    if (VMMIsHwVirtExtForced(pVM) && !pVM->hwaccm.s.fAllowed)
+#endif
+    {
+        AssertLogRelMsgFailed(("VMMIsHwVirtExtForced=%RTbool fAllowed=%RTbool\n",
+                               VMMIsHwVirtExtForced(pVM), pVM->hwaccm.s.fAllowed));
+        return VERR_HWACCM_CONFIG_MISMATCH;
+    }
+
+    if (VMMIsHwVirtExtForced(pVM))
+        pVM->fHWACCMEnabled = true;
+
+#if HC_ARCH_BITS == 32
+    /* 64-bit mode is configurable and it depends on both the kernel mode and VT-x.
+     * (To use the default, don't set 64bitEnabled in CFGM.) */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "64bitEnabled", &pVM->hwaccm.s.fAllow64BitGuests, false);
+    AssertLogRelRCReturn(rc, rc);
+    if (pVM->hwaccm.s.fAllow64BitGuests)
+    {
+# ifdef RT_OS_DARWIN
+        if (!VMMIsHwVirtExtForced(pVM))
+# else
+        if (!pVM->hwaccm.s.fAllowed)
+# endif
+            return VM_SET_ERROR(pVM, VERR_INVALID_PARAMETER, "64-bit guest support was requested without also enabling HWVirtEx (VT-x/AMD-V).");
+    }
+#else
+    /* On 64-bit hosts 64-bit guest support is enabled by default, but allow this to be overridden
+     * via VBoxInternal/HWVirtExt/64bitEnabled=0. (ConsoleImpl2.cpp doesn't set this to false for 64-bit.) */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "64bitEnabled", &pVM->hwaccm.s.fAllow64BitGuests, true);
+    AssertLogRelRCReturn(rc, rc);
+#endif
+
+
+    /** Determine the init method for AMD-V and VT-x; either one global init for each host CPU
+     *  or local init each time we wish to execute guest code.
+     *
+     *  Default false for Mac OS X and Windows due to the higher risk of conflicts with other hypervisors.
+     */
+    rc = CFGMR3QueryBoolDef(pHWVirtExt, "Exclusive", &pVM->hwaccm.s.fGlobalInit,
+#if defined(RT_OS_DARWIN) || defined(RT_OS_WINDOWS)
+                            false
+#else
+                            true
+#endif
+                           );
+
+    /* Max number of resume loops. */
+    rc = CFGMR3QueryU32Def(pHWVirtExt, "MaxResumeLoops", &pVM->hwaccm.s.cMaxResumeLoops, 0 /* set by R0 later */);
+    AssertRC(rc);
+
+    return rc;
+}
+
+/**
+ * Initializes the per-VCPU HWACCM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+static int hwaccmR3InitCPU(PVM pVM)
+{
+    LogFlow(("HWACCMR3InitCPU\n"));
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        pVCpu->hwaccm.s.fActive = false;
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRPatchSuccess,   STAMTYPE_COUNTER, "/HWACCM/TPR/Patch/Success",  STAMUNIT_OCCURENCES,    "Number of times an instruction was successfully patched.");
+    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRPatchFailure,   STAMTYPE_COUNTER, "/HWACCM/TPR/Patch/Failed",   STAMUNIT_OCCURENCES,    "Number of unsuccessful patch attempts.");
+    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRReplaceSuccess, STAMTYPE_COUNTER, "/HWACCM/TPR/Replace/Success",STAMUNIT_OCCURENCES,    "Number of times an instruction was successfully patched.");
+    STAM_REG(pVM, &pVM->hwaccm.s.StatTPRReplaceFailure, STAMTYPE_COUNTER, "/HWACCM/TPR/Replace/Failed", STAMUNIT_OCCURENCES,    "Number of unsuccessful patch attempts.");
+
+    /*
+     * Statistics.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+        int    rc;
+
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatPoke, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of RTMpPokeCpu",
+                             "/PROF/HWACCM/CPU%d/Poke", i);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatSpinPoke, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of poke wait",
+                             "/PROF/HWACCM/CPU%d/PokeWait", i);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatSpinPokeFailed, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of poke wait when RTMpPokeCpu fails",
+                             "/PROF/HWACCM/CPU%d/PokeWaitFailed", i);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatEntry, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode entry",
+                             "/PROF/HWACCM/CPU%d/SwitchToGC", i);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit1, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode exit part 1",
+                             "/PROF/HWACCM/CPU%d/SwitchFromGC_1", i);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of VMXR0RunGuestCode exit part 2",
+                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2", i);
+        AssertRC(rc);
+# if 1 /* temporary for tracking down darwin holdup. */
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2Sub1, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Temporary - I/O",
+                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2/Sub1", i);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2Sub2, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Temporary - CRx RWs",
+                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2/Sub2", i);
+        AssertRC(rc);
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExit2Sub3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Temporary - Exceptions",
+                             "/PROF/HWACCM/CPU%d/SwitchFromGC_2/Sub3", i);
+        AssertRC(rc);
+# endif
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatInGC, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of vmlaunch",
+                             "/PROF/HWACCM/CPU%d/InGC", i);
+        AssertRC(rc);
+
+# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+        rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatWorldSwitch3264, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, "Profiling of the 32/64 switcher",
+                             "/PROF/HWACCM/CPU%d/Switcher3264", i);
+        AssertRC(rc);
+# endif
+
+# define HWACCM_REG_COUNTER(a, b) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Profiling of vmlaunch", b, i); \
+        AssertRC(rc);
+
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitShadowNM,           "/HWACCM/CPU%d/Exit/Trap/Shw/#NM");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestNM,            "/HWACCM/CPU%d/Exit/Trap/Gst/#NM");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitShadowPF,           "/HWACCM/CPU%d/Exit/Trap/Shw/#PF");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestPF,            "/HWACCM/CPU%d/Exit/Trap/Gst/#PF");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestUD,            "/HWACCM/CPU%d/Exit/Trap/Gst/#UD");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestSS,            "/HWACCM/CPU%d/Exit/Trap/Gst/#SS");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestNP,            "/HWACCM/CPU%d/Exit/Trap/Gst/#NP");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestGP,            "/HWACCM/CPU%d/Exit/Trap/Gst/#GP");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestMF,            "/HWACCM/CPU%d/Exit/Trap/Gst/#MF");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestDE,            "/HWACCM/CPU%d/Exit/Trap/Gst/#DE");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitGuestDB,            "/HWACCM/CPU%d/Exit/Trap/Gst/#DB");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitInvpg,              "/HWACCM/CPU%d/Exit/Instr/Invlpg");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitInvd,               "/HWACCM/CPU%d/Exit/Instr/Invd");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitCpuid,              "/HWACCM/CPU%d/Exit/Instr/Cpuid");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitRdtsc,              "/HWACCM/CPU%d/Exit/Instr/Rdtsc");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitRdpmc,              "/HWACCM/CPU%d/Exit/Instr/Rdpmc");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitRdmsr,              "/HWACCM/CPU%d/Exit/Instr/Rdmsr");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitWrmsr,              "/HWACCM/CPU%d/Exit/Instr/Wrmsr");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitMwait,              "/HWACCM/CPU%d/Exit/Instr/Mwait");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitMonitor,            "/HWACCM/CPU%d/Exit/Instr/Monitor");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitDRxWrite,           "/HWACCM/CPU%d/Exit/Instr/DR/Write");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitDRxRead,            "/HWACCM/CPU%d/Exit/Instr/DR/Read");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitCLTS,               "/HWACCM/CPU%d/Exit/Instr/CLTS");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitLMSW,               "/HWACCM/CPU%d/Exit/Instr/LMSW");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitCli,                "/HWACCM/CPU%d/Exit/Instr/Cli");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitSti,                "/HWACCM/CPU%d/Exit/Instr/Sti");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitPushf,              "/HWACCM/CPU%d/Exit/Instr/Pushf");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitPopf,               "/HWACCM/CPU%d/Exit/Instr/Popf");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIret,               "/HWACCM/CPU%d/Exit/Instr/Iret");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitInt,                "/HWACCM/CPU%d/Exit/Instr/Int");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitHlt,                "/HWACCM/CPU%d/Exit/Instr/Hlt");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIOWrite,            "/HWACCM/CPU%d/Exit/IO/Write");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIORead,             "/HWACCM/CPU%d/Exit/IO/Read");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIOStringWrite,      "/HWACCM/CPU%d/Exit/IO/WriteString");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIOStringRead,       "/HWACCM/CPU%d/Exit/IO/ReadString");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitIrqWindow,          "/HWACCM/CPU%d/Exit/IrqWindow");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitMaxResume,          "/HWACCM/CPU%d/Exit/MaxResume");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatExitPreemptPending,     "/HWACCM/CPU%d/Exit/PreemptPending");
+
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatSwitchGuestIrq,         "/HWACCM/CPU%d/Switch/IrqPending");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatSwitchToR3,             "/HWACCM/CPU%d/Switch/ToR3");
+
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatIntInject,              "/HWACCM/CPU%d/Irq/Inject");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatIntReinject,            "/HWACCM/CPU%d/Irq/Reinject");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatPendingHostIrq,         "/HWACCM/CPU%d/Irq/PendingOnHost");
+
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPage,              "/HWACCM/CPU%d/Flush/Page");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPageManual,        "/HWACCM/CPU%d/Flush/Page/Virt");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPhysPageManual,    "/HWACCM/CPU%d/Flush/Page/Phys");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLB,               "/HWACCM/CPU%d/Flush/TLB");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBManual,         "/HWACCM/CPU%d/Flush/TLB/Manual");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBCRxChange,      "/HWACCM/CPU%d/Flush/TLB/CRx");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushPageInvlpg,        "/HWACCM/CPU%d/Flush/Page/Invlpg");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch,    "/HWACCM/CPU%d/Flush/TLB/Switch");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch,  "/HWACCM/CPU%d/Flush/TLB/Skipped");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushASID,              "/HWACCM/CPU%d/Flush/TLB/ASID");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFlushTLBInvlpga,        "/HWACCM/CPU%d/Flush/TLB/PhysInvl");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTlbShootdown,           "/HWACCM/CPU%d/Flush/Shootdown/Page");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTlbShootdownFlush,      "/HWACCM/CPU%d/Flush/Shootdown/TLB");
+
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTSCOffset,              "/HWACCM/CPU%d/TSC/Offset");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTSCIntercept,           "/HWACCM/CPU%d/TSC/Intercept");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow,   "/HWACCM/CPU%d/TSC/InterceptOverflow");
+
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDRxArmed,               "/HWACCM/CPU%d/Debug/Armed");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDRxContextSwitch,       "/HWACCM/CPU%d/Debug/ContextSwitch");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDRxIOCheck,             "/HWACCM/CPU%d/Debug/IOCheck");
+
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatLoadMinimal,            "/HWACCM/CPU%d/Load/Minimal");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatLoadFull,               "/HWACCM/CPU%d/Load/Full");
+
+#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatFpu64SwitchBack,        "/HWACCM/CPU%d/Switch64/Fpu");
+        HWACCM_REG_COUNTER(&pVCpu->hwaccm.s.StatDebug64SwitchBack,      "/HWACCM/CPU%d/Switch64/Debug");
+#endif
+
+        for (unsigned j=0;j<RT_ELEMENTS(pVCpu->hwaccm.s.StatExitCRxWrite);j++)
+        {
+            rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExitCRxWrite[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Profiling of CRx writes",
+                                "/HWACCM/CPU%d/Exit/Instr/CR/Write/%x", i, j);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExitCRxRead[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Profiling of CRx reads",
+                                "/HWACCM/CPU%d/Exit/Instr/CR/Read/%x", i, j);
+            AssertRC(rc);
+        }
+
+#undef HWACCM_REG_COUNTER
+
+        pVCpu->hwaccm.s.paStatExitReason = NULL;
+
+        rc = MMHyperAlloc(pVM, MAX_EXITREASON_STAT*sizeof(*pVCpu->hwaccm.s.paStatExitReason), 0, MM_TAG_HWACCM, (void **)&pVCpu->hwaccm.s.paStatExitReason);
+        AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+            const char * const *papszDesc = ASMIsIntelCpu() ? &g_apszVTxExitReasons[0] : &g_apszAmdVExitReasons[0];
+            for (int j=0;j<MAX_EXITREASON_STAT;j++)
+            {
+                if (papszDesc[j])
+                {
+                    rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.paStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                                        papszDesc[j], "/HWACCM/CPU%d/Exit/Reason/%02x", i, j);
+                    AssertRC(rc);
+                }
+            }
+            rc = STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.StatExitReasonNPF, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Nested page fault", "/HWACCM/CPU%d/Exit/Reason/#NPF", i);
+            AssertRC(rc);
+        }
+        pVCpu->hwaccm.s.paStatExitReasonR0 = MMHyperR3ToR0(pVM, pVCpu->hwaccm.s.paStatExitReason);
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        Assert(pVCpu->hwaccm.s.paStatExitReasonR0 != NIL_RTR0PTR || !VMMIsHwVirtExtForced(pVM));
+# else
+        Assert(pVCpu->hwaccm.s.paStatExitReasonR0 != NIL_RTR0PTR);
+# endif
+
+        rc = MMHyperAlloc(pVM, sizeof(STAMCOUNTER) * 256, 8, MM_TAG_HWACCM, (void **)&pVCpu->hwaccm.s.paStatInjectedIrqs);
+        AssertRCReturn(rc, rc);
+        pVCpu->hwaccm.s.paStatInjectedIrqsR0 = MMHyperR3ToR0(pVM, pVCpu->hwaccm.s.paStatInjectedIrqs);
+# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        Assert(pVCpu->hwaccm.s.paStatInjectedIrqsR0 != NIL_RTR0PTR || !VMMIsHwVirtExtForced(pVM));
+# else
+        Assert(pVCpu->hwaccm.s.paStatInjectedIrqsR0 != NIL_RTR0PTR);
+# endif
+        for (unsigned j = 0; j < 255; j++)
+            STAMR3RegisterF(pVM, &pVCpu->hwaccm.s.paStatInjectedIrqs[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Forwarded interrupts.",
+                            (j < 0x20) ? "/HWACCM/CPU%d/Interrupt/Trap/%02X" : "/HWACCM/CPU%d/Interrupt/IRQ/%02X", i, j);
+
+    }
+#endif /* VBOX_WITH_STATISTICS */
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    /* Magic marker for searching in crash dumps. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
+        strcpy((char *)pCache->aMagic, "VMCSCACHE Magic");
+        pCache->uMagic = UINT64_C(0xDEADBEEFDEADBEEF);
+    }
+#endif
+    return VINF_SUCCESS;
+}
+
+/**
+ * Called when a init phase has completed.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM.
+ * @param   enmWhat             The phase that completed.
+ */
+VMMR3_INT_DECL(int) HWACCMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    switch (enmWhat)
+    {
+    case VMINITCOMPLETED_RING3:
+        return hwaccmR3InitCPU(pVM);
+    default:
+        return VINF_SUCCESS;
+    }
+}
+
+/**
+ * Turns off normal raw mode features
+ *
+ * @param   pVM         The VM to operate on.
+ */
+static void hwaccmR3DisableRawMode(PVM pVM)
+{
+    /* Disable PATM & CSAM. */
+    PATMR3AllowPatching(pVM, false);
+    CSAMDisableScanning(pVM);
+
+    /* Turn off IDT/LDT/GDT and TSS monitoring and sycing. */
+    SELMR3DisableMonitoring(pVM);
+    TRPMR3DisableMonitoring(pVM);
+
+    /* Disable the switcher code (safety precaution). */
+    VMMR3DisableSwitcher(pVM);
+
+    /* Disable mapping of the hypervisor into the shadow page table. */
+    PGMR3MappingsDisable(pVM);
+
+    /* Disable the switcher */
+    VMMR3DisableSwitcher(pVM);
+
+    /* Reinit the paging mode to force the new shadow mode. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
+    }
+}
+
+/**
+ * Initialize VT-x or AMD-V.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(int) HWACCMR3InitFinalizeR0(PVM pVM)
+{
+    int rc;
+
+    /* Hack to allow users to work around broken BIOSes that incorrectly set EFER.SVME, which makes us believe somebody else
+     * is already using AMD-V.
+     */
+    if (    !pVM->hwaccm.s.vmx.fSupported
+        &&  !pVM->hwaccm.s.svm.fSupported
+        &&  pVM->hwaccm.s.lLastError == VERR_SVM_IN_USE /* implies functional AMD-V */
+        &&  RTEnvExist("VBOX_HWVIRTEX_IGNORE_SVM_IN_USE"))
+    {
+        LogRel(("HWACCM: VBOX_HWVIRTEX_IGNORE_SVM_IN_USE active!\n"));
+        pVM->hwaccm.s.svm.fSupported        = true;
+        pVM->hwaccm.s.svm.fIgnoreInUseError = true;
+    }
+    else
+    if (    !pVM->hwaccm.s.vmx.fSupported
+        &&  !pVM->hwaccm.s.svm.fSupported)
+    {
+        LogRel(("HWACCM: No VT-x or AMD-V CPU extension found. Reason %Rrc\n", pVM->hwaccm.s.lLastError));
+        LogRel(("HWACCM: VMX MSR_IA32_FEATURE_CONTROL=%RX64\n", pVM->hwaccm.s.vmx.msr.feature_ctrl));
+
+        if (VMMIsHwVirtExtForced(pVM))
+        {
+            switch (pVM->hwaccm.s.lLastError)
+            {
+            case VERR_VMX_NO_VMX:
+                return VM_SET_ERROR(pVM, VERR_VMX_NO_VMX, "VT-x is not available.");
+            case VERR_VMX_IN_VMX_ROOT_MODE:
+                return VM_SET_ERROR(pVM, VERR_VMX_IN_VMX_ROOT_MODE, "VT-x is being used by another hypervisor.");
+            case VERR_SVM_IN_USE:
+                return VM_SET_ERROR(pVM, VERR_SVM_IN_USE, "AMD-V is being used by another hypervisor.");
+            case VERR_SVM_NO_SVM:
+                return VM_SET_ERROR(pVM, VERR_SVM_NO_SVM, "AMD-V is not available.");
+            case VERR_SVM_DISABLED:
+                return VM_SET_ERROR(pVM, VERR_SVM_DISABLED, "AMD-V is disabled in the BIOS.");
+            default:
+                return pVM->hwaccm.s.lLastError;
+            }
+        }
+        return VINF_SUCCESS;
+    }
+
+    if (pVM->hwaccm.s.vmx.fSupported)
+    {
+        rc = SUPR3QueryVTxSupported();
+        if (RT_FAILURE(rc))
+        {
+#ifdef RT_OS_LINUX
+            LogRel(("HWACCM: The host kernel does not support VT-x -- Linux 2.6.13 or newer required!\n"));
+#else
+            LogRel(("HWACCM: The host kernel does not support VT-x!\n"));
+#endif
+            if (   pVM->cCpus > 1
+                || VMMIsHwVirtExtForced(pVM))
+                return rc;
+
+            /* silently fall back to raw mode */
+            return VINF_SUCCESS;
+        }
+    }
+
+    if (!pVM->hwaccm.s.fAllowed)
+        return VINF_SUCCESS;    /* nothing to do */
+
+    /* Enable VT-x or AMD-V on all host CPUs. */
+    rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_HWACC_ENABLE, 0, NULL);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("HWACCMR3InitFinalize: SUPR3CallVMMR0Ex VMMR0_DO_HWACC_ENABLE failed with %Rrc\n", rc));
+        return rc;
+    }
+    Assert(!pVM->fHWACCMEnabled || VMMIsHwVirtExtForced(pVM));
+
+    pVM->hwaccm.s.fHasIoApic = PDMHasIoApic(pVM);
+    /* No TPR patching is required when the IO-APIC is not enabled for this VM. (Main should have taken care of this already) */
+    if (!pVM->hwaccm.s.fHasIoApic)
+    {
+        Assert(!pVM->hwaccm.s.fTRPPatchingAllowed); /* paranoia */
+        pVM->hwaccm.s.fTRPPatchingAllowed = false;
+    }
+
+    bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+    if (pVM->hwaccm.s.vmx.fSupported)
+    {
+        Log(("pVM->hwaccm.s.vmx.fSupported = %d\n", pVM->hwaccm.s.vmx.fSupported));
+
+        if (    pVM->hwaccm.s.fInitialized == false
+            &&  pVM->hwaccm.s.vmx.msr.feature_ctrl != 0)
+        {
+            uint64_t val;
+            RTGCPHYS GCPhys = 0;
+
+            LogRel(("HWACCM: Host CR4=%08X\n", pVM->hwaccm.s.vmx.hostCR4));
+            LogRel(("HWACCM: MSR_IA32_FEATURE_CONTROL      = %RX64\n", pVM->hwaccm.s.vmx.msr.feature_ctrl));
+            LogRel(("HWACCM: MSR_IA32_VMX_BASIC_INFO       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_basic_info));
+            LogRel(("HWACCM: VMCS id                       = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
+            LogRel(("HWACCM: VMCS size                     = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
+            LogRel(("HWACCM: VMCS physical address limit   = %s\n", MSR_IA32_VMX_BASIC_INFO_VMCS_PHYS_WIDTH(pVM->hwaccm.s.vmx.msr.vmx_basic_info) ? "< 4 GB" : "None"));
+            LogRel(("HWACCM: VMCS memory type              = %x\n", MSR_IA32_VMX_BASIC_INFO_VMCS_MEM_TYPE(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
+            LogRel(("HWACCM: Dual monitor treatment        = %d\n", MSR_IA32_VMX_BASIC_INFO_VMCS_DUAL_MON(pVM->hwaccm.s.vmx.msr.vmx_basic_info)));
+
+            LogRel(("HWACCM: MSR_IA32_VMX_PINBASED_CTLS    = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.u));
+            val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI\n"));
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER\n"));
+            val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER *must* be set\n"));
+
+            LogRel(("HWACCM: MSR_IA32_VMX_PROCBASED_CTLS   = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.u));
+            val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL\n"));
+
+            val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_NMI_WINDOW_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_IO_BITMAPS *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL *must* be set\n"));
+
+            if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
+            {
+                LogRel(("HWACCM: MSR_IA32_VMX_PROCBASED_CTLS2  = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.u));
+                val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_EPT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_EPT\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_X2APIC)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_X2APIC\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VPID)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VPID\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT\n"));
+
+                val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_INSTR_EXIT *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP_EXIT *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_X2APIC)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_X2APIC *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_EPT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_EPT *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_VPID)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_VPID *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE *must* be set\n"));
+                if (val & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT)
+                    LogRel(("HWACCM:    VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT *must* be set\n"));
+            }
+
+            LogRel(("HWACCM: MSR_IA32_VMX_ENTRY_CTLS       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_entry.u));
+            val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR\n"));
+            val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR *must* be set\n"));
+
+            LogRel(("HWACCM: MSR_IA32_VMX_EXIT_CTLS        = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_exit.u));
+            val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER\n"));
+            val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64 *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXTERNAL_IRQ *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR *must* be set\n"));
+            if (val & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
+                LogRel(("HWACCM:    VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER *must* be set\n"));
+
+            if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps)
+            {
+                LogRel(("HWACCM: MSR_IA32_VMX_EPT_VPID_CAPS    = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_eptcaps));
+
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_RWX_X_ONLY)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_RWX_X_ONLY\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_RWX_W_ONLY)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_RWX_W_ONLY\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_RWX_WX_ONLY)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_RWX_WX_ONLY\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_21_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_21_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_30_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_30_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_39_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_39_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_48_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_48_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_GAW_57_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_GAW_57_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_UC)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_UC\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WC)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WC\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WT)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WT\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WP)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WP\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_EMT_WB)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_EMT_WB\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_21_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_21_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_30_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_30_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_39_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_39_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_SP_48_BITS)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_SP_48_BITS\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL\n"));
+                if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT_GLOBAL)
+                    LogRel(("HWACCM:    MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT_GLOBAL\n"));
+            }
+
+            LogRel(("HWACCM: MSR_IA32_VMX_MISC             = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_misc));
+            if (MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(pVM->hwaccm.s.vmx.msr.vmx_misc) == pVM->hwaccm.s.vmx.cPreemptTimerShift)
+                LogRel(("HWACCM:    MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT %x\n", MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(pVM->hwaccm.s.vmx.msr.vmx_misc)));
+            else
+                LogRel(("HWACCM:    MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT %x - erratum detected, using %x instead\n", MSR_IA32_VMX_MISC_PREEMPT_TSC_BIT(pVM->hwaccm.s.vmx.msr.vmx_misc), pVM->hwaccm.s.vmx.cPreemptTimerShift));
+            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_ACTIVITY_STATES %x\n", MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hwaccm.s.vmx.msr.vmx_misc)));
+            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_CR3_TARGET      %x\n", MSR_IA32_VMX_MISC_CR3_TARGET(pVM->hwaccm.s.vmx.msr.vmx_misc)));
+            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_MAX_MSR         %x\n", MSR_IA32_VMX_MISC_MAX_MSR(pVM->hwaccm.s.vmx.msr.vmx_misc)));
+            LogRel(("HWACCM:    MSR_IA32_VMX_MISC_MSEG_ID         %x\n", MSR_IA32_VMX_MISC_MSEG_ID(pVM->hwaccm.s.vmx.msr.vmx_misc)));
+
+            LogRel(("HWACCM: MSR_IA32_VMX_CR0_FIXED0       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0));
+            LogRel(("HWACCM: MSR_IA32_VMX_CR0_FIXED1       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1));
+            LogRel(("HWACCM: MSR_IA32_VMX_CR4_FIXED0       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0));
+            LogRel(("HWACCM: MSR_IA32_VMX_CR4_FIXED1       = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1));
+            LogRel(("HWACCM: MSR_IA32_VMX_VMCS_ENUM        = %RX64\n", pVM->hwaccm.s.vmx.msr.vmx_vmcs_enum));
+
+            LogRel(("HWACCM: TPR shadow physaddr           = %RHp\n", pVM->hwaccm.s.vmx.pAPICPhys));
+
+            /* Paranoia */
+            AssertRelease(MSR_IA32_VMX_MISC_MAX_MSR(pVM->hwaccm.s.vmx.msr.vmx_misc) >= 512);
+
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                LogRel(("HWACCM: VCPU%d: MSR bitmap physaddr      = %RHp\n", i, pVM->aCpus[i].hwaccm.s.vmx.pMSRBitmapPhys));
+                LogRel(("HWACCM: VCPU%d: VMCS physaddr            = %RHp\n", i, pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys));
+            }
+
+#ifdef HWACCM_VTX_WITH_EPT
+            if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_EPT)
+                pVM->hwaccm.s.fNestedPaging = pVM->hwaccm.s.fAllowNestedPaging;
+#endif /* HWACCM_VTX_WITH_EPT */
+#ifdef HWACCM_VTX_WITH_VPID
+            if (    (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VPID)
+                &&  !pVM->hwaccm.s.fNestedPaging)    /* VPID and EPT are mutually exclusive. */
+                pVM->hwaccm.s.vmx.fVPID = pVM->hwaccm.s.vmx.fAllowVPID;
+#endif /* HWACCM_VTX_WITH_VPID */
+
+            /* Unrestricted guest execution relies on EPT. */
+            if (    pVM->hwaccm.s.fNestedPaging
+                &&  (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE))
+            {
+                pVM->hwaccm.s.vmx.fUnrestrictedGuest = true;
+            }
+
+            /* Only try once. */
+            pVM->hwaccm.s.fInitialized = true;
+
+            if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
+            {
+                /* Allocate three pages for the TSS we need for real mode emulation. (2 pages for the IO bitmap) */
+                rc = PDMR3VMMDevHeapAlloc(pVM, HWACCM_VTX_TOTAL_DEVHEAP_MEM, (RTR3PTR *)&pVM->hwaccm.s.vmx.pRealModeTSS);
+                if (RT_SUCCESS(rc))
+                {
+                    /* The I/O bitmap starts right after the virtual interrupt redirection bitmap. */
+                    ASMMemZero32(pVM->hwaccm.s.vmx.pRealModeTSS, sizeof(*pVM->hwaccm.s.vmx.pRealModeTSS));
+                    pVM->hwaccm.s.vmx.pRealModeTSS->offIoBitmap = sizeof(*pVM->hwaccm.s.vmx.pRealModeTSS);
+                    /* Bit set to 0 means redirection enabled. */
+                    memset(pVM->hwaccm.s.vmx.pRealModeTSS->IntRedirBitmap, 0x0, sizeof(pVM->hwaccm.s.vmx.pRealModeTSS->IntRedirBitmap));
+                    /* Allow all port IO, so the VT-x IO intercepts do their job. */
+                    memset(pVM->hwaccm.s.vmx.pRealModeTSS + 1, 0, PAGE_SIZE*2);
+                    *((unsigned char *)pVM->hwaccm.s.vmx.pRealModeTSS + HWACCM_VTX_TSS_SIZE - 2) = 0xff;
+
+                    /* Construct a 1024 element page directory with 4 MB pages for the identity mapped page table used in
+                     * real and protected mode without paging with EPT.
+                     */
+                    pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable = (PX86PD)((char *)pVM->hwaccm.s.vmx.pRealModeTSS + PAGE_SIZE * 3);
+                    for (unsigned i=0;i<X86_PG_ENTRIES;i++)
+                    {
+                        pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable->a[i].u  = _4M * i;
+                        pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable->a[i].u |= X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_G;
+                    }
+
+                    /* We convert it here every time as pci regions could be reconfigured. */
+                    rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
+                    AssertRC(rc);
+                    LogRel(("HWACCM: Real Mode TSS guest physaddr  = %RGp\n", GCPhys));
+
+                    rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
+                    AssertRC(rc);
+                    LogRel(("HWACCM: Non-Paging Mode EPT CR3       = %RGp\n", GCPhys));
+                }
+                else
+                {
+                    LogRel(("HWACCM: No real mode VT-x support (PDMR3VMMDevHeapAlloc returned %Rrc)\n", rc));
+                    pVM->hwaccm.s.vmx.pRealModeTSS = NULL;
+                    pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable = NULL;
+                }
+            }
+
+            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_HWACC_SETUP_VM, 0, NULL);
+            AssertRC(rc);
+            if (rc == VINF_SUCCESS)
+            {
+                pVM->fHWACCMEnabled = true;
+                pVM->hwaccm.s.vmx.fEnabled = true;
+                hwaccmR3DisableRawMode(pVM);
+
+                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
+#ifdef VBOX_ENABLE_64_BITS_GUESTS
+                if (pVM->hwaccm.s.fAllow64BitGuests)
+                {
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);            /* 64 bits only on Intel CPUs */
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LAHF);
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
+                }
+                else
+                /* Turn on NXE if PAE has been enabled *and* the host has turned on NXE (we reuse the host EFER in the switcher) */
+                /* Todo: this needs to be fixed properly!! */
+                if (    CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE)
+                    &&  (pVM->hwaccm.s.vmx.hostEFER & MSR_K6_EFER_NXE))
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
+
+                LogRel((pVM->hwaccm.s.fAllow64BitGuests
+                        ? "HWACCM: 32-bit and 64-bit guests supported.\n"
+                        : "HWACCM: 32-bit guests supported.\n"));
+#else
+                LogRel(("HWACCM: 32-bit guests supported.\n"));
+#endif
+                LogRel(("HWACCM: VMX enabled!\n"));
+                if (pVM->hwaccm.s.fNestedPaging)
+                {
+                    LogRel(("HWACCM: Enabled nested paging\n"));
+                    LogRel(("HWACCM: EPT root page                 = %RHp\n", PGMGetHyperCR3(VMMGetCpu(pVM))));
+                    if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
+                        LogRel(("HWACCM: Unrestricted guest execution enabled!\n"));
+
+#if HC_ARCH_BITS == 64
+                    if (pVM->hwaccm.s.fLargePages)
+                    {
+                        /* Use large (2 MB) pages for our EPT PDEs where possible. */
+                        PGMSetLargePageUsage(pVM, true);
+                        LogRel(("HWACCM: Large page support enabled!\n"));
+                    }
+#endif
+                }
+                else
+                    Assert(!pVM->hwaccm.s.vmx.fUnrestrictedGuest);
+
+                if (pVM->hwaccm.s.vmx.fVPID)
+                    LogRel(("HWACCM: Enabled VPID\n"));
+
+                if (   pVM->hwaccm.s.fNestedPaging
+                    || pVM->hwaccm.s.vmx.fVPID)
+                {
+                    LogRel(("HWACCM: enmFlushPage    %d\n", pVM->hwaccm.s.vmx.enmFlushPage));
+                    LogRel(("HWACCM: enmFlushContext %d\n", pVM->hwaccm.s.vmx.enmFlushContext));
+                }
+
+                /* TPR patching status logging. */
+                if (pVM->hwaccm.s.fTRPPatchingAllowed)
+                {
+                    if (    (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
+                        &&  (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
+                    {
+                        pVM->hwaccm.s.fTRPPatchingAllowed = false;  /* not necessary as we have a hardware solution. */
+                        LogRel(("HWACCM: TPR Patching not required (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).\n"));
+                    }
+                    else
+                    {
+                        uint32_t u32Eax, u32Dummy;
+
+                        /* TPR patching needs access to the MSR_K8_LSTAR msr. */
+                        ASMCpuId(0x80000000, &u32Eax, &u32Dummy, &u32Dummy, &u32Dummy);
+                        if (    u32Eax < 0x80000001
+                            ||  !(ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
+                        {
+                            pVM->hwaccm.s.fTRPPatchingAllowed = false;
+                            LogRel(("HWACCM: TPR patching disabled (long mode not supported).\n"));
+                        }
+                    }
+                }
+                LogRel(("HWACCM: TPR Patching %s.\n", (pVM->hwaccm.s.fTRPPatchingAllowed) ? "enabled" : "disabled"));
+
+                /*
+                 * Check for preemption timer config override and log the state of it.
+                 */
+                if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
+                {
+                    PCFGMNODE pCfgHwAccM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "HWACCM");
+                    int rc2 = CFGMR3QueryBoolDef(pCfgHwAccM, "UsePreemptTimer", &pVM->hwaccm.s.vmx.fUsePreemptTimer, true);
+                    AssertLogRelRC(rc2);
+                }
+                if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
+                    LogRel(("HWACCM: Using the VMX-preemption timer (cPreemptTimerShift=%u)\n", pVM->hwaccm.s.vmx.cPreemptTimerShift));
+            }
+            else
+            {
+                LogRel(("HWACCM: VMX setup failed with rc=%Rrc!\n", rc));
+                LogRel(("HWACCM: Last instruction error %x\n", pVM->aCpus[0].hwaccm.s.vmx.lasterror.ulInstrError));
+                pVM->fHWACCMEnabled = false;
+            }
+        }
+    }
+    else
+    if (pVM->hwaccm.s.svm.fSupported)
+    {
+        Log(("pVM->hwaccm.s.svm.fSupported = %d\n", pVM->hwaccm.s.svm.fSupported));
+
+        if (pVM->hwaccm.s.fInitialized == false)
+        {
+            /* Erratum 170 which requires a forced TLB flush for each world switch:
+             * See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
+             *
+             * All BH-G1/2 and DH-G1/2 models include a fix:
+             * Athlon X2:   0x6b 1/2
+             *              0x68 1/2
+             * Athlon 64:   0x7f 1
+             *              0x6f 2
+             * Sempron:     0x7f 1/2
+             *              0x6f 2
+             *              0x6c 2
+             *              0x7c 2
+             * Turion 64:   0x68 2
+             *
+             */
+            uint32_t u32Dummy;
+            uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
+            ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
+            u32BaseFamily= (u32Version >> 8) & 0xf;
+            u32Family    = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
+            u32Model     = ((u32Version >> 4) & 0xf);
+            u32Model     = u32Model | ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
+            u32Stepping  = u32Version & 0xf;
+            if (    u32Family == 0xf
+                &&  !((u32Model == 0x68 || u32Model == 0x6b || u32Model == 0x7f) &&  u32Stepping >= 1)
+                &&  !((u32Model == 0x6f || u32Model == 0x6c || u32Model == 0x7c) &&  u32Stepping >= 2))
+            {
+                LogRel(("HWACMM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
+            }
+
+            LogRel(("HWACMM: cpuid 0x80000001.u32AMDFeatureECX = %RX32\n", pVM->hwaccm.s.cpuid.u32AMDFeatureECX));
+            LogRel(("HWACMM: cpuid 0x80000001.u32AMDFeatureEDX = %RX32\n", pVM->hwaccm.s.cpuid.u32AMDFeatureEDX));
+            LogRel(("HWACCM: AMD HWCR MSR                      = %RX64\n", pVM->hwaccm.s.svm.msrHWCR));
+            LogRel(("HWACCM: AMD-V revision                    = %X\n", pVM->hwaccm.s.svm.u32Rev));
+            LogRel(("HWACCM: AMD-V max ASID                    = %d\n", pVM->hwaccm.s.uMaxASID));
+            LogRel(("HWACCM: AMD-V features                    = %X\n", pVM->hwaccm.s.svm.u32Features));
+
+            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_NESTED_PAGING)
+                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_NESTED_PAGING\n"));
+            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_LBR_VIRT)
+                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_LBR_VIRT\n"));
+            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_SVM_LOCK)
+                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_SVM_LOCK\n"));
+            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
+                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_NRIP_SAVE\n"));
+            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_SSE_3_5_DISABLE)
+                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_SSE_3_5_DISABLE\n"));
+            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER)
+                LogRel(("HWACCM:    AMD_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER\n"));
+
+            /* Only try once. */
+            pVM->hwaccm.s.fInitialized = true;
+
+            if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_NESTED_PAGING)
+                pVM->hwaccm.s.fNestedPaging = pVM->hwaccm.s.fAllowNestedPaging;
+
+            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_HWACC_SETUP_VM, 0, NULL);
+            AssertRC(rc);
+            if (rc == VINF_SUCCESS)
+            {
+                pVM->fHWACCMEnabled = true;
+                pVM->hwaccm.s.svm.fEnabled = true;
+
+                if (pVM->hwaccm.s.fNestedPaging)
+                {
+                    LogRel(("HWACCM:    Enabled nested paging\n"));
+#if HC_ARCH_BITS == 64
+                    if (pVM->hwaccm.s.fLargePages)
+                    {
+                        /* Use large (2 MB) pages for our nested paging PDEs where possible. */
+                        PGMSetLargePageUsage(pVM, true);
+                        LogRel(("HWACCM:    Large page support enabled!\n"));
+                    }
+#endif
+                }
+
+                hwaccmR3DisableRawMode(pVM);
+                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SEP);
+                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_SYSCALL);
+                CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP);
+#ifdef VBOX_ENABLE_64_BITS_GUESTS
+                if (pVM->hwaccm.s.fAllow64BitGuests)
+                {
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE);
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LAHF);
+                }
+                else
+                /* Turn on NXE if PAE has been enabled. */
+                if (CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_PAE))
+                    CPUMSetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NXE);
+#endif
+
+                LogRel((pVM->hwaccm.s.fAllow64BitGuests
+                        ? "HWACCM:    32-bit and 64-bit guest supported.\n"
+                        : "HWACCM:    32-bit guest supported.\n"));
+
+                LogRel(("HWACCM:    TPR Patching %s.\n", (pVM->hwaccm.s.fTRPPatchingAllowed) ? "enabled" : "disabled"));
+            }
+            else
+            {
+                pVM->fHWACCMEnabled = false;
+            }
+        }
+    }
+    if (pVM->fHWACCMEnabled)
+        LogRel(("HWACCM:    VT-x/AMD-V init method: %s\n", (pVM->hwaccm.s.fGlobalInit) ? "GLOBAL" : "LOCAL"));
+    RTLogRelSetBuffering(fOldBuffered);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The VM.
+ */
+VMMR3DECL(void) HWACCMR3Relocate(PVM pVM)
+{
+    Log(("HWACCMR3Relocate to %RGv\n", MMHyperGetArea(pVM, 0)));
+
+    /* Fetch the current paging mode during the relocate callback during state loading. */
+    if (VMR3GetState(pVM) == VMSTATE_LOADING)
+    {
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PVMCPU pVCpu = &pVM->aCpus[i];
+
+            pVCpu->hwaccm.s.enmShadowMode            = PGMGetShadowMode(pVCpu);
+            Assert(pVCpu->hwaccm.s.vmx.enmCurrGuestMode == PGMGetGuestMode(pVCpu));
+            pVCpu->hwaccm.s.vmx.enmCurrGuestMode     = PGMGetGuestMode(pVCpu);
+        }
+    }
+#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    if (pVM->fHWACCMEnabled)
+    {
+        int rc;
+
+        switch(PGMGetHostMode(pVM))
+        {
+        case PGMMODE_32_BIT:
+            pVM->hwaccm.s.pfnHost32ToGuest64R0 = VMMR3GetHostToGuestSwitcher(pVM, VMMSWITCHER_32_TO_AMD64);
+            break;
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+            pVM->hwaccm.s.pfnHost32ToGuest64R0 = VMMR3GetHostToGuestSwitcher(pVM, VMMSWITCHER_PAE_TO_AMD64);
+            break;
+
+        default:
+            AssertFailed();
+            break;
+        }
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "VMXGCStartVM64", &pVM->hwaccm.s.pfnVMXGCStartVM64);
+        AssertReleaseMsgRC(rc, ("VMXGCStartVM64 -> rc=%Rrc\n", rc));
+
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "SVMGCVMRun64",   &pVM->hwaccm.s.pfnSVMGCVMRun64);
+        AssertReleaseMsgRC(rc, ("SVMGCVMRun64 -> rc=%Rrc\n", rc));
+
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "HWACCMSaveGuestFPU64",   &pVM->hwaccm.s.pfnSaveGuestFPU64);
+        AssertReleaseMsgRC(rc, ("HWACCMSetupFPU64 -> rc=%Rrc\n", rc));
+
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "HWACCMSaveGuestDebug64",   &pVM->hwaccm.s.pfnSaveGuestDebug64);
+        AssertReleaseMsgRC(rc, ("HWACCMSetupDebug64 -> rc=%Rrc\n", rc));
+
+# ifdef DEBUG
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       "HWACCMTestSwitcher64",   &pVM->hwaccm.s.pfnTest64);
+        AssertReleaseMsgRC(rc, ("HWACCMTestSwitcher64 -> rc=%Rrc\n", rc));
+# endif
+    }
+#endif
+    return;
+}
+
+/**
+ * Checks hardware accelerated raw mode is allowed.
+ *
+ * @returns boolean
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(bool) HWACCMR3IsAllowed(PVM pVM)
+{
+    return pVM->hwaccm.s.fAllowed;
+}
+
+/**
+ * Notification callback which is called whenever there is a chance that a CR3
+ * value might have changed.
+ *
+ * This is called by PGM.
+ *
+ * @param   pVM            The VM to operate on.
+ * @param   pVCpu          The VMCPU to operate on.
+ * @param   enmShadowMode  New shadow paging mode.
+ * @param   enmGuestMode   New guest paging mode.
+ */
+VMMR3DECL(void) HWACCMR3PagingModeChanged(PVM pVM, PVMCPU pVCpu, PGMMODE enmShadowMode, PGMMODE enmGuestMode)
+{
+    /* Ignore page mode changes during state loading. */
+    if (VMR3GetState(pVCpu->pVMR3) == VMSTATE_LOADING)
+        return;
+
+    pVCpu->hwaccm.s.enmShadowMode = enmShadowMode;
+
+    if (   pVM->hwaccm.s.vmx.fEnabled
+        && pVM->fHWACCMEnabled)
+    {
+        if (    pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
+            &&  enmGuestMode >= PGMMODE_PROTECTED)
+        {
+            PCPUMCTX pCtx;
+
+            pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+
+            /* After a real mode switch to protected mode we must force
+             * CPL to 0. Our real mode emulation had to set it to 3.
+             */
+            pCtx->ssHid.Attr.n.u2Dpl  = 0;
+        }
+    }
+
+    if (pVCpu->hwaccm.s.vmx.enmCurrGuestMode != enmGuestMode)
+    {
+        /* Keep track of paging mode changes. */
+        pVCpu->hwaccm.s.vmx.enmPrevGuestMode = pVCpu->hwaccm.s.vmx.enmCurrGuestMode;
+        pVCpu->hwaccm.s.vmx.enmCurrGuestMode = enmGuestMode;
+
+        /* Did we miss a change, because all code was executed in the recompiler? */
+        if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == enmGuestMode)
+        {
+            Log(("HWACCMR3PagingModeChanged missed %s->%s transition (prev %s)\n", PGMGetModeName(pVCpu->hwaccm.s.vmx.enmPrevGuestMode), PGMGetModeName(pVCpu->hwaccm.s.vmx.enmCurrGuestMode), PGMGetModeName(pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode)));
+            pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = pVCpu->hwaccm.s.vmx.enmPrevGuestMode;
+        }
+    }
+
+    /* Reset the contents of the read cache. */
+    PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
+    for (unsigned j=0;j<pCache->Read.cValidEntries;j++)
+        pCache->Read.aFieldVal[j] = 0;
+}
+
+/**
+ * Terminates the HWACCM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) HWACCMR3Term(PVM pVM)
+{
+    if (pVM->hwaccm.s.vmx.pRealModeTSS)
+    {
+        PDMR3VMMDevHeapFree(pVM, pVM->hwaccm.s.vmx.pRealModeTSS);
+        pVM->hwaccm.s.vmx.pRealModeTSS       = 0;
+    }
+    hwaccmR3TermCPU(pVM);
+    return 0;
+}
+
+/**
+ * Terminates the per-VCPU HWACCM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+static int hwaccmR3TermCPU(PVM pVM)
+{
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+#ifdef VBOX_WITH_STATISTICS
+        if (pVCpu->hwaccm.s.paStatExitReason)
+        {
+            MMHyperFree(pVM, pVCpu->hwaccm.s.paStatExitReason);
+            pVCpu->hwaccm.s.paStatExitReason   = NULL;
+            pVCpu->hwaccm.s.paStatExitReasonR0 = NIL_RTR0PTR;
+        }
+        if (pVCpu->hwaccm.s.paStatInjectedIrqs)
+        {
+            MMHyperFree(pVM, pVCpu->hwaccm.s.paStatInjectedIrqs);
+            pVCpu->hwaccm.s.paStatInjectedIrqs   = NULL;
+            pVCpu->hwaccm.s.paStatInjectedIrqsR0 = NIL_RTR0PTR;
+        }
+#endif
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+        memset(pVCpu->hwaccm.s.vmx.VMCSCache.aMagic, 0, sizeof(pVCpu->hwaccm.s.vmx.VMCSCache.aMagic));
+        pVCpu->hwaccm.s.vmx.VMCSCache.uMagic = 0;
+        pVCpu->hwaccm.s.vmx.VMCSCache.uPos = 0xffffffff;
+#endif
+    }
+    return 0;
+}
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by HWACCMR3Reset and CPU hot plugging.
+ *
+ * @param   pVCpu   The CPU to reset.
+ */
+VMMR3DECL(void) HWACCMR3ResetCpu(PVMCPU pVCpu)
+{
+    /* On first entry we'll sync everything. */
+    pVCpu->hwaccm.s.fContextUseFlags = HWACCM_CHANGED_ALL;
+
+    pVCpu->hwaccm.s.vmx.cr0_mask = 0;
+    pVCpu->hwaccm.s.vmx.cr4_mask = 0;
+
+    pVCpu->hwaccm.s.fActive        = false;
+    pVCpu->hwaccm.s.Event.fPending = false;
+
+    /* Reset state information for real-mode emulation in VT-x. */
+    pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
+    pVCpu->hwaccm.s.vmx.enmPrevGuestMode     = PGMMODE_REAL;
+    pVCpu->hwaccm.s.vmx.enmCurrGuestMode     = PGMMODE_REAL;
+
+    /* Reset the contents of the read cache. */
+    PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
+    for (unsigned j=0;j<pCache->Read.cValidEntries;j++)
+        pCache->Read.aFieldVal[j] = 0;
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    /* Magic marker for searching in crash dumps. */
+    strcpy((char *)pCache->aMagic, "VMCSCACHE Magic");
+    pCache->uMagic = UINT64_C(0xDEADBEEFDEADBEEF);
+#endif
+}
+
+/**
+ * The VM is being reset.
+ *
+ * For the HWACCM component this means that any GDT/LDT/TSS monitors
+ * needs to be removed.
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) HWACCMR3Reset(PVM pVM)
+{
+    LogFlow(("HWACCMR3Reset:\n"));
+
+    if (pVM->fHWACCMEnabled)
+        hwaccmR3DisableRawMode(pVM);
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        HWACCMR3ResetCpu(pVCpu);
+    }
+
+    /* Clear all patch information. */
+    pVM->hwaccm.s.pGuestPatchMem         = 0;
+    pVM->hwaccm.s.pFreeGuestPatchMem     = 0;
+    pVM->hwaccm.s.cbGuestPatchMem        = 0;
+    pVM->hwaccm.s.cPatches           = 0;
+    pVM->hwaccm.s.PatchTree          = 0;
+    pVM->hwaccm.s.fTPRPatchingActive = false;
+    ASMMemZero32(pVM->hwaccm.s.aPatches, sizeof(pVM->hwaccm.s.aPatches));
+}
+
+/**
+ * Callback to patch a TPR instruction (vmmcall or mov cr8)
+ *
+ * @returns VBox strict status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU for the EMT we're being called on.
+ * @param   pvUser  Unused
+ *
+ */
+DECLCALLBACK(VBOXSTRICTRC) hwaccmR3RemovePatches(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMCPUID idCpu = (VMCPUID)(uintptr_t)pvUser;
+
+    /* Only execute the handler on the VCPU the original patch request was issued. */
+    if (pVCpu->idCpu != idCpu)
+        return VINF_SUCCESS;
+
+    Log(("hwaccmR3RemovePatches\n"));
+    for (unsigned i = 0; i < pVM->hwaccm.s.cPatches; i++)
+    {
+        uint8_t         szInstr[15];
+        PHWACCMTPRPATCH pPatch = &pVM->hwaccm.s.aPatches[i];
+        RTGCPTR         pInstrGC = (RTGCPTR)pPatch->Core.Key;
+        int             rc;
+
+#ifdef LOG_ENABLED
+        char            szOutput[256];
+
+        rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, CPUMGetGuestCS(pVCpu), pInstrGC, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                szOutput, sizeof(szOutput), NULL);
+        if (RT_SUCCESS(rc))
+            Log(("Patched instr: %s\n", szOutput));
+#endif
+
+        /* Check if the instruction is still the same. */
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, szInstr, pInstrGC, pPatch->cbNewOp);
+        if (rc != VINF_SUCCESS)
+        {
+            Log(("Patched code removed? (rc=%Rrc0\n", rc));
+            continue;   /* swapped out or otherwise removed; skip it. */
+        }
+
+        if (memcmp(szInstr, pPatch->aNewOpcode, pPatch->cbNewOp))
+        {
+            Log(("Patched instruction was changed! (rc=%Rrc0\n", rc));
+            continue;   /* skip it. */
+        }
+
+        rc = PGMPhysSimpleWriteGCPtr(pVCpu, pInstrGC, pPatch->aOpcode, pPatch->cbOp);
+        AssertRC(rc);
+
+#ifdef LOG_ENABLED
+        rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, CPUMGetGuestCS(pVCpu), pInstrGC, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                szOutput, sizeof(szOutput), NULL);
+        if (RT_SUCCESS(rc))
+            Log(("Original instr: %s\n", szOutput));
+#endif
+    }
+    pVM->hwaccm.s.cPatches           = 0;
+    pVM->hwaccm.s.PatchTree          = 0;
+    pVM->hwaccm.s.pFreeGuestPatchMem = pVM->hwaccm.s.pGuestPatchMem;
+    pVM->hwaccm.s.fTPRPatchingActive = false;
+    return VINF_SUCCESS;
+}
+
+/**
+ * Enable patching in a VT-x/AMD-V guest
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   idCpu       VCPU to execute hwaccmR3RemovePatches on
+ * @param   pPatchMem   Patch memory range
+ * @param   cbPatchMem  Size of the memory range
+ */
+int hwaccmR3EnablePatching(PVM pVM, VMCPUID idCpu, RTRCPTR pPatchMem, unsigned cbPatchMem)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, hwaccmR3RemovePatches, (void *)idCpu);
+    AssertRC(rc);
+
+    pVM->hwaccm.s.pGuestPatchMem      = pPatchMem;
+    pVM->hwaccm.s.pFreeGuestPatchMem  = pPatchMem;
+    pVM->hwaccm.s.cbGuestPatchMem     = cbPatchMem;
+    return VINF_SUCCESS;
+}
+
+/**
+ * Enable patching in a VT-x/AMD-V guest
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pPatchMem   Patch memory range
+ * @param   cbPatchMem  Size of the memory range
+ */
+VMMR3DECL(int)  HWACMMR3EnablePatching(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    Log(("HWACMMR3EnablePatching %RGv size %x\n", pPatchMem, cbPatchMem));
+    if (pVM->cCpus > 1)
+    {
+        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
+        int rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE,
+                                    (PFNRT)hwaccmR3EnablePatching, 4, pVM, VMMGetCpuId(pVM), (RTRCPTR)pPatchMem, cbPatchMem);
+        AssertRC(rc);
+        return rc;
+    }
+    return hwaccmR3EnablePatching(pVM, VMMGetCpuId(pVM), (RTRCPTR)pPatchMem, cbPatchMem);
+}
+
+/**
+ * Disable patching in a VT-x/AMD-V guest
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pPatchMem   Patch memory range
+ * @param   cbPatchMem  Size of the memory range
+ */
+VMMR3DECL(int)  HWACMMR3DisablePatching(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    Log(("HWACMMR3DisablePatching %RGv size %x\n", pPatchMem, cbPatchMem));
+
+    Assert(pVM->hwaccm.s.pGuestPatchMem == pPatchMem);
+    Assert(pVM->hwaccm.s.cbGuestPatchMem == cbPatchMem);
+
+    /* @todo Potential deadlock when other VCPUs are waiting on the IOM lock (we own it)!! */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, hwaccmR3RemovePatches, (void *)VMMGetCpuId(pVM));
+    AssertRC(rc);
+
+    pVM->hwaccm.s.pGuestPatchMem      = 0;
+    pVM->hwaccm.s.pFreeGuestPatchMem  = 0;
+    pVM->hwaccm.s.cbGuestPatchMem     = 0;
+    pVM->hwaccm.s.fTPRPatchingActive = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback to patch a TPR instruction (vmmcall or mov cr8)
+ *
+ * @returns VBox strict status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU for the EMT we're being called on.
+ * @param   pvUser  User specified CPU context
+ *
+ */
+DECLCALLBACK(VBOXSTRICTRC) hwaccmR3ReplaceTprInstr(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMCPUID      idCpu  = (VMCPUID)(uintptr_t)pvUser;
+    PCPUMCTX     pCtx   = CPUMQueryGuestCtxPtr(pVCpu);
+    PDISCPUSTATE pDis   = &pVCpu->hwaccm.s.DisState;
+    unsigned     cbOp;
+
+    /* Only execute the handler on the VCPU the original patch request was issued. (the other CPU(s) might not yet have switched to protected mode) */
+    if (pVCpu->idCpu != idCpu)
+        return VINF_SUCCESS;
+
+    Log(("hwaccmR3ReplaceTprInstr: %RGv\n", pCtx->rip));
+
+    /* Two or more VCPUs were racing to patch this instruction. */
+    PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+    if (pPatch)
+        return VINF_SUCCESS;
+
+    Assert(pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches));
+
+    int rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
+    AssertRC(rc);
+    if (    rc == VINF_SUCCESS
+        &&  pDis->pCurInstr->opcode == OP_MOV
+        &&  cbOp >= 3)
+    {
+        uint8_t         aVMMCall[3] = { 0xf, 0x1, 0xd9};
+        uint32_t        idx = pVM->hwaccm.s.cPatches;
+
+        pPatch = &pVM->hwaccm.s.aPatches[idx];
+
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pPatch->aOpcode, pCtx->rip, cbOp);
+        AssertRC(rc);
+
+        pPatch->cbOp     = cbOp;
+
+        if (pDis->param1.flags == USE_DISPLACEMENT32)
+        {
+            /* write. */
+            if (pDis->param2.flags == USE_REG_GEN32)
+            {
+                pPatch->enmType     = HWACCMTPRINSTR_WRITE_REG;
+                pPatch->uSrcOperand = pDis->param2.base.reg_gen;
+            }
+            else
+            {
+                Assert(pDis->param2.flags == USE_IMMEDIATE32);
+                pPatch->enmType     = HWACCMTPRINSTR_WRITE_IMM;
+                pPatch->uSrcOperand = pDis->param2.parval;
+            }
+            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, aVMMCall, sizeof(aVMMCall));
+            AssertRC(rc);
+
+            memcpy(pPatch->aNewOpcode, aVMMCall, sizeof(aVMMCall));
+            pPatch->cbNewOp = sizeof(aVMMCall);
+        }
+        else
+        {
+            RTGCPTR  oldrip   = pCtx->rip;
+            uint32_t oldcbOp  = cbOp;
+            uint32_t uMmioReg = pDis->param1.base.reg_gen;
+
+            /* read */
+            Assert(pDis->param1.flags == USE_REG_GEN32);
+
+            /* Found:
+                *   mov eax, dword [fffe0080]        (5 bytes)
+                * Check if next instruction is:
+                *   shr eax, 4
+                */
+            pCtx->rip += cbOp;
+            rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
+            pCtx->rip = oldrip;
+            if (    rc == VINF_SUCCESS
+                &&  pDis->pCurInstr->opcode == OP_SHR
+                &&  pDis->param1.flags == USE_REG_GEN32
+                &&  pDis->param1.base.reg_gen == uMmioReg
+                &&  pDis->param2.flags == USE_IMMEDIATE8
+                &&  pDis->param2.parval == 4
+                &&  oldcbOp + cbOp < sizeof(pVM->hwaccm.s.aPatches[idx].aOpcode))
+            {
+                uint8_t szInstr[15];
+
+                /* Replacing two instructions now. */
+                rc = PGMPhysSimpleReadGCPtr(pVCpu, &pPatch->aOpcode, pCtx->rip, oldcbOp + cbOp);
+                AssertRC(rc);
+
+                pPatch->cbOp = oldcbOp + cbOp;
+
+                /* 0xF0, 0x0F, 0x20, 0xC0 = mov eax, cr8 */
+                szInstr[0] = 0xF0;
+                szInstr[1] = 0x0F;
+                szInstr[2] = 0x20;
+                szInstr[3] = 0xC0 | pDis->param1.base.reg_gen;
+                for (unsigned i = 4; i < pPatch->cbOp; i++)
+                    szInstr[i] = 0x90;  /* nop */
+
+                rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, szInstr, pPatch->cbOp);
+                AssertRC(rc);
+
+                memcpy(pPatch->aNewOpcode, szInstr, pPatch->cbOp);
+                pPatch->cbNewOp = pPatch->cbOp;
+
+                Log(("Acceptable read/shr candidate!\n"));
+                pPatch->enmType = HWACCMTPRINSTR_READ_SHR4;
+            }
+            else
+            {
+                pPatch->enmType     = HWACCMTPRINSTR_READ;
+                pPatch->uDstOperand = pDis->param1.base.reg_gen;
+
+                rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, aVMMCall, sizeof(aVMMCall));
+                AssertRC(rc);
+
+                memcpy(pPatch->aNewOpcode, aVMMCall, sizeof(aVMMCall));
+                pPatch->cbNewOp = sizeof(aVMMCall);
+            }
+        }
+
+        pPatch->Core.Key = pCtx->eip;
+        rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
+        AssertRC(rc);
+
+        pVM->hwaccm.s.cPatches++;
+        STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRReplaceSuccess);
+        return VINF_SUCCESS;
+    }
+
+    /* Save invalid patch, so we will not try again. */
+    uint32_t  idx = pVM->hwaccm.s.cPatches;
+
+#ifdef LOG_ENABLED
+    char      szOutput[256];
+    rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                            szOutput, sizeof(szOutput), NULL);
+    if (RT_SUCCESS(rc))
+        Log(("Failed to patch instr: %s\n", szOutput));
+#endif
+
+    pPatch = &pVM->hwaccm.s.aPatches[idx];
+    pPatch->Core.Key = pCtx->eip;
+    pPatch->enmType  = HWACCMTPRINSTR_INVALID;
+    rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
+    AssertRC(rc);
+    pVM->hwaccm.s.cPatches++;
+    STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRReplaceFailure);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Callback to patch a TPR instruction (jump to generated code)
+ *
+ * @returns VBox strict status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU for the EMT we're being called on.
+ * @param   pvUser  User specified CPU context
+ *
+ */
+DECLCALLBACK(VBOXSTRICTRC) hwaccmR3PatchTprInstr(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMCPUID      idCpu  = (VMCPUID)(uintptr_t)pvUser;
+    PCPUMCTX     pCtx   = CPUMQueryGuestCtxPtr(pVCpu);
+    PDISCPUSTATE pDis   = &pVCpu->hwaccm.s.DisState;
+    unsigned     cbOp;
+    int          rc;
+#ifdef LOG_ENABLED
+    RTGCPTR      pInstr;
+    char         szOutput[256];
+#endif
+
+    /* Only execute the handler on the VCPU the original patch request was issued. (the other CPU(s) might not yet have switched to protected mode) */
+    if (pVCpu->idCpu != idCpu)
+        return VINF_SUCCESS;
+
+    Assert(pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches));
+
+    /* Two or more VCPUs were racing to patch this instruction. */
+    PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+    if (pPatch)
+    {
+        Log(("hwaccmR3PatchTprInstr: already patched %RGv\n", pCtx->rip));
+        return VINF_SUCCESS;
+    }
+
+    Log(("hwaccmR3PatchTprInstr %RGv\n", pCtx->rip));
+
+    rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
+    AssertRC(rc);
+    if (    rc == VINF_SUCCESS
+        &&  pDis->pCurInstr->opcode == OP_MOV
+        &&  cbOp >= 5)
+    {
+        uint32_t        idx = pVM->hwaccm.s.cPatches;
+        uint8_t         aPatch[64];
+        uint32_t        off = 0;
+
+        pPatch = &pVM->hwaccm.s.aPatches[idx];
+
+#ifdef LOG_ENABLED
+        rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                szOutput, sizeof(szOutput), NULL);
+        if (RT_SUCCESS(rc))
+            Log(("Original instr: %s\n", szOutput));
+#endif
+
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pPatch->aOpcode, pCtx->rip, cbOp);
+        AssertRC(rc);
+
+        pPatch->cbOp    = cbOp;
+        pPatch->enmType = HWACCMTPRINSTR_JUMP_REPLACEMENT;
+
+        if (pDis->param1.flags == USE_DISPLACEMENT32)
+        {
+            /*
+                * TPR write:
+                *
+                * push ECX                      [51]
+                * push EDX                      [52]
+                * push EAX                      [50]
+                * xor EDX,EDX                   [31 D2]
+                * mov EAX,EAX                   [89 C0]
+                *  or
+                * mov EAX,0000000CCh            [B8 CC 00 00 00]
+                * mov ECX,0C0000082h            [B9 82 00 00 C0]
+                * wrmsr                         [0F 30]
+                * pop EAX                       [58]
+                * pop EDX                       [5A]
+                * pop ECX                       [59]
+                * jmp return_address            [E9 return_address]
+                *
+                */
+            bool fUsesEax = (pDis->param2.flags == USE_REG_GEN32 && pDis->param2.base.reg_gen == USE_REG_EAX);
+
+            aPatch[off++] = 0x51;    /* push ecx */
+            aPatch[off++] = 0x52;    /* push edx */
+            if (!fUsesEax)
+                aPatch[off++] = 0x50;    /* push eax */
+            aPatch[off++] = 0x31;    /* xor edx, edx */
+            aPatch[off++] = 0xD2;
+            if (pDis->param2.flags == USE_REG_GEN32)
+            {
+                if (!fUsesEax)
+                {
+                    aPatch[off++] = 0x89;    /* mov eax, src_reg */
+                    aPatch[off++] = MAKE_MODRM(3, pDis->param2.base.reg_gen, USE_REG_EAX);
+                }
+            }
+            else
+            {
+                Assert(pDis->param2.flags == USE_IMMEDIATE32);
+                aPatch[off++] = 0xB8;    /* mov eax, immediate */
+                *(uint32_t *)&aPatch[off] = pDis->param2.parval;
+                off += sizeof(uint32_t);
+            }
+            aPatch[off++] = 0xB9;    /* mov ecx, 0xc0000082 */
+            *(uint32_t *)&aPatch[off] = MSR_K8_LSTAR;
+            off += sizeof(uint32_t);
+
+            aPatch[off++] = 0x0F;    /* wrmsr */
+            aPatch[off++] = 0x30;
+            if (!fUsesEax)
+                aPatch[off++] = 0x58;    /* pop eax */
+            aPatch[off++] = 0x5A;    /* pop edx */
+            aPatch[off++] = 0x59;    /* pop ecx */
+        }
+        else
+        {
+            /*
+                * TPR read:
+                *
+                * push ECX                      [51]
+                * push EDX                      [52]
+                * push EAX                      [50]
+                * mov ECX,0C0000082h            [B9 82 00 00 C0]
+                * rdmsr                         [0F 32]
+                * mov EAX,EAX                   [89 C0]
+                * pop EAX                       [58]
+                * pop EDX                       [5A]
+                * pop ECX                       [59]
+                * jmp return_address            [E9 return_address]
+                *
+                */
+            Assert(pDis->param1.flags == USE_REG_GEN32);
+
+            if (pDis->param1.base.reg_gen != USE_REG_ECX)
+                aPatch[off++] = 0x51;    /* push ecx */
+            if (pDis->param1.base.reg_gen != USE_REG_EDX)
+                aPatch[off++] = 0x52;    /* push edx */
+            if (pDis->param1.base.reg_gen != USE_REG_EAX)
+                aPatch[off++] = 0x50;    /* push eax */
+
+            aPatch[off++] = 0x31;    /* xor edx, edx */
+            aPatch[off++] = 0xD2;
+
+            aPatch[off++] = 0xB9;    /* mov ecx, 0xc0000082 */
+            *(uint32_t *)&aPatch[off] = MSR_K8_LSTAR;
+            off += sizeof(uint32_t);
+
+            aPatch[off++] = 0x0F;    /* rdmsr */
+            aPatch[off++] = 0x32;
+
+            if (pDis->param1.base.reg_gen != USE_REG_EAX)
+            {
+                aPatch[off++] = 0x89;    /* mov dst_reg, eax */
+                aPatch[off++] = MAKE_MODRM(3, USE_REG_EAX, pDis->param1.base.reg_gen);
+            }
+
+            if (pDis->param1.base.reg_gen != USE_REG_EAX)
+                aPatch[off++] = 0x58;    /* pop eax */
+            if (pDis->param1.base.reg_gen != USE_REG_EDX)
+                aPatch[off++] = 0x5A;    /* pop edx */
+            if (pDis->param1.base.reg_gen != USE_REG_ECX)
+                aPatch[off++] = 0x59;    /* pop ecx */
+        }
+        aPatch[off++] = 0xE9;    /* jmp return_address */
+        *(RTRCUINTPTR *)&aPatch[off] = ((RTRCUINTPTR)pCtx->eip + cbOp) - ((RTRCUINTPTR)pVM->hwaccm.s.pFreeGuestPatchMem + off + 4);
+        off += sizeof(RTRCUINTPTR);
+
+        if (pVM->hwaccm.s.pFreeGuestPatchMem + off <= pVM->hwaccm.s.pGuestPatchMem + pVM->hwaccm.s.cbGuestPatchMem)
+        {
+            /* Write new code to the patch buffer. */
+            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pVM->hwaccm.s.pFreeGuestPatchMem, aPatch, off);
+            AssertRC(rc);
+
+#ifdef LOG_ENABLED
+            pInstr = pVM->hwaccm.s.pFreeGuestPatchMem;
+            while (true)
+            {
+                uint32_t cb;
+
+                rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pInstr, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                        szOutput, sizeof(szOutput), &cb);
+                if (RT_SUCCESS(rc))
+                    Log(("Patch instr %s\n", szOutput));
+
+                pInstr += cb;
+
+                if (pInstr >= pVM->hwaccm.s.pFreeGuestPatchMem + off)
+                    break;
+            }
+#endif
+
+            pPatch->aNewOpcode[0] = 0xE9;
+            *(RTRCUINTPTR *)&pPatch->aNewOpcode[1] = ((RTRCUINTPTR)pVM->hwaccm.s.pFreeGuestPatchMem) - ((RTRCUINTPTR)pCtx->eip + 5);
+
+            /* Overwrite the TPR instruction with a jump. */
+            rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->eip, pPatch->aNewOpcode, 5);
+            AssertRC(rc);
+
+#ifdef LOG_ENABLED
+            rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                    szOutput, sizeof(szOutput), NULL);
+            if (RT_SUCCESS(rc))
+                Log(("Jump: %s\n", szOutput));
+#endif
+            pVM->hwaccm.s.pFreeGuestPatchMem += off;
+            pPatch->cbNewOp = 5;
+
+            pPatch->Core.Key = pCtx->eip;
+            rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
+            AssertRC(rc);
+
+            pVM->hwaccm.s.cPatches++;
+            pVM->hwaccm.s.fTPRPatchingActive = true;
+            STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRPatchSuccess);
+            return VINF_SUCCESS;
+        }
+        else
+            Log(("Ran out of space in our patch buffer!\n"));
+    }
+
+    /* Save invalid patch, so we will not try again. */
+    uint32_t  idx = pVM->hwaccm.s.cPatches;
+
+#ifdef LOG_ENABLED
+    rc = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip, DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                            szOutput, sizeof(szOutput), NULL);
+    if (RT_SUCCESS(rc))
+        Log(("Failed to patch instr: %s\n", szOutput));
+#endif
+
+    pPatch = &pVM->hwaccm.s.aPatches[idx];
+    pPatch->Core.Key = pCtx->eip;
+    pPatch->enmType  = HWACCMTPRINSTR_INVALID;
+    rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
+    AssertRC(rc);
+    pVM->hwaccm.s.cPatches++;
+    STAM_COUNTER_INC(&pVM->hwaccm.s.StatTPRPatchFailure);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Attempt to patch TPR mmio instructions
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VM CPU to operate on.
+ * @param   pCtx        CPU context
+ */
+VMMR3DECL(int) HWACCMR3PatchTprInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, (pVM->hwaccm.s.pGuestPatchMem) ? hwaccmR3PatchTprInstr : hwaccmR3ReplaceTprInstr, (void *)pVCpu->idCpu);
+    AssertRC(rc);
+    return rc;
+}
+
+/**
+ * Force execution of the current IO code in the recompiler
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pCtx        Partial VM execution context
+ */
+VMMR3DECL(int) HWACCMR3EmulateIoBlock(PVM pVM, PCPUMCTX pCtx)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    Assert(pVM->fHWACCMEnabled);
+    Log(("HWACCMR3EmulateIoBlock\n"));
+
+    /* This is primarily intended to speed up Grub, so we don't care about paged protected mode. */
+    if (HWACCMCanEmulateIoBlockEx(pCtx))
+    {
+        Log(("HWACCMR3EmulateIoBlock -> enabled\n"));
+        pVCpu->hwaccm.s.EmulateIoBlock.fEnabled         = true;
+        pVCpu->hwaccm.s.EmulateIoBlock.GCPtrFunctionEip = pCtx->rip;
+        pVCpu->hwaccm.s.EmulateIoBlock.cr0              = pCtx->cr0;
+        return VINF_EM_RESCHEDULE_REM;
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * Checks if we can currently use hardware accelerated raw mode.
+ *
+ * @returns boolean
+ * @param   pVM         The VM to operate on.
+ * @param   pCtx        Partial VM execution context
+ */
+VMMR3DECL(bool) HWACCMR3CanExecuteGuest(PVM pVM, PCPUMCTX pCtx)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    Assert(pVM->fHWACCMEnabled);
+
+    /* If we're still executing the IO code, then return false. */
+    if (    RT_UNLIKELY(pVCpu->hwaccm.s.EmulateIoBlock.fEnabled)
+        &&  pCtx->rip <  pVCpu->hwaccm.s.EmulateIoBlock.GCPtrFunctionEip + 0x200
+        &&  pCtx->rip >  pVCpu->hwaccm.s.EmulateIoBlock.GCPtrFunctionEip - 0x200
+        &&  pCtx->cr0 == pVCpu->hwaccm.s.EmulateIoBlock.cr0)
+        return false;
+
+    pVCpu->hwaccm.s.EmulateIoBlock.fEnabled = false;
+
+    /* AMD-V supports real & protected mode with or without paging. */
+    if (pVM->hwaccm.s.svm.fEnabled)
+    {
+        pVCpu->hwaccm.s.fActive = true;
+        return true;
+    }
+
+    pVCpu->hwaccm.s.fActive = false;
+
+    /* Note! The context supplied by REM is partial. If we add more checks here, be sure to verify that REM provides this info! */
+    Assert((pVM->hwaccm.s.vmx.fUnrestrictedGuest && !pVM->hwaccm.s.vmx.pRealModeTSS) || (!pVM->hwaccm.s.vmx.fUnrestrictedGuest && pVM->hwaccm.s.vmx.pRealModeTSS));
+
+    bool fSupportsRealMode = pVM->hwaccm.s.vmx.fUnrestrictedGuest || PDMVMMDevHeapIsEnabled(pVM);
+    if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
+    {
+        /** The VMM device heap is a requirement for emulating real mode or protected mode without paging when the unrestricted guest execution feature is missing. */
+        if (fSupportsRealMode)
+        {
+            if (CPUMIsGuestInRealModeEx(pCtx))
+            {
+                /* VT-x will not allow high selector bases in v86 mode; fall back to the recompiler in that case.
+                 * The base must also be equal to (sel << 4).
+                 */
+                if (   (   pCtx->cs != (pCtx->csHid.u64Base >> 4)
+                        && pCtx->csHid.u64Base != 0xffff0000 /* we can deal with the BIOS code as it's also mapped into the lower region. */)
+                    || pCtx->ds != (pCtx->dsHid.u64Base >> 4)
+                    || pCtx->es != (pCtx->esHid.u64Base >> 4)
+                    || pCtx->fs != (pCtx->fsHid.u64Base >> 4)
+                    || pCtx->gs != (pCtx->gsHid.u64Base >> 4)
+                    || pCtx->ss != (pCtx->ssHid.u64Base >> 4))
+                {
+                    return false;
+                }
+            }
+            else
+            {
+                PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
+                /* Verify the requirements for executing code in protected mode. VT-x can't handle the CPU state right after a switch
+                 * from real to protected mode. (all sorts of RPL & DPL assumptions)
+                 */
+                if (    pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
+                    &&  enmGuestMode >= PGMMODE_PROTECTED)
+                {
+                    if (   (pCtx->cs & X86_SEL_RPL)
+                        || (pCtx->ds & X86_SEL_RPL)
+                        || (pCtx->es & X86_SEL_RPL)
+                        || (pCtx->fs & X86_SEL_RPL)
+                        || (pCtx->gs & X86_SEL_RPL)
+                        || (pCtx->ss & X86_SEL_RPL))
+                    {
+                        return false;
+                    }
+                }
+                /* VT-x also chokes on invalid tr or ldtr selectors (minix) */
+                if (    pCtx->gdtr.cbGdt
+                    &&  (   pCtx->tr > pCtx->gdtr.cbGdt
+                         || pCtx->ldtr > pCtx->gdtr.cbGdt))
+                {
+                        return false;
+                }
+            }
+        }
+        else
+        {
+            if (    !CPUMIsGuestInLongModeEx(pCtx)
+                &&  !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
+            {
+                /** @todo   This should (probably) be set on every excursion to the REM,
+                 *          however it's too risky right now. So, only apply it when we go
+                 *          back to REM for real mode execution. (The XP hack below doesn't
+                 *          work reliably without this.)
+                 *  Update: Implemented in EM.cpp, see #ifdef EM_NOTIFY_HWACCM.  */
+                pVM->aCpus[0].hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL_GUEST;
+
+                if (    !pVM->hwaccm.s.fNestedPaging        /* requires a fake PD for real *and* protected mode without paging - stored in the VMM device heap*/
+                    ||  CPUMIsGuestInRealModeEx(pCtx))      /* requires a fake TSS for real mode - stored in the VMM device heap */
+                    return false;
+
+                /* Too early for VT-x; Solaris guests will fail with a guru meditation otherwise; same for XP. */
+                if (pCtx->idtr.pIdt == 0 || pCtx->idtr.cbIdt == 0 || pCtx->tr == 0)
+                    return false;
+
+                /* The guest is about to complete the switch to protected mode. Wait a bit longer. */
+                /* Windows XP; switch to protected mode; all selectors are marked not present in the
+                 * hidden registers (possible recompiler bug; see load_seg_vm) */
+                if (pCtx->csHid.Attr.n.u1Present == 0)
+                    return false;
+                if (pCtx->ssHid.Attr.n.u1Present == 0)
+                    return false;
+
+                /* Windows XP: possible same as above, but new recompiler requires new heuristics?
+                   VT-x doesn't seem to like something about the guest state and this stuff avoids it. */
+                /** @todo This check is actually wrong, it doesn't take the direction of the
+                 *        stack segment into account. But, it does the job for now. */
+                if (pCtx->rsp >= pCtx->ssHid.u32Limit)
+                    return false;
+    #if 0
+                if (    pCtx->cs >= pCtx->gdtr.cbGdt
+                    ||  pCtx->ss >= pCtx->gdtr.cbGdt
+                    ||  pCtx->ds >= pCtx->gdtr.cbGdt
+                    ||  pCtx->es >= pCtx->gdtr.cbGdt
+                    ||  pCtx->fs >= pCtx->gdtr.cbGdt
+                    ||  pCtx->gs >= pCtx->gdtr.cbGdt)
+                    return false;
+    #endif
+            }
+        }
+    }
+
+    if (pVM->hwaccm.s.vmx.fEnabled)
+    {
+        uint32_t mask;
+
+        /* if bit N is set in cr0_fixed0, then it must be set in the guest's cr0. */
+        mask = (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed0;
+        /* Note: We ignore the NE bit here on purpose; see vmmr0\hwaccmr0.cpp for details. */
+        mask &= ~X86_CR0_NE;
+
+        if (fSupportsRealMode)
+        {
+            /* Note: We ignore the PE & PG bits here on purpose; we emulate real and protected mode without paging. */
+            mask &= ~(X86_CR0_PG|X86_CR0_PE);
+        }
+        else
+        {
+            /* We support protected mode without paging using identity mapping. */
+            mask &= ~X86_CR0_PG;
+        }
+        if ((pCtx->cr0 & mask) != mask)
+            return false;
+
+        /* if bit N is cleared in cr0_fixed1, then it must be zero in the guest's cr0. */
+        mask = (uint32_t)~pVM->hwaccm.s.vmx.msr.vmx_cr0_fixed1;
+        if ((pCtx->cr0 & mask) != 0)
+            return false;
+
+        /* if bit N is set in cr4_fixed0, then it must be set in the guest's cr4. */
+        mask  = (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
+        mask &= ~X86_CR4_VMXE;
+        if ((pCtx->cr4 & mask) != mask)
+            return false;
+
+        /* if bit N is cleared in cr4_fixed1, then it must be zero in the guest's cr4. */
+        mask = (uint32_t)~pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed1;
+        if ((pCtx->cr4 & mask) != 0)
+            return false;
+
+        pVCpu->hwaccm.s.fActive = true;
+        return true;
+    }
+
+    return false;
+}
+
+/**
+ * Checks if we need to reschedule due to VMM device heap changes
+ *
+ * @returns boolean
+ * @param   pVM         The VM to operate on.
+ * @param   pCtx        VM execution context
+ */
+VMMR3DECL(bool) HWACCMR3IsRescheduleRequired(PVM pVM, PCPUMCTX pCtx)
+{
+    /** The VMM device heap is a requirement for emulating real mode or protected mode without paging when the unrestricted guest execution feature is missing. (VT-x only) */
+    if (    pVM->hwaccm.s.vmx.fEnabled
+        &&  !pVM->hwaccm.s.vmx.fUnrestrictedGuest
+        &&  !CPUMIsGuestInPagedProtectedModeEx(pCtx)
+        &&  !PDMVMMDevHeapIsEnabled(pVM)
+        &&  (pVM->hwaccm.s.fNestedPaging || CPUMIsGuestInRealModeEx(pCtx)))
+        return true;
+
+    return false;
+}
+
+
+/**
+ * Notification from EM about a rescheduling into hardware assisted execution
+ * mode.
+ *
+ * @param   pVCpu       Pointer to the current virtual cpu structure.
+ */
+VMMR3DECL(void) HWACCMR3NotifyScheduled(PVMCPU pVCpu)
+{
+    pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL_GUEST;
+}
+
+/**
+ * Notification from EM about returning from instruction emulation (REM / EM).
+ *
+ * @param   pVCpu       Pointer to the current virtual cpu structure.
+ */
+VMMR3DECL(void) HWACCMR3NotifyEmulated(PVMCPU pVCpu)
+{
+    pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL_GUEST;
+}
+
+/**
+ * Checks if we are currently using hardware accelerated raw mode.
+ *
+ * @returns boolean
+ * @param   pVCpu        The VMCPU to operate on.
+ */
+VMMR3DECL(bool) HWACCMR3IsActive(PVMCPU pVCpu)
+{
+    return pVCpu->hwaccm.s.fActive;
+}
+
+/**
+ * Checks if we are currently using nested paging.
+ *
+ * @returns boolean
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(bool) HWACCMR3IsNestedPagingActive(PVM pVM)
+{
+    return pVM->hwaccm.s.fNestedPaging;
+}
+
+/**
+ * Checks if we are currently using VPID in VT-x mode.
+ *
+ * @returns boolean
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(bool) HWACCMR3IsVPIDActive(PVM pVM)
+{
+    return pVM->hwaccm.s.vmx.fVPID;
+}
+
+
+/**
+ * Checks if internal events are pending. In that case we are not allowed to dispatch interrupts.
+ *
+ * @returns boolean
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(bool) HWACCMR3IsEventPending(PVMCPU pVCpu)
+{
+    return HWACCMIsEnabled(pVCpu->pVMR3) && pVCpu->hwaccm.s.Event.fPending;
+}
+
+/**
+ * Checks if the VMX-preemption timer is being used.
+ *
+ * @returns true if it is, false if it isn't.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(bool) HWACCMR3IsVmxPreemptionTimerUsed(PVM pVM)
+{
+    return HWACCMIsEnabled(pVM)
+        && pVM->hwaccm.s.vmx.fEnabled
+        && pVM->hwaccm.s.vmx.fUsePreemptTimer;
+}
+
+/**
+ * Restart an I/O instruction that was refused in ring-0
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VERR_NOT_FOUND if no pending I/O instruction.
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   pCtx        VCPU register context
+ */
+VMMR3DECL(VBOXSTRICTRC) HWACCMR3RestartPendingIOInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    HWACCMPENDINGIO enmType = pVCpu->hwaccm.s.PendingIO.enmType;
+
+    pVCpu->hwaccm.s.PendingIO.enmType = HWACCMPENDINGIO_INVALID;
+
+    if (    pVCpu->hwaccm.s.PendingIO.GCPtrRip != pCtx->rip
+        ||  enmType  == HWACCMPENDINGIO_INVALID)
+        return VERR_NOT_FOUND;
+
+    VBOXSTRICTRC rcStrict;
+    switch (enmType)
+    {
+    case HWACCMPENDINGIO_PORT_READ:
+    {
+        uint32_t uAndVal = pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal;
+        uint32_t u32Val  = 0;
+
+        rcStrict = IOMIOPortRead(pVM, pVCpu->hwaccm.s.PendingIO.s.Port.uPort,
+                                 &u32Val,
+                                 pVCpu->hwaccm.s.PendingIO.s.Port.cbSize);
+        if (IOM_SUCCESS(rcStrict))
+        {
+            /* Write back to the EAX register. */
+            pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
+            pCtx->rip = pVCpu->hwaccm.s.PendingIO.GCPtrRipNext;
+        }
+        break;
+    }
+
+    case HWACCMPENDINGIO_PORT_WRITE:
+        rcStrict = IOMIOPortWrite(pVM, pVCpu->hwaccm.s.PendingIO.s.Port.uPort,
+                                  pCtx->eax & pVCpu->hwaccm.s.PendingIO.s.Port.uAndVal,
+                                  pVCpu->hwaccm.s.PendingIO.s.Port.cbSize);
+        if (IOM_SUCCESS(rcStrict))
+            pCtx->rip = pVCpu->hwaccm.s.PendingIO.GCPtrRipNext;
+        break;
+
+    default:
+        AssertFailed();
+        return VERR_INTERNAL_ERROR;
+    }
+
+    return rcStrict;
+}
+
+/**
+ * Inject an NMI into a running VM (only VCPU 0!)
+ *
+ * @returns boolean
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int)  HWACCMR3InjectNMI(PVM pVM)
+{
+    VMCPU_FF_SET(&pVM->aCpus[0], VMCPU_FF_INTERRUPT_NMI);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Check fatal VT-x/AMD-V error and produce some meaningful
+ * log release message.
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   iStatusCode VBox status code
+ */
+VMMR3DECL(void) HWACCMR3CheckError(PVM pVM, int iStatusCode)
+{
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        switch(iStatusCode)
+        {
+        case VERR_VMX_INVALID_VMCS_FIELD:
+            break;
+
+        case VERR_VMX_INVALID_VMCS_PTR:
+            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Current pointer %RGp vs %RGp\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.u64VMCSPhys, pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys));
+            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Current VMCS version %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulVMCSRevision));
+            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Entered Cpu %d\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.idEnteredCpu));
+            LogRel(("VERR_VMX_INVALID_VMCS_PTR: CPU%d Current Cpu %d\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.idCurrentCpu));
+            break;
+
+        case VERR_VMX_UNABLE_TO_START_VM:
+            LogRel(("VERR_VMX_UNABLE_TO_START_VM: CPU%d instruction error %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulInstrError));
+            LogRel(("VERR_VMX_UNABLE_TO_START_VM: CPU%d exit reason       %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulExitReason));
+#if 0 /* @todo dump the current control fields to the release log */
+            if (pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulInstrError == VMX_ERROR_VMENTRY_INVALID_CONTROL_FIELDS)
+            {
+
+            }
+#endif
+            break;
+
+        case VERR_VMX_UNABLE_TO_RESUME_VM:
+            LogRel(("VERR_VMX_UNABLE_TO_RESUME_VM: CPU%d instruction error %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulInstrError));
+            LogRel(("VERR_VMX_UNABLE_TO_RESUME_VM: CPU%d exit reason       %x\n", i, pVM->aCpus[i].hwaccm.s.vmx.lasterror.ulExitReason));
+            break;
+
+        case VERR_VMX_INVALID_VMXON_PTR:
+            break;
+        }
+    }
+}
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) hwaccmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    int rc;
+
+    Log(("hwaccmR3Save:\n"));
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        /*
+         * Save the basic bits - fortunately all the other things can be resynced on load.
+         */
+        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.Event.fPending);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.Event.errCode);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutU64(pSSM, pVM->aCpus[i].hwaccm.s.Event.intInfo);
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.vmx.enmLastSeenGuestMode);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.vmx.enmCurrGuestMode);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3PutU32(pSSM, pVM->aCpus[i].hwaccm.s.vmx.enmPrevGuestMode);
+        AssertRCReturn(rc, rc);
+    }
+#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
+    rc = SSMR3PutGCPtr(pSSM, pVM->hwaccm.s.pGuestPatchMem);
+    AssertRCReturn(rc, rc);
+    rc = SSMR3PutGCPtr(pSSM, pVM->hwaccm.s.pFreeGuestPatchMem);
+    AssertRCReturn(rc, rc);
+    rc = SSMR3PutU32(pSSM, pVM->hwaccm.s.cbGuestPatchMem);
+    AssertRCReturn(rc, rc);
+
+    /* Store all the guest patch records too. */
+    rc = SSMR3PutU32(pSSM, pVM->hwaccm.s.cPatches);
+    AssertRCReturn(rc, rc);
+
+    for (unsigned i = 0; i < pVM->hwaccm.s.cPatches; i++)
+    {
+        PHWACCMTPRPATCH pPatch = &pVM->hwaccm.s.aPatches[i];
+
+        rc = SSMR3PutU32(pSSM, pPatch->Core.Key);
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutMem(pSSM, pPatch->aOpcode, sizeof(pPatch->aOpcode));
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutU32(pSSM, pPatch->cbOp);
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutMem(pSSM, pPatch->aNewOpcode, sizeof(pPatch->aNewOpcode));
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutU32(pSSM, pPatch->cbNewOp);
+        AssertRCReturn(rc, rc);
+
+        AssertCompileSize(HWACCMTPRINSTR, 4);
+        rc = SSMR3PutU32(pSSM, (uint32_t)pPatch->enmType);
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutU32(pSSM, pPatch->uSrcOperand);
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutU32(pSSM, pPatch->uDstOperand);
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutU32(pSSM, pPatch->pJumpTarget);
+        AssertRCReturn(rc, rc);
+
+        rc = SSMR3PutU32(pSSM, pPatch->cFaults);
+        AssertRCReturn(rc, rc);
+    }
+#endif
+    return VINF_SUCCESS;
+}
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) hwaccmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int rc;
+
+    Log(("hwaccmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (   uVersion != HWACCM_SSM_VERSION
+        && uVersion != HWACCM_SSM_VERSION_NO_PATCHING
+        && uVersion != HWACCM_SSM_VERSION_2_0_X)
+    {
+        AssertMsgFailed(("hwaccmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        rc = SSMR3GetU32(pSSM, &pVM->aCpus[i].hwaccm.s.Event.fPending);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3GetU32(pSSM, &pVM->aCpus[i].hwaccm.s.Event.errCode);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3GetU64(pSSM, &pVM->aCpus[i].hwaccm.s.Event.intInfo);
+        AssertRCReturn(rc, rc);
+
+        if (uVersion >= HWACCM_SSM_VERSION_NO_PATCHING)
+        {
+            uint32_t val;
+
+            rc = SSMR3GetU32(pSSM, &val);
+            AssertRCReturn(rc, rc);
+            pVM->aCpus[i].hwaccm.s.vmx.enmLastSeenGuestMode = (PGMMODE)val;
+
+            rc = SSMR3GetU32(pSSM, &val);
+            AssertRCReturn(rc, rc);
+            pVM->aCpus[i].hwaccm.s.vmx.enmCurrGuestMode = (PGMMODE)val;
+
+            rc = SSMR3GetU32(pSSM, &val);
+            AssertRCReturn(rc, rc);
+            pVM->aCpus[i].hwaccm.s.vmx.enmPrevGuestMode = (PGMMODE)val;
+        }
+    }
+#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
+    if (uVersion > HWACCM_SSM_VERSION_NO_PATCHING)
+    {
+        rc = SSMR3GetGCPtr(pSSM, &pVM->hwaccm.s.pGuestPatchMem);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3GetGCPtr(pSSM, &pVM->hwaccm.s.pFreeGuestPatchMem);
+        AssertRCReturn(rc, rc);
+        rc = SSMR3GetU32(pSSM, &pVM->hwaccm.s.cbGuestPatchMem);
+        AssertRCReturn(rc, rc);
+
+        /* Fetch all TPR patch records. */
+        rc = SSMR3GetU32(pSSM, &pVM->hwaccm.s.cPatches);
+        AssertRCReturn(rc, rc);
+
+        for (unsigned i = 0; i < pVM->hwaccm.s.cPatches; i++)
+        {
+            PHWACCMTPRPATCH pPatch = &pVM->hwaccm.s.aPatches[i];
+
+            rc = SSMR3GetU32(pSSM, &pPatch->Core.Key);
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetMem(pSSM, pPatch->aOpcode, sizeof(pPatch->aOpcode));
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetU32(pSSM, &pPatch->cbOp);
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetMem(pSSM, pPatch->aNewOpcode, sizeof(pPatch->aNewOpcode));
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetU32(pSSM, &pPatch->cbNewOp);
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetU32(pSSM, (uint32_t *)&pPatch->enmType);
+            AssertRCReturn(rc, rc);
+
+            if (pPatch->enmType == HWACCMTPRINSTR_JUMP_REPLACEMENT)
+                pVM->hwaccm.s.fTPRPatchingActive = true;
+
+            Assert(pPatch->enmType == HWACCMTPRINSTR_JUMP_REPLACEMENT || pVM->hwaccm.s.fTPRPatchingActive == false);
+
+            rc = SSMR3GetU32(pSSM, &pPatch->uSrcOperand);
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetU32(pSSM, &pPatch->uDstOperand);
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetU32(pSSM, &pPatch->cFaults);
+            AssertRCReturn(rc, rc);
+
+            rc = SSMR3GetU32(pSSM, &pPatch->pJumpTarget);
+            AssertRCReturn(rc, rc);
+
+            Log(("hwaccmR3Load: patch %d\n", i));
+            Log(("Key       = %x\n", pPatch->Core.Key));
+            Log(("cbOp      = %d\n", pPatch->cbOp));
+            Log(("cbNewOp   = %d\n", pPatch->cbNewOp));
+            Log(("type      = %d\n", pPatch->enmType));
+            Log(("srcop     = %d\n", pPatch->uSrcOperand));
+            Log(("dstop     = %d\n", pPatch->uDstOperand));
+            Log(("cFaults   = %d\n", pPatch->cFaults));
+            Log(("target    = %x\n", pPatch->pJumpTarget));
+            rc = RTAvloU32Insert(&pVM->hwaccm.s.PatchTree, &pPatch->Core);
+            AssertRC(rc);
+        }
+    }
+#endif
+
+    /* Recheck all VCPUs if we can go straight into hwaccm execution mode. */
+    if (HWACCMIsEnabled(pVM))
+    {
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PVMCPU pVCpu = &pVM->aCpus[i];
+
+            HWACCMR3CanExecuteGuest(pVM, CPUMQueryGuestCtxPtr(pVCpu));
+        }
+    }
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/IOM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/IOM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/IOM.cpp	(revision 35333)
@@ -0,0 +1,2082 @@
+/* $Id$ */
+/** @file
+ * IOM - Input / Output Monitor.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_iom        IOM - The Input / Output Monitor
+ *
+ * The input/output monitor will handle I/O exceptions routing them to the
+ * appropriate device. It implements an API to register and deregister virtual
+ * I/0 port handlers and memory mapped I/O handlers. A handler is PDM devices
+ * and a set of callback functions.
+ *
+ * @see grp_iom
+ *
+ *
+ * @section sec_iom_rawmode     Raw-Mode
+ *
+ * In raw-mode I/O port access is trapped (\#GP(0)) by ensuring that the actual
+ * IOPL is 0 regardless of what the guest IOPL is. The \#GP handler use the
+ * disassembler (DIS) to figure which instruction caused it (there are a number
+ * of instructions in addition to the I/O ones) and if it's an I/O port access
+ * it will hand it to IOMGCIOPortHandler (via EMInterpretPortIO).
+ * IOMGCIOPortHandler will lookup the port in the AVL tree of registered
+ * handlers. If found, the handler will be called otherwise default action is
+ * taken. (Default action is to write into the void and read all set bits.)
+ *
+ * Memory Mapped I/O (MMIO) is implemented as a slightly special case of PGM
+ * access handlers. An MMIO range is registered with IOM which then registers it
+ * with the PGM access handler sub-system. The access handler catches all
+ * access and will be called in the context of a \#PF handler. In RC and R0 this
+ * handler is IOMMMIOHandler while in ring-3 it's IOMR3MMIOHandler (although in
+ * ring-3 there can be alternative ways). IOMMMIOHandler will attempt to emulate
+ * the instruction that is doing the access and pass the corresponding reads /
+ * writes to the device.
+ *
+ * Emulating I/O port access is less complex and should be slightly faster than
+ * emulating MMIO, so in most cases we should encourage the OS to use port I/O.
+ * Devices which are frequently accessed should register GC handlers to speed up
+ * execution.
+ *
+ *
+ * @section sec_iom_hwaccm     Hardware Assisted Virtualization Mode
+ *
+ * When running in hardware assisted virtualization mode we'll be doing much the
+ * same things as in raw-mode. The main difference is that we're running in the
+ * host ring-0 context and that we don't get faults (\#GP(0) and \#PG) but
+ * exits.
+ *
+ *
+ * @section sec_iom_rem         Recompiled Execution Mode
+ *
+ * When running in the recompiler things are different. I/O port access is
+ * handled by calling IOMIOPortRead and IOMIOPortWrite directly. While MMIO can
+ * be handled in one of two ways. The normal way is that we have a registered a
+ * special RAM range with the recompiler and in the three callbacks (for byte,
+ * word and dword access) we call IOMMMIORead and IOMMMIOWrite directly. The
+ * alternative ways that the physical memory access which goes via PGM will take
+ * care of it by calling IOMR3MMIOHandler via the PGM access handler machinery
+ * - this shouldn't happen but it is an alternative...
+ *
+ *
+ * @section sec_iom_other       Other Accesses
+ *
+ * I/O ports aren't really exposed in any other way, unless you count the
+ * instruction interpreter in EM, but that's just what we're doing in the
+ * raw-mode \#GP(0) case really. Now, it's possible to call IOMIOPortRead and
+ * IOMIOPortWrite directly to talk to a device, but this is really bad behavior
+ * and should only be done as temporary hacks (the PC BIOS device used to setup
+ * the CMOS this way back in the dark ages).
+ *
+ * MMIO has similar direct routes as the I/O ports and these shouldn't be used
+ * for the same reasons and with the same restrictions. OTOH since MMIO is
+ * mapped into the physical memory address space, it can be accessed in a number
+ * of ways thru PGM.
+ *
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM
+#include <VBox/iom.h>
+#include <VBox/cpum.h>
+#include <VBox/pgm.h>
+#include <VBox/sup.h>
+#include <VBox/mm.h>
+#include <VBox/stam.h>
+#include <VBox/dbgf.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pdmdev.h>
+#include "IOMInternal.h"
+#include <VBox/vm.h>
+
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static void iomR3FlushCache(PVM pVM);
+static DECLCALLBACK(int) iomR3RelocateIOPortCallback(PAVLROIOPORTNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int) iomR3RelocateMMIOCallback(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+static DECLCALLBACK(void) iomR3IOPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(int) iomR3IOPortDummyIn(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t *pu32, unsigned cb);
+static DECLCALLBACK(int) iomR3IOPortDummyOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t u32, unsigned cb);
+static DECLCALLBACK(int) iomR3IOPortDummyInStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrDst, PRTGCUINTREG pcTransfer, unsigned cb);
+static DECLCALLBACK(int) iomR3IOPortDummyOutStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrSrc, PRTGCUINTREG pcTransfer, unsigned cb);
+
+#ifdef VBOX_WITH_STATISTICS
+static const char *iomR3IOPortGetStandardName(RTIOPORT Port);
+#endif
+
+
+/**
+ * Initializes the IOM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) IOMR3Init(PVM pVM)
+{
+    LogFlow(("IOMR3Init:\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, iom.s, 32);
+    AssertCompile(sizeof(pVM->iom.s) <= sizeof(pVM->iom.padding));
+    AssertCompileMemberAlignment(IOM, EmtLock, sizeof(uintptr_t));
+
+    /*
+     * Setup any fixed pointers and offsets.
+     */
+    pVM->iom.s.offVM = RT_OFFSETOF(VM, iom);
+
+    /*
+     * Initialize the REM critical section.
+     */
+    int rc = PDMR3CritSectInit(pVM, &pVM->iom.s.EmtLock, RT_SRC_POS, "IOM EMT Lock");
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Allocate the trees structure.
+     */
+    rc = MMHyperAlloc(pVM, sizeof(*pVM->iom.s.pTreesR3), 0, MM_TAG_IOM, (void **)&pVM->iom.s.pTreesR3);
+    if (RT_SUCCESS(rc))
+    {
+        pVM->iom.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->iom.s.pTreesR3);
+        pVM->iom.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->iom.s.pTreesR3);
+        pVM->iom.s.pfnMMIOHandlerRC = NIL_RTGCPTR;
+        pVM->iom.s.pfnMMIOHandlerR0 = NIL_RTR0PTR;
+
+        /*
+         * Info.
+         */
+        DBGFR3InfoRegisterInternal(pVM, "ioport", "Dumps all IOPort ranges. No arguments.", &iomR3IOPortInfo);
+        DBGFR3InfoRegisterInternal(pVM, "mmio", "Dumps all MMIO ranges. No arguments.", &iomR3MMIOInfo);
+
+        /*
+         * Statistics.
+         */
+        STAM_REG(pVM, &pVM->iom.s.StatRZMMIOHandler,      STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler",                      STAMUNIT_TICKS_PER_CALL, "Profiling of the IOMMMIOHandler() body, only success calls.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO1Byte,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access1",              STAMUNIT_OCCURENCES,     "MMIO access by 1 byte counter.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO2Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access2",              STAMUNIT_OCCURENCES,     "MMIO access by 2 bytes counter.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO4Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access4",              STAMUNIT_OCCURENCES,     "MMIO access by 4 bytes counter.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZMMIO8Bytes,       STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Access8",              STAMUNIT_OCCURENCES,     "MMIO access by 8 bytes counter.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZMMIOFailures,     STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/MMIOFailures",         STAMUNIT_OCCURENCES,     "Number of times IOMMMIOHandler() didn't service the request.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstMov,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOV",             STAMUNIT_TICKS_PER_CALL, "Profiling of the MOV instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstCmp,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/CMP",             STAMUNIT_TICKS_PER_CALL, "Profiling of the CMP instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstAnd,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/AND",             STAMUNIT_TICKS_PER_CALL, "Profiling of the AND instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstOr,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/OR",              STAMUNIT_TICKS_PER_CALL, "Profiling of the OR instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstXor,          STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XOR",             STAMUNIT_TICKS_PER_CALL, "Profiling of the XOR instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstBt,           STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/BT",              STAMUNIT_TICKS_PER_CALL, "Profiling of the BT instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstTest,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/TEST",            STAMUNIT_TICKS_PER_CALL, "Profiling of the TEST instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstXchg,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/XCHG",            STAMUNIT_TICKS_PER_CALL, "Profiling of the XCHG instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstStos,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/STOS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the STOS instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstLods,         STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/LODS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the LODS instruction emulation.");
+#ifdef IOM_WITH_MOVS_SUPPORT
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovs,     STAMTYPE_PROFILE_ADV, "/IOM/RZ-MMIOHandler/Inst/MOVS",            STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsToMMIO,   STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/ToMMIO",     STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - Mem2MMIO.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsFromMMIO, STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/FromMMIO",   STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2Mem.");
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstMovsMMIO,     STAMTYPE_PROFILE, "/IOM/RZ-MMIOHandler/Inst/MOVS/MMIO2MMIO",  STAMUNIT_TICKS_PER_CALL, "Profiling of the MOVS instruction emulation - MMIO2MMIO.");
+#endif
+        STAM_REG(pVM, &pVM->iom.s.StatRZInstOther,        STAMTYPE_COUNTER, "/IOM/RZ-MMIOHandler/Inst/Other",           STAMUNIT_OCCURENCES,     "Other instructions counter.");
+        STAM_REG(pVM, &pVM->iom.s.StatR3MMIOHandler,      STAMTYPE_COUNTER, "/IOM/R3-MMIOHandler",                      STAMUNIT_OCCURENCES,     "Number of calls to IOMR3MMIOHandler.");
+        STAM_REG(pVM, &pVM->iom.s.StatInstIn,             STAMTYPE_COUNTER, "/IOM/IOWork/In",                           STAMUNIT_OCCURENCES,     "Counter of any IN instructions.");
+        STAM_REG(pVM, &pVM->iom.s.StatInstOut,            STAMTYPE_COUNTER, "/IOM/IOWork/Out",                          STAMUNIT_OCCURENCES,     "Counter of any OUT instructions.");
+        STAM_REG(pVM, &pVM->iom.s.StatInstIns,            STAMTYPE_COUNTER, "/IOM/IOWork/Ins",                          STAMUNIT_OCCURENCES,     "Counter of any INS instructions.");
+        STAM_REG(pVM, &pVM->iom.s.StatInstOuts,           STAMTYPE_COUNTER, "/IOM/IOWork/Outs",                         STAMUNIT_OCCURENCES,     "Counter of any OUTS instructions.");
+    }
+
+    /* Redundant, but just in case we change something in the future */
+    iomR3FlushCache(pVM);
+
+    LogFlow(("IOMR3Init: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Flushes the IOM port & statistics lookup cache
+ *
+ * @param   pVM     The VM.
+ */
+static void iomR3FlushCache(PVM pVM)
+{
+    iomLock(pVM);
+    /*
+     * Caching of port and statistics (saves some time in rep outs/ins instruction emulation)
+     */
+    pVM->iom.s.pRangeLastReadR0  = NIL_RTR0PTR;
+    pVM->iom.s.pRangeLastWriteR0 = NIL_RTR0PTR;
+    pVM->iom.s.pStatsLastReadR0  = NIL_RTR0PTR;
+    pVM->iom.s.pStatsLastWriteR0 = NIL_RTR0PTR;
+    pVM->iom.s.pMMIORangeLastR0  = NIL_RTR0PTR;
+    pVM->iom.s.pMMIOStatsLastR0  = NIL_RTR0PTR;
+
+    pVM->iom.s.pRangeLastReadR3  = NULL;
+    pVM->iom.s.pRangeLastWriteR3 = NULL;
+    pVM->iom.s.pStatsLastReadR3  = NULL;
+    pVM->iom.s.pStatsLastWriteR3 = NULL;
+    pVM->iom.s.pMMIORangeLastR3  = NULL;
+    pVM->iom.s.pMMIOStatsLastR3  = NULL;
+
+    pVM->iom.s.pRangeLastReadRC  = NIL_RTRCPTR;
+    pVM->iom.s.pRangeLastWriteRC = NIL_RTRCPTR;
+    pVM->iom.s.pStatsLastReadRC  = NIL_RTRCPTR;
+    pVM->iom.s.pStatsLastWriteRC = NIL_RTRCPTR;
+    pVM->iom.s.pMMIORangeLastRC  = NIL_RTRCPTR;
+    pVM->iom.s.pMMIOStatsLastRC  = NIL_RTRCPTR;
+
+    iomUnlock(pVM);
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) IOMR3Reset(PVM pVM)
+{
+    iomR3FlushCache(pVM);
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * The IOM will update the addresses used by the switcher.
+ *
+ * @param   pVM     The VM.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void) IOMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("IOMR3Relocate: offDelta=%d\n", offDelta));
+
+    /*
+     * Apply relocations to the GC callbacks.
+     */
+    pVM->iom.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->iom.s.pTreesR3);
+    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeRC, true, iomR3RelocateIOPortCallback, &offDelta);
+    RTAvlroGCPhysDoWithAll(&pVM->iom.s.pTreesR3->MMIOTree,     true, iomR3RelocateMMIOCallback,   &offDelta);
+
+    if (pVM->iom.s.pfnMMIOHandlerRC)
+        pVM->iom.s.pfnMMIOHandlerRC += offDelta;
+
+    /*
+     * Apply relocations to the cached GC handlers
+     */
+    if (pVM->iom.s.pRangeLastReadRC)
+        pVM->iom.s.pRangeLastReadRC  += offDelta;
+    if (pVM->iom.s.pRangeLastWriteRC)
+        pVM->iom.s.pRangeLastWriteRC += offDelta;
+    if (pVM->iom.s.pStatsLastReadRC)
+        pVM->iom.s.pStatsLastReadRC  += offDelta;
+    if (pVM->iom.s.pStatsLastWriteRC)
+        pVM->iom.s.pStatsLastWriteRC += offDelta;
+    if (pVM->iom.s.pMMIORangeLastRC)
+        pVM->iom.s.pMMIORangeLastRC  += offDelta;
+    if (pVM->iom.s.pMMIOStatsLastRC)
+        pVM->iom.s.pMMIOStatsLastRC  += offDelta;
+}
+
+
+/**
+ * Callback function for relocating a I/O port range.
+ *
+ * @returns 0 (continue enum)
+ * @param   pNode       Pointer to a IOMIOPORTRANGERC node.
+ * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
+ *                      not certain the delta will fit in a void pointer for all possible configs.
+ */
+static DECLCALLBACK(int) iomR3RelocateIOPortCallback(PAVLROIOPORTNODECORE pNode, void *pvUser)
+{
+    PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)pNode;
+    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
+
+    Assert(pRange->pDevIns);
+    pRange->pDevIns                 += offDelta;
+    if (pRange->pfnOutCallback)
+        pRange->pfnOutCallback      += offDelta;
+    if (pRange->pfnInCallback)
+        pRange->pfnInCallback       += offDelta;
+    if (pRange->pfnOutStrCallback)
+        pRange->pfnOutStrCallback   += offDelta;
+    if (pRange->pfnInStrCallback)
+        pRange->pfnInStrCallback    += offDelta;
+    if (pRange->pvUser > _64K)
+        pRange->pvUser              += offDelta;
+    return 0;
+}
+
+
+/**
+ * Callback function for relocating a MMIO range.
+ *
+ * @returns 0 (continue enum)
+ * @param   pNode       Pointer to a IOMMMIORANGE node.
+ * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
+ *                      not certain the delta will fit in a void pointer for all possible configs.
+ */
+static DECLCALLBACK(int) iomR3RelocateMMIOCallback(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pNode;
+    RTGCINTPTR    offDelta = *(PRTGCINTPTR)pvUser;
+
+    if (pRange->pDevInsRC)
+        pRange->pDevInsRC           += offDelta;
+    if (pRange->pfnWriteCallbackRC)
+        pRange->pfnWriteCallbackRC  += offDelta;
+    if (pRange->pfnReadCallbackRC)
+        pRange->pfnReadCallbackRC   += offDelta;
+    if (pRange->pfnFillCallbackRC)
+        pRange->pfnFillCallbackRC   += offDelta;
+    if (pRange->pvUserRC > _64K)
+        pRange->pvUserRC            += offDelta;
+
+    return 0;
+}
+
+
+/**
+ * Terminates the IOM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) IOMR3Term(PVM pVM)
+{
+    /*
+     * IOM is not owning anything but automatically freed resources,
+     * so there's nothing to do here.
+     */
+    return VINF_SUCCESS;
+}
+
+#ifdef VBOX_WITH_STATISTICS
+
+/**
+ * Create the statistics node for an I/O port.
+ *
+ * @returns Pointer to new stats node.
+ *
+ * @param   pVM         VM handle.
+ * @param   Port        Port.
+ * @param   pszDesc     Description.
+ */
+PIOMIOPORTSTATS iomR3IOPortStatsCreate(PVM pVM, RTIOPORT Port, const char *pszDesc)
+{
+    Assert(IOMIsLockOwner(pVM));
+    /* check if it already exists. */
+    PIOMIOPORTSTATS pPort = (PIOMIOPORTSTATS)RTAvloIOPortGet(&pVM->iom.s.pTreesR3->IOPortStatTree, Port);
+    if (pPort)
+        return pPort;
+
+    /* allocate stats node. */
+    int rc = MMHyperAlloc(pVM, sizeof(*pPort), 0, MM_TAG_IOM_STATS, (void **)&pPort);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+    {
+        /* insert into the tree. */
+        pPort->Core.Key = Port;
+        if (RTAvloIOPortInsert(&pVM->iom.s.pTreesR3->IOPortStatTree, &pPort->Core))
+        {
+            /* put a name on common ports. */
+            if (!pszDesc)
+                pszDesc = iomR3IOPortGetStandardName(Port);
+
+            /* register the statistics counters. */
+            rc = STAMR3RegisterF(pVM, &pPort->InR3,     STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-In-R3", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->OutR3,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-Out-R3", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->InRZ,     STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-In-RZ", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->OutRZ,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-Out-RZ", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->InRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-In-RZtoR3", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->OutRZToR3,STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, pszDesc,    "/IOM/Ports/%04x-Out-RZtoR3", Port); AssertRC(rc);
+
+            /* Profiling */
+            rc = STAMR3RegisterF(pVM, &pPort->ProfInR3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-In-R3/Prof", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->ProfOutR3,STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-Out-R3/Prof", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->ProfInRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-In-RZ/Prof", Port); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pPort->ProfOutRZ,STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc,"/IOM/Ports/%04x-Out-RZ/Prof", Port); AssertRC(rc);
+
+            return pPort;
+        }
+        AssertMsgFailed(("what! Port=%d\n", Port));
+        MMHyperFree(pVM, pPort);
+    }
+    return NULL;
+}
+
+
+/**
+ * Create the statistics node for an MMIO address.
+ *
+ * @returns Pointer to new stats node.
+ *
+ * @param   pVM         VM handle.
+ * @param   GCPhys      The address.
+ * @param   pszDesc     Description.
+ */
+PIOMMMIOSTATS iomR3MMIOStatsCreate(PVM pVM, RTGCPHYS GCPhys, const char *pszDesc)
+{
+    Assert(IOMIsLockOwner(pVM));
+#ifdef DEBUG_sandervl
+    AssertGCPhys32(GCPhys);
+#endif
+    /* check if it already exists. */
+    PIOMMMIOSTATS pStats = (PIOMMMIOSTATS)RTAvloGCPhysGet(&pVM->iom.s.pTreesR3->MMIOStatTree, GCPhys);
+    if (pStats)
+        return pStats;
+
+    /* allocate stats node. */
+    int rc = MMHyperAlloc(pVM, sizeof(*pStats), 0, MM_TAG_IOM_STATS, (void **)&pStats);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+    {
+        /* insert into the tree. */
+        pStats->Core.Key = GCPhys;
+        if (RTAvloGCPhysInsert(&pVM->iom.s.pTreesR3->MMIOStatTree, &pStats->Core))
+        {
+            rc = STAMR3RegisterF(pVM, &pStats->Accesses,    STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,     pszDesc, "/IOM/MMIO/%RGp",              GCPhys); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pStats->ProfReadR3,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Read-R3",      GCPhys); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pStats->ProfWriteR3, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Write-R3",     GCPhys); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pStats->ProfReadRZ,  STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Read-RZ",      GCPhys); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pStats->ProfWriteRZ, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, pszDesc, "/IOM/MMIO/%RGp/Write-RZ",     GCPhys); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pStats->ReadRZToR3,  STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,     pszDesc, "/IOM/MMIO/%RGp/Read-RZtoR3",  GCPhys); AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pStats->WriteRZToR3, STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,     pszDesc, "/IOM/MMIO/%RGp/Write-RZtoR3", GCPhys); AssertRC(rc);
+
+            return pStats;
+        }
+        AssertMsgFailed(("what! GCPhys=%RGp\n", GCPhys));
+        MMHyperFree(pVM, pStats);
+    }
+    return NULL;
+}
+
+#endif /* VBOX_WITH_STATISTICS */
+
+/**
+ * Registers a I/O port ring-3 handler.
+ *
+ * This API is called by PDM on behalf of a device. Devices must first register
+ * ring-3 ranges before any GC and R0 ranges can be registered using IOMR3IOPortRegisterRC()
+ * and IOMR3IOPortRegisterR0().
+ *
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 VM handle.
+ * @param   pDevIns             PDM device instance owning the port range.
+ * @param   PortStart           First port number in the range.
+ * @param   cPorts              Number of ports to register.
+ * @param   pvUser              User argument for the callbacks.
+ * @param   pfnOutCallback      Pointer to function which is gonna handle OUT operations in R3.
+ * @param   pfnInCallback       Pointer to function which is gonna handle IN operations in R3.
+ * @param   pfnOutStrCallback   Pointer to function which is gonna handle string OUT operations in R3.
+ * @param   pfnInStrCallback    Pointer to function which is gonna handle string IN operations in R3.
+ * @param   pszDesc             Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int) IOMR3IOPortRegisterR3(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts, RTHCPTR pvUser,
+                                     R3PTRTYPE(PFNIOMIOPORTOUT) pfnOutCallback, R3PTRTYPE(PFNIOMIOPORTIN) pfnInCallback,
+                                     R3PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback, R3PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback, const char *pszDesc)
+{
+    LogFlow(("IOMR3IOPortRegisterR3: pDevIns=%p PortStart=%#x cPorts=%#x pvUser=%RHv pfnOutCallback=%#x pfnInCallback=%#x pfnOutStrCallback=%#x pfnInStrCallback=%#x pszDesc=%s\n",
+             pDevIns, PortStart, cPorts, pvUser, pfnOutCallback, pfnInCallback, pfnOutStrCallback, pfnInStrCallback, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    if (    (RTUINT)PortStart + cPorts <= (RTUINT)PortStart
+        ||  (RTUINT)PortStart + cPorts > 0x10000)
+    {
+        AssertMsgFailed(("Invalid port range %#x-%#x (inclusive)! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
+        return VERR_IOM_INVALID_IOPORT_RANGE;
+    }
+    if (!pfnOutCallback && !pfnInCallback)
+    {
+        AssertMsgFailed(("no handlers specfied for %#x-%#x (inclusive)! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!pfnOutCallback)
+        pfnOutCallback = iomR3IOPortDummyOut;
+    if (!pfnInCallback)
+        pfnInCallback = iomR3IOPortDummyIn;
+    if (!pfnOutStrCallback)
+        pfnOutStrCallback = iomR3IOPortDummyOutStr;
+    if (!pfnInStrCallback)
+        pfnInStrCallback = iomR3IOPortDummyInStr;
+
+    /* Flush the IO port lookup cache */
+    iomR3FlushCache(pVM);
+
+    /*
+     * Allocate new range record and initialize it.
+     */
+    PIOMIOPORTRANGER3 pRange;
+    int rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
+    if (RT_SUCCESS(rc))
+    {
+        pRange->Core.Key        = PortStart;
+        pRange->Core.KeyLast    = PortStart + (cPorts - 1);
+        pRange->Port            = PortStart;
+        pRange->cPorts          = cPorts;
+        pRange->pvUser          = pvUser;
+        pRange->pDevIns         = pDevIns;
+        pRange->pfnOutCallback  = pfnOutCallback;
+        pRange->pfnInCallback   = pfnInCallback;
+        pRange->pfnOutStrCallback = pfnOutStrCallback;
+        pRange->pfnInStrCallback = pfnInStrCallback;
+        pRange->pszDesc         = pszDesc;
+
+        /*
+         * Try Insert it.
+         */
+        iomLock(pVM);
+        if (RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeR3, &pRange->Core))
+        {
+            #ifdef VBOX_WITH_STATISTICS
+            for (unsigned iPort = 0; iPort < cPorts; iPort++)
+                iomR3IOPortStatsCreate(pVM, PortStart + iPort, pszDesc);
+            #endif
+            iomUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+        iomUnlock(pVM);
+
+        /* conflict. */
+        DBGFR3Info(pVM, "ioport", NULL, NULL);
+        AssertMsgFailed(("Port range %#x-%#x (%s) conflicts with existing range(s)!\n", PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
+        MMHyperFree(pVM, pRange);
+        rc = VERR_IOM_IOPORT_RANGE_CONFLICT;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Registers a I/O port RC handler.
+ *
+ * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
+ * using IOMIOPortRegisterR3() before calling this function.
+ *
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 VM handle.
+ * @param   pDevIns             PDM device instance owning the port range.
+ * @param   PortStart           First port number in the range.
+ * @param   cPorts              Number of ports to register.
+ * @param   pvUser              User argument for the callbacks.
+ * @param   pfnOutCallback      Pointer to function which is gonna handle OUT operations in GC.
+ * @param   pfnInCallback       Pointer to function which is gonna handle IN operations in GC.
+ * @param   pfnOutStrCallback   Pointer to function which is gonna handle string OUT operations in GC.
+ * @param   pfnInStrCallback    Pointer to function which is gonna handle string IN operations in GC.
+ * @param   pszDesc             Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int)  IOMR3IOPortRegisterRC(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts, RTRCPTR pvUser,
+                                      RCPTRTYPE(PFNIOMIOPORTOUT) pfnOutCallback, RCPTRTYPE(PFNIOMIOPORTIN) pfnInCallback,
+                                      RCPTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback, RCPTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback, const char *pszDesc)
+{
+    LogFlow(("IOMR3IOPortRegisterRC: pDevIns=%p PortStart=%#x cPorts=%#x pvUser=%RRv pfnOutCallback=%RRv pfnInCallback=%RRv pfnOutStrCallback=%RRv  pfnInStrCallback=%RRv pszDesc=%s\n",
+             pDevIns, PortStart, cPorts, pvUser, pfnOutCallback, pfnInCallback, pfnOutStrCallback, pfnInStrCallback, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    if (    (RTUINT)PortStart + cPorts <= (RTUINT)PortStart
+        ||  (RTUINT)PortStart + cPorts > 0x10000)
+    {
+        AssertMsgFailed(("Invalid port range %#x-%#x! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
+        return VERR_IOM_INVALID_IOPORT_RANGE;
+    }
+    RTIOPORT PortLast = PortStart + (cPorts - 1);
+    if (!pfnOutCallback && !pfnInCallback)
+    {
+        AssertMsgFailed(("Invalid port range %#x-%#x! No callbacks! (%s)\n", PortStart, PortLast, pszDesc));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    iomLock(pVM);
+
+    /*
+     * Validate that there are ring-3 ranges for the ports.
+     */
+    RTIOPORT Port = PortStart;
+    while (Port <= PortLast && Port >= PortStart)
+    {
+        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeR3, Port);
+        if (!pRange)
+        {
+            AssertMsgFailed(("No R3! Port=#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
+            iomUnlock(pVM);
+            return VERR_IOM_NO_HC_IOPORT_RANGE;
+        }
+#ifndef IOM_NO_PDMINS_CHECKS
+# ifndef IN_RC
+        if (pRange->pDevIns != pDevIns)
+# else
+        if (pRange->pDevIns != MMHyperRCToCC(pVM, pDevIns))
+# endif
+        {
+            AssertMsgFailed(("Not owner! Port=%#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
+            iomUnlock(pVM);
+            return VERR_IOM_NOT_IOPORT_RANGE_OWNER;
+        }
+#endif
+        Port = pRange->Core.KeyLast + 1;
+    }
+
+    /* Flush the IO port lookup cache */
+    iomR3FlushCache(pVM);
+
+    /*
+     * Allocate new range record and initialize it.
+     */
+    PIOMIOPORTRANGERC pRange;
+    int rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
+    if (RT_SUCCESS(rc))
+    {
+        pRange->Core.Key        = PortStart;
+        pRange->Core.KeyLast    = PortLast;
+        pRange->Port            = PortStart;
+        pRange->cPorts          = cPorts;
+        pRange->pvUser          = pvUser;
+        pRange->pfnOutCallback  = pfnOutCallback;
+        pRange->pfnInCallback   = pfnInCallback;
+        pRange->pfnOutStrCallback = pfnOutStrCallback;
+        pRange->pfnInStrCallback = pfnInStrCallback;
+        pRange->pDevIns         = MMHyperCCToRC(pVM, pDevIns);
+        pRange->pszDesc         = pszDesc;
+
+        /*
+         * Insert it.
+         */
+        if (RTAvlroIOPortInsert(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeRC, &pRange->Core))
+        {
+            iomUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+
+        /* conflict. */
+        AssertMsgFailed(("Port range %#x-%#x (%s) conflicts with existing range(s)!\n", PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
+        MMHyperFree(pVM, pRange);
+        rc = VERR_IOM_IOPORT_RANGE_CONFLICT;
+    }
+    iomUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Registers a Port IO R0 handler.
+ *
+ * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
+ * using IOMR3IOPortRegisterR3() before calling this function.
+ *
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 VM handle.
+ * @param   pDevIns             PDM device instance owning the port range.
+ * @param   PortStart           First port number in the range.
+ * @param   cPorts              Number of ports to register.
+ * @param   pvUser              User argument for the callbacks.
+ * @param   pfnOutCallback      Pointer to function which is gonna handle OUT operations in GC.
+ * @param   pfnInCallback       Pointer to function which is gonna handle IN operations in GC.
+ * @param   pfnOutStrCallback   Pointer to function which is gonna handle OUT operations in GC.
+ * @param   pfnInStrCallback    Pointer to function which is gonna handle IN operations in GC.
+ * @param   pszDesc             Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int)  IOMR3IOPortRegisterR0(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts, RTR0PTR pvUser,
+                                      R0PTRTYPE(PFNIOMIOPORTOUT) pfnOutCallback, R0PTRTYPE(PFNIOMIOPORTIN) pfnInCallback,
+                                      R0PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback, R0PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback,
+                                      const char *pszDesc)
+{
+    LogFlow(("IOMR3IOPortRegisterR0: pDevIns=%p PortStart=%#x cPorts=%#x pvUser=%RHv pfnOutCallback=%RHv pfnInCallback=%RHv pfnOutStrCallback=%RHv  pfnInStrCallback=%RHv pszDesc=%s\n",
+             pDevIns, PortStart, cPorts, pvUser, pfnOutCallback, pfnInCallback, pfnOutStrCallback, pfnInStrCallback, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    if (    (RTUINT)PortStart + cPorts <= (RTUINT)PortStart
+        ||  (RTUINT)PortStart + cPorts > 0x10000)
+    {
+        AssertMsgFailed(("Invalid port range %#x-%#x! (%s)\n", PortStart, (RTUINT)PortStart + (cPorts - 1), pszDesc));
+        return VERR_IOM_INVALID_IOPORT_RANGE;
+    }
+    RTIOPORT PortLast = PortStart + (cPorts - 1);
+    if (!pfnOutCallback && !pfnInCallback)
+    {
+        AssertMsgFailed(("Invalid port range %#x-%#x! No callbacks! (%s)\n", PortStart, PortLast, pszDesc));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    iomLock(pVM);
+    /*
+     * Validate that there are ring-3 ranges for the ports.
+     */
+    RTIOPORT Port = PortStart;
+    while (Port <= PortLast && Port >= PortStart)
+    {
+        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeR3, Port);
+        if (!pRange)
+        {
+            AssertMsgFailed(("No R3! Port=#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
+            iomUnlock(pVM);
+            return VERR_IOM_NO_HC_IOPORT_RANGE;
+        }
+#ifndef IOM_NO_PDMINS_CHECKS
+# ifndef IN_RC
+        if (pRange->pDevIns != pDevIns)
+# else
+        if (pRange->pDevIns != MMHyperRCToCC(pVM, pDevIns))
+# endif
+        {
+            AssertMsgFailed(("Not owner! Port=%#x %#x-%#x! (%s)\n", Port, PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
+            iomUnlock(pVM);
+            return VERR_IOM_NOT_IOPORT_RANGE_OWNER;
+        }
+#endif
+        Port = pRange->Core.KeyLast + 1;
+    }
+
+    /* Flush the IO port lookup cache */
+    iomR3FlushCache(pVM);
+
+    /*
+     * Allocate new range record and initialize it.
+     */
+    PIOMIOPORTRANGER0 pRange;
+    int rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
+    if (RT_SUCCESS(rc))
+    {
+        pRange->Core.Key        = PortStart;
+        pRange->Core.KeyLast    = PortLast;
+        pRange->Port            = PortStart;
+        pRange->cPorts          = cPorts;
+        pRange->pvUser          = pvUser;
+        pRange->pfnOutCallback  = pfnOutCallback;
+        pRange->pfnInCallback   = pfnInCallback;
+        pRange->pfnOutStrCallback = pfnOutStrCallback;
+        pRange->pfnInStrCallback = pfnInStrCallback;
+        pRange->pDevIns         = MMHyperR3ToR0(pVM, pDevIns);
+        pRange->pszDesc         = pszDesc;
+
+        /*
+         * Insert it.
+         */
+        if (RTAvlroIOPortInsert(&pVM->iom.s.CTX_SUFF(pTrees)->IOPortTreeR0, &pRange->Core))
+        {
+            iomUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+
+        /* conflict. */
+        AssertMsgFailed(("Port range %#x-%#x (%s) conflicts with existing range(s)!\n", PortStart, (unsigned)PortStart + cPorts - 1, pszDesc));
+        MMHyperFree(pVM, pRange);
+        rc = VERR_IOM_IOPORT_RANGE_CONFLICT;
+    }
+    iomUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Deregisters a I/O Port range.
+ *
+ * The specified range must be registered using IOMR3IOPortRegister previous to
+ * this call. The range does can be a smaller part of the range specified to
+ * IOMR3IOPortRegister, but it can never be larger.
+ *
+ * This function will remove GC, R0 and R3 context port handlers for this range.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The virtual machine.
+ * @param   pDevIns             The device instance associated with the range.
+ * @param   PortStart           First port number in the range.
+ * @param   cPorts              Number of ports to remove starting at PortStart.
+ *
+ * @remark  This function mainly for PCI PnP Config and will not do
+ *          all the checks you might expect it to do.
+ */
+VMMR3DECL(int)  IOMR3IOPortDeregister(PVM pVM, PPDMDEVINS pDevIns, RTIOPORT PortStart, RTUINT cPorts)
+{
+    LogFlow(("IOMR3IOPortDeregister: pDevIns=%p PortStart=%#x cPorts=%#x\n", pDevIns, PortStart, cPorts));
+
+    /*
+     * Validate input.
+     */
+    if (    (RTUINT)PortStart + cPorts < (RTUINT)PortStart
+        ||  (RTUINT)PortStart + cPorts > 0x10000)
+    {
+        AssertMsgFailed(("Invalid port range %#x-%#x!\n", PortStart, (unsigned)PortStart + cPorts - 1));
+        return VERR_IOM_INVALID_IOPORT_RANGE;
+    }
+
+    iomLock(pVM);
+
+    /* Flush the IO port lookup cache */
+    iomR3FlushCache(pVM);
+
+    /*
+     * Check ownership.
+     */
+    RTIOPORT PortLast = PortStart + (cPorts - 1);
+    RTIOPORT Port = PortStart;
+    while (Port <= PortLast && Port >= PortStart)
+    {
+        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeR3, Port);
+        if (pRange)
+        {
+            Assert(Port <= pRange->Core.KeyLast);
+#ifndef IOM_NO_PDMINS_CHECKS
+            if (pRange->pDevIns != pDevIns)
+            {
+                AssertMsgFailed(("Removal of ports in range %#x-%#x rejected because not owner of %#x-%#x (%s)\n",
+                                 PortStart, PortLast, pRange->Core.Key, pRange->Core.KeyLast, pRange->pszDesc));
+                iomUnlock(pVM);
+                return VERR_IOM_NOT_IOPORT_RANGE_OWNER;
+            }
+#endif /* !IOM_NO_PDMINS_CHECKS */
+            Port = pRange->Core.KeyLast;
+        }
+        Port++;
+    }
+
+    /*
+     * Remove any RC ranges first.
+     */
+    int     rc = VINF_SUCCESS;
+    Port = PortStart;
+    while (Port <= PortLast && Port >= PortStart)
+    {
+        /*
+         * Try find range.
+         */
+        PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeRC, Port);
+        if (pRange)
+        {
+            if (   pRange->Core.Key     == Port
+                && pRange->Core.KeyLast <= PortLast)
+            {
+                /*
+                 * Kick out the entire range.
+                 */
+                void *pv = RTAvlroIOPortRemove(&pVM->iom.s.pTreesR3->IOPortTreeRC, Port);
+                Assert(pv == (void *)pRange); NOREF(pv);
+                Port += pRange->cPorts;
+                MMHyperFree(pVM, pRange);
+            }
+            else if (pRange->Core.Key == Port)
+            {
+                /*
+                 * Cut of the head of the range, done.
+                 */
+                pRange->cPorts  -= Port - pRange->Port;
+                pRange->Core.Key = Port;
+                pRange->Port     = Port;
+                break;
+            }
+            else if (pRange->Core.KeyLast <= PortLast)
+            {
+                /*
+                 * Just cut of the tail.
+                 */
+                unsigned c = pRange->Core.KeyLast - Port + 1;
+                pRange->Core.KeyLast -= c;
+                pRange->cPorts -= c;
+                Port += c;
+            }
+            else
+            {
+                /*
+                 * Split the range, done.
+                 */
+                Assert(pRange->Core.KeyLast > PortLast && pRange->Core.Key < Port);
+                /* create tail. */
+                PIOMIOPORTRANGERC pRangeNew;
+                int rc2 = MMHyperAlloc(pVM, sizeof(*pRangeNew), 0, MM_TAG_IOM, (void **)&pRangeNew);
+                if (RT_FAILURE(rc2))
+                {
+                    iomUnlock(pVM);
+                    return rc2;
+                }
+                *pRangeNew = *pRange;
+                pRangeNew->Core.Key     = PortLast;
+                pRangeNew->Port         = PortLast;
+                pRangeNew->cPorts       = pRangeNew->Core.KeyLast - PortLast + 1;
+
+                LogFlow(("IOMR3IOPortDeregister (rc): split the range; new %x\n", pRangeNew->Core.Key));
+
+                /* adjust head */
+                pRange->Core.KeyLast  = Port - 1;
+                pRange->cPorts        = Port - pRange->Port;
+
+                /* insert */
+                if (!RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeRC, &pRangeNew->Core))
+                {
+                    AssertMsgFailed(("This cannot happen!\n"));
+                    MMHyperFree(pVM, pRangeNew);
+                    rc = VERR_INTERNAL_ERROR;
+                }
+                break;
+            }
+        }
+        else /* next port */
+            Port++;
+    } /* for all ports - RC. */
+
+
+    /*
+     * Remove any R0 ranges.
+     */
+    Port = PortStart;
+    while (Port <= PortLast && Port >= PortStart)
+    {
+        /*
+         * Try find range.
+         */
+        PIOMIOPORTRANGER0 pRange = (PIOMIOPORTRANGER0)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeR0, Port);
+        if (pRange)
+        {
+            if (   pRange->Core.Key     == Port
+                && pRange->Core.KeyLast <= PortLast)
+            {
+                /*
+                 * Kick out the entire range.
+                 */
+                void *pv = RTAvlroIOPortRemove(&pVM->iom.s.pTreesR3->IOPortTreeR0, Port);
+                Assert(pv == (void *)pRange); NOREF(pv);
+                Port += pRange->cPorts;
+                MMHyperFree(pVM, pRange);
+            }
+            else if (pRange->Core.Key == Port)
+            {
+                /*
+                 * Cut of the head of the range, done.
+                 */
+                pRange->cPorts  -= Port - pRange->Port;
+                pRange->Core.Key = Port;
+                pRange->Port     = Port;
+                break;
+            }
+            else if (pRange->Core.KeyLast <= PortLast)
+            {
+                /*
+                 * Just cut of the tail.
+                 */
+                unsigned c = pRange->Core.KeyLast - Port + 1;
+                pRange->Core.KeyLast -= c;
+                pRange->cPorts -= c;
+                Port += c;
+            }
+            else
+            {
+                /*
+                 * Split the range, done.
+                 */
+                Assert(pRange->Core.KeyLast > PortLast && pRange->Core.Key < Port);
+                /* create tail. */
+                PIOMIOPORTRANGER0 pRangeNew;
+                int rc2 = MMHyperAlloc(pVM, sizeof(*pRangeNew), 0, MM_TAG_IOM, (void **)&pRangeNew);
+                if (RT_FAILURE(rc2))
+                {
+                    iomUnlock(pVM);
+                    return rc2;
+                }
+                *pRangeNew = *pRange;
+                pRangeNew->Core.Key     = PortLast;
+                pRangeNew->Port         = PortLast;
+                pRangeNew->cPorts       = pRangeNew->Core.KeyLast - PortLast + 1;
+
+                LogFlow(("IOMR3IOPortDeregister (r0): split the range; new %x\n", pRangeNew->Core.Key));
+
+                /* adjust head */
+                pRange->Core.KeyLast  = Port - 1;
+                pRange->cPorts        = Port - pRange->Port;
+
+                /* insert */
+                if (!RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeR0, &pRangeNew->Core))
+                {
+                    AssertMsgFailed(("This cannot happen!\n"));
+                    MMHyperFree(pVM, pRangeNew);
+                    rc = VERR_INTERNAL_ERROR;
+                }
+                break;
+            }
+        }
+        else /* next port */
+            Port++;
+    } /* for all ports - R0. */
+
+    /*
+     * And the same procedure for ring-3 ranges.
+     */
+    Port = PortStart;
+    while (Port <= PortLast && Port >= PortStart)
+    {
+        /*
+         * Try find range.
+         */
+        PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pVM->iom.s.pTreesR3->IOPortTreeR3, Port);
+        if (pRange)
+        {
+            if (   pRange->Core.Key     == Port
+                && pRange->Core.KeyLast <= PortLast)
+            {
+                /*
+                 * Kick out the entire range.
+                 */
+                void *pv = RTAvlroIOPortRemove(&pVM->iom.s.pTreesR3->IOPortTreeR3, Port);
+                Assert(pv == (void *)pRange); NOREF(pv);
+                Port += pRange->cPorts;
+                MMHyperFree(pVM, pRange);
+            }
+            else if (pRange->Core.Key == Port)
+            {
+                /*
+                 * Cut of the head of the range, done.
+                 */
+                pRange->cPorts  -= Port - pRange->Port;
+                pRange->Core.Key = Port;
+                pRange->Port     = Port;
+                break;
+            }
+            else if (pRange->Core.KeyLast <= PortLast)
+            {
+                /*
+                 * Just cut of the tail.
+                 */
+                unsigned c = pRange->Core.KeyLast - Port + 1;
+                pRange->Core.KeyLast -= c;
+                pRange->cPorts -= c;
+                Port += c;
+            }
+            else
+            {
+                /*
+                 * Split the range, done.
+                 */
+                Assert(pRange->Core.KeyLast > PortLast && pRange->Core.Key < Port);
+                /* create tail. */
+                PIOMIOPORTRANGER3 pRangeNew;
+                int rc2 = MMHyperAlloc(pVM, sizeof(*pRangeNew), 0, MM_TAG_IOM, (void **)&pRangeNew);
+                if (RT_FAILURE(rc2))
+                {
+                    iomUnlock(pVM);
+                    return rc2;
+                }
+                *pRangeNew = *pRange;
+                pRangeNew->Core.Key     = PortLast;
+                pRangeNew->Port         = PortLast;
+                pRangeNew->cPorts       = pRangeNew->Core.KeyLast - PortLast + 1;
+
+                LogFlow(("IOMR3IOPortDeregister (r3): split the range; new %x\n", pRangeNew->Core.Key));
+
+                /* adjust head */
+                pRange->Core.KeyLast  = Port - 1;
+                pRange->cPorts        = Port - pRange->Port;
+
+                /* insert */
+                if (!RTAvlroIOPortInsert(&pVM->iom.s.pTreesR3->IOPortTreeR3, &pRangeNew->Core))
+                {
+                    AssertMsgFailed(("This cannot happen!\n"));
+                    MMHyperFree(pVM, pRangeNew);
+                    rc = VERR_INTERNAL_ERROR;
+                }
+                break;
+            }
+        }
+        else /* next port */
+            Port++;
+    } /* for all ports - ring-3. */
+
+    /* done */
+    iomUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Dummy Port I/O Handler for IN operations.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pvUser      User argument.
+ * @param   Port        Port number used for the IN operation.
+ * @param   pu32        Where to store the result.
+ * @param   cb          Number of bytes read.
+ */
+static DECLCALLBACK(int) iomR3IOPortDummyIn(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t *pu32, unsigned cb)
+{
+    switch (cb)
+    {
+        case 1: *pu32 = 0xff; break;
+        case 2: *pu32 = 0xffff; break;
+        case 4: *pu32 = UINT32_C(0xffffffff); break;
+        default:
+            AssertReleaseMsgFailed(("cb=%d\n", cb));
+            return VERR_INTERNAL_ERROR;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dummy Port I/O Handler for string IN operations.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pvUser      User argument.
+ * @param   Port        Port number used for the string IN operation.
+ * @param   pGCPtrDst   Pointer to the destination buffer (GC, incremented appropriately).
+ * @param   pcTransfer  Pointer to the number of transfer units to read, on return remaining transfer units.
+ * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
+ */
+static DECLCALLBACK(int) iomR3IOPortDummyInStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrDst, PRTGCUINTREG pcTransfer, unsigned cb)
+{
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dummy Port I/O Handler for OUT operations.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pvUser      User argument.
+ * @param   Port        Port number used for the OUT operation.
+ * @param   u32         The value to output.
+ * @param   cb          The value size in bytes.
+ */
+static DECLCALLBACK(int) iomR3IOPortDummyOut(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, uint32_t u32, unsigned cb)
+{
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dummy Port I/O Handler for string OUT operations.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pvUser      User argument.
+ * @param   Port        Port number used for the string OUT operation.
+ * @param   pGCPtrSrc   Pointer to the source buffer (GC, incremented appropriately).
+ * @param   pcTransfer  Pointer to the number of transfer units to write, on return remaining transfer units.
+ * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
+ */
+static DECLCALLBACK(int) iomR3IOPortDummyOutStr(PPDMDEVINS pDevIns, void *pvUser, RTIOPORT Port, RTGCPTR *pGCPtrSrc, PRTGCUINTREG pcTransfer, unsigned cb)
+{
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Display a single I/O port ring-3 range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to I/O port HC range.
+ * @param   pvUser  Pointer to info output callback structure.
+ */
+static DECLCALLBACK(int) iomR3IOPortInfoOneR3(PAVLROIOPORTNODECORE pNode, void *pvUser)
+{
+    PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)pNode;
+    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
+    pHlp->pfnPrintf(pHlp,
+                    "%04x-%04x %p %p %p %p %s\n",
+                    pRange->Core.Key,
+                    pRange->Core.KeyLast,
+                    pRange->pDevIns,
+                    pRange->pfnInCallback,
+                    pRange->pfnOutCallback,
+                    pRange->pvUser,
+                    pRange->pszDesc);
+    return 0;
+}
+
+
+/**
+ * Display a single I/O port GC range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to IOPORT GC range.
+ * @param   pvUser  Pointer to info output callback structure.
+ */
+static DECLCALLBACK(int) iomR3IOPortInfoOneRC(PAVLROIOPORTNODECORE pNode, void *pvUser)
+{
+    PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)pNode;
+    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
+    pHlp->pfnPrintf(pHlp,
+                    "%04x-%04x %RRv %RRv %RRv %RRv %s\n",
+                    pRange->Core.Key,
+                    pRange->Core.KeyLast,
+                    pRange->pDevIns,
+                    pRange->pfnInCallback,
+                    pRange->pfnOutCallback,
+                    pRange->pvUser,
+                    pRange->pszDesc);
+    return 0;
+}
+
+
+/**
+ * Display all registered I/O port ranges.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) iomR3IOPortInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    pHlp->pfnPrintf(pHlp,
+                    "I/O Port R3 ranges (pVM=%p)\n"
+                    "Range     %.*s %.*s %.*s %.*s Description\n",
+                    pVM,
+                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
+                    sizeof(RTHCPTR) * 2,      "In              ",
+                    sizeof(RTHCPTR) * 2,      "Out             ",
+                    sizeof(RTHCPTR) * 2,      "pvUser          ");
+    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeR3, true, iomR3IOPortInfoOneR3, (void *)pHlp);
+
+    pHlp->pfnPrintf(pHlp,
+                    "I/O Port R0 ranges (pVM=%p)\n"
+                    "Range     %.*s %.*s %.*s %.*s Description\n",
+                    pVM,
+                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
+                    sizeof(RTHCPTR) * 2,      "In              ",
+                    sizeof(RTHCPTR) * 2,      "Out             ",
+                    sizeof(RTHCPTR) * 2,      "pvUser          ");
+    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeR0, true, iomR3IOPortInfoOneR3, (void *)pHlp);
+
+    pHlp->pfnPrintf(pHlp,
+                    "I/O Port GC ranges (pVM=%p)\n"
+                    "Range     %.*s %.*s %.*s %.*s Description\n",
+                    pVM,
+                    sizeof(RTRCPTR) * 2,      "pDevIns         ",
+                    sizeof(RTRCPTR) * 2,      "In              ",
+                    sizeof(RTRCPTR) * 2,      "Out             ",
+                    sizeof(RTRCPTR) * 2,      "pvUser          ");
+    RTAvlroIOPortDoWithAll(&pVM->iom.s.pTreesR3->IOPortTreeRC, true, iomR3IOPortInfoOneRC, (void *)pHlp);
+
+    if (pVM->iom.s.pRangeLastReadRC)
+    {
+        PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)MMHyperRCToCC(pVM, pVM->iom.s.pRangeLastReadRC);
+        pHlp->pfnPrintf(pHlp, "RC Read  Ports: %#04x-%#04x %RRv %s\n",
+                        pRange->Port, pRange->Port + pRange->cPorts, pVM->iom.s.pRangeLastReadRC, pRange->pszDesc);
+    }
+    if (pVM->iom.s.pStatsLastReadRC)
+    {
+        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperRCToCC(pVM, pVM->iom.s.pStatsLastReadRC);
+        pHlp->pfnPrintf(pHlp, "RC Read  Stats: %#04x %RRv\n",
+                        pRange->Core.Key, pVM->iom.s.pStatsLastReadRC);
+    }
+
+    if (pVM->iom.s.pRangeLastWriteRC)
+    {
+        PIOMIOPORTRANGERC pRange = (PIOMIOPORTRANGERC)MMHyperRCToCC(pVM, pVM->iom.s.pRangeLastWriteRC);
+        pHlp->pfnPrintf(pHlp, "RC Write Ports: %#04x-%#04x %RRv %s\n",
+                        pRange->Port, pRange->Port + pRange->cPorts, pVM->iom.s.pRangeLastWriteRC, pRange->pszDesc);
+    }
+    if (pVM->iom.s.pStatsLastWriteRC)
+    {
+        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperRCToCC(pVM, pVM->iom.s.pStatsLastWriteRC);
+        pHlp->pfnPrintf(pHlp, "RC Write Stats: %#04x %RRv\n",
+                        pRange->Core.Key, pVM->iom.s.pStatsLastWriteRC);
+    }
+
+    if (pVM->iom.s.pRangeLastReadR3)
+    {
+        PIOMIOPORTRANGER3 pRange = pVM->iom.s.pRangeLastReadR3;
+        pHlp->pfnPrintf(pHlp, "R3 Read  Ports: %#04x-%#04x %p %s\n",
+                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
+    }
+    if (pVM->iom.s.pStatsLastReadR3)
+    {
+        PIOMIOPORTSTATS pRange = pVM->iom.s.pStatsLastReadR3;
+        pHlp->pfnPrintf(pHlp, "R3 Read  Stats: %#04x %p\n",
+                        pRange->Core.Key, pRange);
+    }
+
+    if (pVM->iom.s.pRangeLastWriteR3)
+    {
+        PIOMIOPORTRANGER3 pRange = pVM->iom.s.pRangeLastWriteR3;
+        pHlp->pfnPrintf(pHlp, "R3 Write Ports: %#04x-%#04x %p %s\n",
+                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
+    }
+    if (pVM->iom.s.pStatsLastWriteR3)
+    {
+        PIOMIOPORTSTATS pRange = pVM->iom.s.pStatsLastWriteR3;
+        pHlp->pfnPrintf(pHlp, "R3 Write Stats: %#04x %p\n",
+                        pRange->Core.Key, pRange);
+    }
+
+    if (pVM->iom.s.pRangeLastReadR0)
+    {
+        PIOMIOPORTRANGER0 pRange = (PIOMIOPORTRANGER0)MMHyperR0ToCC(pVM, pVM->iom.s.pRangeLastReadR0);
+        pHlp->pfnPrintf(pHlp, "R0 Read  Ports: %#04x-%#04x %p %s\n",
+                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
+    }
+    if (pVM->iom.s.pStatsLastReadR0)
+    {
+        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperR0ToCC(pVM, pVM->iom.s.pStatsLastReadR0);
+        pHlp->pfnPrintf(pHlp, "R0 Read  Stats: %#04x %p\n",
+                        pRange->Core.Key, pRange);
+    }
+
+    if (pVM->iom.s.pRangeLastWriteR0)
+    {
+        PIOMIOPORTRANGER0 pRange = (PIOMIOPORTRANGER0)MMHyperR0ToCC(pVM, pVM->iom.s.pRangeLastWriteR0);
+        pHlp->pfnPrintf(pHlp, "R0 Write Ports: %#04x-%#04x %p %s\n",
+                        pRange->Port, pRange->Port + pRange->cPorts, pRange, pRange->pszDesc);
+    }
+    if (pVM->iom.s.pStatsLastWriteR0)
+    {
+        PIOMIOPORTSTATS pRange = (PIOMIOPORTSTATS)MMHyperR0ToCC(pVM, pVM->iom.s.pStatsLastWriteR0);
+        pHlp->pfnPrintf(pHlp, "R0 Write Stats: %#04x %p\n",
+                        pRange->Core.Key, pRange);
+    }
+}
+
+
+/**
+ * Registers a Memory Mapped I/O R3 handler.
+ *
+ * This API is called by PDM on behalf of a device. Devices must register ring-3 ranges
+ * before any GC and R0 ranges can be registered using IOMR3MMIORegisterRC() and IOMR3MMIORegisterR0().
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 VM handle.
+ * @param   pDevIns             PDM device instance owning the MMIO range.
+ * @param   GCPhysStart         First physical address in the range.
+ * @param   cbRange             The size of the range (in bytes).
+ * @param   pvUser              User argument for the callbacks.
+ * @param   pfnWriteCallback    Pointer to function which is gonna handle Write operations.
+ * @param   pfnReadCallback     Pointer to function which is gonna handle Read operations.
+ * @param   pfnFillCallback     Pointer to function which is gonna handle Fill/memset operations.
+ * @param   pszDesc             Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int)  IOMR3MMIORegisterR3(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTHCPTR pvUser,
+                                    R3PTRTYPE(PFNIOMMMIOWRITE) pfnWriteCallback, R3PTRTYPE(PFNIOMMMIOREAD) pfnReadCallback,
+                                    R3PTRTYPE(PFNIOMMMIOFILL) pfnFillCallback, const char *pszDesc)
+{
+    LogFlow(("IOMR3MMIORegisterR3: pDevIns=%p GCPhysStart=%RGp cbRange=%#x pvUser=%RHv pfnWriteCallback=%#x pfnReadCallback=%#x pfnFillCallback=%#x pszDesc=%s\n",
+             pDevIns, GCPhysStart, cbRange, pvUser, pfnWriteCallback, pfnReadCallback, pfnFillCallback, pszDesc));
+    int rc;
+
+    /*
+     * Validate input.
+     */
+    if (GCPhysStart + (cbRange - 1) < GCPhysStart)
+    {
+        AssertMsgFailed(("Wrapped! %RGp %#x bytes\n", GCPhysStart, cbRange));
+        return VERR_IOM_INVALID_MMIO_RANGE;
+    }
+    /** @todo implement per-device locks for MMIO access. */
+    AssertReturn(!pDevIns->pCritSectR3, VERR_INTERNAL_ERROR_2);
+
+    /*
+     * Resolve the GC/R0 handler addresses lazily because of init order.
+     */
+    if (pVM->iom.s.pfnMMIOHandlerR0 == NIL_RTR0PTR)
+    {
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "IOMMMIOHandler", &pVM->iom.s.pfnMMIOHandlerRC);
+        AssertLogRelRCReturn(rc, rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "IOMMMIOHandler", &pVM->iom.s.pfnMMIOHandlerR0);
+        AssertLogRelRCReturn(rc, rc);
+    }
+
+    /*
+     * For the 2nd+ instance, mangle the description string so it's unique.
+     * (PGM requires this.)
+     */
+    if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a string cache. */
+    {
+        pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_IOM, "%s [%u]", pszDesc, pDevIns->iInstance);
+        if (!pszDesc)
+            return VERR_NO_MEMORY;
+    }
+
+
+    /*
+     * Allocate new range record and initialize it.
+     */
+    PIOMMMIORANGE pRange;
+    rc = MMHyperAlloc(pVM, sizeof(*pRange), 0, MM_TAG_IOM, (void **)&pRange);
+    if (RT_SUCCESS(rc))
+    {
+        pRange->Core.Key            = GCPhysStart;
+        pRange->Core.KeyLast        = GCPhysStart + (cbRange - 1);
+        pRange->GCPhys              = GCPhysStart;
+        pRange->cb                  = cbRange;
+        pRange->pszDesc             = pszDesc;
+
+        pRange->pvUserR3            = pvUser;
+        pRange->pDevInsR3           = pDevIns;
+        pRange->pfnReadCallbackR3   = pfnReadCallback;
+        pRange->pfnWriteCallbackR3  = pfnWriteCallback;
+        pRange->pfnFillCallbackR3   = pfnFillCallback;
+
+        //pRange->pvUserR0            = NIL_RTR0PTR;
+        //pRange->pDevInsR0           = NIL_RTR0PTR;
+        //pRange->pfnReadCallbackR0   = NIL_RTR0PTR;
+        //pRange->pfnWriteCallbackR0  = NIL_RTR0PTR;
+        //pRange->pfnFillCallbackR0   = NIL_RTR0PTR;
+
+        //pRange->pvUserRC            = NIL_RTRCPTR;
+        //pRange->pDevInsRC           = NIL_RTRCPTR;
+        //pRange->pfnReadCallbackRC   = NIL_RTRCPTR;
+        //pRange->pfnWriteCallbackRC  = NIL_RTRCPTR;
+        //pRange->pfnFillCallbackRC   = NIL_RTRCPTR;
+
+        /*
+         * Try register it with PGM and then insert it into the tree.
+         */
+        iomLock(pVM);
+        iomR3FlushCache(pVM);
+        rc = PGMR3PhysMMIORegister(pVM, GCPhysStart, cbRange,
+                                   IOMR3MMIOHandler, pRange,
+                                   pVM->iom.s.pfnMMIOHandlerR0, MMHyperR3ToR0(pVM, pRange),
+                                   pVM->iom.s.pfnMMIOHandlerRC, MMHyperR3ToRC(pVM, pRange), pszDesc);
+        if (RT_SUCCESS(rc))
+        {
+            if (RTAvlroGCPhysInsert(&pVM->iom.s.pTreesR3->MMIOTree, &pRange->Core))
+            {
+                iomUnlock(pVM);
+                return VINF_SUCCESS;
+            }
+
+            /* bail out */
+            iomUnlock(pVM);
+            DBGFR3Info(pVM, "mmio", NULL, NULL);
+            AssertMsgFailed(("This cannot happen!\n"));
+            rc = VERR_INTERNAL_ERROR;
+        }
+        else
+            iomUnlock(pVM);
+
+        MMHyperFree(pVM, pRange);
+    }
+    if (pDevIns->iInstance > 0)
+        MMR3HeapFree((void *)pszDesc);
+    return rc;
+}
+
+
+/**
+ * Registers a Memory Mapped I/O RC handler range.
+ *
+ * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
+ * using IOMMMIORegisterR3() before calling this function.
+ *
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 VM handle.
+ * @param   pDevIns             PDM device instance owning the MMIO range.
+ * @param   GCPhysStart         First physical address in the range.
+ * @param   cbRange             The size of the range (in bytes).
+ * @param   pvUser              User argument for the callbacks.
+ * @param   pfnWriteCallback    Pointer to function which is gonna handle Write operations.
+ * @param   pfnReadCallback     Pointer to function which is gonna handle Read operations.
+ * @param   pfnFillCallback     Pointer to function which is gonna handle Fill/memset operations.
+ */
+VMMR3DECL(int)  IOMR3MMIORegisterRC(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTGCPTR pvUser,
+                                    RCPTRTYPE(PFNIOMMMIOWRITE) pfnWriteCallback, RCPTRTYPE(PFNIOMMMIOREAD) pfnReadCallback,
+                                    RCPTRTYPE(PFNIOMMMIOFILL) pfnFillCallback)
+{
+    LogFlow(("IOMR3MMIORegisterRC: pDevIns=%p GCPhysStart=%RGp cbRange=%#x pvUser=%RGv pfnWriteCallback=%#x pfnReadCallback=%#x pfnFillCallback=%#x\n",
+             pDevIns, GCPhysStart, cbRange, pvUser, pfnWriteCallback, pfnReadCallback, pfnFillCallback));
+
+    /*
+     * Validate input.
+     */
+    if (!pfnWriteCallback && !pfnReadCallback)
+    {
+        AssertMsgFailed(("No callbacks! %RGp LB%#x %s\n", GCPhysStart, cbRange));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Find the MMIO range and check that the input matches.
+     */
+    iomLock(pVM);
+    PIOMMMIORANGE pRange = iomMMIOGetRange(&pVM->iom.s, GCPhysStart);
+    AssertReturnStmt(pRange, iomUnlock(pVM), VERR_IOM_MMIO_RANGE_NOT_FOUND);
+    AssertReturnStmt(pRange->pDevInsR3 == pDevIns, iomUnlock(pVM), VERR_IOM_NOT_MMIO_RANGE_OWNER);
+    AssertReturnStmt(pRange->GCPhys == GCPhysStart, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
+    AssertReturnStmt(pRange->cb == cbRange, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
+
+    pRange->pvUserRC          = pvUser;
+    pRange->pfnReadCallbackRC = pfnReadCallback;
+    pRange->pfnWriteCallbackRC= pfnWriteCallback;
+    pRange->pfnFillCallbackRC = pfnFillCallback;
+    pRange->pDevInsRC         = MMHyperCCToRC(pVM, pDevIns);
+    iomUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers a Memory Mapped I/O R0 handler range.
+ *
+ * This API is called by PDM on behalf of a device. Devices must first register ring-3 ranges
+ * using IOMMR3MIORegisterHC() before calling this function.
+ *
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 VM handle.
+ * @param   pDevIns             PDM device instance owning the MMIO range.
+ * @param   GCPhysStart         First physical address in the range.
+ * @param   cbRange             The size of the range (in bytes).
+ * @param   pvUser              User argument for the callbacks.
+ * @param   pfnWriteCallback    Pointer to function which is gonna handle Write operations.
+ * @param   pfnReadCallback     Pointer to function which is gonna handle Read operations.
+ * @param   pfnFillCallback     Pointer to function which is gonna handle Fill/memset operations.
+ */
+VMMR3DECL(int)  IOMR3MMIORegisterR0(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTR0PTR pvUser,
+                                    R0PTRTYPE(PFNIOMMMIOWRITE) pfnWriteCallback,
+                                    R0PTRTYPE(PFNIOMMMIOREAD) pfnReadCallback,
+                                    R0PTRTYPE(PFNIOMMMIOFILL) pfnFillCallback)
+{
+    LogFlow(("IOMR3MMIORegisterR0: pDevIns=%p GCPhysStart=%RGp cbRange=%#x pvUser=%RHv pfnWriteCallback=%#x pfnReadCallback=%#x pfnFillCallback=%#x\n",
+             pDevIns, GCPhysStart, cbRange, pvUser, pfnWriteCallback, pfnReadCallback, pfnFillCallback));
+
+    /*
+     * Validate input.
+     */
+    if (!pfnWriteCallback && !pfnReadCallback)
+    {
+        AssertMsgFailed(("No callbacks! %RGp LB%#x %s\n", GCPhysStart, cbRange));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Find the MMIO range and check that the input matches.
+     */
+    iomLock(pVM);
+    PIOMMMIORANGE pRange = iomMMIOGetRange(&pVM->iom.s, GCPhysStart);
+    AssertReturnStmt(pRange, iomUnlock(pVM), VERR_IOM_MMIO_RANGE_NOT_FOUND);
+    AssertReturnStmt(pRange->pDevInsR3 == pDevIns, iomUnlock(pVM), VERR_IOM_NOT_MMIO_RANGE_OWNER);
+    AssertReturnStmt(pRange->GCPhys == GCPhysStart, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
+    AssertReturnStmt(pRange->cb == cbRange, iomUnlock(pVM), VERR_IOM_INVALID_MMIO_RANGE);
+
+    pRange->pvUserR0          = pvUser;
+    pRange->pfnReadCallbackR0 = pfnReadCallback;
+    pRange->pfnWriteCallbackR0= pfnWriteCallback;
+    pRange->pfnFillCallbackR0 = pfnFillCallback;
+    pRange->pDevInsR0         = MMHyperCCToR0(pVM, pDevIns);
+    iomUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a Memory Mapped I/O handler range.
+ *
+ * Registered GC, R0, and R3 ranges are affected.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The virtual machine.
+ * @param   pDevIns             Device instance which the MMIO region is registered.
+ * @param   GCPhysStart         First physical address (GC) in the range.
+ * @param   cbRange             Number of bytes to deregister.
+ *
+ * @remark  This function mainly for PCI PnP Config and will not do
+ *          all the checks you might expect it to do.
+ */
+VMMR3DECL(int)  IOMR3MMIODeregister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange)
+{
+    LogFlow(("IOMR3MMIODeregister: pDevIns=%p GCPhysStart=%RGp cbRange=%#x\n", pDevIns, GCPhysStart, cbRange));
+
+    /*
+     * Validate input.
+     */
+    RTGCPHYS GCPhysLast = GCPhysStart + (cbRange - 1);
+    if (GCPhysLast < GCPhysStart)
+    {
+        AssertMsgFailed(("Wrapped! %#x LB%#x\n", GCPhysStart, cbRange));
+        return VERR_IOM_INVALID_MMIO_RANGE;
+    }
+
+    iomLock(pVM);
+
+    /*
+     * Check ownership and such for the entire area.
+     */
+    RTGCPHYS GCPhys = GCPhysStart;
+    while (GCPhys <= GCPhysLast && GCPhys >= GCPhysStart)
+    {
+        PIOMMMIORANGE pRange = iomMMIOGetRange(&pVM->iom.s, GCPhys);
+        if (!pRange)
+        {
+            iomUnlock(pVM);
+            return VERR_IOM_MMIO_RANGE_NOT_FOUND;
+        }
+        AssertMsgReturnStmt(pRange->pDevInsR3 == pDevIns,
+                            ("Not owner! GCPhys=%RGp %RGp LB%#x %s\n", GCPhys, GCPhysStart, cbRange, pRange->pszDesc),
+                            iomUnlock(pVM),
+                            VERR_IOM_NOT_MMIO_RANGE_OWNER);
+        AssertMsgReturnStmt(pRange->Core.KeyLast <= GCPhysLast,
+                            ("Incomplete R3 range! GCPhys=%RGp %RGp LB%#x %s\n", GCPhys, GCPhysStart, cbRange, pRange->pszDesc),
+                            iomUnlock(pVM),
+                            VERR_IOM_INCOMPLETE_MMIO_RANGE);
+
+        /* next */
+        Assert(GCPhys <= pRange->Core.KeyLast);
+        GCPhys = pRange->Core.KeyLast + 1;
+    }
+
+    /*
+     * Do the actual removing of the MMIO ranges.
+     */
+    GCPhys = GCPhysStart;
+    while (GCPhys <= GCPhysLast && GCPhys >= GCPhysStart)
+    {
+        iomR3FlushCache(pVM);
+
+        PIOMMMIORANGE pRange = (PIOMMMIORANGE)RTAvlroGCPhysRemove(&pVM->iom.s.pTreesR3->MMIOTree, GCPhys);
+        Assert(pRange);
+        Assert(pRange->Core.Key == GCPhys && pRange->Core.KeyLast <= GCPhysLast);
+        iomUnlock(pVM);
+
+        /* remove it from PGM */
+        int rc = PGMR3PhysMMIODeregister(pVM, GCPhys, pRange->cb);
+        AssertRC(rc);
+
+        iomLock(pVM);
+
+        /* advance and free. */
+        GCPhys = pRange->Core.KeyLast + 1;
+        if (pDevIns->iInstance > 0)
+            MMR3HeapFree((void *)pRange->pszDesc);
+        MMHyperFree(pVM, pRange);
+    }
+
+    iomUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * For TM only!
+ *
+ * @returns Pointer to the critical section.
+ * @param   pVM                 The VM handle.
+ */
+VMMR3DECL(PPDMCRITSECT) IOMR3GetCritSect(PVM pVM)
+{
+    return &pVM->iom.s.EmtLock;
+}
+
+
+/**
+ * Display a single MMIO range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to MMIO R3 range.
+ * @param   pvUser  Pointer to info output callback structure.
+ */
+static DECLCALLBACK(int) iomR3MMIOInfoOne(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PIOMMMIORANGE pRange = (PIOMMMIORANGE)pNode;
+    PCDBGFINFOHLP pHlp = (PCDBGFINFOHLP)pvUser;
+    pHlp->pfnPrintf(pHlp,
+                    "%RGp-%RGp %RHv %RHv %RHv %RHv %RHv %s\n",
+                    pRange->Core.Key,
+                    pRange->Core.KeyLast,
+                    pRange->pDevInsR3,
+                    pRange->pfnReadCallbackR3,
+                    pRange->pfnWriteCallbackR3,
+                    pRange->pfnFillCallbackR3,
+                    pRange->pvUserR3,
+                    pRange->pszDesc);
+    pHlp->pfnPrintf(pHlp,
+                    "%*s %RHv %RHv %RHv %RHv %RHv\n",
+                    sizeof(RTGCPHYS) * 2 * 2 + 1, "R0",
+                    pRange->pDevInsR0,
+                    pRange->pfnReadCallbackR0,
+                    pRange->pfnWriteCallbackR0,
+                    pRange->pfnFillCallbackR0,
+                    pRange->pvUserR0);
+    pHlp->pfnPrintf(pHlp,
+                    "%*s %RRv %RRv %RRv %RRv %RRv\n",
+                    sizeof(RTGCPHYS) * 2 * 2 + 1, "RC",
+                    pRange->pDevInsRC,
+                    pRange->pfnReadCallbackRC,
+                    pRange->pfnWriteCallbackRC,
+                    pRange->pfnFillCallbackRC,
+                    pRange->pvUserRC);
+    return 0;
+}
+
+
+/**
+ * Display registered MMIO ranges to the log.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) iomR3MMIOInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    pHlp->pfnPrintf(pHlp,
+                    "MMIO ranges (pVM=%p)\n"
+                    "%.*s %.*s %.*s %.*s %.*s %.*s %s\n",
+                    pVM,
+                    sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range                    ",
+                    sizeof(RTHCPTR) * 2,      "pDevIns         ",
+                    sizeof(RTHCPTR) * 2,      "Read            ",
+                    sizeof(RTHCPTR) * 2,      "Write           ",
+                    sizeof(RTHCPTR) * 2,      "Fill            ",
+                    sizeof(RTHCPTR) * 2,      "pvUser          ",
+                                              "Description");
+    RTAvlroGCPhysDoWithAll(&pVM->iom.s.pTreesR3->MMIOTree, true, iomR3MMIOInfoOne, (void *)pHlp);
+}
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Tries to come up with the standard name for a port.
+ *
+ * @returns Pointer to readonly string if known.
+ * @returns NULL if unknown port number.
+ *
+ * @param   Port    The port to name.
+ */
+static const char *iomR3IOPortGetStandardName(RTIOPORT Port)
+{
+    switch (Port)
+    {
+        case 0x00: case 0x10: case 0x20: case 0x30: case 0x40: case 0x50:            case 0x70:
+        case 0x01: case 0x11: case 0x21: case 0x31: case 0x41: case 0x51: case 0x61: case 0x71:
+        case 0x02: case 0x12: case 0x22: case 0x32: case 0x42: case 0x52: case 0x62: case 0x72:
+        case 0x03: case 0x13: case 0x23: case 0x33: case 0x43: case 0x53: case 0x63: case 0x73:
+        case 0x04: case 0x14: case 0x24: case 0x34: case 0x44: case 0x54:            case 0x74:
+        case 0x05: case 0x15: case 0x25: case 0x35: case 0x45: case 0x55: case 0x65: case 0x75:
+        case 0x06: case 0x16: case 0x26: case 0x36: case 0x46: case 0x56: case 0x66: case 0x76:
+        case 0x07: case 0x17: case 0x27: case 0x37: case 0x47: case 0x57: case 0x67: case 0x77:
+        case 0x08: case 0x18: case 0x28: case 0x38: case 0x48: case 0x58: case 0x68: case 0x78:
+        case 0x09: case 0x19: case 0x29: case 0x39: case 0x49: case 0x59: case 0x69: case 0x79:
+        case 0x0a: case 0x1a: case 0x2a: case 0x3a: case 0x4a: case 0x5a: case 0x6a: case 0x7a:
+        case 0x0b: case 0x1b: case 0x2b: case 0x3b: case 0x4b: case 0x5b: case 0x6b: case 0x7b:
+        case 0x0c: case 0x1c: case 0x2c: case 0x3c: case 0x4c: case 0x5c: case 0x6c: case 0x7c:
+        case 0x0d: case 0x1d: case 0x2d: case 0x3d: case 0x4d: case 0x5d: case 0x6d: case 0x7d:
+        case 0x0e: case 0x1e: case 0x2e: case 0x3e: case 0x4e: case 0x5e: case 0x6e: case 0x7e:
+        case 0x0f: case 0x1f: case 0x2f: case 0x3f: case 0x4f: case 0x5f: case 0x6f: case 0x7f:
+
+        case 0x80: case 0x90: case 0xa0: case 0xb0: case 0xc0: case 0xd0: case 0xe0: case 0xf0:
+        case 0x81: case 0x91: case 0xa1: case 0xb1: case 0xc1: case 0xd1: case 0xe1: case 0xf1:
+        case 0x82: case 0x92: case 0xa2: case 0xb2: case 0xc2: case 0xd2: case 0xe2: case 0xf2:
+        case 0x83: case 0x93: case 0xa3: case 0xb3: case 0xc3: case 0xd3: case 0xe3: case 0xf3:
+        case 0x84: case 0x94: case 0xa4: case 0xb4: case 0xc4: case 0xd4: case 0xe4: case 0xf4:
+        case 0x85: case 0x95: case 0xa5: case 0xb5: case 0xc5: case 0xd5: case 0xe5: case 0xf5:
+        case 0x86: case 0x96: case 0xa6: case 0xb6: case 0xc6: case 0xd6: case 0xe6: case 0xf6:
+        case 0x87: case 0x97: case 0xa7: case 0xb7: case 0xc7: case 0xd7: case 0xe7: case 0xf7:
+        case 0x88: case 0x98: case 0xa8: case 0xb8: case 0xc8: case 0xd8: case 0xe8: case 0xf8:
+        case 0x89: case 0x99: case 0xa9: case 0xb9: case 0xc9: case 0xd9: case 0xe9: case 0xf9:
+        case 0x8a: case 0x9a: case 0xaa: case 0xba: case 0xca: case 0xda: case 0xea: case 0xfa:
+        case 0x8b: case 0x9b: case 0xab: case 0xbb: case 0xcb: case 0xdb: case 0xeb: case 0xfb:
+        case 0x8c: case 0x9c: case 0xac: case 0xbc: case 0xcc: case 0xdc: case 0xec: case 0xfc:
+        case 0x8d: case 0x9d: case 0xad: case 0xbd: case 0xcd: case 0xdd: case 0xed: case 0xfd:
+        case 0x8e: case 0x9e: case 0xae: case 0xbe: case 0xce: case 0xde: case 0xee: case 0xfe:
+        case 0x8f: case 0x9f: case 0xaf: case 0xbf: case 0xcf: case 0xdf: case 0xef: case 0xff:
+            return "System Reserved";
+
+        case 0x60:
+        case 0x64:
+            return "Keyboard & Mouse";
+
+        case 0x378:
+        case 0x379:
+        case 0x37a:
+        case 0x37b:
+        case 0x37c:
+        case 0x37d:
+        case 0x37e:
+        case 0x37f:
+        case 0x3bc:
+        case 0x3bd:
+        case 0x3be:
+        case 0x3bf:
+        case 0x278:
+        case 0x279:
+        case 0x27a:
+        case 0x27b:
+        case 0x27c:
+        case 0x27d:
+        case 0x27e:
+        case 0x27f:
+            return "LPT1/2/3";
+
+        case 0x3f8:
+        case 0x3f9:
+        case 0x3fa:
+        case 0x3fb:
+        case 0x3fc:
+        case 0x3fd:
+        case 0x3fe:
+        case 0x3ff:
+            return "COM1";
+
+        case 0x2f8:
+        case 0x2f9:
+        case 0x2fa:
+        case 0x2fb:
+        case 0x2fc:
+        case 0x2fd:
+        case 0x2fe:
+        case 0x2ff:
+            return "COM2";
+
+        case 0x3e8:
+        case 0x3e9:
+        case 0x3ea:
+        case 0x3eb:
+        case 0x3ec:
+        case 0x3ed:
+        case 0x3ee:
+        case 0x3ef:
+            return "COM3";
+
+        case 0x2e8:
+        case 0x2e9:
+        case 0x2ea:
+        case 0x2eb:
+        case 0x2ec:
+        case 0x2ed:
+        case 0x2ee:
+        case 0x2ef:
+            return "COM4";
+
+        case 0x200:
+        case 0x201:
+        case 0x202:
+        case 0x203:
+        case 0x204:
+        case 0x205:
+        case 0x206:
+        case 0x207:
+            return "Joystick";
+
+        case 0x3f0:
+        case 0x3f1:
+        case 0x3f2:
+        case 0x3f3:
+        case 0x3f4:
+        case 0x3f5:
+        case 0x3f6:
+        case 0x3f7:
+            return "Floppy";
+
+        case 0x1f0:
+        case 0x1f1:
+        case 0x1f2:
+        case 0x1f3:
+        case 0x1f4:
+        case 0x1f5:
+        case 0x1f6:
+        case 0x1f7:
+        //case 0x3f6:
+        //case 0x3f7:
+            return "IDE 1st";
+
+        case 0x170:
+        case 0x171:
+        case 0x172:
+        case 0x173:
+        case 0x174:
+        case 0x175:
+        case 0x176:
+        case 0x177:
+        case 0x376:
+        case 0x377:
+            return "IDE 2nd";
+
+        case 0x1e0:
+        case 0x1e1:
+        case 0x1e2:
+        case 0x1e3:
+        case 0x1e4:
+        case 0x1e5:
+        case 0x1e6:
+        case 0x1e7:
+        case 0x3e6:
+        case 0x3e7:
+            return "IDE 3rd";
+
+        case 0x160:
+        case 0x161:
+        case 0x162:
+        case 0x163:
+        case 0x164:
+        case 0x165:
+        case 0x166:
+        case 0x167:
+        case 0x366:
+        case 0x367:
+            return "IDE 4th";
+
+        case 0x130: case 0x140: case 0x150:
+        case 0x131: case 0x141: case 0x151:
+        case 0x132: case 0x142: case 0x152:
+        case 0x133: case 0x143: case 0x153:
+        case 0x134: case 0x144: case 0x154:
+        case 0x135: case 0x145: case 0x155:
+        case 0x136: case 0x146: case 0x156:
+        case 0x137: case 0x147: case 0x157:
+        case 0x138: case 0x148: case 0x158:
+        case 0x139: case 0x149: case 0x159:
+        case 0x13a: case 0x14a: case 0x15a:
+        case 0x13b: case 0x14b: case 0x15b:
+        case 0x13c: case 0x14c: case 0x15c:
+        case 0x13d: case 0x14d: case 0x15d:
+        case 0x13e: case 0x14e: case 0x15e:
+        case 0x13f: case 0x14f: case 0x15f:
+        case 0x220: case 0x230:
+        case 0x221: case 0x231:
+        case 0x222: case 0x232:
+        case 0x223: case 0x233:
+        case 0x224: case 0x234:
+        case 0x225: case 0x235:
+        case 0x226: case 0x236:
+        case 0x227: case 0x237:
+        case 0x228: case 0x238:
+        case 0x229: case 0x239:
+        case 0x22a: case 0x23a:
+        case 0x22b: case 0x23b:
+        case 0x22c: case 0x23c:
+        case 0x22d: case 0x23d:
+        case 0x22e: case 0x23e:
+        case 0x22f: case 0x23f:
+        case 0x330: case 0x340: case 0x350:
+        case 0x331: case 0x341: case 0x351:
+        case 0x332: case 0x342: case 0x352:
+        case 0x333: case 0x343: case 0x353:
+        case 0x334: case 0x344: case 0x354:
+        case 0x335: case 0x345: case 0x355:
+        case 0x336: case 0x346: case 0x356:
+        case 0x337: case 0x347: case 0x357:
+        case 0x338: case 0x348: case 0x358:
+        case 0x339: case 0x349: case 0x359:
+        case 0x33a: case 0x34a: case 0x35a:
+        case 0x33b: case 0x34b: case 0x35b:
+        case 0x33c: case 0x34c: case 0x35c:
+        case 0x33d: case 0x34d: case 0x35d:
+        case 0x33e: case 0x34e: case 0x35e:
+        case 0x33f: case 0x34f: case 0x35f:
+            return "SCSI (typically)";
+
+        case 0x320:
+        case 0x321:
+        case 0x322:
+        case 0x323:
+        case 0x324:
+        case 0x325:
+        case 0x326:
+        case 0x327:
+            return "XT HD";
+
+        case 0x3b0:
+        case 0x3b1:
+        case 0x3b2:
+        case 0x3b3:
+        case 0x3b4:
+        case 0x3b5:
+        case 0x3b6:
+        case 0x3b7:
+        case 0x3b8:
+        case 0x3b9:
+        case 0x3ba:
+        case 0x3bb:
+            return "VGA";
+
+        case 0x3c0: case 0x3d0:
+        case 0x3c1: case 0x3d1:
+        case 0x3c2: case 0x3d2:
+        case 0x3c3: case 0x3d3:
+        case 0x3c4: case 0x3d4:
+        case 0x3c5: case 0x3d5:
+        case 0x3c6: case 0x3d6:
+        case 0x3c7: case 0x3d7:
+        case 0x3c8: case 0x3d8:
+        case 0x3c9: case 0x3d9:
+        case 0x3ca: case 0x3da:
+        case 0x3cb: case 0x3db:
+        case 0x3cc: case 0x3dc:
+        case 0x3cd: case 0x3dd:
+        case 0x3ce: case 0x3de:
+        case 0x3cf: case 0x3df:
+            return "VGA/EGA";
+
+        case 0x240: case 0x260: case 0x280:
+        case 0x241: case 0x261: case 0x281:
+        case 0x242: case 0x262: case 0x282:
+        case 0x243: case 0x263: case 0x283:
+        case 0x244: case 0x264: case 0x284:
+        case 0x245: case 0x265: case 0x285:
+        case 0x246: case 0x266: case 0x286:
+        case 0x247: case 0x267: case 0x287:
+        case 0x248: case 0x268: case 0x288:
+        case 0x249: case 0x269: case 0x289:
+        case 0x24a: case 0x26a: case 0x28a:
+        case 0x24b: case 0x26b: case 0x28b:
+        case 0x24c: case 0x26c: case 0x28c:
+        case 0x24d: case 0x26d: case 0x28d:
+        case 0x24e: case 0x26e: case 0x28e:
+        case 0x24f: case 0x26f: case 0x28f:
+        case 0x300:
+        case 0x301:
+        case 0x388:
+        case 0x389:
+        case 0x38a:
+        case 0x38b:
+            return "Sound Card (typically)";
+
+        default:
+            return NULL;
+    }
+}
+#endif /* VBOX_WITH_STATISTICS */
+
Index: /trunk/src/VBox/VMM/VMMR3/MM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/MM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/MM.cpp	(revision 35333)
@@ -0,0 +1,804 @@
+/* $Id$ */
+/** @file
+ * MM - Memory Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_mm     MM - The Memory Manager
+ *
+ * The memory manager is in charge of the following memory:
+ *      - Hypervisor Memory Area (HMA) - Address space management.
+ *      - Hypervisor Heap - A memory heap that lives in all contexts.
+ *      - Tagged ring-3 heap.
+ *      - Page pools - Primarily used by PGM for shadow page tables.
+ *      - Locked process memory - Guest RAM and other. (reduce/obsolete this)
+ *      - Physical guest memory (RAM & ROM) - Moving to PGM. (obsolete this)
+ *
+ * The global memory manager (GMM) is the global counter part / partner of MM.
+ * MM will provide therefore ring-3 callable interfaces for some of the GMM APIs
+ * related to resource tracking (PGM is the user).
+ *
+ * @see grp_mm
+ *
+ *
+ * @section sec_mm_hma  Hypervisor Memory Area
+ *
+ * The HMA is used when executing in raw-mode. We borrow, with the help of
+ * PGMMap, some unused space (one or more page directory entries to be precise)
+ * in the guest's virtual memory context. PGM will monitor the guest's virtual
+ * address space for changes and relocate the HMA when required.
+ *
+ * To give some idea what's in the HMA, study the 'info hma' output:
+ * @verbatim
+VBoxDbg> info hma
+Hypervisor Memory Area (HMA) Layout: Base 00000000a0000000, 0x00800000 bytes
+00000000a05cc000-00000000a05cd000                  DYNAMIC                  fence
+00000000a05c4000-00000000a05cc000                  DYNAMIC                  Dynamic mapping
+00000000a05c3000-00000000a05c4000                  DYNAMIC                  fence
+00000000a05b8000-00000000a05c3000                  DYNAMIC                  Paging
+00000000a05b6000-00000000a05b8000                  MMIO2   0000000000000000 PCNetShMem
+00000000a0536000-00000000a05b6000                  MMIO2   0000000000000000 VGA VRam
+00000000a0523000-00000000a0536000 00002aaab3d0c000 LOCKED  autofree         alloc once (PDM_DEVICE)
+00000000a0522000-00000000a0523000                  DYNAMIC                  fence
+00000000a051e000-00000000a0522000 00002aaab36f5000 LOCKED  autofree         VBoxDD2GC.gc
+00000000a051d000-00000000a051e000                  DYNAMIC                  fence
+00000000a04eb000-00000000a051d000 00002aaab36c3000 LOCKED  autofree         VBoxDDGC.gc
+00000000a04ea000-00000000a04eb000                  DYNAMIC                  fence
+00000000a04e9000-00000000a04ea000 00002aaab36c2000 LOCKED  autofree         ram range (High ROM Region)
+00000000a04e8000-00000000a04e9000                  DYNAMIC                  fence
+00000000a040e000-00000000a04e8000 00002aaab2e6d000 LOCKED  autofree         VMMGC.gc
+00000000a0208000-00000000a040e000 00002aaab2c67000 LOCKED  autofree         alloc once (PATM)
+00000000a01f7000-00000000a0208000 00002aaaab92d000 LOCKED  autofree         alloc once (SELM)
+00000000a01e7000-00000000a01f7000 00002aaaab5e8000 LOCKED  autofree         alloc once (SELM)
+00000000a01e6000-00000000a01e7000                  DYNAMIC                  fence
+00000000a01e5000-00000000a01e6000 00002aaaab5e7000 HCPHYS  00000000c363c000 Core Code
+00000000a01e4000-00000000a01e5000                  DYNAMIC                  fence
+00000000a01e3000-00000000a01e4000 00002aaaaab26000 HCPHYS  00000000619cf000 GIP
+00000000a01a2000-00000000a01e3000 00002aaaabf32000 LOCKED  autofree         alloc once (PGM_PHYS)
+00000000a016b000-00000000a01a2000 00002aaab233f000 LOCKED  autofree         alloc once (PGM_POOL)
+00000000a016a000-00000000a016b000                  DYNAMIC                  fence
+00000000a0165000-00000000a016a000                  DYNAMIC                  CR3 mapping
+00000000a0164000-00000000a0165000                  DYNAMIC                  fence
+00000000a0024000-00000000a0164000 00002aaab215f000 LOCKED  autofree         Heap
+00000000a0023000-00000000a0024000                  DYNAMIC                  fence
+00000000a0001000-00000000a0023000 00002aaab1d24000 LOCKED  pages            VM
+00000000a0000000-00000000a0001000                  DYNAMIC                  fence
+ @endverbatim
+ *
+ *
+ * @section sec_mm_hyperheap    Hypervisor Heap
+ *
+ * The heap is accessible from ring-3, ring-0 and the raw-mode context. That
+ * said, it's not necessarily mapped into ring-0 on if that's possible since we
+ * don't wish to waste kernel address space without a good reason.
+ *
+ * Allocations within the heap are always in the same relative position in all
+ * contexts, so, it's possible to use offset based linking. In fact, the heap is
+ * internally using offset based linked lists tracking heap blocks. We use
+ * offset linked AVL trees and lists in a lot of places where share structures
+ * between RC, R3 and R0, so this is a strict requirement of the heap. However
+ * this means that we cannot easily extend the heap since the extension won't
+ * necessarily be in the continuation of the current heap memory in all (or any)
+ * context.
+ *
+ * All allocations are tagged. Per tag allocation statistics will be maintaining
+ * and exposed thru STAM when VBOX_WITH_STATISTICS is defined.
+ *
+ *
+ * @section sec_mm_r3heap   Tagged Ring-3 Heap
+ *
+ * The ring-3 heap is a wrapper around the RTMem API adding allocation
+ * statistics and automatic cleanup on VM destruction.
+ *
+ * Per tag allocation statistics will be maintaining and exposed thru STAM when
+ * VBOX_WITH_STATISTICS is defined.
+ *
+ *
+ * @section sec_mm_page     Page Pool
+ *
+ * The MM manages a page pool from which other components can allocate locked,
+ * page aligned and page sized memory objects. The pool provides facilities to
+ * convert back and forth between (host) physical and virtual addresses (within
+ * the pool of course). Several specialized interfaces are provided for the most
+ * common allocations and conversions to save the caller from bothersome casting
+ * and extra parameter passing.
+ *
+ *
+ * @section sec_mm_locked   Locked Process Memory
+ *
+ * MM manages the locked process memory. This is used for a bunch of things
+ * (count the LOCKED entries in the 'info hma' output found in @ref sec_mm_hma),
+ * but the main consumer of memory is currently for guest RAM. There is an
+ * ongoing rewrite that will move all the guest RAM allocation to PGM and
+ * GMM.
+ *
+ * The locking of memory is something doing in cooperation with the VirtualBox
+ * support driver, SUPDrv (aka. VBoxDrv), thru the support library API,
+ * SUPR3 (aka. SUPLib).
+ *
+ *
+ * @section sec_mm_phys     Physical Guest Memory
+ *
+ * MM is currently managing the physical memory for the guest. It relies heavily
+ * on PGM for this. There is an ongoing rewrite that will move this to PGM. (The
+ * rewrite is driven by the need for more flexible guest ram allocation, but
+ * also motivated by the fact that MMPhys is just adding stupid bureaucracy and
+ * that MMR3PhysReserve is a totally weird artifact that must go away.)
+ *
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM
+#include <VBox/mm.h>
+#include <VBox/pgm.h>
+#include <VBox/cfgm.h>
+#include <VBox/ssm.h>
+#include <VBox/gmm.h>
+#include "MMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <VBox/log.h>
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The current saved state version of MM. */
+#define MM_SAVED_STATE_VERSION      2
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) mmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) mmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+
+
+
+
+/**
+ * Initializes the MM members of the UVM.
+ *
+ * This is currently only the ring-3 heap.
+ *
+ * @returns VBox status code.
+ * @param   pUVM    Pointer to the user mode VM structure.
+ */
+VMMR3DECL(int) MMR3InitUVM(PUVM pUVM)
+{
+    /*
+     * Assert sizes and order.
+     */
+    AssertCompile(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
+    AssertRelease(sizeof(pUVM->mm.s) <= sizeof(pUVM->mm.padding));
+    Assert(!pUVM->mm.s.pHeap);
+
+    /*
+     * Init the heap.
+     */
+    int rc = mmR3HeapCreateU(pUVM, &pUVM->mm.s.pHeap);
+    if (RT_SUCCESS(rc))
+    {
+        rc = mmR3UkHeapCreateU(pUVM, &pUVM->mm.s.pUkHeap);
+        if (RT_SUCCESS(rc))
+            return VINF_SUCCESS;
+        mmR3HeapDestroy(pUVM->mm.s.pHeap);
+        pUVM->mm.s.pHeap = NULL;
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes the MM.
+ *
+ * MM is managing the virtual address space (among other things) and
+ * setup the hypervisor memory area mapping in the VM structure and
+ * the hypervisor alloc-only-heap. Assuming the current init order
+ * and components the hypervisor memory area looks like this:
+ *      -# VM Structure.
+ *      -# Hypervisor alloc only heap (also call Hypervisor memory region).
+ *      -# Core code.
+ *
+ * MM determines the virtual address of the hypervisor memory area by
+ * checking for location at previous run. If that property isn't available
+ * it will choose a default starting location, currently 0xa0000000.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) MMR3Init(PVM pVM)
+{
+    LogFlow(("MMR3Init\n"));
+
+    /*
+     * Assert alignment, sizes and order.
+     */
+    AssertRelease(!(RT_OFFSETOF(VM, mm.s) & 31));
+    AssertRelease(sizeof(pVM->mm.s) <= sizeof(pVM->mm.padding));
+    AssertMsg(pVM->mm.s.offVM == 0, ("Already initialized!\n"));
+
+    /*
+     * Init the structure.
+     */
+    pVM->mm.s.offVM = RT_OFFSETOF(VM, mm);
+    pVM->mm.s.offLookupHyper = NIL_OFFSET;
+
+    /*
+     * Init the page pool.
+     */
+    int rc = mmR3PagePoolInit(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Init the hypervisor related stuff.
+         */
+        rc = mmR3HyperInit(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Register the saved state data unit.
+             */
+            rc = SSMR3RegisterInternal(pVM, "mm", 1, MM_SAVED_STATE_VERSION, sizeof(uint32_t) * 2,
+                                       NULL, NULL, NULL,
+                                       NULL, mmR3Save, NULL,
+                                       NULL, mmR3Load, NULL);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Statistics.
+                 */
+                STAM_REG(pVM, &pVM->mm.s.cBasePages,   STAMTYPE_U64, "/MM/Reserved/cBasePages",   STAMUNIT_PAGES, "Reserved number of base pages, ROM and Shadow ROM included.");
+                STAM_REG(pVM, &pVM->mm.s.cHandyPages,  STAMTYPE_U32, "/MM/Reserved/cHandyPages",  STAMUNIT_PAGES, "Reserved number of handy pages.");
+                STAM_REG(pVM, &pVM->mm.s.cShadowPages, STAMTYPE_U32, "/MM/Reserved/cShadowPages", STAMUNIT_PAGES, "Reserved number of shadow paging pages.");
+                STAM_REG(pVM, &pVM->mm.s.cFixedPages,  STAMTYPE_U32, "/MM/Reserved/cFixedPages",  STAMUNIT_PAGES, "Reserved number of fixed pages (MMIO2).");
+                STAM_REG(pVM, &pVM->mm.s.cbRamBase,    STAMTYPE_U64, "/MM/cbRamBase",             STAMUNIT_BYTES, "Size of the base RAM.");
+
+                return rc;
+            }
+
+            /* .... failure .... */
+        }
+    }
+    MMR3Term(pVM);
+    return rc;
+}
+
+
+/**
+ * Initializes the MM parts which depends on PGM being initialized.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @remark  No cleanup necessary since MMR3Term() will be called on failure.
+ */
+VMMR3DECL(int) MMR3InitPaging(PVM pVM)
+{
+    LogFlow(("MMR3InitPaging:\n"));
+
+    /*
+     * Query the CFGM values.
+     */
+    int rc;
+    PCFGMNODE pMMCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "MM");
+    if (!pMMCfg)
+    {
+        rc = CFGMR3InsertNode(CFGMR3GetRoot(pVM), "MM", &pMMCfg);
+        AssertRCReturn(rc, rc);
+    }
+
+    /** @cfgm{RamSize, uint64_t, 0, 16TB, 0}
+     * Specifies the size of the base RAM that is to be set up during
+     * VM initialization.
+     */
+    uint64_t cbRam;
+    rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        cbRam = 0;
+    else
+        AssertMsgRCReturn(rc, ("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc), rc);
+    AssertLogRelMsg(!(cbRam & ~X86_PTE_PAE_PG_MASK), ("%RGp X86_PTE_PAE_PG_MASK=%RX64\n", cbRam, X86_PTE_PAE_PG_MASK));
+    AssertLogRelMsgReturn(cbRam <= GMM_GCPHYS_LAST, ("cbRam=%RGp GMM_GCPHYS_LAST=%RX64\n", cbRam, GMM_GCPHYS_LAST), VERR_OUT_OF_RANGE);
+    cbRam &= X86_PTE_PAE_PG_MASK;
+    pVM->mm.s.cbRamBase = cbRam;
+
+    /** @cfgm{RamHoleSize, uint32_t, 0, 4032MB, 512MB}
+     * Specifies the size of the memory hole. The memory hole is used
+     * to avoid mapping RAM to the range normally used for PCI memory regions.
+     * Must be aligned on a 4MB boundary. */
+    uint32_t cbRamHole;
+    rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
+    AssertLogRelMsgRCReturn(rc, ("Configuration error: Failed to query integer \"RamHoleSize\", rc=%Rrc.\n", rc), rc);
+    AssertLogRelMsgReturn(cbRamHole <= 4032U * _1M,
+                          ("Configuration error: \"RamHoleSize\"=%#RX32 is too large.\n", cbRamHole), VERR_OUT_OF_RANGE);
+    AssertLogRelMsgReturn(cbRamHole > 16 * _1M,
+                          ("Configuration error: \"RamHoleSize\"=%#RX32 is too large.\n", cbRamHole), VERR_OUT_OF_RANGE);
+    AssertLogRelMsgReturn(!(cbRamHole & (_4M - 1)),
+                          ("Configuration error: \"RamHoleSize\"=%#RX32 is misaligned.\n", cbRamHole), VERR_OUT_OF_RANGE);
+    uint64_t const offRamHole = _4G - cbRamHole;
+    if (cbRam < offRamHole)
+        Log(("MM: %RU64 bytes of RAM\n", cbRam));
+    else
+        Log(("MM: %RU64 bytes of RAM with a hole at %RU64 up to 4GB.\n", cbRam, offRamHole));
+
+    /** @cfgm{MM/Policy, string, no overcommitment}
+     * Specifies the policy to use when reserving memory for this VM. The recognized
+     * value is 'no overcommitment' (default). See GMMPOLICY.
+     */
+    GMMOCPOLICY enmOcPolicy;
+    char sz[64];
+    rc = CFGMR3QueryString(CFGMR3GetRoot(pVM), "Policy", sz, sizeof(sz));
+    if (RT_SUCCESS(rc))
+    {
+        if (    !RTStrICmp(sz, "no_oc")
+            ||  !RTStrICmp(sz, "no overcommitment"))
+            enmOcPolicy = GMMOCPOLICY_NO_OC;
+        else
+            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, "Unknown \"MM/Policy\" value \"%s\"", sz);
+    }
+    else if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        enmOcPolicy = GMMOCPOLICY_NO_OC;
+    else
+        AssertMsgFailedReturn(("Configuration error: Failed to query string \"MM/Policy\", rc=%Rrc.\n", rc), rc);
+
+    /** @cfgm{MM/Priority, string, normal}
+     * Specifies the memory priority of this VM. The priority comes into play when the
+     * system is overcommitted and the VMs needs to be milked for memory. The recognized
+     * values are 'low', 'normal' (default) and 'high'. See GMMPRIORITY.
+     */
+    GMMPRIORITY enmPriority;
+    rc = CFGMR3QueryString(CFGMR3GetRoot(pVM), "Priority", sz, sizeof(sz));
+    if (RT_SUCCESS(rc))
+    {
+        if (!RTStrICmp(sz, "low"))
+            enmPriority = GMMPRIORITY_LOW;
+        else if (!RTStrICmp(sz, "normal"))
+            enmPriority = GMMPRIORITY_NORMAL;
+        else if (!RTStrICmp(sz, "high"))
+            enmPriority = GMMPRIORITY_HIGH;
+        else
+            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, "Unknown \"MM/Priority\" value \"%s\"", sz);
+    }
+    else if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        enmPriority = GMMPRIORITY_NORMAL;
+    else
+        AssertMsgFailedReturn(("Configuration error: Failed to query string \"MM/Priority\", rc=%Rrc.\n", rc), rc);
+
+    /*
+     * Make the initial memory reservation with GMM.
+     */
+    uint64_t cBasePages = (cbRam >> PAGE_SHIFT) + pVM->mm.s.cBasePages;
+    rc = GMMR3InitialReservation(pVM,
+                                 RT_MAX(cBasePages + pVM->mm.s.cHandyPages, 1),
+                                 RT_MAX(pVM->mm.s.cShadowPages, 1),
+                                 RT_MAX(pVM->mm.s.cFixedPages, 1),
+                                 enmOcPolicy,
+                                 enmPriority);
+    if (RT_FAILURE(rc))
+    {
+        if (rc == VERR_GMM_MEMORY_RESERVATION_DECLINED)
+            return VMSetError(pVM, rc, RT_SRC_POS,
+                              N_("Insufficient free memory to start the VM (cbRam=%#RX64 enmOcPolicy=%d enmPriority=%d)"),
+                              cbRam, enmOcPolicy, enmPriority);
+        return VMSetError(pVM, rc, RT_SRC_POS, "GMMR3InitialReservation(,%#RX64,0,0,%d,%d)",
+                          cbRam >> PAGE_SHIFT, enmOcPolicy, enmPriority);
+    }
+
+    /*
+     * If RamSize is 0 we're done now.
+     */
+    if (cbRam < PAGE_SIZE)
+    {
+        Log(("MM: No RAM configured\n"));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Setup the base ram (PGM).
+     */
+    if (cbRam > offRamHole)
+    {
+        rc = PGMR3PhysRegisterRam(pVM, 0, offRamHole, "Base RAM");
+        if (RT_SUCCESS(rc))
+            rc = PGMR3PhysRegisterRam(pVM, _4G, cbRam - offRamHole, "Above 4GB Base RAM");
+    }
+    else
+        rc = PGMR3PhysRegisterRam(pVM, 0, RT_MIN(cbRam, offRamHole), "Base RAM");
+
+    /*
+     * Enabled mmR3UpdateReservation here since we don't want the
+     * PGMR3PhysRegisterRam calls above mess things up.
+     */
+    pVM->mm.s.fDoneMMR3InitPaging = true;
+    AssertMsg(pVM->mm.s.cBasePages == cBasePages || RT_FAILURE(rc), ("%RX64 != %RX64\n", pVM->mm.s.cBasePages, cBasePages));
+
+    LogFlow(("MMR3InitPaging: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Terminates the MM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) MMR3Term(PVM pVM)
+{
+    /*
+     * Destroy the page pool. (first as it used the hyper heap)
+     */
+    mmR3PagePoolTerm(pVM);
+
+    /* Clean up the hypervisor heap. */
+    mmR3HyperTerm(pVM);
+
+    /*
+     * Zero stuff to detect after termination use of the MM interface
+     */
+    pVM->mm.s.offLookupHyper = NIL_OFFSET;
+    pVM->mm.s.pHyperHeapR3   = NULL;        /* freed above. */
+    pVM->mm.s.pHyperHeapR0   = NIL_RTR0PTR; /* freed above. */
+    pVM->mm.s.pHyperHeapRC   = NIL_RTRCPTR; /* freed above. */
+    pVM->mm.s.offVM          = 0;           /* init assertion on this */
+
+    /*
+     * Destroy the User-kernel heap here since the support driver session
+     * may have been terminated by the time we get to MMR3TermUVM.
+     */
+    mmR3UkHeapDestroy(pVM->pUVM->mm.s.pUkHeap);
+    pVM->pUVM->mm.s.pUkHeap = NULL;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates the UVM part of MM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3DECL(void) MMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * Destroy the heaps.
+     */
+    if (pUVM->mm.s.pUkHeap)
+    {
+        mmR3UkHeapDestroy(pUVM->mm.s.pUkHeap);
+        pUVM->mm.s.pUkHeap = NULL;
+    }
+    mmR3HeapDestroy(pUVM->mm.s.pHeap);
+    pUVM->mm.s.pHeap = NULL;
+}
+
+
+/**
+ * Reset notification.
+ *
+ * @param   pVM             The VM handle.
+ */
+VMMR3DECL(void) MMR3Reset(PVM pVM)
+{
+    /* nothing to do anylonger. */
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) mmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("mmR3Save:\n"));
+
+    /* (PGM saves the physical memory.) */
+    SSMR3PutU64(pSSM, pVM->mm.s.cBasePages);
+    return SSMR3PutU64(pSSM, pVM->mm.s.cbRamBase);
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion       Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) mmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("mmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (    SSM_VERSION_MAJOR_CHANGED(uVersion, MM_SAVED_STATE_VERSION)
+        ||  !uVersion)
+    {
+        AssertMsgFailed(("mmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Check the cBasePages and cbRamBase values.
+     */
+    int rc;
+    RTUINT cb1;
+
+    /* cBasePages (ignored) */
+    uint64_t cPages;
+    if (uVersion >= 2)
+        rc = SSMR3GetU64(pSSM, &cPages);
+    else
+    {
+        rc = SSMR3GetUInt(pSSM, &cb1);
+        cPages = cb1 >> PAGE_SHIFT;
+    }
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* cbRamBase */
+    uint64_t cb;
+    if (uVersion != 1)
+        rc = SSMR3GetU64(pSSM, &cb);
+    else
+    {
+        rc = SSMR3GetUInt(pSSM, &cb1);
+        cb = cb1;
+    }
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertLogRelMsgReturn(cb == pVM->mm.s.cbRamBase,
+                          ("Memory configuration has changed. cbRamBase=%#RX64 save=%#RX64\n", pVM->mm.s.cbRamBase, cb),
+                          VERR_SSM_LOAD_MEMORY_SIZE_MISMATCH);
+
+    /* (PGM restores the physical memory.) */
+    return rc;
+}
+
+
+/**
+ * Updates GMM with memory reservation changes.
+ *
+ * Called when MM::cbRamRegistered, MM::cShadowPages or MM::cFixedPages changes.
+ *
+ * @returns VBox status code - see GMMR0UpdateReservation.
+ * @param   pVM             The shared VM structure.
+ */
+int mmR3UpdateReservation(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    if (pVM->mm.s.fDoneMMR3InitPaging)
+        return GMMR3UpdateReservation(pVM,
+                                      RT_MAX(pVM->mm.s.cBasePages + pVM->mm.s.cHandyPages, 1),
+                                      RT_MAX(pVM->mm.s.cShadowPages, 1),
+                                      RT_MAX(pVM->mm.s.cFixedPages, 1));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Interface for PGM to increase the reservation of RAM and ROM pages.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM             The shared VM structure.
+ * @param   cAddBasePages   The number of pages to add.
+ */
+VMMR3DECL(int) MMR3IncreaseBaseReservation(PVM pVM, uint64_t cAddBasePages)
+{
+    uint64_t cOld = pVM->mm.s.cBasePages;
+    pVM->mm.s.cBasePages += cAddBasePages;
+    LogFlow(("MMR3IncreaseBaseReservation: +%RU64 (%RU64 -> %RU64\n", cAddBasePages, cOld, pVM->mm.s.cBasePages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserved physical memory for the RAM (%#RX64 -> %#RX64 + %#RX32)"),
+                   cOld, pVM->mm.s.cBasePages, pVM->mm.s.cHandyPages);
+        pVM->mm.s.cBasePages = cOld;
+    }
+    return rc;
+}
+
+
+/**
+ * Interface for PGM to make reservations for handy pages in addition to the
+ * base memory.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM             The shared VM structure.
+ * @param   cHandyPages     The number of handy pages.
+ */
+VMMR3DECL(int) MMR3ReserveHandyPages(PVM pVM, uint32_t cHandyPages)
+{
+    AssertReturn(!pVM->mm.s.cHandyPages, VERR_WRONG_ORDER);
+
+    pVM->mm.s.cHandyPages = cHandyPages;
+    LogFlow(("MMR3ReserveHandyPages: %RU32 (base %RU64)\n", pVM->mm.s.cHandyPages, pVM->mm.s.cBasePages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserved physical memory for the RAM (%#RX64 + %#RX32)"),
+                   pVM->mm.s.cBasePages, pVM->mm.s.cHandyPages);
+        pVM->mm.s.cHandyPages = 0;
+    }
+    return rc;
+}
+
+
+/**
+ * Interface for PGM to adjust the reservation of fixed pages.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM                 The shared VM structure.
+ * @param   cDeltaFixedPages    The number of pages to add (positive) or subtract (negative).
+ * @param   pszDesc             Some description associated with the reservation.
+ */
+VMMR3DECL(int) MMR3AdjustFixedReservation(PVM pVM, int32_t cDeltaFixedPages, const char *pszDesc)
+{
+    const uint32_t cOld = pVM->mm.s.cFixedPages;
+    pVM->mm.s.cFixedPages += cDeltaFixedPages;
+    LogFlow(("MMR3AdjustFixedReservation: %d (%u -> %u)\n", cDeltaFixedPages, cOld, pVM->mm.s.cFixedPages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserve physical memory (%#x -> %#x; %s)"),
+                   cOld, pVM->mm.s.cFixedPages, pszDesc);
+        pVM->mm.s.cFixedPages = cOld;
+    }
+    return rc;
+}
+
+
+/**
+ * Interface for PGM to update the reservation of shadow pages.
+ *
+ * This can be called before MMR3InitPaging.
+ *
+ * @returns VBox status code. Will set VM error on failure.
+ * @param   pVM             The shared VM structure.
+ * @param   cShadowPages    The new page count.
+ */
+VMMR3DECL(int) MMR3UpdateShadowReservation(PVM pVM, uint32_t cShadowPages)
+{
+    const uint32_t cOld = pVM->mm.s.cShadowPages;
+    pVM->mm.s.cShadowPages = cShadowPages;
+    LogFlow(("MMR3UpdateShadowReservation: %u -> %u\n", cOld, pVM->mm.s.cShadowPages));
+    int rc = mmR3UpdateReservation(pVM);
+    if (RT_FAILURE(rc))
+    {
+        VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to reserve physical memory for shadow page tables (%#x -> %#x)"), cOld, pVM->mm.s.cShadowPages);
+        pVM->mm.s.cShadowPages = cOld;
+    }
+    return rc;
+}
+
+
+/**
+ * Convert HC Physical address to HC Virtual address.
+ *
+ * @returns VBox status.
+ * @param   pVM         VM handle.
+ * @param   HCPhys      The host context virtual address.
+ * @param   ppv         Where to store the resulting address.
+ * @thread  The Emulation Thread.
+ *
+ * @remarks Avoid whenever possible.
+ *          Intended for the debugger facility only.
+ * @todo    Rename to indicate the special usage.
+ */
+VMMR3DECL(int) MMR3HCPhys2HCVirt(PVM pVM, RTHCPHYS HCPhys, void **ppv)
+{
+    /*
+     * Try page tables.
+     */
+    int rc = MMPagePhys2PageTry(pVM, HCPhys, ppv);
+    if (RT_SUCCESS(rc))
+        return rc;
+
+    /*
+     * Iterate thru the lookup records for HMA.
+     */
+    uint32_t off = HCPhys & PAGE_OFFSET_MASK;
+    HCPhys &= X86_PTE_PAE_PG_MASK;
+    PMMLOOKUPHYPER pCur = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.CTX_SUFF(pHyperHeap) + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pCur->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                PCRTHCPHYS  paHCPhysPages = pCur->u.Locked.paHCPhysPages;
+                size_t      iPage         = pCur->cb >> PAGE_SHIFT;
+                while (iPage-- > 0)
+                    if (paHCPhysPages[iPage] == HCPhys)
+                    {
+                        *ppv = (char *)pCur->u.Locked.pvR3 + (iPage << PAGE_SHIFT) + off;
+                        return VINF_SUCCESS;
+                    }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+                if (pCur->u.HCPhys.HCPhys - HCPhys < pCur->cb)
+                {
+                    *ppv = (uint8_t *)pCur->u.HCPhys.pvR3 + pCur->u.HCPhys.HCPhys - HCPhys + off;
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:  /* (for now we'll not allow these kind of conversions) */
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pCur->enmType));
+                break;
+        }
+
+        /* next */
+        if (pCur->offNext ==  (int32_t)NIL_OFFSET)
+            break;
+        pCur = (PMMLOOKUPHYPER)((uint8_t *)pCur + pCur->offNext);
+    }
+    /* give up */
+    return VERR_INVALID_POINTER;
+}
+
+
+
+/**
+ * Get the size of the base RAM.
+ * This usually means the size of the first contiguous block of physical memory.
+ *
+ * @returns The guest base RAM size.
+ * @param   pVM         The VM handle.
+ * @thread  Any.
+ *
+ * @deprecated
+ */
+VMMR3DECL(uint64_t) MMR3PhysGetRamSize(PVM pVM)
+{
+    return pVM->mm.s.cbRamBase;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/MMHeap.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/MMHeap.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/MMHeap.cpp	(revision 35333)
@@ -0,0 +1,694 @@
+/* $Id$ */
+/** @file
+ * MM - Memory Manager - Heap.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HEAP
+#include <VBox/mm.h>
+#include <VBox/stam.h>
+#include <VBox/pgm.h>
+#include "MMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero);
+
+
+
+/**
+ * Allocate and initialize a heap structure and it's associated substructures.
+ *
+ * @returns VBox status.
+ * @param   pVM     The handle to the VM the heap should be associated with.
+ * @param   ppHeap  Where to store the heap pointer.
+ */
+int mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap)
+{
+    PMMHEAP pHeap = (PMMHEAP)RTMemAllocZ(sizeof(MMHEAP) + sizeof(MMHEAPSTAT));
+    if (pHeap)
+    {
+        int rc = RTCritSectInit(&pHeap->Lock);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize the global stat record.
+             */
+            pHeap->pUVM = pUVM;
+            pHeap->Stat.pHeap = pHeap;
+#ifdef MMR3HEAP_WITH_STATISTICS
+            PMMHEAPSTAT pStat = &pHeap->Stat;
+            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFrees",           STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
+            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
+                                                                        STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/R3Heap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
+#endif
+            *ppHeap = pHeap;
+            return VINF_SUCCESS;
+        }
+        AssertRC(rc);
+        RTMemFree(pHeap);
+    }
+    AssertMsgFailed(("failed to allocate heap structure\n"));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Destroy a heap.
+ *
+ * @param   pHeap   Heap handle.
+ */
+void mmR3HeapDestroy(PMMHEAP pHeap)
+{
+    /*
+     * Start by deleting the lock, that'll trap anyone
+     * attempting to use the heap.
+     */
+    RTCritSectDelete(&pHeap->Lock);
+
+    /*
+     * Walk the node list and free all the memory.
+     */
+    PMMHEAPHDR  pHdr = pHeap->pHead;
+    while (pHdr)
+    {
+        void *pv = pHdr;
+        pHdr = pHdr->pNext;
+        RTMemFree(pv);
+    }
+
+    /*
+     * Free the stat nodes.
+     */
+    /** @todo free all nodes in a AVL tree. */
+    RTMemFree(pHeap);
+}
+
+
+/**
+ * Allocate memory associating it with the VM for collective cleanup.
+ *
+ * The memory will be allocated from the default heap but a header
+ * is added in which we keep track of which VM it belongs to and chain
+ * all the allocations together so they can be freed in one go.
+ *
+ * This interface is typically used for memory block which will not be
+ * freed during the life of the VM.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAllocU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
+{
+    Assert(pUVM->mm.s.pHeap);
+    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
+}
+
+
+/**
+ * Allocate memory associating it with the VM for collective cleanup.
+ *
+ * The memory will be allocated from the default heap but a header
+ * is added in which we keep track of which VM it belongs to and chain
+ * all the allocations together so they can be freed in one go.
+ *
+ * This interface is typically used for memory block which will not be
+ * freed during the life of the VM.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
+}
+
+
+/**
+ * Same as MMR3HeapAllocU().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    Assert(pUVM->mm.s.pHeap);
+    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, false);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3HeapAlloc().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, false);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3HeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAllocZU(PUVM pUVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
+}
+
+
+/**
+ * Same as MMR3HeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ */
+VMMR3DECL(void *) MMR3HeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize)
+{
+    return mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
+}
+
+
+/**
+ * Same as MMR3HeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocZExU(PUVM pUVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    Assert(pUVM->mm.s.pHeap);
+    void *pv = mmR3HeapAlloc(pUVM->mm.s.pHeap, enmTag, cbSize, true);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3HeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ */
+VMMR3DECL(int) MMR3HeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv)
+{
+    void *pv = mmR3HeapAlloc(pVM->pUVM->mm.s.pHeap, enmTag, cbSize, true);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Allocate memory from the heap.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pHeap       Heap handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   cbSize      Size of the block.
+ * @param   fZero       Whether or not to zero the memory block.
+ */
+void *mmR3HeapAlloc(PMMHEAP pHeap, MMTAG enmTag, size_t cbSize, bool fZero)
+{
+#ifdef MMR3HEAP_WITH_STATISTICS
+    RTCritSectEnter(&pHeap->Lock);
+
+    /*
+     * Find/alloc statistics nodes.
+     */
+    pHeap->Stat.cAllocations++;
+    PMMHEAPSTAT pStat = (PMMHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+    if (pStat)
+    {
+        pStat->cAllocations++;
+
+        RTCritSectLeave(&pHeap->Lock);
+    }
+    else
+    {
+        pStat = (PMMHEAPSTAT)RTMemAllocZ(sizeof(MMHEAPSTAT));
+        if (!pStat)
+        {
+            pHeap->Stat.cFailures++;
+            AssertMsgFailed(("Failed to allocate heap stat record.\n"));
+            RTCritSectLeave(&pHeap->Lock);
+            return NULL;
+        }
+        pStat->Core.Key = (AVLULKEY)enmTag;
+        pStat->pHeap    = pHeap;
+        RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
+
+        pStat->cAllocations++;
+        RTCritSectLeave(&pHeap->Lock);
+
+        /* register the statistics */
+        PUVM pUVM = pHeap->pUVM;
+        const char *pszTag = mmGetTagName(enmTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/R3Heap/%s", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3HeapAlloc() calls.",        "/MM/R3Heap/%s/cAllocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapRealloc() calls.",      "/MM/R3Heap/%s/cReallocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3HeapFree() calls.",         "/MM/R3Heap/%s/cFrees", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/R3Heap/%s/cFailures", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/R3Heap/%s/cbAllocated", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/R3Heap/%s/cbFreed", pszTag);
+    }
+#endif
+
+    /*
+     * Validate input.
+     */
+    if (cbSize == 0)
+    {
+#ifdef MMR3HEAP_WITH_STATISTICS
+        RTCritSectEnter(&pHeap->Lock);
+        pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+        RTCritSectLeave(&pHeap->Lock);
+#endif
+        return NULL;
+    }
+
+    /*
+     * Allocate heap block.
+     */
+    cbSize = RT_ALIGN_Z(cbSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
+    PMMHEAPHDR  pHdr = (PMMHEAPHDR)(fZero ? RTMemAllocZ(cbSize) : RTMemAlloc(cbSize));
+    if (!pHdr)
+    {
+        AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cbSize, enmTag, &enmTag));
+#ifdef MMR3HEAP_WITH_STATISTICS
+        RTCritSectEnter(&pHeap->Lock);
+        pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+        RTCritSectLeave(&pHeap->Lock);
+#endif
+        return NULL;
+    }
+    Assert(!((uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1)));
+
+    RTCritSectEnter(&pHeap->Lock);
+
+    /*
+     * Init and link in the header.
+     */
+    pHdr->pNext = NULL;
+    pHdr->pPrev = pHeap->pTail;
+    if (pHdr->pPrev)
+        pHdr->pPrev->pNext = pHdr;
+    else
+        pHeap->pHead = pHdr;
+    pHeap->pTail = pHdr;
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pHdr->pStat  = pStat;
+#else
+    pHdr->pStat  = &pHeap->Stat;
+#endif
+    pHdr->cbSize = cbSize;
+
+    /*
+     * Update statistics
+     */
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pStat->cbAllocated          += cbSize;
+    pStat->cbCurAllocated       += cbSize;
+    pHeap->Stat.cbAllocated     += cbSize;
+    pHeap->Stat.cbCurAllocated  += cbSize;
+#endif
+
+    RTCritSectLeave(&pHeap->Lock);
+
+    return pHdr + 1;
+}
+
+
+/**
+ * Reallocate memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
+ *
+ * @returns Pointer to reallocated memory.
+ * @param   pv          Pointer to the memory block to reallocate.
+ *                      Must not be NULL!
+ * @param   cbNewSize   New block size.
+ */
+VMMR3DECL(void *) MMR3HeapRealloc(void *pv, size_t cbNewSize)
+{
+    AssertMsg(pv, ("Invalid pointer pv=%p\n", pv));
+    if (!pv)
+        return NULL;
+
+    /*
+     * If newsize is zero then this is a free.
+     */
+    if (!cbNewSize)
+    {
+        MMR3HeapFree(pv);
+        return NULL;
+    }
+
+    /*
+     * Validate header.
+     */
+    PMMHEAPHDR  pHdr = (PMMHEAPHDR)pv - 1;
+    if (    pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1)
+        ||  (uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))
+    {
+        AssertMsgFailed(("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
+        return NULL;
+    }
+    Assert(pHdr->pStat != NULL);
+    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
+    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
+
+    PMMHEAP pHeap = pHdr->pStat->pHeap;
+
+#ifdef MMR3HEAP_WITH_STATISTICS
+    RTCritSectEnter(&pHeap->Lock);
+    pHdr->pStat->cReallocations++;
+    pHeap->Stat.cReallocations++;
+    RTCritSectLeave(&pHeap->Lock);
+#endif
+
+    /*
+     * Reallocate the block.
+     */
+    cbNewSize = RT_ALIGN_Z(cbNewSize, MMR3HEAP_SIZE_ALIGNMENT) + sizeof(MMHEAPHDR);
+    PMMHEAPHDR pHdrNew = (PMMHEAPHDR)RTMemRealloc(pHdr, cbNewSize);
+    if (!pHdrNew)
+    {
+#ifdef MMR3HEAP_WITH_STATISTICS
+        RTCritSectEnter(&pHeap->Lock);
+        pHdr->pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+        RTCritSectLeave(&pHeap->Lock);
+#endif
+        return NULL;
+    }
+
+    /*
+     * Update pointers.
+     */
+    if (pHdrNew != pHdr)
+    {
+        RTCritSectEnter(&pHeap->Lock);
+        if (pHdrNew->pPrev)
+            pHdrNew->pPrev->pNext = pHdrNew;
+        else
+            pHeap->pHead = pHdrNew;
+
+        if (pHdrNew->pNext)
+            pHdrNew->pNext->pPrev = pHdrNew;
+        else
+            pHeap->pTail = pHdrNew;
+        RTCritSectLeave(&pHeap->Lock);
+    }
+
+    /*
+     * Update statistics.
+     */
+#ifdef MMR3HEAP_WITH_STATISTICS
+    RTCritSectEnter(&pHeap->Lock);
+    pHdrNew->pStat->cbAllocated += cbNewSize - pHdrNew->cbSize;
+    pHeap->Stat.cbAllocated += cbNewSize - pHdrNew->cbSize;
+    RTCritSectLeave(&pHeap->Lock);
+#endif
+
+    pHdrNew->cbSize = cbNewSize;
+
+    return pHdrNew + 1;
+}
+
+
+/**
+ * Duplicates the specified string.
+ *
+ * @returns Pointer to the duplicate.
+ * @returns NULL on failure or when input NULL.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   psz         The string to duplicate. NULL is allowed.
+ */
+VMMR3DECL(char *) MMR3HeapStrDupU(PUVM pUVM, MMTAG enmTag, const char *psz)
+{
+    if (!psz)
+        return NULL;
+    AssertPtr(psz);
+
+    size_t cch = strlen(psz) + 1;
+    char *pszDup = (char *)MMR3HeapAllocU(pUVM, enmTag, cch);
+    if (pszDup)
+        memcpy(pszDup, psz, cch);
+    return pszDup;
+}
+
+
+/**
+ * Duplicates the specified string.
+ *
+ * @returns Pointer to the duplicate.
+ * @returns NULL on failure or when input NULL.
+ * @param   pVM         The VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM. See MM_TAG_*.
+ * @param   psz         The string to duplicate. NULL is allowed.
+ */
+VMMR3DECL(char *) MMR3HeapStrDup(PVM pVM, MMTAG enmTag, const char *psz)
+{
+    return MMR3HeapStrDupU(pVM->pUVM, enmTag, psz);
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pVM         The VM
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   ...         Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintf(PVM pVM, MMTAG enmTag, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    char *psz = MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
+    va_end(va);
+    return psz;
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   ...         Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintfU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    char *psz = MMR3HeapAPrintfVU(pUVM, enmTag, pszFormat, va);
+    va_end(va);
+    return psz;
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pVM         The VM
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   va          Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintfV(PVM pVM, MMTAG enmTag, const char *pszFormat, va_list va)
+{
+    return MMR3HeapAPrintfVU(pVM->pUVM, enmTag, pszFormat, va);
+}
+
+
+/**
+ * Allocating string printf.
+ *
+ * @returns Pointer to the string.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   enmTag      The statistics tag.
+ * @param   pszFormat   The format string.
+ * @param   va          Format arguments.
+ */
+VMMR3DECL(char *)    MMR3HeapAPrintfVU(PUVM pUVM, MMTAG enmTag, const char *pszFormat, va_list va)
+{
+    /*
+     * The lazy bird way.
+     */
+    char *psz;
+    int cch = RTStrAPrintfV(&psz, pszFormat, va);
+    if (cch < 0)
+        return NULL;
+    Assert(psz[cch] == '\0');
+    char *pszRet = (char *)MMR3HeapAllocU(pUVM, enmTag, cch + 1);
+    if (pszRet)
+        memcpy(pszRet, psz, cch + 1);
+    RTStrFree(psz);
+    return pszRet;
+}
+
+
+/**
+ * Releases memory allocated with MMR3HeapAlloc() or MMR3HeapRealloc().
+ *
+ * @param   pv          Pointer to the memory block to free.
+ */
+VMMR3DECL(void) MMR3HeapFree(void *pv)
+{
+    /* Ignore NULL pointers. */
+    if (!pv)
+        return;
+
+    /*
+     * Validate header.
+     */
+    PMMHEAPHDR  pHdr = (PMMHEAPHDR)pv - 1;
+    if (    pHdr->cbSize & (MMR3HEAP_SIZE_ALIGNMENT - 1)
+        ||  (uintptr_t)pHdr & (RTMEM_ALIGNMENT - 1))
+    {
+        AssertMsgFailed(("Invalid heap header! pv=%p, size=%#x\n", pv, pHdr->cbSize));
+        return;
+    }
+    Assert(pHdr->pStat != NULL);
+    Assert(!((uintptr_t)pHdr->pNext & (RTMEM_ALIGNMENT - 1)));
+    Assert(!((uintptr_t)pHdr->pPrev & (RTMEM_ALIGNMENT - 1)));
+
+    /*
+     * Update statistics
+     */
+    PMMHEAP pHeap = pHdr->pStat->pHeap;
+    RTCritSectEnter(&pHeap->Lock);
+
+#ifdef MMR3HEAP_WITH_STATISTICS
+    pHdr->pStat->cFrees++;
+    pHeap->Stat.cFrees++;
+    pHdr->pStat->cbFreed            += pHdr->cbSize;
+    pHeap->Stat.cbFreed             += pHdr->cbSize;
+    pHdr->pStat->cbCurAllocated     -= pHdr->cbSize;
+    pHeap->Stat.cbCurAllocated      -= pHdr->cbSize;
+#endif
+
+    /*
+     * Unlink it.
+     */
+    if (pHdr->pPrev)
+        pHdr->pPrev->pNext = pHdr->pNext;
+    else
+        pHeap->pHead = pHdr->pNext;
+
+    if (pHdr->pNext)
+        pHdr->pNext->pPrev = pHdr->pPrev;
+    else
+        pHeap->pTail = pHdr->pPrev;
+
+    RTCritSectLeave(&pHeap->Lock);
+
+    /*
+     * Free the memory.
+     */
+    RTMemFree(pHdr);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/MMHyper.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/MMHyper.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/MMHyper.cpp	(revision 35333)
@@ -0,0 +1,1407 @@
+/* $Id$ */
+/** @file
+ * MM - Memory Manager - Hypervisor Memory Area.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HYPER
+#include <VBox/pgm.h>
+#include <VBox/mm.h>
+#include <VBox/dbgf.h>
+#include "MMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include "internal/pgm.h"
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(bool) mmR3HyperRelocateCallback(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser);
+static int mmR3HyperMap(PVM pVM, const size_t cb, const char *pszDesc, PRTGCPTR pGCPtr, PMMLOOKUPHYPER *ppLookup);
+static int mmR3HyperHeapCreate(PVM pVM, const size_t cb, PMMHYPERHEAP *ppHeap, PRTR0PTR pR0PtrHeap);
+static int mmR3HyperHeapMap(PVM pVM, PMMHYPERHEAP pHeap, PRTGCPTR ppHeapGC);
+static DECLCALLBACK(void) mmR3HyperInfoHma(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+DECLINLINE(uint32_t) mmR3ComputeHyperHeapSize(PVM pVM, bool fCanUseLargerHeap)
+{
+    bool fHwVirtExtForced = VMMIsHwVirtExtForced(pVM);
+
+    /* Newer chipset allows for much more devices, putting additional pressure on hyperheap */
+    /* @todo: maybe base hyperheap size upon number of devices attached too */
+    if (fCanUseLargerHeap)
+        return _2M + pVM->cCpus * 2 * _64K;
+
+    if (pVM->cCpus > 1)
+        return _1M + pVM->cCpus * 2 * _64K;
+
+    if (fHwVirtExtForced)
+    {
+        uint64_t cbRam = 0;
+        CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
+
+        /* Need a bit more space for large memory guests. (@todo: only for shadow paging!) */
+        if (cbRam >= _4G)
+            return _1M;
+        else
+            return 640 * _1K;
+    }
+    else
+        /* Size must be kept like this for saved state compatibility (only for raw mode though). */
+        return 1280*_1K;
+}
+
+
+/**
+ * Initializes the hypervisor related MM stuff without
+ * calling down to PGM.
+ *
+ * PGM is not initialized at this  point, PGM relies on
+ * the heap to initialize.
+ *
+ * @returns VBox status.
+ */
+int mmR3HyperInit(PVM pVM)
+{
+    LogFlow(("mmR3HyperInit:\n"));
+
+    /*
+     * Decide Hypervisor mapping in the guest context
+     * And setup various hypervisor area and heap parameters.
+     */
+    pVM->mm.s.pvHyperAreaGC = (RTGCPTR)MM_HYPER_AREA_ADDRESS;
+    pVM->mm.s.cbHyperArea   = MM_HYPER_AREA_MAX_SIZE;
+    AssertRelease(RT_ALIGN_T(pVM->mm.s.pvHyperAreaGC, 1 << X86_PD_SHIFT, RTGCPTR) == pVM->mm.s.pvHyperAreaGC);
+    Assert(pVM->mm.s.pvHyperAreaGC < 0xff000000);
+
+    /** @todo @bugref{1865}, @bugref{3202}: Change the cbHyperHeap default
+     *        depending on whether VT-x/AMD-V is enabled or not! Don't waste
+     *        precious kernel space on heap for the PATM.
+     */
+    PCFGMNODE pMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "MM");
+    bool fCanUseLargerHeap = false;
+    int rc = CFGMR3QueryBoolDef(pMM, "CanUseLargerHeap", &fCanUseLargerHeap, false);
+    uint32_t cbHyperHeap = mmR3ComputeHyperHeapSize(pVM, fCanUseLargerHeap);
+    rc = CFGMR3QueryU32Def(pMM, "cbHyperHeap", &cbHyperHeap, cbHyperHeap);
+    AssertLogRelRCReturn(rc, rc);
+
+    cbHyperHeap = RT_ALIGN_32(cbHyperHeap, PAGE_SIZE);
+    LogRel(("MM: cbHyperHeap=%#x (%u)\n", cbHyperHeap, cbHyperHeap));
+
+    /*
+     * Allocate the hypervisor heap.
+     *
+     * (This must be done before we start adding memory to the
+     * hypervisor static area because lookup records are allocated from it.)
+     */
+    rc = mmR3HyperHeapCreate(pVM, cbHyperHeap, &pVM->mm.s.pHyperHeapR3, &pVM->mm.s.pHyperHeapR0);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Make a small head fence to fend of accidental sequential access.
+         */
+        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+
+        /*
+         * Map the VM structure into the hypervisor space.
+         */
+        AssertRelease(pVM->cbSelf == RT_UOFFSETOF(VM, aCpus[pVM->cCpus]));
+        RTGCPTR GCPtr;
+        rc = MMR3HyperMapPages(pVM, pVM, pVM->pVMR0, RT_ALIGN_Z(pVM->cbSelf, PAGE_SIZE) >> PAGE_SHIFT, pVM->paVMPagesR3, "VM", &GCPtr);
+        if (RT_SUCCESS(rc))
+        {
+            pVM->pVMRC = (RTRCPTR)GCPtr;
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+                pVM->aCpus[i].pVMRC = pVM->pVMRC;
+
+            /* Reserve a page for fencing. */
+            MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+
+            /*
+             * Map the heap into the hypervisor space.
+             */
+            rc = mmR3HyperHeapMap(pVM, pVM->mm.s.pHyperHeapR3, &GCPtr);
+            if (RT_SUCCESS(rc))
+            {
+                pVM->mm.s.pHyperHeapRC = (RTRCPTR)GCPtr;
+                Assert(pVM->mm.s.pHyperHeapRC == GCPtr);
+
+                /*
+                 * Register info handlers.
+                 */
+                DBGFR3InfoRegisterInternal(pVM, "hma", "Show the layout of the Hypervisor Memory Area.", mmR3HyperInfoHma);
+
+                LogFlow(("mmR3HyperInit: returns VINF_SUCCESS\n"));
+                return VINF_SUCCESS;
+            }
+            /* Caller will do proper cleanup. */
+        }
+    }
+
+    LogFlow(("mmR3HyperInit: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Cleans up the hypervisor heap.
+ *
+ * @returns VBox status.
+ */
+int mmR3HyperTerm(PVM pVM)
+{
+    if (pVM->mm.s.pHyperHeapR3)
+        PDMR3CritSectDelete(&pVM->mm.s.pHyperHeapR3->Lock);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Finalizes the HMA mapping.
+ *
+ * This is called later during init, most (all) HMA allocations should be done
+ * by the time this function is called.
+ *
+ * @returns VBox status.
+ */
+VMMR3DECL(int) MMR3HyperInitFinalize(PVM pVM)
+{
+    LogFlow(("MMR3HyperInitFinalize:\n"));
+
+    /*
+     * Initialize the hyper heap critical section.
+     */
+    int rc = PDMR3CritSectInit(pVM, &pVM->mm.s.pHyperHeapR3->Lock, RT_SRC_POS, "MM-HYPER");
+    AssertRC(rc);
+
+    /*
+     * Adjust and create the HMA mapping.
+     */
+    while ((RTINT)pVM->mm.s.offHyperNextStatic + 64*_1K < (RTINT)pVM->mm.s.cbHyperArea - _4M)
+        pVM->mm.s.cbHyperArea -= _4M;
+    rc = PGMR3MapPT(pVM, pVM->mm.s.pvHyperAreaGC, pVM->mm.s.cbHyperArea, 0 /*fFlags*/,
+                    mmR3HyperRelocateCallback, NULL, "Hypervisor Memory Area");
+    if (RT_FAILURE(rc))
+        return rc;
+    pVM->mm.s.fPGMInitialized = true;
+
+    /*
+     * Do all the delayed mappings.
+     */
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uintptr_t)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        RTGCPTR     GCPtr = pVM->mm.s.pvHyperAreaGC + pLookup->off;
+        uint32_t    cPages = pLookup->cb >> PAGE_SHIFT;
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                PCRTHCPHYS paHCPhysPages = pLookup->u.Locked.paHCPhysPages;
+                for (uint32_t i = 0; i < cPages; i++)
+                {
+                    rc = PGMMap(pVM, GCPtr + (i << PAGE_SHIFT), paHCPhysPages[i], PAGE_SIZE, 0);
+                    AssertRCReturn(rc, rc);
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+                rc = PGMMap(pVM, GCPtr, pLookup->u.HCPhys.HCPhys, pLookup->cb, 0);
+                break;
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            {
+                const RTGCPHYS  GCPhys = pLookup->u.GCPhys.GCPhys;
+                const uint32_t  cb = pLookup->cb;
+                for (uint32_t off = 0; off < cb; off += PAGE_SIZE)
+                {
+                    RTHCPHYS HCPhys;
+                    rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys + off, &HCPhys);
+                    if (RT_FAILURE(rc))
+                        break;
+                    rc = PGMMap(pVM, GCPtr + off, HCPhys, PAGE_SIZE, 0);
+                    if (RT_FAILURE(rc))
+                        break;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            {
+                const RTGCPHYS offEnd = pLookup->u.MMIO2.off + pLookup->cb;
+                for (RTGCPHYS offCur = pLookup->u.MMIO2.off; offCur < offEnd; offCur += PAGE_SIZE)
+                {
+                    RTHCPHYS HCPhys;
+                    rc = PGMR3PhysMMIO2GetHCPhys(pVM, pLookup->u.MMIO2.pDevIns, pLookup->u.MMIO2.iRegion, offCur, &HCPhys);
+                    if (RT_FAILURE(rc))
+                        break;
+                    rc = PGMMap(pVM, GCPtr + (offCur - pLookup->u.MMIO2.off), HCPhys, PAGE_SIZE, 0);
+                    if (RT_FAILURE(rc))
+                        break;
+                }
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                /* do nothing here since these are either fences or managed by someone else using PGM. */
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("rc=%Rrc cb=%d off=%#RX32 enmType=%d pszDesc=%s\n",
+                             rc, pLookup->cb, pLookup->off, pLookup->enmType, pLookup->pszDesc));
+            return rc;
+        }
+
+        /* next */
+        if (pLookup->offNext == (int32_t)NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uintptr_t)pLookup + pLookup->offNext);
+    }
+
+    LogFlow(("MMR3HyperInitFinalize: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Callback function which will be called when PGM is trying to find
+ * a new location for the mapping.
+ *
+ * The callback is called in two modes, 1) the check mode and 2) the relocate mode.
+ * In 1) the callback should say if it objects to a suggested new location. If it
+ * accepts the new location, it is called again for doing it's relocation.
+ *
+ *
+ * @returns true if the location is ok.
+ * @returns false if another location should be found.
+ * @param   pVM         The VM handle.
+ * @param   GCPtrOld    The old virtual address.
+ * @param   GCPtrNew    The new virtual address.
+ * @param   enmMode     Used to indicate the callback mode.
+ * @param   pvUser      User argument. Ignored.
+ * @remark  The return value is no a failure indicator, it's an acceptance
+ *          indicator. Relocation can not fail!
+ */
+static DECLCALLBACK(bool) mmR3HyperRelocateCallback(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
+{
+    switch (enmMode)
+    {
+        /*
+         * Verify location - all locations are good for us.
+         */
+        case PGMRELOCATECALL_SUGGEST:
+            return true;
+
+        /*
+         * Execute the relocation.
+         */
+        case PGMRELOCATECALL_RELOCATE:
+        {
+            /*
+             * Accepted!
+             */
+            AssertMsg(GCPtrOld == pVM->mm.s.pvHyperAreaGC, ("GCPtrOld=%RGv pVM->mm.s.pvHyperAreaGC=%RGv\n", GCPtrOld, pVM->mm.s.pvHyperAreaGC));
+            Log(("Relocating the hypervisor from %RGv to %RGv\n", GCPtrOld, GCPtrNew));
+
+            /*
+             * Relocate the VM structure and ourselves.
+             */
+            RTGCINTPTR offDelta = GCPtrNew - GCPtrOld;
+            pVM->pVMRC                          += offDelta;
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+                pVM->aCpus[i].pVMRC              = pVM->pVMRC;
+
+            pVM->mm.s.pvHyperAreaGC             += offDelta;
+            Assert(pVM->mm.s.pvHyperAreaGC < _4G);
+            pVM->mm.s.pHyperHeapRC              += offDelta;
+            pVM->mm.s.pHyperHeapR3->pbHeapRC    += offDelta;
+            pVM->mm.s.pHyperHeapR3->pVMRC        = pVM->pVMRC;
+
+            /*
+             * Relocate the rest.
+             */
+            VMR3Relocate(pVM, offDelta);
+            return true;
+        }
+
+        default:
+            AssertMsgFailed(("Invalid relocation mode %d\n", enmMode));
+    }
+
+    return false;
+}
+
+/**
+ * Service a VMMCALLRING3_MMHYPER_LOCK call.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+VMMR3DECL(int) MMR3LockCall(PVM pVM)
+{
+    PMMHYPERHEAP pHeap = pVM->mm.s.CTX_SUFF(pHyperHeap);
+
+    int rc = PDMR3CritSectEnterEx(&pHeap->Lock, true /* fHostCall */);
+    AssertRC(rc);
+    return rc;
+}
+
+/**
+ * Maps contiguous HC physical memory into the hypervisor region in the GC.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         VM handle.
+ * @param   pvR3        Ring-3 address of the memory. Must be page aligned!
+ * @param   pvR0        Optional ring-0 address of the memory.
+ * @param   HCPhys      Host context physical address of the memory to be
+ *                      mapped. Must be page aligned!
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     Description.
+ * @param   pGCPtr      Where to store the GC address.
+ */
+VMMR3DECL(int) MMR3HyperMapHCPhys(PVM pVM, void *pvR3, RTR0PTR pvR0, RTHCPHYS HCPhys, size_t cb, const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapHCPhys: pvR3=%p pvR0=%p HCPhys=%RHp cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", pvR3, pvR0, HCPhys, (int)cb, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    AssertReturn(RT_ALIGN_P(pvR3, PAGE_SIZE) == pvR3, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(pvR0, PAGE_SIZE, RTR0PTR) == pvR0, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(HCPhys, PAGE_SIZE, RTHCPHYS) == HCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    uint32_t cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cbAligned, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_HCPHYS;
+        pLookup->u.HCPhys.pvR3   = pvR3;
+        pLookup->u.HCPhys.pvR0   = pvR0;
+        pLookup->u.HCPhys.HCPhys = HCPhys;
+
+        /*
+         * Update the page table.
+         */
+        if (pVM->mm.s.fPGMInitialized)
+            rc = PGMMap(pVM, GCPtr, HCPhys, cbAligned, 0);
+        if (RT_SUCCESS(rc))
+            *pGCPtr = GCPtr;
+    }
+    return rc;
+}
+
+
+/**
+ * Maps contiguous GC physical memory into the hypervisor region in the GC.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         VM handle.
+ * @param   GCPhys      Guest context physical address of the memory to be mapped. Must be page aligned!
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     Mapping description.
+ * @param   pGCPtr      Where to store the GC address.
+ */
+VMMR3DECL(int) MMR3HyperMapGCPhys(PVM pVM, RTGCPHYS GCPhys, size_t cb, const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapGCPhys: GCPhys=%RGp cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", GCPhys, (int)cb, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_GCPHYS;
+        pLookup->u.GCPhys.GCPhys = GCPhys;
+
+        /*
+         * Update the page table.
+         */
+        for (unsigned off = 0; off < cb; off += PAGE_SIZE)
+        {
+            RTHCPHYS HCPhys;
+            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhys + off, &HCPhys);
+            AssertRC(rc);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("rc=%Rrc GCPhys=%RGp off=%#x %s\n", rc, GCPhys, off, pszDesc));
+                break;
+            }
+            if (pVM->mm.s.fPGMInitialized)
+            {
+                rc = PGMMap(pVM, GCPtr + off, HCPhys, PAGE_SIZE, 0);
+                AssertRC(rc);
+                if (RT_FAILURE(rc))
+                {
+                    AssertMsgFailed(("rc=%Rrc GCPhys=%RGp off=%#x %s\n", rc, GCPhys, off, pszDesc));
+                    break;
+                }
+            }
+        }
+
+        if (RT_SUCCESS(rc) && pGCPtr)
+            *pGCPtr = GCPtr;
+    }
+    return rc;
+}
+
+
+/**
+ * Maps a portion of an MMIO2 region into the hypervisor region.
+ *
+ * Callers of this API must never deregister the MMIO2 region before the
+ * VM is powered off. If this becomes a requirement MMR3HyperUnmapMMIO2
+ * API will be needed to perform cleanups.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pDevIns     The device owning the MMIO2 memory.
+ * @param   iRegion     The region.
+ * @param   off         The offset into the region. Will be rounded down to closest page boundary.
+ * @param   cb          The number of bytes to map. Will be rounded up to the closest page boundary.
+ * @param   pszDesc     Mapping description.
+ * @param   pRCPtr      Where to store the RC address.
+ */
+VMMR3DECL(int) MMR3HyperMapMMIO2(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
+                                const char *pszDesc, PRTRCPTR pRCPtr)
+{
+    LogFlow(("MMR3HyperMapMMIO2: pDevIns=%p iRegion=%#x off=%RGp cb=%RGp pszDesc=%p:{%s} pRCPtr=%p\n",
+             pDevIns, iRegion, off, cb, pszDesc, pszDesc, pRCPtr));
+    int rc;
+
+    /*
+     * Validate input.
+     */
+    AssertReturn(pszDesc && *pszDesc, VERR_INVALID_PARAMETER);
+    AssertReturn(off + cb > off, VERR_INVALID_PARAMETER);
+    uint32_t const offPage = off & PAGE_OFFSET_MASK;
+    off &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    cb += offPage;
+    cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+    const RTGCPHYS offEnd = off + cb;
+    AssertReturn(offEnd > off, VERR_INVALID_PARAMETER);
+    for (RTGCPHYS offCur = off; offCur < offEnd; offCur += PAGE_SIZE)
+    {
+        RTHCPHYS HCPhys;
+        rc = PGMR3PhysMMIO2GetHCPhys(pVM, pDevIns, iRegion, offCur, &HCPhys);
+        AssertMsgRCReturn(rc, ("rc=%Rrc - iRegion=%d off=%RGp\n", rc, iRegion, off), rc);
+    }
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_MMIO2;
+        pLookup->u.MMIO2.pDevIns = pDevIns;
+        pLookup->u.MMIO2.iRegion = iRegion;
+        pLookup->u.MMIO2.off = off;
+
+        /*
+         * Update the page table.
+         */
+        if (pVM->mm.s.fPGMInitialized)
+        {
+            for (RTGCPHYS offCur = off; offCur < offEnd; offCur += PAGE_SIZE)
+            {
+                RTHCPHYS HCPhys;
+                rc = PGMR3PhysMMIO2GetHCPhys(pVM, pDevIns, iRegion, offCur, &HCPhys);
+                AssertRCReturn(rc, VERR_INTERNAL_ERROR);
+                rc = PGMMap(pVM, GCPtr + (offCur - off), HCPhys, PAGE_SIZE, 0);
+                if (RT_FAILURE(rc))
+                {
+                    AssertMsgFailed(("rc=%Rrc offCur=%RGp %s\n", rc, offCur, pszDesc));
+                    break;
+                }
+            }
+        }
+
+        if (RT_SUCCESS(rc))
+        {
+            GCPtr |= offPage;
+            *pRCPtr = GCPtr;
+            AssertLogRelReturn(*pRCPtr == GCPtr, VERR_INTERNAL_ERROR);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Maps locked R3 virtual memory into the hypervisor region in the GC.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         VM handle.
+ * @param   pvR3        The ring-3 address of the memory, must be page aligned.
+ * @param   pvR0        The ring-0 address of the memory, must be page aligned. (optional)
+ * @param   cPages      The number of pages.
+ * @param   paPages     The page descriptors.
+ * @param   pszDesc     Mapping description.
+ * @param   pGCPtr      Where to store the GC address corresponding to pvR3.
+ */
+VMMR3DECL(int) MMR3HyperMapPages(PVM pVM, void *pvR3, RTR0PTR pvR0, size_t cPages, PCSUPPAGE paPages, const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapPages: pvR3=%p pvR0=%p cPages=%zu paPages=%p pszDesc=%p:{%s} pGCPtr=%p\n",
+             pvR3, pvR0, cPages, paPages, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pvR3, VERR_INVALID_POINTER);
+    AssertPtrReturn(paPages, VERR_INVALID_POINTER);
+    AssertReturn(cPages > 0, VERR_PAGE_COUNT_OUT_OF_RANGE);
+    AssertReturn(cPages <= VBOX_MAX_ALLOC_PAGE_COUNT, VERR_PAGE_COUNT_OUT_OF_RANGE);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pGCPtr, VERR_INVALID_PARAMETER);
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cPages << PAGE_SHIFT, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Copy the physical page addresses and tell PGM about them.
+         */
+        PRTHCPHYS paHCPhysPages = (PRTHCPHYS)MMR3HeapAlloc(pVM, MM_TAG_MM, sizeof(RTHCPHYS) * cPages);
+        if (paHCPhysPages)
+        {
+            for (size_t i = 0; i < cPages; i++)
+            {
+                AssertReleaseMsgReturn(paPages[i].Phys != 0 && paPages[i].Phys != NIL_RTHCPHYS && !(paPages[i].Phys & PAGE_OFFSET_MASK),
+                                       ("i=%#zx Phys=%RHp %s\n", i, paPages[i].Phys, pszDesc),
+                                       VERR_INTERNAL_ERROR);
+                paHCPhysPages[i] = paPages[i].Phys;
+            }
+
+            if (pVM->mm.s.fPGMInitialized)
+            {
+                for (size_t i = 0; i < cPages; i++)
+                {
+                    rc = PGMMap(pVM, GCPtr + (i << PAGE_SHIFT), paHCPhysPages[i], PAGE_SIZE, 0);
+                    AssertRCBreak(rc);
+                }
+            }
+            if (RT_SUCCESS(rc))
+            {
+                pLookup->enmType = MMLOOKUPHYPERTYPE_LOCKED;
+                pLookup->u.Locked.pvR3          = pvR3;
+                pLookup->u.Locked.pvR0          = pvR0;
+                pLookup->u.Locked.paHCPhysPages = paHCPhysPages;
+
+                /* done. */
+                *pGCPtr   = GCPtr;
+                return rc;
+            }
+            /* Don't care about failure clean, we're screwed if this fails anyway. */
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Reserves a hypervisor memory area.
+ * Most frequent usage is fence pages and dynamically mappings like the guest PD and PDPT.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         VM handle.
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     Mapping description.
+ * @param   pGCPtr      Where to store the assigned GC address. Optional.
+ */
+VMMR3DECL(int) MMR3HyperReserve(PVM pVM, unsigned cb, const char *pszDesc, PRTGCPTR pGCPtr)
+{
+    LogFlow(("MMR3HyperMapHCRam: cb=%d pszDesc=%p:{%s} pGCPtr=%p\n", (int)cb, pszDesc, pszDesc, pGCPtr));
+
+    /*
+     * Validate input.
+     */
+    if (    cb <= 0
+        ||  !pszDesc
+        ||  !*pszDesc)
+    {
+        AssertMsgFailed(("Invalid parameter\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Add the memory to the hypervisor area.
+     */
+    RTGCPTR         GCPtr;
+    PMMLOOKUPHYPER  pLookup;
+    int rc = mmR3HyperMap(pVM, cb, pszDesc, &GCPtr, &pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        pLookup->enmType = MMLOOKUPHYPERTYPE_DYNAMIC;
+        if (pGCPtr)
+            *pGCPtr = GCPtr;
+        return VINF_SUCCESS;
+    }
+    return rc;
+}
+
+
+/**
+ * Adds memory to the hypervisor memory arena.
+ *
+ * @return VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   cb          Size of the memory. Will be rounded up to nearest page.
+ * @param   pszDesc     The description of the memory.
+ * @param   pGCPtr      Where to store the GC address.
+ * @param   ppLookup    Where to store the pointer to the lookup record.
+ * @remark  We assume the threading structure of VBox imposes natural
+ *          serialization of most functions, this one included.
+ */
+static int mmR3HyperMap(PVM pVM, const size_t cb, const char *pszDesc, PRTGCPTR pGCPtr, PMMLOOKUPHYPER *ppLookup)
+{
+    /*
+     * Validate input.
+     */
+    const uint32_t cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    if (pVM->mm.s.offHyperNextStatic + cbAligned >= pVM->mm.s.cbHyperArea) /* don't use the last page, it's a fence. */
+    {
+        AssertMsgFailed(("Out of static mapping space in the HMA! offHyperAreaGC=%x cbAligned=%x cbHyperArea=%x\n",
+                         pVM->mm.s.offHyperNextStatic, cbAligned, pVM->mm.s.cbHyperArea));
+        return VERR_NO_MEMORY;
+    }
+
+    /*
+     * Allocate lookup record.
+     */
+    PMMLOOKUPHYPER  pLookup;
+    int rc = MMHyperAlloc(pVM, sizeof(*pLookup), 1, MM_TAG_MM, (void **)&pLookup);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Initialize it and insert it.
+         */
+        pLookup->offNext        = pVM->mm.s.offLookupHyper;
+        pLookup->cb             = cbAligned;
+        pLookup->off            = pVM->mm.s.offHyperNextStatic;
+        pVM->mm.s.offLookupHyper = (uint8_t *)pLookup - (uint8_t *)pVM->mm.s.pHyperHeapR3;
+        if (pLookup->offNext != (int32_t)NIL_OFFSET)
+            pLookup->offNext   -= pVM->mm.s.offLookupHyper;
+        pLookup->enmType        = MMLOOKUPHYPERTYPE_INVALID;
+        memset(&pLookup->u, 0xff, sizeof(pLookup->u));
+        pLookup->pszDesc        = pszDesc;
+
+        /* Mapping. */
+        *pGCPtr = pVM->mm.s.pvHyperAreaGC + pVM->mm.s.offHyperNextStatic;
+        pVM->mm.s.offHyperNextStatic += cbAligned;
+
+        /* Return pointer. */
+        *ppLookup = pLookup;
+    }
+
+    AssertRC(rc);
+    LogFlow(("mmR3HyperMap: returns %Rrc *pGCPtr=%RGv\n", rc, *pGCPtr));
+    return rc;
+}
+
+
+/**
+ * Allocates a new heap.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   cb          The size of the new heap.
+ * @param   ppHeap      Where to store the heap pointer on successful return.
+ * @param   pR0PtrHeap  Where to store the ring-0 address of the heap on
+ *                      success.
+ */
+static int mmR3HyperHeapCreate(PVM pVM, const size_t cb, PMMHYPERHEAP *ppHeap, PRTR0PTR pR0PtrHeap)
+{
+    /*
+     * Allocate the hypervisor heap.
+     */
+    const uint32_t  cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    uint32_t const  cPages = cbAligned >> PAGE_SHIFT;
+    PSUPPAGE        paPages = (PSUPPAGE)MMR3HeapAlloc(pVM, MM_TAG_MM, cPages * sizeof(paPages[0]));
+    if (!paPages)
+        return VERR_NO_MEMORY;
+    void           *pv;
+    RTR0PTR         pvR0 = NIL_RTR0PTR;
+    int rc = SUPR3PageAllocEx(cPages,
+                              0 /*fFlags*/,
+                              &pv,
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+                              VMMIsHwVirtExtForced(pVM) ? &pvR0 : NULL,
+#else
+                              NULL,
+#endif
+                              paPages);
+    if (RT_SUCCESS(rc))
+    {
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        if (!VMMIsHwVirtExtForced(pVM))
+            pvR0 = NIL_RTR0PTR;
+#else
+        pvR0 = (uintptr_t)pv;
+#endif
+        memset(pv, 0, cbAligned);
+
+        /*
+         * Initialize the heap and first free chunk.
+         */
+        PMMHYPERHEAP pHeap = (PMMHYPERHEAP)pv;
+        pHeap->u32Magic             = MMHYPERHEAP_MAGIC;
+        pHeap->pbHeapR3             = (uint8_t *)pHeap + MMYPERHEAP_HDR_SIZE;
+        pHeap->pbHeapR0             = pvR0 != NIL_RTR0PTR ? pvR0 + MMYPERHEAP_HDR_SIZE : NIL_RTR0PTR;
+        //pHeap->pbHeapRC           = 0; // set by mmR3HyperHeapMap()
+        pHeap->pVMR3                = pVM;
+        pHeap->pVMR0                = pVM->pVMR0;
+        pHeap->pVMRC                = pVM->pVMRC;
+        pHeap->cbHeap               = cbAligned - MMYPERHEAP_HDR_SIZE;
+        pHeap->cbFree               = pHeap->cbHeap - sizeof(MMHYPERCHUNK);
+        //pHeap->offFreeHead        = 0;
+        //pHeap->offFreeTail        = 0;
+        pHeap->offPageAligned       = pHeap->cbHeap;
+        //pHeap->HyperHeapStatTree  = 0;
+        pHeap->paPages              = paPages;
+
+        PMMHYPERCHUNKFREE pFree = (PMMHYPERCHUNKFREE)pHeap->pbHeapR3;
+        pFree->cb                   = pHeap->cbFree;
+        //pFree->core.offNext       = 0;
+        MMHYPERCHUNK_SET_TYPE(&pFree->core, MMHYPERCHUNK_FLAGS_FREE);
+        pFree->core.offHeap         = -(int32_t)MMYPERHEAP_HDR_SIZE;
+        //pFree->offNext            = 0;
+        //pFree->offPrev            = 0;
+
+        STAMR3Register(pVM, &pHeap->cbHeap, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, "/MM/HyperHeap/cbHeap",  STAMUNIT_BYTES, "The heap size.");
+        STAMR3Register(pVM, &pHeap->cbFree, STAMTYPE_U32, STAMVISIBILITY_ALWAYS, "/MM/HyperHeap/cbFree",  STAMUNIT_BYTES, "The free space.");
+
+        *ppHeap = pHeap;
+        *pR0PtrHeap = pvR0;
+        return VINF_SUCCESS;
+    }
+    AssertMsgFailed(("SUPR3PageAllocEx(%d,,,,) -> %Rrc\n", cbAligned >> PAGE_SHIFT, rc));
+
+    *ppHeap = NULL;
+    return rc;
+}
+
+/**
+ * Allocates a new heap.
+ */
+static int mmR3HyperHeapMap(PVM pVM, PMMHYPERHEAP pHeap, PRTGCPTR ppHeapGC)
+{
+    Assert(RT_ALIGN_Z(pHeap->cbHeap + MMYPERHEAP_HDR_SIZE, PAGE_SIZE) == pHeap->cbHeap + MMYPERHEAP_HDR_SIZE);
+    Assert(pHeap->paPages);
+    int rc = MMR3HyperMapPages(pVM,
+                               pHeap,
+                               pHeap->pbHeapR0 != NIL_RTR0PTR ? pHeap->pbHeapR0 - MMYPERHEAP_HDR_SIZE : NIL_RTR0PTR,
+                               (pHeap->cbHeap + MMYPERHEAP_HDR_SIZE) >> PAGE_SHIFT,
+                               pHeap->paPages,
+                               "Heap", ppHeapGC);
+    if (RT_SUCCESS(rc))
+    {
+        pHeap->pVMRC    = pVM->pVMRC;
+        pHeap->pbHeapRC = *ppHeapGC + MMYPERHEAP_HDR_SIZE;
+        /* Reserve a page for fencing. */
+        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+
+        /* We won't need these any more. */
+        MMR3HeapFree(pHeap->paPages);
+        pHeap->paPages = NULL;
+    }
+    return rc;
+}
+
+
+/**
+ * Allocates memory in the Hypervisor (GC VMM) area which never will
+ * be freed and doesn't have any offset based relation to other heap blocks.
+ *
+ * The latter means that two blocks allocated by this API will not have the
+ * same relative position to each other in GC and HC. In short, never use
+ * this API for allocating nodes for an offset based AVL tree!
+ *
+ * The returned memory is of course zeroed.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   cb          Number of bytes to allocate.
+ * @param   uAlignment  Required memory alignment in bytes.
+ *                      Values are 0,8,16,32 and PAGE_SIZE.
+ *                      0 -> default alignment, i.e. 8 bytes.
+ * @param   enmTag      The statistics tag.
+ * @param   ppv         Where to store the address to the allocated
+ *                      memory.
+ * @remark  This is assumed not to be used at times when serialization is required.
+ */
+VMMR3DECL(int) MMR3HyperAllocOnceNoRel(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, void **ppv)
+{
+    return MMR3HyperAllocOnceNoRelEx(pVM, cb, uAlignment, enmTag, 0/*fFlags*/, ppv);
+}
+
+
+/**
+ * Allocates memory in the Hypervisor (GC VMM) area which never will
+ * be freed and doesn't have any offset based relation to other heap blocks.
+ *
+ * The latter means that two blocks allocated by this API will not have the
+ * same relative position to each other in GC and HC. In short, never use
+ * this API for allocating nodes for an offset based AVL tree!
+ *
+ * The returned memory is of course zeroed.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   cb          Number of bytes to allocate.
+ * @param   uAlignment  Required memory alignment in bytes.
+ *                      Values are 0,8,16,32 and PAGE_SIZE.
+ *                      0 -> default alignment, i.e. 8 bytes.
+ * @param   enmTag      The statistics tag.
+ * @param   fFlags      Flags, see MMHYPER_AONR_FLAGS_KERNEL_MAPPING.
+ * @param   ppv         Where to store the address to the allocated memory.
+ * @remark  This is assumed not to be used at times when serialization is required.
+ */
+VMMR3DECL(int) MMR3HyperAllocOnceNoRelEx(PVM pVM, size_t cb, unsigned uAlignment, MMTAG enmTag, uint32_t fFlags, void **ppv)
+{
+    AssertMsg(cb >= 8, ("Hey! Do you really mean to allocate less than 8 bytes?! cb=%d\n", cb));
+    Assert(!(fFlags & ~(MMHYPER_AONR_FLAGS_KERNEL_MAPPING)));
+
+    /*
+     * Choose between allocating a new chunk of HMA memory
+     * and the heap. We will only do BIG allocations from HMA and
+     * only at creation time.
+     */
+    if (   (   cb < _64K
+            && (   uAlignment != PAGE_SIZE
+                || cb < 48*_1K)
+            && !(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING)
+           )
+        ||  VMR3GetState(pVM) != VMSTATE_CREATING
+       )
+    {
+        Assert(!(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING));
+        int rc = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
+        if (    rc != VERR_MM_HYPER_NO_MEMORY
+            ||  cb <= 8*_1K)
+        {
+            Log2(("MMR3HyperAllocOnceNoRel: cb=%#zx uAlignment=%#x returns %Rrc and *ppv=%p\n",
+                  cb, uAlignment, rc, *ppv));
+            return rc;
+        }
+    }
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /*
+     * Set MMHYPER_AONR_FLAGS_KERNEL_MAPPING if we're in going to execute in ring-0.
+     */
+    if (VMMIsHwVirtExtForced(pVM))
+        fFlags |= MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
+#endif
+
+    /*
+     * Validate alignment.
+     */
+    switch (uAlignment)
+    {
+        case 0:
+        case 8:
+        case 16:
+        case 32:
+        case PAGE_SIZE:
+            break;
+        default:
+            AssertMsgFailed(("Invalid alignment %u\n", uAlignment));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate the pages and map them into HMA space.
+     */
+    uint32_t const  cbAligned = RT_ALIGN_32(cb, PAGE_SIZE);
+    AssertReturn(cbAligned >= cb, VERR_INVALID_PARAMETER);
+    uint32_t const  cPages    = cbAligned >> PAGE_SHIFT;
+    PSUPPAGE        paPages   = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(paPages[0]));
+    if (!paPages)
+        return VERR_NO_TMP_MEMORY;
+    void           *pvPages;
+    RTR0PTR         pvR0 = NIL_RTR0PTR;
+    int rc = SUPR3PageAllocEx(cPages,
+                              0 /*fFlags*/,
+                              &pvPages,
+                              fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING ? &pvR0 : NULL,
+                              paPages);
+    if (RT_SUCCESS(rc))
+    {
+        if (!(fFlags & MMHYPER_AONR_FLAGS_KERNEL_MAPPING))
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pvR0 = NIL_RTR0PTR;
+#else
+            pvR0 = (RTR0PTR)pvPages;
+#endif
+
+        memset(pvPages, 0, cbAligned);
+
+        RTGCPTR GCPtr;
+        rc = MMR3HyperMapPages(pVM,
+                               pvPages,
+                               pvR0,
+                               cPages,
+                               paPages,
+                               MMR3HeapAPrintf(pVM, MM_TAG_MM, "alloc once (%s)", mmGetTagName(enmTag)),
+                               &GCPtr);
+        if (RT_SUCCESS(rc))
+        {
+            *ppv = pvPages;
+            Log2(("MMR3HyperAllocOnceNoRel: cbAligned=%#x uAlignment=%#x returns VINF_SUCCESS and *ppv=%p\n",
+                  cbAligned, uAlignment, *ppv));
+            MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+            return rc;
+        }
+        AssertMsgFailed(("Failed to allocate %zd bytes! %Rrc\n", cbAligned, rc));
+        SUPR3PageFreeEx(pvPages, cPages);
+
+
+        /*
+         * HACK ALERT! Try allocate it off the heap so that we don't freak
+         * out during vga/vmmdev mmio2 allocation with certain ram sizes.
+         */
+        /** @todo make a proper fix for this so we will never end up in this kind of situation! */
+        Log(("MMR3HyperAllocOnceNoRel: MMR3HyperMapHCRam failed with rc=%Rrc, try MMHyperAlloc(,%#x,,) instead\n",  rc, cb));
+        int rc2 = MMHyperAlloc(pVM, cb, uAlignment, enmTag, ppv);
+        if (RT_SUCCESS(rc2))
+        {
+            Log2(("MMR3HyperAllocOnceNoRel: cb=%#x uAlignment=%#x returns %Rrc and *ppv=%p\n",
+                  cb, uAlignment, rc, *ppv));
+            return rc;
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to allocate %zd bytes! %Rrc\n", cbAligned, rc));
+
+    if (rc == VERR_NO_MEMORY)
+        rc = VERR_MM_HYPER_NO_MEMORY;
+    LogRel(("MMR3HyperAllocOnceNoRel: cb=%#zx uAlignment=%#x returns %Rrc\n", cb, uAlignment, rc));
+    return rc;
+}
+
+
+/**
+ * Lookus up a ring-3 pointer to HMA.
+ *
+ * @returns The lookup record on success, NULL on failure.
+ * @param   pVM                 The VM handle.
+ * @param   pvR3                The ring-3 address to look up.
+ */
+DECLINLINE(PMMLOOKUPHYPER) mmR3HyperLookupR3(PVM pVM, void *pvR3)
+{
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.Locked.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup;
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.HCPhys.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup;
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                /** @todo ?    */
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                return NULL;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            return NULL;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+}
+
+
+/**
+ * Set / unset guard status on one or more hyper heap pages.
+ *
+ * @returns VBox status code (first failure).
+ * @param   pVM                 The VM handle.
+ * @param   pvStart             The hyper heap page address. Must be page
+ *                              aligned.
+ * @param   cb                  The number of bytes. Must be page aligned.
+ * @param   fSet                Whether to set or unset guard page status.
+ */
+VMMR3DECL(int) MMR3HyperSetGuard(PVM pVM, void *pvStart, size_t cb, bool fSet)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(!((uintptr_t)pvStart & PAGE_OFFSET_MASK), VERR_INVALID_POINTER);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(cb <= UINT32_MAX, VERR_INVALID_PARAMETER);
+    PMMLOOKUPHYPER pLookup = mmR3HyperLookupR3(pVM, pvStart);
+    AssertReturn(pLookup, VERR_INVALID_PARAMETER);
+    AssertReturn(pLookup->enmType == MMLOOKUPHYPERTYPE_LOCKED, VERR_INVALID_PARAMETER);
+
+    /*
+     * Get down to business.
+     * Note! We quietly ignore errors from the support library since the
+     *       protection stuff isn't possible to implement on all platforms.
+     */
+    uint8_t    *pbR3  = (uint8_t *)pLookup->u.Locked.pvR3;
+    RTR0PTR     R0Ptr = pLookup->u.Locked.pvR0 != (uintptr_t)pLookup->u.Locked.pvR3
+                      ? pLookup->u.Locked.pvR0
+                      : NIL_RTR0PTR;
+    uint32_t    off   = (uint32_t)((uint8_t *)pvStart - pbR3);
+    int         rc;
+    if (fSet)
+    {
+        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, pvStart), cb, 0);
+        SUPR3PageProtect(pbR3, R0Ptr, off, (uint32_t)cb, RTMEM_PROT_NONE);
+    }
+    else
+    {
+        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, pvStart), cb, X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
+        SUPR3PageProtect(pbR3, R0Ptr, off, (uint32_t)cb, RTMEM_PROT_READ | RTMEM_PROT_WRITE);
+    }
+    return rc;
+}
+
+
+/**
+ * Convert hypervisor HC virtual address to HC physical address.
+ *
+ * @returns HC physical address.
+ * @param   pVM         VM Handle
+ * @param   pvR3        Host context virtual address.
+ */
+VMMR3DECL(RTHCPHYS) MMR3HyperHCVirt2HCPhys(PVM pVM, void *pvR3)
+{
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.Locked.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup->u.Locked.paHCPhysPages[off >> PAGE_SHIFT] | (off & PAGE_OFFSET_MASK);
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                unsigned off = (uint8_t *)pvR3 - (uint8_t *)pLookup->u.HCPhys.pvR3;
+                if (off < pLookup->cb)
+                    return pLookup->u.HCPhys.HCPhys + off;
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                /* can (or don't want to) convert these kind of records. */
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+
+    AssertMsgFailed(("pvR3=%p is not inside the hypervisor memory area!\n", pvR3));
+    return NIL_RTHCPHYS;
+}
+
+
+/**
+ * Implements the return case of MMR3HyperQueryInfoFromHCPhys.
+ *
+ * @returns VINF_SUCCESS, VINF_BUFFER_OVERFLOW.
+ * @param   pVM                 The VM handle.
+ * @param   HCPhys              The host physical address to look for.
+ * @param   pLookup             The HMA lookup entry corresponding to HCPhys.
+ * @param   pszWhat             Where to return the description.
+ * @param   cbWhat              Size of the return buffer.
+ * @param   pcbAlloc            Where to return the size of whatever it is.
+ */
+static int mmR3HyperQueryInfoFromHCPhysFound(PVM pVM, RTHCPHYS HCPhys, PMMLOOKUPHYPER pLookup,
+                                             char *pszWhat, size_t cbWhat, uint32_t *pcbAlloc)
+{
+    *pcbAlloc = pLookup->cb;
+    int rc = RTStrCopy(pszWhat, cbWhat, pLookup->pszDesc);
+    return rc == VERR_BUFFER_OVERFLOW ? VINF_BUFFER_OVERFLOW : rc;
+}
+
+
+/**
+ * Scans the HMA for the physical page and reports back a description if found.
+ *
+ * @returns VINF_SUCCESS, VINF_BUFFER_OVERFLOW, VERR_NOT_FOUND.
+ * @param   pVM                 The VM handle.
+ * @param   HCPhys              The host physical address to look for.
+ * @param   pszWhat             Where to return the description.
+ * @param   cbWhat              Size of the return buffer.
+ * @param   pcbAlloc            Where to return the size of whatever it is.
+ */
+VMMR3_INT_DECL(int) MMR3HyperQueryInfoFromHCPhys(PVM pVM, RTHCPHYS HCPhys, char *pszWhat, size_t cbWhat, uint32_t *pcbAlloc)
+{
+    RTHCPHYS        HCPhysPage = HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK;
+    PMMLOOKUPHYPER  pLookup    = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+            {
+                uint32_t i = pLookup->cb >> PAGE_SHIFT;
+                while (i-- > 0)
+                    if (pLookup->u.Locked.paHCPhysPages[i] == HCPhysPage)
+                        return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+            {
+                if (pLookup->u.HCPhys.HCPhys - HCPhysPage < pLookup->cb)
+                    return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
+                break;
+            }
+
+            case MMLOOKUPHYPERTYPE_MMIO2:
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+            {
+                /* brute force. */
+                uint32_t i = pLookup->cb >> PAGE_SHIFT;
+                while (i-- > 0)
+                {
+                    RTGCPTR     GCPtr = pLookup->off + pVM->mm.s.pvHyperAreaGC;
+                    RTHCPHYS    HCPhysCur;
+                    int rc = PGMMapGetPage(pVM, GCPtr, NULL, &HCPhysCur);
+                    if (RT_SUCCESS(rc) && HCPhysCur == HCPhysPage)
+                        return mmR3HyperQueryInfoFromHCPhysFound(pVM, HCPhys, pLookup, pszWhat, cbWhat, pcbAlloc);
+                }
+                break;
+            }
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+#if 0 /* unused, not implemented */
+/**
+ * Convert hypervisor HC physical address to HC virtual address.
+ *
+ * @returns HC virtual address.
+ * @param   pVM         VM Handle
+ * @param   HCPhys      Host context physical address.
+ */
+VMMR3DECL(void *) MMR3HyperHCPhys2HCVirt(PVM pVM, RTHCPHYS HCPhys)
+{
+    void *pv;
+    int rc = MMR3HyperHCPhys2HCVirtEx(pVM, HCPhys, &pv);
+    if (RT_SUCCESS(rc))
+        return pv;
+    AssertMsgFailed(("Invalid address HCPhys=%x rc=%d\n", HCPhys, rc));
+    return NULL;
+}
+
+
+/**
+ * Convert hypervisor HC physical address to HC virtual address.
+ *
+ * @returns VBox status.
+ * @param   pVM         VM Handle
+ * @param   HCPhys      Host context physical address.
+ * @param   ppv         Where to store the HC virtual address.
+ */
+VMMR3DECL(int)   MMR3HyperHCPhys2HCVirtEx(PVM pVM, RTHCPHYS HCPhys, void **ppv)
+{
+    /*
+     * Linear search.
+     */
+    /** @todo implement when actually used. */
+    return VERR_INVALID_POINTER;
+}
+#endif /* unused, not implemented */
+
+
+/**
+ * Read hypervisor memory from GC virtual address.
+ *
+ * @returns VBox status.
+ * @param   pVM         VM handle.
+ * @param   pvDst       Destination address (HC of course).
+ * @param   GCPtr       GC virtual address.
+ * @param   cb          Number of bytes to read.
+ *
+ * @remarks For DBGF only.
+ */
+VMMR3DECL(int) MMR3HyperReadGCVirt(PVM pVM, void *pvDst, RTGCPTR GCPtr, size_t cb)
+{
+    if (GCPtr - pVM->mm.s.pvHyperAreaGC >= pVM->mm.s.cbHyperArea)
+        return VERR_INVALID_POINTER;
+    return PGMR3MapRead(pVM, pvDst, GCPtr, cb);
+}
+
+
+/**
+ * Info handler for 'hma', it dumps the list of lookup records for the hypervisor memory area.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pHlp        Callback functions for doing output.
+ * @param   pszArgs     Argument string. Optional and specific to the handler.
+ */
+static DECLCALLBACK(void) mmR3HyperInfoHma(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    pHlp->pfnPrintf(pHlp, "Hypervisor Memory Area (HMA) Layout: Base %RGv, 0x%08x bytes\n",
+                    pVM->mm.s.pvHyperAreaGC, pVM->mm.s.cbHyperArea);
+
+    PMMLOOKUPHYPER  pLookup = (PMMLOOKUPHYPER)((uint8_t *)pVM->mm.s.pHyperHeapR3 + pVM->mm.s.offLookupHyper);
+    for (;;)
+    {
+        switch (pLookup->enmType)
+        {
+            case MMLOOKUPHYPERTYPE_LOCKED:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %RHv %RHv LOCKED  %-*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                pLookup->u.Locked.pvR3,
+                                pLookup->u.Locked.pvR0,
+                                sizeof(RTHCPTR) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_HCPHYS:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %RHv %RHv HCPHYS  %RHp %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                pLookup->u.HCPhys.pvR3,
+                                pLookup->u.HCPhys.pvR0,
+                                pLookup->u.HCPhys.HCPhys,
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_GCPHYS:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s GCPHYS  %RGp%*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
+                                pLookup->u.GCPhys.GCPhys, RT_ABS((int)(sizeof(RTHCPHYS) - sizeof(RTGCPHYS))) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_MMIO2:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s MMIO2   %RGp%*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
+                                pLookup->u.MMIO2.off, RT_ABS((int)(sizeof(RTHCPHYS) - sizeof(RTGCPHYS))) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            case MMLOOKUPHYPERTYPE_DYNAMIC:
+                pHlp->pfnPrintf(pHlp, "%RGv-%RGv %*s DYNAMIC %*s %s\n",
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC,
+                                pLookup->off + pVM->mm.s.pvHyperAreaGC + pLookup->cb,
+                                sizeof(RTHCPTR) * 2 * 2 + 1, "",
+                                sizeof(RTHCPTR) * 2, "",
+                                pLookup->pszDesc);
+                break;
+
+            default:
+                AssertMsgFailed(("enmType=%d\n", pLookup->enmType));
+                break;
+        }
+
+        /* next */
+        if ((unsigned)pLookup->offNext == NIL_OFFSET)
+            break;
+        pLookup = (PMMLOOKUPHYPER)((uint8_t *)pLookup + pLookup->offNext);
+    }
+}
Index: /trunk/src/VBox/VMM/VMMR3/MMPagePool.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/MMPagePool.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/MMPagePool.cpp	(revision 35333)
@@ -0,0 +1,527 @@
+/* $Id$ */
+/** @file
+ * MM - Memory Manager - Page Pool.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_POOL
+#include <VBox/mm.h>
+#include <VBox/pgm.h>
+#include <VBox/stam.h>
+#include "MMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include "internal/pgm.h"
+#include <iprt/alloc.h>
+#include <iprt/assert.h>
+#define USE_INLINE_ASM_BIT_OPS
+#ifdef USE_INLINE_ASM_BIT_OPS
+# include <iprt/asm.h>
+#endif
+#include <iprt/string.h>
+
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+#ifdef IN_RING3
+static void *   mmR3PagePoolAlloc(PMMPAGEPOOL pPool);
+static void     mmR3PagePoolFree(PMMPAGEPOOL pPool, void *pv);
+#endif
+
+
+/**
+ * Initializes the page pool
+ *
+ * @return  VBox status.
+ * @param   pVM     VM handle.
+ * @thread  The Emulation Thread.
+ */
+int mmR3PagePoolInit(PVM pVM)
+{
+    AssertMsg(!pVM->mm.s.pPagePoolR3, ("Already initialized!\n"));
+
+    /*
+     * Allocate the pool structures.
+     */
+    /** @todo @bufref{1865},@bufref{3202}: mapping the page pool page into
+     *        ring-0. Need to change the wasy we allocate it... */
+    AssertReleaseReturn(sizeof(*pVM->mm.s.pPagePoolR3) + sizeof(*pVM->mm.s.pPagePoolLowR3) < PAGE_SIZE, VERR_INTERNAL_ERROR);
+    int rc = SUPR3PageAllocEx(1, 0 /*fFlags*/, (void **)&pVM->mm.s.pPagePoolR3, NULL /*pR0Ptr*/, NULL /*paPages*/);
+    if (RT_FAILURE(rc))
+        return rc;
+    memset(pVM->mm.s.pPagePoolR3, 0, PAGE_SIZE);
+    pVM->mm.s.pPagePoolR3->pVM = pVM;
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cPages,         STAMTYPE_U32,     "/MM/Page/Def/cPages",        STAMUNIT_PAGES, "Number of pages in the default pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cFreePages,     STAMTYPE_U32,     "/MM/Page/Def/cFreePages",    STAMUNIT_PAGES, "Number of free pages in the default pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cSubPools,      STAMTYPE_U32,     "/MM/Page/Def/cSubPools",     STAMUNIT_COUNT, "Number of sub pools in the default pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cAllocCalls,    STAMTYPE_COUNTER, "/MM/Page/Def/cAllocCalls",   STAMUNIT_CALLS, "Number of MMR3PageAlloc() calls for the default pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cFreeCalls,     STAMTYPE_COUNTER, "/MM/Page/Def/cFreeCalls",    STAMUNIT_CALLS, "Number of MMR3PageFree()+MMR3PageFreeByPhys() calls for the default pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cToPhysCalls,   STAMTYPE_COUNTER, "/MM/Page/Def/cToPhysCalls",  STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for this pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cToVirtCalls,   STAMTYPE_COUNTER, "/MM/Page/Def/cToVirtCalls",  STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the default pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolR3->cErrors,        STAMTYPE_COUNTER, "/MM/Page/Def/cErrors",       STAMUNIT_ERRORS,"Number of errors for the default pool.");
+
+    pVM->mm.s.pPagePoolLowR3 = pVM->mm.s.pPagePoolR3 + 1;
+    pVM->mm.s.pPagePoolLowR3->pVM = pVM;
+    pVM->mm.s.pPagePoolLowR3->fLow = true;
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cPages,      STAMTYPE_U32,     "/MM/Page/Low/cPages",        STAMUNIT_PAGES, "Number of pages in the <4GB pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cFreePages,  STAMTYPE_U32,     "/MM/Page/Low/cFreePages",    STAMUNIT_PAGES, "Number of free pages in the <4GB pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cSubPools,   STAMTYPE_U32,     "/MM/Page/Low/cSubPools",     STAMUNIT_COUNT, "Number of sub pools in the <4GB pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cAllocCalls, STAMTYPE_COUNTER, "/MM/Page/Low/cAllocCalls",   STAMUNIT_CALLS, "Number of MMR3PageAllocLow() calls for the <4GB pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cFreeCalls,  STAMTYPE_COUNTER, "/MM/Page/Low/cFreeCalls",    STAMUNIT_CALLS, "Number of MMR3PageFreeLow()+MMR3PageFreeByPhys() calls for the <4GB pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cToPhysCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToPhysCalls",  STAMUNIT_CALLS, "Number of MMR3Page2Phys() calls for the <4GB pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cToVirtCalls,STAMTYPE_COUNTER, "/MM/Page/Low/cToVirtCalls",  STAMUNIT_CALLS, "Number of MMR3PagePhys2Page()+MMR3PageFreeByPhys() calls for the <4GB pool.");
+    STAM_REG(pVM, &pVM->mm.s.pPagePoolLowR3->cErrors,     STAMTYPE_COUNTER, "/MM/Page/Low/cErrors",       STAMUNIT_ERRORS,"Number of errors for the <4GB pool.");
+
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    pVM->mm.s.pPagePoolR0 = (uintptr_t)pVM->mm.s.pPagePoolR3;
+    pVM->mm.s.pPagePoolLowR0 = (uintptr_t)pVM->mm.s.pPagePoolLowR3;
+#endif
+
+    /** @todo init a mutex? */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Release all locks and free the allocated memory.
+ *
+ * @param   pVM     VM handle.
+ * @thread  The Emulation Thread.
+ */
+void mmR3PagePoolTerm(PVM pVM)
+{
+    if (pVM->mm.s.pPagePoolR3)
+    {
+        /*
+         * Unlock all memory held by subpools and free the memory.
+         * (The MM Heap will free the memory used for internal stuff.)
+         */
+        Assert(!pVM->mm.s.pPagePoolR3->fLow);
+        PMMPAGESUBPOOL  pSubPool = pVM->mm.s.pPagePoolR3->pHead;
+        while (pSubPool)
+        {
+            int rc = SUPR3PageFreeEx(pSubPool->pvPages, pSubPool->cPages);
+            AssertMsgRC(rc, ("SUPR3PageFreeEx(%p) failed with rc=%Rrc\n", pSubPool->pvPages, rc));
+            pSubPool->pvPages = NULL;
+
+            /* next */
+            pSubPool = pSubPool->pNext;
+        }
+        pVM->mm.s.pPagePoolR3 = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pVM->mm.s.pPagePoolR0 = NIL_RTR0PTR;
+#endif
+    }
+
+    if (pVM->mm.s.pPagePoolLowR3)
+    {
+        /*
+         * Free the memory.
+         */
+        Assert(pVM->mm.s.pPagePoolLowR3->fLow);
+        PMMPAGESUBPOOL  pSubPool = pVM->mm.s.pPagePoolLowR3->pHead;
+        while (pSubPool)
+        {
+            int rc = SUPR3LowFree(pSubPool->pvPages, pSubPool->cPages);
+            AssertMsgRC(rc, ("SUPR3LowFree(%p) failed with rc=%d\n", pSubPool->pvPages, rc));
+            pSubPool->pvPages = NULL;
+
+            /* next */
+            pSubPool = pSubPool->pNext;
+        }
+        pVM->mm.s.pPagePoolLowR3 = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pVM->mm.s.pPagePoolLowR0 = NIL_RTR0PTR;
+#endif
+    }
+}
+
+
+/**
+ * Allocates a page from the page pool.
+ *
+ * @returns Pointer to allocated page(s).
+ * @returns NULL on failure.
+ * @param   pPool   Pointer to the page pool.
+ * @thread  The Emulation Thread.
+ */
+DECLINLINE(void *) mmR3PagePoolAlloc(PMMPAGEPOOL pPool)
+{
+    VM_ASSERT_EMT(pPool->pVM);
+    STAM_COUNTER_INC(&pPool->cAllocCalls);
+
+    /*
+     * Walk free list.
+     */
+    if (pPool->pHeadFree)
+    {
+        PMMPAGESUBPOOL  pSub = pPool->pHeadFree;
+        /* decrement free count and unlink if no more free entries. */
+        if (!--pSub->cPagesFree)
+            pPool->pHeadFree = pSub->pNextFree;
+#ifdef VBOX_WITH_STATISTICS
+        pPool->cFreePages--;
+#endif
+
+        /* find free spot in bitmap. */
+#ifdef USE_INLINE_ASM_BIT_OPS
+        const int iPage = ASMBitFirstClear(pSub->auBitmap, pSub->cPages);
+        if (iPage >= 0)
+        {
+            Assert(!ASMBitTest(pSub->auBitmap, iPage));
+            ASMBitSet(pSub->auBitmap, iPage);
+            return (uint8_t *)pSub->pvPages + PAGE_SIZE * iPage;
+        }
+#else
+        unsigned   *pu = &pSub->auBitmap[0];
+        unsigned   *puEnd = &pSub->auBitmap[pSub->cPages / (sizeof(pSub->auBitmap) * 8)];
+        while (pu < puEnd)
+        {
+            unsigned u;
+            if ((u = *pu) != ~0U)
+            {
+                unsigned iBit = 0;
+                unsigned uMask = 1;
+                while (iBit < sizeof(pSub->auBitmap[0]) * 8)
+                {
+                    if (!(u & uMask))
+                    {
+                        *pu |= uMask;
+                        return (uint8_t *)pSub->pvPages
+                            + PAGE_SIZE * (iBit + ((uint8_t *)pu - (uint8_t *)&pSub->auBitmap[0]) * 8);
+                    }
+                    iBit++;
+                    uMask <<= 1;
+                }
+                STAM_COUNTER_INC(&pPool->cErrors);
+                AssertMsgFailed(("how odd, expected to find a free bit in %#x, but didn't\n", u));
+            }
+            /* next */
+            pu++;
+        }
+#endif
+        STAM_COUNTER_INC(&pPool->cErrors);
+#ifdef VBOX_WITH_STATISTICS
+        pPool->cFreePages++;
+#endif
+        AssertMsgFailed(("how strange, expected to find a free bit in %p, but didn't (%d pages supposed to be free!)\n", pSub, pSub->cPagesFree + 1));
+    }
+
+    /*
+     * Allocate new subpool.
+     */
+    unsigned        cPages = !pPool->fLow ? 128 : 32;
+    PMMPAGESUBPOOL  pSub;
+    int rc = MMHyperAlloc(pPool->pVM,
+                          RT_OFFSETOF(MMPAGESUBPOOL, auBitmap[cPages / (sizeof(pSub->auBitmap[0] * 8))])
+                          + (sizeof(SUPPAGE) + sizeof(MMPPLOOKUPHCPHYS)) * cPages
+                          + sizeof(MMPPLOOKUPHCPTR),
+                          0,
+                          MM_TAG_MM_PAGE,
+                          (void **)&pSub);
+    if (RT_FAILURE(rc))
+        return NULL;
+
+    PSUPPAGE paPhysPages = (PSUPPAGE)&pSub->auBitmap[cPages / (sizeof(pSub->auBitmap[0]) * 8)];
+    Assert((uintptr_t)paPhysPages >= (uintptr_t)&pSub->auBitmap[1]);
+    if (!pPool->fLow)
+    {
+        rc = SUPR3PageAllocEx(cPages,
+                              0 /* fFlags */,
+                              &pSub->pvPages,
+                              NULL,
+                              paPhysPages);
+        if (RT_FAILURE(rc))
+            rc = VMSetError(pPool->pVM, rc, RT_SRC_POS,
+                            N_("Failed to lock host %zd bytes of memory (out of memory)"), (size_t)cPages << PAGE_SHIFT);
+    }
+    else
+        rc = SUPR3LowAlloc(cPages, &pSub->pvPages, NULL, paPhysPages);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Setup the sub structure and allocate the requested page.
+         */
+        pSub->cPages    = cPages;
+        pSub->cPagesFree= cPages - 1;
+        pSub->paPhysPages = paPhysPages;
+        memset(pSub->auBitmap, 0, cPages / 8);
+        /* allocate first page. */
+        pSub->auBitmap[0] |= 1;
+        /* link into free chain. */
+        pSub->pNextFree = pPool->pHeadFree;
+        pPool->pHeadFree= pSub;
+        /* link into main chain. */
+        pSub->pNext     = pPool->pHead;
+        pPool->pHead    = pSub;
+        /* update pool statistics. */
+        pPool->cSubPools++;
+        pPool->cPages  += cPages;
+#ifdef VBOX_WITH_STATISTICS
+        pPool->cFreePages += cPages - 1;
+#endif
+
+        /*
+         * Initialize the physical pages with backpointer to subpool.
+         */
+        unsigned i = cPages;
+        while (i-- > 0)
+        {
+            AssertMsg(paPhysPages[i].Phys && !(paPhysPages[i].Phys & PAGE_OFFSET_MASK),
+                      ("i=%d Phys=%d\n", i, paPhysPages[i].Phys));
+            paPhysPages[i].uReserved = (RTHCUINTPTR)pSub;
+        }
+
+        /*
+         * Initialize the physical lookup record with backpointers to the physical pages.
+         */
+        PMMPPLOOKUPHCPHYS paLookupPhys = (PMMPPLOOKUPHCPHYS)&paPhysPages[cPages];
+        i = cPages;
+        while (i-- > 0)
+        {
+            paLookupPhys[i].pPhysPage = &paPhysPages[i];
+            paLookupPhys[i].Core.Key = paPhysPages[i].Phys;
+            RTAvlHCPhysInsert(&pPool->pLookupPhys, &paLookupPhys[i].Core);
+        }
+
+        /*
+         * And the one record for virtual memory lookup.
+         */
+        PMMPPLOOKUPHCPTR   pLookupVirt = (PMMPPLOOKUPHCPTR)&paLookupPhys[cPages];
+        pLookupVirt->pSubPool = pSub;
+        pLookupVirt->Core.Key = pSub->pvPages;
+        RTAvlPVInsert(&pPool->pLookupVirt, &pLookupVirt->Core);
+
+        /* return allocated page (first). */
+        return pSub->pvPages;
+    }
+
+    MMHyperFree(pPool->pVM, pSub);
+    STAM_COUNTER_INC(&pPool->cErrors);
+    if (pPool->fLow)
+        VMSetError(pPool->pVM, rc, RT_SRC_POS,
+                   N_("Failed to expand page pool for memory below 4GB. Current size: %d pages"),
+                   pPool->cPages);
+    AssertMsgFailed(("Failed to expand pool%s. rc=%Rrc poolsize=%d\n",
+                     pPool->fLow ? " (<4GB)" : "", rc, pPool->cPages));
+    return NULL;
+}
+
+
+/**
+ * Frees a page from the page pool.
+ *
+ * @param   pPool   Pointer to the page pool.
+ * @param   pv      Pointer to the page to free.
+ *                  I.e. pointer returned by mmR3PagePoolAlloc().
+ * @thread  The Emulation Thread.
+ */
+DECLINLINE(void) mmR3PagePoolFree(PMMPAGEPOOL pPool, void *pv)
+{
+    VM_ASSERT_EMT(pPool->pVM);
+    STAM_COUNTER_INC(&pPool->cFreeCalls);
+
+    /*
+     * Lookup the virtual address.
+     */
+    PMMPPLOOKUPHCPTR pLookup = (PMMPPLOOKUPHCPTR)RTAvlPVGetBestFit(&pPool->pLookupVirt, pv, false);
+    if (    !pLookup
+        ||  (uint8_t *)pv >= (uint8_t *)pLookup->pSubPool->pvPages + (pLookup->pSubPool->cPages << PAGE_SHIFT)
+        )
+    {
+        STAM_COUNTER_INC(&pPool->cErrors);
+        AssertMsgFailed(("invalid pointer %p\n", pv));
+        return;
+    }
+
+    /*
+     * Free the page.
+     */
+    PMMPAGESUBPOOL  pSubPool = pLookup->pSubPool;
+    /* clear bitmap bit */
+    const unsigned  iPage = ((uint8_t *)pv - (uint8_t *)pSubPool->pvPages) >> PAGE_SHIFT;
+#ifdef USE_INLINE_ASM_BIT_OPS
+    Assert(ASMBitTest(pSubPool->auBitmap, iPage));
+    ASMBitClear(pSubPool->auBitmap, iPage);
+#else
+    unsigned    iBit   = iPage % (sizeof(pSubPool->auBitmap[0]) * 8);
+    unsigned    iIndex = iPage / (sizeof(pSubPool->auBitmap[0]) * 8);
+    pSubPool->auBitmap[iIndex] &= ~(1 << iBit);
+#endif
+    /* update stats. */
+    pSubPool->cPagesFree++;
+#ifdef VBOX_WITH_STATISTICS
+    pPool->cFreePages++;
+#endif
+    if (pSubPool->cPagesFree == 1)
+    {
+        pSubPool->pNextFree = pPool->pHeadFree;
+        pPool->pHeadFree = pSubPool;
+    }
+}
+
+
+/**
+ * Allocates a page from the page pool.
+ *
+ * This function may returns pages which has physical addresses any
+ * where. If you require a page to be within the first 4GB of physical
+ * memory, use MMR3PageAllocLow().
+ *
+ * @returns Pointer to the allocated page page.
+ * @returns NULL on failure.
+ * @param   pVM         VM handle.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(void *) MMR3PageAlloc(PVM pVM)
+{
+    /* Note: unprotected by locks; currently fine as it's used during init or under the PGM lock */
+    return mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3);
+}
+
+
+/**
+ * Allocates a page from the page pool and return its physical address.
+ *
+ * This function may returns pages which has physical addresses any
+ * where. If you require a page to be within the first 4GB of physical
+ * memory, use MMR3PageAllocLow().
+ *
+ * @returns Pointer to the allocated page page.
+ * @returns NIL_RTHCPHYS on failure.
+ * @param   pVM         VM handle.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(RTHCPHYS) MMR3PageAllocPhys(PVM pVM)
+{
+    /* Note: unprotected by locks; currently fine as it's used during init or under the PGM lock */
+    /** @todo optimize this, it's the most common case now. */
+    void *pv = mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3);
+    if (pv)
+        return mmPagePoolPtr2Phys(pVM->mm.s.pPagePoolR3, pv);
+    return NIL_RTHCPHYS;
+}
+
+
+/**
+ * Frees a page allocated from the page pool by MMR3PageAlloc() or
+ * MMR3PageAllocPhys().
+ *
+ * @param   pVM         VM handle.
+ * @param   pvPage      Pointer to the page.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(void) MMR3PageFree(PVM pVM, void *pvPage)
+{
+    mmR3PagePoolFree(pVM->mm.s.pPagePoolR3, pvPage);
+}
+
+
+/**
+ * Allocates a page from the low page pool.
+ *
+ * @returns Pointer to the allocated page.
+ * @returns NULL on failure.
+ * @param   pVM         VM handle.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(void *) MMR3PageAllocLow(PVM pVM)
+{
+    return mmR3PagePoolAlloc(pVM->mm.s.pPagePoolLowR3);
+}
+
+
+/**
+ * Frees a page allocated from the page pool by MMR3PageAllocLow().
+ *
+ * @param   pVM         VM handle.
+ * @param   pvPage      Pointer to the page.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(void) MMR3PageFreeLow(PVM pVM, void *pvPage)
+{
+    mmR3PagePoolFree(pVM->mm.s.pPagePoolLowR3, pvPage);
+}
+
+
+/**
+ * Free a page allocated from the page pool by physical address.
+ * This works for pages allocated by MMR3PageAlloc(), MMR3PageAllocPhys()
+ * and MMR3PageAllocLow().
+ *
+ * @param   pVM         VM handle.
+ * @param   HCPhysPage  The physical address of the page to be freed.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(void) MMR3PageFreeByPhys(PVM pVM, RTHCPHYS HCPhysPage)
+{
+    void *pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePoolR3, HCPhysPage);
+    if (!pvPage)
+        pvPage = mmPagePoolPhys2Ptr(pVM->mm.s.pPagePoolLowR3, HCPhysPage);
+    if (pvPage)
+        mmR3PagePoolFree(pVM->mm.s.pPagePoolR3, pvPage);
+    else
+        AssertMsgFailed(("Invalid address HCPhysPT=%#x\n", HCPhysPage));
+}
+
+
+/**
+ * Gets the HC pointer to the dummy page.
+ *
+ * The dummy page is used as a place holder to prevent potential bugs
+ * from doing really bad things to the system.
+ *
+ * @returns Pointer to the dummy page.
+ * @param   pVM         VM handle.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(void *) MMR3PageDummyHCPtr(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    if (!pVM->mm.s.pvDummyPage)
+    {
+        pVM->mm.s.pvDummyPage = mmR3PagePoolAlloc(pVM->mm.s.pPagePoolR3);
+        AssertRelease(pVM->mm.s.pvDummyPage);
+        pVM->mm.s.HCPhysDummyPage = mmPagePoolPtr2Phys(pVM->mm.s.pPagePoolR3, pVM->mm.s.pvDummyPage);
+        AssertRelease(!(pVM->mm.s.HCPhysDummyPage & ~X86_PTE_PAE_PG_MASK));
+    }
+    return pVM->mm.s.pvDummyPage;
+}
+
+
+/**
+ * Gets the HC Phys to the dummy page.
+ *
+ * The dummy page is used as a place holder to prevent potential bugs
+ * from doing really bad things to the system.
+ *
+ * @returns Pointer to the dummy page.
+ * @param   pVM         VM handle.
+ * @thread  The Emulation Thread.
+ */
+VMMR3DECL(RTHCPHYS) MMR3PageDummyHCPhys(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    if (!pVM->mm.s.pvDummyPage)
+        MMR3PageDummyHCPtr(pVM);
+    return pVM->mm.s.HCPhysDummyPage;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/MMUkHeap.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/MMUkHeap.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/MMUkHeap.cpp	(revision 35333)
@@ -0,0 +1,422 @@
+/* $Id$ */
+/** @file
+ * MM - Memory Manager - Ring-3 Heap with kernel accessible mapping.
+ */
+
+/*
+ * Copyright (C) 2006-2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HEAP
+#include <VBox/mm.h>
+#include <VBox/stam.h>
+#include "MMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/heap.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr);
+
+
+
+/**
+ * Create a User-kernel heap.
+ *
+ * This does not require SUPLib to be initialized as we'll lazily allocate the
+ * kernel accessible memory on the first alloc call.
+ *
+ * @returns VBox status.
+ * @param   pVM     The handle to the VM the heap should be associated with.
+ * @param   ppHeap  Where to store the heap pointer.
+ */
+int mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap)
+{
+    PMMUKHEAP pHeap = (PMMUKHEAP)MMR3HeapAllocZU(pUVM, MM_TAG_MM, sizeof(MMUKHEAP));
+    if (pHeap)
+    {
+        int rc = RTCritSectInit(&pHeap->Lock);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize the global stat record.
+             */
+            pHeap->pUVM = pUVM;
+#ifdef MMUKHEAP_WITH_STATISTICS
+            PMMUKHEAPSTAT pStat = &pHeap->Stat;
+            STAMR3RegisterU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cAllocations",     STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cReallocations",   STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFrees",           STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.");
+            STAMR3RegisterU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cFailures",        STAMUNIT_COUNT, "Number of failures.");
+            STAMR3RegisterU(pUVM, &pStat->cbCurAllocated, sizeof(pStat->cbCurAllocated) == sizeof(uint32_t) ? STAMTYPE_U32 : STAMTYPE_U64,
+                                                                        STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbCurAllocated",   STAMUNIT_BYTES, "Number of bytes currently allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbAllocated",      STAMUNIT_BYTES, "Total number of bytes allocated.");
+            STAMR3RegisterU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, "/MM/UkHeap/cbFreed",          STAMUNIT_BYTES, "Total number of bytes freed.");
+#endif
+            *ppHeap = pHeap;
+            return VINF_SUCCESS;
+        }
+        AssertRC(rc);
+        MMR3HeapFree(pHeap);
+    }
+    AssertMsgFailed(("failed to allocate heap structure\n"));
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Destroy a User-kernel heap.
+ *
+ * @param   pHeap   Heap handle.
+ */
+void mmR3UkHeapDestroy(PMMUKHEAP pHeap)
+{
+    /*
+     * Start by deleting the lock, that'll trap anyone
+     * attempting to use the heap.
+     */
+    RTCritSectDelete(&pHeap->Lock);
+
+    /*
+     * Walk the sub-heaps and free them.
+     */
+    while (pHeap->pSubHeapHead)
+    {
+        PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+        pHeap->pSubHeapHead = pSubHeap->pNext;
+        SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
+        //MMR3HeapFree(pSubHeap); - rely on the automatic cleanup.
+    }
+    //MMR3HeapFree(pHeap->stats);
+    //MMR3HeapFree(pHeap);
+}
+
+
+/**
+ * Allocate memory associating it with the VM for collective cleanup.
+ *
+ * The memory will be allocated from the default heap but a header
+ * is added in which we keep track of which VM it belongs to and chain
+ * all the allocations together so they can be freed in one go.
+ *
+ * This interface is typically used for memory block which will not be
+ * freed during the life of the VM.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(void *) MMR3UkHeapAlloc(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
+{
+    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
+}
+
+
+/**
+ * Same as MMR3UkHeapAlloc().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(int) MMR3UkHeapAllocEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
+{
+    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, false, pR0Ptr);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Same as MMR3UkHeapAlloc() only the memory is zeroed.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(void *) MMR3UkHeapAllocZ(PVM pVM, MMTAG enmTag, size_t cbSize, PRTR0PTR pR0Ptr)
+{
+    return mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
+}
+
+
+/**
+ * Same as MMR3UkHeapAllocZ().
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pVM         VM handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cbSize      Size of the block.
+ * @param   ppv         Where to store the pointer to the allocated memory on success.
+ * @param   pR0Ptr      Where to return the ring-0 address of the memory.
+ */
+VMMR3DECL(int) MMR3UkHeapAllocZEx(PVM pVM, MMTAG enmTag, size_t cbSize, void **ppv, PRTR0PTR pR0Ptr)
+{
+    void *pv = mmR3UkHeapAlloc(pVM->pUVM->mm.s.pUkHeap, enmTag, cbSize, true, pR0Ptr);
+    if (pv)
+    {
+        *ppv = pv;
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/***
+ * Worker for mmR3UkHeapAlloc that creates and adds a new sub-heap.
+ *
+ * @returns Pointer to the new sub-heap.
+ * @param   pHeap       The heap
+ * @param   cbSubHeap   The size of the sub-heap.
+ */
+static PMMUKHEAPSUB mmR3UkHeapAddSubHeap(PMMUKHEAP pHeap, size_t cbSubHeap)
+{
+    PMMUKHEAPSUB pSubHeap = (PMMUKHEAPSUB)MMR3HeapAllocU(pHeap->pUVM, MM_TAG_MM/*_UK_HEAP*/, sizeof(*pSubHeap));
+    if (pSubHeap)
+    {
+        pSubHeap->cb = cbSubHeap;
+        int rc = SUPR3PageAllocEx(pSubHeap->cb >> PAGE_SHIFT, 0, &pSubHeap->pv, &pSubHeap->pvR0, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTHeapSimpleInit(&pSubHeap->hSimple, pSubHeap->pv, pSubHeap->cb);
+            if (RT_SUCCESS(rc))
+            {
+                pSubHeap->pNext = pHeap->pSubHeapHead;
+                pHeap->pSubHeapHead = pSubHeap;
+                return pSubHeap;
+            }
+
+            /* bail out */
+            SUPR3PageFreeEx(pSubHeap->pv, pSubHeap->cb >> PAGE_SHIFT);
+        }
+        MMR3HeapFree(pSubHeap);
+    }
+    return NULL;
+}
+
+
+/**
+ * Allocate memory from the heap.
+ *
+ * @returns Pointer to allocated memory.
+ * @param   pHeap       Heap handle.
+ * @param   enmTag      Statistics tag. Statistics are collected on a per tag
+ *                      basis in addition to a global one. Thus we can easily
+ *                      identify how memory is used by the VM.
+ * @param   cb          Size of the block.
+ * @param   fZero       Whether or not to zero the memory block.
+ * @param   pR0Ptr      Where to return the ring-0 pointer.
+ */
+static void *mmR3UkHeapAlloc(PMMUKHEAP pHeap, MMTAG enmTag, size_t cb, bool fZero, PRTR0PTR pR0Ptr)
+{
+    if (pR0Ptr)
+        *pR0Ptr = NIL_RTR0PTR;
+    RTCritSectEnter(&pHeap->Lock);
+
+#ifdef MMUKHEAP_WITH_STATISTICS
+    /*
+     * Find/alloc statistics nodes.
+     */
+    pHeap->Stat.cAllocations++;
+    PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+    if (pStat)
+        pStat->cAllocations++;
+    else
+    {
+        pStat = (PMMUKHEAPSTAT)MMR3HeapAllocZU(pHeap->pUVM, MM_TAG_MM, sizeof(MMUKHEAPSTAT));
+        if (!pStat)
+        {
+            pHeap->Stat.cFailures++;
+            AssertMsgFailed(("Failed to allocate heap stat record.\n"));
+            RTCritSectLeave(&pHeap->Lock);
+            return NULL;
+        }
+        pStat->Core.Key = (AVLULKEY)enmTag;
+        RTAvlULInsert(&pHeap->pStatTree, &pStat->Core);
+
+        pStat->cAllocations++;
+
+        /* register the statistics */
+        PUVM pUVM = pHeap->pUVM;
+        const char *pszTag = mmGetTagName(enmTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbCurAllocated, STAMTYPE_U32, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Number of bytes currently allocated.",    "/MM/UkHeap/%s", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cAllocations,   STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number or MMR3UkHeapAlloc() calls.",      "/MM/UkHeap/%s/cAllocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cReallocations, STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapRealloc() calls.",    "/MM/UkHeap/%s/cReallocations", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFrees,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_CALLS, "Number of MMR3UkHeapFree() calls.",       "/MM/UkHeap/%s/cFrees", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cFailures,      STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_COUNT, "Number of failures.",                     "/MM/UkHeap/%s/cFailures", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbAllocated,    STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes allocated.",        "/MM/UkHeap/%s/cbAllocated", pszTag);
+        STAMR3RegisterFU(pUVM, &pStat->cbFreed,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS,  STAMUNIT_BYTES, "Total number of bytes freed.",            "/MM/UkHeap/%s/cbFreed", pszTag);
+    }
+#endif
+
+    /*
+     * Validate input.
+     */
+    if (cb == 0)
+    {
+#ifdef MMUKHEAP_WITH_STATISTICS
+        pStat->cFailures++;
+        pHeap->Stat.cFailures++;
+#endif
+        RTCritSectLeave(&pHeap->Lock);
+        return NULL;
+    }
+
+    /*
+     * Allocate heap block.
+     */
+    cb = RT_ALIGN_Z(cb, MMUKHEAP_SIZE_ALIGNMENT);
+    void *pv = NULL;
+    PMMUKHEAPSUB pSubHeapPrev = NULL;
+    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+    while (pSubHeap)
+    {
+        if (fZero)
+            pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        else
+            pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        if (pv)
+        {
+            /* Move the sub-heap with free memory to the head. */
+            if (pSubHeapPrev)
+            {
+                pSubHeapPrev->pNext = pSubHeap->pNext;
+                pSubHeap->pNext = pHeap->pSubHeapHead;
+                pHeap->pSubHeapHead = pSubHeap;
+            }
+            break;
+        }
+        pSubHeapPrev = pSubHeap;
+        pSubHeap = pSubHeap->pNext;
+    }
+    if (RT_UNLIKELY(!pv))
+    {
+        /*
+         * Add another sub-heap.
+         */
+        pSubHeap = mmR3UkHeapAddSubHeap(pHeap, RT_MAX(RT_ALIGN_Z(cb, PAGE_SIZE) + PAGE_SIZE * 16, _256K));
+        if (pSubHeap)
+        {
+            if (fZero)
+                pv = RTHeapSimpleAllocZ(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+            else
+                pv = RTHeapSimpleAlloc(pSubHeap->hSimple, cb, MMUKHEAP_SIZE_ALIGNMENT);
+        }
+        if (RT_UNLIKELY(!pv))
+        {
+            AssertMsgFailed(("Failed to allocate heap block %d, enmTag=%x(%.4s).\n", cb, enmTag, &enmTag));
+#ifdef MMUKHEAP_WITH_STATISTICS
+            pStat->cFailures++;
+            pHeap->Stat.cFailures++;
+#endif
+            RTCritSectLeave(&pHeap->Lock);
+            return NULL;
+        }
+    }
+
+    /*
+     * Update statistics
+     */
+#ifdef MMUKHEAP_WITH_STATISTICS
+    size_t cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
+    pStat->cbAllocated          += cbActual;
+    pStat->cbCurAllocated       += cbActual;
+    pHeap->Stat.cbAllocated     += cbActual;
+    pHeap->Stat.cbCurAllocated  += cbActual;
+#endif
+
+    if (pR0Ptr)
+        *pR0Ptr = (uintptr_t)pv - (uintptr_t)pSubHeap->pv + pSubHeap->pvR0;
+    RTCritSectLeave(&pHeap->Lock);
+    return pv;
+}
+
+
+/**
+ * Releases memory allocated with MMR3UkHeapAlloc() and MMR3UkHeapAllocZ()
+ *
+ * @param   pVM         The VM handle.
+ * @param   pv          Pointer to the memory block to free.
+ */
+VMMR3DECL(void) MMR3UkHeapFree(PVM pVM, void *pv, MMTAG enmTag)
+{
+    /* Ignore NULL pointers. */
+    if (!pv)
+        return;
+
+    PMMUKHEAP pHeap = pVM->pUVM->mm.s.pUkHeap;
+    RTCritSectEnter(&pHeap->Lock);
+
+    /*
+     * Find the sub-heap and block
+     */
+#ifdef MMUKHEAP_WITH_STATISTICS
+    size_t cbActual = 0;
+#endif
+    PMMUKHEAPSUB pSubHeap = pHeap->pSubHeapHead;
+    while (pSubHeap)
+    {
+        if ((uintptr_t)pv - (uintptr_t)pSubHeap->pv < pSubHeap->cb)
+        {
+#ifdef MMUKHEAP_WITH_STATISTICS
+            cbActual = RTHeapSimpleSize(pSubHeap->hSimple, pv);
+            PMMUKHEAPSTAT pStat = (PMMUKHEAPSTAT)RTAvlULGet(&pHeap->pStatTree, (AVLULKEY)enmTag);
+            if (pStat)
+            {
+                pStat->cFrees++;
+                pStat->cbCurAllocated     -= cbActual;
+                pStat->cbFreed            += cbActual;
+            }
+            pHeap->Stat.cFrees++;
+            pHeap->Stat.cbFreed           += cbActual;
+            pHeap->Stat.cbCurAllocated    -= cbActual;
+#endif
+            RTHeapSimpleFree(pSubHeap->hSimple, pv);
+
+            RTCritSectLeave(&pHeap->Lock);
+            return;
+        }
+    }
+    AssertMsgFailed(("pv=%p\n", pv));
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDM.cpp	(revision 35333)
@@ -0,0 +1,2138 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page   pg_pdm      PDM - The Pluggable Device & Driver Manager
+ *
+ * VirtualBox is designed to be very configurable, i.e. the ability to select
+ * virtual devices and configure them uniquely for a VM.  For this reason
+ * virtual devices are not statically linked with the VMM but loaded, linked and
+ * instantiated at runtime by PDM using the information found in the
+ * Configuration Manager (CFGM).
+ *
+ * While the chief purpose of PDM is to manager of devices their drivers, it
+ * also serves as somewhere to put usful things like cross context queues, cross
+ * context synchronization (like critsect), VM centric thread management,
+ * asynchronous I/O framework, and so on.
+ *
+ * @see grp_pdm
+ *
+ *
+ * @section sec_pdm_dev     The Pluggable Devices
+ *
+ * Devices register themselves when the module containing them is loaded.  PDM
+ * will call the entry point 'VBoxDevicesRegister' when loading a device module.
+ * The device module will then use the supplied callback table to check the VMM
+ * version and to register its devices.  Each device have an unique (for the
+ * configured VM) name.  The name is not only used in PDM but also in CFGM (to
+ * organize device and device instance settings) and by anyone who wants to talk
+ * to a specific device instance.
+ *
+ * When all device modules have been successfully loaded PDM will instantiate
+ * those devices which are configured for the VM.  Note that a device may have
+ * more than one instance, see network adaptors for instance.  When
+ * instantiating a device PDM provides device instance memory and a callback
+ * table (aka Device Helpers / DevHlp) with the VM APIs which the device
+ * instance is trusted with.
+ *
+ * Some devices are trusted devices, most are not.  The trusted devices are an
+ * integrated part of the VM and can obtain the VM handle from their device
+ * instance handles, thus enabling them to call any VM api.  Untrusted devices
+ * can only use the callbacks provided during device instantiation.
+ *
+ * The main purpose in having DevHlps rather than just giving all the devices
+ * the VM handle and let them call the internal VM APIs directly, is both to
+ * create a binary interface that can be supported across releases and to
+ * create a barrier between devices and the VM.  (The trusted / untrusted bit
+ * hasn't turned out to be of much use btw., but it's easy to maintain so there
+ * isn't any point in removing it.)
+ *
+ * A device can provide a ring-0 and/or a raw-mode context extension to improve
+ * the VM performance by handling exits and traps (respectively) without
+ * requiring context switches (to ring-3).  Callbacks for MMIO and I/O ports can
+ * needs to be registered specifically for the additional contexts for this to
+ * make sense.  Also, the device has to be trusted to be loaded into R0/RC
+ * because of the extra privilege it entails.  Note that raw-mode code and data
+ * will be subject to relocation.
+ *
+ *
+ * @section sec_pdm_special_devs    Special Devices
+ *
+ * Several kinds of devices interacts with the VMM and/or other device and PDM
+ * will work like a mediator for these. The typical pattern is that the device
+ * calls a special registration device helper with a set of callbacks, PDM
+ * responds by copying this and providing a pointer to a set helper callbacks
+ * for that particular kind of device. Unlike interfaces where the callback
+ * table pointer is used a 'this' pointer, these arrangements will use the
+ * device instance pointer (PPDMDEVINS) as a kind of 'this' pointer.
+ *
+ * For an example of this kind of setup, see the PIC. The PIC registers itself
+ * by calling PDMDEVHLPR3::pfnPICRegister.  PDM saves the device instance,
+ * copies the callback tables (PDMPICREG), resolving the ring-0 and raw-mode
+ * addresses in the process, and hands back the pointer to a set of helper
+ * methods (PDMPICHLPR3).  The PCI device then queries the ring-0 and raw-mode
+ * helpers using PDMPICHLPR3::pfnGetR0Helpers and PDMPICHLPR3::pfnGetRCHelpers.
+ * The PCI device repeats ths pfnGetRCHelpers call in it's relocation method
+ * since the address changes when RC is relocated.
+ *
+ * @see grp_pdm_device
+ *
+ *
+ * @section sec_pdm_usbdev  The Pluggable USB Devices
+ *
+ * USB devices are handled a little bit differently than other devices.  The
+ * general concepts wrt. pluggability are mostly the same, but the details
+ * varies.  The registration entry point is 'VBoxUsbRegister', the device
+ * instance is PDMUSBINS and the callbacks helpers are different.  Also, USB
+ * device are restricted to ring-3 and cannot have any ring-0 or raw-mode
+ * extensions (at least not yet).
+ *
+ * The way USB devices work differs greatly from other devices though since they
+ * aren't attaches directly to the PCI/ISA/whatever system buses but via a
+ * USB host control (OHCI, UHCI or EHCI).  USB devices handles USB requests
+ * (URBs) and does not register I/O ports, MMIO ranges or PCI bus
+ * devices/functions.
+ *
+ * @see grp_pdm_usbdev
+ *
+ *
+ * @section sec_pdm_drv     The Pluggable Drivers
+ *
+ * The VM devices are often accessing host hardware or OS facilities.  For most
+ * devices these facilities can be abstracted in one or more levels.  These
+ * abstractions are called drivers.
+ *
+ * For instance take a DVD/CD drive.  This can be connected to a SCSI
+ * controller, an ATA controller or a SATA controller.  The basics of the DVD/CD
+ * drive implementation remains the same - eject, insert, read, seek, and such.
+ * (For the scsi case, you might wanna speak SCSI directly to, but that can of
+ * course be fixed - see SCSI passthru.)  So, it
+ * makes much sense to have a generic CD/DVD driver which implements this.
+ *
+ * Then the media 'inserted' into the DVD/CD drive can be a ISO image, or it can
+ * be read from a real CD or DVD drive (there are probably other custom formats
+ * someone could desire to read or construct too).  So, it would make sense to
+ * have abstracted interfaces for dealing with this in a generic way so the
+ * cdrom unit doesn't have to implement it all.  Thus we have created the
+ * CDROM/DVD media driver family.
+ *
+ * So, for this example the IDE controller #1 (i.e. secondary) will have
+ * the DVD/CD Driver attached to it's LUN #0 (master).  When a media is mounted
+ * the DVD/CD Driver will have a ISO, HostDVD or RAW (media) Driver attached.
+ *
+ * It is possible to configure many levels of drivers inserting filters, loggers,
+ * or whatever you desire into the chain.  We're using this for network sniffing
+ * for instance.
+ *
+ * The drivers are loaded in a similar manner to that of the device, namely by
+ * iterating a keyspace in CFGM, load the modules listed there and call
+ * 'VBoxDriversRegister' with a callback table.
+ *
+ * @see grp_pdm_driver
+ *
+ *
+ * @section sec_pdm_ifs     Interfaces
+ *
+ * The pluggable drivers and devices exposes one standard interface (callback
+ * table) which is used to construct, destruct, attach, detach,( ++,) and query
+ * other interfaces. A device will query the interfaces required for it's
+ * operation during init and hot-plug.  PDM may query some interfaces during
+ * runtime mounting too.
+ *
+ * An interface here means a function table contained within the device or
+ * driver instance data. Its method are invoked with the function table pointer
+ * as the first argument and they will calculate the address of the device or
+ * driver instance data from it. (This is one of the aspects which *might* have
+ * been better done in C++.)
+ *
+ * @see grp_pdm_interfaces
+ *
+ *
+ * @section sec_pdm_utils   Utilities
+ *
+ * As mentioned earlier, PDM is the location of any usful constructs that doesn't
+ * quite fit into IPRT. The next subsections will discuss these.
+ *
+ * One thing these APIs all have in common is that resources will be associated
+ * with a device / driver and automatically freed after it has been destroyed if
+ * the destructor didn't do this.
+ *
+ *
+ * @subsection sec_pdm_async_completion     Async I/O
+ *
+ * The PDM Async I/O API provides a somewhat platform agnostic interface for
+ * asynchronous I/O.  For reasons of performance and complexity this does not
+ * build upon any IPRT API.
+ *
+ * @todo more details.
+ *
+ * @see grp_pdm_async_completion
+ *
+ *
+ * @subsection sec_pdm_async_task   Async Task - not implemented
+ *
+ * @todo implement and describe
+ *
+ * @see grp_pdm_async_task
+ *
+ *
+ * @subsection sec_pdm_critsect     Critical Section
+ *
+ * The PDM Critical Section API is currently building on the IPRT API with the
+ * same name.  It adds the possibility to use critical sections in ring-0 and
+ * raw-mode as well as in ring-3.  There are certain restrictions on the RC and
+ * R0 usage though since we're not able to wait on it, nor wake up anyone that
+ * is waiting on it.  These restrictions origins with the use of a ring-3 event
+ * semaphore.  In a later incarnation we plan to replace the ring-3 event
+ * semaphore with a ring-0 one, thus enabling us to wake up waiters while
+ * exectuing in ring-0 and making the hardware assisted execution mode more
+ * efficient. (Raw-mode won't benefit much from this, naturally.)
+ *
+ * @see grp_pdm_critsect
+ *
+ *
+ * @subsection sec_pdm_queue        Queue
+ *
+ * The PDM Queue API is for queuing one or more tasks for later consumption in
+ * ring-3 by EMT, and optionally forcing a delayed or ASAP return to ring-3.  The
+ * queues can also be run on a timer basis as an alternative to the ASAP thing.
+ * The queue will be flushed at forced action time.
+ *
+ * A queue can also be used by another thread (a I/O worker for instance) to
+ * send work / events over to the EMT.
+ *
+ * @see grp_pdm_queue
+ *
+ *
+ * @subsection sec_pdm_task        Task - not implemented yet
+ *
+ * The PDM Task API is for flagging a task for execution at a later point when
+ * we're back in ring-3, optionally forcing the ring-3 return to happen ASAP.
+ * As you can see the concept is similar to queues only simpler.
+ *
+ * A task can also be scheduled by another thread (a I/O worker for instance) as
+ * a mean of getting something done in EMT.
+ *
+ * @see grp_pdm_task
+ *
+ *
+ * @subsection sec_pdm_thread       Thread
+ *
+ * The PDM Thread API is there to help devices and drivers manage their threads
+ * correctly wrt. power on, suspend, resume, power off and destruction.
+ *
+ * The general usage pattern for threads in the employ of devices and drivers is
+ * that they shuffle data or requests while the VM is running and stop doing
+ * this when the VM is paused or powered down. Rogue threads running while the
+ * VM is paused can cause the state to change during saving or have other
+ * unwanted side effects. The PDM Threads API ensures that this won't happen.
+ *
+ * @see grp_pdm_thread
+ *
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/pgm.h>
+#include <VBox/ssm.h>
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/vmm.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/sup.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/ldr.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The PDM saved state version. */
+#define PDM_SAVED_STATE_VERSION             4
+#define PDM_SAVED_STATE_VERSION_PRE_NMI_FF  3
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) pdmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
+static DECLCALLBACK(int) pdmR3SaveExec(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) pdmR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int) pdmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM);
+
+
+
+/**
+ * Initializes the PDM part of the UVM.
+ *
+ * This doesn't really do much right now but has to be here for the sake
+ * of completeness.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3DECL(int) PDMR3InitUVM(PUVM pUVM)
+{
+    AssertCompile(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
+    AssertRelease(sizeof(pUVM->pdm.s) <= sizeof(pUVM->pdm.padding));
+    pUVM->pdm.s.pModules   = NULL;
+    pUVM->pdm.s.pCritSects = NULL;
+    return RTCritSectInit(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Initializes the PDM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) PDMR3Init(PVM pVM)
+{
+    LogFlow(("PDMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertRelease(!(RT_OFFSETOF(VM, pdm.s) & 31));
+    AssertRelease(sizeof(pVM->pdm.s) <= sizeof(pVM->pdm.padding));
+    AssertCompileMemberAlignment(PDM, CritSect, sizeof(uintptr_t));
+    /*
+     * Init the structure.
+     */
+    pVM->pdm.s.offVM = RT_OFFSETOF(VM, pdm.s);
+    pVM->pdm.s.GCPhysVMMDevHeap = NIL_RTGCPHYS;
+
+    /*
+     * Initialize sub components.
+     */
+    int rc = pdmR3CritSectInitStats(pVM);
+    if (RT_SUCCESS(rc))
+        rc = PDMR3CritSectInit(pVM, &pVM->pdm.s.CritSect, RT_SRC_POS, "PDM");
+    if (RT_SUCCESS(rc))
+        rc = pdmR3LdrInitU(pVM->pUVM);
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+    if (RT_SUCCESS(rc))
+        rc = pdmR3AsyncCompletionInit(pVM);
+#endif
+    if (RT_SUCCESS(rc))
+        rc = pdmR3BlkCacheInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pdmR3DrvInit(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pdmR3DevInit(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Register the saved state data unit.
+         */
+        rc = SSMR3RegisterInternal(pVM, "pdm", 1, PDM_SAVED_STATE_VERSION, 128,
+                                   NULL, pdmR3LiveExec, NULL,
+                                   NULL, pdmR3SaveExec, NULL,
+                                   pdmR3LoadPrep, pdmR3LoadExec, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            LogFlow(("PDM: Successfully initialized\n"));
+            return rc;
+        }
+    }
+
+    /*
+     * Cleanup and return failure.
+     */
+    PDMR3Term(pVM);
+    LogFlow(("PDMR3Init: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM         VM handle.
+ * @param   offDelta    Relocation delta relative to old location.
+ * @remark  The loader subcomponent is relocated by PDMR3LdrRelocate() very
+ *          early in the relocation phase.
+ */
+VMMR3DECL(void) PDMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("PDMR3Relocate\n"));
+
+    /*
+     * Queues.
+     */
+    pdmR3QueueRelocate(pVM, offDelta);
+    pVM->pdm.s.pDevHlpQueueRC = PDMQueueRCPtr(pVM->pdm.s.pDevHlpQueueR3);
+
+    /*
+     * Critical sections.
+     */
+    pdmR3CritSectRelocate(pVM);
+
+    /*
+     * The registered PIC.
+     */
+    if (pVM->pdm.s.Pic.pDevInsRC)
+    {
+        pVM->pdm.s.Pic.pDevInsRC            += offDelta;
+        pVM->pdm.s.Pic.pfnSetIrqRC          += offDelta;
+        pVM->pdm.s.Pic.pfnGetInterruptRC    += offDelta;
+    }
+
+    /*
+     * The registered APIC.
+     */
+    if (pVM->pdm.s.Apic.pDevInsRC)
+    {
+        pVM->pdm.s.Apic.pDevInsRC           += offDelta;
+        pVM->pdm.s.Apic.pfnGetInterruptRC   += offDelta;
+        pVM->pdm.s.Apic.pfnSetBaseRC        += offDelta;
+        pVM->pdm.s.Apic.pfnGetBaseRC        += offDelta;
+        pVM->pdm.s.Apic.pfnSetTPRRC         += offDelta;
+        pVM->pdm.s.Apic.pfnGetTPRRC         += offDelta;
+        pVM->pdm.s.Apic.pfnBusDeliverRC     += offDelta;
+        if (pVM->pdm.s.Apic.pfnLocalInterruptRC)
+            pVM->pdm.s.Apic.pfnLocalInterruptRC += offDelta;
+        pVM->pdm.s.Apic.pfnWriteMSRRC       += offDelta;
+        pVM->pdm.s.Apic.pfnReadMSRRC        += offDelta;
+    }
+
+    /*
+     * The registered I/O APIC.
+     */
+    if (pVM->pdm.s.IoApic.pDevInsRC)
+    {
+        pVM->pdm.s.IoApic.pDevInsRC         += offDelta;
+        pVM->pdm.s.IoApic.pfnSetIrqRC       += offDelta;
+        if (pVM->pdm.s.IoApic.pfnSendMsiRC)
+            pVM->pdm.s.IoApic.pfnSendMsiRC      += offDelta;
+    }
+
+    /*
+     * The register PCI Buses.
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pdm.s.aPciBuses); i++)
+    {
+        if (pVM->pdm.s.aPciBuses[i].pDevInsRC)
+        {
+            pVM->pdm.s.aPciBuses[i].pDevInsRC   += offDelta;
+            pVM->pdm.s.aPciBuses[i].pfnSetIrqRC += offDelta;
+        }
+    }
+
+    /*
+     * Devices & Drivers.
+     */
+    PCPDMDEVHLPRC pDevHlpRC;
+    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pDevHlpRC);
+    AssertReleaseMsgRC(rc, ("rc=%Rrc when resolving g_pdmRCDevHlp\n", rc));
+
+    PCPDMDRVHLPRC pDrvHlpRC;
+    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pDrvHlpRC);
+    AssertReleaseMsgRC(rc, ("rc=%Rrc when resolving g_pdmRCDevHlp\n", rc));
+
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+        {
+            pDevIns->pHlpRC             = pDevHlpRC;
+            pDevIns->pvInstanceDataRC   = MMHyperR3ToRC(pVM, pDevIns->pvInstanceDataR3);
+            if (pDevIns->pCritSectR3)
+                pDevIns->pCritSectRC    = MMHyperR3ToRC(pVM, pDevIns->pCritSectR3);
+            pDevIns->Internal.s.pVMRC   = pVM->pVMRC;
+            if (pDevIns->Internal.s.pPciBusR3)
+                pDevIns->Internal.s.pPciBusRC    = MMHyperR3ToRC(pVM, pDevIns->Internal.s.pPciBusR3);
+            if (pDevIns->Internal.s.pPciDeviceR3)
+                pDevIns->Internal.s.pPciDeviceRC = MMHyperR3ToRC(pVM, pDevIns->Internal.s.pPciDeviceR3);
+            if (pDevIns->pReg->pfnRelocate)
+            {
+                LogFlow(("PDMR3Relocate: Relocating device '%s'/%d\n",
+                         pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnRelocate(pDevIns, offDelta);
+            }
+        }
+
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+        {
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+            {
+                if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+                {
+                    pDrvIns->pHlpRC = pDrvHlpRC;
+                    pDrvIns->pvInstanceDataRC = MMHyperR3ToRC(pVM, pDrvIns->pvInstanceDataR3);
+                    pDrvIns->Internal.s.pVMRC = pVM->pVMRC;
+                    if (pDrvIns->pReg->pfnRelocate)
+                    {
+                        LogFlow(("PDMR3Relocate: Relocating driver '%s'/%u attached to '%s'/%d/%u\n",
+                                 pDrvIns->pReg->szName, pDrvIns->iInstance,
+                                 pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun));
+                        pDrvIns->pReg->pfnRelocate(pDrvIns, offDelta);
+                    }
+                }
+            }
+        }
+
+    }
+}
+
+
+/**
+ * Worker for pdmR3Term that terminates a LUN chain.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pLun        The head of the chain.
+ * @param   pszDevice   The name of the device (for logging).
+ * @param   iInstance   The device instance number (for logging).
+ */
+static void pdmR3TermLuns(PVM pVM, PPDMLUN pLun, const char *pszDevice, unsigned iInstance)
+{
+    for (; pLun; pLun = pLun->pNext)
+    {
+        /*
+         * Destroy them one at a time from the bottom up.
+         * (The serial device/drivers depends on this - bad.)
+         */
+        PPDMDRVINS pDrvIns = pLun->pBottom;
+        pLun->pBottom = pLun->pTop = NULL;
+        while (pDrvIns)
+        {
+            PPDMDRVINS pDrvNext = pDrvIns->Internal.s.pUp;
+
+            if (pDrvIns->pReg->pfnDestruct)
+            {
+                LogFlow(("pdmR3DevTerm: Destroying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, pLun->iLun, pszDevice, iInstance));
+                pDrvIns->pReg->pfnDestruct(pDrvIns);
+            }
+            pDrvIns->Internal.s.pDrv->cInstances--;
+
+            TMR3TimerDestroyDriver(pVM, pDrvIns);
+            //PDMR3QueueDestroyDriver(pVM, pDrvIns);
+            //pdmR3ThreadDestroyDriver(pVM, pDrvIns);
+            SSMR3DeregisterDriver(pVM, pDrvIns, NULL, 0);
+
+            pDrvIns = pDrvNext;
+        }
+    }
+}
+
+
+/**
+ * Terminates the PDM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) PDMR3Term(PVM pVM)
+{
+    LogFlow(("PDMR3Term:\n"));
+    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+
+    /*
+     * Iterate the device instances and attach drivers, doing
+     * relevant destruction processing.
+     *
+     * N.B. There is no need to mess around freeing memory allocated
+     *      from any MM heap since MM will do that in its Term function.
+     */
+    /* usb ones first. */
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        pdmR3TermLuns(pVM, pUsbIns->Internal.s.pLuns, pUsbIns->pReg->szName, pUsbIns->iInstance);
+
+        if (pUsbIns->pReg->pfnDestruct)
+        {
+            LogFlow(("pdmR3DevTerm: Destroying - device '%s'/%d\n",
+                     pUsbIns->pReg->szName, pUsbIns->iInstance));
+            pUsbIns->pReg->pfnDestruct(pUsbIns);
+        }
+
+        //TMR3TimerDestroyUsb(pVM, pUsbIns);
+        //SSMR3DeregisterUsb(pVM, pUsbIns, NULL, 0);
+        pdmR3ThreadDestroyUsb(pVM, pUsbIns);
+    }
+
+    /* then the 'normal' ones. */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        pdmR3TermLuns(pVM, pDevIns->Internal.s.pLunsR3, pDevIns->pReg->szName, pDevIns->iInstance);
+
+        if (pDevIns->pReg->pfnDestruct)
+        {
+            LogFlow(("pdmR3DevTerm: Destroying - device '%s'/%d\n",
+                     pDevIns->pReg->szName, pDevIns->iInstance));
+            pDevIns->pReg->pfnDestruct(pDevIns);
+        }
+
+        TMR3TimerDestroyDevice(pVM, pDevIns);
+        //SSMR3DeregisterDriver(pVM, pDevIns, NULL, 0);
+        pdmR3CritSectDeleteDevice(pVM, pDevIns);
+        //pdmR3ThreadDestroyDevice(pVM, pDevIns);
+        //PDMR3QueueDestroyDevice(pVM, pDevIns);
+        PGMR3PhysMMIO2Deregister(pVM, pDevIns, UINT32_MAX);
+    }
+
+    /*
+     * Destroy all threads.
+     */
+    pdmR3ThreadDestroyAll(pVM);
+
+    /*
+     * Destroy the block cache.
+     */
+    pdmR3BlkCacheTerm(pVM);
+
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+    /*
+     * Free async completion managers.
+     */
+    pdmR3AsyncCompletionTerm(pVM);
+#endif
+
+    /*
+     * Free modules.
+     */
+    pdmR3LdrTermU(pVM->pUVM);
+
+    /*
+     * Destroy the PDM lock.
+     */
+    PDMR3CritSectDelete(&pVM->pdm.s.CritSect);
+    /* The MiscCritSect is deleted by PDMR3CritSectTerm. */
+
+    LogFlow(("PDMR3Term: returns %Rrc\n", VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates the PDM part of the UVM.
+ *
+ * This will unload any modules left behind.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3DECL(void) PDMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * In the normal cause of events we will now call pdmR3LdrTermU for
+     * the second time. In the case of init failure however, this might
+     * the first time, which is why we do it.
+     */
+    pdmR3LdrTermU(pUVM);
+
+    Assert(pUVM->pdm.s.pCritSects == NULL);
+    RTCritSectDelete(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Bits that are saved in pass 0 and in the final pass.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The saved state handle.
+ */
+static void pdmR3SaveBoth(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Save the list of device instances so we can check that they're all still
+     * there when we load the state and that nothing new has been added.
+     */
+    uint32_t i = 0;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3, i++)
+    {
+        SSMR3PutU32(pSSM, i);
+        SSMR3PutStrZ(pSSM, pDevIns->pReg->szName);
+        SSMR3PutU32(pSSM, pDevIns->iInstance);
+    }
+    SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
+}
+
+
+/**
+ * Live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The saved state handle.
+ * @param   uPass           The pass.
+ */
+static DECLCALLBACK(int) pdmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    LogFlow(("pdmR3LiveExec:\n"));
+    AssertReturn(uPass == 0, VERR_INTERNAL_ERROR_4);
+    pdmR3SaveBoth(pVM, pSSM);
+    return VINF_SSM_DONT_CALL_AGAIN;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The saved state handle.
+ */
+static DECLCALLBACK(int) pdmR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("pdmR3SaveExec:\n"));
+
+    /*
+     * Save interrupt and DMA states.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[idCpu];
+        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC));
+        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC));
+        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_NMI));
+        SSMR3PutU32(pSSM, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_SMI));
+    }
+    SSMR3PutU32(pSSM, VM_FF_ISSET(pVM, VM_FF_PDM_DMA));
+
+    pdmR3SaveBoth(pVM, pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Prepare state load operation.
+ *
+ * This will dispatch pending operations and clear the FFs governed by PDM and its devices.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pSSM        The SSM handle.
+ */
+static DECLCALLBACK(int) pdmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("pdmR3LoadPrep: %s%s\n",
+             VM_FF_ISSET(pVM, VM_FF_PDM_QUEUES)     ? " VM_FF_PDM_QUEUES" : "",
+             VM_FF_ISSET(pVM, VM_FF_PDM_DMA)        ? " VM_FF_PDM_DMA" : ""));
+#ifdef LOG_ENABLED
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[idCpu];
+        LogFlow(("pdmR3LoadPrep: VCPU %u %s%s\n", idCpu,
+                VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC) ? " VMCPU_FF_INTERRUPT_APIC" : "",
+                VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC)  ? " VMCPU_FF_INTERRUPT_PIC" : ""));
+    }
+#endif
+
+    /*
+     * In case there is work pending that will raise an interrupt,
+     * start a DMA transfer, or release a lock. (unlikely)
+     */
+    if (VM_FF_ISSET(pVM, VM_FF_PDM_QUEUES))
+        PDMR3QueueFlushAll(pVM);
+
+    /* Clear the FFs. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[idCpu];
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_SMI);
+    }
+    VM_FF_CLEAR(pVM, VM_FF_PDM_DMA);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) pdmR3LoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int rc;
+
+    LogFlow(("pdmR3LoadExec: uPass=%#x\n", uPass));
+
+    /*
+     * Validate version.
+     */
+    if (    uVersion != PDM_SAVED_STATE_VERSION
+        &&  uVersion != PDM_SAVED_STATE_VERSION_PRE_NMI_FF)
+    {
+        AssertMsgFailed(("Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    if (uPass == SSM_PASS_FINAL)
+    {
+        /*
+         * Load the interrupt and DMA states.
+         */
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            PVMCPU pVCpu = &pVM->aCpus[idCpu];
+
+            /* APIC interrupt */
+            uint32_t fInterruptPending = 0;
+            rc = SSMR3GetU32(pSSM, &fInterruptPending);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (fInterruptPending & ~1)
+            {
+                AssertMsgFailed(("fInterruptPending=%#x (APIC)\n", fInterruptPending));
+                return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+            }
+            AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC));
+            if (fInterruptPending)
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+
+            /* PIC interrupt */
+            fInterruptPending = 0;
+            rc = SSMR3GetU32(pSSM, &fInterruptPending);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (fInterruptPending & ~1)
+            {
+                AssertMsgFailed(("fInterruptPending=%#x (PIC)\n", fInterruptPending));
+                return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+            }
+            AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC));
+            if (fInterruptPending)
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+
+            if (uVersion > PDM_SAVED_STATE_VERSION_PRE_NMI_FF)
+            {
+                /* NMI interrupt */
+                fInterruptPending = 0;
+                rc = SSMR3GetU32(pSSM, &fInterruptPending);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (fInterruptPending & ~1)
+                {
+                    AssertMsgFailed(("fInterruptPending=%#x (NMI)\n", fInterruptPending));
+                    return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+                }
+                AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_NMI));
+                if (fInterruptPending)
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+
+                /* SMI interrupt */
+                fInterruptPending = 0;
+                rc = SSMR3GetU32(pSSM, &fInterruptPending);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (fInterruptPending & ~1)
+                {
+                    AssertMsgFailed(("fInterruptPending=%#x (SMI)\n", fInterruptPending));
+                    return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+                }
+                AssertRelease(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_SMI));
+                if (fInterruptPending)
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI);
+            }
+        }
+
+        /* DMA pending */
+        uint32_t fDMAPending = 0;
+        rc = SSMR3GetU32(pSSM, &fDMAPending);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (fDMAPending & ~1)
+        {
+            AssertMsgFailed(("fDMAPending=%#x\n", fDMAPending));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+        if (fDMAPending)
+            VM_FF_SET(pVM, VM_FF_PDM_DMA);
+        Log(("pdmR3LoadExec: VM_FF_PDM_DMA=%RTbool\n", VM_FF_ISSET(pVM, VM_FF_PDM_DMA)));
+    }
+
+    /*
+     * Load the list of devices and verify that they are all there.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_FOUND;
+
+    for (uint32_t i = 0; ; i++)
+    {
+        /* Get the sequence number / terminator. */
+        uint32_t    u32Sep;
+        rc = SSMR3GetU32(pSSM, &u32Sep);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u32Sep == UINT32_MAX)
+            break;
+        if (u32Sep != i)
+            AssertMsgFailedReturn(("Out of sequence. u32Sep=%#x i=%#x\n", u32Sep, i), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /* Get the name and instance number. */
+        char szName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
+        rc = SSMR3GetStrZ(pSSM, szName, sizeof(szName));
+        if (RT_FAILURE(rc))
+            return rc;
+        uint32_t iInstance;
+        rc = SSMR3GetU32(pSSM, &iInstance);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* Try locate it. */
+        PPDMDEVINS pDevIns;
+        for (pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+            if (   !strcmp(szName, pDevIns->pReg->szName)
+                && pDevIns->iInstance == iInstance)
+            {
+                AssertLogRelMsgReturn(!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_FOUND),
+                                      ("%s/#%u\n", pDevIns->pReg->szName, pDevIns->iInstance),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_FOUND;
+                break;
+            }
+        if (!pDevIns)
+        {
+            LogRel(("Device '%s'/%d not found in current config\n", szName, iInstance));
+            if (SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT)
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Device '%s'/%d not found in current config"), szName, iInstance);
+        }
+    }
+
+    /*
+     * Check that no additional devices were configured.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_FOUND))
+        {
+            LogRel(("Device '%s'/%d not found in the saved state\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            if (SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT)
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Device '%s'/%d not found in the saved state"),
+                                        pDevIns->pReg->szName, pDevIns->iInstance);
+        }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(int) pdmR3PowerOnDrv(PPDMDRVINS pDrvIns, const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    Assert(pDrvIns->Internal.s.fVMSuspended);
+    if (pDrvIns->pReg->pfnPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                 pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+        int rc = VINF_SUCCESS; pDrvIns->pReg->pfnPowerOn(pDrvIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
+                    pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance, rc));
+            return rc;
+        }
+    }
+    pDrvIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one USB device instance.
+ *
+ * @returns VBox status code.
+ * @param   pUsbIns             The USB device instance.
+ */
+DECLINLINE(int) pdmR3PowerOnUsb(PPDMUSBINS pUsbIns)
+{
+    Assert(pUsbIns->Internal.s.fVMSuspended);
+    if (pUsbIns->pReg->pfnVMPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        int rc = VINF_SUCCESS; pUsbIns->pReg->pfnVMPowerOn(pUsbIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: device '%s'/%d -> %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pUsbIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3PowerOn that deals with one device instance.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns             The device instance.
+ */
+DECLINLINE(int) pdmR3PowerOnDev(PPDMDEVINS pDevIns)
+{
+    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
+    if (pDevIns->pReg->pfnPowerOn)
+    {
+        LogFlow(("PDMR3PowerOn: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        int rc = VINF_SUCCESS; pDevIns->pReg->pfnPowerOn(pDevIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3PowerOn: device '%s'/%d -> %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This function will notify all the devices and their
+ * attached drivers about the VM now being powered on.
+ *
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(void) PDMR3PowerOn(PVM pVM)
+{
+    LogFlow(("PDMR3PowerOn:\n"));
+
+    /*
+     * Iterate thru the device instances and USB device instances,
+     * processing the drivers associated with those.
+     */
+    int rc = VINF_SUCCESS;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3PowerOnDrv(pDrvIns, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3PowerOnDev(pDevIns);
+    }
+
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3PowerOnDrv(pDrvIns, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3PowerOnUsb(pUsbIns);
+    }
+#endif
+
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+    pdmR3AsyncCompletionResume(pVM);
+#endif
+
+    /*
+     * Resume all threads.
+     */
+    if (RT_SUCCESS(rc))
+        pdmR3ThreadResumeAll(pVM);
+
+    /*
+     * On failure, clean up via PDMR3Suspend.
+     */
+    if (RT_FAILURE(rc))
+        PDMR3Suspend(pVM);
+
+    LogFlow(("PDMR3PowerOn: returns %Rrc\n", rc));
+    return /*rc*/;
+}
+
+
+/**
+ * Worker for PDMR3Reset that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pcAsync             The asynchronous reset notification counter.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3ResetDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
+                               const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMReset)
+    {
+        pDrvIns->Internal.s.fVMReset = true;
+        if (pDrvIns->pReg->pfnReset)
+        {
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Reset: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+                pDrvIns->pReg->pfnReset(pDrvIns);
+                if (pDrvIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Reset: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                pDrvIns->Internal.s.pfnAsyncNotify = false;
+                LogFlow(("PDMR3Reset: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMReset = false;
+                (*pcAsync)++;
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3Reset that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pcAsync             The asynchronous reset notification counter.
+ */
+DECLINLINE(void) pdmR3ResetUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
+{
+    if (!pUsbIns->Internal.s.fVMReset)
+    {
+        pUsbIns->Internal.s.fVMReset = true;
+        if (pUsbIns->pReg->pfnVMReset)
+        {
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Reset: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->pReg->pfnVMReset(pUsbIns);
+                if (pUsbIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Reset: Async notification started - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3Reset: Async notification completed - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMReset = false;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3Reset that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pcAsync             The asynchronous reset notification counter.
+ */
+DECLINLINE(void) pdmR3ResetDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_RESET))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_RESET;
+        if (pDevIns->pReg->pfnReset)
+        {
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Reset: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnReset(pDevIns);
+                if (pDevIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Reset: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3Reset: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_RESET;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by PDMR3Reset and CPU hot plugging.
+ *
+ * @param   pVCpu               The virtual CPU handle.
+ */
+VMMR3DECL(void) PDMR3ResetCpu(PVMCPU pVCpu)
+{
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_SMI);
+}
+
+
+/**
+ * This function will notify all the devices and their attached drivers about
+ * the VM now being reset.
+ *
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(void) PDMR3Reset(PVM pVM)
+{
+    LogFlow(("PDMR3Reset:\n"));
+
+    /*
+     * Clear all the reset flags.
+     */
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_RESET;
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                pDrvIns->Internal.s.fVMReset = false;
+    }
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        pUsbIns->Internal.s.fVMReset = false;
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                pDrvIns->Internal.s.fVMReset = false;
+    }
+#endif
+
+    /*
+     * The outer loop repeats until there are no more async requests.
+     */
+    unsigned cAsync;
+    for (unsigned iLoop = 0; ; iLoop++)
+    {
+        /*
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         */
+        cAsync = 0;
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            if (cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3ResetDrv(pDrvIns, &cAsync, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+                        if (cAsync == cAsyncStart)
+                pdmR3ResetDev(pDevIns, &cAsync);
+        }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3ResetDrv(pDrvIns, &cAsync, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (cAsync == cAsyncStart)
+                pdmR3ResetUsb(pUsbIns, &cAsync);
+        }
+#endif
+        if (!cAsync)
+            break;
+
+        /*
+         * Process requests.
+         */
+        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
+         *        bit I hope... */
+        int rc = VMR3AsyncPdmNotificationWaitU(&pVM->pUVM->aCpus[0]);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = VMR3ReqProcessU(pVM->pUVM, 0/*idDstCpu*/);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+    }
+
+    /*
+     * Clear all pending interrupts and DMA operations.
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        PDMR3ResetCpu(&pVM->aCpus[idCpu]);
+    VM_FF_CLEAR(pVM, VM_FF_PDM_DMA);
+
+    LogFlow(("PDMR3Reset: returns void\n"));
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pcAsync             The asynchronous suspend notification counter.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3SuspendDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
+                                 const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMSuspended)
+    {
+        pDrvIns->Internal.s.fVMSuspended = true;
+        if (pDrvIns->pReg->pfnSuspend)
+        {
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+                pDrvIns->pReg->pfnSuspend(pDrvIns);
+                if (pDrvIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Suspend: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                pDrvIns->Internal.s.pfnAsyncNotify = false;
+                LogFlow(("PDMR3Suspend: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pcAsync             The asynchronous suspend notification counter.
+ */
+DECLINLINE(void) pdmR3SuspendUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
+{
+    if (!pUsbIns->Internal.s.fVMSuspended)
+    {
+        pUsbIns->Internal.s.fVMSuspended = true;
+        if (pUsbIns->pReg->pfnVMSuspend)
+        {
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->pReg->pfnVMSuspend(pUsbIns);
+                if (pUsbIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Suspend: Async notification started - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3Suspend: Async notification completed - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3Suspend that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pcAsync             The asynchronous suspend notification counter.
+ */
+DECLINLINE(void) pdmR3SuspendDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
+        if (pDevIns->pReg->pfnSuspend)
+        {
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3Suspend: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnSuspend(pDevIns);
+                if (pDevIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3Suspend: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3Suspend: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * This function will notify all the devices and their attached drivers about
+ * the VM now being suspended.
+ *
+ * @param   pVM     The VM Handle.
+ * @thread  EMT(0)
+ */
+VMMR3DECL(void) PDMR3Suspend(PVM pVM)
+{
+    LogFlow(("PDMR3Suspend:\n"));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * The outer loop repeats until there are no more async requests.
+     *
+     * Note! We depend on the suspended indicators to be in the desired state
+     *       and we do not reset them before starting because this allows
+     *       PDMR3PowerOn and PDMR3Resume to use PDMR3Suspend for cleaning up
+     *       on failure.
+     */
+    unsigned cAsync;
+    for (unsigned iLoop = 0; ; iLoop++)
+    {
+        /*
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         *
+         * The attached drivers are normally processed first.  Some devices
+         * (like DevAHCI) though needs to be notified before the drivers so
+         * that it doesn't kick off any new requests after the drivers stopped
+         * taking any. (DrvVD changes to read-only in this particular case.)
+         */
+        cAsync = 0;
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION)
+                pdmR3SuspendDev(pDevIns, &cAsync);
+
+            if (cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3SuspendDrv(pDrvIns, &cAsync, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+            if (    cAsync == cAsyncStart
+                && !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_SUSPEND_NOTIFICATION))
+                pdmR3SuspendDev(pDevIns, &cAsync);
+        }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3SuspendDrv(pDrvIns, &cAsync, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (cAsync == cAsyncStart)
+                pdmR3SuspendUsb(pUsbIns, &cAsync);
+        }
+#endif
+        if (!cAsync)
+            break;
+
+        /*
+         * Process requests.
+         */
+        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
+         *        bit I hope... */
+        int rc = VMR3AsyncPdmNotificationWaitU(&pVM->pUVM->aCpus[0]);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = VMR3ReqProcessU(pVM->pUVM, 0/*idDstCpu*/);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+    }
+
+    /*
+     * Suspend all threads.
+     */
+    pdmR3ThreadSuspendAll(pVM);
+
+    LogFlow(("PDMR3Suspend: returns void\n"));
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(int) pdmR3ResumeDrv(PPDMDRVINS pDrvIns, const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    Assert(pDrvIns->Internal.s.fVMSuspended);
+    if (pDrvIns->pReg->pfnResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                 pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+        int rc = VINF_SUCCESS; pDrvIns->pReg->pfnResume(pDrvIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: driver '%s'/%d on LUN#%d of device '%s'/%d -> %Rrc\n",
+                    pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance, rc));
+            return rc;
+        }
+    }
+    pDrvIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one USB device instance.
+ *
+ * @returns VBox status code.
+ * @param   pUsbIns             The USB device instance.
+ */
+DECLINLINE(int) pdmR3ResumeUsb(PPDMUSBINS pUsbIns)
+{
+    Assert(pUsbIns->Internal.s.fVMSuspended);
+    if (pUsbIns->pReg->pfnVMResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        int rc = VINF_SUCCESS; pUsbIns->pReg->pfnVMResume(pUsbIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: device '%s'/%d -> %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pUsbIns->Internal.s.fVMSuspended = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for PDMR3Resume that deals with one device instance.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns             The device instance.
+ */
+DECLINLINE(int) pdmR3ResumeDev(PPDMDEVINS pDevIns)
+{
+    Assert(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED);
+    if (pDevIns->pReg->pfnResume)
+    {
+        LogFlow(("PDMR3Resume: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        int rc = VINF_SUCCESS; pDevIns->pReg->pfnResume(pDevIns);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDMR3Resume: device '%s'/%d -> %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * This function will notify all the devices and their
+ * attached drivers about the VM now being resumed.
+ *
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(void) PDMR3Resume(PVM pVM)
+{
+    LogFlow(("PDMR3Resume:\n"));
+
+    /*
+     * Iterate thru the device instances and USB device instances,
+     * processing the drivers associated with those.
+     */
+    int rc = VINF_SUCCESS;
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances;  pDevIns && RT_SUCCESS(rc);  pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;  pLun && RT_SUCCESS(rc);  pLun    = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3ResumeDrv(pDrvIns, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3ResumeDev(pDevIns);
+    }
+
+#ifdef VBOX_WITH_USB
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;  pUsbIns && RT_SUCCESS(rc);  pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns;  pLun && RT_SUCCESS(rc);  pLun = pLun->pNext)
+            for (PPDMDRVINS pDrvIns = pLun->pTop;  pDrvIns && RT_SUCCESS(rc);  pDrvIns = pDrvIns->Internal.s.pDown)
+                rc = pdmR3ResumeDrv(pDrvIns, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun);
+        if (RT_SUCCESS(rc))
+            rc = pdmR3ResumeUsb(pUsbIns);
+    }
+#endif
+
+    /*
+     * Resume all threads.
+     */
+    if (RT_SUCCESS(rc))
+        pdmR3ThreadResumeAll(pVM);
+
+    /*
+     * Resume the block cache.
+     */
+    if (RT_SUCCESS(rc))
+        pdmR3BlkCacheResume(pVM);
+
+    /*
+     * On failure, clean up via PDMR3Suspend.
+     */
+    if (RT_FAILURE(rc))
+        PDMR3Suspend(pVM);
+
+    LogFlow(("PDMR3Resume: returns %Rrc\n", rc));
+    return /*rc*/;
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one driver.
+ *
+ * @param   pDrvIns             The driver instance.
+ * @param   pcAsync             The asynchronous power off notification counter.
+ * @param   pszDeviceName       The parent device name.
+ * @param   iDevInstance        The parent device instance number.
+ * @param   iLun                The parent LUN number.
+ */
+DECLINLINE(bool) pdmR3PowerOffDrv(PPDMDRVINS pDrvIns, unsigned *pcAsync,
+                                  const char *pszDeviceName, uint32_t iDevInstance, uint32_t iLun)
+{
+    if (!pDrvIns->Internal.s.fVMSuspended)
+    {
+        pDrvIns->Internal.s.fVMSuspended = true;
+        if (pDrvIns->pReg->pfnPowerOff)
+        {
+            if (!pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+                pDrvIns->pReg->pfnPowerOff(pDrvIns);
+                if (pDrvIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3PowerOff: Async notification started - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            else if (pDrvIns->Internal.s.pfnAsyncNotify(pDrvIns))
+            {
+                pDrvIns->Internal.s.pfnAsyncNotify = false;
+                LogFlow(("PDMR3PowerOff: Async notification completed - driver '%s'/%d on LUN#%d of device '%s'/%d\n",
+                         pDrvIns->pReg->szName, pDrvIns->iInstance, iLun, pszDeviceName, iDevInstance));
+            }
+            if (pDrvIns->Internal.s.pfnAsyncNotify)
+            {
+                pDrvIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one USB device instance.
+ *
+ * @param   pUsbIns             The USB device instance.
+ * @param   pcAsync             The asynchronous power off notification counter.
+ */
+DECLINLINE(void) pdmR3PowerOffUsb(PPDMUSBINS pUsbIns, unsigned *pcAsync)
+{
+    if (!pUsbIns->Internal.s.fVMSuspended)
+    {
+        pUsbIns->Internal.s.fVMSuspended = true;
+        if (pUsbIns->pReg->pfnVMPowerOff)
+        {
+            if (!pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->pReg->pfnVMPowerOff(pUsbIns);
+                if (pUsbIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3PowerOff: Async notification started - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+            }
+            else if (pUsbIns->Internal.s.pfnAsyncNotify(pUsbIns))
+            {
+                LogFlow(("PDMR3PowerOff: Async notification completed - device '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+                pUsbIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pUsbIns->Internal.s.pfnAsyncNotify)
+            {
+                pUsbIns->Internal.s.fVMSuspended = false;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * Worker for PDMR3PowerOff that deals with one device instance.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pcAsync             The asynchronous power off notification counter.
+ */
+DECLINLINE(void) pdmR3PowerOffDev(PPDMDEVINS pDevIns, unsigned *pcAsync)
+{
+    if (!(pDevIns->Internal.s.fIntFlags & PDMDEVINSINT_FLAGS_SUSPENDED))
+    {
+        pDevIns->Internal.s.fIntFlags |= PDMDEVINSINT_FLAGS_SUSPENDED;
+        if (pDevIns->pReg->pfnPowerOff)
+        {
+            if (!pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                LogFlow(("PDMR3PowerOff: Notifying - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->pReg->pfnPowerOff(pDevIns);
+                if (pDevIns->Internal.s.pfnAsyncNotify)
+                    LogFlow(("PDMR3PowerOff: Async notification started - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+            }
+            else if (pDevIns->Internal.s.pfnAsyncNotify(pDevIns))
+            {
+                LogFlow(("PDMR3PowerOff: Async notification completed - device '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+                pDevIns->Internal.s.pfnAsyncNotify = NULL;
+            }
+            if (pDevIns->Internal.s.pfnAsyncNotify)
+            {
+                pDevIns->Internal.s.fIntFlags &= ~PDMDEVINSINT_FLAGS_SUSPENDED;
+                (*pcAsync)++;
+            }
+        }
+    }
+}
+
+
+/**
+ * This function will notify all the devices and their
+ * attached drivers about the VM being powered off.
+ *
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(void) PDMR3PowerOff(PVM pVM)
+{
+    LogFlow(("PDMR3PowerOff:\n"));
+
+    /*
+     * The outer loop repeats until there are no more async requests.
+     */
+    unsigned cAsync;
+    for (unsigned iLoop = 0; ; iLoop++)
+    {
+        /*
+         * Iterate thru the device instances and USB device instances,
+         * processing the drivers associated with those.
+         *
+         * The attached drivers are normally processed first.  Some devices
+         * (like DevAHCI) though needs to be notified before the drivers so
+         * that it doesn't kick off any new requests after the drivers stopped
+         * taking any. (DrvVD changes to read-only in this particular case.)
+         */
+        cAsync = 0;
+        for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION)
+                pdmR3PowerOffDev(pDevIns, &cAsync);
+
+            if (cAsync == cAsyncStart)
+                for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                        if (!pdmR3PowerOffDrv(pDrvIns, &cAsync, pDevIns->pReg->szName, pDevIns->iInstance, pLun->iLun))
+                            break;
+
+            if (    cAsync == cAsyncStart
+                && !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_FIRST_POWEROFF_NOTIFICATION))
+                pdmR3PowerOffDev(pDevIns, &cAsync);
+        }
+
+#ifdef VBOX_WITH_USB
+        for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        {
+            unsigned const cAsyncStart = cAsync;
+
+            for (PPDMLUN pLun = pUsbIns->Internal.s.pLuns; pLun; pLun = pLun->pNext)
+                for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                    if (!pdmR3PowerOffDrv(pDrvIns, &cAsync, pUsbIns->pReg->szName, pUsbIns->iInstance, pLun->iLun))
+                        break;
+
+            if (cAsync == cAsyncStart)
+                pdmR3PowerOffUsb(pUsbIns, &cAsync);
+        }
+#endif
+        if (!cAsync)
+            break;
+
+        /*
+         * Process requests.
+         */
+        /** @todo This is utterly nuts and completely unsafe... will get back to it in a
+         *        bit I hope... */
+        int rc = VMR3AsyncPdmNotificationWaitU(&pVM->pUVM->aCpus[0]);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = VMR3ReqProcessU(pVM->pUVM, 0/*idDstCpu*/);
+        AssertReleaseMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+    }
+
+    /*
+     * Suspend all threads.
+     */
+    pdmR3ThreadSuspendAll(pVM);
+
+    LogFlow(("PDMR3PowerOff: returns void\n"));
+}
+
+
+/**
+ * Queries the base interface of a device instance.
+ *
+ * The caller can use this to query other interfaces the device implements
+ * and use them to talk to the device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   ppBase          Where to store the pointer to the base device interface on success.
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryDevice(PVM pVM, const char *pszDevice, unsigned iInstance, PPDMIBASE *ppBase)
+{
+    LogFlow(("PDMR3DeviceQuery: pszDevice=%p:{%s} iInstance=%u ppBase=%p\n", pszDevice, pszDevice, iInstance, ppBase));
+
+    /*
+     * Iterate registered devices looking for the device.
+     */
+    size_t cchDevice = strlen(pszDevice);
+    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
+    {
+        if (    pDev->cchName == cchDevice
+            &&  !memcmp(pDev->pReg->szName, pszDevice, cchDevice))
+        {
+            /*
+             * Iterate device instances.
+             */
+            for (PPDMDEVINS pDevIns = pDev->pInstances; pDevIns; pDevIns = pDevIns->Internal.s.pPerDeviceNextR3)
+            {
+                if (pDevIns->iInstance == iInstance)
+                {
+                    if (pDevIns->IBase.pfnQueryInterface)
+                    {
+                        *ppBase = &pDevIns->IBase;
+                        LogFlow(("PDMR3DeviceQuery: return VINF_SUCCESS and *ppBase=%p\n", *ppBase));
+                        return VINF_SUCCESS;
+                    }
+
+                    LogFlow(("PDMR3DeviceQuery: returns VERR_PDM_DEVICE_INSTANCE_NO_IBASE\n"));
+                    return VERR_PDM_DEVICE_INSTANCE_NO_IBASE;
+                }
+            }
+
+            LogFlow(("PDMR3DeviceQuery: returns VERR_PDM_DEVICE_INSTANCE_NOT_FOUND\n"));
+            return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND;
+        }
+    }
+
+    LogFlow(("PDMR3QueryDevice: returns VERR_PDM_DEVICE_NOT_FOUND\n"));
+    return VERR_PDM_DEVICE_NOT_FOUND;
+}
+
+
+/**
+ * Queries the base interface of a device LUN.
+ *
+ * This differs from PDMR3QueryLun by that it returns the interface on the
+ * device and not the top level driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppBase          Where to store the base interface pointer.
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryDeviceLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMIBASE *ppBase)
+{
+    LogFlow(("PDMR3QueryLun: pszDevice=%p:{%s} iInstance=%u iLun=%u ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, ppBase));
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        *ppBase = pLun->pBase;
+        LogFlow(("PDMR3QueryDeviceLun: return VINF_SUCCESS and *ppBase=%p\n", *ppBase));
+        return VINF_SUCCESS;
+    }
+    LogFlow(("PDMR3QueryDeviceLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Query the interface of the top level driver on a LUN.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppBase          Where to store the base interface pointer.
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMIBASE *ppBase)
+{
+    LogFlow(("PDMR3QueryLun: pszDevice=%p:{%s} iInstance=%u iLun=%u ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, ppBase));
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLun->pTop)
+        {
+            *ppBase = &pLun->pTop->IBase;
+            LogFlow(("PDMR3QueryLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
+            return VINF_SUCCESS;
+        }
+        rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+    LogFlow(("PDMR3QueryLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Query the interface of a named driver on a LUN.
+ *
+ * If the driver appears more than once in the driver chain, the first instance
+ * is returned.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   pszDriver       The driver name.
+ * @param   ppBase          Where to store the base interface pointer.
+ *
+ * @remark  We're not doing any locking ATM, so don't try call this at times when the
+ *          device chain is known to be updated.
+ */
+VMMR3DECL(int) PDMR3QueryDriverOnLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, const char *pszDriver, PPPDMIBASE ppBase)
+{
+    LogFlow(("PDMR3QueryDriverOnLun: pszDevice=%p:{%s} iInstance=%u iLun=%u pszDriver=%p:{%s} ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, pszDriver, pszDriver, ppBase));
+
+    /*
+     * Find the LUN.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        if (pLun->pTop)
+        {
+            for (PPDMDRVINS pDrvIns = pLun->pTop; pDrvIns; pDrvIns = pDrvIns->Internal.s.pDown)
+                if (!strcmp(pDrvIns->pReg->szName, pszDriver))
+                {
+                    *ppBase = &pDrvIns->IBase;
+                    LogFlow(("PDMR3QueryDriverOnLun: return %Rrc and *ppBase=%p\n", VINF_SUCCESS, *ppBase));
+                    return VINF_SUCCESS;
+
+                }
+            rc = VERR_PDM_DRIVER_NOT_FOUND;
+        }
+        else
+            rc = VERR_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+    LogFlow(("PDMR3QueryDriverOnLun: returns %Rrc\n", rc));
+    return rc;
+}
+
+/**
+ * Executes pending DMA transfers.
+ * Forced Action handler.
+ *
+ * @param   pVM             VM handle.
+ */
+VMMR3DECL(void) PDMR3DmaRun(PVM pVM)
+{
+    /* Note! Not really SMP safe; restrict it to VCPU 0. */
+    if (VMMGetCpuId(pVM) != 0)
+        return;
+
+    if (VM_FF_TESTANDCLEAR(pVM, VM_FF_PDM_DMA))
+    {
+        if (pVM->pdm.s.pDmac)
+        {
+            bool fMore = pVM->pdm.s.pDmac->Reg.pfnRun(pVM->pdm.s.pDmac->pDevIns);
+            if (fMore)
+                VM_FF_SET(pVM, VM_FF_PDM_DMA);
+        }
+    }
+}
+
+
+/**
+ * Service a VMMCALLRING3_PDM_LOCK call.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+VMMR3DECL(int) PDMR3LockCall(PVM pVM)
+{
+    return PDMR3CritSectEnterEx(&pVM->pdm.s.CritSect, true /* fHostCall */);
+}
+
+
+/**
+ * Registers the VMM device heap
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   GCPhys          The physical address.
+ * @param   pvHeap          Ring-3 pointer.
+ * @param   cbSize          Size of the heap.
+ */
+VMMR3DECL(int) PDMR3RegisterVMMDevHeap(PVM pVM, RTGCPHYS GCPhys, RTR3PTR pvHeap, unsigned cbSize)
+{
+    Assert(pVM->pdm.s.pvVMMDevHeap == NULL);
+
+    Log(("PDMR3RegisterVMMDevHeap %RGp %RHv %x\n", GCPhys, pvHeap, cbSize));
+    pVM->pdm.s.pvVMMDevHeap     = pvHeap;
+    pVM->pdm.s.GCPhysVMMDevHeap = GCPhys;
+    pVM->pdm.s.cbVMMDevHeap     = cbSize;
+    pVM->pdm.s.cbVMMDevHeapLeft = cbSize;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unregisters the VMM device heap
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   GCPhys          The physical address.
+ */
+VMMR3DECL(int) PDMR3UnregisterVMMDevHeap(PVM pVM, RTGCPHYS GCPhys)
+{
+    Assert(pVM->pdm.s.GCPhysVMMDevHeap == GCPhys);
+
+    Log(("PDMR3UnregisterVMMDevHeap %RGp\n", GCPhys));
+    pVM->pdm.s.pvVMMDevHeap     = NULL;
+    pVM->pdm.s.GCPhysVMMDevHeap = NIL_RTGCPHYS;
+    pVM->pdm.s.cbVMMDevHeap     = 0;
+    pVM->pdm.s.cbVMMDevHeapLeft = 0;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Allocates memory from the VMM device heap
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   cbSize          Allocation size.
+ * @param   pv              Ring-3 pointer. (out)
+ */
+VMMR3DECL(int) PDMR3VMMDevHeapAlloc(PVM pVM, unsigned cbSize, RTR3PTR *ppv)
+{
+#ifdef DEBUG_bird
+    if (!cbSize || cbSize > pVM->pdm.s.cbVMMDevHeapLeft)
+        return VERR_NO_MEMORY;
+#else
+    AssertReturn(cbSize && cbSize <= pVM->pdm.s.cbVMMDevHeapLeft, VERR_NO_MEMORY);
+#endif
+
+    Log(("PDMR3VMMDevHeapAlloc %x\n", cbSize));
+
+    /** @todo not a real heap as there's currently only one user. */
+    *ppv = pVM->pdm.s.pvVMMDevHeap;
+    pVM->pdm.s.cbVMMDevHeapLeft = 0;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees memory from the VMM device heap
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pv              Ring-3 pointer.
+ */
+VMMR3DECL(int) PDMR3VMMDevHeapFree(PVM pVM, RTR3PTR pv)
+{
+    Log(("PDMR3VMMDevHeapFree %RHv\n", pv));
+
+    /** @todo not a real heap as there's currently only one user. */
+    pVM->pdm.s.cbVMMDevHeapLeft = pVM->pdm.s.cbVMMDevHeap;
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletion.cpp	(revision 35333)
@@ -0,0 +1,1609 @@
+/* $Id$ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ */
+
+/*
+ * Copyright (C) 2006-2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/rem.h>
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/mem.h>
+#include <iprt/critsect.h>
+#include <iprt/tcp.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+#include <VBox/pdmasynccompletion.h>
+#include "PDMAsyncCompletionInternal.h"
+
+/**
+ * Async I/O type.
+ */
+typedef enum PDMASYNCCOMPLETIONTEMPLATETYPE
+{
+    /** Device . */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_DEV = 1,
+    /** Driver consumer. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_DRV,
+    /** Internal consumer. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL,
+    /** Usb consumer. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE_USB
+} PDMASYNCTEMPLATETYPE;
+
+/**
+ * PDM Async I/O template.
+ */
+typedef struct PDMASYNCCOMPLETIONTEMPLATE
+{
+    /** Pointer to the next template in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE)    pNext;
+    /** Pointer to the previous template in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE)    pPrev;
+    /** Type specific data. */
+    union
+    {
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DEV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEDEV)   pfnCompleted;
+            /** Pointer to the device instance owning the template. */
+            R3PTRTYPE(PPDMDEVINS)               pDevIns;
+        } Dev;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DRV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEDRV)   pfnCompleted;
+            /** Pointer to the driver instance owning the template. */
+            R3PTRTYPE(PPDMDRVINS)               pDrvIns;
+            /** User argument given during template creation.
+             *  This is only here to make things much easier
+             *  for DrVVD. */
+            void                               *pvTemplateUser;
+        } Drv;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEINT)   pfnCompleted;
+            /** Pointer to user data. */
+            R3PTRTYPE(void *)                   pvUser;
+        } Int;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_USB */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMASYNCCOMPLETEUSB)   pfnCompleted;
+            /** Pointer to the usb instance owning the template. */
+            R3PTRTYPE(PPDMUSBINS)               pUsbIns;
+        } Usb;
+    } u;
+    /** Template type. */
+    PDMASYNCCOMPLETIONTEMPLATETYPE          enmType;
+    /** Pointer to the VM. */
+    R3PTRTYPE(PVM)                          pVM;
+    /** Use count of the template. */
+    volatile uint32_t                       cUsed;
+} PDMASYNCCOMPLETIONTEMPLATE;
+
+/**
+ * Bandwidth control manager instance data
+ */
+typedef struct PDMACBWMGR
+{
+    /** Pointer to the next manager in the list. */
+    struct PDMACBWMGR                          *pNext;
+    /** Pointer to the shared UVM structure. */
+    PPDMASYNCCOMPLETIONEPCLASS                  pEpClass;
+    /** Identifier of the manager. */
+    char                                       *pszId;
+    /** Maximum number of bytes the endpoints are allowed to transfer (Max is 4GB/s currently) */
+    volatile uint32_t                           cbTransferPerSecMax;
+    /** Number of bytes we start with */
+    volatile uint32_t                           cbTransferPerSecStart;
+    /** Step after each update */
+    volatile uint32_t                           cbTransferPerSecStep;
+    /** Number of bytes we are allowed to transfer till the next update.
+     * Reset by the refresh timer. */
+    volatile uint32_t                           cbTransferAllowed;
+    /** Timestamp of the last update */
+    volatile uint64_t                           tsUpdatedLast;
+    /** Reference counter - How many endpoints are associated with this manager. */
+    volatile uint32_t                           cRefs;
+} PDMACBWMGR;
+/** Pointer to a bandwidth control manager pointer. */
+typedef PPDMACBWMGR *PPPDMACBWMGR;
+
+static void pdmR3AsyncCompletionPutTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, PPDMASYNCCOMPLETIONTASK pTask);
+
+/**
+ * Internal worker for the creation apis
+ *
+ * @returns VBox status.
+ * @param   pVM           VM handle.
+ * @param   ppTemplate    Where to store the template handle.
+ */
+static int pdmR3AsyncCompletionTemplateCreate(PVM pVM, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE enmType)
+{
+    PUVM pUVM = pVM->pUVM;
+
+    if (ppTemplate == NULL)
+    {
+        AssertMsgFailed(("ppTemplate is NULL\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION, sizeof(PDMASYNCCOMPLETIONTEMPLATE), (void **)&pTemplate);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize fields.
+     */
+    pTemplate->pVM = pVM;
+    pTemplate->cUsed = 0;
+    pTemplate->enmType = enmType;
+
+    /*
+     * Add template to the global VM template list.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    pTemplate->pNext = pUVM->pdm.s.pAsyncCompletionTemplates;
+    if (pUVM->pdm.s.pAsyncCompletionTemplates)
+        pUVM->pdm.s.pAsyncCompletionTemplates->pPrev = pTemplate;
+    pUVM->pdm.s.pAsyncCompletionTemplates = pTemplate;
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+    *ppTemplate = pTemplate;
+    return VINF_SUCCESS;
+}
+
+/**
+ * Creates a async completion template for a device instance.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The device instance.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pszDesc         Description.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEDEV pfnCompleted, const char *pszDesc)
+{
+    LogFlow(("%s: pDevIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
+              __FUNCTION__, pDevIns, ppTemplate, pfnCompleted, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    if (!pfnCompleted)
+    {
+        AssertMsgFailed(("No completion callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppTemplate)
+    {
+        AssertMsgFailed(("Template pointer is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_DEV);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Dev.pDevIns = pDevIns;
+        pTemplate->u.Dev.pfnCompleted = pfnCompleted;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created device template %p: pfnCompleted=%p pDevIns=%p\n",
+             pTemplate, pfnCompleted, pDevIns));
+    }
+
+    return rc;
+}
+
+/**
+ * Creates a async completion template for a driver instance.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDrvIns         The driver instance.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pvTemplateUser  Template user argument
+ * @param   pszDesc         Description.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEDRV pfnCompleted, void *pvTemplateUser, const char *pszDesc)
+{
+    LogFlow(("%s: pDrvIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
+              __FUNCTION__, pDrvIns, ppTemplate, pfnCompleted, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    if (!pfnCompleted)
+    {
+        AssertMsgFailed(("No completion callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppTemplate)
+    {
+        AssertMsgFailed(("Template pointer is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_DRV);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Drv.pDrvIns        = pDrvIns;
+        pTemplate->u.Drv.pfnCompleted   = pfnCompleted;
+        pTemplate->u.Drv.pvTemplateUser = pvTemplateUser;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created driver template %p: pfnCompleted=%p pDrvIns=%p\n",
+             pTemplate, pfnCompleted, pDrvIns));
+    }
+
+    return rc;
+}
+
+/**
+ * Creates a async completion template for a USB device instance.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pUsbIns         The USB device instance.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pszDesc         Description.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEUSB pfnCompleted, const char *pszDesc)
+{
+    LogFlow(("%s: pUsbIns=%p ppTemplate=%p pfnCompleted=%p pszDesc=%s\n",
+              __FUNCTION__, pUsbIns, ppTemplate, pfnCompleted, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    if (!pfnCompleted)
+    {
+        AssertMsgFailed(("No completion callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppTemplate)
+    {
+        AssertMsgFailed(("Template pointer is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_USB);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Usb.pUsbIns = pUsbIns;
+        pTemplate->u.Usb.pfnCompleted = pfnCompleted;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created usb template %p: pfnCompleted=%p pDevIns=%p\n",
+             pTemplate, pfnCompleted, pUsbIns));
+    }
+
+    return rc;
+}
+
+/**
+ * Creates a async completion template for internally by the VMM.
+ *
+ * The template is used when creating new completion tasks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   ppTemplate      Where to store the template pointer on success.
+ * @param   pfnCompleted    The completion callback routine.
+ * @param   pvUser2         The 2nd user argument for the callback.
+ * @param   pszDesc         Description.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateCreateInternal(PVM pVM, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate, PFNPDMASYNCCOMPLETEINT pfnCompleted, void *pvUser2, const char *pszDesc)
+{
+    LogFlow(("%s: ppTemplate=%p pfnCompleted=%p pvUser2=%p pszDesc=%s\n",
+              __FUNCTION__, ppTemplate, pfnCompleted, pvUser2, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    if (!pfnCompleted)
+    {
+        AssertMsgFailed(("No completion callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppTemplate)
+    {
+        AssertMsgFailed(("Template pointer is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the template.
+     */
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate;
+    int rc = pdmR3AsyncCompletionTemplateCreate(pVM, &pTemplate, PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL);
+    if (RT_SUCCESS(rc))
+    {
+        pTemplate->u.Int.pvUser = pvUser2;
+        pTemplate->u.Int.pfnCompleted = pfnCompleted;
+
+        *ppTemplate = pTemplate;
+        Log(("PDM: Created internal template %p: pfnCompleted=%p pvUser2=%p\n",
+             pTemplate, pfnCompleted, pvUser2));
+    }
+
+    return rc;
+}
+
+/**
+ * Destroys the specified async completion template.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if the template is still in use.
+ *
+ * @param   pTemplate       The template in question.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroy(PPDMASYNCCOMPLETIONTEMPLATE pTemplate)
+{
+    LogFlow(("%s: pTemplate=%p\n", __FUNCTION__, pTemplate));
+
+    if (!pTemplate)
+    {
+        AssertMsgFailed(("pTemplate is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Check if the template is still used.
+     */
+    if (pTemplate->cUsed > 0)
+    {
+        AssertMsgFailed(("Template is still in use\n"));
+        return VERR_PDM_ASYNC_TEMPLATE_BUSY;
+    }
+
+    /*
+     * Unlink the template from the list.
+     */
+    PUVM pUVM = pTemplate->pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+
+    PPDMASYNCCOMPLETIONTEMPLATE pPrev = pTemplate->pPrev;
+    PPDMASYNCCOMPLETIONTEMPLATE pNext = pTemplate->pNext;
+
+    if (pPrev)
+        pPrev->pNext = pNext;
+    else
+        pUVM->pdm.s.pAsyncCompletionTemplates = pNext;
+
+    if (pNext)
+        pNext->pPrev = pPrev;
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+    /*
+     * Free the template.
+     */
+    MMR3HeapFree(pTemplate);
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Destroys all the specified async completion templates for the given device instance.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The device instance.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+        return VERR_INVALID_PARAMETER;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
+    while (pTemplate)
+    {
+        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_DEV
+            &&  pTemplate->u.Dev.pDevIns == pDevIns)
+        {
+            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
+            pTemplate = pTemplate->pNext;
+            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
+            if (RT_FAILURE(rc))
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+        }
+        else
+            pTemplate = pTemplate->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Destroys all the specified async completion templates for the given driver instance.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDrvIns         The driver instance.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDrvIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+        return VERR_INVALID_PARAMETER;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
+    while (pTemplate)
+    {
+        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_DRV
+            &&  pTemplate->u.Drv.pDrvIns == pDrvIns)
+        {
+            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
+            pTemplate = pTemplate->pNext;
+            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
+            if (RT_FAILURE(rc))
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+        }
+        else
+            pTemplate = pTemplate->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+/**
+ * Destroys all the specified async completion templates for the given USB device instance.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PDM_ASYNC_TEMPLATE_BUSY if one or more of the templates are still in use.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pUsbIns         The USB device instance.
+ */
+VMMR3DECL(int) PDMR3AsyncCompletionTemplateDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pUsbIns)
+        return VERR_INVALID_PARAMETER;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pUVM->pdm.s.pAsyncCompletionTemplates;
+    while (pTemplate)
+    {
+        if (    pTemplate->enmType == PDMASYNCCOMPLETIONTEMPLATETYPE_USB
+            &&  pTemplate->u.Usb.pUsbIns == pUsbIns)
+        {
+            PPDMASYNCCOMPLETIONTEMPLATE pTemplateDestroy = pTemplate;
+            pTemplate = pTemplate->pNext;
+            int rc = PDMR3AsyncCompletionTemplateDestroy(pTemplateDestroy);
+            if (RT_FAILURE(rc))
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+        }
+        else
+            pTemplate = pTemplate->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+
+static PPDMACBWMGR pdmacBwMgrFindById(PPDMASYNCCOMPLETIONEPCLASS pEpClass, const char *pcszId)
+{
+    PPDMACBWMGR pBwMgr = NULL;
+
+    if (VALID_PTR(pcszId))
+    {
+        int rc;
+        rc = RTCritSectEnter(&pEpClass->CritSect);
+        AssertRC(rc);
+
+        pBwMgr = pEpClass->pBwMgrsHead;
+        while (   pBwMgr
+               && RTStrCmp(pBwMgr->pszId, pcszId))
+            pBwMgr = pBwMgr->pNext;
+
+        rc = RTCritSectLeave(&pEpClass->CritSect);
+        AssertRC(rc);
+    }
+
+    return pBwMgr;
+}
+
+static void pdmacBwMgrLink(PPDMACBWMGR pBwMgr)
+{
+    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pBwMgr->pEpClass;
+    int rc;
+
+    rc = RTCritSectEnter(&pEpClass->CritSect);
+    AssertRC(rc);
+
+    pBwMgr->pNext = pEpClass->pBwMgrsHead;
+    pEpClass->pBwMgrsHead = pBwMgr;
+
+    rc = RTCritSectLeave(&pEpClass->CritSect);
+    AssertRC(rc);
+}
+
+static void pdmacBwMgrUnlink(PPDMACBWMGR pBwMgr)
+{
+    int rc;
+    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pBwMgr->pEpClass;
+
+    rc = RTCritSectEnter(&pEpClass->CritSect);
+    AssertRC(rc);
+
+    if (pBwMgr == pEpClass->pBwMgrsHead)
+        pEpClass->pBwMgrsHead = pBwMgr->pNext;
+    else
+    {
+        PPDMACBWMGR pPrev = pEpClass->pBwMgrsHead;
+        while (   pPrev
+               && pPrev->pNext != pBwMgr)
+            pPrev = pPrev->pNext;
+
+        AssertPtr(pPrev);
+        pPrev->pNext = pBwMgr->pNext;
+    }
+
+    rc = RTCritSectLeave(&pEpClass->CritSect);
+    AssertRC(rc);
+}
+
+static int pdmacAsyncCompletionBwMgrCreate(PPDMASYNCCOMPLETIONEPCLASS pEpClass, const char *pcszBwMgr, uint32_t cbTransferPerSecMax,
+                                           uint32_t cbTransferPerSecStart, uint32_t cbTransferPerSecStep)
+{
+    int rc = VINF_SUCCESS;
+    PPDMACBWMGR pBwMgr;
+
+    LogFlowFunc(("pEpClass=%#p pcszBwMgr=%#p{%s} cbTransferPerSecMax=%u cbTransferPerSecStart=%u cbTransferPerSecStep=%u\n",
+                 pEpClass, pcszBwMgr, cbTransferPerSecMax, cbTransferPerSecStart, cbTransferPerSecStep));
+
+    AssertPtrReturn(pEpClass, VERR_INVALID_POINTER);
+    AssertPtrReturn(pcszBwMgr, VERR_INVALID_POINTER);
+    AssertReturn(*pcszBwMgr != '\0', VERR_INVALID_PARAMETER);
+
+    pBwMgr = pdmacBwMgrFindById(pEpClass, pcszBwMgr);
+    if (!pBwMgr)
+    {
+        rc = MMR3HeapAllocZEx(pEpClass->pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                              sizeof(PDMACBWMGR),
+                              (void **)&pBwMgr);
+        if (RT_SUCCESS(rc))
+        {
+            pBwMgr->pszId = RTStrDup(pcszBwMgr);
+            if (pBwMgr->pszId)
+            {
+                pBwMgr->pEpClass              = pEpClass;
+                pBwMgr->cRefs                 = 0;
+
+                /* Init I/O flow control. */
+                pBwMgr->cbTransferPerSecMax   = cbTransferPerSecMax;
+                pBwMgr->cbTransferPerSecStart = cbTransferPerSecStart;
+                pBwMgr->cbTransferPerSecStep  = cbTransferPerSecStep;
+
+                pBwMgr->cbTransferAllowed     = pBwMgr->cbTransferPerSecStart;
+                pBwMgr->tsUpdatedLast         = RTTimeSystemNanoTS();
+
+                pdmacBwMgrLink(pBwMgr);
+                rc = VINF_SUCCESS;
+            }
+            else
+            {
+                rc = VERR_NO_MEMORY;
+                MMR3HeapFree(pBwMgr);
+            }
+        }
+    }
+    else
+        rc = VERR_ALREADY_EXISTS;
+
+    LogFlowFunc(("returns rc=%Rrc\n", rc));
+    return rc;
+}
+
+DECLINLINE(void) pdmacBwMgrRef(PPDMACBWMGR pBwMgr)
+{
+    ASMAtomicIncU32(&pBwMgr->cRefs);
+}
+
+DECLINLINE(void) pdmacBwMgrUnref(PPDMACBWMGR pBwMgr)
+{
+    Assert(pBwMgr->cRefs > 0);
+    ASMAtomicDecU32(&pBwMgr->cRefs);
+}
+
+bool pdmacEpIsTransferAllowed(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint32_t cbTransfer, RTMSINTERVAL *pmsWhenNext)
+{
+    bool fAllowed = true;
+    PPDMACBWMGR pBwMgr = ASMAtomicReadPtrT(&pEndpoint->pBwMgr, PPDMACBWMGR);
+
+    LogFlowFunc(("pEndpoint=%p pBwMgr=%p cbTransfer=%u\n", pEndpoint, pBwMgr, cbTransfer));
+
+    if (pBwMgr)
+    {
+        uint32_t cbOld = ASMAtomicSubU32(&pBwMgr->cbTransferAllowed, cbTransfer);
+        if (RT_LIKELY(cbOld >= cbTransfer))
+            fAllowed = true;
+        else
+        {
+            fAllowed = false;
+
+            /* We are out of resources  Check if we can update again. */
+            uint64_t tsNow          = RTTimeSystemNanoTS();
+            uint64_t tsUpdatedLast  = ASMAtomicUoReadU64(&pBwMgr->tsUpdatedLast);
+
+            if (tsNow - tsUpdatedLast >= (1000*1000*1000))
+            {
+                if (ASMAtomicCmpXchgU64(&pBwMgr->tsUpdatedLast, tsNow, tsUpdatedLast))
+                {
+                    if (pBwMgr->cbTransferPerSecStart < pBwMgr->cbTransferPerSecMax)
+                    {
+                       pBwMgr->cbTransferPerSecStart = RT_MIN(pBwMgr->cbTransferPerSecMax, pBwMgr->cbTransferPerSecStart + pBwMgr->cbTransferPerSecStep);
+                       LogFlow(("AIOMgr: Increasing maximum bandwidth to %u bytes/sec\n", pBwMgr->cbTransferPerSecStart));
+                    }
+
+                    /* Update */
+                    ASMAtomicWriteU32(&pBwMgr->cbTransferAllowed, pBwMgr->cbTransferPerSecStart - cbTransfer);
+                    fAllowed = true;
+                    LogFlow(("AIOMgr: Refreshed bandwidth\n"));
+                }
+            }
+            else
+            {
+                ASMAtomicAddU32(&pBwMgr->cbTransferAllowed, cbTransfer);
+                *pmsWhenNext = ((1000*1000*1000) - (tsNow - tsUpdatedLast)) / (1000*1000);
+            }
+        }
+    }
+
+    LogFlowFunc(("fAllowed=%RTbool\n", fAllowed));
+
+    return fAllowed;
+}
+
+void pdmR3AsyncCompletionCompleteTask(PPDMASYNCCOMPLETIONTASK pTask, int rc, bool fCallCompletionHandler)
+{
+    LogFlow(("%s: pTask=%#p fCallCompletionHandler=%RTbool\n", __FUNCTION__, pTask, fCallCompletionHandler));
+
+    if (fCallCompletionHandler)
+    {
+        PPDMASYNCCOMPLETIONTEMPLATE pTemplate = pTask->pEndpoint->pTemplate;
+
+        switch (pTemplate->enmType)
+        {
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_DEV:
+            {
+                pTemplate->u.Dev.pfnCompleted(pTemplate->u.Dev.pDevIns, pTask->pvUser, rc);
+                break;
+            }
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_DRV:
+            {
+                pTemplate->u.Drv.pfnCompleted(pTemplate->u.Drv.pDrvIns, pTemplate->u.Drv.pvTemplateUser, pTask->pvUser, rc);
+                break;
+            }
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_USB:
+            {
+                pTemplate->u.Usb.pfnCompleted(pTemplate->u.Usb.pUsbIns, pTask->pvUser, rc);
+                break;
+            }
+            case PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL:
+            {
+                pTemplate->u.Int.pfnCompleted(pTemplate->pVM, pTask->pvUser, pTemplate->u.Int.pvUser, rc);
+                break;
+            }
+            default:
+                AssertMsgFailed(("Unknown template type!\n"));
+        }
+    }
+
+    pdmR3AsyncCompletionPutTask(pTask->pEndpoint, pTask);
+}
+
+/**
+ * Worker initializing a endpoint class.
+ *
+ * @returns VBox status code.
+ * @param   pVM        Pointer to the shared VM instance data.
+ * @param   pEpClass   Pointer to the endpoint class structure.
+ * @param   pCfgHandle Pointer to the the CFGM tree.
+ */
+int pdmR3AsyncCompletionEpClassInit(PVM pVM, PCPDMASYNCCOMPLETIONEPCLASSOPS pEpClassOps, PCFGMNODE pCfgHandle)
+{
+    int rc = VINF_SUCCESS;
+
+    /* Validate input. */
+    if (   !pEpClassOps
+        || (pEpClassOps->u32Version != PDMAC_EPCLASS_OPS_VERSION)
+        || (pEpClassOps->u32VersionEnd != PDMAC_EPCLASS_OPS_VERSION))
+        AssertMsgFailedReturn(("Invalid endpoint class data\n"), VERR_VERSION_MISMATCH);
+
+    LogFlowFunc((": pVM=%p pEpClassOps=%p{%s}\n", pVM, pEpClassOps, pEpClassOps->pcszName));
+
+    /* Allocate global class data. */
+    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = NULL;
+
+    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                          pEpClassOps->cbEndpointClassGlobal,
+                          (void **)&pEndpointClass);
+    if (RT_SUCCESS(rc))
+    {
+        /* Initialize common data. */
+        pEndpointClass->pVM          = pVM;
+        pEndpointClass->pEndpointOps = pEpClassOps;
+
+        rc = RTCritSectInit(&pEndpointClass->CritSect);
+        if (RT_SUCCESS(rc))
+        {
+            PCFGMNODE pCfgNodeClass = CFGMR3GetChild(pCfgHandle, pEpClassOps->pcszName);
+
+            /* Create task cache */
+            rc = RTMemCacheCreate(&pEndpointClass->hMemCacheTasks, pEpClassOps->cbTask,
+                                  0, UINT32_MAX, NULL, NULL, NULL, 0);
+            if (RT_SUCCESS(rc))
+            {
+                /* Call the specific endpoint class initializer. */
+                rc = pEpClassOps->pfnInitialize(pEndpointClass, pCfgNodeClass);
+                if (RT_SUCCESS(rc))
+                {
+                    /* Create all bandwidth groups for resource control. */
+                    PCFGMNODE pCfgBwGrp = CFGMR3GetChild(pCfgNodeClass, "BwGroups");
+
+                    if (pCfgBwGrp)
+                    {
+                        for (PCFGMNODE pCur = CFGMR3GetFirstChild(pCfgBwGrp); pCur; pCur = CFGMR3GetNextChild(pCur))
+                        {
+                            uint32_t cbMax, cbStart, cbStep;
+                            size_t cchName = CFGMR3GetNameLen(pCur) + 1;
+                            char *pszBwGrpId = (char *)RTMemAllocZ(cchName);
+
+                            if (!pszBwGrpId)
+                            {
+                                rc = VERR_NO_MEMORY;
+                                break;
+                            }
+
+                            rc = CFGMR3GetName(pCur, pszBwGrpId, cchName);
+                            AssertRC(rc);
+
+                            if (RT_SUCCESS(rc))
+                                rc = CFGMR3QueryU32(pCur, "Max", &cbMax);
+                            if (RT_SUCCESS(rc))
+                                rc = CFGMR3QueryU32Def(pCur, "Start", &cbStart, cbMax);
+                            if (RT_SUCCESS(rc))
+                                rc = CFGMR3QueryU32Def(pCur, "Step", &cbStep, 0);
+                            if (RT_SUCCESS(rc))
+                                rc = pdmacAsyncCompletionBwMgrCreate(pEndpointClass, pszBwGrpId, cbMax, cbStart, cbStep);
+
+                            RTMemFree(pszBwGrpId);
+
+                            if (RT_FAILURE(rc))
+                                break;
+                        }
+                    }
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        PUVM pUVM = pVM->pUVM;
+                        AssertMsg(!pUVM->pdm.s.apAsyncCompletionEndpointClass[pEpClassOps->enmClassType],
+                                  ("Endpoint class was already initialized\n"));
+
+                        pUVM->pdm.s.apAsyncCompletionEndpointClass[pEpClassOps->enmClassType] = pEndpointClass;
+                        LogFlowFunc((": Initialized endpoint class \"%s\" rc=%Rrc\n", pEpClassOps->pcszName, rc));
+                        return VINF_SUCCESS;
+                    }
+                }
+                RTMemCacheDestroy(pEndpointClass->hMemCacheTasks);
+            }
+            RTCritSectDelete(&pEndpointClass->CritSect);
+        }
+        MMR3HeapFree(pEndpointClass);
+    }
+
+    LogFlowFunc((": Failed to initialize endpoint class rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+/**
+ * Worker terminating all endpoint classes.
+ *
+ * @returns nothing
+ * @param   pEndpointClass    Pointer to the endpoint class to terminate.
+ *
+ * @remarks This method ensures that any still open endpoint is closed.
+ */
+static void pdmR3AsyncCompletionEpClassTerminate(PPDMASYNCCOMPLETIONEPCLASS pEndpointClass)
+{
+    int rc = VINF_SUCCESS;
+    PVM pVM = pEndpointClass->pVM;
+
+    /* Close all still open endpoints. */
+    while (pEndpointClass->pEndpointsHead)
+        PDMR3AsyncCompletionEpClose(pEndpointClass->pEndpointsHead);
+
+    /* Destroy the bandwidth managers. */
+    PPDMACBWMGR pBwMgr = pEndpointClass->pBwMgrsHead;
+    while (pBwMgr)
+    {
+        PPDMACBWMGR pFree = pBwMgr;
+        pBwMgr = pBwMgr->pNext;
+        MMR3HeapFree(pFree);
+    }
+
+    /* Call the termination callback of the class. */
+    pEndpointClass->pEndpointOps->pfnTerminate(pEndpointClass);
+
+    RTMemCacheDestroy(pEndpointClass->hMemCacheTasks);
+    RTCritSectDelete(&pEndpointClass->CritSect);
+
+    /* Free the memory of the class finally and clear the entry in the class array. */
+    pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[pEndpointClass->pEndpointOps->enmClassType] = NULL;
+    MMR3HeapFree(pEndpointClass);
+}
+
+/**
+ * Initialize the async completion manager.
+ *
+ * @returns VBox status code
+ * @param   pVM Pointer to the shared VM structure.
+ */
+int pdmR3AsyncCompletionInit(PVM pVM)
+{
+    int  rc   = VINF_SUCCESS;
+    PUVM pUVM = pVM->pUVM;
+
+    LogFlowFunc((": pVM=%p\n", pVM));
+
+    VM_ASSERT_EMT(pVM);
+
+    PCFGMNODE pCfgRoot            = CFGMR3GetRoot(pVM);
+    PCFGMNODE pCfgAsyncCompletion = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "AsyncCompletion");
+
+    rc = pdmR3AsyncCompletionEpClassInit(pVM, &g_PDMAsyncCompletionEndpointClassFile, pCfgAsyncCompletion);
+
+    LogFlowFunc((": pVM=%p rc=%Rrc\n", pVM, rc));
+
+    return rc;
+}
+
+/**
+ * Terminates the async completion manager.
+ *
+ * @returns VBox status code
+ * @param   pVM Pointer to the shared VM structure.
+ */
+int pdmR3AsyncCompletionTerm(PVM pVM)
+{
+    LogFlowFunc((": pVM=%p\n", pVM));
+    PUVM pUVM = pVM->pUVM;
+
+    for (size_t i = 0; i < RT_ELEMENTS(pUVM->pdm.s.apAsyncCompletionEndpointClass); i++)
+        if (pUVM->pdm.s.apAsyncCompletionEndpointClass[i])
+            pdmR3AsyncCompletionEpClassTerminate(pUVM->pdm.s.apAsyncCompletionEndpointClass[i]);
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Resume worker for the async completion manager.
+ *
+ * @returns nothing.
+ * @param   pVM Pointer to the shared VM structure.
+ */
+void pdmR3AsyncCompletionResume(PVM pVM)
+{
+    LogFlowFunc((": pVM=%p\n", pVM));
+    PUVM pUVM = pVM->pUVM;
+
+    /* Log the bandwidth groups and all assigned endpoints. */
+    for (size_t i = 0; i < RT_ELEMENTS(pUVM->pdm.s.apAsyncCompletionEndpointClass); i++)
+        if (pUVM->pdm.s.apAsyncCompletionEndpointClass[i])
+        {
+            PPDMASYNCCOMPLETIONEPCLASS pEpClass = pUVM->pdm.s.apAsyncCompletionEndpointClass[i];
+            PPDMASYNCCOMPLETIONENDPOINT pEp;
+            PPDMACBWMGR pBwMgr = pEpClass->pBwMgrsHead;
+
+            if (pBwMgr)
+                LogRel(("AIOMgr: Bandwidth groups for class '%s'\n", i == PDMASYNCCOMPLETIONEPCLASSTYPE_FILE
+                                                                     ? "File" : "<Unknown>"));
+
+            while (pBwMgr)
+            {
+                LogRel(("AIOMgr:     Id:    %s\n", pBwMgr->pszId));
+                LogRel(("AIOMgr:     Max:   %u B/s\n", pBwMgr->cbTransferPerSecMax));
+                LogRel(("AIOMgr:     Start: %u B/s\n", pBwMgr->cbTransferPerSecStart));
+                LogRel(("AIOMgr:     Step:  %u B/s\n", pBwMgr->cbTransferPerSecStep));
+                LogRel(("AIOMgr:     Endpoints:\n"));
+
+                pEp = pEpClass->pEndpointsHead;
+                while (pEp)
+                {
+                    if (pEp->pBwMgr == pBwMgr)
+                        LogRel(("AIOMgr:         %s\n", pEp->pszUri));
+
+                    pEp = pEp->pNext;
+                }
+
+                pBwMgr = pBwMgr->pNext;
+            }
+
+            /* Print all endpoints without assigned bandwidth groups. */
+            pEp = pEpClass->pEndpointsHead;
+            if (pEp)
+                LogRel(("AIOMgr: Endpoints without assigned bandwidth groups:\n"));
+
+            while (pEp)
+            {
+                if (!pEp->pBwMgr)
+                    LogRel(("AIOMgr:     %s\n", pEp->pszUri));
+
+                pEp = pEp->pNext;
+            }
+        }
+}
+
+/**
+ * Tries to get a free task from the endpoint or class cache
+ * allocating the task if it fails.
+ *
+ * @returns Pointer to a new and initialized task or NULL
+ * @param   pEndpoint    The endpoint the task is for.
+ * @param   pvUser       Opaque user data for the task.
+ */
+static PPDMASYNCCOMPLETIONTASK pdmR3AsyncCompletionGetTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, void *pvUser)
+{
+    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
+    PPDMASYNCCOMPLETIONTASK pTask = NULL;
+
+    pTask = (PPDMASYNCCOMPLETIONTASK)RTMemCacheAlloc(pEndpointClass->hMemCacheTasks);
+
+    if (RT_LIKELY(pTask))
+    {
+        /* Initialize common parts. */
+        pTask->pvUser    = pvUser;
+        pTask->pEndpoint = pEndpoint;
+        /* Clear list pointers for safety. */
+        pTask->pPrev     = NULL;
+        pTask->pNext     = NULL;
+        pTask->tsNsStart = RTTimeNanoTS();
+#ifdef VBOX_WITH_STATISTICS
+        STAM_COUNTER_INC(&pEndpoint->StatIoOpsStarted);
+#endif
+    }
+
+    return pTask;
+}
+
+/**
+ * Puts a task in one of the caches.
+ *
+ * @returns nothing.
+ * @param   pEndpoint    The endpoint the task belongs to.
+ * @param   pTask        The task to cache.
+ */
+static void pdmR3AsyncCompletionPutTask(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, PPDMASYNCCOMPLETIONTASK pTask)
+{
+    PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
+    uint64_t tsRun  = RTTimeNanoTS() - pTask->tsNsStart;
+
+    if (RT_UNLIKELY(tsRun >= (uint64_t)10*1000*1000*1000))
+    {
+        LogRel(("AsyncCompletion: Task %#p completed after %llu seconds\n", pTask, tsRun / ((uint64_t)1000*1000*1000)));
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    uint64_t iStatIdx;
+
+    if (tsRun < 1000)
+    {
+        /* Update nanoseconds statistics */
+        iStatIdx = tsRun / 100;
+        STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesNs[iStatIdx]);
+    }
+    else
+    {
+        tsRun /= 1000;
+
+        if (tsRun < 1000)
+        {
+            /* Update microsecnds statistics */
+            iStatIdx = tsRun / 100;
+            STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesMicroSec[iStatIdx]);
+        }
+        else
+        {
+            tsRun /= 1000;
+
+            if (tsRun < 1000)
+            {
+                /* Update milliseconds statistics */
+                iStatIdx = tsRun / 100;
+                STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesMs[iStatIdx]);
+            }
+            else
+            {
+                tsRun /= 1000;
+
+                if (tsRun < 1000)
+                {
+                    /* Update seconds statistics */
+                    iStatIdx = tsRun / 10;
+                    STAM_COUNTER_INC(&pEndpoint->StatTaskRunTimesSec[iStatIdx]);
+                }
+                else
+                    STAM_COUNTER_INC(&pEndpoint->StatTaskRunOver100Sec);
+            }
+        }
+    }
+
+    STAM_COUNTER_INC(&pEndpoint->StatIoOpsCompleted);
+    pEndpoint->cIoOpsCompleted++;
+    uint64_t tsMsCur = RTTimeMilliTS();
+    uint64_t tsInterval = tsMsCur - pEndpoint->tsIntervalStartMs;
+
+    if (tsInterval >= 1000)
+    {
+        pEndpoint->StatIoOpsPerSec.c = pEndpoint->cIoOpsCompleted / (tsInterval / 1000);
+        pEndpoint->tsIntervalStartMs = tsMsCur;
+        pEndpoint->cIoOpsCompleted = 0;
+    }
+#endif
+
+    RTMemCacheFree(pEndpointClass->hMemCacheTasks, pTask);
+}
+
+static PPDMASYNCCOMPLETIONENDPOINT pdmR3AsyncCompletionFindEndpointWithUri(PPDMASYNCCOMPLETIONEPCLASS pEndpointClass,
+                                                                           const char *pszUri)
+{
+    PPDMASYNCCOMPLETIONENDPOINT pEndpoint = pEndpointClass->pEndpointsHead;
+
+    while (pEndpoint)
+    {
+        if (!RTStrCmp(pEndpoint->pszUri, pszUri))
+            return pEndpoint;
+
+        pEndpoint = pEndpoint->pNext;
+    }
+
+    return NULL;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionEpCreateForFile(PPPDMASYNCCOMPLETIONENDPOINT ppEndpoint,
+                                                   const char *pszFilename, uint32_t fFlags,
+                                                   PPDMASYNCCOMPLETIONTEMPLATE pTemplate)
+{
+    int rc = VINF_SUCCESS;
+
+    LogFlowFunc((": ppEndpoint=%p pszFilename=%p{%s} fFlags=%u pTemplate=%p\n",
+                 ppEndpoint, pszFilename, pszFilename, fFlags, pTemplate));
+
+    /* Sanity checks. */
+    AssertReturn(VALID_PTR(ppEndpoint),  VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(pszFilename), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(pTemplate),   VERR_INVALID_POINTER);
+
+    /* Check that the flags are valid. */
+    AssertReturn(((~(PDMACEP_FILE_FLAGS_READ_ONLY | PDMACEP_FILE_FLAGS_DONT_LOCK) & fFlags) == 0),
+                 VERR_INVALID_PARAMETER);
+
+    PVM  pVM  = pTemplate->pVM;
+    PUVM pUVM = pVM->pUVM;
+    PPDMASYNCCOMPLETIONEPCLASS  pEndpointClass = pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
+    PPDMASYNCCOMPLETIONENDPOINT pEndpoint = NULL;
+
+    AssertMsg(pEndpointClass, ("File endpoint class was not initialized\n"));
+
+    /* Search for a already opened endpoint for this file. */
+    pEndpoint = pdmR3AsyncCompletionFindEndpointWithUri(pEndpointClass, pszFilename);
+
+    if(!pEndpoint)
+    {
+        rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                              pEndpointClass->pEndpointOps->cbEndpoint,
+                              (void **)&pEndpoint);
+        if (RT_SUCCESS(rc))
+        {
+
+            /* Initialize common parts. */
+            pEndpoint->pNext             = NULL;
+            pEndpoint->pPrev             = NULL;
+            pEndpoint->pEpClass          = pEndpointClass;
+            pEndpoint->pTemplate         = pTemplate;
+            pEndpoint->pszUri            = RTStrDup(pszFilename);
+            pEndpoint->cUsers            = 1;
+            pEndpoint->pBwMgr            = NULL;
+
+            if (   pEndpoint->pszUri
+                && RT_SUCCESS(rc))
+            {
+                /* Call the initializer for the endpoint. */
+                rc = pEndpointClass->pEndpointOps->pfnEpInitialize(pEndpoint, pszFilename, fFlags);
+                if (RT_SUCCESS(rc))
+                {
+                    /* Link it into the list of endpoints. */
+                    rc = RTCritSectEnter(&pEndpointClass->CritSect);
+                    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+                    pEndpoint->pNext = pEndpointClass->pEndpointsHead;
+                    if (pEndpointClass->pEndpointsHead)
+                        pEndpointClass->pEndpointsHead->pPrev = pEndpoint;
+
+                    pEndpointClass->pEndpointsHead = pEndpoint;
+                    pEndpointClass->cEndpoints++;
+
+                    rc = RTCritSectLeave(&pEndpointClass->CritSect);
+                    AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+                    /* Reference the template. */
+                    ASMAtomicIncU32(&pTemplate->cUsed);
+
+#ifdef VBOX_WITH_STATISTICS
+                    /* Init the statistics part */
+                    for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesNs); i++)
+                    {
+                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesNs[i], STAMTYPE_COUNTER,
+                                             STAMVISIBILITY_USED,
+                                             STAMUNIT_OCCURENCES,
+                                             "Nanosecond resolution runtime statistics",
+                                             "/PDM/AsyncCompletion/File/%s/TaskRun1Ns-%u-%u",
+                                             RTPathFilename(pEndpoint->pszUri),
+                                             i*100, i*100+100-1);
+                        if (RT_FAILURE(rc))
+                            break;
+                    }
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMicroSec); i++)
+                        {
+                            rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesMicroSec[i], STAMTYPE_COUNTER,
+                                                 STAMVISIBILITY_USED,
+                                                 STAMUNIT_OCCURENCES,
+                                                 "Microsecond resolution runtime statistics",
+                                                 "/PDM/AsyncCompletion/File/%s/TaskRun2MicroSec-%u-%u",
+                                                 RTPathFilename(pEndpoint->pszUri),
+                                                 i*100, i*100+100-1);
+                            if (RT_FAILURE(rc))
+                                break;
+                        }
+                    }
+
+                   if (RT_SUCCESS(rc))
+                   {
+                       for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
+                       {
+                           rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesMs[i], STAMTYPE_COUNTER,
+                                                STAMVISIBILITY_USED,
+                                                STAMUNIT_OCCURENCES,
+                                                "Milliseconds resolution runtime statistics",
+                                                "/PDM/AsyncCompletion/File/%s/TaskRun3Ms-%u-%u",
+                                                RTPathFilename(pEndpoint->pszUri),
+                                                i*100, i*100+100-1);
+                           if (RT_FAILURE(rc))
+                               break;
+                       }
+                   }
+
+                   if (RT_SUCCESS(rc))
+                   {
+                       for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
+                       {
+                            rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunTimesSec[i], STAMTYPE_COUNTER,
+                                                 STAMVISIBILITY_USED,
+                                                 STAMUNIT_OCCURENCES,
+                                                 "Second resolution runtime statistics",
+                                                 "/PDM/AsyncCompletion/File/%s/TaskRun4Sec-%u-%u",
+                                                 RTPathFilename(pEndpoint->pszUri),
+                                                 i*10, i*10+10-1);
+                            if (RT_FAILURE(rc))
+                                break;
+                        }
+                    }
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatTaskRunOver100Sec, STAMTYPE_COUNTER,
+                                             STAMVISIBILITY_USED,
+                                             STAMUNIT_OCCURENCES,
+                                             "Tasks which ran more than 100sec",
+                                             "/PDM/AsyncCompletion/File/%s/TaskRunSecGreater100Sec",
+                                             RTPathFilename(pEndpoint->pszUri));
+                    }
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsPerSec, STAMTYPE_COUNTER,
+                                             STAMVISIBILITY_ALWAYS,
+                                             STAMUNIT_OCCURENCES,
+                                             "Processed I/O operations per second",
+                                             "/PDM/AsyncCompletion/File/%s/IoOpsPerSec",
+                                             RTPathFilename(pEndpoint->pszUri));
+                    }
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsStarted, STAMTYPE_COUNTER,
+                                             STAMVISIBILITY_ALWAYS,
+                                             STAMUNIT_OCCURENCES,
+                                             "Started I/O operations for this endpoint",
+                                             "/PDM/AsyncCompletion/File/%s/IoOpsStarted",
+                                             RTPathFilename(pEndpoint->pszUri));
+                    }
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = STAMR3RegisterF(pVM, &pEndpoint->StatIoOpsCompleted, STAMTYPE_COUNTER,
+                                             STAMVISIBILITY_ALWAYS,
+                                             STAMUNIT_OCCURENCES,
+                                             "Completed I/O operations for this endpoint",
+                                             "/PDM/AsyncCompletion/File/%s/IoOpsCompleted",
+                                             RTPathFilename(pEndpoint->pszUri));
+                    }
+
+                    pEndpoint->tsIntervalStartMs = RTTimeMilliTS();
+#endif
+
+                    *ppEndpoint = pEndpoint;
+
+                    LogFlowFunc((": Created endpoint for %s: rc=%Rrc\n", pszFilename, rc));
+                    return VINF_SUCCESS;
+                }
+                RTStrFree(pEndpoint->pszUri);
+            }
+            MMR3HeapFree(pEndpoint);
+        }
+    }
+    else
+    {
+        /* Endpoint found. */
+        pEndpoint->cUsers++;
+
+        *ppEndpoint = pEndpoint;
+        return VINF_SUCCESS;
+    }
+
+    LogFlowFunc((": Creation of endpoint for %s failed: rc=%Rrc\n", pszFilename, rc));
+
+    return rc;
+}
+
+VMMR3DECL(void) PDMR3AsyncCompletionEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    LogFlowFunc((": pEndpoint=%p\n", pEndpoint));
+
+    /* Sanity checks. */
+    AssertReturnVoid(VALID_PTR(pEndpoint));
+
+    pEndpoint->cUsers--;
+
+    /* If the last user closed the endpoint we will free it. */
+    if (!pEndpoint->cUsers)
+    {
+        PPDMASYNCCOMPLETIONEPCLASS pEndpointClass = pEndpoint->pEpClass;
+        pEndpointClass->pEndpointOps->pfnEpClose(pEndpoint);
+
+        /* Drop reference from the template. */
+        ASMAtomicDecU32(&pEndpoint->pTemplate->cUsed);
+
+        /* Unlink the endpoint from the list. */
+        int rc = RTCritSectEnter(&pEndpointClass->CritSect);
+        AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+        PPDMASYNCCOMPLETIONENDPOINT pEndpointNext = pEndpoint->pNext;
+        PPDMASYNCCOMPLETIONENDPOINT pEndpointPrev = pEndpoint->pPrev;
+
+        if (pEndpointPrev)
+            pEndpointPrev->pNext = pEndpointNext;
+        else
+            pEndpointClass->pEndpointsHead = pEndpointNext;
+        if (pEndpointNext)
+            pEndpointNext->pPrev = pEndpointPrev;
+
+        pEndpointClass->cEndpoints--;
+
+        rc = RTCritSectLeave(&pEndpointClass->CritSect);
+        AssertMsg(RT_SUCCESS(rc), ("Failed to enter critical section rc=%Rrc\n", rc));
+
+#ifdef VBOX_WITH_STATISTICS
+        /* Deregister the statistics part */
+        PVM pVM = pEndpointClass->pVM;
+
+        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesNs); i++)
+            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesNs[i]);
+        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMicroSec); i++)
+            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesMicroSec[i]);
+        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
+            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesMs[i]);
+        for (unsigned i = 0; i < RT_ELEMENTS(pEndpoint->StatTaskRunTimesMs); i++)
+            STAMR3Deregister(pVM, &pEndpoint->StatTaskRunTimesSec[i]);
+
+        STAMR3Deregister(pVM, &pEndpoint->StatTaskRunOver100Sec);
+        STAMR3Deregister(pVM, &pEndpoint->StatIoOpsPerSec);
+        STAMR3Deregister(pVM, &pEndpoint->StatIoOpsStarted);
+        STAMR3Deregister(pVM, &pEndpoint->StatIoOpsCompleted);
+#endif
+
+        RTStrFree(pEndpoint->pszUri);
+        MMR3HeapFree(pEndpoint);
+    }
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionEpRead(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                          PCRTSGSEG paSegments, unsigned cSegments,
+                                          size_t cbRead, void *pvUser,
+                                          PPPDMASYNCCOMPLETIONTASK ppTask)
+{
+    int rc = VINF_SUCCESS;
+
+    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(paSegments), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(ppTask), VERR_INVALID_POINTER);
+    AssertReturn(cSegments > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(cbRead > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(off >= 0, VERR_INVALID_PARAMETER);
+
+    PPDMASYNCCOMPLETIONTASK pTask;
+
+    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
+    if (!pTask)
+        return VERR_NO_MEMORY;
+
+    rc = pEndpoint->pEpClass->pEndpointOps->pfnEpRead(pTask, pEndpoint, off,
+                                                      paSegments, cSegments, cbRead);
+    if (RT_SUCCESS(rc))
+    {
+        *ppTask = pTask;
+    }
+    else
+        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionEpWrite(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                           PCRTSGSEG paSegments, unsigned cSegments,
+                                           size_t cbWrite, void *pvUser,
+                                           PPPDMASYNCCOMPLETIONTASK ppTask)
+{
+    int rc = VINF_SUCCESS;
+
+    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(paSegments), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(ppTask), VERR_INVALID_POINTER);
+    AssertReturn(cSegments > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(cbWrite > 0, VERR_INVALID_PARAMETER);
+    AssertReturn(off >= 0, VERR_INVALID_PARAMETER);
+
+    PPDMASYNCCOMPLETIONTASK pTask;
+
+    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
+    if (!pTask)
+        return VERR_NO_MEMORY;
+
+    rc = pEndpoint->pEpClass->pEndpointOps->pfnEpWrite(pTask, pEndpoint, off,
+                                                       paSegments, cSegments, cbWrite);
+    if (RT_SUCCESS(rc))
+    {
+        *ppTask = pTask;
+    }
+    else
+        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionEpFlush(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                           void *pvUser,
+                                           PPPDMASYNCCOMPLETIONTASK ppTask)
+{
+    int rc = VINF_SUCCESS;
+
+    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(ppTask), VERR_INVALID_POINTER);
+
+    PPDMASYNCCOMPLETIONTASK pTask;
+
+    pTask = pdmR3AsyncCompletionGetTask(pEndpoint, pvUser);
+    if (!pTask)
+        return VERR_NO_MEMORY;
+
+    rc = pEndpoint->pEpClass->pEndpointOps->pfnEpFlush(pTask, pEndpoint);
+    if (RT_SUCCESS(rc))
+    {
+        *ppTask = pTask;
+    }
+    else
+        pdmR3AsyncCompletionPutTask(pEndpoint, pTask);
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionEpGetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                             uint64_t *pcbSize)
+{
+    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
+    AssertReturn(VALID_PTR(pcbSize), VERR_INVALID_POINTER);
+
+    if (pEndpoint->pEpClass->pEndpointOps->pfnEpGetSize)
+        return pEndpoint->pEpClass->pEndpointOps->pfnEpGetSize(pEndpoint, pcbSize);
+    else
+        return VERR_NOT_SUPPORTED;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionEpSetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                             uint64_t cbSize)
+{
+    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
+
+    if (pEndpoint->pEpClass->pEndpointOps->pfnEpSetSize)
+        return pEndpoint->pEpClass->pEndpointOps->pfnEpSetSize(pEndpoint, cbSize);
+    else
+        return VERR_NOT_SUPPORTED;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionEpSetBwMgr(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                              const char *pcszBwMgr)
+{
+    int rc = VINF_SUCCESS;
+    AssertReturn(VALID_PTR(pEndpoint), VERR_INVALID_POINTER);
+    PPDMACBWMGR pBwMgrOld = NULL;
+    PPDMACBWMGR pBwMgrNew = NULL;
+
+    if (pcszBwMgr)
+    {
+        pBwMgrNew = pdmacBwMgrFindById(pEndpoint->pEpClass, pcszBwMgr);
+
+        if (pBwMgrNew)
+            pdmacBwMgrRef(pBwMgrNew);
+        else
+            rc = VERR_NOT_FOUND;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        pBwMgrOld = ASMAtomicXchgPtrT(&pEndpoint->pBwMgr, pBwMgrNew, PPDMACBWMGR);
+
+        if (pBwMgrOld)
+            pdmacBwMgrUnref(pBwMgrOld);
+    }
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionTaskCancel(PPDMASYNCCOMPLETIONTASK pTask)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
+VMMR3DECL(int) PDMR3AsyncCompletionBwMgrSetMaxForFile(PVM pVM, const char *pcszBwMgr, uint32_t cbMaxNew)
+{
+    int rc = VINF_SUCCESS;
+    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
+    AssertPtrReturn(pcszBwMgr, VERR_INVALID_POINTER);
+    PPDMACBWMGR pBwMgr;
+
+    PPDMASYNCCOMPLETIONEPCLASS pEpClass = pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
+
+    pBwMgr = pdmacBwMgrFindById(pEpClass, pcszBwMgr);
+    if (pBwMgr)
+    {
+        /*
+         * Set the new value for the start and max value to let the manager pick up
+         * the new limit immediately.
+         */
+        ASMAtomicXchgU32(&pBwMgr->cbTransferPerSecMax, cbMaxNew);
+        ASMAtomicXchgU32(&pBwMgr->cbTransferPerSecStart, cbMaxNew);
+    }
+    else
+        rc = VERR_NOT_FOUND;
+
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFile.cpp	(revision 35333)
@@ -0,0 +1,1225 @@
+/* $Id$ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ */
+
+/*
+ * Copyright (C) 2006-2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/dbg.h>
+#include <VBox/uvm.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/critsect.h>
+#include <iprt/env.h>
+#include <iprt/file.h>
+#include <iprt/mem.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/path.h>
+
+#if defined(RT_OS_DARWIN) || defined(RT_OS_SOLARIS) || defined(RT_OS_FREEBSD)
+# include <errno.h>
+# include <sys/ioctl.h>
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <unistd.h>
+#endif
+
+#ifdef RT_OS_WINDOWS
+# define _WIN32_WINNT 0x0500
+# include <windows.h>
+# include <winioctl.h>
+#endif
+#ifdef RT_OS_DARWIN
+# include <sys/disk.h>
+#endif /* RT_OS_DARWIN */
+#ifdef RT_OS_SOLARIS
+# include <stropts.h>
+# include <sys/dkio.h>
+# include <sys/vtoc.h>
+#endif /* RT_OS_SOLARIS */
+#ifdef RT_OS_FREEBSD
+# include <sys/disk.h>
+#endif /* RT_OS_FREEBSD */
+
+#include "PDMAsyncCompletionFileInternal.h"
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+static DECLCALLBACK(int) pdmacEpFileErrorInject(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR pArgs, unsigned cArgs, PDBGCVAR pResult);
+#endif
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+static const DBGCVARDESC g_aInjectErrorArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "direction",    "write/read." },
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "filename",     "Filename." },
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "errcode",      "IPRT error code." },
+};
+
+/** Command descriptors. */
+static const DBGCCMD g_aCmds[] =
+{
+    /* pszCmd,       cArgsMin, cArgsMax, paArgDesc,                cArgDescs,    pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
+    { "injecterror",        3, 3,        &g_aInjectErrorArgs[0],           3,           NULL,             0,     pdmacEpFileErrorInject,        "",   "Inject error into I/O subsystem." },
+};
+#endif
+
+/**
+ * Frees a task.
+ *
+ * @returns nothing.
+ * @param   pEndpoint    Pointer to the endpoint the segment was for.
+ * @param   pTask        The task to free.
+ */
+void pdmacFileTaskFree(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                       PPDMACTASKFILE pTask)
+{
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+
+    LogFlowFunc((": pEndpoint=%p pTask=%p\n", pEndpoint, pTask));
+
+    /* Try the per endpoint cache first. */
+    if (pEndpoint->cTasksCached < pEpClass->cTasksCacheMax)
+    {
+        /* Add it to the list. */
+        pEndpoint->pTasksFreeTail->pNext = pTask;
+        pEndpoint->pTasksFreeTail        = pTask;
+        ASMAtomicIncU32(&pEndpoint->cTasksCached);
+    }
+    else
+    {
+        Log(("Freeing task %p because all caches are full\n", pTask));
+        MMR3HeapFree(pTask);
+    }
+}
+
+/**
+ * Allocates a task segment
+ *
+ * @returns Pointer to the new task segment or NULL
+ * @param   pEndpoint    Pointer to the endpoint
+ */
+PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    PPDMACTASKFILE pTask = NULL;
+
+    /* Try the small per endpoint cache first. */
+    if (pEndpoint->pTasksFreeHead == pEndpoint->pTasksFreeTail)
+    {
+        /* Try the bigger endpoint class cache. */
+        PPDMASYNCCOMPLETIONEPCLASSFILE pEndpointClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+
+        /*
+         * Allocate completely new.
+         * If this fails we return NULL.
+         */
+        int rc = MMR3HeapAllocZEx(pEndpointClass->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                                  sizeof(PDMACTASKFILE),
+                                  (void **)&pTask);
+        if (RT_FAILURE(rc))
+            pTask = NULL;
+
+        LogFlow(("Allocated task %p\n", pTask));
+    }
+    else
+    {
+        /* Grab a free task from the head. */
+        AssertMsg(pEndpoint->cTasksCached > 0, ("No tasks cached but list contains more than one element\n"));
+
+        pTask = pEndpoint->pTasksFreeHead;
+        pEndpoint->pTasksFreeHead = pTask->pNext;
+        ASMAtomicDecU32(&pEndpoint->cTasksCached);
+    }
+
+    pTask->pNext = NULL;
+
+    return pTask;
+}
+
+PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    PPDMACTASKFILE pTasks = NULL;
+
+    /*
+     * Get pending tasks.
+     */
+    pTasks = ASMAtomicXchgPtrT(&pEndpoint->pTasksNewHead, NULL, PPDMACTASKFILE);
+
+    /* Reverse the list to process in FIFO order. */
+    if (pTasks)
+    {
+        PPDMACTASKFILE pTask = pTasks;
+
+        pTasks = NULL;
+
+        while (pTask)
+        {
+            PPDMACTASKFILE pCur = pTask;
+            pTask = pTask->pNext;
+            pCur->pNext = pTasks;
+            pTasks = pCur;
+        }
+    }
+
+    return pTasks;
+}
+
+static void pdmacFileAioMgrWakeup(PPDMACEPFILEMGR pAioMgr)
+{
+    bool fWokenUp = ASMAtomicXchgBool(&pAioMgr->fWokenUp, true);
+
+    if (!fWokenUp)
+    {
+        int rc = VINF_SUCCESS;
+        bool fWaitingEventSem = ASMAtomicReadBool(&pAioMgr->fWaitingEventSem);
+
+        if (fWaitingEventSem)
+            rc = RTSemEventSignal(pAioMgr->EventSem);
+
+        AssertRC(rc);
+    }
+}
+
+static int pdmacFileAioMgrWaitForBlockingEvent(PPDMACEPFILEMGR pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT enmEvent)
+{
+    int rc = VINF_SUCCESS;
+
+    ASMAtomicWriteU32((volatile uint32_t *)&pAioMgr->enmBlockingEvent, enmEvent);
+    Assert(!pAioMgr->fBlockingEventPending);
+    ASMAtomicXchgBool(&pAioMgr->fBlockingEventPending, true);
+
+    /* Wakeup the async I/O manager */
+    pdmacFileAioMgrWakeup(pAioMgr);
+
+    /* Wait for completion. */
+    rc = RTSemEventWait(pAioMgr->EventSemBlock, RT_INDEFINITE_WAIT);
+    AssertRC(rc);
+
+    ASMAtomicXchgBool(&pAioMgr->fBlockingEventPending, false);
+    ASMAtomicWriteU32((volatile uint32_t *)&pAioMgr->enmBlockingEvent, PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID);
+
+    return rc;
+}
+
+int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc;
+
+    LogFlowFunc(("pAioMgr=%#p pEndpoint=%#p{%s}\n", pAioMgr, pEndpoint, pEndpoint->Core.pszUri));
+
+    /* Update the assigned I/O manager. */
+    ASMAtomicWritePtr(&pEndpoint->pAioMgr, pAioMgr);
+
+    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint, pEndpoint);
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT);
+    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+
+static int pdmacFileAioMgrRemoveEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc;
+
+    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint, pEndpoint);
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT);
+    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+
+static int pdmacFileAioMgrCloseEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc;
+
+    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    ASMAtomicWritePtr(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint, pEndpoint);
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT);
+    ASMAtomicWriteNullPtr(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+
+static int pdmacFileAioMgrShutdown(PPDMACEPFILEMGR pAioMgr)
+{
+    int rc;
+
+    rc = RTCritSectEnter(&pAioMgr->CritSectBlockingEvent);
+    AssertRCReturn(rc, rc);
+
+    rc = pdmacFileAioMgrWaitForBlockingEvent(pAioMgr, PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN);
+
+    RTCritSectLeave(&pAioMgr->CritSectBlockingEvent);
+
+    return rc;
+}
+
+int pdmacFileEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
+{
+    PPDMACTASKFILE pNext;
+    do
+    {
+        pNext = pEndpoint->pTasksNewHead;
+        pTask->pNext = pNext;
+    } while (!ASMAtomicCmpXchgPtr(&pEndpoint->pTasksNewHead, pTask, pNext));
+
+    pdmacFileAioMgrWakeup(ASMAtomicReadPtrT(&pEndpoint->pAioMgr, PPDMACEPFILEMGR));
+
+    return VINF_SUCCESS;
+}
+
+void pdmacFileEpTaskCompleted(PPDMACTASKFILE pTask, void *pvUser, int rc)
+{
+    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pvUser;
+
+    LogFlowFunc(("pTask=%#p pvUser=%#p rc=%Rrc\n", pTask, pvUser, rc));
+
+    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+    {
+        pdmR3AsyncCompletionCompleteTask(&pTaskFile->Core, rc, true);
+    }
+    else
+    {
+        Assert((uint32_t)pTask->DataSeg.cbSeg == pTask->DataSeg.cbSeg && (int32_t)pTask->DataSeg.cbSeg >= 0);
+        uint32_t uOld = ASMAtomicSubS32(&pTaskFile->cbTransferLeft, (int32_t)pTask->DataSeg.cbSeg);
+
+        /* The first error will be returned. */
+        if (RT_FAILURE(rc))
+            ASMAtomicCmpXchgS32(&pTaskFile->rc, rc, VINF_SUCCESS);
+#ifdef VBOX_WITH_DEBUGGER
+        else
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pTaskFile->Core.pEndpoint;
+
+            /* Overwrite with injected error code. */
+            if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                rc = ASMAtomicXchgS32(&pEpFile->rcReqRead, VINF_SUCCESS);
+            else
+                rc = ASMAtomicXchgS32(&pEpFile->rcReqWrite, VINF_SUCCESS);
+
+            if (RT_FAILURE(rc))
+                ASMAtomicCmpXchgS32(&pTaskFile->rc, rc, VINF_SUCCESS);
+        }
+#endif
+
+        if (!(uOld - pTask->DataSeg.cbSeg)
+            && !ASMAtomicXchgBool(&pTaskFile->fCompleted, true))
+            pdmR3AsyncCompletionCompleteTask(&pTaskFile->Core, pTaskFile->rc, true);
+    }
+}
+
+DECLINLINE(void) pdmacFileEpTaskInit(PPDMASYNCCOMPLETIONTASK pTask, size_t cbTransfer)
+{
+    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
+
+    Assert((uint32_t)cbTransfer == cbTransfer && (int32_t)cbTransfer >= 0);
+    ASMAtomicWriteS32(&pTaskFile->cbTransferLeft, (int32_t)cbTransfer);
+    ASMAtomicWriteBool(&pTaskFile->fCompleted, false);
+    ASMAtomicWriteS32(&pTaskFile->rc, VINF_SUCCESS);
+}
+
+int pdmacFileEpTaskInitiate(PPDMASYNCCOMPLETIONTASK pTask,
+                            PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                            PCRTSGSEG paSegments, size_t cSegments,
+                            size_t cbTransfer, PDMACTASKFILETRANSFER enmTransfer)
+{
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
+    PPDMACEPFILEMGR pAioMgr = pEpFile->pAioMgr;
+
+    Assert(   (enmTransfer == PDMACTASKFILETRANSFER_READ)
+           || (enmTransfer == PDMACTASKFILETRANSFER_WRITE));
+
+    for (unsigned i = 0; i < cSegments; i++)
+    {
+        PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEpFile);
+        AssertPtr(pIoTask);
+
+        pIoTask->pEndpoint       = pEpFile;
+        pIoTask->enmTransferType = enmTransfer;
+        pIoTask->Off             = off;
+        pIoTask->DataSeg.cbSeg   = paSegments[i].cbSeg;
+        pIoTask->DataSeg.pvSeg   = paSegments[i].pvSeg;
+        pIoTask->pvUser          = pTaskFile;
+        pIoTask->pfnCompleted    = pdmacFileEpTaskCompleted;
+
+        /* Send it off to the I/O manager. */
+        pdmacFileEpAddTask(pEpFile, pIoTask);
+        off        += paSegments[i].cbSeg;
+        cbTransfer -= paSegments[i].cbSeg;
+    }
+
+    AssertMsg(!cbTransfer, ("Incomplete transfer %u bytes left\n", cbTransfer));
+
+    if (ASMAtomicReadS32(&pTaskFile->cbTransferLeft) == 0
+        && !ASMAtomicXchgBool(&pTaskFile->fCompleted, true))
+        pdmR3AsyncCompletionCompleteTask(pTask, pTaskFile->rc, false);
+    else
+        rc = VINF_AIO_TASK_PENDING;
+
+    return rc;
+}
+
+/**
+ * Creates a new async I/O manager.
+ *
+ * @returns VBox status code.
+ * @param   pEpClass    Pointer to the endpoint class data.
+ * @param   ppAioMgr    Where to store the pointer to the new async I/O manager on success.
+ * @param   enmMgrType  Wanted manager type - can be overwritten by the global override.
+ */
+int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr,
+                          PDMACEPFILEMGRTYPE enmMgrType)
+{
+    int rc = VINF_SUCCESS;
+    PPDMACEPFILEMGR pAioMgrNew;
+
+    LogFlowFunc((": Entered\n"));
+
+    rc = MMR3HeapAllocZEx(pEpClass->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION, sizeof(PDMACEPFILEMGR), (void **)&pAioMgrNew);
+    if (RT_SUCCESS(rc))
+    {
+        if (enmMgrType < pEpClass->enmMgrTypeOverride)
+            pAioMgrNew->enmMgrType = enmMgrType;
+        else
+            pAioMgrNew->enmMgrType = pEpClass->enmMgrTypeOverride;
+
+        rc = RTSemEventCreate(&pAioMgrNew->EventSem);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTSemEventCreate(&pAioMgrNew->EventSemBlock);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTCritSectInit(&pAioMgrNew->CritSectBlockingEvent);
+                if (RT_SUCCESS(rc))
+                {
+                    /* Init the rest of the manager. */
+                    if (pAioMgrNew->enmMgrType != PDMACEPFILEMGRTYPE_SIMPLE)
+                        rc = pdmacFileAioMgrNormalInit(pAioMgrNew);
+
+                    if (RT_SUCCESS(rc))
+                    {
+                        pAioMgrNew->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+
+                        rc = RTThreadCreateF(&pAioMgrNew->Thread,
+                                             pAioMgrNew->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE
+                                             ? pdmacFileAioMgrFailsafe
+                                             : pdmacFileAioMgrNormal,
+                                             pAioMgrNew,
+                                             0,
+                                             RTTHREADTYPE_IO,
+                                             0,
+                                             "AioMgr%d-%s", pEpClass->cAioMgrs,
+                                             pAioMgrNew->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE
+                                             ? "F"
+                                             : "N");
+                        if (RT_SUCCESS(rc))
+                        {
+                            /* Link it into the list. */
+                            RTCritSectEnter(&pEpClass->CritSect);
+                            pAioMgrNew->pNext = pEpClass->pAioMgrHead;
+                            if (pEpClass->pAioMgrHead)
+                                pEpClass->pAioMgrHead->pPrev = pAioMgrNew;
+                            pEpClass->pAioMgrHead = pAioMgrNew;
+                            pEpClass->cAioMgrs++;
+                            RTCritSectLeave(&pEpClass->CritSect);
+
+                            *ppAioMgr = pAioMgrNew;
+
+                            Log(("PDMAC: Successfully created new file AIO Mgr {%s}\n", RTThreadGetName(pAioMgrNew->Thread)));
+                            return VINF_SUCCESS;
+                        }
+                        pdmacFileAioMgrNormalDestroy(pAioMgrNew);
+                    }
+                    RTCritSectDelete(&pAioMgrNew->CritSectBlockingEvent);
+                }
+                RTSemEventDestroy(pAioMgrNew->EventSem);
+            }
+            RTSemEventDestroy(pAioMgrNew->EventSemBlock);
+        }
+        MMR3HeapFree(pAioMgrNew);
+    }
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+/**
+ * Destroys a async I/O manager.
+ *
+ * @returns nothing.
+ * @param   pAioMgr    The async I/O manager to destroy.
+ */
+static void pdmacFileAioMgrDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile, PPDMACEPFILEMGR pAioMgr)
+{
+    int rc = pdmacFileAioMgrShutdown(pAioMgr);
+    AssertRC(rc);
+
+    /* Unlink from the list. */
+    rc = RTCritSectEnter(&pEpClassFile->CritSect);
+    AssertRC(rc);
+
+    PPDMACEPFILEMGR pPrev = pAioMgr->pPrev;
+    PPDMACEPFILEMGR pNext = pAioMgr->pNext;
+
+    if (pPrev)
+        pPrev->pNext = pNext;
+    else
+        pEpClassFile->pAioMgrHead = pNext;
+
+    if (pNext)
+        pNext->pPrev = pPrev;
+
+    pEpClassFile->cAioMgrs--;
+    rc = RTCritSectLeave(&pEpClassFile->CritSect);
+    AssertRC(rc);
+
+    /* Free the resources. */
+    RTCritSectDelete(&pAioMgr->CritSectBlockingEvent);
+    RTSemEventDestroy(pAioMgr->EventSem);
+    if (pAioMgr->enmMgrType != PDMACEPFILEMGRTYPE_SIMPLE)
+        pdmacFileAioMgrNormalDestroy(pAioMgr);
+
+    MMR3HeapFree(pAioMgr);
+}
+
+static int pdmacFileMgrTypeFromName(const char *pszVal, PPDMACEPFILEMGRTYPE penmMgrType)
+{
+    int rc = VINF_SUCCESS;
+
+    if (!RTStrCmp(pszVal, "Simple"))
+        *penmMgrType = PDMACEPFILEMGRTYPE_SIMPLE;
+    else if (!RTStrCmp(pszVal, "Async"))
+        *penmMgrType = PDMACEPFILEMGRTYPE_ASYNC;
+    else
+        rc = VERR_CFGM_CONFIG_UNKNOWN_VALUE;
+
+    return rc;
+}
+
+static const char *pdmacFileMgrTypeToName(PDMACEPFILEMGRTYPE enmMgrType)
+{
+    if (enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
+        return "Simple";
+    if (enmMgrType == PDMACEPFILEMGRTYPE_ASYNC)
+        return "Async";
+
+    return NULL;
+}
+
+static int pdmacFileBackendTypeFromName(const char *pszVal, PPDMACFILEEPBACKEND penmBackendType)
+{
+    int rc = VINF_SUCCESS;
+
+    if (!RTStrCmp(pszVal, "Buffered"))
+        *penmBackendType = PDMACFILEEPBACKEND_BUFFERED;
+    else if (!RTStrCmp(pszVal, "NonBuffered"))
+        *penmBackendType = PDMACFILEEPBACKEND_NON_BUFFERED;
+    else
+        rc = VERR_CFGM_CONFIG_UNKNOWN_VALUE;
+
+    return rc;
+}
+
+static const char *pdmacFileBackendTypeToName(PDMACFILEEPBACKEND enmBackendType)
+{
+    if (enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
+        return "Buffered";
+    if (enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
+        return "NonBuffered";
+
+    return NULL;
+}
+
+/**
+ * Get the size of the given file.
+ * Works for block devices too.
+ *
+ * @returns VBox status code.
+ * @param   hFile    The file handle.
+ * @param   pcbSize  Where to store the size of the file on success.
+ */
+static int pdmacFileEpNativeGetSize(RTFILE hFile, uint64_t *pcbSize)
+{
+    int rc = VINF_SUCCESS;
+    uint64_t cbSize = 0;
+
+    rc = RTFileGetSize(hFile, &cbSize);
+    if (RT_SUCCESS(rc) && (cbSize != 0))
+        *pcbSize = cbSize;
+    else
+    {
+#ifdef RT_OS_WINDOWS
+        DISK_GEOMETRY DriveGeo;
+        DWORD cbDriveGeo;
+        if (DeviceIoControl((HANDLE)hFile,
+                            IOCTL_DISK_GET_DRIVE_GEOMETRY, NULL, 0,
+                            &DriveGeo, sizeof(DriveGeo), &cbDriveGeo, NULL))
+        {
+            if (   DriveGeo.MediaType == FixedMedia
+                || DriveGeo.MediaType == RemovableMedia)
+            {
+                cbSize =     DriveGeo.Cylinders.QuadPart
+                         *   DriveGeo.TracksPerCylinder
+                         *   DriveGeo.SectorsPerTrack
+                         *   DriveGeo.BytesPerSector;
+
+                GET_LENGTH_INFORMATION DiskLenInfo;
+                DWORD junk;
+                if (DeviceIoControl((HANDLE)hFile,
+                                    IOCTL_DISK_GET_LENGTH_INFO, NULL, 0,
+                                    &DiskLenInfo, sizeof(DiskLenInfo), &junk, (LPOVERLAPPED)NULL))
+                {
+                    /* IOCTL_DISK_GET_LENGTH_INFO is supported -- override cbSize. */
+                    cbSize = DiskLenInfo.Length.QuadPart;
+                }
+
+                rc = VINF_SUCCESS;
+            }
+            else
+            {
+                rc = VERR_INVALID_PARAMETER;
+            }
+        }
+        else
+        {
+            rc = RTErrConvertFromWin32(GetLastError());
+        }
+#elif defined(RT_OS_DARWIN)
+        struct stat DevStat;
+        if (!fstat(hFile, &DevStat) && S_ISBLK(DevStat.st_mode))
+        {
+            uint64_t cBlocks;
+            uint32_t cbBlock;
+            if (!ioctl(hFile, DKIOCGETBLOCKCOUNT, &cBlocks))
+            {
+                if (!ioctl(hFile, DKIOCGETBLOCKSIZE, &cbBlock))
+                    cbSize = cBlocks * cbBlock;
+                else
+                    rc = RTErrConvertFromErrno(errno);
+            }
+            else
+                rc = RTErrConvertFromErrno(errno);
+        }
+        else
+            rc = VERR_INVALID_PARAMETER;
+#elif defined(RT_OS_SOLARIS)
+        struct stat DevStat;
+        if (!fstat(hFile, &DevStat) && (   S_ISBLK(DevStat.st_mode)
+                                        || S_ISCHR(DevStat.st_mode)))
+        {
+            struct dk_minfo mediainfo;
+            if (!ioctl(hFile, DKIOCGMEDIAINFO, &mediainfo))
+                cbSize = mediainfo.dki_capacity * mediainfo.dki_lbsize;
+            else
+                rc = RTErrConvertFromErrno(errno);
+        }
+        else
+            rc = VERR_INVALID_PARAMETER;
+#elif defined(RT_OS_FREEBSD)
+        struct stat DevStat;
+        if (!fstat(hFile, &DevStat) && S_ISCHR(DevStat.st_mode))
+        {
+            off_t cbMedia = 0;
+            if (!ioctl(hFile, DIOCGMEDIASIZE, &cbMedia))
+            {
+                cbSize = cbMedia;
+            }
+            else
+                rc = RTErrConvertFromErrno(errno);
+        }
+        else
+            rc = VERR_INVALID_PARAMETER;
+#else
+        /* Could be a block device */
+        rc = RTFileSeek(hFile, 0, RTFILE_SEEK_END, &cbSize);
+#endif
+
+        if (RT_SUCCESS(rc) && (cbSize != 0))
+            *pcbSize = cbSize;
+        else if (RT_SUCCESS(rc))
+            rc = VERR_NOT_SUPPORTED;
+    }
+
+    return rc;
+}
+
+#ifdef VBOX_WITH_DEBUGGER
+/**
+ * Error inject callback.
+ */
+static DECLCALLBACK(int) pdmacEpFileErrorInject(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR pArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    bool fWrite;
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile;
+
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return DBGCCmdHlpPrintf(pCmdHlp, "error: The command requires a VM to be selected.\n");
+    if (    cArgs != 3
+        ||  pArgs[0].enmType != DBGCVAR_TYPE_STRING
+        ||  pArgs[1].enmType != DBGCVAR_TYPE_STRING
+        ||  pArgs[2].enmType != DBGCVAR_TYPE_STRING)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: parser error, invalid arguments.\n");
+
+    pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pVM->pUVM->pdm.s.apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_FILE];
+
+    /* Syntax is "read|write <filename> <status code>" */
+    if (!RTStrCmp(pArgs[0].u.pszString, "read"))
+        fWrite = false;
+    else if (!RTStrCmp(pArgs[0].u.pszString, "write"))
+        fWrite = true;
+    else
+    {
+        DBGCCmdHlpPrintf(pCmdHlp, "error: invalid transefr direction '%s'.\n", pArgs[0].u.pszString);
+        return VINF_SUCCESS;
+    }
+
+    /* Search for the matching endpoint. */
+    RTCritSectEnter(&pEpClassFile->Core.CritSect);
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpClassFile->Core.pEndpointsHead;
+
+    while (pEpFile)
+    {
+        if (!RTStrCmp(pArgs[1].u.pszString, RTPathFilename(pEpFile->Core.pszUri)))
+            break;
+        pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEpFile->Core.pNext;
+    }
+
+    if (pEpFile)
+    {
+        int rcToInject = RTStrToInt32(pArgs[2].u.pszString);
+
+        if (fWrite)
+            ASMAtomicXchgS32(&pEpFile->rcReqWrite, rcToInject);
+        else
+            ASMAtomicXchgS32(&pEpFile->rcReqRead, rcToInject);
+
+            DBGCCmdHlpPrintf(pCmdHlp, "Injected %Rrc into '%s' for %s\n",
+                             rcToInject, pArgs[1].u.pszString, pArgs[0].u.pszString);
+    }
+    else
+        DBGCCmdHlpPrintf(pCmdHlp, "No file with name '%s' found\n", NULL, pArgs[1].u.pszString);
+
+    RTCritSectLeave(&pEpClassFile->Core.CritSect);
+    return VINF_SUCCESS;
+}
+#endif
+
+static int pdmacFileInitialize(PPDMASYNCCOMPLETIONEPCLASS pClassGlobals, PCFGMNODE pCfgNode)
+{
+    int rc = VINF_SUCCESS;
+    RTFILEAIOLIMITS AioLimits; /** < Async I/O limitations. */
+
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pClassGlobals;
+
+    rc = RTFileAioGetLimits(&AioLimits);
+#ifdef DEBUG
+    if (RT_SUCCESS(rc) && RTEnvExist("VBOX_ASYNC_IO_FAILBACK"))
+        rc = VERR_ENV_VAR_NOT_FOUND;
+#endif
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("AIO: Async I/O manager not supported (rc=%Rrc). Falling back to simple manager\n",
+                rc));
+        pEpClassFile->enmMgrTypeOverride = PDMACEPFILEMGRTYPE_SIMPLE;
+        pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_BUFFERED;
+    }
+    else
+    {
+        pEpClassFile->uBitmaskAlignment   = AioLimits.cbBufferAlignment ? ~((RTR3UINTPTR)AioLimits.cbBufferAlignment - 1) : RTR3UINTPTR_MAX;
+        pEpClassFile->cReqsOutstandingMax = AioLimits.cReqsOutstandingMax;
+
+        if (pCfgNode)
+        {
+            /* Query the default manager type */
+            char *pszVal = NULL;
+            rc = CFGMR3QueryStringAllocDef(pCfgNode, "IoMgr", &pszVal, "Async");
+            AssertLogRelRCReturn(rc, rc);
+
+            rc = pdmacFileMgrTypeFromName(pszVal, &pEpClassFile->enmMgrTypeOverride);
+            MMR3HeapFree(pszVal);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            LogRel(("AIOMgr: Default manager type is \"%s\"\n", pdmacFileMgrTypeToName(pEpClassFile->enmMgrTypeOverride)));
+
+            /* Query default backend type */
+            rc = CFGMR3QueryStringAllocDef(pCfgNode, "FileBackend", &pszVal, "NonBuffered");
+            AssertLogRelRCReturn(rc, rc);
+
+            rc = pdmacFileBackendTypeFromName(pszVal, &pEpClassFile->enmEpBackendDefault);
+            MMR3HeapFree(pszVal);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            LogRel(("AIOMgr: Default file backend is \"%s\"\n", pdmacFileBackendTypeToName(pEpClassFile->enmEpBackendDefault)));
+
+#ifdef RT_OS_LINUX
+            if (   pEpClassFile->enmMgrTypeOverride == PDMACEPFILEMGRTYPE_ASYNC
+                && pEpClassFile->enmEpBackendDefault == PDMACFILEEPBACKEND_BUFFERED)
+            {
+                LogRel(("AIOMgr: Linux does not support buffered async I/O, changing to non buffered\n"));
+                pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_NON_BUFFERED;
+            }
+#endif
+        }
+        else
+        {
+            /* No configuration supplied, set defaults */
+            pEpClassFile->enmEpBackendDefault = PDMACFILEEPBACKEND_NON_BUFFERED;
+            pEpClassFile->enmMgrTypeOverride  = PDMACEPFILEMGRTYPE_ASYNC;
+        }
+    }
+
+    /* Init critical section. */
+    rc = RTCritSectInit(&pEpClassFile->CritSect);
+
+#ifdef VBOX_WITH_DEBUGGER
+    /* Install the error injection handler. */
+    if (RT_SUCCESS(rc))
+    {
+        rc = DBGCRegisterCommands(&g_aCmds[0], 1);
+        AssertRC(rc);
+    }
+#endif
+
+    return rc;
+}
+
+static void pdmacFileTerminate(PPDMASYNCCOMPLETIONEPCLASS pClassGlobals)
+{
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pClassGlobals;
+
+    /* All endpoints should be closed at this point. */
+    AssertMsg(!pEpClassFile->Core.pEndpointsHead, ("There are still endpoints left\n"));
+
+    /* Destroy all left async I/O managers. */
+    while (pEpClassFile->pAioMgrHead)
+        pdmacFileAioMgrDestroy(pEpClassFile, pEpClassFile->pAioMgrHead);
+
+    RTCritSectDelete(&pEpClassFile->CritSect);
+}
+
+static int pdmacFileEpInitialize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                 const char *pszUri, uint32_t fFlags)
+{
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;
+    PDMACEPFILEMGRTYPE enmMgrType = pEpClassFile->enmMgrTypeOverride;
+    PDMACFILEEPBACKEND enmEpBackend = pEpClassFile->enmEpBackendDefault;
+
+    AssertMsgReturn((fFlags & ~(PDMACEP_FILE_FLAGS_READ_ONLY | PDMACEP_FILE_FLAGS_DONT_LOCK)) == 0,
+                    ("PDMAsyncCompletion: Invalid flag specified\n"), VERR_INVALID_PARAMETER);
+
+    unsigned fFileFlags = RTFILE_O_OPEN;
+
+    if (fFlags & PDMACEP_FILE_FLAGS_READ_ONLY)
+        fFileFlags |= RTFILE_O_READ | RTFILE_O_DENY_NONE;
+    else
+    {
+        fFileFlags |= RTFILE_O_READWRITE;
+
+        /*
+         * Opened in read/write mode. Check whether the caller wants to
+         * avoid the lock. Return an error in case caching is enabled
+         * because this can lead to data corruption.
+         */
+        if (fFlags & PDMACEP_FILE_FLAGS_DONT_LOCK)
+            fFileFlags |= RTFILE_O_DENY_NONE;
+        else
+            fFileFlags |= RTFILE_O_DENY_WRITE;
+    }
+
+    if (enmMgrType == PDMACEPFILEMGRTYPE_ASYNC)
+        fFileFlags |= RTFILE_O_ASYNC_IO;
+
+    if (enmEpBackend == PDMACFILEEPBACKEND_NON_BUFFERED)
+    {
+        /*
+         * We only disable the cache if the size of the file is a multiple of 512.
+         * Certain hosts like Windows, Linux and Solaris require that transfer sizes
+         * are aligned to the volume sector size.
+         * If not we just make sure that the data is written to disk with RTFILE_O_WRITE_THROUGH
+         * which will trash the host cache but ensures that the host cache will not
+         * contain dirty buffers.
+         */
+        RTFILE File = NIL_RTFILE;
+
+        rc = RTFileOpen(&File, pszUri, RTFILE_O_READ | RTFILE_O_OPEN | RTFILE_O_DENY_NONE);
+        if (RT_SUCCESS(rc))
+        {
+            uint64_t cbSize;
+
+            rc = pdmacFileEpNativeGetSize(File, &cbSize);
+            Assert(RT_FAILURE(rc) || cbSize != 0);
+
+            if (RT_SUCCESS(rc) && ((cbSize % 512) == 0))
+                fFileFlags |= RTFILE_O_NO_CACHE;
+            else
+            {
+                /* Downgrade to the buffered backend */
+                enmEpBackend = PDMACFILEEPBACKEND_BUFFERED;
+
+#ifdef RT_OS_LINUX
+                fFileFlags &= ~RTFILE_O_ASYNC_IO;
+                enmMgrType   = PDMACEPFILEMGRTYPE_SIMPLE;
+#endif
+            }
+            RTFileClose(File);
+        }
+    }
+
+    /* Open with final flags. */
+    rc = RTFileOpen(&pEpFile->File, pszUri, fFileFlags);
+    if ((rc == VERR_INVALID_FUNCTION) || (rc == VERR_INVALID_PARAMETER))
+    {
+        LogRel(("pdmacFileEpInitialize: RTFileOpen %s / %08x failed with %Rrc\n",
+               pszUri, fFileFlags, rc));
+        /*
+         * Solaris doesn't support directio on ZFS so far. :-\
+         * Trying to enable it returns VERR_INVALID_FUNCTION
+         * (ENOTTY). Remove it and hope for the best.
+         * ZFS supports write throttling in case applications
+         * write more data than can be synced to the disk
+         * without blocking the whole application.
+         *
+         * On Linux we have the same problem with cifs.
+         * Have to disable async I/O here too because it requires O_DIRECT.
+         */
+        fFileFlags &= ~RTFILE_O_NO_CACHE;
+        enmEpBackend = PDMACFILEEPBACKEND_BUFFERED;
+
+#ifdef RT_OS_LINUX
+        fFileFlags &= ~RTFILE_O_ASYNC_IO;
+        enmMgrType   = PDMACEPFILEMGRTYPE_SIMPLE;
+#endif
+
+        /* Open again. */
+        rc = RTFileOpen(&pEpFile->File, pszUri, fFileFlags);
+
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("pdmacFileEpInitialize: RTFileOpen %s / %08x failed AGAIN(!) with %Rrc\n",
+                        pszUri, fFileFlags, rc));
+        }
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        pEpFile->fFlags = fFileFlags;
+
+        rc = pdmacFileEpNativeGetSize(pEpFile->File, (uint64_t *)&pEpFile->cbFile);
+        Assert(RT_FAILURE(rc) || pEpFile->cbFile != 0);
+
+        if (RT_SUCCESS(rc))
+        {
+            /* Initialize the segment cache */
+            rc = MMR3HeapAllocZEx(pEpClassFile->Core.pVM, MM_TAG_PDM_ASYNC_COMPLETION,
+                                  sizeof(PDMACTASKFILE),
+                                  (void **)&pEpFile->pTasksFreeHead);
+            if (RT_SUCCESS(rc))
+            {
+                PPDMACEPFILEMGR pAioMgr = NULL;
+
+                pEpFile->pTasksFreeTail = pEpFile->pTasksFreeHead;
+                pEpFile->cTasksCached   = 0;
+                pEpFile->enmBackendType = enmEpBackend;
+                /*
+                 * Disable async flushes on Solaris for now.
+                 * They cause weird hangs which needs more investigations.
+                 */
+#ifndef RT_OS_SOLARIS
+                pEpFile->fAsyncFlushSupported = true;
+#else
+                pEpFile->fAsyncFlushSupported = false;
+#endif
+
+                if (enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
+                {
+                    /* Simple mode. Every file has its own async I/O manager. */
+                    rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, PDMACEPFILEMGRTYPE_SIMPLE);
+                    AssertRC(rc);
+                }
+                else
+                {
+                    pAioMgr = pEpClassFile->pAioMgrHead;
+
+                    /* Check for an idling manager of the same type */
+                    while (pAioMgr)
+                    {
+                        if (pAioMgr->enmMgrType == enmMgrType)
+                            break;
+                        pAioMgr = pAioMgr->pNext;
+                    }
+
+                    if (!pAioMgr)
+                    {
+                        rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgr, enmMgrType);
+                        AssertRC(rc);
+                    }
+                }
+
+                pEpFile->AioMgr.pTreeRangesLocked = (PAVLRFOFFTREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
+                if (!pEpFile->AioMgr.pTreeRangesLocked)
+                    rc = VERR_NO_MEMORY;
+                else
+                {
+                    pEpFile->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
+
+                    /* Assign the endpoint to the thread. */
+                    rc = pdmacFileAioMgrAddEndpoint(pAioMgr, pEpFile);
+                    if (RT_FAILURE(rc))
+                    {
+                        RTMemFree(pEpFile->AioMgr.pTreeRangesLocked);
+                        MMR3HeapFree(pEpFile->pTasksFreeHead);
+                    }
+                }
+            }
+        }
+
+        if (RT_FAILURE(rc))
+            RTFileClose(pEpFile->File);
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    if (RT_SUCCESS(rc))
+    {
+        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatRead,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to read from the endpoint",
+                       "/PDM/AsyncCompletion/File/%s/Read", RTPathFilename(pEpFile->Core.pszUri));
+
+        STAMR3RegisterF(pEpClassFile->Core.pVM, &pEpFile->StatWrite,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to write to the endpoint",
+                       "/PDM/AsyncCompletion/File/%s/Write", RTPathFilename(pEpFile->Core.pszUri));
+    }
+#endif
+
+    if (RT_SUCCESS(rc))
+        LogRel(("AIOMgr: Endpoint for file '%s' (flags %08x) created successfully\n", pszUri, pEpFile->fFlags));
+
+    return rc;
+}
+
+static int pdmacFileEpRangesLockedDestroy(PAVLRFOFFNODECORE pNode, void *pvUser)
+{
+    AssertMsgFailed(("The locked ranges tree should be empty at that point\n"));
+    return VINF_SUCCESS;
+}
+
+static int pdmacFileEpClose(PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->pEpClass;
+
+    /* Make sure that all tasks finished for this endpoint. */
+    rc = pdmacFileAioMgrCloseEndpoint(pEpFile->pAioMgr, pEpFile);
+    AssertRC(rc);
+
+    /*
+     * If the async I/O manager is in failsafe mode this is the only endpoint
+     * he processes and thus can be destroyed now.
+     */
+    if (pEpFile->pAioMgr->enmMgrType == PDMACEPFILEMGRTYPE_SIMPLE)
+        pdmacFileAioMgrDestroy(pEpClassFile, pEpFile->pAioMgr);
+
+    /* Free cached tasks. */
+    PPDMACTASKFILE pTask = pEpFile->pTasksFreeHead;
+
+    while (pTask)
+    {
+        PPDMACTASKFILE pTaskFree = pTask;
+        pTask = pTask->pNext;
+        MMR3HeapFree(pTaskFree);
+    }
+
+    /* Destroy the locked ranges tree now. */
+    RTAvlrFileOffsetDestroy(pEpFile->AioMgr.pTreeRangesLocked, pdmacFileEpRangesLockedDestroy, NULL);
+
+    RTFileClose(pEpFile->File);
+
+#ifdef VBOX_WITH_STATISTICS
+    STAMR3Deregister(pEpClassFile->Core.pVM, &pEpFile->StatRead);
+    STAMR3Deregister(pEpClassFile->Core.pVM, &pEpFile->StatWrite);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+static int pdmacFileEpRead(PPDMASYNCCOMPLETIONTASK pTask,
+                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                           PCRTSGSEG paSegments, size_t cSegments,
+                           size_t cbRead)
+{
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    LogFlowFunc(("pTask=%#p pEndpoint=%#p off=%RTfoff paSegments=%#p cSegments=%zu cbRead=%zu\n",
+                 pTask, pEndpoint, off, paSegments, cSegments, cbRead));
+
+    STAM_PROFILE_ADV_START(&pEpFile->StatRead, Read);
+
+    pdmacFileEpTaskInit(pTask, cbRead);
+
+    rc = pdmacFileEpTaskInitiate(pTask, pEndpoint, off, paSegments, cSegments, cbRead,
+                                 PDMACTASKFILETRANSFER_READ);
+
+    STAM_PROFILE_ADV_STOP(&pEpFile->StatRead, Read);
+
+    return rc;
+}
+
+static int pdmacFileEpWrite(PPDMASYNCCOMPLETIONTASK pTask,
+                            PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                            PCRTSGSEG paSegments, size_t cSegments,
+                            size_t cbWrite)
+{
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    if (RT_UNLIKELY(pEpFile->fReadonly))
+        return VERR_NOT_SUPPORTED;
+
+    STAM_PROFILE_ADV_START(&pEpFile->StatWrite, Write);
+
+    pdmacFileEpTaskInit(pTask, cbWrite);
+
+    rc = pdmacFileEpTaskInitiate(pTask, pEndpoint, off, paSegments, cSegments, cbWrite,
+                                 PDMACTASKFILETRANSFER_WRITE);
+
+    STAM_PROFILE_ADV_STOP(&pEpFile->StatWrite, Write);
+
+    return rc;
+}
+
+static int pdmacFileEpFlush(PPDMASYNCCOMPLETIONTASK pTask,
+                            PPDMASYNCCOMPLETIONENDPOINT pEndpoint)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+    PPDMASYNCCOMPLETIONTASKFILE pTaskFile = (PPDMASYNCCOMPLETIONTASKFILE)pTask;
+
+    if (RT_UNLIKELY(pEpFile->fReadonly))
+        return VERR_NOT_SUPPORTED;
+
+    pdmacFileEpTaskInit(pTask, 0);
+
+    PPDMACTASKFILE pIoTask = pdmacFileTaskAlloc(pEpFile);
+    if (RT_UNLIKELY(!pIoTask))
+        return VERR_NO_MEMORY;
+
+    pIoTask->pEndpoint       = pEpFile;
+    pIoTask->enmTransferType = PDMACTASKFILETRANSFER_FLUSH;
+    pIoTask->pvUser          = pTaskFile;
+    pIoTask->pfnCompleted    = pdmacFileEpTaskCompleted;
+    pdmacFileEpAddTask(pEpFile, pIoTask);
+
+    return VINF_AIO_TASK_PENDING;
+}
+
+static int pdmacFileEpGetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t *pcbSize)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    *pcbSize = ASMAtomicReadU64(&pEpFile->cbFile);
+
+    return VINF_SUCCESS;
+}
+
+static int pdmacFileEpSetSize(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint64_t cbSize)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpFile = (PPDMASYNCCOMPLETIONENDPOINTFILE)pEndpoint;
+
+    ASMAtomicWriteU64(&pEpFile->cbFile, cbSize);
+    return RTFileSetSize(pEpFile->File, cbSize);
+}
+
+const PDMASYNCCOMPLETIONEPCLASSOPS g_PDMAsyncCompletionEndpointClassFile =
+{
+    /* u32Version */
+    PDMAC_EPCLASS_OPS_VERSION,
+    /* pcszName */
+    "File",
+    /* enmClassType */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_FILE,
+    /* cbEndpointClassGlobal */
+    sizeof(PDMASYNCCOMPLETIONEPCLASSFILE),
+    /* cbEndpoint */
+    sizeof(PDMASYNCCOMPLETIONENDPOINTFILE),
+    /* cbTask */
+    sizeof(PDMASYNCCOMPLETIONTASKFILE),
+    /* pfnInitialize */
+    pdmacFileInitialize,
+    /* pfnTerminate */
+    pdmacFileTerminate,
+    /* pfnEpInitialize. */
+    pdmacFileEpInitialize,
+    /* pfnEpClose */
+    pdmacFileEpClose,
+    /* pfnEpRead */
+    pdmacFileEpRead,
+    /* pfnEpWrite */
+    pdmacFileEpWrite,
+    /* pfnEpFlush */
+    pdmacFileEpFlush,
+    /* pfnEpGetSize */
+    pdmacFileEpGetSize,
+    /* pfnEpSetSize */
+    pdmacFileEpSetSize,
+    /* u32VersionEnd */
+    PDMAC_EPCLASS_OPS_VERSION
+};
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileFailsafe.cpp	(revision 35333)
@@ -0,0 +1,223 @@
+/* $Id$ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ * Simple File I/O manager.
+ */
+
+/*
+ * Copyright (C) 2006-2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+
+#include "PDMAsyncCompletionFileInternal.h"
+
+
+static int pdmacFileAioMgrFailsafeProcessEndpointTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                          PPDMACTASKFILE pTasks)
+{
+    int rc = VINF_SUCCESS;
+
+    while (pTasks)
+    {
+        PPDMACTASKFILE pCurr = pTasks;
+
+        pTasks = pTasks->pNext;
+
+        switch (pCurr->enmTransferType)
+        {
+            case PDMACTASKFILETRANSFER_FLUSH:
+            {
+                rc = RTFileFlush(pEndpoint->File);
+                break;
+            }
+            case PDMACTASKFILETRANSFER_READ:
+            case PDMACTASKFILETRANSFER_WRITE:
+            {
+                if (pCurr->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                {
+                    rc = RTFileReadAt(pEndpoint->File, pCurr->Off,
+                                      pCurr->DataSeg.pvSeg,
+                                      pCurr->DataSeg.cbSeg,
+                                      NULL);
+                }
+                else
+                {
+                    if (RT_UNLIKELY((uint64_t)pCurr->Off + pCurr->DataSeg.cbSeg > pEndpoint->cbFile))
+                    {
+                        ASMAtomicWriteU64(&pEndpoint->cbFile, pCurr->Off + pCurr->DataSeg.cbSeg);
+                        RTFileSetSize(pEndpoint->File, pCurr->Off + pCurr->DataSeg.cbSeg);
+                    }
+
+                    rc = RTFileWriteAt(pEndpoint->File, pCurr->Off,
+                                       pCurr->DataSeg.pvSeg,
+                                       pCurr->DataSeg.cbSeg,
+                                       NULL);
+                }
+
+                break;
+            }
+            default:
+                AssertMsgFailed(("Invalid transfer type %d\n", pTasks->enmTransferType));
+        }
+
+        pCurr->pfnCompleted(pCurr, pCurr->pvUser, rc);
+        pdmacFileTaskFree(pEndpoint, pCurr);
+    }
+
+    return VINF_SUCCESS;
+}
+
+static int pdmacFileAioMgrFailsafeProcessEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc = VINF_SUCCESS;
+    PPDMACTASKFILE pTasks = pEndpoint->AioMgr.pReqsPendingHead;
+
+    pEndpoint->AioMgr.pReqsPendingHead = NULL;
+    pEndpoint->AioMgr.pReqsPendingTail = NULL;
+
+    /* Process the request pending list first in case the endpoint was migrated due to an error. */
+    if (pTasks)
+        rc = pdmacFileAioMgrFailsafeProcessEndpointTaskList(pEndpoint, pTasks);
+
+    if (RT_SUCCESS(rc))
+    {
+        pTasks = pdmacFileEpGetNewTasks(pEndpoint);
+
+        if (pTasks)
+            rc = pdmacFileAioMgrFailsafeProcessEndpointTaskList(pEndpoint, pTasks);
+    }
+
+    return rc;
+}
+
+/**
+ * A fallback method in case something goes wrong with the normal
+ * I/O manager.
+ */
+int pdmacFileAioMgrFailsafe(RTTHREAD ThreadSelf, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
+
+    while (   (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
+           || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING))
+    {
+        ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
+        if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
+            rc = RTSemEventWait(pAioMgr->EventSem, RT_INDEFINITE_WAIT);
+        ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
+        AssertRC(rc);
+
+        LogFlow(("Got woken up\n"));
+        ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
+
+        /* Process endpoint events first. */
+        PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
+        while (pEndpoint)
+        {
+            rc = pdmacFileAioMgrFailsafeProcessEndpoint(pEndpoint);
+            AssertRC(rc);
+            pEndpoint = pEndpoint->AioMgr.pEndpointNext;
+        }
+
+        /* Now check for an external blocking event. */
+        if (pAioMgr->fBlockingEventPending)
+        {
+            switch (pAioMgr->enmBlockingEvent)
+            {
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = pAioMgr->BlockingEventData.AddEndpoint.pEndpoint;
+                    AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
+
+                    pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
+
+                    pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
+                    pEndpointNew->AioMgr.pEndpointPrev = NULL;
+                    if (pAioMgr->pEndpointsHead)
+                        pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
+                    pAioMgr->pEndpointsHead = pEndpointNew;
+
+                    pAioMgr->cEndpoints++;
+
+                    /*
+                     * Process the task list the first time. There might be pending requests
+                     * if the endpoint was migrated from another endpoint.
+                     */
+                    rc = pdmacFileAioMgrFailsafeProcessEndpoint(pEndpointNew);
+                    AssertRC(rc);
+                    break;
+                }
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint;
+                    AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
+
+                    pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
+
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEndpointRemove->AioMgr.pEndpointPrev;
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEndpointRemove->AioMgr.pEndpointNext;
+
+                    if (pPrev)
+                        pPrev->AioMgr.pEndpointNext = pNext;
+                    else
+                        pAioMgr->pEndpointsHead = pNext;
+
+                    if (pNext)
+                        pNext->AioMgr.pEndpointPrev = pPrev;
+
+                    pAioMgr->cEndpoints--;
+                    break;
+                }
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint;
+                    AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to Close\n"));
+
+                    pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
+
+                    /* Make sure all tasks finished. */
+                    rc = pdmacFileAioMgrFailsafeProcessEndpoint(pEndpointClose);
+                    AssertRC(rc);
+                    break;
+                }
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
+                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
+                    break;
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
+                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
+                    break;
+                case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
+                    pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+                    break;
+                default:
+                    AssertMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
+            }
+
+            ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
+            pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
+
+            /* Release the waiting thread. */
+            rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+            AssertRC(rc);
+        }
+    }
+
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMAsyncCompletionFileNormal.cpp	(revision 35333)
@@ -0,0 +1,1705 @@
+/* $Id$ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ * Async File I/O manager.
+ */
+
+/*
+ * Copyright (C) 2006-2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+#define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
+#include <iprt/types.h>
+#include <iprt/asm.h>
+#include <iprt/file.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+
+#include "PDMAsyncCompletionFileInternal.h"
+
+/** The update period for the I/O load statistics in ms. */
+#define PDMACEPFILEMGR_LOAD_UPDATE_PERIOD 1000
+/** Maximum number of requests a manager will handle. */
+#define PDMACEPFILEMGR_REQS_STEP 512
+
+/*******************************************************************************
+*   Internal functions                                                         *
+*******************************************************************************/
+static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
+                                                PPDMACEPFILEMGR pAioMgr,
+                                                PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
+                                                         PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                         PPDMACFILERANGELOCK pRangeLock);
+
+static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
+                                               int rc, size_t cbTransfered);
+
+int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr)
+{
+    int rc = VINF_SUCCESS;
+
+    pAioMgr->cRequestsActiveMax = PDMACEPFILEMGR_REQS_STEP;
+
+    rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, RTFILEAIO_UNLIMITED_REQS);
+    if (rc == VERR_OUT_OF_RANGE)
+        rc = RTFileAioCtxCreate(&pAioMgr->hAioCtx, pAioMgr->cRequestsActiveMax);
+
+    if (RT_SUCCESS(rc))
+    {
+        /* Initialize request handle array. */
+        pAioMgr->iFreeEntry       = 0;
+        pAioMgr->cReqEntries      = pAioMgr->cRequestsActiveMax;
+        pAioMgr->pahReqsFree      = (RTFILEAIOREQ *)RTMemAllocZ(pAioMgr->cReqEntries * sizeof(RTFILEAIOREQ));
+        pAioMgr->msBwLimitExpired = RT_INDEFINITE_WAIT;
+
+        if (pAioMgr->pahReqsFree)
+        {
+            /* Create the range lock memcache. */
+            rc = RTMemCacheCreate(&pAioMgr->hMemCacheRangeLocks, sizeof(PDMACFILERANGELOCK),
+                                  0, UINT32_MAX, NULL, NULL, NULL, 0);
+            if (RT_SUCCESS(rc))
+                return VINF_SUCCESS;
+
+            RTMemFree(pAioMgr->pahReqsFree);
+        }
+        else
+        {
+            RTFileAioCtxDestroy(pAioMgr->hAioCtx);
+            rc = VERR_NO_MEMORY;
+        }
+    }
+
+    return rc;
+}
+
+void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr)
+{
+    RTFileAioCtxDestroy(pAioMgr->hAioCtx);
+
+    while (pAioMgr->iFreeEntry > 0)
+    {
+        pAioMgr->iFreeEntry--;
+        Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] != NIL_RTFILEAIOREQ);
+        RTFileAioReqDestroy(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry]);
+    }
+
+    RTMemFree(pAioMgr->pahReqsFree);
+    RTMemCacheDestroy(pAioMgr->hMemCacheRangeLocks);
+}
+
+/**
+ * Sorts the endpoint list with insertion sort.
+ */
+static void pdmacFileAioMgrNormalEndpointsSortByLoad(PPDMACEPFILEMGR pAioMgr)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEpPrev, pEpCurr, pEpNextToSort;
+
+    pEpPrev = pAioMgr->pEndpointsHead;
+    pEpCurr = pEpPrev->AioMgr.pEndpointNext;
+
+    while (pEpCurr)
+    {
+        /* Remember the next element to sort because the list might change. */
+        pEpNextToSort = pEpCurr->AioMgr.pEndpointNext;
+
+        /* Unlink the current element from the list. */
+        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
+        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
+
+        if (pPrev)
+            pPrev->AioMgr.pEndpointNext = pNext;
+        else
+            pAioMgr->pEndpointsHead = pNext;
+
+        if (pNext)
+            pNext->AioMgr.pEndpointPrev = pPrev;
+
+        /* Go back until we reached the place to insert the current endpoint into. */
+        while (pEpPrev && (pEpPrev->AioMgr.cReqsPerSec < pEpCurr->AioMgr.cReqsPerSec))
+            pEpPrev = pEpPrev->AioMgr.pEndpointPrev;
+
+        /* Link the endpoint into the list. */
+        if (pEpPrev)
+            pNext = pEpPrev->AioMgr.pEndpointNext;
+        else
+            pNext = pAioMgr->pEndpointsHead;
+
+        pEpCurr->AioMgr.pEndpointNext = pNext;
+        pEpCurr->AioMgr.pEndpointPrev = pEpPrev;
+
+        if (pNext)
+            pNext->AioMgr.pEndpointPrev = pEpCurr;
+
+        if (pEpPrev)
+            pEpPrev->AioMgr.pEndpointNext = pEpCurr;
+        else
+            pAioMgr->pEndpointsHead = pEpCurr;
+
+        pEpCurr = pEpNextToSort;
+    }
+
+#ifdef DEBUG
+    /* Validate sorting algorithm */
+    unsigned cEndpoints = 0;
+    pEpCurr = pAioMgr->pEndpointsHead;
+
+    AssertMsg(pEpCurr, ("No endpoint in the list?\n"));
+    AssertMsg(!pEpCurr->AioMgr.pEndpointPrev, ("First element in the list points to previous element\n"));
+
+    while (pEpCurr)
+    {
+        cEndpoints++;
+
+        PPDMASYNCCOMPLETIONENDPOINTFILE pNext = pEpCurr->AioMgr.pEndpointNext;
+        PPDMASYNCCOMPLETIONENDPOINTFILE pPrev = pEpCurr->AioMgr.pEndpointPrev;
+
+        Assert(!pNext || pNext->AioMgr.cReqsPerSec <= pEpCurr->AioMgr.cReqsPerSec);
+        Assert(!pPrev || pPrev->AioMgr.cReqsPerSec >= pEpCurr->AioMgr.cReqsPerSec);
+
+        pEpCurr = pNext;
+    }
+
+    AssertMsg(cEndpoints == pAioMgr->cEndpoints, ("Endpoints lost during sort!\n"));
+
+#endif
+}
+
+/**
+ * Removes an endpoint from the currently assigned manager.
+ *
+ * @returns TRUE if there are still requests pending on the current manager for this endpoint.
+ *          FALSE otherwise.
+ * @param   pEndpointRemove    The endpoint to remove.
+ */
+static bool pdmacFileAioMgrNormalRemoveEndpoint(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove)
+{
+    PPDMASYNCCOMPLETIONENDPOINTFILE pPrev   = pEndpointRemove->AioMgr.pEndpointPrev;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pNext   = pEndpointRemove->AioMgr.pEndpointNext;
+    PPDMACEPFILEMGR                 pAioMgr = pEndpointRemove->pAioMgr;
+
+    pAioMgr->cEndpoints--;
+
+    if (pPrev)
+        pPrev->AioMgr.pEndpointNext = pNext;
+    else
+        pAioMgr->pEndpointsHead = pNext;
+
+    if (pNext)
+        pNext->AioMgr.pEndpointPrev = pPrev;
+
+    /* Make sure that there is no request pending on this manager for the endpoint. */
+    if (!pEndpointRemove->AioMgr.cRequestsActive)
+    {
+        Assert(!pEndpointRemove->pFlushReq);
+
+        /* Reopen the file so that the new endpoint can re-associate with the file */
+        RTFileClose(pEndpointRemove->File);
+        int rc = RTFileOpen(&pEndpointRemove->File, pEndpointRemove->Core.pszUri, pEndpointRemove->fFlags);
+        AssertRC(rc);
+        return false;
+    }
+
+    return true;
+}
+
+static bool pdmacFileAioMgrNormalIsBalancePossible(PPDMACEPFILEMGR pAioMgr)
+{
+    /* Balancing doesn't make sense with only one endpoint. */
+    if (pAioMgr->cEndpoints == 1)
+        return false;
+
+    /* Doesn't make sens to move endpoints if only one produces the whole load */
+    unsigned cEndpointsWithLoad = 0;
+
+    PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
+
+    while (pCurr)
+    {
+        if (pCurr->AioMgr.cReqsPerSec)
+            cEndpointsWithLoad++;
+
+        pCurr = pCurr->AioMgr.pEndpointNext;
+    }
+
+    return (cEndpointsWithLoad > 1);
+}
+
+/**
+ * Creates a new I/O manager and spreads the I/O load of the endpoints
+ * between the given I/O manager and the new one.
+ *
+ * @returns nothing.
+ * @param   pAioMgr    The I/O manager with high I/O load.
+ */
+static void pdmacFileAioMgrNormalBalanceLoad(PPDMACEPFILEMGR pAioMgr)
+{
+    PPDMACEPFILEMGR pAioMgrNew = NULL;
+    int rc = VINF_SUCCESS;
+
+    /*
+     * Check if balancing would improve the situation.
+     */
+    if (pdmacFileAioMgrNormalIsBalancePossible(pAioMgr))
+    {
+        PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
+
+        rc = pdmacFileAioMgrCreate(pEpClassFile, &pAioMgrNew, PDMACEPFILEMGRTYPE_ASYNC);
+        if (RT_SUCCESS(rc))
+        {
+            /* We will sort the list by request count per second. */
+            pdmacFileAioMgrNormalEndpointsSortByLoad(pAioMgr);
+
+            /* Now move some endpoints to the new manager. */
+            unsigned cReqsHere  = pAioMgr->pEndpointsHead->AioMgr.cReqsPerSec;
+            unsigned cReqsOther = 0;
+            PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead->AioMgr.pEndpointNext;
+
+            while (pCurr)
+            {
+                if (cReqsHere <= cReqsOther)
+                {
+                    /*
+                     * The other manager has more requests to handle now.
+                     * We will keep the current endpoint.
+                     */
+                    Log(("Keeping endpoint %#p{%s} with %u reqs/s\n", pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
+                    cReqsHere += pCurr->AioMgr.cReqsPerSec;
+                    pCurr = pCurr->AioMgr.pEndpointNext;
+                }
+                else
+                {
+                    /* Move to other endpoint. */
+                    Log(("Moving endpoint %#p{%s} with %u reqs/s to other manager\n", pCurr, pCurr->Core.pszUri, pCurr->AioMgr.cReqsPerSec));
+                    cReqsOther += pCurr->AioMgr.cReqsPerSec;
+
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pMove = pCurr;
+
+                    pCurr = pCurr->AioMgr.pEndpointNext;
+
+                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pMove);
+
+                    if (fReqsPending)
+                    {
+                        pMove->enmState          = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
+                        pMove->AioMgr.fMoving    = true;
+                        pMove->AioMgr.pAioMgrDst = pAioMgrNew;
+                    }
+                    else
+                    {
+                        pMove->AioMgr.fMoving    = false;
+                        pMove->AioMgr.pAioMgrDst = NULL;
+                        pdmacFileAioMgrAddEndpoint(pAioMgrNew, pMove);
+                    }
+                }
+            }
+        }
+        else
+        {
+            /* Don't process further but leave a log entry about reduced performance. */
+            LogRel(("AIOMgr: Could not create new I/O manager (rc=%Rrc). Expect reduced performance\n", rc));
+        }
+    }
+    else
+        Log(("AIOMgr: Load balancing would not improve anything\n"));
+}
+
+/**
+ * Increase the maximum number of active requests for the given I/O manager.
+ *
+ * @returns VBox status code.
+ * @param   pAioMgr    The I/O manager to grow.
+ */
+static int pdmacFileAioMgrNormalGrow(PPDMACEPFILEMGR pAioMgr)
+{
+    int rc = VINF_SUCCESS;
+    RTFILEAIOCTX hAioCtxNew = NIL_RTFILEAIOCTX;
+
+    LogFlowFunc(("pAioMgr=%#p\n", pAioMgr));
+
+    AssertMsg(    pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING
+              && !pAioMgr->cRequestsActive,
+              ("Invalid state of the I/O manager\n"));
+
+#ifdef RT_OS_WINDOWS
+    /*
+     * Reopen the files of all assigned endpoints first so we can assign them to the new
+     * I/O context.
+     */
+    PPDMASYNCCOMPLETIONENDPOINTFILE pCurr = pAioMgr->pEndpointsHead;
+
+    while (pCurr)
+    {
+        RTFileClose(pCurr->File);
+        rc = RTFileOpen(&pCurr->File, pCurr->Core.pszUri, pCurr->fFlags);
+        AssertRC(rc);
+
+        pCurr = pCurr->AioMgr.pEndpointNext;
+    }
+#endif
+
+    /* Create the new bigger context. */
+    pAioMgr->cRequestsActiveMax += PDMACEPFILEMGR_REQS_STEP;
+
+    rc = RTFileAioCtxCreate(&hAioCtxNew, RTFILEAIO_UNLIMITED_REQS);
+    if (rc == VERR_OUT_OF_RANGE)
+        rc = RTFileAioCtxCreate(&hAioCtxNew, pAioMgr->cRequestsActiveMax);
+
+    if (RT_SUCCESS(rc))
+    {
+        /* Close the old context. */
+        rc = RTFileAioCtxDestroy(pAioMgr->hAioCtx);
+        AssertRC(rc);
+
+        pAioMgr->hAioCtx = hAioCtxNew;
+
+        /* Create a new I/O task handle array */
+        uint32_t cReqEntriesNew = pAioMgr->cRequestsActiveMax + 1;
+        RTFILEAIOREQ *pahReqNew = (RTFILEAIOREQ *)RTMemAllocZ(cReqEntriesNew * sizeof(RTFILEAIOREQ));
+
+        if (pahReqNew)
+        {
+            /* Copy the cached request handles. */
+            for (uint32_t iReq = 0; iReq < pAioMgr->cReqEntries; iReq++)
+                pahReqNew[iReq] = pAioMgr->pahReqsFree[iReq];
+
+            RTMemFree(pAioMgr->pahReqsFree);
+            pAioMgr->pahReqsFree = pahReqNew;
+            pAioMgr->cReqEntries = cReqEntriesNew;
+            LogFlowFunc(("I/O manager increased to handle a maximum of %u requests\n",
+                         pAioMgr->cRequestsActiveMax));
+        }
+        else
+            rc = VERR_NO_MEMORY;
+    }
+
+#ifdef RT_OS_WINDOWS
+    /* Assign the file to the new context. */
+    pCurr = pAioMgr->pEndpointsHead;
+
+    while (pCurr)
+    {
+        rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pCurr->File);
+        AssertRC(rc);
+
+        pCurr = pCurr->AioMgr.pEndpointNext;
+    }
+#endif
+
+    if (RT_FAILURE(rc))
+    {
+        LogFlow(("Increasing size of the I/O manager failed with rc=%Rrc\n", rc));
+        pAioMgr->cRequestsActiveMax -= PDMACEPFILEMGR_REQS_STEP;
+    }
+
+    pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+    LogFlowFunc(("returns rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+/**
+ * Checks if a given status code is fatal.
+ * Non fatal errors can be fixed by migrating the endpoint to a
+ * failsafe manager.
+ *
+ * @returns true If the error is fatal and migrating to a failsafe manager doesn't help
+ *          false If the error can be fixed by a migration. (image on NFS disk for example)
+ * @param   rcReq    The status code to check.
+ */
+DECLINLINE(bool) pdmacFileAioMgrNormalRcIsFatal(int rcReq)
+{
+    return    rcReq == VERR_DEV_IO_ERROR
+           || rcReq == VERR_FILE_IO_ERROR
+           || rcReq == VERR_DISK_IO_ERROR
+           || rcReq == VERR_DISK_FULL
+           || rcReq == VERR_FILE_TOO_BIG;
+}
+
+/**
+ * Error handler which will create the failsafe managers and destroy the failed I/O manager.
+ *
+ * @returns VBox status code
+ * @param   pAioMgr    The I/O manager the error occurred on.
+ * @param   rc         The error code.
+ */
+static int pdmacFileAioMgrNormalErrorHandler(PPDMACEPFILEMGR pAioMgr, int rc, RT_SRC_POS_DECL)
+{
+    LogRel(("AIOMgr: I/O manager %#p encountered a critical error (rc=%Rrc) during operation. Falling back to failsafe mode. Expect reduced performance\n",
+            pAioMgr, rc));
+    LogRel(("AIOMgr: Error happened in %s:(%u){%s}\n", RT_SRC_POS_ARGS));
+    LogRel(("AIOMgr: Please contact the product vendor\n"));
+
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pAioMgr->pEndpointsHead->Core.pEpClass;
+
+    pAioMgr->enmState = PDMACEPFILEMGRSTATE_FAULT;
+    ASMAtomicWriteU32((volatile uint32_t *)&pEpClassFile->enmMgrTypeOverride, PDMACEPFILEMGRTYPE_SIMPLE);
+
+    AssertMsgFailed(("Implement\n"));
+    return VINF_SUCCESS;
+}
+
+/**
+ * Put a list of tasks in the pending request list of an endpoint.
+ */
+DECLINLINE(void) pdmacFileAioMgrEpAddTaskList(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTaskHead)
+{
+    /* Add the rest of the tasks to the pending list */
+    if (!pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        Assert(!pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingHead = pTaskHead;
+    }
+    else
+    {
+        Assert(pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingTail->pNext = pTaskHead;
+    }
+
+    /* Update the tail. */
+    while (pTaskHead->pNext)
+        pTaskHead = pTaskHead->pNext;
+
+    pEndpoint->AioMgr.pReqsPendingTail = pTaskHead;
+    pTaskHead->pNext = NULL;
+}
+
+/**
+ * Put one task in the pending request list of an endpoint.
+ */
+DECLINLINE(void) pdmacFileAioMgrEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask)
+{
+    /* Add the rest of the tasks to the pending list */
+    if (!pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        Assert(!pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingHead = pTask;
+    }
+    else
+    {
+        Assert(pEndpoint->AioMgr.pReqsPendingTail);
+        pEndpoint->AioMgr.pReqsPendingTail->pNext = pTask;
+    }
+
+    pEndpoint->AioMgr.pReqsPendingTail = pTask;
+    pTask->pNext = NULL;
+}
+
+/**
+ * Allocates a async I/O request.
+ *
+ * @returns Handle to the request.
+ * @param   pAioMgr    The I/O manager.
+ */
+static RTFILEAIOREQ pdmacFileAioMgrNormalRequestAlloc(PPDMACEPFILEMGR pAioMgr)
+{
+    RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
+
+    /* Get a request handle. */
+    if (pAioMgr->iFreeEntry > 0)
+    {
+        pAioMgr->iFreeEntry--;
+        hReq = pAioMgr->pahReqsFree[pAioMgr->iFreeEntry];
+        pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = NIL_RTFILEAIOREQ;
+        Assert(hReq != NIL_RTFILEAIOREQ);
+    }
+    else
+    {
+        int rc = RTFileAioReqCreate(&hReq);
+        AssertRC(rc);
+    }
+
+    return hReq;
+}
+
+/**
+ * Frees a async I/O request handle.
+ *
+ * @returns nothing.
+ * @param   pAioMgr    The I/O manager.
+ * @param   hReq       The I/O request handle to free.
+ */
+static void pdmacFileAioMgrNormalRequestFree(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
+{
+    Assert(pAioMgr->iFreeEntry < pAioMgr->cReqEntries);
+    Assert(pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] == NIL_RTFILEAIOREQ);
+
+    pAioMgr->pahReqsFree[pAioMgr->iFreeEntry] = hReq;
+    pAioMgr->iFreeEntry++;
+}
+
+/**
+ * Wrapper around RTFIleAioCtxSubmit() which is also doing error handling.
+ */
+static int pdmacFileAioMgrNormalReqsEnqueue(PPDMACEPFILEMGR pAioMgr,
+                                            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                            PRTFILEAIOREQ pahReqs, unsigned cReqs)
+{
+    int rc;
+
+    pAioMgr->cRequestsActive += cReqs;
+    pEndpoint->AioMgr.cRequestsActive += cReqs;
+
+    LogFlow(("Enqueuing %d requests. I/O manager has a total of %d active requests now\n", cReqs, pAioMgr->cRequestsActive));
+    LogFlow(("Endpoint has a total of %d active requests now\n", pEndpoint->AioMgr.cRequestsActive));
+
+    rc = RTFileAioCtxSubmit(pAioMgr->hAioCtx, pahReqs, cReqs);
+    if (RT_FAILURE(rc))
+    {
+        if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
+        {
+            PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+
+            /* Append any not submitted task to the waiting list. */
+            for (size_t i = 0; i < cReqs; i++)
+            {
+                int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
+
+                if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
+                {
+                    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
+
+                    Assert(pTask->hReq == pahReqs[i]);
+                    pdmacFileAioMgrEpAddTask(pEndpoint, pTask);
+                    pAioMgr->cRequestsActive--;
+                    pEndpoint->AioMgr.cRequestsActive--;
+
+                    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+                    {
+                        /* Clear the pending flush */
+                        Assert(pEndpoint->pFlushReq == pTask);
+                        pEndpoint->pFlushReq = NULL;
+                    }
+                }
+            }
+
+            pAioMgr->cRequestsActiveMax = pAioMgr->cRequestsActive;
+
+            /* Print an entry in the release log */
+            if (RT_UNLIKELY(!pEpClass->fOutOfResourcesWarningPrinted))
+            {
+                pEpClass->fOutOfResourcesWarningPrinted = true;
+                LogRel(("AIOMgr: Host limits number of active IO requests to %u. Expect a performance impact.\n",
+                        pAioMgr->cRequestsActive));
+            }
+
+            LogFlow(("Removed requests. I/O manager has a total of %u active requests now\n", pAioMgr->cRequestsActive));
+            LogFlow(("Endpoint has a total of %u active requests now\n", pEndpoint->AioMgr.cRequestsActive));
+            rc = VINF_SUCCESS;
+        }
+        else /* Another kind of error happened (full disk, ...) */
+        {
+            /* An error happened. Find out which one caused the error and resubmit all other tasks. */
+            for (size_t i = 0; i < cReqs; i++)
+            {
+                int rcReq = RTFileAioReqGetRC(pahReqs[i], NULL);
+
+                if (rcReq == VERR_FILE_AIO_NOT_SUBMITTED)
+                {
+                    /* We call ourself again to do any error handling which might come up now. */
+                    rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &pahReqs[i], 1);
+                    AssertRC(rc);
+                }
+                else if (rcReq != VERR_FILE_AIO_IN_PROGRESS)
+                {
+                    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(pahReqs[i]);
+
+                    pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, pahReqs[i], rcReq, 0);
+                }
+            }
+
+
+            if (    pEndpoint->pFlushReq
+                && !pAioMgr->cRequestsActive
+                && !pEndpoint->fAsyncFlushSupported)
+            {
+                /*
+                 * Complete a pending flush if we don't have requests enqueued and the host doesn't support
+                 * the async flush API.
+                 * Happens only if this we just noticed that this is not supported
+                 * and the only active request was a flush.
+                 */
+                PPDMACTASKFILE pFlush = pEndpoint->pFlushReq;
+                pEndpoint->pFlushReq = NULL;
+                pFlush->pfnCompleted(pFlush, pFlush->pvUser, VINF_SUCCESS);
+                pdmacFileTaskFree(pEndpoint, pFlush);
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+static bool pdmacFileAioMgrNormalIsRangeLocked(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                               RTFOFF offStart, size_t cbRange,
+                                               PPDMACTASKFILE pTask)
+{
+    PPDMACFILERANGELOCK pRangeLock = NULL; /** < Range lock */
+
+    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
+              || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ,
+                 ("Invalid task type %d\n", pTask->enmTransferType));
+
+    pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetRangeGet(pEndpoint->AioMgr.pTreeRangesLocked, offStart);
+    if (!pRangeLock)
+    {
+        pRangeLock = (PPDMACFILERANGELOCK)RTAvlrFileOffsetGetBestFit(pEndpoint->AioMgr.pTreeRangesLocked, offStart, true);
+        /* Check if we intersect with the range. */
+        if (   !pRangeLock
+            || !(   (pRangeLock->Core.Key) <= (offStart + (RTFOFF)cbRange - 1)
+                && (pRangeLock->Core.KeyLast) >= offStart))
+        {
+            pRangeLock = NULL; /* False alarm */
+        }
+    }
+
+    /* Check whether we have one of the situations explained below */
+    if (   pRangeLock
+#if 0 /** @todo: later. For now we will just block all requests if they interfere */
+        && (   (pRangeLock->fReadLock && pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+            || (!pRangeLock->fReadLock)
+#endif
+        )
+    {
+        /* Add to the list. */
+        pTask->pNext = NULL;
+
+        if (!pRangeLock->pWaitingTasksHead)
+        {
+            Assert(!pRangeLock->pWaitingTasksTail);
+            pRangeLock->pWaitingTasksHead = pTask;
+            pRangeLock->pWaitingTasksTail = pTask;
+        }
+        else
+        {
+            AssertPtr(pRangeLock->pWaitingTasksTail);
+            pRangeLock->pWaitingTasksTail->pNext = pTask;
+            pRangeLock->pWaitingTasksTail = pTask;
+        }
+        return true;
+    }
+
+    return false;
+}
+
+static int pdmacFileAioMgrNormalRangeLock(PPDMACEPFILEMGR pAioMgr,
+                                          PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                          RTFOFF offStart, size_t cbRange,
+                                          PPDMACTASKFILE pTask)
+{
+    AssertMsg(!pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbRange, pTask),
+              ("Range is already locked offStart=%RTfoff cbRange=%u\n",
+               offStart, cbRange));
+
+    PPDMACFILERANGELOCK pRangeLock = (PPDMACFILERANGELOCK)RTMemCacheAlloc(pAioMgr->hMemCacheRangeLocks);
+    if (!pRangeLock)
+        return VERR_NO_MEMORY;
+
+    /* Init the lock. */
+    pRangeLock->Core.Key          = offStart;
+    pRangeLock->Core.KeyLast      = offStart + cbRange - 1;
+    pRangeLock->cRefs             = 1;
+    pRangeLock->fReadLock         = pTask->enmTransferType == PDMACTASKFILETRANSFER_READ;
+    pRangeLock->pWaitingTasksHead = NULL;
+    pRangeLock->pWaitingTasksTail = NULL;
+
+    bool fInserted = RTAvlrFileOffsetInsert(pEndpoint->AioMgr.pTreeRangesLocked, &pRangeLock->Core);
+    AssertMsg(fInserted, ("Range lock was not inserted!\n"));
+
+    /* Let the task point to its lock. */
+    pTask->pRangeLock = pRangeLock;
+
+    return VINF_SUCCESS;
+}
+
+static PPDMACTASKFILE pdmacFileAioMgrNormalRangeLockFree(PPDMACEPFILEMGR pAioMgr,
+                                                         PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                         PPDMACFILERANGELOCK pRangeLock)
+{
+    PPDMACTASKFILE pTasksWaitingHead;
+
+    AssertPtr(pRangeLock);
+    Assert(pRangeLock->cRefs == 1);
+
+    RTAvlrFileOffsetRemove(pEndpoint->AioMgr.pTreeRangesLocked, pRangeLock->Core.Key);
+    pTasksWaitingHead = pRangeLock->pWaitingTasksHead;
+    pRangeLock->pWaitingTasksHead = NULL;
+    pRangeLock->pWaitingTasksTail = NULL;
+    RTMemCacheFree(pAioMgr->hMemCacheRangeLocks, pRangeLock);
+
+    return pTasksWaitingHead;
+}
+
+static int pdmacFileAioMgrNormalTaskPrepareBuffered(PPDMACEPFILEMGR pAioMgr,
+                                                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                    PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
+{
+    int rc = VINF_SUCCESS;
+    RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+    void *pvBuf = pTask->DataSeg.pvSeg;
+
+    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
+                || (uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) <= pEndpoint->cbFile,
+                ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
+                pTask->Off, pTask->DataSeg.cbSeg, pEndpoint->cbFile));
+
+    pTask->fPrefetch = false;
+    pTask->cbBounceBuffer = 0;
+
+    /*
+     * Before we start to setup the request we have to check whether there is a task
+     * already active which range intersects with ours. We have to defer execution
+     * of this task in two cases:
+     *     - The pending task is a write and the current is either read or write
+     *     - The pending task is a read and the current task is a write task.
+     *
+     * To check whether a range is currently "locked" we use the AVL tree where every pending task
+     * is stored by its file offset range. The current task will be added to the active task
+     * and will be executed when the active one completes. (The method below
+     * which checks whether a range is already used will add the task)
+     *
+     * This is necessary because of the requirement to align all requests to a 512 boundary
+     * which is enforced by the host OS (Linux and Windows atm). It is possible that
+     * we have to process unaligned tasks and need to align them using bounce buffers.
+     * While the data is fetched from the file another request might arrive writing to
+     * the same range. This will result in data corruption if both are executed concurrently.
+     */
+    bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, pTask->Off, pTask->DataSeg.cbSeg, pTask);
+
+    if (!fLocked)
+    {
+        /* Get a request handle. */
+        hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
+        AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
+
+        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+        {
+            /* Grow the file if needed. */
+            if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
+            {
+                ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
+                RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
+            }
+
+            rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
+                                          pTask->Off, pTask->DataSeg.pvSeg,
+                                          pTask->DataSeg.cbSeg, pTask);
+        }
+        else
+            rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
+                                         pTask->Off, pTask->DataSeg.pvSeg,
+                                         pTask->DataSeg.cbSeg, pTask);
+        AssertRC(rc);
+
+        rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, pTask->Off,
+                                            pTask->DataSeg.cbSeg,
+                                            pTask);
+
+        if (RT_SUCCESS(rc))
+        {
+            pTask->hReq = hReq;
+            *phReq = hReq;
+        }
+    }
+    else
+        LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
+
+    return rc;
+}
+
+static int pdmacFileAioMgrNormalTaskPrepareNonBuffered(PPDMACEPFILEMGR pAioMgr,
+                                                       PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                                                       PPDMACTASKFILE pTask, PRTFILEAIOREQ phReq)
+{
+    int rc = VINF_SUCCESS;
+    RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
+    PPDMASYNCCOMPLETIONEPCLASSFILE pEpClassFile = (PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass;
+    void *pvBuf = pTask->DataSeg.pvSeg;
+
+    /*
+     * Check if the alignment requirements are met.
+     * Offset, transfer size and buffer address
+     * need to be on a 512 boundary.
+     */
+    RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
+    size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg + (pTask->Off - offStart), 512);
+    PDMACTASKFILETRANSFER enmTransferType = pTask->enmTransferType;
+
+    AssertMsg(   pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE
+                || (uint64_t)(offStart + cbToTransfer) <= pEndpoint->cbFile,
+                ("Read exceeds file size offStart=%RTfoff cbToTransfer=%d cbFile=%llu\n",
+                offStart, cbToTransfer, pEndpoint->cbFile));
+
+    pTask->fPrefetch = false;
+
+    /*
+     * Before we start to setup the request we have to check whether there is a task
+     * already active which range intersects with ours. We have to defer execution
+     * of this task in two cases:
+     *     - The pending task is a write and the current is either read or write
+     *     - The pending task is a read and the current task is a write task.
+     *
+     * To check whether a range is currently "locked" we use the AVL tree where every pending task
+     * is stored by its file offset range. The current task will be added to the active task
+     * and will be executed when the active one completes. (The method below
+     * which checks whether a range is already used will add the task)
+     *
+     * This is necessary because of the requirement to align all requests to a 512 boundary
+     * which is enforced by the host OS (Linux and Windows atm). It is possible that
+     * we have to process unaligned tasks and need to align them using bounce buffers.
+     * While the data is fetched from the file another request might arrive writing to
+     * the same range. This will result in data corruption if both are executed concurrently.
+     */
+    bool fLocked = pdmacFileAioMgrNormalIsRangeLocked(pEndpoint, offStart, cbToTransfer, pTask);
+
+    if (!fLocked)
+    {
+        /* Get a request handle. */
+        hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
+        AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
+
+        if (   RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
+            || RT_UNLIKELY(offStart != pTask->Off)
+            || ((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) != (RTR3UINTPTR)pvBuf))
+        {
+            LogFlow(("Using bounce buffer for task %#p cbToTransfer=%zd cbSeg=%zd offStart=%RTfoff off=%RTfoff\n",
+                     pTask, cbToTransfer, pTask->DataSeg.cbSeg, offStart, pTask->Off));
+
+            /* Create bounce buffer. */
+            pTask->cbBounceBuffer = cbToTransfer;
+
+            AssertMsg(pTask->Off >= offStart, ("Overflow in calculation Off=%llu offStart=%llu\n",
+                      pTask->Off, offStart));
+            pTask->offBounceBuffer = pTask->Off - offStart;
+
+            /** @todo: I think we need something like a RTMemAllocAligned method here.
+             * Current assumption is that the maximum alignment is 4096byte
+             * (GPT disk on Windows)
+             * so we can use RTMemPageAlloc here.
+             */
+            pTask->pvBounceBuffer = RTMemPageAlloc(cbToTransfer);
+            if (RT_LIKELY(pTask->pvBounceBuffer))
+            {
+                pvBuf = pTask->pvBounceBuffer;
+
+                if (pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+                {
+                    if (   RT_UNLIKELY(cbToTransfer != pTask->DataSeg.cbSeg)
+                        || RT_UNLIKELY(offStart != pTask->Off))
+                    {
+                        /* We have to fill the buffer first before we can update the data. */
+                        LogFlow(("Prefetching data for task %#p\n", pTask));
+                        pTask->fPrefetch = true;
+                        enmTransferType = PDMACTASKFILETRANSFER_READ;
+                    }
+                    else
+                        memcpy(pvBuf, pTask->DataSeg.pvSeg, pTask->DataSeg.cbSeg);
+                }
+            }
+            else
+                rc = VERR_NO_MEMORY;
+        }
+        else
+            pTask->cbBounceBuffer = 0;
+
+        if (RT_SUCCESS(rc))
+        {
+            AssertMsg((pEpClassFile->uBitmaskAlignment & (RTR3UINTPTR)pvBuf) == (RTR3UINTPTR)pvBuf,
+                      ("AIO: Alignment restrictions not met! pvBuf=%p uBitmaskAlignment=%p\n", pvBuf, pEpClassFile->uBitmaskAlignment));
+
+            if (enmTransferType == PDMACTASKFILETRANSFER_WRITE)
+            {
+                /* Grow the file if needed. */
+                if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
+                {
+                    ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
+                    RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
+                }
+
+                rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
+                                              offStart, pvBuf, cbToTransfer, pTask);
+            }
+            else
+                rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File,
+                                             offStart, pvBuf, cbToTransfer, pTask);
+            AssertRC(rc);
+
+            rc = pdmacFileAioMgrNormalRangeLock(pAioMgr, pEndpoint, offStart, cbToTransfer, pTask);
+
+            if (RT_SUCCESS(rc))
+            {
+                pTask->hReq = hReq;
+                *phReq = hReq;
+            }
+            else
+            {
+                /* Cleanup */
+                if (pTask->cbBounceBuffer)
+                    RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
+            }
+        }
+    }
+    else
+        LogFlow(("Task %#p was deferred because the access range is locked\n", pTask));
+
+    return rc;
+}
+
+static int pdmacFileAioMgrNormalProcessTaskList(PPDMACTASKFILE pTaskHead,
+                                                PPDMACEPFILEMGR pAioMgr,
+                                                PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    RTFILEAIOREQ  apReqs[20];
+    unsigned      cRequests = 0;
+    unsigned      cMaxRequests = pAioMgr->cRequestsActiveMax - pAioMgr->cRequestsActive;
+    int           rc = VINF_SUCCESS;
+
+    AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
+              ("Trying to process request lists of a non active endpoint!\n"));
+
+    /* Go through the list and queue the requests until we get a flush request */
+    while (   pTaskHead
+           && !pEndpoint->pFlushReq
+           && (pAioMgr->cRequestsActive + cRequests < pAioMgr->cRequestsActiveMax)
+           && RT_SUCCESS(rc))
+    {
+        RTMSINTERVAL msWhenNext;
+        PPDMACTASKFILE pCurr = pTaskHead;
+
+        if (!pdmacEpIsTransferAllowed(&pEndpoint->Core, (uint32_t)pCurr->DataSeg.cbSeg, &msWhenNext))
+        {
+            pAioMgr->msBwLimitExpired = RT_MIN(pAioMgr->msBwLimitExpired, msWhenNext);
+            break;
+        }
+
+        pTaskHead = pTaskHead->pNext;
+
+        pCurr->pNext = NULL;
+
+        AssertMsg(VALID_PTR(pCurr->pEndpoint) && (pCurr->pEndpoint == pEndpoint),
+                  ("Endpoints do not match\n"));
+
+        switch (pCurr->enmTransferType)
+        {
+            case PDMACTASKFILETRANSFER_FLUSH:
+            {
+                /* If there is no data transfer request this flush request finished immediately. */
+                if (pEndpoint->fAsyncFlushSupported)
+                {
+                    /* Issue a flush to the host. */
+                    RTFILEAIOREQ hReq = pdmacFileAioMgrNormalRequestAlloc(pAioMgr);
+                    AssertMsg(hReq != NIL_RTFILEAIOREQ, ("Out of request handles\n"));
+
+                    LogFlow(("Flush request %#p\n", hReq));
+
+                    rc = RTFileAioReqPrepareFlush(hReq, pEndpoint->File, pCurr);
+                    if (RT_FAILURE(rc))
+                    {
+                        pEndpoint->fAsyncFlushSupported = false;
+                        pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+                        rc = VINF_SUCCESS; /* Fake success */
+                    }
+                    else
+                    {
+                        pCurr->hReq = hReq;
+                        apReqs[cRequests] = hReq;
+                        pEndpoint->AioMgr.cReqsProcessed++;
+                        cRequests++;
+                    }
+                }
+
+                if (   !pEndpoint->AioMgr.cRequestsActive
+                    && !pEndpoint->fAsyncFlushSupported)
+                {
+                    pCurr->pfnCompleted(pCurr, pCurr->pvUser, VINF_SUCCESS);
+                    pdmacFileTaskFree(pEndpoint, pCurr);
+                }
+                else
+                {
+                    Assert(!pEndpoint->pFlushReq);
+                    pEndpoint->pFlushReq = pCurr;
+                }
+                break;
+            }
+            case PDMACTASKFILETRANSFER_READ:
+            case PDMACTASKFILETRANSFER_WRITE:
+            {
+                RTFILEAIOREQ hReq = NIL_RTFILEAIOREQ;
+
+                if (pCurr->hReq == NIL_RTFILEAIOREQ)
+                {
+                    if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_BUFFERED)
+                        rc = pdmacFileAioMgrNormalTaskPrepareBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
+                    else if (pEndpoint->enmBackendType == PDMACFILEEPBACKEND_NON_BUFFERED)
+                        rc = pdmacFileAioMgrNormalTaskPrepareNonBuffered(pAioMgr, pEndpoint, pCurr, &hReq);
+                    else
+                        AssertMsgFailed(("Invalid backend type %d\n", pEndpoint->enmBackendType));
+
+                    AssertRC(rc);
+                }
+                else
+                {
+                    LogFlow(("Task %#p has I/O request %#p already\n", pCurr, pCurr->hReq));
+                    hReq = pCurr->hReq;
+                }
+
+                LogFlow(("Read/Write request %#p\n", hReq));
+
+                if (hReq != NIL_RTFILEAIOREQ)
+                {
+                    apReqs[cRequests] = hReq;
+                    cRequests++;
+                }
+                break;
+            }
+            default:
+                AssertMsgFailed(("Invalid transfer type %d\n", pCurr->enmTransferType));
+        } /* switch transfer type */
+
+        /* Queue the requests if the array is full. */
+        if (cRequests == RT_ELEMENTS(apReqs))
+        {
+            rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
+            cRequests = 0;
+            AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                      ("Unexpected return code\n"));
+        }
+    }
+
+    if (cRequests)
+    {
+        rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, apReqs, cRequests);
+        AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                  ("Unexpected return code rc=%Rrc\n", rc));
+    }
+
+    if (pTaskHead)
+    {
+        /* Add the rest of the tasks to the pending list */
+        pdmacFileAioMgrEpAddTaskList(pEndpoint, pTaskHead);
+
+        if (RT_UNLIKELY(   pAioMgr->cRequestsActiveMax == pAioMgr->cRequestsActive
+                        && !pEndpoint->pFlushReq))
+        {
+#if 0
+            /*
+             * The I/O manager has no room left for more requests
+             * but there are still requests to process.
+             * Create a new I/O manager and let it handle some endpoints.
+             */
+            pdmacFileAioMgrNormalBalanceLoad(pAioMgr);
+#else
+            /* Grow the I/O manager */
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_GROWING;
+#endif
+        }
+    }
+
+    /* Insufficient resources are not fatal. */
+    if (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES)
+        rc = VINF_SUCCESS;
+
+    return rc;
+}
+
+/**
+ * Adds all pending requests for the given endpoint
+ * until a flush request is encountered or there is no
+ * request anymore.
+ *
+ * @returns VBox status code.
+ * @param   pAioMgr    The async I/O manager for the endpoint
+ * @param   pEndpoint  The endpoint to get the requests from.
+ */
+static int pdmacFileAioMgrNormalQueueReqs(PPDMACEPFILEMGR pAioMgr,
+                                          PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint)
+{
+    int rc = VINF_SUCCESS;
+    PPDMACTASKFILE pTasksHead = NULL;
+
+    AssertMsg(pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
+              ("Trying to process request lists of a non active endpoint!\n"));
+
+    Assert(!pEndpoint->pFlushReq);
+
+    /* Check the pending list first */
+    if (pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        LogFlow(("Queuing pending requests first\n"));
+
+        pTasksHead = pEndpoint->AioMgr.pReqsPendingHead;
+        /*
+         * Clear the list as the processing routine will insert them into the list
+         * again if it gets a flush request.
+         */
+        pEndpoint->AioMgr.pReqsPendingHead = NULL;
+        pEndpoint->AioMgr.pReqsPendingTail = NULL;
+        rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
+        AssertRC(rc);
+    }
+
+    if (!pEndpoint->pFlushReq && !pEndpoint->AioMgr.pReqsPendingHead)
+    {
+        /* Now the request queue. */
+        pTasksHead = pdmacFileEpGetNewTasks(pEndpoint);
+        if (pTasksHead)
+        {
+            rc = pdmacFileAioMgrNormalProcessTaskList(pTasksHead, pAioMgr, pEndpoint);
+            AssertRC(rc);
+        }
+    }
+
+    return rc;
+}
+
+static int pdmacFileAioMgrNormalProcessBlockingEvent(PPDMACEPFILEMGR pAioMgr)
+{
+    int rc = VINF_SUCCESS;
+    bool fNotifyWaiter = false;
+
+    LogFlowFunc((": Enter\n"));
+
+    Assert(pAioMgr->fBlockingEventPending);
+
+    switch (pAioMgr->enmBlockingEvent)
+    {
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT:
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointNew = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.AddEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
+            AssertMsg(VALID_PTR(pEndpointNew), ("Adding endpoint event without a endpoint to add\n"));
+
+            pEndpointNew->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE;
+
+            pEndpointNew->AioMgr.pEndpointNext = pAioMgr->pEndpointsHead;
+            pEndpointNew->AioMgr.pEndpointPrev = NULL;
+            if (pAioMgr->pEndpointsHead)
+                pAioMgr->pEndpointsHead->AioMgr.pEndpointPrev = pEndpointNew;
+            pAioMgr->pEndpointsHead = pEndpointNew;
+
+            /* Assign the completion point to this file. */
+            rc = RTFileAioCtxAssociateWithFile(pAioMgr->hAioCtx, pEndpointNew->File);
+            fNotifyWaiter = true;
+            pAioMgr->cEndpoints++;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT:
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointRemove = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.RemoveEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
+            AssertMsg(VALID_PTR(pEndpointRemove), ("Removing endpoint event without a endpoint to remove\n"));
+
+            pEndpointRemove->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING;
+            fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointRemove);
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT:
+        {
+            PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointClose = ASMAtomicReadPtrT(&pAioMgr->BlockingEventData.CloseEndpoint.pEndpoint, PPDMASYNCCOMPLETIONENDPOINTFILE);
+            AssertMsg(VALID_PTR(pEndpointClose), ("Close endpoint event without a endpoint to close\n"));
+
+            if (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
+            {
+                LogFlowFunc((": Closing endpoint %#p{%s}\n", pEndpointClose, pEndpointClose->Core.pszUri));
+
+                /* Make sure all tasks finished. Process the queues a last time first. */
+                rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpointClose);
+                AssertRC(rc);
+
+                pEndpointClose->enmState = PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING;
+                fNotifyWaiter = !pdmacFileAioMgrNormalRemoveEndpoint(pEndpointClose);
+            }
+            else if (   (pEndpointClose->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING)
+                     && (!pEndpointClose->AioMgr.cRequestsActive))
+                fNotifyWaiter = true;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN:
+        {
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_SHUTDOWN;
+            if (!pAioMgr->cRequestsActive)
+                fNotifyWaiter = true;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND:
+        {
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_SUSPENDING;
+            break;
+        }
+        case PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME:
+        {
+            pAioMgr->enmState = PDMACEPFILEMGRSTATE_RUNNING;
+            fNotifyWaiter = true;
+            break;
+        }
+        default:
+            AssertReleaseMsgFailed(("Invalid event type %d\n", pAioMgr->enmBlockingEvent));
+    }
+
+    if (fNotifyWaiter)
+    {
+        ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
+        pAioMgr->enmBlockingEvent = PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID;
+
+        /* Release the waiting thread. */
+        LogFlow(("Signalling waiter\n"));
+        rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+        AssertRC(rc);
+    }
+
+    LogFlowFunc((": Leave\n"));
+    return rc;
+}
+
+/**
+ * Checks all endpoints for pending events or new requests.
+ *
+ * @returns VBox status code.
+ * @param   pAioMgr    The I/O manager handle.
+ */
+static int pdmacFileAioMgrNormalCheckEndpoints(PPDMACEPFILEMGR pAioMgr)
+{
+    /* Check the assigned endpoints for new tasks if there isn't a flush request active at the moment. */
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint = pAioMgr->pEndpointsHead;
+
+    pAioMgr->msBwLimitExpired = RT_INDEFINITE_WAIT;
+
+    while (pEndpoint)
+    {
+        if (!pEndpoint->pFlushReq
+            && (pEndpoint->enmState == PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
+            && !pEndpoint->AioMgr.fMoving)
+        {
+            rc = pdmacFileAioMgrNormalQueueReqs(pAioMgr, pEndpoint);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+        else if (   !pEndpoint->AioMgr.cRequestsActive
+                 && pEndpoint->enmState != PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE)
+        {
+            /* Reopen the file so that the new endpoint can re-associate with the file */
+            RTFileClose(pEndpoint->File);
+            rc = RTFileOpen(&pEndpoint->File, pEndpoint->Core.pszUri, pEndpoint->fFlags);
+            AssertRC(rc);
+
+            if (pEndpoint->AioMgr.fMoving)
+            {
+                pEndpoint->AioMgr.fMoving = false;
+                pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
+            }
+            else
+            {
+                Assert(pAioMgr->fBlockingEventPending);
+                ASMAtomicWriteBool(&pAioMgr->fBlockingEventPending, false);
+
+                /* Release the waiting thread. */
+                LogFlow(("Signalling waiter\n"));
+                rc = RTSemEventSignal(pAioMgr->EventSemBlock);
+                AssertRC(rc);
+            }
+        }
+
+        pEndpoint = pEndpoint->AioMgr.pEndpointNext;
+    }
+
+    return rc;
+}
+
+/**
+ * Wrapper around pdmacFileAioMgrNormalReqCompleteRc().
+ */
+static void pdmacFileAioMgrNormalReqComplete(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq)
+{
+    size_t cbTransfered = 0;
+    int rcReq = RTFileAioReqGetRC(hReq, &cbTransfered);
+
+    pdmacFileAioMgrNormalReqCompleteRc(pAioMgr, hReq, rcReq, cbTransfered);
+}
+
+static void pdmacFileAioMgrNormalReqCompleteRc(PPDMACEPFILEMGR pAioMgr, RTFILEAIOREQ hReq,
+                                               int rcReq, size_t cbTransfered)
+{
+    int rc = VINF_SUCCESS;
+    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+    PPDMACTASKFILE pTask = (PPDMACTASKFILE)RTFileAioReqGetUser(hReq);
+    PPDMACTASKFILE pTasksWaiting;
+
+    LogFlowFunc(("pAioMgr=%#p hReq=%#p\n", pAioMgr, hReq));
+
+    pEndpoint = pTask->pEndpoint;
+
+    pTask->hReq = NIL_RTFILEAIOREQ;
+
+    pAioMgr->cRequestsActive--;
+    pEndpoint->AioMgr.cRequestsActive--;
+    pEndpoint->AioMgr.cReqsProcessed++;
+
+    /*
+     * It is possible that the request failed on Linux with kernels < 2.6.23
+     * if the passed buffer was allocated with remap_pfn_range or if the file
+     * is on an NFS endpoint which does not support async and direct I/O at the same time.
+     * The endpoint will be migrated to a failsafe manager in case a request fails.
+     */
+    if (RT_FAILURE(rcReq))
+    {
+        /* Free bounce buffers and the IPRT request. */
+        pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+
+        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+        {
+            LogFlow(("Async flushes are not supported for this endpoint, disabling\n"));
+            pEndpoint->fAsyncFlushSupported = false;
+            AssertMsg(pEndpoint->pFlushReq == pTask, ("Failed flush request doesn't match active one\n"));
+            /* The other method will take over now. */
+        }
+        else
+        {
+            /* Free the lock and process pending tasks if necessary */
+            pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
+            rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
+            AssertRC(rc);
+
+            if (pTask->cbBounceBuffer)
+                RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
+
+            /*
+             * Fatal errors are reported to the guest and non-fatal errors
+             * will cause a migration to the failsafe manager in the hope
+             * that the error disappears.
+             */
+            if (!pdmacFileAioMgrNormalRcIsFatal(rcReq))
+            {
+                /* Queue the request on the pending list. */
+                pTask->pNext = pEndpoint->AioMgr.pReqsPendingHead;
+                pEndpoint->AioMgr.pReqsPendingHead = pTask;
+
+                /* Create a new failsafe manager if necessary. */
+                if (!pEndpoint->AioMgr.fMoving)
+                {
+                    PPDMACEPFILEMGR pAioMgrFailsafe;
+
+                    LogRel(("%s: Request %#p failed with rc=%Rrc, migrating endpoint %s to failsafe manager.\n",
+                            RTThreadGetName(pAioMgr->Thread), pTask, rcReq, pEndpoint->Core.pszUri));
+
+                    pEndpoint->AioMgr.fMoving = true;
+
+                    rc = pdmacFileAioMgrCreate((PPDMASYNCCOMPLETIONEPCLASSFILE)pEndpoint->Core.pEpClass,
+                                                &pAioMgrFailsafe, PDMACEPFILEMGRTYPE_SIMPLE);
+                    AssertRC(rc);
+
+                    pEndpoint->AioMgr.pAioMgrDst = pAioMgrFailsafe;
+
+                    /* Update the flags to open the file with. Disable async I/O and enable the host cache. */
+                    pEndpoint->fFlags &= ~(RTFILE_O_ASYNC_IO | RTFILE_O_NO_CACHE);
+                }
+
+                /* If this was the last request for the endpoint migrate it to the new manager. */
+                if (!pEndpoint->AioMgr.cRequestsActive)
+                {
+                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
+                    Assert(!fReqsPending);
+
+                    rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
+                    AssertRC(rc);
+                }
+            }
+            else
+            {
+                pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
+                pdmacFileTaskFree(pEndpoint, pTask);
+            }
+        }
+    }
+    else
+    {
+        if (pTask->enmTransferType == PDMACTASKFILETRANSFER_FLUSH)
+        {
+            /* Clear pending flush */
+            AssertMsg(pEndpoint->pFlushReq == pTask, ("Completed flush request doesn't match active one\n"));
+            pEndpoint->pFlushReq = NULL;
+            pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+
+            /* Call completion callback */
+            LogFlow(("Flush task=%#p completed with %Rrc\n", pTask, rcReq));
+            pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
+            pdmacFileTaskFree(pEndpoint, pTask);
+        }
+        else
+        {
+            /*
+             * Restart an incomplete transfer.
+             * This usually means that the request will return an error now
+             * but to get the cause of the error (disk full, file too big, I/O error, ...)
+             * the transfer needs to be continued.
+             */
+            if (RT_UNLIKELY(   cbTransfered < pTask->DataSeg.cbSeg
+                            || (   pTask->cbBounceBuffer
+                                && cbTransfered < pTask->cbBounceBuffer)))
+            {
+                RTFOFF offStart;
+                size_t cbToTransfer;
+                uint8_t *pbBuf = NULL;
+
+                LogFlow(("Restarting incomplete transfer %#p (%zu bytes transferred)\n",
+                         pTask, cbTransfered));
+                Assert(cbTransfered % 512 == 0);
+
+                if (pTask->cbBounceBuffer)
+                {
+                    AssertPtr(pTask->pvBounceBuffer);
+                    offStart     = (pTask->Off & ~((RTFOFF)512-1)) + cbTransfered;
+                    cbToTransfer = pTask->cbBounceBuffer - cbTransfered;
+                    pbBuf        = (uint8_t *)pTask->pvBounceBuffer + cbTransfered;
+                }
+                else
+                {
+                    Assert(!pTask->pvBounceBuffer);
+                    offStart     = pTask->Off + cbTransfered;
+                    cbToTransfer = pTask->DataSeg.cbSeg - cbTransfered;
+                    pbBuf        = (uint8_t *)pTask->DataSeg.pvSeg + cbTransfered;
+                }
+
+                if (pTask->fPrefetch || pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                {
+                    rc = RTFileAioReqPrepareRead(hReq, pEndpoint->File, offStart,
+                                                 pbBuf, cbToTransfer, pTask);
+                }
+                else
+                {
+                    AssertMsg(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE,
+                                  ("Invalid transfer type\n"));
+                    rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File, offStart,
+                                                  pbBuf, cbToTransfer, pTask);
+                }
+                AssertRC(rc);
+
+                pTask->hReq = hReq;
+                rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
+                AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                          ("Unexpected return code rc=%Rrc\n", rc));
+            }
+            else if (pTask->fPrefetch)
+            {
+                Assert(pTask->enmTransferType == PDMACTASKFILETRANSFER_WRITE);
+                Assert(pTask->cbBounceBuffer);
+
+                memcpy(((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
+                        pTask->DataSeg.pvSeg,
+                        pTask->DataSeg.cbSeg);
+
+                /* Write it now. */
+                pTask->fPrefetch = false;
+                size_t cbToTransfer = RT_ALIGN_Z(pTask->DataSeg.cbSeg, 512);
+                RTFOFF offStart = pTask->Off & ~(RTFOFF)(512-1);
+
+                /* Grow the file if needed. */
+                if (RT_UNLIKELY((uint64_t)(pTask->Off + pTask->DataSeg.cbSeg) > pEndpoint->cbFile))
+                {
+                    ASMAtomicWriteU64(&pEndpoint->cbFile, pTask->Off + pTask->DataSeg.cbSeg);
+                    RTFileSetSize(pEndpoint->File, pTask->Off + pTask->DataSeg.cbSeg);
+                }
+
+                rc = RTFileAioReqPrepareWrite(hReq, pEndpoint->File,
+                                              offStart, pTask->pvBounceBuffer, cbToTransfer, pTask);
+                AssertRC(rc);
+                pTask->hReq = hReq;
+                rc = pdmacFileAioMgrNormalReqsEnqueue(pAioMgr, pEndpoint, &hReq, 1);
+                AssertMsg(RT_SUCCESS(rc) || (rc == VERR_FILE_AIO_INSUFFICIENT_RESSOURCES),
+                          ("Unexpected return code rc=%Rrc\n", rc));
+            }
+            else
+            {
+                if (RT_SUCCESS(rc) && pTask->cbBounceBuffer)
+                {
+                    if (pTask->enmTransferType == PDMACTASKFILETRANSFER_READ)
+                        memcpy(pTask->DataSeg.pvSeg,
+                               ((uint8_t *)pTask->pvBounceBuffer) + pTask->offBounceBuffer,
+                               pTask->DataSeg.cbSeg);
+
+                    RTMemPageFree(pTask->pvBounceBuffer, pTask->cbBounceBuffer);
+                }
+
+                pdmacFileAioMgrNormalRequestFree(pAioMgr, hReq);
+
+                /* Free the lock and process pending tasks if necessary */
+                pTasksWaiting = pdmacFileAioMgrNormalRangeLockFree(pAioMgr, pEndpoint, pTask->pRangeLock);
+                if (pTasksWaiting)
+                {
+                    rc = pdmacFileAioMgrNormalProcessTaskList(pTasksWaiting, pAioMgr, pEndpoint);
+                    AssertRC(rc);
+                }
+
+                /* Call completion callback */
+                LogFlow(("Task=%#p completed with %Rrc\n", pTask, rcReq));
+                pTask->pfnCompleted(pTask, pTask->pvUser, rcReq);
+                pdmacFileTaskFree(pEndpoint, pTask);
+
+                /*
+                 * If there is no request left on the endpoint but a flush request is set
+                 * it completed now and we notify the owner.
+                 * Furthermore we look for new requests and continue.
+                 */
+                if (!pEndpoint->AioMgr.cRequestsActive && pEndpoint->pFlushReq)
+                {
+                    /* Call completion callback */
+                    pTask = pEndpoint->pFlushReq;
+                    pEndpoint->pFlushReq = NULL;
+
+                    AssertMsg(pTask->pEndpoint == pEndpoint, ("Endpoint of the flush request does not match assigned one\n"));
+
+                    pTask->pfnCompleted(pTask, pTask->pvUser, VINF_SUCCESS);
+                    pdmacFileTaskFree(pEndpoint, pTask);
+                }
+                else if (RT_UNLIKELY(!pEndpoint->AioMgr.cRequestsActive && pEndpoint->AioMgr.fMoving))
+                {
+                    /* If the endpoint is about to be migrated do it now. */
+                    bool fReqsPending = pdmacFileAioMgrNormalRemoveEndpoint(pEndpoint);
+                    Assert(!fReqsPending);
+
+                    rc = pdmacFileAioMgrAddEndpoint(pEndpoint->AioMgr.pAioMgrDst, pEndpoint);
+                    AssertRC(rc);
+                }
+            }
+        } /* Not a flush request */
+    } /* request completed successfully */
+}
+
+/** Helper macro for checking for error codes. */
+#define CHECK_RC(pAioMgr, rc) \
+    if (RT_FAILURE(rc)) \
+    {\
+        int rc2 = pdmacFileAioMgrNormalErrorHandler(pAioMgr, rc, RT_SRC_POS);\
+        return rc2;\
+    }
+
+/**
+ * The normal I/O manager using the RTFileAio* API
+ *
+ * @returns VBox status code.
+ * @param ThreadSelf    Handle of the thread.
+ * @param pvUser        Opaque user data.
+ */
+int pdmacFileAioMgrNormal(RTTHREAD ThreadSelf, void *pvUser)
+{
+    int rc                  = VINF_SUCCESS;
+    PPDMACEPFILEMGR pAioMgr = (PPDMACEPFILEMGR)pvUser;
+    uint64_t uMillisEnd     = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
+
+    while (   (pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING)
+           || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_SUSPENDING)
+           || (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
+    {
+        if (!pAioMgr->cRequestsActive)
+        {
+            ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, true);
+            if (!ASMAtomicReadBool(&pAioMgr->fWokenUp))
+                rc = RTSemEventWait(pAioMgr->EventSem, pAioMgr->msBwLimitExpired);
+            ASMAtomicWriteBool(&pAioMgr->fWaitingEventSem, false);
+            Assert(RT_SUCCESS(rc) || rc == VERR_TIMEOUT);
+
+            LogFlow(("Got woken up\n"));
+            ASMAtomicWriteBool(&pAioMgr->fWokenUp, false);
+        }
+
+        /* Check for an external blocking event first. */
+        if (pAioMgr->fBlockingEventPending)
+        {
+            rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
+            CHECK_RC(pAioMgr, rc);
+        }
+
+        if (RT_LIKELY(    pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING
+                      ||  pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING))
+        {
+            /* We got woken up because an endpoint issued new requests. Queue them. */
+            rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
+            CHECK_RC(pAioMgr, rc);
+
+            while (pAioMgr->cRequestsActive)
+            {
+                RTFILEAIOREQ apReqs[20];
+                uint32_t     cReqsCompleted = 0;
+                size_t       cReqsWait;
+
+                if (pAioMgr->cRequestsActive > RT_ELEMENTS(apReqs))
+                    cReqsWait = RT_ELEMENTS(apReqs);
+                else
+                    cReqsWait = pAioMgr->cRequestsActive;
+
+                LogFlow(("Waiting for %d of %d tasks to complete\n", 1, cReqsWait));
+
+                rc = RTFileAioCtxWait(pAioMgr->hAioCtx,
+                                      1,
+                                      RT_INDEFINITE_WAIT, apReqs,
+                                      cReqsWait, &cReqsCompleted);
+                if (RT_FAILURE(rc) && (rc != VERR_INTERRUPTED))
+                    CHECK_RC(pAioMgr, rc);
+
+                LogFlow(("%d tasks completed\n", cReqsCompleted));
+
+                for (uint32_t i = 0; i < cReqsCompleted; i++)
+                    pdmacFileAioMgrNormalReqComplete(pAioMgr, apReqs[i]);
+
+                /* Check for an external blocking event before we go to sleep again. */
+                if (pAioMgr->fBlockingEventPending)
+                {
+                    rc = pdmacFileAioMgrNormalProcessBlockingEvent(pAioMgr);
+                    CHECK_RC(pAioMgr, rc);
+                }
+
+                /* Update load statistics. */
+                uint64_t uMillisCurr = RTTimeMilliTS();
+                if (uMillisCurr > uMillisEnd)
+                {
+                    PPDMASYNCCOMPLETIONENDPOINTFILE pEndpointCurr = pAioMgr->pEndpointsHead;
+
+                    /* Calculate timespan. */
+                    uMillisCurr -= uMillisEnd;
+
+                    while (pEndpointCurr)
+                    {
+                        pEndpointCurr->AioMgr.cReqsPerSec    = pEndpointCurr->AioMgr.cReqsProcessed / (uMillisCurr + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD);
+                        pEndpointCurr->AioMgr.cReqsProcessed = 0;
+                        pEndpointCurr = pEndpointCurr->AioMgr.pEndpointNext;
+                    }
+
+                    /* Set new update interval */
+                    uMillisEnd = RTTimeMilliTS() + PDMACEPFILEMGR_LOAD_UPDATE_PERIOD;
+                }
+
+                /* Check endpoints for new requests. */
+                if (pAioMgr->enmState != PDMACEPFILEMGRSTATE_GROWING)
+                {
+                    rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
+                    CHECK_RC(pAioMgr, rc);
+                }
+            } /* while requests are active. */
+
+            if (pAioMgr->enmState == PDMACEPFILEMGRSTATE_GROWING)
+            {
+                rc = pdmacFileAioMgrNormalGrow(pAioMgr);
+                AssertRC(rc);
+                Assert(pAioMgr->enmState == PDMACEPFILEMGRSTATE_RUNNING);
+
+                rc = pdmacFileAioMgrNormalCheckEndpoints(pAioMgr);
+                CHECK_RC(pAioMgr, rc);
+            }
+        } /* if still running */
+    } /* while running */
+
+    LogFlowFunc(("rc=%Rrc\n", rc));
+    return rc;
+}
+
+#undef CHECK_RC
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMBlkCache.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMBlkCache.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMBlkCache.cpp	(revision 35333)
@@ -0,0 +1,2612 @@
+/* $Id$ */
+/** @file
+ * PDM Block Cache.
+ */
+
+/*
+ * Copyright (C) 2006-2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_pdm_block_cache     PDM Block Cache - The I/O cache
+ * This component implements an I/O cache based on the 2Q cache algorithm.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_BLK_CACHE
+#include "PDMInternal.h"
+#include <iprt/asm.h>
+#include <iprt/mem.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/stam.h>
+#include <VBox/uvm.h>
+#include <VBox/vm.h>
+
+#include "PDMBlkCacheInternal.h"
+
+#ifdef VBOX_STRICT
+# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) \
+    do \
+    { \
+     AssertMsg(RTCritSectIsOwner(&Cache->CritSect), \
+               ("Thread does not own critical section\n"));\
+    } while(0)
+
+# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) \
+    do \
+    { \
+        AssertMsg(RTSemRWIsWriteOwner(pEpCache->SemRWEntries), \
+                  ("Thread is not exclusive owner of the per endpoint RW semaphore\n")); \
+    } while(0)
+
+# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) \
+    do \
+    { \
+        AssertMsg(RTSemRWIsReadOwner(pEpCache->SemRWEntries), \
+                  ("Thread is not read owner of the per endpoint RW semaphore\n")); \
+    } while(0)
+
+#else
+# define PDMACFILECACHE_IS_CRITSECT_OWNER(Cache) do { } while(0)
+# define PDMACFILECACHE_EP_IS_SEMRW_WRITE_OWNER(pEpCache) do { } while(0)
+# define PDMACFILECACHE_EP_IS_SEMRW_READ_OWNER(pEpCache) do { } while(0)
+#endif
+
+#define PDM_BLK_CACHE_SAVED_STATE_VERSION 1
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+
+static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
+                                               uint64_t off, size_t cbData, uint8_t *pbBuffer);
+static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry);
+
+/**
+ * Decrement the reference counter of the given cache entry.
+ *
+ * @returns nothing.
+ * @param   pEntry    The entry to release.
+ */
+DECLINLINE(void) pdmBlkCacheEntryRelease(PPDMBLKCACHEENTRY pEntry)
+{
+    AssertMsg(pEntry->cRefs > 0, ("Trying to release a not referenced entry\n"));
+    ASMAtomicDecU32(&pEntry->cRefs);
+}
+
+/**
+ * Increment the reference counter of the given cache entry.
+ *
+ * @returns nothing.
+ * @param   pEntry    The entry to reference.
+ */
+DECLINLINE(void) pdmBlkCacheEntryRef(PPDMBLKCACHEENTRY pEntry)
+{
+    ASMAtomicIncU32(&pEntry->cRefs);
+}
+
+#ifdef DEBUG
+static void pdmBlkCacheValidate(PPDMBLKCACHEGLOBAL pCache)
+{
+    /* Amount of cached data should never exceed the maximum amount. */
+    AssertMsg(pCache->cbCached <= pCache->cbMax,
+              ("Current amount of cached data exceeds maximum\n"));
+
+    /* The amount of cached data in the LRU and FRU list should match cbCached */
+    AssertMsg(pCache->LruRecentlyUsedIn.cbCached + pCache->LruFrequentlyUsed.cbCached == pCache->cbCached,
+              ("Amount of cached data doesn't match\n"));
+
+    AssertMsg(pCache->LruRecentlyUsedOut.cbCached <= pCache->cbRecentlyUsedOutMax,
+              ("Paged out list exceeds maximum\n"));
+}
+#endif
+
+DECLINLINE(void) pdmBlkCacheLockEnter(PPDMBLKCACHEGLOBAL pCache)
+{
+    RTCritSectEnter(&pCache->CritSect);
+#ifdef DEBUG
+    pdmBlkCacheValidate(pCache);
+#endif
+}
+
+DECLINLINE(void) pdmBlkCacheLockLeave(PPDMBLKCACHEGLOBAL pCache)
+{
+#ifdef DEBUG
+    pdmBlkCacheValidate(pCache);
+#endif
+    RTCritSectLeave(&pCache->CritSect);
+}
+
+DECLINLINE(void) pdmBlkCacheSub(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
+{
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+    pCache->cbCached -= cbAmount;
+}
+
+DECLINLINE(void) pdmBlkCacheAdd(PPDMBLKCACHEGLOBAL pCache, uint32_t cbAmount)
+{
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+    pCache->cbCached += cbAmount;
+}
+
+DECLINLINE(void) pdmBlkCacheListAdd(PPDMBLKLRULIST pList, uint32_t cbAmount)
+{
+    pList->cbCached += cbAmount;
+}
+
+DECLINLINE(void) pdmBlkCacheListSub(PPDMBLKLRULIST pList, uint32_t cbAmount)
+{
+    pList->cbCached -= cbAmount;
+}
+
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+/**
+ * Checks consistency of a LRU list.
+ *
+ * @returns nothing
+ * @param    pList         The LRU list to check.
+ * @param    pNotInList    Element which is not allowed to occur in the list.
+ */
+static void pdmBlkCacheCheckList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pNotInList)
+{
+    PPDMBLKCACHEENTRY pCurr = pList->pHead;
+
+    /* Check that there are no double entries and no cycles in the list. */
+    while (pCurr)
+    {
+        PPDMBLKCACHEENTRY pNext = pCurr->pNext;
+
+        while (pNext)
+        {
+            AssertMsg(pCurr != pNext,
+                      ("Entry %#p is at least two times in list %#p or there is a cycle in the list\n",
+                      pCurr, pList));
+            pNext = pNext->pNext;
+        }
+
+        AssertMsg(pCurr != pNotInList, ("Not allowed entry %#p is in list\n", pCurr));
+
+        if (!pCurr->pNext)
+            AssertMsg(pCurr == pList->pTail, ("End of list reached but last element is not list tail\n"));
+
+        pCurr = pCurr->pNext;
+    }
+}
+#endif
+
+/**
+ * Unlinks a cache entry from the LRU list it is assigned to.
+ *
+ * @returns nothing.
+ * @param   pEntry    The entry to unlink.
+ */
+static void pdmBlkCacheEntryRemoveFromList(PPDMBLKCACHEENTRY pEntry)
+{
+    PPDMBLKLRULIST pList = pEntry->pList;
+    PPDMBLKCACHEENTRY pPrev, pNext;
+
+    LogFlowFunc((": Deleting entry %#p from list %#p\n", pEntry, pList));
+
+    AssertPtr(pList);
+
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, NULL);
+#endif
+
+    pPrev = pEntry->pPrev;
+    pNext = pEntry->pNext;
+
+    AssertMsg(pEntry != pPrev, ("Entry links to itself as previous element\n"));
+    AssertMsg(pEntry != pNext, ("Entry links to itself as next element\n"));
+
+    if (pPrev)
+        pPrev->pNext = pNext;
+    else
+    {
+        pList->pHead = pNext;
+
+        if (pNext)
+            pNext->pPrev = NULL;
+    }
+
+    if (pNext)
+        pNext->pPrev = pPrev;
+    else
+    {
+        pList->pTail = pPrev;
+
+        if (pPrev)
+            pPrev->pNext = NULL;
+    }
+
+    pEntry->pList    = NULL;
+    pEntry->pPrev    = NULL;
+    pEntry->pNext    = NULL;
+    pdmBlkCacheListSub(pList, pEntry->cbData);
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, pEntry);
+#endif
+}
+
+/**
+ * Adds a cache entry to the given LRU list unlinking it from the currently
+ * assigned list if needed.
+ *
+ * @returns nothing.
+ * @param    pList    List to the add entry to.
+ * @param    pEntry   Entry to add.
+ */
+static void pdmBlkCacheEntryAddToList(PPDMBLKLRULIST pList, PPDMBLKCACHEENTRY pEntry)
+{
+    LogFlowFunc((": Adding entry %#p to list %#p\n", pEntry, pList));
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, NULL);
+#endif
+
+    /* Remove from old list if needed */
+    if (pEntry->pList)
+        pdmBlkCacheEntryRemoveFromList(pEntry);
+
+    pEntry->pNext = pList->pHead;
+    if (pList->pHead)
+        pList->pHead->pPrev = pEntry;
+    else
+    {
+        Assert(!pList->pTail);
+        pList->pTail = pEntry;
+    }
+
+    pEntry->pPrev    = NULL;
+    pList->pHead     = pEntry;
+    pdmBlkCacheListAdd(pList, pEntry->cbData);
+    pEntry->pList    = pList;
+#ifdef PDMACFILECACHE_WITH_LRULIST_CHECKS
+    pdmBlkCacheCheckList(pList, NULL);
+#endif
+}
+
+/**
+ * Destroys a LRU list freeing all entries.
+ *
+ * @returns nothing
+ * @param   pList    Pointer to the LRU list to destroy.
+ *
+ * @note The caller must own the critical section of the cache.
+ */
+static void pdmBlkCacheDestroyList(PPDMBLKLRULIST pList)
+{
+    while (pList->pHead)
+    {
+        PPDMBLKCACHEENTRY pEntry = pList->pHead;
+
+        pList->pHead = pEntry->pNext;
+
+        AssertMsg(!(pEntry->fFlags & (PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)),
+                  ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
+
+        RTMemPageFree(pEntry->pbData, pEntry->cbData);
+        RTMemFree(pEntry);
+    }
+}
+
+/**
+ * Tries to remove the given amount of bytes from a given list in the cache
+ * moving the entries to one of the given ghosts lists
+ *
+ * @returns Amount of data which could be freed.
+ * @param    pCache           Pointer to the global cache data.
+ * @param    cbData           The amount of the data to free.
+ * @param    pListSrc         The source list to evict data from.
+ * @param    pGhostListSrc    The ghost list removed entries should be moved to
+ *                            NULL if the entry should be freed.
+ * @param    fReuseBuffer     Flag whether a buffer should be reused if it has the same size
+ * @param    ppbBuf           Where to store the address of the buffer if an entry with the
+ *                            same size was found and fReuseBuffer is true.
+ *
+ * @note    This function may return fewer bytes than requested because entries
+ *          may be marked as non evictable if they are used for I/O at the
+ *          moment.
+ */
+static size_t pdmBlkCacheEvictPagesFrom(PPDMBLKCACHEGLOBAL pCache, size_t cbData,
+                                        PPDMBLKLRULIST pListSrc, PPDMBLKLRULIST pGhostListDst,
+                                        bool fReuseBuffer, uint8_t **ppbBuffer)
+{
+    size_t cbEvicted = 0;
+
+    PDMACFILECACHE_IS_CRITSECT_OWNER(pCache);
+
+    AssertMsg(cbData > 0, ("Evicting 0 bytes not possible\n"));
+    AssertMsg(   !pGhostListDst
+              || (pGhostListDst == &pCache->LruRecentlyUsedOut),
+              ("Destination list must be NULL or the recently used but paged out list\n"));
+
+    if (fReuseBuffer)
+    {
+        AssertPtr(ppbBuffer);
+        *ppbBuffer = NULL;
+    }
+
+    /* Start deleting from the tail. */
+    PPDMBLKCACHEENTRY pEntry = pListSrc->pTail;
+
+    while ((cbEvicted < cbData) && pEntry)
+    {
+        PPDMBLKCACHEENTRY pCurr = pEntry;
+
+        pEntry = pEntry->pPrev;
+
+        /* We can't evict pages which are currently in progress or dirty but not in progress */
+        if (   !(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
+            && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
+        {
+            /* Ok eviction candidate. Grab the endpoint semaphore and check again
+             * because somebody else might have raced us. */
+            PPDMBLKCACHE pBlkCache = pCurr->pBlkCache;
+            RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+
+            if (!(pCurr->fFlags & PDMBLKCACHE_NOT_EVICTABLE)
+                && (ASMAtomicReadU32(&pCurr->cRefs) == 0))
+            {
+                LogFlow(("Evicting entry %#p (%u bytes)\n", pCurr, pCurr->cbData));
+
+                if (fReuseBuffer && (pCurr->cbData == cbData))
+                {
+                    STAM_COUNTER_INC(&pCache->StatBuffersReused);
+                    *ppbBuffer = pCurr->pbData;
+                }
+                else if (pCurr->pbData)
+                    RTMemPageFree(pCurr->pbData, pCurr->cbData);
+
+                pCurr->pbData = NULL;
+                cbEvicted += pCurr->cbData;
+
+                pdmBlkCacheEntryRemoveFromList(pCurr);
+                pdmBlkCacheSub(pCache, pCurr->cbData);
+
+                if (pGhostListDst)
+                {
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                    PPDMBLKCACHEENTRY pGhostEntFree = pGhostListDst->pTail;
+
+                    /* We have to remove the last entries from the paged out list. */
+                    while (   ((pGhostListDst->cbCached + pCurr->cbData) > pCache->cbRecentlyUsedOutMax)
+                           && pGhostEntFree)
+                    {
+                        PPDMBLKCACHEENTRY pFree = pGhostEntFree;
+                        PPDMBLKCACHE pBlkCacheFree = pFree->pBlkCache;
+
+                        pGhostEntFree = pGhostEntFree->pPrev;
+
+                        RTSemRWRequestWrite(pBlkCacheFree->SemRWEntries, RT_INDEFINITE_WAIT);
+
+                        if (ASMAtomicReadU32(&pFree->cRefs) == 0)
+                        {
+                            pdmBlkCacheEntryRemoveFromList(pFree);
+
+                            STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                            RTAvlrU64Remove(pBlkCacheFree->pTree, pFree->Core.Key);
+                            STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                            RTMemFree(pFree);
+                        }
+
+                        RTSemRWReleaseWrite(pBlkCacheFree->SemRWEntries);
+                    }
+
+                    if (pGhostListDst->cbCached + pCurr->cbData > pCache->cbRecentlyUsedOutMax)
+                    {
+                        /* Couldn't remove enough entries. Delete */
+                        STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                        RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
+                        STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                        RTMemFree(pCurr);
+                    }
+                    else
+                        pdmBlkCacheEntryAddToList(pGhostListDst, pCurr);
+                }
+                else
+                {
+                    /* Delete the entry from the AVL tree it is assigned to. */
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrU64Remove(pCurr->pBlkCache->pTree, pCurr->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                    RTMemFree(pCurr);
+                }
+            }
+
+        }
+        else
+            LogFlow(("Entry %#p (%u bytes) is still in progress and can't be evicted\n", pCurr, pCurr->cbData));
+    }
+
+    return cbEvicted;
+}
+
+static bool pdmBlkCacheReclaim(PPDMBLKCACHEGLOBAL pCache, size_t cbData, bool fReuseBuffer, uint8_t **ppbBuffer)
+{
+    size_t cbRemoved = 0;
+
+    if ((pCache->cbCached + cbData) < pCache->cbMax)
+        return true;
+    else if ((pCache->LruRecentlyUsedIn.cbCached + cbData) > pCache->cbRecentlyUsedInMax)
+    {
+        /* Try to evict as many bytes as possible from A1in */
+        cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruRecentlyUsedIn,
+                                                 &pCache->LruRecentlyUsedOut, fReuseBuffer, ppbBuffer);
+
+        /*
+         * If it was not possible to remove enough entries
+         * try the frequently accessed cache.
+         */
+        if (cbRemoved < cbData)
+        {
+            Assert(!fReuseBuffer || !*ppbBuffer); /* It is not possible that we got a buffer with the correct size but we didn't freed enough data. */
+
+            /*
+             * If we removed something we can't pass the reuse buffer flag anymore because
+             * we don't need to evict that much data
+             */
+            if (!cbRemoved)
+                cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
+                                                          NULL, fReuseBuffer, ppbBuffer);
+            else
+                cbRemoved += pdmBlkCacheEvictPagesFrom(pCache, cbData - cbRemoved, &pCache->LruFrequentlyUsed,
+                                                          NULL, false, NULL);
+        }
+    }
+    else
+    {
+        /* We have to remove entries from frequently access list. */
+        cbRemoved = pdmBlkCacheEvictPagesFrom(pCache, cbData, &pCache->LruFrequentlyUsed,
+                                                 NULL, fReuseBuffer, ppbBuffer);
+    }
+
+    LogFlowFunc((": removed %u bytes, requested %u\n", cbRemoved, cbData));
+    return (cbRemoved >= cbData);
+}
+
+DECLINLINE(int) pdmBlkCacheEnqueue(PPDMBLKCACHE pBlkCache, uint64_t off, PPDMBLKCACHEIOXFER pIoXfer)
+{
+    int rc = VINF_SUCCESS;
+
+    switch (pBlkCache->enmType)
+    {
+        case PDMBLKCACHETYPE_DEV:
+        {
+            rc = pBlkCache->u.Dev.pfnXferEnqueue(pBlkCache->u.Dev.pDevIns,
+                                                 pIoXfer->enmXferDir,
+                                                 off, pIoXfer->cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        case PDMBLKCACHETYPE_DRV:
+        {
+            rc = pBlkCache->u.Drv.pfnXferEnqueue(pBlkCache->u.Drv.pDrvIns,
+                                                 pIoXfer->enmXferDir,
+                                                 off, pIoXfer->cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        case PDMBLKCACHETYPE_USB:
+        {
+            rc = pBlkCache->u.Usb.pfnXferEnqueue(pBlkCache->u.Usb.pUsbIns,
+                                                 pIoXfer->enmXferDir,
+                                                 off, pIoXfer->cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        case PDMBLKCACHETYPE_INTERNAL:
+        {
+            rc = pBlkCache->u.Int.pfnXferEnqueue(pBlkCache->u.Int.pvUser,
+                                                 pIoXfer->enmXferDir,
+                                                 off, pIoXfer->cbXfer,
+                                                 &pIoXfer->SgBuf, pIoXfer);
+            break;
+        }
+        default:
+            AssertMsgFailed(("Unknown block cache type!\n"));
+    }
+
+    return rc;
+}
+
+/**
+ * Initiates a read I/O task for the given entry.
+ *
+ * @returns VBox status code.
+ * @param   pEntry    The entry to fetch the data to.
+ */
+static int pdmBlkCacheEntryReadFromMedium(PPDMBLKCACHEENTRY pEntry)
+{
+    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
+    LogFlowFunc((": Reading data into cache entry %#p\n", pEntry));
+
+    /* Make sure no one evicts the entry while it is accessed. */
+    pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
+
+    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
+    if (RT_UNLIKELY(!pIoXfer))
+        return VERR_NO_MEMORY;
+
+    AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
+
+    pIoXfer->fIoCache = true;
+    pIoXfer->pEntry = pEntry;
+    pIoXfer->SgSeg.pvSeg = pEntry->pbData;
+    pIoXfer->SgSeg.cbSeg = pEntry->cbData;
+    pIoXfer->cbXfer      = pEntry->cbData;
+    pIoXfer->enmXferDir  = PDMBLKCACHEXFERDIR_READ;
+    RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
+
+    return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
+}
+
+/**
+ * Initiates a write I/O task for the given entry.
+ *
+ * @returns nothing.
+ * @param    pEntry The entry to read the data from.
+ */
+static int pdmBlkCacheEntryWriteToMedium(PPDMBLKCACHEENTRY pEntry)
+{
+    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
+    LogFlowFunc((": Writing data from cache entry %#p\n", pEntry));
+
+    /* Make sure no one evicts the entry while it is accessed. */
+    pEntry->fFlags |= PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
+
+    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
+    if (RT_UNLIKELY(!pIoXfer))
+        return VERR_NO_MEMORY;
+
+    AssertMsg(pEntry->pbData, ("Entry is in ghost state\n"));
+
+    pIoXfer->fIoCache = true;
+    pIoXfer->pEntry = pEntry;
+    pIoXfer->SgSeg.pvSeg = pEntry->pbData;
+    pIoXfer->SgSeg.cbSeg = pEntry->cbData;
+    pIoXfer->cbXfer      = pEntry->cbData;
+    pIoXfer->enmXferDir  = PDMBLKCACHEXFERDIR_WRITE;
+    RTSgBufInit(&pIoXfer->SgBuf, &pIoXfer->SgSeg, 1);
+
+    return pdmBlkCacheEnqueue(pBlkCache, pEntry->Core.Key, pIoXfer);
+}
+
+/**
+ * Passthrough a part of a request directly to the I/O manager
+ * handling the endpoint.
+ *
+ * @returns VBox status code.
+ * @param   pEndpoint          The endpoint.
+ * @param   pTask              The task.
+ * @param   pIoMemCtx          The I/O memory context to use.
+ * @param   offStart           Offset to start transfer from.
+ * @param   cbData             Amount of data to transfer.
+ * @param   enmTransferType    The transfer type (read/write)
+ */
+static int pdmBlkCacheRequestPassthrough(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
+                                         PRTSGBUF pSgBuf, uint64_t offStart, size_t cbData,
+                                         PDMBLKCACHEXFERDIR enmXferDir)
+{
+
+    PPDMBLKCACHEIOXFER pIoXfer = (PPDMBLKCACHEIOXFER)RTMemAllocZ(sizeof(PDMBLKCACHEIOXFER));
+    if (RT_UNLIKELY(!pIoXfer))
+        return VERR_NO_MEMORY;
+
+    ASMAtomicIncU32(&pReq->cXfersPending);
+    pIoXfer->fIoCache    = false;
+    pIoXfer->pReq        = pReq;
+    pIoXfer->cbXfer      = cbData;
+    pIoXfer->enmXferDir  = enmXferDir;
+    if (pSgBuf)
+    {
+        RTSgBufClone(&pIoXfer->SgBuf, pSgBuf);
+        RTSgBufAdvance(pSgBuf, cbData);
+    }
+
+    return pdmBlkCacheEnqueue(pBlkCache, offStart, pIoXfer);
+}
+
+/**
+ * Commit a single dirty entry to the endpoint
+ *
+ * @returns nothing
+ * @param   pEntry    The entry to commit.
+ */
+static void pdmBlkCacheEntryCommit(PPDMBLKCACHEENTRY pEntry)
+{
+    AssertMsg(   (pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY)
+              && !(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
+              ("Invalid flags set for entry %#p\n", pEntry));
+
+    pdmBlkCacheEntryWriteToMedium(pEntry);
+}
+
+/**
+ * Commit all dirty entries for a single endpoint.
+ *
+ * @returns nothing.
+ * @param   pBlkCache    The endpoint cache to commit.
+ */
+static void pdmBlkCacheCommit(PPDMBLKCACHE pBlkCache)
+{
+    uint32_t cbCommitted = 0;
+
+    /* Return if the cache was suspended. */
+    if (pBlkCache->fSuspended)
+        return;
+
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+
+    /* The list is moved to a new header to reduce locking overhead. */
+    RTLISTNODE ListDirtyNotCommitted;
+    RTSPINLOCKTMP Tmp;
+
+    RTListInit(&ListDirtyNotCommitted);
+    RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
+    RTListMove(&ListDirtyNotCommitted, &pBlkCache->ListDirtyNotCommitted);
+    RTSpinlockRelease(pBlkCache->LockList, &Tmp);
+
+    if (!RTListIsEmpty(&ListDirtyNotCommitted))
+    {
+        PPDMBLKCACHEENTRY pEntry = RTListGetFirst(&ListDirtyNotCommitted, PDMBLKCACHEENTRY, NodeNotCommitted);
+
+        while (!RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted))
+        {
+            PPDMBLKCACHEENTRY pNext = RTListNodeGetNext(&pEntry->NodeNotCommitted, PDMBLKCACHEENTRY,
+                                                        NodeNotCommitted);
+            pdmBlkCacheEntryCommit(pEntry);
+            cbCommitted += pEntry->cbData;
+            RTListNodeRemove(&pEntry->NodeNotCommitted);
+            pEntry = pNext;
+        }
+
+        /* Commit the last endpoint */
+        Assert(RTListNodeIsLast(&ListDirtyNotCommitted, &pEntry->NodeNotCommitted));
+        pdmBlkCacheEntryCommit(pEntry);
+        RTListNodeRemove(&pEntry->NodeNotCommitted);
+        AssertMsg(RTListIsEmpty(&ListDirtyNotCommitted),
+                  ("Committed all entries but list is not empty\n"));
+    }
+
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+    AssertMsg(pBlkCache->pCache->cbDirty >= cbCommitted,
+              ("Number of committed bytes exceeds number of dirty bytes\n"));
+    uint32_t cbDirtyOld = ASMAtomicSubU32(&pBlkCache->pCache->cbDirty, cbCommitted);
+
+    /* Reset the commit timer if we don't have any dirty bits. */
+    if (   !(cbDirtyOld - cbCommitted)
+        && pBlkCache->pCache->u32CommitTimeoutMs != 0)
+        TMTimerStop(pBlkCache->pCache->pTimerCommit);
+}
+
+/**
+ * Commit all dirty entries in the cache.
+ *
+ * @returns nothing.
+ * @param   pCache    The global cache instance.
+ */
+static void pdmBlkCacheCommitDirtyEntries(PPDMBLKCACHEGLOBAL pCache)
+{
+    bool fCommitInProgress = ASMAtomicXchgBool(&pCache->fCommitInProgress, true);
+
+    if (!fCommitInProgress)
+    {
+        pdmBlkCacheLockEnter(pCache);
+        Assert(!RTListIsEmpty(&pCache->ListUsers));
+
+        PPDMBLKCACHE pBlkCache = RTListGetFirst(&pCache->ListUsers, PDMBLKCACHE, NodeCacheUser);
+        AssertPtr(pBlkCache);
+
+        while (!RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser))
+        {
+            pdmBlkCacheCommit(pBlkCache);
+
+            pBlkCache = RTListNodeGetNext(&pBlkCache->NodeCacheUser, PDMBLKCACHE,
+                                          NodeCacheUser);
+        }
+
+        /* Commit the last endpoint */
+        Assert(RTListNodeIsLast(&pCache->ListUsers, &pBlkCache->NodeCacheUser));
+        pdmBlkCacheCommit(pBlkCache);
+
+        pdmBlkCacheLockLeave(pCache);
+        ASMAtomicWriteBool(&pCache->fCommitInProgress, false);
+    }
+}
+
+/**
+ * Adds the given entry as a dirty to the cache.
+ *
+ * @returns Flag whether the amount of dirty bytes in the cache exceeds the threshold
+ * @param   pBlkCache    The endpoint cache the entry belongs to.
+ * @param   pEntry       The entry to add.
+ */
+static bool pdmBlkCacheAddDirtyEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
+{
+    bool fDirtyBytesExceeded = false;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+
+    /* If the commit timer is disabled we commit right away. */
+    if (pCache->u32CommitTimeoutMs == 0)
+    {
+        pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
+        pdmBlkCacheEntryCommit(pEntry);
+    }
+    else if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY))
+    {
+        pEntry->fFlags |= PDMBLKCACHE_ENTRY_IS_DIRTY;
+
+        RTSPINLOCKTMP Tmp;
+        RTSpinlockAcquire(pBlkCache->LockList, &Tmp);
+        RTListAppend(&pBlkCache->ListDirtyNotCommitted, &pEntry->NodeNotCommitted);
+        RTSpinlockRelease(pBlkCache->LockList, &Tmp);
+
+        uint32_t cbDirty = ASMAtomicAddU32(&pCache->cbDirty, pEntry->cbData);
+
+        /* Prevent committing if the VM was suspended. */
+        if (RT_LIKELY(!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended)))
+            fDirtyBytesExceeded = (cbDirty + pEntry->cbData >= pCache->cbCommitDirtyThreshold);
+        else if (!cbDirty && pCache->u32CommitTimeoutMs > 0)
+        {
+            /* Arm the commit timer. */
+            TMTimerSetMillies(pCache->pTimerCommit, pCache->u32CommitTimeoutMs);
+        }
+    }
+
+    return fDirtyBytesExceeded;
+}
+
+static PPDMBLKCACHE pdmR3BlkCacheFindById(PPDMBLKCACHEGLOBAL pBlkCacheGlobal, const char *pcszId)
+{
+    bool fFound = false;
+    PPDMBLKCACHE pBlkCache = NULL;
+
+    RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (!RTStrCmp(pBlkCache->pszId, pcszId))
+        {
+            fFound = true;
+            break;
+        }
+    }
+
+    return fFound ?  pBlkCache : NULL;
+}
+
+/**
+ * Commit timer callback.
+ */
+static void pdmBlkCacheCommitTimerCallback(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
+
+    LogFlowFunc(("Commit interval expired, commiting dirty entries\n"));
+
+    if (   ASMAtomicReadU32(&pCache->cbDirty) > 0
+        && !ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
+        pdmBlkCacheCommitDirtyEntries(pCache);
+
+    LogFlowFunc(("Entries committed, going to sleep\n"));
+}
+
+static DECLCALLBACK(int) pdmR3BlkCacheSaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    AssertPtr(pBlkCacheGlobal);
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    SSMR3PutU32(pSSM, pBlkCacheGlobal->cRefs);
+
+    /* Go through the list and save all dirty entries. */
+    PPDMBLKCACHE pBlkCache;
+    RTListForEach(&pBlkCacheGlobal->ListUsers, pBlkCache, PDMBLKCACHE, NodeCacheUser)
+    {
+        uint32_t cEntries = 0;
+        PPDMBLKCACHEENTRY pEntry;
+
+        RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+        SSMR3PutU32(pSSM, strlen(pBlkCache->pszId));
+        SSMR3PutStrZ(pSSM, pBlkCache->pszId);
+
+        /* Count the number of entries to safe. */
+        RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
+        {
+            cEntries++;
+        }
+
+        SSMR3PutU32(pSSM, cEntries);
+
+        /* Walk the list of all dirty entries and save them. */
+        RTListForEach(&pBlkCache->ListDirtyNotCommitted, pEntry, PDMBLKCACHEENTRY, NodeNotCommitted)
+        {
+            /* A few sanity checks. */
+            AssertMsg(!pEntry->cRefs, ("The entry is still referenced\n"));
+            AssertMsg(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY, ("Entry is not dirty\n"));
+            AssertMsg(!(pEntry->fFlags & ~PDMBLKCACHE_ENTRY_IS_DIRTY), ("Invalid flags set\n"));
+            AssertMsg(!pEntry->pWaitingHead && !pEntry->pWaitingTail, ("There are waiting requests\n"));
+            AssertMsg(   pEntry->pList == &pBlkCacheGlobal->LruRecentlyUsedIn
+                      || pEntry->pList == &pBlkCacheGlobal->LruFrequentlyUsed,
+                      ("Invalid list\n"));
+            AssertMsg(pEntry->cbData == pEntry->Core.KeyLast - pEntry->Core.Key + 1,
+                      ("Size and range do not match\n"));
+
+            /* Save */
+            SSMR3PutU64(pSSM, pEntry->Core.Key);
+            SSMR3PutU32(pSSM, pEntry->cbData);
+            SSMR3PutMem(pSSM, pEntry->pbData, pEntry->cbData);
+        }
+
+        RTSemRWReleaseRead(pBlkCache->SemRWEntries);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+    /* Terminator */
+    return SSMR3PutU32(pSSM, UINT32_MAX);
+}
+
+static DECLCALLBACK(int) pdmR3BlkCacheLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int rc = VINF_SUCCESS;
+    uint32_t cRefs;
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    AssertPtr(pBlkCacheGlobal);
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    if (uVersion != PDM_BLK_CACHE_SAVED_STATE_VERSION)
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+
+    SSMR3GetU32(pSSM, &cRefs);
+
+    if (cRefs == pBlkCacheGlobal->cRefs)
+    {
+        char *pszId = NULL;
+
+        while (   cRefs > 0
+               && RT_SUCCESS(rc))
+        {
+            PPDMBLKCACHE pBlkCache = NULL;
+            uint32_t cbId = 0;
+
+            SSMR3GetU32(pSSM, &cbId);
+            Assert(cbId > 0);
+
+            cbId++; /* Include terminator */
+            pszId = (char *)RTMemAllocZ(cbId * sizeof(char));
+            if (!pszId)
+            {
+                rc = VERR_NO_MEMORY;
+                break;
+            }
+
+            rc = SSMR3GetStrZ(pSSM, pszId, cbId);
+            AssertRC(rc);
+
+            /* Search for the block cache with the provided id. */
+            pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pszId);
+            if (!pBlkCache)
+            {
+                rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                      N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
+                break;
+            }
+
+            RTStrFree(pszId);
+            pszId = NULL;
+
+            /* Get the entries */
+            uint32_t cEntries;
+            SSMR3GetU32(pSSM, &cEntries);
+
+            while (cEntries > 0)
+            {
+                PPDMBLKCACHEENTRY pEntry;
+                uint64_t off;
+                uint32_t cbEntry;
+
+                SSMR3GetU64(pSSM, &off);
+                SSMR3GetU32(pSSM, &cbEntry);
+
+                pEntry = pdmBlkCacheEntryAlloc(pBlkCache, off, cbEntry, NULL);
+                if (!pEntry)
+                {
+                    rc = VERR_NO_MEMORY;
+                    break;
+                }
+
+                rc = SSMR3GetMem(pSSM, pEntry->pbData, cbEntry);
+                if (RT_FAILURE(rc))
+                {
+                    RTMemFree(pEntry->pbData);
+                    RTMemFree(pEntry);
+                    break;
+                }
+
+                /* Insert into the tree. */
+                bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
+                Assert(fInserted);
+
+                /* Add to the dirty list. */
+                pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
+                pdmBlkCacheEntryAddToList(&pBlkCacheGlobal->LruRecentlyUsedIn, pEntry);
+                pdmBlkCacheAdd(pBlkCacheGlobal, cbEntry);
+                pdmBlkCacheEntryRelease(pEntry);
+                cEntries--;
+            }
+
+            cRefs--;
+        }
+
+        if (pszId)
+            RTStrFree(pszId);
+    }
+    else
+        rc = SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                              N_("The VM is missing a block device. Please make sure the source and target VMs have compatible storage configurations"));
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t u32 = 0;
+        rc = SSMR3GetU32(pSSM, &u32);
+        if (RT_SUCCESS(rc))
+            AssertMsgReturn(u32 == UINT32_MAX, ("%#x\n", u32), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+    }
+
+    return rc;
+}
+
+int pdmR3BlkCacheInit(PVM pVM)
+{
+    int  rc   = VINF_SUCCESS;
+    PUVM pUVM = pVM->pUVM;
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal;
+
+    LogFlowFunc((": pVM=%p\n", pVM));
+
+    VM_ASSERT_EMT(pVM);
+
+    PCFGMNODE pCfgRoot     = CFGMR3GetRoot(pVM);
+    PCFGMNODE pCfgBlkCache = CFGMR3GetChild(CFGMR3GetChild(pCfgRoot, "PDM"), "BlkCache");
+
+    pBlkCacheGlobal = (PPDMBLKCACHEGLOBAL)RTMemAllocZ(sizeof(PDMBLKCACHEGLOBAL));
+    if (!pBlkCacheGlobal)
+        return VERR_NO_MEMORY;
+
+    RTListInit(&pBlkCacheGlobal->ListUsers);
+    pBlkCacheGlobal->pVM = pVM;
+    pBlkCacheGlobal->cRefs = 0;
+    pBlkCacheGlobal->cbCached  = 0;
+    pBlkCacheGlobal->fCommitInProgress = false;
+
+    /* Initialize members */
+    pBlkCacheGlobal->LruRecentlyUsedIn.pHead    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedIn.pTail    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedIn.cbCached = 0;
+
+    pBlkCacheGlobal->LruRecentlyUsedOut.pHead    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedOut.pTail    = NULL;
+    pBlkCacheGlobal->LruRecentlyUsedOut.cbCached = 0;
+
+    pBlkCacheGlobal->LruFrequentlyUsed.pHead    = NULL;
+    pBlkCacheGlobal->LruFrequentlyUsed.pTail    = NULL;
+    pBlkCacheGlobal->LruFrequentlyUsed.cbCached = 0;
+
+    do
+    {
+        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheSize", &pBlkCacheGlobal->cbMax, 5 * _1M);
+        AssertLogRelRCBreak(rc);
+        LogFlowFunc(("Maximum number of bytes cached %u\n", pBlkCacheGlobal->cbMax));
+
+        pBlkCacheGlobal->cbRecentlyUsedInMax  = (pBlkCacheGlobal->cbMax / 100) * 25; /* 25% of the buffer size */
+        pBlkCacheGlobal->cbRecentlyUsedOutMax = (pBlkCacheGlobal->cbMax / 100) * 50; /* 50% of the buffer size */
+        LogFlowFunc(("cbRecentlyUsedInMax=%u cbRecentlyUsedOutMax=%u\n",
+                     pBlkCacheGlobal->cbRecentlyUsedInMax, pBlkCacheGlobal->cbRecentlyUsedOutMax));
+
+        /** @todo r=aeichner: Experiment to find optimal default values */
+        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitIntervalMs", &pBlkCacheGlobal->u32CommitTimeoutMs, 10000 /* 10sec */);
+        AssertLogRelRCBreak(rc);
+        rc = CFGMR3QueryU32Def(pCfgBlkCache, "CacheCommitThreshold", &pBlkCacheGlobal->cbCommitDirtyThreshold, pBlkCacheGlobal->cbMax / 2);
+        AssertLogRelRCBreak(rc);
+    } while (0);
+
+    if (RT_SUCCESS(rc))
+    {
+        STAMR3Register(pVM, &pBlkCacheGlobal->cbMax,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbMax",
+                       STAMUNIT_BYTES,
+                       "Maximum cache size");
+        STAMR3Register(pVM, &pBlkCacheGlobal->cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCached",
+                       STAMUNIT_BYTES,
+                       "Currently used cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedIn.cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCachedMruIn",
+                       STAMUNIT_BYTES,
+                       "Number of bytes cached in MRU list");
+        STAMR3Register(pVM, &pBlkCacheGlobal->LruRecentlyUsedOut.cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCachedMruOut",
+                       STAMUNIT_BYTES,
+                       "Number of bytes cached in FRU list");
+        STAMR3Register(pVM, &pBlkCacheGlobal->LruFrequentlyUsed.cbCached,
+                       STAMTYPE_U32, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/cbCachedFru",
+                       STAMUNIT_BYTES,
+                       "Number of bytes cached in FRU ghost list");
+
+#ifdef VBOX_WITH_STATISTICS
+        STAMR3Register(pVM, &pBlkCacheGlobal->cHits,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheHits",
+                       STAMUNIT_COUNT, "Number of hits in the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->cPartialHits,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CachePartialHits",
+                       STAMUNIT_COUNT, "Number of partial hits in the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->cMisses,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheMisses",
+                       STAMUNIT_COUNT, "Number of misses when accessing the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatRead,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheRead",
+                       STAMUNIT_BYTES, "Number of bytes read from the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatWritten,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheWritten",
+                       STAMUNIT_BYTES, "Number of bytes written to the cache");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeGet,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheTreeGet",
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to access an entry in the tree");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeInsert,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheTreeInsert",
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to insert an entry in the tree");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatTreeRemove,
+                       STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheTreeRemove",
+                       STAMUNIT_TICKS_PER_CALL, "Time taken to remove an entry an the tree");
+        STAMR3Register(pVM, &pBlkCacheGlobal->StatBuffersReused,
+                       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                       "/PDM/BlkCache/CacheBuffersReused",
+                       STAMUNIT_COUNT, "Number of times a buffer could be reused");
+#endif
+
+        /* Initialize the critical section */
+        rc = RTCritSectInit(&pBlkCacheGlobal->CritSect);
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        /* Create the commit timer */
+        if (pBlkCacheGlobal->u32CommitTimeoutMs > 0)
+            rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL,
+                                         pdmBlkCacheCommitTimerCallback,
+                                         pBlkCacheGlobal,
+                                         "BlkCache-Commit",
+                                         &pBlkCacheGlobal->pTimerCommit);
+
+        if (RT_SUCCESS(rc))
+        {
+            /* Register saved state handler. */
+            rc = SSMR3RegisterInternal(pVM, "pdmblkcache", 0, PDM_BLK_CACHE_SAVED_STATE_VERSION, pBlkCacheGlobal->cbMax,
+                                       NULL, NULL, NULL,
+                                       NULL, pdmR3BlkCacheSaveExec, NULL,
+                                       NULL, pdmR3BlkCacheLoadExec, NULL);
+            if (RT_SUCCESS(rc))
+            {
+                LogRel(("BlkCache: Cache successfully initialised. Cache size is %u bytes\n", pBlkCacheGlobal->cbMax));
+                LogRel(("BlkCache: Cache commit interval is %u ms\n", pBlkCacheGlobal->u32CommitTimeoutMs));
+                LogRel(("BlkCache: Cache commit threshold is %u bytes\n", pBlkCacheGlobal->cbCommitDirtyThreshold));
+                pUVM->pdm.s.pBlkCacheGlobal = pBlkCacheGlobal;
+                return VINF_SUCCESS;
+            }
+        }
+
+        RTCritSectDelete(&pBlkCacheGlobal->CritSect);
+    }
+
+    if (pBlkCacheGlobal)
+        RTMemFree(pBlkCacheGlobal);
+
+    LogFlowFunc((": returns rc=%Rrc\n", pVM, rc));
+    return rc;
+}
+
+void pdmR3BlkCacheTerm(PVM pVM)
+{
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    if (pBlkCacheGlobal)
+    {
+        /* Make sure no one else uses the cache now */
+        pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+        /* Cleanup deleting all cache entries waiting for in progress entries to finish. */
+        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedIn);
+        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruRecentlyUsedOut);
+        pdmBlkCacheDestroyList(&pBlkCacheGlobal->LruFrequentlyUsed);
+
+        pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+        RTCritSectDelete(&pBlkCacheGlobal->CritSect);
+        RTMemFree(pBlkCacheGlobal);
+        pVM->pUVM->pdm.s.pBlkCacheGlobal = NULL;
+    }
+}
+
+int pdmR3BlkCacheResume(PVM pVM)
+{
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    LogFlowFunc(("pVM=%#p\n", pVM));
+
+    if (   pBlkCacheGlobal
+        && ASMAtomicXchgBool(&pBlkCacheGlobal->fIoErrorVmSuspended, false))
+    {
+        /* The VM was suspended because of an I/O error, commit all dirty entries. */
+        pdmBlkCacheCommitDirtyEntries(pBlkCacheGlobal);
+    }
+
+    return VINF_SUCCESS;
+}
+
+static int pdmR3BlkCacheRetain(PVM pVM, PPPDMBLKCACHE ppBlkCache, const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache = NULL;
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+
+    if (!pBlkCacheGlobal)
+        return VERR_NOT_SUPPORTED;
+
+    /*
+     * Check that no other user cache has the same id first,
+     * Unique id's are necessary in case the state is saved.
+     */
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    pBlkCache = pdmR3BlkCacheFindById(pBlkCacheGlobal, pcszId);
+
+    if (!pBlkCache)
+    {
+        pBlkCache = (PPDMBLKCACHE)RTMemAllocZ(sizeof(PDMBLKCACHE));
+
+        if (pBlkCache)
+            pBlkCache->pszId = RTStrDup(pcszId);
+
+        if (   pBlkCache
+            && pBlkCache->pszId)
+        {
+            pBlkCache->fSuspended = false;
+            pBlkCache->pCache = pBlkCacheGlobal;
+            RTListInit(&pBlkCache->ListDirtyNotCommitted);
+
+            rc = RTSpinlockCreate(&pBlkCache->LockList);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTSemRWCreate(&pBlkCache->SemRWEntries);
+                if (RT_SUCCESS(rc))
+                {
+                    pBlkCache->pTree  = (PAVLRU64TREE)RTMemAllocZ(sizeof(AVLRFOFFTREE));
+                    if (pBlkCache->pTree)
+                    {
+#ifdef VBOX_WITH_STATISTICS
+                        STAMR3RegisterF(pBlkCacheGlobal->pVM, &pBlkCache->StatWriteDeferred,
+                                        STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,
+                                        STAMUNIT_COUNT, "Number of deferred writes",
+                                        "/PDM/BlkCache/%s/Cache/DeferredWrites", pBlkCache->pszId);
+#endif
+
+                        /* Add to the list of users. */
+                        pBlkCacheGlobal->cRefs++;
+                        RTListAppend(&pBlkCacheGlobal->ListUsers, &pBlkCache->NodeCacheUser);
+                        pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+                        *ppBlkCache = pBlkCache;
+                        LogFlowFunc(("returns success\n"));
+                        return VINF_SUCCESS;
+                    }
+                    else
+                        rc = VERR_NO_MEMORY;
+
+                    RTSemRWDestroy(pBlkCache->SemRWEntries);
+                }
+
+                RTSpinlockDestroy(pBlkCache->LockList);
+            }
+
+            RTStrFree(pBlkCache->pszId);
+        }
+        else
+            rc = VERR_NO_MEMORY;
+
+        if (pBlkCache)
+            RTMemFree(pBlkCache);
+    }
+    else
+        rc = VERR_ALREADY_EXISTS;
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
+                                         PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
+                                         PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
+                                         const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType = PDMBLKCACHETYPE_DRV;
+        pBlkCache->u.Drv.pfnXferComplete = pfnXferComplete;
+        pBlkCache->u.Drv.pfnXferEnqueue  = pfnXferEnqueue;
+        pBlkCache->u.Drv.pDrvIns         = pDrvIns;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMBLKCACHE ppBlkCache,
+                                         PFNPDMBLKCACHEXFERCOMPLETEDEV pfnXferComplete,
+                                         PFNPDMBLKCACHEXFERENQUEUEDEV pfnXferEnqueue,
+                                         const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType = PDMBLKCACHETYPE_DEV;
+        pBlkCache->u.Dev.pfnXferComplete = pfnXferComplete;
+        pBlkCache->u.Dev.pfnXferEnqueue  = pfnXferEnqueue;
+        pBlkCache->u.Dev.pDevIns         = pDevIns;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMBLKCACHE ppBlkCache,
+                                      PFNPDMBLKCACHEXFERCOMPLETEUSB pfnXferComplete,
+                                      PFNPDMBLKCACHEXFERENQUEUEUSB pfnXferEnqueue,
+                                      const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType = PDMBLKCACHETYPE_USB;
+        pBlkCache->u.Usb.pfnXferComplete = pfnXferComplete;
+        pBlkCache->u.Usb.pfnXferEnqueue  = pfnXferEnqueue;
+        pBlkCache->u.Usb.pUsbIns         = pUsbIns;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRetainInt(PVM pVM, void *pvUser, PPPDMBLKCACHE ppBlkCache,
+                                      PFNPDMBLKCACHEXFERCOMPLETEINT pfnXferComplete,
+                                      PFNPDMBLKCACHEXFERENQUEUEINT pfnXferEnqueue,
+                                      const char *pcszId)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHE pBlkCache;
+
+    rc = pdmR3BlkCacheRetain(pVM, &pBlkCache, pcszId);
+    if (RT_SUCCESS(rc))
+    {
+        pBlkCache->enmType = PDMBLKCACHETYPE_INTERNAL;
+        pBlkCache->u.Int.pfnXferComplete = pfnXferComplete;
+        pBlkCache->u.Int.pfnXferEnqueue  = pfnXferEnqueue;
+        pBlkCache->u.Int.pvUser          = pvUser;
+        *ppBlkCache = pBlkCache;
+    }
+
+    LogFlowFunc(("Leave rc=%Rrc\n", rc));
+    return rc;
+
+}
+
+/**
+ * Callback for the AVL destroy routine. Frees a cache entry for this endpoint.
+ *
+ * @returns IPRT status code.
+ * @param    pNode     The node to destroy.
+ * @param    pvUser    Opaque user data.
+ */
+static int pdmBlkCacheEntryDestroy(PAVLRU64NODECORE pNode, void *pvUser)
+{
+    PPDMBLKCACHEENTRY  pEntry = (PPDMBLKCACHEENTRY)pNode;
+    PPDMBLKCACHEGLOBAL pCache = (PPDMBLKCACHEGLOBAL)pvUser;
+    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
+
+    while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
+    {
+        /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
+        pdmBlkCacheEntryRef(pEntry);
+        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+        pdmBlkCacheLockLeave(pCache);
+
+        RTThreadSleep(250);
+
+        /* Re-enter all locks */
+        pdmBlkCacheLockEnter(pCache);
+        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+        pdmBlkCacheEntryRelease(pEntry);
+    }
+
+    AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
+                ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
+
+    bool fUpdateCache =    pEntry->pList == &pCache->LruFrequentlyUsed
+                        || pEntry->pList == &pCache->LruRecentlyUsedIn;
+
+    pdmBlkCacheEntryRemoveFromList(pEntry);
+
+    if (fUpdateCache)
+        pdmBlkCacheSub(pCache, pEntry->cbData);
+
+    RTMemPageFree(pEntry->pbData, pEntry->cbData);
+    RTMemFree(pEntry);
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Destroys all cache resources used by the given endpoint.
+ *
+ * @returns nothing.
+ * @param    pEndpoint    The endpoint to the destroy.
+ */
+VMMR3DECL(void) PDMR3BlkCacheRelease(PPDMBLKCACHE pBlkCache)
+{
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+
+    /*
+     * Commit all dirty entries now (they are waited on for completion during the
+     * destruction of the AVL tree below).
+     * The exception is if the VM was paused because of an I/O error before.
+     */
+    if (!ASMAtomicReadBool(&pCache->fIoErrorVmSuspended))
+        pdmBlkCacheCommit(pBlkCache);
+
+    /* Make sure nobody is accessing the cache while we delete the tree. */
+    pdmBlkCacheLockEnter(pCache);
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    RTAvlrU64Destroy(pBlkCache->pTree, pdmBlkCacheEntryDestroy, pCache);
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+    RTSpinlockDestroy(pBlkCache->LockList);
+
+    pCache->cRefs--;
+    RTListNodeRemove(&pBlkCache->NodeCacheUser);
+
+    pdmBlkCacheLockLeave(pCache);
+
+    RTSemRWDestroy(pBlkCache->SemRWEntries);
+
+#ifdef VBOX_WITH_STATISTICS
+    STAMR3Deregister(pCache->pVM, &pBlkCache->StatWriteDeferred);
+#endif
+
+    RTStrFree(pBlkCache->pszId);
+    RTMemFree(pBlkCache);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheReleaseDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("%s: pDevIns=%p\n", __FUNCTION__, pDevIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+        return;
+    VM_ASSERT_EMT(pVM);
+
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
+
+    /* Return silently if not supported. */
+    if (!pBlkCacheGlobal)
+        return;
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (    pBlkCache->enmType == PDMBLKCACHETYPE_DEV
+            &&  pBlkCache->u.Dev.pDevIns == pDevIns)
+            PDMR3BlkCacheRelease(pBlkCache);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheReleaseDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("%s: pDrvIns=%p\n", __FUNCTION__, pDrvIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+        return;
+    VM_ASSERT_EMT(pVM);
+
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
+
+    /* Return silently if not supported. */
+    if (!pBlkCacheGlobal)
+        return;
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (    pBlkCache->enmType == PDMBLKCACHETYPE_DRV
+            &&  pBlkCache->u.Drv.pDrvIns == pDrvIns)
+            PDMR3BlkCacheRelease(pBlkCache);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheReleaseUsb(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    LogFlow(("%s: pUsbIns=%p\n", __FUNCTION__, pUsbIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pUsbIns)
+        return;
+    VM_ASSERT_EMT(pVM);
+
+    PPDMBLKCACHEGLOBAL pBlkCacheGlobal = pVM->pUVM->pdm.s.pBlkCacheGlobal;
+    PPDMBLKCACHE pBlkCache, pBlkCacheNext;
+
+    /* Return silently if not supported. */
+    if (!pBlkCacheGlobal)
+        return;
+
+    pdmBlkCacheLockEnter(pBlkCacheGlobal);
+
+    RTListForEachSafe(&pBlkCacheGlobal->ListUsers, pBlkCache, pBlkCacheNext, PDMBLKCACHE, NodeCacheUser)
+    {
+        if (    pBlkCache->enmType == PDMBLKCACHETYPE_USB
+            &&  pBlkCache->u.Usb.pUsbIns == pUsbIns)
+            PDMR3BlkCacheRelease(pBlkCache);
+    }
+
+    pdmBlkCacheLockLeave(pBlkCacheGlobal);
+}
+
+static PPDMBLKCACHEENTRY pdmBlkCacheGetCacheEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off)
+{
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    PPDMBLKCACHEENTRY pEntry = NULL;
+
+    STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
+
+    RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    pEntry = (PPDMBLKCACHEENTRY)RTAvlrU64RangeGet(pBlkCache->pTree, off);
+    if (pEntry)
+        pdmBlkCacheEntryRef(pEntry);
+    RTSemRWReleaseRead(pBlkCache->SemRWEntries);
+
+    STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+
+    return pEntry;
+}
+
+/**
+ * Return the best fit cache entries for the given offset.
+ *
+ * @returns nothing.
+ * @param   pBlkCache    The endpoint cache.
+ * @param   off               The offset.
+ * @param   pEntryAbove       Where to store the pointer to the best fit entry above the
+ *                            the given offset. NULL if not required.
+ */
+static void pdmBlkCacheGetCacheBestFitEntryByOffset(PPDMBLKCACHE pBlkCache, uint64_t off,
+                                                    PPDMBLKCACHEENTRY *ppEntryAbove)
+{
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+
+    STAM_PROFILE_ADV_START(&pCache->StatTreeGet, Cache);
+
+    RTSemRWRequestRead(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    if (ppEntryAbove)
+    {
+        *ppEntryAbove = (PPDMBLKCACHEENTRY)RTAvlrU64GetBestFit(pBlkCache->pTree, off, true /*fAbove*/);
+        if (*ppEntryAbove)
+            pdmBlkCacheEntryRef(*ppEntryAbove);
+    }
+
+    RTSemRWReleaseRead(pBlkCache->SemRWEntries);
+
+    STAM_PROFILE_ADV_STOP(&pCache->StatTreeGet, Cache);
+}
+
+static void pdmBlkCacheInsertEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEENTRY pEntry)
+{
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+
+    STAM_PROFILE_ADV_START(&pCache->StatTreeInsert, Cache);
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    bool fInserted = RTAvlrU64Insert(pBlkCache->pTree, &pEntry->Core);
+    AssertMsg(fInserted, ("Node was not inserted into tree\n"));
+    STAM_PROFILE_ADV_STOP(&pCache->StatTreeInsert, Cache);
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+}
+
+/**
+ * Allocates and initializes a new entry for the cache.
+ * The entry has a reference count of 1.
+ *
+ * @returns Pointer to the new cache entry or NULL if out of memory.
+ * @param   pBlkCache The cache the entry belongs to.
+ * @param   off       Start offset.
+ * @param   cbData    Size of the cache entry.
+ * @param   pbBuffer  Pointer to the buffer to use.
+ *                    NULL if a new buffer should be allocated.
+ *                    The buffer needs to have the same size of the entry.
+ */
+static PPDMBLKCACHEENTRY pdmBlkCacheEntryAlloc(PPDMBLKCACHE pBlkCache,
+                                               uint64_t off, size_t cbData, uint8_t *pbBuffer)
+{
+    PPDMBLKCACHEENTRY pEntryNew = (PPDMBLKCACHEENTRY)RTMemAllocZ(sizeof(PDMBLKCACHEENTRY));
+
+    if (RT_UNLIKELY(!pEntryNew))
+        return NULL;
+
+    pEntryNew->Core.Key      = off;
+    pEntryNew->Core.KeyLast  = off + cbData - 1;
+    pEntryNew->pBlkCache     = pBlkCache;
+    pEntryNew->fFlags        = 0;
+    pEntryNew->cRefs         = 1; /* We are using it now. */
+    pEntryNew->pList         = NULL;
+    pEntryNew->cbData        = cbData;
+    pEntryNew->pWaitingHead  = NULL;
+    pEntryNew->pWaitingTail  = NULL;
+    if (pbBuffer)
+        pEntryNew->pbData    = pbBuffer;
+    else
+        pEntryNew->pbData    = (uint8_t *)RTMemPageAlloc(cbData);
+
+    if (RT_UNLIKELY(!pEntryNew->pbData))
+    {
+        RTMemFree(pEntryNew);
+        return NULL;
+    }
+
+    return pEntryNew;
+}
+
+/**
+ * Checks that a set of flags is set/clear acquiring the R/W semaphore
+ * in exclusive mode.
+ *
+ * @returns true if the flag in fSet is set and the one in fClear is clear.
+ *          false otherwise.
+ *          The R/W semaphore is only held if true is returned.
+ *
+ * @param   pBlkCache   The endpoint cache instance data.
+ * @param   pEntry           The entry to check the flags for.
+ * @param   fSet             The flag which is tested to be set.
+ * @param   fClear           The flag which is tested to be clear.
+ */
+DECLINLINE(bool) pdmBlkCacheEntryFlagIsSetClearAcquireLock(PPDMBLKCACHE pBlkCache,
+                                                           PPDMBLKCACHEENTRY pEntry,
+                                                           uint32_t fSet, uint32_t fClear)
+{
+    uint32_t fFlags = ASMAtomicReadU32(&pEntry->fFlags);
+    bool fPassed = ((fFlags & fSet) && !(fFlags & fClear));
+
+    if (fPassed)
+    {
+        /* Acquire the lock and check again because the completion callback might have raced us. */
+        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+
+        fFlags = ASMAtomicReadU32(&pEntry->fFlags);
+        fPassed = ((fFlags & fSet) && !(fFlags & fClear));
+
+        /* Drop the lock if we didn't passed the test. */
+        if (!fPassed)
+            RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+    }
+
+    return fPassed;
+}
+
+/**
+ * Adds a segment to the waiting list for a cache entry
+ * which is currently in progress.
+ *
+ * @returns nothing.
+ * @param   pEntry    The cache entry to add the segment to.
+ * @param   pSeg      The segment to add.
+ */
+DECLINLINE(void) pdmBlkCacheEntryAddWaiter(PPDMBLKCACHEENTRY pEntry,
+                                           PPDMBLKCACHEWAITER pWaiter)
+{
+    pWaiter->pNext = NULL;
+
+    if (pEntry->pWaitingHead)
+    {
+        AssertPtr(pEntry->pWaitingTail);
+
+        pEntry->pWaitingTail->pNext = pWaiter;
+        pEntry->pWaitingTail = pWaiter;
+    }
+    else
+    {
+        Assert(!pEntry->pWaitingTail);
+
+        pEntry->pWaitingHead = pWaiter;
+        pEntry->pWaitingTail = pWaiter;
+    }
+}
+
+/**
+ * Add a buffer described by the I/O memory context
+ * to the entry waiting for completion.
+ *
+ * @returns VBox status code.
+ * @param   pEntry    The entry to add the buffer to.
+ * @param   pTask     Task associated with the buffer.
+ * @param   pIoMemCtx The memory context to use.
+ * @param   offDiff   Offset from the start of the buffer
+ *                    in the entry.
+ * @param   cbData    Amount of data to wait for onthis entry.
+ * @param   fWrite    Flag whether the task waits because it wants to write
+ *                    to the cache entry.
+ */
+static int pdmBlkCacheEntryWaitersAdd(PPDMBLKCACHEENTRY pEntry,
+                                      PPDMBLKCACHEREQ pReq,
+                                      PRTSGBUF pSgBuf, uint64_t offDiff,
+                                      size_t cbData, bool fWrite)
+{
+    PPDMBLKCACHEWAITER pWaiter  = (PPDMBLKCACHEWAITER)RTMemAllocZ(sizeof(PDMBLKCACHEWAITER));
+    if (!pWaiter)
+        return VERR_NO_MEMORY;
+
+    ASMAtomicIncU32(&pReq->cXfersPending);
+    pWaiter->pReq          = pReq;
+    pWaiter->offCacheEntry = offDiff;
+    pWaiter->cbTransfer    = cbData;
+    pWaiter->fWrite        = fWrite;
+    RTSgBufClone(&pWaiter->SgBuf, pSgBuf);
+    RTSgBufAdvance(pSgBuf, cbData);
+
+    pdmBlkCacheEntryAddWaiter(pEntry, pWaiter);
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Calculate aligned offset and size for a new cache entry
+ * which do not intersect with an already existing entry and the
+ * file end.
+ *
+ * @returns The number of bytes the entry can hold of the requested amount
+ *          of byte.
+ * @param   pEndpoint        The endpoint.
+ * @param   pBlkCache   The endpoint cache.
+ * @param   off              The start offset.
+ * @param   cb               The number of bytes the entry needs to hold at least.
+ * @param   uAlignment       Alignment of the boundary sizes.
+ * @param   poffAligned      Where to store the aligned offset.
+ * @param   pcbAligned       Where to store the aligned size of the entry.
+ */
+static size_t pdmBlkCacheEntryBoundariesCalc(PPDMBLKCACHE pBlkCache,
+                                             uint64_t off, size_t cb,
+                                             unsigned uAlignment,
+                                             uint64_t *poffAligned, size_t *pcbAligned)
+{
+    size_t cbAligned;
+    size_t cbInEntry = 0;
+    uint64_t offAligned;
+    PPDMBLKCACHEENTRY pEntryAbove = NULL;
+
+    /* Get the best fit entries around the offset */
+    pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
+
+    /* Log the info */
+    LogFlow(("%sest fit entry above off=%llu (BestFit=%llu BestFitEnd=%llu BestFitSize=%u)\n",
+             pEntryAbove ? "B" : "No b",
+             off,
+             pEntryAbove ? pEntryAbove->Core.Key : 0,
+             pEntryAbove ? pEntryAbove->Core.KeyLast : 0,
+             pEntryAbove ? pEntryAbove->cbData : 0));
+
+    offAligned = off;
+
+    if (    pEntryAbove
+        &&  off + cb > pEntryAbove->Core.Key)
+    {
+        cbInEntry = pEntryAbove->Core.Key - off;
+        cbAligned = pEntryAbove->Core.Key - offAligned;
+    }
+    else
+    {
+        cbAligned = cb;
+        cbInEntry = cb;
+    }
+
+    /* A few sanity checks */
+    AssertMsg(!pEntryAbove || (offAligned + cbAligned) <= pEntryAbove->Core.Key,
+              ("Aligned size intersects with another cache entry\n"));
+    Assert(cbInEntry <= cbAligned);
+
+    if (pEntryAbove)
+        pdmBlkCacheEntryRelease(pEntryAbove);
+
+    LogFlow(("offAligned=%llu cbAligned=%u\n", offAligned, cbAligned));
+
+    *poffAligned = offAligned;
+    *pcbAligned  = cbAligned;
+
+    return cbInEntry;
+}
+
+/**
+ * Create a new cache entry evicting data from the cache if required.
+ *
+ * @returns Pointer to the new cache entry or NULL
+ *          if not enough bytes could be evicted from the cache.
+ * @param   pEndpoint         The endpoint.
+ * @param   pBlkCache    The endpoint cache.
+ * @param   off               The offset.
+ * @param   cb                Number of bytes the cache entry should have.
+ * @param   uAlignment        Alignment the size of the entry should have.
+ * @param   pcbData           Where to store the number of bytes the new
+ *                            entry can hold. May be lower than actually requested
+ *                            due to another entry intersecting the access range.
+ */
+static PPDMBLKCACHEENTRY pdmBlkCacheEntryCreate(PPDMBLKCACHE pBlkCache,
+                                                uint64_t off, size_t cb,
+                                                unsigned uAlignment,
+                                                size_t *pcbData)
+{
+    uint64_t offStart = 0;
+    size_t cbEntry = 0;
+    PPDMBLKCACHEENTRY pEntryNew = NULL;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    uint8_t *pbBuffer = NULL;
+
+    *pcbData = pdmBlkCacheEntryBoundariesCalc(pBlkCache, off, cb, uAlignment,
+                                              &offStart, &cbEntry);
+
+    pdmBlkCacheLockEnter(pCache);
+    bool fEnough = pdmBlkCacheReclaim(pCache, cbEntry, true, &pbBuffer);
+
+    if (fEnough)
+    {
+        LogFlow(("Evicted enough bytes (%u requested). Creating new cache entry\n", cbEntry));
+
+        pEntryNew = pdmBlkCacheEntryAlloc(pBlkCache, offStart, cbEntry, pbBuffer);
+        if (RT_LIKELY(pEntryNew))
+        {
+            pdmBlkCacheEntryAddToList(&pCache->LruRecentlyUsedIn, pEntryNew);
+            pdmBlkCacheAdd(pCache, cbEntry);
+            pdmBlkCacheLockLeave(pCache);
+
+            pdmBlkCacheInsertEntry(pBlkCache, pEntryNew);
+
+            AssertMsg(   (off >= pEntryNew->Core.Key)
+                      && (off + *pcbData <= pEntryNew->Core.KeyLast + 1),
+                      ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
+                       off, pEntryNew->Core.Key));
+        }
+        else
+            pdmBlkCacheLockLeave(pCache);
+    }
+    else
+        pdmBlkCacheLockLeave(pCache);
+
+    return pEntryNew;
+}
+
+static PPDMBLKCACHEREQ pdmBlkCacheReqAlloc(size_t cbXfer, void *pvUser)
+{
+    PPDMBLKCACHEREQ pReq = (PPDMBLKCACHEREQ)RTMemAlloc(sizeof(PDMBLKCACHEREQ));
+
+    if (RT_LIKELY(pReq))
+    {
+        pReq->pvUser = pvUser;
+        pReq->cbXfer = cbXfer;
+        pReq->rcReq  = VINF_SUCCESS;
+        pReq->cXfersPending = 0;
+    }
+
+    return pReq;
+}
+
+static void pdmBlkCacheReqComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq)
+{
+    switch (pBlkCache->enmType)
+    {
+        case PDMBLKCACHETYPE_DEV:
+        {
+            pBlkCache->u.Dev.pfnXferComplete(pBlkCache->u.Dev.pDevIns,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        case PDMBLKCACHETYPE_DRV:
+        {
+            pBlkCache->u.Drv.pfnXferComplete(pBlkCache->u.Drv.pDrvIns,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        case PDMBLKCACHETYPE_USB:
+        {
+            pBlkCache->u.Usb.pfnXferComplete(pBlkCache->u.Usb.pUsbIns,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        case PDMBLKCACHETYPE_INTERNAL:
+        {
+            pBlkCache->u.Int.pfnXferComplete(pBlkCache->u.Int.pvUser,
+                                             pReq->pvUser, pReq->rcReq);
+            break;
+        }
+        default:
+            AssertMsgFailed(("Unknown block cache type!\n"));
+    }
+
+    RTMemFree(pReq);
+}
+
+static bool pdmBlkCacheReqUpdate(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEREQ pReq,
+                                 size_t cbComplete, int rcReq, bool fCallHandler)
+{
+    if (RT_FAILURE(rcReq))
+        ASMAtomicCmpXchgS32(&pReq->rcReq, rcReq, VINF_SUCCESS);
+
+    AssertMsg(pReq->cbXfer >= cbComplete, ("Completed more than left\n"));
+    AssertMsg(pReq->cXfersPending > 0, ("No transfers are pending for this request\n"));
+    uint32_t cbOld = ASMAtomicSubU32(&pReq->cbXfer, cbComplete);
+    uint32_t cXfersPending = ASMAtomicDecU32(&pReq->cXfersPending);
+
+    if (   !(cbOld - cbComplete)
+        && !cXfersPending)
+    {
+        if (fCallHandler)
+            pdmBlkCacheReqComplete(pBlkCache, pReq);
+        return true;
+    }
+
+    LogFlowFunc(("pReq=%#p cXfersPending=%u cbXfer=%u\n", pReq, cXfersPending, (cbOld - cbComplete)));
+    return false;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheRead(PPDMBLKCACHE pBlkCache, uint64_t off,
+                                 PCRTSGBUF pcSgBuf, size_t cbRead, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    PPDMBLKCACHEENTRY  pEntry;
+    PPDMBLKCACHEREQ    pReq;
+
+    LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbRead=%u pvUser=%#p\n",
+                 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbRead, pvUser));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
+
+    RTSGBUF SgBuf;
+    RTSgBufClone(&SgBuf, pcSgBuf);
+
+    /* Allocate new request structure. */
+    pReq = pdmBlkCacheReqAlloc(cbRead, pvUser);
+    if (RT_UNLIKELY(!pReq))
+        return VERR_NO_MEMORY;
+
+    /* Increment data transfer counter to keep the request valid while we access it. */
+    ASMAtomicIncU32(&pReq->cXfersPending);
+
+    while (cbRead)
+    {
+        size_t cbToRead;
+
+        pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
+
+        /*
+         * If there is no entry we try to create a new one eviciting unused pages
+         * if the cache is full. If this is not possible we will pass the request through
+         * and skip the caching (all entries may be still in progress so they can't
+         * be evicted)
+         * If we have an entry it can be in one of the LRU lists where the entry
+         * contains data (recently used or frequently used LRU) so we can just read
+         * the data we need and put the entry at the head of the frequently used LRU list.
+         * In case the entry is in one of the ghost lists it doesn't contain any data.
+         * We have to fetch it again evicting pages from either T1 or T2 to make room.
+         */
+        if (pEntry)
+        {
+            uint64_t offDiff = off - pEntry->Core.Key;
+
+            AssertMsg(off >= pEntry->Core.Key,
+                      ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
+                      off, pEntry->Core.Key));
+
+            AssertPtr(pEntry->pList);
+
+            cbToRead = RT_MIN(pEntry->cbData - offDiff, cbRead);
+
+            AssertMsg(off + cbToRead <= pEntry->Core.Key + pEntry->Core.KeyLast + 1,
+                      ("Buffer of cache entry exceeded off=%llu cbToRead=%d\n",
+                       off, cbToRead));
+
+            cbRead  -= cbToRead;
+
+            if (!cbRead)
+                STAM_COUNTER_INC(&pCache->cHits);
+            else
+                STAM_COUNTER_INC(&pCache->cPartialHits);
+
+            STAM_COUNTER_ADD(&pCache->StatRead, cbToRead);
+
+            /* Ghost lists contain no data. */
+            if (   (pEntry->pList == &pCache->LruRecentlyUsedIn)
+                || (pEntry->pList == &pCache->LruFrequentlyUsed))
+            {
+                if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                              PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
+                                                              PDMBLKCACHE_ENTRY_IS_DIRTY))
+                {
+                    /* Entry didn't completed yet. Append to the list */
+                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                               &SgBuf, offDiff, cbToRead,
+                                               false /* fWrite */);
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                }
+                else
+                {
+                    /* Read as much as we can from the entry. */
+                    RTSgBufCopyFromBuf(&SgBuf, pEntry->pbData + offDiff, cbToRead);
+                    ASMAtomicSubU32(&pReq->cbXfer, cbToRead);
+                }
+
+                /* Move this entry to the top position */
+                if (pEntry->pList == &pCache->LruFrequentlyUsed)
+                {
+                    pdmBlkCacheLockEnter(pCache);
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheLockLeave(pCache);
+                }
+                /* Release the entry */
+                pdmBlkCacheEntryRelease(pEntry);
+            }
+            else
+            {
+                uint8_t *pbBuffer = NULL;
+
+                LogFlow(("Fetching data for ghost entry %#p from file\n", pEntry));
+
+                pdmBlkCacheLockEnter(pCache);
+                pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
+                bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
+
+                /* Move the entry to Am and fetch it to the cache. */
+                if (fEnough)
+                {
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheAdd(pCache, pEntry->cbData);
+                    pdmBlkCacheLockLeave(pCache);
+
+                    if (pbBuffer)
+                        pEntry->pbData = pbBuffer;
+                    else
+                        pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
+                    AssertPtr(pEntry->pbData);
+
+                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                               &SgBuf, offDiff, cbToRead,
+                                               false /* fWrite */);
+                    pdmBlkCacheEntryReadFromMedium(pEntry);
+                    /* Release the entry */
+                    pdmBlkCacheEntryRelease(pEntry);
+                }
+                else
+                {
+                    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                    pdmBlkCacheLockLeave(pCache);
+
+                    RTMemFree(pEntry);
+
+                    pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                                  &SgBuf, off, cbToRead,
+                                                  PDMBLKCACHEXFERDIR_READ);
+                }
+            }
+        }
+        else
+        {
+#ifdef VBOX_WITH_IO_READ_CACHE
+            /* No entry found for this offset. Create a new entry and fetch the data to the cache. */
+            PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
+                                                                 off, cbRead,
+                                                                 PAGE_SIZE,
+                                                                 &cbToRead);
+
+            cbRead -= cbToRead;
+
+            if (pEntryNew)
+            {
+                if (!cbRead)
+                    STAM_COUNTER_INC(&pCache->cMisses);
+                else
+                    STAM_COUNTER_INC(&pCache->cPartialHits);
+
+                pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
+                                           &SgBuf,
+                                           off - pEntryNew->Core.Key,
+                                           cbToRead,
+                                           false /* fWrite */);
+                pdmBlkCacheEntryReadFromMedium(pEntryNew);
+                pdmBlkCacheEntryRelease(pEntryNew); /* it is protected by the I/O in progress flag now. */
+            }
+            else
+            {
+                /*
+                 * There is not enough free space in the cache.
+                 * Pass the request directly to the I/O manager.
+                 */
+                LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToRead));
+
+                pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                              &SgBuf, off, cbToRead,
+                                              PDMBLKCACHEXFERDIR_READ);
+            }
+#else
+            /* Clip read size if necessary. */
+            PPDMBLKCACHEENTRY pEntryAbove;
+            pdmBlkCacheGetCacheBestFitEntryByOffset(pBlkCache, off, &pEntryAbove);
+
+            if (pEntryAbove)
+            {
+                if (off + cbRead > pEntryAbove->Core.Key)
+                    cbToRead = pEntryAbove->Core.Key - off;
+                else
+                    cbToRead = cbRead;
+
+                pdmBlkCacheEntryRelease(pEntryAbove);
+            }
+            else
+                cbToRead = cbRead;
+
+            cbRead -= cbToRead;
+            pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                          &SgBuf, off, cbToRead,
+                                          PDMBLKCACHEXFERDIR_READ);
+#endif
+        }
+        off += cbToRead;
+    }
+
+    if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
+        rc = VINF_AIO_TASK_PENDING;
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+
+   return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheWrite(PPDMBLKCACHE pBlkCache, uint64_t off,
+                                  PCRTSGBUF pcSgBuf, size_t cbWrite, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEGLOBAL pCache = pBlkCache->pCache;
+    PPDMBLKCACHEENTRY pEntry;
+    PPDMBLKCACHEREQ pReq;
+
+    LogFlowFunc((": pBlkCache=%#p{%s} off=%llu pcSgBuf=%#p cbWrite=%u pvUser=%#p\n",
+                 pBlkCache, pBlkCache->pszId, off, pcSgBuf, cbWrite, pvUser));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
+
+    RTSGBUF SgBuf;
+    RTSgBufClone(&SgBuf, pcSgBuf);
+
+    /* Allocate new request structure. */
+    pReq = pdmBlkCacheReqAlloc(cbWrite, pvUser);
+    if (RT_UNLIKELY(!pReq))
+        return VERR_NO_MEMORY;
+
+    /* Increment data transfer counter to keep the request valid while we access it. */
+    ASMAtomicIncU32(&pReq->cXfersPending);
+
+    while (cbWrite)
+    {
+        size_t cbToWrite;
+
+        pEntry = pdmBlkCacheGetCacheEntryByOffset(pBlkCache, off);
+
+        if (pEntry)
+        {
+            /* Write the data into the entry and mark it as dirty */
+            AssertPtr(pEntry->pList);
+
+            uint64_t offDiff = off - pEntry->Core.Key;
+
+            AssertMsg(off >= pEntry->Core.Key,
+                      ("Overflow in calculation off=%llu OffsetAligned=%llu\n",
+                      off, pEntry->Core.Key));
+
+            cbToWrite = RT_MIN(pEntry->cbData - offDiff, cbWrite);
+            cbWrite  -= cbToWrite;
+
+            if (!cbWrite)
+                STAM_COUNTER_INC(&pCache->cHits);
+            else
+                STAM_COUNTER_INC(&pCache->cPartialHits);
+
+            STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
+
+            /* Ghost lists contain no data. */
+            if (   (pEntry->pList == &pCache->LruRecentlyUsedIn)
+                || (pEntry->pList == &pCache->LruFrequentlyUsed))
+            {
+                /* Check if the entry is dirty. */
+                if (pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                              PDMBLKCACHE_ENTRY_IS_DIRTY,
+                                                              0))
+                {
+                    /* If it is already dirty but not in progress just update the data. */
+                    if (!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS))
+                    {
+                        RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff,
+                                         cbToWrite);
+                        ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
+                    }
+                    else
+                    {
+                        /* The data isn't written to the file yet */
+                        pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                                   &SgBuf, offDiff, cbToWrite,
+                                                   true /* fWrite */);
+                        STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                    }
+
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                }
+                else /* Dirty bit not set */
+                {
+                    /*
+                     * Check if a read is in progress for this entry.
+                     * We have to defer processing in that case.
+                     */
+                    if(pdmBlkCacheEntryFlagIsSetClearAcquireLock(pBlkCache, pEntry,
+                                                                 PDMBLKCACHE_ENTRY_IO_IN_PROGRESS,
+                                                                 0))
+                    {
+                        pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                                   &SgBuf, offDiff, cbToWrite,
+                                                   true /* fWrite */);
+                        STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+                    }
+                    else /* I/O in progress flag not set */
+                    {
+                        /* Write as much as we can into the entry and update the file. */
+                        RTSgBufCopyToBuf(&SgBuf, pEntry->pbData + offDiff, cbToWrite);
+                        ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
+
+                        bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
+                        if (fCommit)
+                            pdmBlkCacheCommitDirtyEntries(pCache);
+                    }
+                } /* Dirty bit not set */
+
+                /* Move this entry to the top position */
+                if (pEntry->pList == &pCache->LruFrequentlyUsed)
+                {
+                    pdmBlkCacheLockEnter(pCache);
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheLockLeave(pCache);
+                }
+
+                pdmBlkCacheEntryRelease(pEntry);
+            }
+            else /* Entry is on the ghost list */
+            {
+                uint8_t *pbBuffer = NULL;
+
+                pdmBlkCacheLockEnter(pCache);
+                pdmBlkCacheEntryRemoveFromList(pEntry); /* Remove it before we remove data, otherwise it may get freed when evicting data. */
+                bool fEnough = pdmBlkCacheReclaim(pCache, pEntry->cbData, true, &pbBuffer);
+
+                if (fEnough)
+                {
+                    /* Move the entry to Am and fetch it to the cache. */
+                    pdmBlkCacheEntryAddToList(&pCache->LruFrequentlyUsed, pEntry);
+                    pdmBlkCacheAdd(pCache, pEntry->cbData);
+                    pdmBlkCacheLockLeave(pCache);
+
+                    if (pbBuffer)
+                        pEntry->pbData = pbBuffer;
+                    else
+                        pEntry->pbData = (uint8_t *)RTMemPageAlloc(pEntry->cbData);
+                    AssertPtr(pEntry->pbData);
+
+                    pdmBlkCacheEntryWaitersAdd(pEntry, pReq,
+                                               &SgBuf, offDiff, cbToWrite,
+                                               true /* fWrite */);
+                    STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                    pdmBlkCacheEntryReadFromMedium(pEntry);
+
+                    /* Release the reference. If it is still needed the I/O in progress flag should protect it now. */
+                    pdmBlkCacheEntryRelease(pEntry);
+                }
+                else
+                {
+                    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+                    STAM_PROFILE_ADV_START(&pCache->StatTreeRemove, Cache);
+                    RTAvlrU64Remove(pBlkCache->pTree, pEntry->Core.Key);
+                    STAM_PROFILE_ADV_STOP(&pCache->StatTreeRemove, Cache);
+                    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+                    pdmBlkCacheLockLeave(pCache);
+
+                    RTMemFree(pEntry);
+                    pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                                  &SgBuf, off, cbToWrite,
+                                                  PDMBLKCACHEXFERDIR_WRITE);
+                }
+            }
+        }
+        else /* No entry found */
+        {
+            /*
+             * No entry found. Try to create a new cache entry to store the data in and if that fails
+             * write directly to the file.
+             */
+            PPDMBLKCACHEENTRY pEntryNew = pdmBlkCacheEntryCreate(pBlkCache,
+                                                                 off, cbWrite,
+                                                                 512, &cbToWrite);
+
+            cbWrite -= cbToWrite;
+
+            if (pEntryNew)
+            {
+                uint64_t offDiff = off - pEntryNew->Core.Key;
+
+                STAM_COUNTER_INC(&pCache->cHits);
+
+                /*
+                 * Check if it is possible to just write the data without waiting
+                 * for it to get fetched first.
+                 */
+                if (!offDiff && pEntryNew->cbData == cbToWrite)
+                {
+                    RTSgBufCopyToBuf(&SgBuf, pEntryNew->pbData, cbToWrite);
+                    ASMAtomicSubU32(&pReq->cbXfer, cbToWrite);
+
+                    bool fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntryNew);
+                    if (fCommit)
+                        pdmBlkCacheCommitDirtyEntries(pCache);
+                    STAM_COUNTER_ADD(&pCache->StatWritten, cbToWrite);
+                }
+                else
+                {
+                    /* Defer the write and fetch the data from the endpoint. */
+                    pdmBlkCacheEntryWaitersAdd(pEntryNew, pReq,
+                                               &SgBuf, offDiff, cbToWrite,
+                                               true /* fWrite */);
+                    STAM_COUNTER_INC(&pBlkCache->StatWriteDeferred);
+                    pdmBlkCacheEntryReadFromMedium(pEntryNew);
+                }
+
+                pdmBlkCacheEntryRelease(pEntryNew);
+            }
+            else
+            {
+                /*
+                 * There is not enough free space in the cache.
+                 * Pass the request directly to the I/O manager.
+                 */
+                LogFlow(("Couldn't evict %u bytes from the cache. Remaining request will be passed through\n", cbToWrite));
+
+                STAM_COUNTER_INC(&pCache->cMisses);
+
+                pdmBlkCacheRequestPassthrough(pBlkCache, pReq,
+                                              &SgBuf, off, cbToWrite,
+                                              PDMBLKCACHEXFERDIR_WRITE);
+            }
+        }
+
+        off += cbToWrite;
+    }
+
+    if (!pdmBlkCacheReqUpdate(pBlkCache, pReq, 0, rc, false))
+        rc = VINF_AIO_TASK_PENDING;
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheFlush(PPDMBLKCACHE pBlkCache, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    PPDMBLKCACHEREQ pReq;
+
+    LogFlowFunc((": pBlkCache=%#p{%s}\n", pBlkCache, pBlkCache->pszId));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+    AssertReturn(!pBlkCache->fSuspended, VERR_INVALID_STATE);
+
+    /* Commit dirty entries in the cache. */
+    pdmBlkCacheCommit(pBlkCache);
+
+    /* Allocate new request structure. */
+    pReq = pdmBlkCacheReqAlloc(0, pvUser);
+    if (RT_UNLIKELY(!pReq))
+        return VERR_NO_MEMORY;
+
+    rc = pdmBlkCacheRequestPassthrough(pBlkCache, pReq, NULL, 0, 0,
+                                       PDMBLKCACHEXFERDIR_FLUSH);
+    AssertRC(rc);
+
+    LogFlowFunc((": Leave rc=%Rrc\n", rc));
+    return VINF_AIO_TASK_PENDING;
+}
+
+/**
+ * Completes a task segment freeing all resources and completes the task handle
+ * if everything was transferred.
+ *
+ * @returns Next task segment handle.
+ * @param   pTaskSeg          Task segment to complete.
+ * @param   rc                Status code to set.
+ */
+static PPDMBLKCACHEWAITER pdmBlkCacheWaiterComplete(PPDMBLKCACHE pBlkCache,
+                                                    PPDMBLKCACHEWAITER pWaiter,
+                                                    int rc)
+{
+    PPDMBLKCACHEWAITER pNext = pWaiter->pNext;
+    PPDMBLKCACHEREQ pReq = pWaiter->pReq;
+
+    pdmBlkCacheReqUpdate(pBlkCache, pWaiter->pReq, pWaiter->cbTransfer, rc, true);
+
+    RTMemFree(pWaiter);
+
+    return pNext;
+}
+
+static void pdmBlkCacheIoXferCompleteEntry(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
+{
+    PPDMBLKCACHEENTRY  pEntry    = hIoXfer->pEntry;
+    PPDMBLKCACHEGLOBAL pCache    = pBlkCache->pCache;
+
+    /* Reference the entry now as we are clearing the I/O in progress flag
+     * which protected the entry till now. */
+    pdmBlkCacheEntryRef(pEntry);
+
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IO_IN_PROGRESS;
+
+    /* Process waiting segment list. The data in entry might have changed in-between. */
+    bool fDirty = false;
+    PPDMBLKCACHEWAITER pComplete = pEntry->pWaitingHead;
+    PPDMBLKCACHEWAITER pCurr     = pComplete;
+
+    AssertMsg((pCurr && pEntry->pWaitingTail) || (!pCurr && !pEntry->pWaitingTail),
+                ("The list tail was not updated correctly\n"));
+    pEntry->pWaitingTail = NULL;
+    pEntry->pWaitingHead = NULL;
+
+    if (hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_WRITE)
+    {
+        /*
+         * An error here is difficult to handle as the original request completed already.
+         * The error is logged for now and the VM is paused.
+         * If the user continues the entry is written again in the hope
+         * the user fixed the problem and the next write succeeds.
+         */
+        if (RT_FAILURE(rcIoXfer))
+        {
+            LogRel(("I/O cache: Error while writing entry at offset %llu (%u bytes) to medium \"%s\" (rc=%Rrc)\n",
+                    pEntry->Core.Key, pEntry->cbData, pBlkCache->pszId, rcIoXfer));
+
+            if (!ASMAtomicXchgBool(&pCache->fIoErrorVmSuspended, true))
+            {
+                int rc = VMSetRuntimeError(pCache->pVM, VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_NO_WAIT, "BLKCACHE_IOERR",
+                                           N_("The I/O cache encountered an error while updating data in medium \"%s\" (rc=%Rrc). "
+                                              "Make sure there is enough free space on the disk and that the disk is working properly. "
+                                              "Operation can be resumed afterwards"),
+                                           pBlkCache->pszId, rcIoXfer);
+                AssertRC(rc);
+            }
+
+            /*
+             * The entry is still marked as dirty which prevents eviction.
+             * Add the waiters to the list again.
+             */
+            pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY; /* Clear so it gets added to the list again. */
+            fDirty = true;
+
+            if (pComplete)
+            {
+                pEntry->pWaitingHead = pComplete;
+                while (pComplete->pNext)
+                    pComplete = pComplete->pNext;
+                pEntry->pWaitingTail = pComplete;
+                pComplete = NULL;
+            }
+        }
+        else
+        {
+            pEntry->fFlags &= ~PDMBLKCACHE_ENTRY_IS_DIRTY;
+
+            while (pCurr)
+            {
+                AssertMsg(pCurr->fWrite, ("Completed write entries should never have read tasks attached\n"));
+
+                RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
+                fDirty = true;
+
+                pCurr = pCurr->pNext;
+            }
+        }
+    }
+    else
+    {
+        AssertMsg(hIoXfer->enmXferDir == PDMBLKCACHEXFERDIR_READ, ("Invalid transfer type\n"));
+        AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IS_DIRTY),
+                  ("Invalid flags set\n"));
+
+        while (pCurr)
+        {
+            if (pCurr->fWrite)
+            {
+                RTSgBufCopyToBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
+                fDirty = true;
+            }
+            else
+                RTSgBufCopyFromBuf(&pCurr->SgBuf, pEntry->pbData + pCurr->offCacheEntry, pCurr->cbTransfer);
+
+            pCurr = pCurr->pNext;
+        }
+    }
+
+    bool fCommit = false;
+    if (fDirty)
+        fCommit = pdmBlkCacheAddDirtyEntry(pBlkCache, pEntry);
+
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+    /* Dereference so that it isn't protected anymore except we issued anyother write for it. */
+    pdmBlkCacheEntryRelease(pEntry);
+
+    if (fCommit)
+        pdmBlkCacheCommitDirtyEntries(pCache);
+
+    /* Complete waiters now. */
+    while (pComplete)
+        pComplete = pdmBlkCacheWaiterComplete(pBlkCache, pComplete, rcIoXfer);
+}
+
+VMMR3DECL(void) PDMR3BlkCacheIoXferComplete(PPDMBLKCACHE pBlkCache, PPDMBLKCACHEIOXFER hIoXfer, int rcIoXfer)
+{
+    LogFlowFunc(("pBlkCache=%#p hIoXfer=%#p rcIoXfer=%Rrc\n", pBlkCache, hIoXfer, rcIoXfer));
+
+    if (hIoXfer->fIoCache)
+        pdmBlkCacheIoXferCompleteEntry(pBlkCache, hIoXfer, rcIoXfer);
+    else
+        pdmBlkCacheReqUpdate(pBlkCache, hIoXfer->pReq, hIoXfer->cbXfer, rcIoXfer, true);
+}
+
+/**
+ * Callback for the AVL do with all routine. Waits for a cachen entry to finish any pending I/O.
+ *
+ * @returns IPRT status code.
+ * @param    pNode     The node to destroy.
+ * @param    pvUser    Opaque user data.
+ */
+static int pdmBlkCacheEntryQuiesce(PAVLRU64NODECORE pNode, void *pvUser)
+{
+    PPDMBLKCACHEENTRY  pEntry = (PPDMBLKCACHEENTRY)pNode;
+    PPDMBLKCACHE pBlkCache = pEntry->pBlkCache;
+
+    while (ASMAtomicReadU32(&pEntry->fFlags) & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS)
+    {
+        /* Leave the locks to let the I/O thread make progress but reference the entry to prevent eviction. */
+        pdmBlkCacheEntryRef(pEntry);
+        RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+        RTThreadSleep(1);
+
+        /* Re-enter all locks and drop the reference. */
+        RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+        pdmBlkCacheEntryRelease(pEntry);
+    }
+
+    AssertMsg(!(pEntry->fFlags & PDMBLKCACHE_ENTRY_IO_IN_PROGRESS),
+                ("Entry is dirty and/or still in progress fFlags=%#x\n", pEntry->fFlags));
+
+    return VINF_SUCCESS;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheSuspend(PPDMBLKCACHE pBlkCache)
+{
+    int rc = VINF_SUCCESS;
+    LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+
+    ASMAtomicXchgBool(&pBlkCache->fSuspended, true);
+
+    /* Wait for all I/O to complete. */
+    RTSemRWRequestWrite(pBlkCache->SemRWEntries, RT_INDEFINITE_WAIT);
+    rc = RTAvlrU64DoWithAll(pBlkCache->pTree, true, pdmBlkCacheEntryQuiesce, NULL);
+    AssertRC(rc);
+    RTSemRWReleaseWrite(pBlkCache->SemRWEntries);
+
+    return rc;
+}
+
+VMMR3DECL(int) PDMR3BlkCacheResume(PPDMBLKCACHE pBlkCache)
+{
+    LogFlowFunc(("pBlkCache=%#p\n", pBlkCache));
+
+    AssertPtrReturn(pBlkCache, VERR_INVALID_POINTER);
+
+    ASMAtomicXchgBool(&pBlkCache->fSuspended, false);
+
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMCritSect.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMCritSect.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMCritSect.cpp	(revision 35333)
@@ -0,0 +1,638 @@
+/* $Id$ */
+/** @file
+ * PDM - Critical Sections, Ring-3.
+ */
+
+/*
+ * Copyright (C) 2006-2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM//_CRITSECT
+#include "PDMInternal.h"
+#include <VBox/pdmcritsect.h>
+#include <VBox/mm.h>
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/lockvalidator.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int pdmR3CritSectDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTINT pCritSect, PPDMCRITSECTINT pPrev, bool fFinal);
+
+
+
+/**
+ * Register statistics related to the critical sections.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+int pdmR3CritSectInitStats(PVM pVM)
+{
+    STAM_REG(pVM, &pVM->pdm.s.StatQueuedCritSectLeaves, STAMTYPE_COUNTER, "/PDM/QueuedCritSectLeaves", STAMUNIT_OCCURENCES,
+             "Number of times a critical section leave request needed to be queued for ring-3 execution.");
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Relocates all the critical sections.
+ *
+ * @param   pVM         The VM handle.
+ */
+void pdmR3CritSectRelocate(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+
+    for (PPDMCRITSECTINT pCur = pUVM->pdm.s.pCritSects;
+         pCur;
+         pCur = pCur->pNext)
+        pCur->pVMRC = pVM->pVMRC;
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Deletes all remaining critical sections.
+ *
+ * This is called at the very end of the termination process.  It is also called
+ * at the end of vmR3CreateU failure cleanup, which may cause it to be called
+ * twice depending on where vmR3CreateU actually failed.  We have to do the
+ * latter call because other components expect the critical sections to be
+ * automatically deleted.
+ *
+ * @returns VBox status.
+ *          First error code, rest is lost.
+ * @param   pVMU        The user mode VM handle.
+ * @remark  Don't confuse this with PDMR3CritSectDelete.
+ */
+VMMDECL(int) PDMR3CritSectTerm(PVM pVM)
+{
+    PUVM    pUVM = pVM->pUVM;
+    int     rc   = VINF_SUCCESS;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+
+    while (pUVM->pdm.s.pCritSects)
+    {
+        int rc2 = pdmR3CritSectDeleteOne(pVM, pUVM, pUVM->pdm.s.pCritSects, NULL, true /* final */);
+        AssertRC(rc2);
+        if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+            rc = rc2;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Initializes a critical section and inserts it into the list.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The Vm handle.
+ * @param   pCritSect       The critical section.
+ * @param   pvKey           The owner key.
+ * @param   RT_SRC_POS_DECL The source position.
+ * @param   pszName         The name of the critical section (for statistics).
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   va              Arguments for the format string.
+ */
+static int pdmR3CritSectInitOne(PVM pVM, PPDMCRITSECTINT pCritSect, void *pvKey, RT_SRC_POS_DECL, const char *pszNameFmt, va_list va)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Allocate the semaphore.
+     */
+    AssertCompile(sizeof(SUPSEMEVENT) == sizeof(pCritSect->Core.EventSem));
+    int rc = SUPSemEventCreate(pVM->pSession, (PSUPSEMEVENT)&pCritSect->Core.EventSem);
+    if (RT_SUCCESS(rc))
+    {
+        /* Only format the name once. */
+        char *pszName = RTStrAPrintf2V(pszNameFmt, va); /** @todo plug the "leak"... */
+        if (pszName)
+        {
+#ifndef PDMCRITSECT_STRICT
+            pCritSect->Core.pValidatorRec = NULL;
+#else
+            rc = RTLockValidatorRecExclCreate(&pCritSect->Core.pValidatorRec,
+# ifdef RT_LOCK_STRICT_ORDER
+                                              RTLockValidatorClassForSrcPos(RT_SRC_POS_ARGS, "%s", pszName),
+# else
+                                              NIL_RTLOCKVALCLASS,
+# endif
+                                              RTLOCKVAL_SUB_CLASS_NONE,
+                                              pCritSect, true, "%s", pszName);
+#endif
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Initialize the structure (first bit is c&p from RTCritSectInitEx).
+                 */
+                pCritSect->Core.u32Magic             = RTCRITSECT_MAGIC;
+                pCritSect->Core.fFlags               = 0;
+                pCritSect->Core.cNestings            = 0;
+                pCritSect->Core.cLockers             = -1;
+                pCritSect->Core.NativeThreadOwner    = NIL_RTNATIVETHREAD;
+                pCritSect->pVMR3                     = pVM;
+                pCritSect->pVMR0                     = pVM->pVMR0;
+                pCritSect->pVMRC                     = pVM->pVMRC;
+                pCritSect->pvKey                     = pvKey;
+                pCritSect->EventToSignal             = NIL_RTSEMEVENT;
+                pCritSect->pNext                     = pVM->pUVM->pdm.s.pCritSects;
+                pCritSect->pszName                   = pszName;
+                pVM->pUVM->pdm.s.pCritSects = pCritSect;
+                STAMR3RegisterF(pVM, &pCritSect->StatContentionRZLock,  STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionRZLock", pCritSect->pszName);
+                STAMR3RegisterF(pVM, &pCritSect->StatContentionRZUnlock,STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionRZUnlock", pCritSect->pszName);
+                STAMR3RegisterF(pVM, &pCritSect->StatContentionR3,      STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,          NULL, "/PDM/CritSects/%s/ContentionR3", pCritSect->pszName);
+#ifdef VBOX_WITH_STATISTICS
+                STAMR3RegisterF(pVM, &pCritSect->StatLocked,        STAMTYPE_PROFILE_ADV, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_OCCURENCE, NULL, "/PDM/CritSects/%s/Locked", pCritSect->pszName);
+#endif
+                return VINF_SUCCESS;
+            }
+
+            RTStrFree(pszName);
+        }
+        else
+            rc = VERR_NO_STR_MEMORY;
+        SUPSemEventClose(pVM->pSession, (SUPSEMEVENT)pCritSect->Core.EventSem);
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes a PDM critical section for internal use.
+ *
+ * The PDM critical sections are derived from the IPRT critical sections, but
+ * works in GC as well.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pDevIns         Device instance.
+ * @param   pCritSect       Pointer to the critical section.
+ * @param   RT_SRC_POS_DECL Use RT_SRC_POS.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   ...             Arguments for the format string.
+ * @thread  EMT(0)
+ */
+VMMR3DECL(int) PDMR3CritSectInit(PVM pVM, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, ...)
+{
+#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
+    AssertCompile(sizeof(pCritSect->padding) >= sizeof(pCritSect->s));
+#endif
+    Assert(RT_ALIGN_P(pCritSect, sizeof(uintptr_t)) == pCritSect);
+    va_list va;
+    va_start(va, pszNameFmt);
+    int rc = pdmR3CritSectInitOne(pVM, &pCritSect->s, pCritSect, RT_SRC_POS_ARGS, pszNameFmt, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Initializes a PDM critical section for a device.
+ *
+ * The PDM critical sections are derived from the IPRT critical sections, but
+ * works in GC as well.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pDevIns         Device instance.
+ * @param   pCritSect       Pointer to the critical section.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   va              Arguments for the format string.
+ */
+int pdmR3CritSectInitDevice(PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                            const char *pszNameFmt, va_list va)
+{
+    return pdmR3CritSectInitOne(pVM, &pCritSect->s, pDevIns, RT_SRC_POS_ARGS, pszNameFmt, va);
+}
+
+
+/**
+ * Initializes a PDM critical section for a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pDrvIns         Driver instance.
+ * @param   pCritSect       Pointer to the critical section.
+ * @param   pszNameFmt      Format string for naming the critical section.  For
+ *                          statistics and lock validation.
+ * @param   ...             Arguments for the format string.
+ */
+int pdmR3CritSectInitDriver(PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                            const char *pszNameFmt, ...)
+{
+    va_list va;
+    va_start(va, pszNameFmt);
+    int rc = pdmR3CritSectInitOne(pVM, &pCritSect->s, pDrvIns, RT_SRC_POS_ARGS, pszNameFmt, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Deletes one critical section.
+ *
+ * @returns Return code from RTCritSectDelete.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pCritSect   The critical section.
+ * @param   pPrev       The previous critical section in the list.
+ * @param   fFinal      Set if this is the final call and statistics shouldn't be deregistered.
+ *
+ * @remarks Caller must have entered the ListCritSect.
+ */
+static int pdmR3CritSectDeleteOne(PVM pVM, PUVM pUVM, PPDMCRITSECTINT pCritSect, PPDMCRITSECTINT pPrev, bool fFinal)
+{
+    /*
+     * Assert free waiters and so on (c&p from RTCritSectDelete).
+     */
+    Assert(pCritSect->Core.u32Magic == RTCRITSECT_MAGIC);
+    Assert(pCritSect->Core.cNestings == 0);
+    Assert(pCritSect->Core.cLockers == -1);
+    Assert(pCritSect->Core.NativeThreadOwner == NIL_RTNATIVETHREAD);
+    Assert(RTCritSectIsOwner(&pUVM->pdm.s.ListCritSect));
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+        pPrev->pNext = pCritSect->pNext;
+    else
+        pUVM->pdm.s.pCritSects = pCritSect->pNext;
+
+    /*
+     * Delete it (parts taken from RTCritSectDelete).
+     * In case someone is waiting we'll signal the semaphore cLockers + 1 times.
+     */
+    ASMAtomicWriteU32(&pCritSect->Core.u32Magic, 0);
+    SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->Core.EventSem;
+    pCritSect->Core.EventSem = NIL_RTSEMEVENT;
+    while (pCritSect->Core.cLockers-- >= 0)
+        SUPSemEventSignal(pVM->pSession, hEvent);
+    ASMAtomicWriteS32(&pCritSect->Core.cLockers, -1);
+    int rc = SUPSemEventClose(pVM->pSession, hEvent);
+    AssertRC(rc);
+    RTLockValidatorRecExclDestroy(&pCritSect->Core.pValidatorRec);
+    pCritSect->pNext   = NULL;
+    pCritSect->pvKey   = NULL;
+    pCritSect->pVMR3   = NULL;
+    pCritSect->pVMR0   = NIL_RTR0PTR;
+    pCritSect->pVMRC   = NIL_RTRCPTR;
+    RTStrFree((char *)pCritSect->pszName);
+    pCritSect->pszName = NULL;
+    if (!fFinal)
+    {
+        STAMR3Deregister(pVM, &pCritSect->StatContentionRZLock);
+        STAMR3Deregister(pVM, &pCritSect->StatContentionRZUnlock);
+        STAMR3Deregister(pVM, &pCritSect->StatContentionR3);
+#ifdef VBOX_WITH_STATISTICS
+        STAMR3Deregister(pVM, &pCritSect->StatLocked);
+#endif
+    }
+    return rc;
+}
+
+
+/**
+ * Deletes all critical sections with a give initializer key.
+ *
+ * @returns VBox status code.
+ *          The entire list is processed on failure, so we'll only
+ *          return the first error code. This shouldn't be a problem
+ *          since errors really shouldn't happen here.
+ * @param   pVM     The VM handle.
+ * @param   pvKey   The initializer key.
+ */
+static int pdmR3CritSectDeleteByKey(PVM pVM, void *pvKey)
+{
+    /*
+     * Iterate the list and match key.
+     */
+    PUVM            pUVM  = pVM->pUVM;
+    int             rc    = VINF_SUCCESS;
+    PPDMCRITSECTINT pPrev = NULL;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMCRITSECTINT pCur  = pUVM->pdm.s.pCritSects;
+    while (pCur)
+    {
+        if (pCur->pvKey == pvKey)
+        {
+            int rc2 = pdmR3CritSectDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
+            AssertRC(rc2);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Deletes all undeleted critical sections initialized by a given device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pDevIns     The device handle.
+ */
+int pdmR3CritSectDeleteDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    return pdmR3CritSectDeleteByKey(pVM, pDevIns);
+}
+
+
+/**
+ * Deletes all undeleted critical sections initialized by a given driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pDrvIns     The driver handle.
+ */
+int pdmR3CritSectDeleteDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    return pdmR3CritSectDeleteByKey(pVM, pDrvIns);
+}
+
+
+/**
+ * Deletes the critical section.
+ *
+ * @returns VBox status code.
+ * @param   pCritSect           The PDM critical section to destroy.
+ */
+VMMR3DECL(int) PDMR3CritSectDelete(PPDMCRITSECT pCritSect)
+{
+    if (!RTCritSectIsInitialized(&pCritSect->s.Core))
+        return VINF_SUCCESS;
+
+    /*
+     * Find and unlink it.
+     */
+    PVM             pVM   = pCritSect->s.pVMR3;
+    PUVM            pUVM  = pVM->pUVM;
+    AssertReleaseReturn(pVM, VERR_INTERNAL_ERROR);
+    PPDMCRITSECTINT pPrev = NULL;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMCRITSECTINT pCur  = pUVM->pdm.s.pCritSects;
+    while (pCur)
+    {
+        if (pCur == &pCritSect->s)
+        {
+            int rc = pdmR3CritSectDeleteOne(pVM, pUVM, pCur, pPrev, false /* not final */);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            return rc;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertReleaseMsgFailed(("pCritSect=%p wasn't found!\n", pCritSect));
+    return VERR_INTERNAL_ERROR;
+}
+
+
+/**
+ * Gets the name of the critical section.
+ *
+ *
+ * @returns Pointer to the critical section name (read only) on success,
+ *          NULL on failure (invalid critical section).
+ * @param   pCritSect           The critical section.
+ */
+VMMR3DECL(const char *) PDMR3CritSectName(PCPDMCRITSECT pCritSect)
+{
+    AssertPtrReturn(pCritSect, NULL);
+    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, NULL);
+    return pCritSect->s.pszName;
+}
+
+
+/**
+ * Yield the critical section if someone is waiting on it.
+ *
+ * When yielding, we'll leave the critical section and try to make sure the
+ * other waiting threads get a chance of entering before we reclaim it.
+ *
+ * @retval  true if yielded.
+ * @retval  false if not yielded.
+ * @param   pCritSect           The critical section.
+ */
+VMMR3DECL(bool) PDMR3CritSectYield(PPDMCRITSECT pCritSect)
+{
+    AssertPtrReturn(pCritSect, false);
+    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, false);
+    Assert(pCritSect->s.Core.NativeThreadOwner == RTThreadNativeSelf());
+
+    /* No recursion allowed here. */
+    int32_t const cNestings = pCritSect->s.Core.cNestings;
+    AssertReturn(cNestings == 1, false);
+
+    int32_t const cLockers  = ASMAtomicReadS32(&pCritSect->s.Core.cLockers);
+    if (cLockers < cNestings)
+        return false;
+
+#ifdef PDMCRITSECT_STRICT
+    RTLOCKVALSRCPOS const SrcPos = pCritSect->s.Core.pValidatorRec->SrcPos;
+#endif
+    PDMCritSectLeave(pCritSect);
+
+    /*
+     * If we're lucky, then one of the waiters has entered the lock already.
+     * We spin a little bit in hope for this to happen so we can avoid the
+     * yield detour.
+     */
+    if (ASMAtomicUoReadS32(&pCritSect->s.Core.cNestings) == 0)
+    {
+        int cLoops = 20;
+        while (   cLoops > 0
+               && ASMAtomicUoReadS32(&pCritSect->s.Core.cNestings) == 0
+               && ASMAtomicUoReadS32(&pCritSect->s.Core.cLockers)  >= 0)
+        {
+            ASMNopPause();
+            cLoops--;
+        }
+        if (cLoops == 0)
+            RTThreadYield();
+    }
+
+#ifdef PDMCRITSECT_STRICT
+    int rc = PDMCritSectEnterDebug(pCritSect, VERR_INTERNAL_ERROR,
+                                   SrcPos.uId, SrcPos.pszFile, SrcPos.uLine, SrcPos.pszFunction);
+#else
+    int rc = PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
+#endif
+    AssertLogRelRC(rc);
+    return true;
+}
+
+
+/**
+ * Schedule a event semaphore for signalling upon critsect exit.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_TOO_MANY_SEMAPHORES if an event was already scheduled.
+ * @returns VERR_NOT_OWNER if we're not the critsect owner.
+ * @returns VERR_SEM_DESTROYED if RTCritSectDelete was called while waiting.
+ *
+ * @param   pCritSect       The critical section.
+ * @param   EventToSignal   The semaphore that should be signalled.
+ */
+VMMR3DECL(int) PDMR3CritSectScheduleExitEvent(PPDMCRITSECT pCritSect, RTSEMEVENT EventToSignal)
+{
+    Assert(EventToSignal != NIL_RTSEMEVENT);
+    if (RT_UNLIKELY(!RTCritSectIsOwner(&pCritSect->s.Core)))
+        return VERR_NOT_OWNER;
+    if (RT_LIKELY(   pCritSect->s.EventToSignal == NIL_RTSEMEVENT
+                  || pCritSect->s.EventToSignal == EventToSignal))
+    {
+        pCritSect->s.EventToSignal = EventToSignal;
+        return VINF_SUCCESS;
+    }
+    return VERR_TOO_MANY_SEMAPHORES;
+}
+
+
+/**
+ * Counts the critical sections owned by the calling thread, optionally
+ * returning a comma separated list naming them.
+ *
+ * This is for diagnostic purposes only.
+ *
+ * @returns Lock count.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pszNames        Where to return the critical section names.
+ * @param   cbNames         The size of the buffer.
+ */
+VMMR3DECL(uint32_t) PDMR3CritSectCountOwned(PVM pVM, char *pszNames, size_t cbNames)
+{
+    /*
+     * Init the name buffer.
+     */
+    size_t cchLeft = cbNames;
+    if (cchLeft)
+    {
+        cchLeft--;
+        pszNames[0] = pszNames[cchLeft] = '\0';
+    }
+
+    /*
+     * Iterate the critical sections.
+     */
+    /* This is unsafe, but wtf. */
+    RTNATIVETHREAD const    hNativeThread = RTThreadNativeSelf();
+    uint32_t                cCritSects = 0;
+    for (PPDMCRITSECTINT pCur = pVM->pUVM->pdm.s.pCritSects;
+         pCur;
+         pCur = pCur->pNext)
+    {
+        /* Same as RTCritSectIsOwner(). */
+        if (pCur->Core.NativeThreadOwner == hNativeThread)
+        {
+            cCritSects++;
+
+            /*
+             * Copy the name if there is space. Fun stuff.
+             */
+            if (cchLeft)
+            {
+                /* try add comma. */
+                if (cCritSects != 1)
+                {
+                    *pszNames++ = ',';
+                    if (--cchLeft)
+                    {
+                        *pszNames++ = ' ';
+                        cchLeft--;
+                    }
+                }
+
+                /* try copy the name. */
+                if (cchLeft)
+                {
+                    size_t const cchName = strlen(pCur->pszName);
+                    if (cchName < cchLeft)
+                    {
+                        memcpy(pszNames, pCur->pszName, cchName);
+                        pszNames += cchName;
+                        cchLeft -= cchName;
+                    }
+                    else
+                    {
+                        if (cchLeft > 2)
+                        {
+                            memcpy(pszNames, pCur->pszName, cchLeft - 2);
+                            pszNames += cchLeft - 2;
+                            cchLeft = 2;
+                        }
+                        while (cchLeft-- > 0)
+                            *pszNames++ = '+';
+                    }
+                }
+                *pszNames = '\0';
+            }
+        }
+    }
+
+    return cCritSects;
+}
+
+
+/**
+ * Leave all critical sections the calling thread owns.
+ *
+ * @param   pVM         The VM handle.
+ */
+void PDMR3CritSectLeaveAll(PVM pVM)
+{
+    RTNATIVETHREAD const hNativeSelf = RTThreadNativeSelf();
+    PUVM                 pUVM        = pVM->pUVM;
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMCRITSECTINT pCur = pUVM->pdm.s.pCritSects;
+         pCur;
+         pCur = pCur->pNext)
+    {
+        while (     pCur->Core.NativeThreadOwner == hNativeSelf
+               &&   pCur->Core.cNestings > 0)
+            PDMCritSectLeave((PPDMCRITSECT)pCur);
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMDevHlp.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMDevHlp.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMDevHlp.cpp	(revision 35333)
@@ -0,0 +1,3471 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Device Helpers.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/pgm.h>
+#include <VBox/iom.h>
+#include <VBox/rem.h>
+#include <VBox/dbgf.h>
+#include <VBox/vmapi.h>
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/vmm.h>
+
+#include <VBox/version.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** @def PDM_DEVHLP_DEADLOCK_DETECTION
+ * Define this to enable the deadlock detection when accessing physical memory.
+ */
+#if /*defined(DEBUG_bird) ||*/ defined(DOXYGEN_RUNNING)
+# define PDM_DEVHLP_DEADLOCK_DETECTION /**< @todo enable DevHlp deadlock detection! */
+#endif
+
+
+
+/**
+ * Wrapper around PDMR3LdrGetSymbolRCLazy.
+ */
+DECLINLINE(int) pdmR3DevGetSymbolRCLazy(PPDMDEVINS pDevIns, const char *pszSymbol, PRTRCPTR ppvValue)
+{
+    return PDMR3LdrGetSymbolRCLazy(pDevIns->Internal.s.pVMR3,
+                                   pDevIns->Internal.s.pDevR3->pReg->szRCMod,
+                                   pDevIns->Internal.s.pDevR3->pszRCSearchPath,
+                                   pszSymbol, ppvValue);
+}
+
+
+/**
+ * Wrapper around PDMR3LdrGetSymbolR0Lazy.
+ */
+DECLINLINE(int) pdmR3DevGetSymbolR0Lazy(PPDMDEVINS pDevIns, const char *pszSymbol, PRTR0PTR ppvValue)
+{
+    return PDMR3LdrGetSymbolR0Lazy(pDevIns->Internal.s.pVMR3,
+                                   pDevIns->Internal.s.pDevR3->pReg->szR0Mod,
+                                   pDevIns->Internal.s.pDevR3->pszR0SearchPath,
+                                   pszSymbol, ppvValue);
+}
+
+
+/** @name R3 DevHlp
+ * @{
+ */
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_IOPortRegister(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts, RTHCPTR pvUser, PFNIOMIOPORTOUT pfnOut, PFNIOMIOPORTIN pfnIn,
+                                                    PFNIOMIOPORTOUTSTRING pfnOutStr, PFNIOMIOPORTINSTRING pfnInStr, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_IOPortRegister: caller='%s'/%d: Port=%#x cPorts=%#x pvUser=%p pfnOut=%p pfnIn=%p pfnOutStr=%p pfnInStr=%p p32_tszDesc=%p:{%s}\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             Port, cPorts, pvUser, pfnOut, pfnIn, pfnOutStr, pfnInStr, pszDesc, pszDesc));
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+
+#if 0 /** @todo needs a real string cache for this */
+    if (pDevIns->iInstance > 0)
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+#endif
+
+    int rc = IOMR3IOPortRegisterR3(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts, pvUser, pfnOut, pfnIn, pfnOutStr, pfnInStr, pszDesc);
+
+    LogFlow(("pdmR3DevHlp_IOPortRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortRegisterRC} */
+static DECLCALLBACK(int) pdmR3DevHlp_IOPortRegisterRC(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts, RTRCPTR pvUser,
+                                                      const char *pszOut, const char *pszIn,
+                                                      const char *pszOutStr, const char *pszInStr, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_IOPortRegisterRC: caller='%s'/%d: Port=%#x cPorts=%#x pvUser=%p pszOut=%p:{%s} pszIn=%p:{%s} pszOutStr=%p:{%s} pszInStr=%p:{%s} pszDesc=%p:{%s}\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             Port, cPorts, pvUser, pszOut, pszOut, pszIn, pszIn, pszOutStr, pszOutStr, pszInStr, pszInStr, pszDesc, pszDesc));
+
+    /*
+     * Resolve the functions (one of the can be NULL).
+     */
+    int rc = VINF_SUCCESS;
+    if (    pDevIns->pReg->szRCMod[0]
+        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC))
+    {
+        RTRCPTR RCPtrIn = NIL_RTRCPTR;
+        if (pszIn)
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszIn, &RCPtrIn);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszIn)\n", pDevIns->pReg->szRCMod, pszIn));
+        }
+        RTRCPTR RCPtrOut = NIL_RTRCPTR;
+        if (pszOut && RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszOut, &RCPtrOut);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOut)\n", pDevIns->pReg->szRCMod, pszOut));
+        }
+        RTRCPTR RCPtrInStr = NIL_RTRCPTR;
+        if (pszInStr && RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszInStr, &RCPtrInStr);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszInStr)\n", pDevIns->pReg->szRCMod, pszInStr));
+        }
+        RTRCPTR RCPtrOutStr = NIL_RTRCPTR;
+        if (pszOutStr && RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszOutStr, &RCPtrOutStr);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOutStr)\n", pDevIns->pReg->szRCMod, pszOutStr));
+        }
+
+        if (RT_SUCCESS(rc))
+        {
+#if 0 /** @todo needs a real string cache for this */
+            if (pDevIns->iInstance > 0)
+            {
+                 char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
+                 if (pszDesc2)
+                     pszDesc = pszDesc2;
+            }
+#endif
+
+            rc = IOMR3IOPortRegisterRC(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts, pvUser, RCPtrOut, RCPtrIn, RCPtrOutStr, RCPtrInStr, pszDesc);
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("No GC module for this driver!\n"));
+        rc = VERR_INVALID_PARAMETER;
+    }
+
+    LogFlow(("pdmR3DevHlp_IOPortRegisterRC: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortRegisterR0} */
+static DECLCALLBACK(int) pdmR3DevHlp_IOPortRegisterR0(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts, RTR0PTR pvUser,
+                                                      const char *pszOut, const char *pszIn,
+                                                      const char *pszOutStr, const char *pszInStr, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_IOPortRegisterR0: caller='%s'/%d: Port=%#x cPorts=%#x pvUser=%p pszOut=%p:{%s} pszIn=%p:{%s} pszOutStr=%p:{%s} pszInStr=%p:{%s} pszDesc=%p:{%s}\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             Port, cPorts, pvUser, pszOut, pszOut, pszIn, pszIn, pszOutStr, pszOutStr, pszInStr, pszInStr, pszDesc, pszDesc));
+
+    /*
+     * Resolve the functions (one of the can be NULL).
+     */
+    int rc = VINF_SUCCESS;
+    if (    pDevIns->pReg->szR0Mod[0]
+        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0))
+    {
+        R0PTRTYPE(PFNIOMIOPORTIN) pfnR0PtrIn = 0;
+        if (pszIn)
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszIn, &pfnR0PtrIn);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszIn)\n", pDevIns->pReg->szR0Mod, pszIn));
+        }
+        R0PTRTYPE(PFNIOMIOPORTOUT) pfnR0PtrOut = 0;
+        if (pszOut && RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszOut, &pfnR0PtrOut);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOut)\n", pDevIns->pReg->szR0Mod, pszOut));
+        }
+        R0PTRTYPE(PFNIOMIOPORTINSTRING) pfnR0PtrInStr = 0;
+        if (pszInStr && RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszInStr, &pfnR0PtrInStr);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszInStr)\n", pDevIns->pReg->szR0Mod, pszInStr));
+        }
+        R0PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnR0PtrOutStr = 0;
+        if (pszOutStr && RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszOutStr, &pfnR0PtrOutStr);
+            AssertMsgRC(rc, ("Failed to resolve %s.%s (pszOutStr)\n", pDevIns->pReg->szR0Mod, pszOutStr));
+        }
+
+        if (RT_SUCCESS(rc))
+        {
+#if 0 /** @todo needs a real string cache for this */
+            if (pDevIns->iInstance > 0)
+            {
+                 char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
+                 if (pszDesc2)
+                     pszDesc = pszDesc2;
+            }
+#endif
+
+            rc = IOMR3IOPortRegisterR0(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts, pvUser, pfnR0PtrOut, pfnR0PtrIn, pfnR0PtrOutStr, pfnR0PtrInStr, pszDesc);
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("No R0 module for this driver!\n"));
+        rc = VERR_INVALID_PARAMETER;
+    }
+
+    LogFlow(("pdmR3DevHlp_IOPortRegisterR0: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIOPortDeregister} */
+static DECLCALLBACK(int) pdmR3DevHlp_IOPortDeregister(PPDMDEVINS pDevIns, RTIOPORT Port, RTUINT cPorts)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_IOPortDeregister: caller='%s'/%d: Port=%#x cPorts=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             Port, cPorts));
+
+    int rc = IOMR3IOPortDeregister(pDevIns->Internal.s.pVMR3, pDevIns, Port, cPorts);
+
+    LogFlow(("pdmR3DevHlp_IOPortDeregister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMIORegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIORegister(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTHCPTR pvUser,
+                                                  PFNIOMMMIOWRITE pfnWrite, PFNIOMMMIOREAD pfnRead, PFNIOMMMIOFILL pfnFill,
+                                                  const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIORegister: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvUser=%p pfnWrite=%p pfnRead=%p pfnFill=%p pszDesc=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvUser, pfnWrite, pfnRead, pfnFill, pszDesc, pszDesc));
+
+/** @todo IOMR3MMIORegisterR3 mangles the description, move it here. */
+    int rc = IOMR3MMIORegisterR3(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvUser, pfnWrite, pfnRead, pfnFill, pszDesc);
+
+    LogFlow(("pdmR3DevHlp_MMIORegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMIORegisterRC} */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIORegisterRC(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTGCPTR pvUser,
+                                                    const char *pszWrite, const char *pszRead, const char *pszFill,
+                                                    const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIORegisterRC: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvUser=%p pszWrite=%p:{%s} pszRead=%p:{%s} pszFill=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvUser, pszWrite, pszWrite, pszRead, pszRead, pszFill, pszFill));
+
+/** @todo pszDesc is unused here, drop it.  */
+
+    /*
+     * Resolve the functions.
+     * Not all function have to present, leave it to IOM to enforce this.
+     */
+    int rc = VINF_SUCCESS;
+    if (    pDevIns->pReg->szRCMod[0]
+        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC))
+    {
+        RTRCPTR RCPtrWrite = NIL_RTRCPTR;
+        if (pszWrite)
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pszWrite, &RCPtrWrite);
+
+        RTRCPTR RCPtrRead = NIL_RTRCPTR;
+        int rc2 = VINF_SUCCESS;
+        if (pszRead)
+            rc2 = pdmR3DevGetSymbolRCLazy(pDevIns, pszRead, &RCPtrRead);
+
+        RTRCPTR RCPtrFill = NIL_RTRCPTR;
+        int rc3 = VINF_SUCCESS;
+        if (pszFill)
+            rc3 = pdmR3DevGetSymbolRCLazy(pDevIns, pszFill, &RCPtrFill);
+
+        if (RT_SUCCESS(rc) && RT_SUCCESS(rc2) && RT_SUCCESS(rc3))
+            rc = IOMR3MMIORegisterRC(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvUser, RCPtrWrite, RCPtrRead, RCPtrFill);
+        else
+        {
+            AssertMsgRC(rc,  ("Failed to resolve %s.%s (pszWrite)\n", pDevIns->pReg->szRCMod, pszWrite));
+            AssertMsgRC(rc2, ("Failed to resolve %s.%s (pszRead)\n",  pDevIns->pReg->szRCMod, pszRead));
+            AssertMsgRC(rc3, ("Failed to resolve %s.%s (pszFill)\n",  pDevIns->pReg->szRCMod, pszFill));
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+            if (RT_FAILURE(rc3) && RT_SUCCESS(rc))
+                rc = rc3;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("No GC module for this driver!\n"));
+        rc = VERR_INVALID_PARAMETER;
+    }
+
+    LogFlow(("pdmR3DevHlp_MMIORegisterRC: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMIORegisterR0} */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIORegisterR0(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, RTR0PTR pvUser,
+                                                    const char *pszWrite, const char *pszRead, const char *pszFill,
+                                                    const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIORegisterHC: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvUser=%p pszWrite=%p:{%s} pszRead=%p:{%s} pszFill=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvUser, pszWrite, pszWrite, pszRead, pszRead, pszFill, pszFill));
+
+/** @todo pszDesc is unused here, remove it.  */
+
+    /*
+     * Resolve the functions.
+     * Not all function have to present, leave it to IOM to enforce this.
+     */
+    int rc = VINF_SUCCESS;
+    if (    pDevIns->pReg->szR0Mod[0]
+        &&  (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0))
+    {
+        R0PTRTYPE(PFNIOMMMIOWRITE) pfnR0PtrWrite = 0;
+        if (pszWrite)
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pszWrite, &pfnR0PtrWrite);
+        R0PTRTYPE(PFNIOMMMIOREAD) pfnR0PtrRead = 0;
+        int rc2 = VINF_SUCCESS;
+        if (pszRead)
+            rc2 = pdmR3DevGetSymbolR0Lazy(pDevIns, pszRead, &pfnR0PtrRead);
+        R0PTRTYPE(PFNIOMMMIOFILL) pfnR0PtrFill = 0;
+        int rc3 = VINF_SUCCESS;
+        if (pszFill)
+            rc3 = pdmR3DevGetSymbolR0Lazy(pDevIns, pszFill, &pfnR0PtrFill);
+        if (RT_SUCCESS(rc) && RT_SUCCESS(rc2) && RT_SUCCESS(rc3))
+            rc = IOMR3MMIORegisterR0(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvUser, pfnR0PtrWrite, pfnR0PtrRead, pfnR0PtrFill);
+        else
+        {
+            AssertMsgRC(rc,  ("Failed to resolve %s.%s (pszWrite)\n", pDevIns->pReg->szR0Mod, pszWrite));
+            AssertMsgRC(rc2, ("Failed to resolve %s.%s (pszRead)\n",  pDevIns->pReg->szR0Mod, pszRead));
+            AssertMsgRC(rc3, ("Failed to resolve %s.%s (pszFill)\n",  pDevIns->pReg->szR0Mod, pszFill));
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+            if (RT_FAILURE(rc3) && RT_SUCCESS(rc))
+                rc = rc3;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("No R0 module for this driver!\n"));
+        rc = VERR_INVALID_PARAMETER;
+    }
+
+    LogFlow(("pdmR3DevHlp_MMIORegisterR0: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMIODeregister} */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIODeregister(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIODeregister: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange));
+
+    int rc = IOMR3MMIODeregister(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange);
+
+    LogFlow(("pdmR3DevHlp_MMIODeregister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnMMIO2Register
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Register(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void **ppv, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIO2Register: caller='%s'/%d: iRegion=%#x cb=%#RGp fFlags=%RX32 ppv=%p pszDescp=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, cb, fFlags, ppv, pszDesc, pszDesc));
+
+/** @todo PGMR3PhysMMIO2Register mangles the description, move it here and
+ *        use a real string cache. */
+    int rc = PGMR3PhysMMIO2Register(pDevIns->Internal.s.pVMR3, pDevIns, iRegion, cb, fFlags, ppv, pszDesc);
+
+    LogFlow(("pdmR3DevHlp_MMIO2Register: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnMMIO2Deregister
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Deregister(PPDMDEVINS pDevIns, uint32_t iRegion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIO2Deregister: caller='%s'/%d: iRegion=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iRegion));
+
+    AssertReturn(iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
+
+    int rc = PGMR3PhysMMIO2Deregister(pDevIns->Internal.s.pVMR3, pDevIns, iRegion);
+
+    LogFlow(("pdmR3DevHlp_MMIO2Deregister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnMMIO2Map
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Map(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIO2Map: caller='%s'/%d: iRegion=%#x GCPhys=%#RGp\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, GCPhys));
+
+    int rc = PGMR3PhysMMIO2Map(pDevIns->Internal.s.pVMR3, pDevIns, iRegion, GCPhys);
+
+    LogFlow(("pdmR3DevHlp_MMIO2Map: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnMMIO2Unmap
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIO2Unmap(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_MMIO2Unmap: caller='%s'/%d: iRegion=%#x GCPhys=%#RGp\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, GCPhys));
+
+    int rc = PGMR3PhysMMIO2Unmap(pDevIns->Internal.s.pVMR3, pDevIns, iRegion, GCPhys);
+
+    LogFlow(("pdmR3DevHlp_MMIO2Unmap: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnMMHyperMapMMIO2
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_MMHyperMapMMIO2(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
+                                                     const char *pszDesc, PRTRCPTR pRCPtr)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_MMHyperMapMMIO2: caller='%s'/%d: iRegion=%#x off=%RGp cb=%RGp pszDesc=%p:{%s} pRCPtr=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, off, cb, pszDesc, pszDesc, pRCPtr));
+
+    if (pDevIns->iInstance > 0)
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+
+    int rc = MMR3HyperMapMMIO2(pVM, pDevIns, iRegion, off, cb, pszDesc, pRCPtr);
+
+    LogFlow(("pdmR3DevHlp_MMHyperMapMMIO2: caller='%s'/%d: returns %Rrc *pRCPtr=%RRv\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *pRCPtr));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnMMIO2MapKernel
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_MMIO2MapKernel(PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
+                                                    const char *pszDesc, PRTR0PTR pR0Ptr)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_MMIO2MapKernel: caller='%s'/%d: iRegion=%#x off=%RGp cb=%RGp pszDesc=%p:{%s} pR0Ptr=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, off, cb, pszDesc, pszDesc, pR0Ptr));
+
+    if (pDevIns->iInstance > 0)
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+
+    int rc = PGMR3PhysMMIO2MapKernel(pVM, pDevIns, iRegion, off, cb, pszDesc, pR0Ptr);
+
+    LogFlow(("pdmR3DevHlp_MMIO2MapKernel: caller='%s'/%d: returns %Rrc *pR0Ptr=%RHv\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *pR0Ptr));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnROMRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_ROMRegister(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange,
+                                                 const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_ROMRegister: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x pvBinary=%p cbBinary=%#x fFlags=%#RX32 pszDesc=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, pvBinary, cbBinary, fFlags, pszDesc, pszDesc));
+
+/** @todo can we mangle pszDesc? */
+    int rc = PGMR3PhysRomRegister(pDevIns->Internal.s.pVMR3, pDevIns, GCPhysStart, cbRange, pvBinary, cbBinary, fFlags, pszDesc);
+
+    LogFlow(("pdmR3DevHlp_ROMRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnROMProtectShadow} */
+static DECLCALLBACK(int) pdmR3DevHlp_ROMProtectShadow(PPDMDEVINS pDevIns, RTGCPHYS GCPhysStart, RTUINT cbRange, PGMROMPROT enmProt)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_ROMProtectShadow: caller='%s'/%d: GCPhysStart=%RGp cbRange=%#x enmProt=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhysStart, cbRange, enmProt));
+
+    int rc = PGMR3PhysRomProtect(pDevIns->Internal.s.pVMR3, GCPhysStart, cbRange, enmProt);
+
+    LogFlow(("pdmR3DevHlp_ROMProtectShadow: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSSMRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_SSMRegister(PPDMDEVINS pDevIns, uint32_t uVersion, size_t cbGuess, const char *pszBefore,
+                                                 PFNSSMDEVLIVEPREP pfnLivePrep, PFNSSMDEVLIVEEXEC pfnLiveExec, PFNSSMDEVLIVEVOTE pfnLiveVote,
+                                                 PFNSSMDEVSAVEPREP pfnSavePrep, PFNSSMDEVSAVEEXEC pfnSaveExec, PFNSSMDEVSAVEDONE pfnSaveDone,
+                                                 PFNSSMDEVLOADPREP pfnLoadPrep, PFNSSMDEVLOADEXEC pfnLoadExec, PFNSSMDEVLOADDONE pfnLoadDone)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_SSMRegister: caller='%s'/%d: uVersion=#x cbGuess=%#x pszBefore=%p:{%s}\n"
+             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoadDone=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uVersion, cbGuess, pszBefore, pszBefore,
+             pfnLivePrep, pfnLiveExec, pfnLiveVote,
+             pfnSavePrep, pfnSaveExec, pfnSaveDone,
+             pfnLoadPrep, pfnLoadExec, pfnLoadDone));
+
+    int rc = SSMR3RegisterDevice(pDevIns->Internal.s.pVMR3, pDevIns, pDevIns->pReg->szName, pDevIns->iInstance,
+                                 uVersion, cbGuess, pszBefore,
+                                 pfnLivePrep, pfnLiveExec, pfnLiveVote,
+                                 pfnSavePrep, pfnSaveExec, pfnSaveDone,
+                                 pfnLoadPrep, pfnLoadExec, pfnLoadDone);
+
+    LogFlow(("pdmR3DevHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimerCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_TMTimerCreate(PPDMDEVINS pDevIns, TMCLOCK enmClock, PFNTMTIMERDEV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
+
+    if (pDevIns->iInstance > 0) /** @todo use a string cache here later. */
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s [%u]", pszDesc, pDevIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+
+    int rc = TMR3TimerCreateDevice(pVM, pDevIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer);
+
+    LogFlow(("pdmR3DevHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMUtcNow} */
+static DECLCALLBACK(PRTTIMESPEC) pdmR3DevHlp_TMUtcNow(PPDMDEVINS pDevIns, PRTTIMESPEC pTime)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMUtcNow: caller='%s'/%d: pTime=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pTime));
+
+    pTime = TMR3UtcNow(pDevIns->Internal.s.pVMR3, pTime);
+
+    LogFlow(("pdmR3DevHlp_TMUtcNow: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, RTTimeSpecGetNano(pTime)));
+    return pTime;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGet} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGet(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGet: caller='%s'\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint64_t u64Time = TMVirtualSyncGet(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGet: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Time));
+    return u64Time;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGetFreq} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGetFreq(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetFreq: caller='%s'\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint64_t u64Freq = TMVirtualGetFreq(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetFreq: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Freq));
+    return u64Freq;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnTMTimeVirtGetNano} */
+static DECLCALLBACK(uint64_t) pdmR3DevHlp_TMTimeVirtGetNano(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetNano: caller='%s'\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    uint64_t u64Time = TMVirtualSyncGet(pDevIns->Internal.s.pVMR3);
+    uint64_t u64Nano = TMVirtualToNano(pDevIns->Internal.s.pVMR3, u64Time);
+
+    LogFlow(("pdmR3DevHlp_TMTimeVirtGetNano: caller='%s'/%d: returns %RU64\n", pDevIns->pReg->szName, pDevIns->iInstance, u64Nano));
+    return u64Nano;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysRead} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysRead(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysRead: caller='%s'/%d: GCPhys=%RGp pvBuf=%p cbRead=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, pvBuf, cbRead));
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc;
+    if (VM_IS_EMT(pVM))
+        rc = PGMPhysRead(pVM, GCPhys, pvBuf, cbRead);
+    else
+        rc = PGMR3PhysReadExternal(pVM, GCPhys, pvBuf, cbRead);
+
+    Log(("pdmR3DevHlp_PhysRead: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysWrite} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysWrite(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysWrite: caller='%s'/%d: GCPhys=%RGp pvBuf=%p cbWrite=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, pvBuf, cbWrite));
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc;
+    if (VM_IS_EMT(pVM))
+        rc = PGMPhysWrite(pVM, GCPhys, pvBuf, cbWrite);
+    else
+        rc = PGMR3PhysWriteExternal(pVM, GCPhys, pvBuf, cbWrite, pDevIns->pReg->szName);
+
+    Log(("pdmR3DevHlp_PhysWrite: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPhys2CCPtr} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPhys2CCPtr(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t fFlags, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysGCPhys2CCPtr: caller='%s'/%d: GCPhys=%RGp fFlags=%#x ppv=%p pLock=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, fFlags, ppv, pLock));
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc = PGMR3PhysGCPhys2CCPtrExternal(pVM, GCPhys, ppv, pLock);
+
+    Log(("pdmR3DevHlp_PhysGCPhys2CCPtr: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPhys2CCPtrReadOnly} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, uint32_t fFlags, const void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly: caller='%s'/%d: GCPhys=%RGp fFlags=%#x ppv=%p pLock=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPhys, fFlags, ppv, pLock));
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    if (!VM_IS_EMT(pVM))
+    {
+        char szNames[128];
+        uint32_t cLocks = PDMR3CritSectCountOwned(pVM, szNames, sizeof(szNames));
+        AssertMsg(cLocks == 0, ("cLocks=%u %s\n", cLocks, szNames));
+    }
+#endif
+
+    int rc = PGMR3PhysGCPhys2CCPtrReadOnlyExternal(pVM, GCPhys, ppv, pLock);
+
+    Log(("pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysReleasePageMappingLock} */
+static DECLCALLBACK(void) pdmR3DevHlp_PhysReleasePageMappingLock(PPDMDEVINS pDevIns, PPGMPAGEMAPLOCK pLock)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DevHlp_PhysReleasePageMappingLock: caller='%s'/%d: pLock=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pLock));
+
+    PGMPhysReleasePageMappingLock(pVM, pLock);
+
+    Log(("pdmR3DevHlp_PhysReleasePageMappingLock: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysReadGCVirt} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysReadGCVirt(PPDMDEVINS pDevIns, void *pvDst, RTGCPTR GCVirtSrc, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PhysReadGCVirt: caller='%s'/%d: pvDst=%p GCVirt=%RGv cb=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pvDst, GCVirtSrc, cb));
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        return VERR_ACCESS_DENIED;
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    /** @todo SMP. */
+#endif
+
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCVirtSrc, cb);
+
+    LogFlow(("pdmR3DevHlp_PhysReadGCVirt: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysWriteGCVirt} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysWriteGCVirt(PPDMDEVINS pDevIns, RTGCPTR GCVirtDst, const void *pvSrc, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PhysWriteGCVirt: caller='%s'/%d: GCVirtDst=%RGv pvSrc=%p cb=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCVirtDst, pvSrc, cb));
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        return VERR_ACCESS_DENIED;
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    /** @todo SMP. */
+#endif
+
+    int rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCVirtDst, pvSrc, cb);
+
+    LogFlow(("pdmR3DevHlp_PhysWriteGCVirt: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPhysGCPtr2GCPhys} */
+static DECLCALLBACK(int) pdmR3DevHlp_PhysGCPtr2GCPhys(PPDMDEVINS pDevIns, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PhysGCPtr2GCPhys: caller='%s'/%d: GCPtr=%RGv pGCPhys=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, GCPtr, pGCPhys));
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        return VERR_ACCESS_DENIED;
+#if defined(VBOX_STRICT) && defined(PDM_DEVHLP_DEADLOCK_DETECTION)
+    /** @todo SMP. */
+#endif
+
+    int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, pGCPhys);
+
+    LogFlow(("pdmR3DevHlp_PhysGCPtr2GCPhys: caller='%s'/%d: returns %Rrc *pGCPhys=%RGp\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *pGCPhys));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapAlloc} */
+static DECLCALLBACK(void *) pdmR3DevHlp_MMHeapAlloc(PPDMDEVINS pDevIns, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: cb=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cb));
+
+    void *pv = MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE_USER, cb);
+
+    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapAllocZ} */
+static DECLCALLBACK(void *) pdmR3DevHlp_MMHeapAllocZ(PPDMDEVINS pDevIns, size_t cb)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MMHeapAllocZ: caller='%s'/%d: cb=%#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cb));
+
+    void *pv = MMR3HeapAllocZ(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE_USER, cb);
+
+    LogFlow(("pdmR3DevHlp_MMHeapAllocZ: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnMMHeapFree} */
+static DECLCALLBACK(void) pdmR3DevHlp_MMHeapFree(PPDMDEVINS pDevIns, void *pv)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_MMHeapFree: caller='%s'/%d: pv=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pv));
+
+    MMR3HeapFree(pv);
+
+    LogFlow(("pdmR3DevHlp_MMHeapAlloc: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMState} */
+static DECLCALLBACK(VMSTATE) pdmR3DevHlp_VMState(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+
+    VMSTATE enmVMState = VMR3GetState(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             enmVMState, VMR3GetStateName(enmVMState)));
+    return enmVMState;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMTeleportedAndNotFullyResumedYet} */
+static DECLCALLBACK(bool) pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+
+    bool fRc = VMR3TeleportedAndNotFullyResumedYet(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_VMState: caller='%s'/%d: returns %RTbool\n", pDevIns->pReg->szName, pDevIns->iInstance,
+             fRc));
+    return fRc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetError} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetError(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pDevIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetErrorV(PPDMDEVINS pDevIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    int rc2 = VMSetErrorV(pDevIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetRuntimeError} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetRuntimeError(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, args);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSetRuntimeErrorV(PPDMDEVINS pDevIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    int rc = VMSetRuntimeErrorV(pDevIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFStopV} */
+static DECLCALLBACK(int) pdmR3DevHlp_DBGFStopV(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction, const char *pszFormat, va_list args)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+#ifdef LOG_ENABLED
+    va_list va2;
+    va_copy(va2, args);
+    LogFlow(("pdmR3DevHlp_DBGFStopV: caller='%s'/%d: pszFile=%p:{%s} iLine=%d pszFunction=%p:{%s} pszFormat=%p:{%s} (%N)\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pszFile, pszFile, iLine, pszFunction, pszFunction, pszFormat, pszFormat, pszFormat, &va2));
+    va_end(va2);
+#endif
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3EventSrcV(pVM, DBGFEVENT_DEV_STOP, pszFile, iLine, pszFunction, pszFormat, args);
+    if (rc == VERR_DBGF_NOT_ATTACHED)
+        rc = VINF_SUCCESS;
+
+    LogFlow(("pdmR3DevHlp_DBGFStopV: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDBGFInfoRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_DBGFInfoRegister(PPDMDEVINS pDevIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDEV pfnHandler)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3InfoRegisterDevice(pVM, pszName, pszDesc, pfnHandler, pDevIns);
+
+    LogFlow(("pdmR3DevHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegister} */
+static DECLCALLBACK(void) pdmR3DevHlp_STAMRegister(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    STAM_REG(pVM, pvSample, enmType, pszName, enmUnit, pszDesc);
+    NOREF(pVM);
+}
+
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegisterF} */
+static DECLCALLBACK(void) pdmR3DevHlp_STAMRegisterF(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, ...)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    AssertRC(rc);
+
+    NOREF(pVM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSTAMRegisterV} */
+static DECLCALLBACK(void) pdmR3DevHlp_STAMRegisterV(PPDMDEVINS pDevIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    AssertRC(rc);
+
+    NOREF(pVM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIRegister(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: pPciDev=%p:{.config={%#.256Rhxs}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pPciDev->config));
+
+    /*
+     * Validate input.
+     */
+    if (!pPciDev)
+    {
+        Assert(pPciDev);
+        LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc (pPciDev)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!pPciDev->config[0] && !pPciDev->config[1])
+    {
+        Assert(pPciDev->config[0] || pPciDev->config[1]);
+        LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc (vendor)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (pDevIns->Internal.s.pPciDeviceR3)
+    {
+        /** @todo the PCI device vs. PDM device designed is a bit flawed if we have to
+         * support a PDM device with multiple PCI devices. This might become a problem
+         * when upgrading the chipset for instance because of multiple functions in some
+         * devices...
+         */
+        AssertMsgFailed(("Only one PCI device per device is currently implemented!\n"));
+        return VERR_INTERNAL_ERROR;
+    }
+
+    /*
+     * Choose the PCI bus for the device.
+     *
+     * This is simple. If the device was configured for a particular bus, the PCIBusNo
+     * configuration value will be set. If not the default bus is 0.
+     */
+    int rc;
+    PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3;
+    if (!pBus)
+    {
+        uint8_t u8Bus;
+        rc = CFGMR3QueryU8Def(pDevIns->Internal.s.pCfgHandle, "PCIBusNo", &u8Bus, 0);
+        AssertLogRelMsgRCReturn(rc, ("Configuration error: PCIBusNo query failed with rc=%Rrc (%s/%d)\n",
+                                     rc, pDevIns->pReg->szName, pDevIns->iInstance), rc);
+        AssertLogRelMsgReturn(u8Bus < RT_ELEMENTS(pVM->pdm.s.aPciBuses),
+                              ("Configuration error: PCIBusNo=%d, max is %d. (%s/%d)\n", u8Bus,
+                               RT_ELEMENTS(pVM->pdm.s.aPciBuses), pDevIns->pReg->szName, pDevIns->iInstance),
+                              VERR_PDM_NO_PCI_BUS);
+        pBus = pDevIns->Internal.s.pPciBusR3 = &pVM->pdm.s.aPciBuses[u8Bus];
+    }
+    if (pBus->pDevInsR3)
+    {
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+            pDevIns->Internal.s.pPciBusR0 = MMHyperR3ToR0(pVM, pDevIns->Internal.s.pPciBusR3);
+        else
+            pDevIns->Internal.s.pPciBusR0 = NIL_RTR0PTR;
+
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+            pDevIns->Internal.s.pPciBusRC = MMHyperR3ToRC(pVM, pDevIns->Internal.s.pPciBusR3);
+        else
+            pDevIns->Internal.s.pPciBusRC = NIL_RTRCPTR;
+
+        /*
+         * Check the configuration for PCI device and function assignment.
+         */
+        int iDev = -1;
+        uint8_t     u8Device;
+        rc = CFGMR3QueryU8(pDevIns->Internal.s.pCfgHandle, "PCIDeviceNo", &u8Device);
+        if (RT_SUCCESS(rc))
+        {
+            if (u8Device > 31)
+            {
+                AssertMsgFailed(("Configuration error: PCIDeviceNo=%d, max is 31. (%s/%d)\n",
+                                 u8Device, pDevIns->pReg->szName, pDevIns->iInstance));
+                return VERR_INTERNAL_ERROR;
+            }
+
+            uint8_t     u8Function;
+            rc = CFGMR3QueryU8(pDevIns->Internal.s.pCfgHandle, "PCIFunctionNo", &u8Function);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("Configuration error: PCIDeviceNo, but PCIFunctionNo query failed with rc=%Rrc (%s/%d)\n",
+                                 rc, pDevIns->pReg->szName, pDevIns->iInstance));
+                return rc;
+            }
+            if (u8Function > 7)
+            {
+                AssertMsgFailed(("Configuration error: PCIFunctionNo=%d, max is 7. (%s/%d)\n",
+                                 u8Function, pDevIns->pReg->szName, pDevIns->iInstance));
+                return VERR_INTERNAL_ERROR;
+            }
+            iDev = (u8Device << 3) | u8Function;
+        }
+        else if (rc != VERR_CFGM_VALUE_NOT_FOUND)
+        {
+            AssertMsgFailed(("Configuration error: PCIDeviceNo query failed with rc=%Rrc (%s/%d)\n",
+                             rc, pDevIns->pReg->szName, pDevIns->iInstance));
+            return rc;
+        }
+
+        /*
+         * Call the pci bus device to do the actual registration.
+         */
+        pdmLock(pVM);
+        rc = pBus->pfnRegisterR3(pBus->pDevInsR3, pPciDev, pDevIns->pReg->szName, iDev);
+        pdmUnlock(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            pPciDev->pDevIns = pDevIns;
+
+            pDevIns->Internal.s.pPciDeviceR3 = pPciDev;
+            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+                pDevIns->Internal.s.pPciDeviceR0 = MMHyperR3ToR0(pVM, pPciDev);
+            else
+                pDevIns->Internal.s.pPciDeviceR0 = NIL_RTR0PTR;
+
+            if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+                pDevIns->Internal.s.pPciDeviceRC = MMHyperR3ToRC(pVM, pPciDev);
+            else
+                pDevIns->Internal.s.pPciDeviceRC = NIL_RTRCPTR;
+
+            Log(("PDM: Registered device '%s'/%d as PCI device %d on bus %d\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, pPciDev->devfn, pDevIns->Internal.s.pPciBusR3->iBus));
+        }
+    }
+    else
+    {
+        AssertLogRelMsgFailed(("Configuration error: No PCI bus available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_PCI_BUS;
+    }
+
+    LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIIORegionRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIIORegionRegister(PPDMDEVINS pDevIns, int iRegion, uint32_t cbRegion, PCIADDRESSSPACE enmType, PFNPCIIOREGIONMAP pfnCallback)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: iRegion=%d cbRegion=%#x enmType=%d pfnCallback=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iRegion, cbRegion, enmType, pfnCallback));
+
+    /*
+     * Validate input.
+     */
+    if (iRegion < 0 || iRegion >= PCI_NUM_REGIONS)
+    {
+        Assert(iRegion >= 0 && iRegion < PCI_NUM_REGIONS);
+        LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (iRegion)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    switch (enmType)
+    {
+        case PCI_ADDRESS_SPACE_IO:
+            /*
+             * Sanity check: don't allow to register more than 32K of the PCI I/O space.
+             */
+            AssertMsgReturn(cbRegion <= _32K,
+                            ("caller='%s'/%d: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cbRegion),
+                            VERR_INVALID_PARAMETER);
+            break;
+
+        case PCI_ADDRESS_SPACE_MEM:
+        case PCI_ADDRESS_SPACE_MEM_PREFETCH:
+            /*
+             * Sanity check: don't allow to register more than 512MB of the PCI MMIO space for
+             * now. If this limit is increased beyond 2GB, adapt the aligned check below as well!
+             */
+            AssertMsgReturn(cbRegion <= 512 * _1M,
+                            ("caller='%s'/%d: %#x\n", pDevIns->pReg->szName, pDevIns->iInstance, cbRegion),
+                            VERR_INVALID_PARAMETER);
+            break;
+        default:
+            AssertMsgFailed(("enmType=%#x is unknown\n", enmType));
+            LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (enmType)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+            return VERR_INVALID_PARAMETER;
+    }
+    if (!pfnCallback)
+    {
+        Assert(pfnCallback);
+        LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc (callback)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    AssertRelease(VMR3GetState(pVM) != VMSTATE_RUNNING);
+
+    /*
+     * Must have a PCI device registered!
+     */
+    int rc;
+    PPCIDEVICE pPciDev = pDevIns->Internal.s.pPciDeviceR3;
+    if (pPciDev)
+    {
+        /*
+         * We're currently restricted to page aligned MMIO regions.
+         */
+        if (    (enmType == PCI_ADDRESS_SPACE_MEM || enmType == PCI_ADDRESS_SPACE_MEM_PREFETCH)
+            &&  cbRegion != RT_ALIGN_32(cbRegion, PAGE_SIZE))
+        {
+            Log(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: aligning cbRegion %#x -> %#x\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, cbRegion, RT_ALIGN_32(cbRegion, PAGE_SIZE)));
+            cbRegion = RT_ALIGN_32(cbRegion, PAGE_SIZE);
+        }
+
+        /*
+         * For registering PCI MMIO memory or PCI I/O memory, the size of the region must be a power of 2!
+         */
+        int iLastSet = ASMBitLastSetU32(cbRegion);
+        Assert(iLastSet > 0);
+        uint32_t cbRegionAligned = RT_BIT_32(iLastSet - 1);
+        if (cbRegion > cbRegionAligned)
+            cbRegion = cbRegionAligned * 2; /* round up */
+
+        PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3;
+        Assert(pBus);
+        pdmLock(pVM);
+        rc = pBus->pfnIORegionRegisterR3(pBus->pDevInsR3, pPciDev, iRegion, cbRegion, enmType, pfnCallback);
+        pdmUnlock(pVM);
+    }
+    else
+    {
+        AssertMsgFailed(("No PCI device registered!\n"));
+        rc = VERR_PDM_NOT_PCI_DEVICE;
+    }
+
+    LogFlow(("pdmR3DevHlp_PCIIORegionRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetConfigCallbacks} */
+static DECLCALLBACK(void) pdmR3DevHlp_PCISetConfigCallbacks(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PFNPCICONFIGREAD pfnRead, PPFNPCICONFIGREAD ppfnReadOld,
+                                                            PFNPCICONFIGWRITE pfnWrite, PPFNPCICONFIGWRITE ppfnWriteOld)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PCISetConfigCallbacks: caller='%s'/%d: pPciDev=%p pfnRead=%p ppfnReadOld=%p pfnWrite=%p ppfnWriteOld=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciDev, pfnRead, ppfnReadOld, pfnWrite, ppfnWriteOld));
+
+    /*
+     * Validate input and resolve defaults.
+     */
+    AssertPtr(pfnRead);
+    AssertPtr(pfnWrite);
+    AssertPtrNull(ppfnReadOld);
+    AssertPtrNull(ppfnWriteOld);
+    AssertPtrNull(pPciDev);
+
+    if (!pPciDev)
+        pPciDev = pDevIns->Internal.s.pPciDeviceR3;
+    AssertReleaseMsg(pPciDev, ("You must register your device first!\n"));
+    PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3;
+    AssertRelease(pBus);
+    AssertRelease(VMR3GetState(pVM) != VMSTATE_RUNNING);
+
+    /*
+     * Do the job.
+     */
+    pdmLock(pVM);
+    pBus->pfnSetConfigCallbacksR3(pBus->pDevInsR3, pPciDev, pfnRead, ppfnReadOld, pfnWrite, ppfnWriteOld);
+    pdmUnlock(pVM);
+
+    LogFlow(("pdmR3DevHlp_PCISetConfigCallbacks: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetIrq} */
+static DECLCALLBACK(void) pdmR3DevHlp_PCISetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
+
+    /*
+     * Validate input.
+     */
+    /** @todo iIrq and iLevel checks. */
+
+    /*
+     * Must have a PCI device registered!
+     */
+    PPCIDEVICE pPciDev = pDevIns->Internal.s.pPciDeviceR3;
+    if (pPciDev)
+    {
+        PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3; /** @todo the bus should be associated with the PCI device not the PDM device. */
+        Assert(pBus);
+        PVM pVM = pDevIns->Internal.s.pVMR3;
+        pdmLock(pVM);
+        pBus->pfnSetIrqR3(pBus->pDevInsR3, pPciDev, iIrq, iLevel);
+        pdmUnlock(pVM);
+    }
+    else
+        AssertReleaseMsgFailed(("No PCI device registered!\n"));
+
+    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCISetIrqNoWait} */
+static DECLCALLBACK(void) pdmR3DevHlp_PCISetIrqNoWait(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    pdmR3DevHlp_PCISetIrq(pDevIns, iIrq, iLevel);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIRegisterMsi} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIRegisterMsi(PPDMDEVINS pDevIns, PPDMMSIREG pMsiReg)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_PCIRegisterMsi: caller='%s'/%d: %d MSI vectors %d MSI-X vectors\n", pDevIns->pReg->szName, pDevIns->iInstance, pMsiReg->cMsiVectors,pMsiReg->cMsixVectors ));
+    int rc = VINF_SUCCESS;
+
+    /*
+     * Must have a PCI device registered!
+     */
+    PPCIDEVICE pPciDev = pDevIns->Internal.s.pPciDeviceR3;
+    if (pPciDev)
+    {
+        PPDMPCIBUS pBus = pDevIns->Internal.s.pPciBusR3; /** @todo the bus should be associated with the PCI device not the PDM device. */
+        Assert(pBus);
+
+        PVM pVM = pDevIns->Internal.s.pVMR3;
+        pdmLock(pVM);
+        if (!pBus->pfnRegisterMsiR3)
+            rc = VERR_NOT_IMPLEMENTED;
+        else
+            rc = pBus->pfnRegisterMsiR3(pBus->pDevInsR3, pPciDev, pMsiReg);
+        pdmUnlock(pVM);
+    }
+    else
+        AssertReleaseMsgFailed(("No PCI device registered!\n"));
+
+    LogFlow(("pdmR3DevHlp_PCISetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return rc;
+}
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnISASetIrq} */
+static DECLCALLBACK(void) pdmR3DevHlp_ISASetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_ISASetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
+
+    /*
+     * Validate input.
+     */
+    /** @todo iIrq and iLevel checks. */
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    PDMIsaSetIrq(pVM, iIrq, iLevel);    /* (The API takes the lock.) */
+
+    LogFlow(("pdmR3DevHlp_ISASetIrq: caller='%s'/%d: returns void\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnISASetIrqNoWait} */
+static DECLCALLBACK(void) pdmR3DevHlp_ISASetIrqNoWait(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    pdmR3DevHlp_ISASetIrq(pDevIns, iIrq, iLevel);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDriverAttach} */
+static DECLCALLBACK(int) pdmR3DevHlp_DriverAttach(PPDMDEVINS pDevIns, RTUINT iLun, PPDMIBASE pBaseInterface, PPDMIBASE *ppBaseInterface, const char *pszDesc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: iLun=%d pBaseInterface=%p ppBaseInterface=%p pszDesc=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iLun, pBaseInterface, ppBaseInterface, pszDesc, pszDesc));
+
+    /*
+     * Lookup the LUN, it might already be registered.
+     */
+    PPDMLUN pLunPrev = NULL;
+    PPDMLUN pLun = pDevIns->Internal.s.pLunsR3;
+    for (; pLun; pLunPrev = pLun, pLun = pLun->pNext)
+        if (pLun->iLun == iLun)
+            break;
+
+    /*
+     * Create the LUN if if wasn't found, else check if driver is already attached to it.
+     */
+    if (!pLun)
+    {
+        if (    !pBaseInterface
+            ||  !pszDesc
+            ||  !*pszDesc)
+        {
+            Assert(pBaseInterface);
+            Assert(pszDesc || *pszDesc);
+            return VERR_INVALID_PARAMETER;
+        }
+
+        pLun = (PPDMLUN)MMR3HeapAlloc(pVM, MM_TAG_PDM_LUN, sizeof(*pLun));
+        if (!pLun)
+            return VERR_NO_MEMORY;
+
+        pLun->iLun      = iLun;
+        pLun->pNext     = pLunPrev ? pLunPrev->pNext : NULL;
+        pLun->pTop      = NULL;
+        pLun->pBottom   = NULL;
+        pLun->pDevIns   = pDevIns;
+        pLun->pUsbIns   = NULL;
+        pLun->pszDesc   = pszDesc;
+        pLun->pBase     = pBaseInterface;
+        if (!pLunPrev)
+            pDevIns->Internal.s.pLunsR3 = pLun;
+        else
+            pLunPrev->pNext = pLun;
+        Log(("pdmR3DevHlp_DriverAttach: Registered LUN#%d '%s' with device '%s'/%d.\n",
+             iLun, pszDesc, pDevIns->pReg->szName, pDevIns->iInstance));
+    }
+    else if (pLun->pTop)
+    {
+        AssertMsgFailed(("Already attached! The device should keep track of such things!\n"));
+        LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_PDM_DRIVER_ALREADY_ATTACHED));
+        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+    Assert(pLun->pBase == pBaseInterface);
+
+
+    /*
+     * Get the attached driver configuration.
+     */
+    int rc;
+    PCFGMNODE pNode = CFGMR3GetChildF(pDevIns->Internal.s.pCfgHandle, "LUN#%u", iLun);
+    if (pNode)
+        rc = pdmR3DrvInstantiate(pVM, pNode, pBaseInterface, NULL /*pDrvAbove*/, pLun, ppBaseInterface);
+    else
+        rc = VERR_PDM_NO_ATTACHED_DRIVER;
+
+
+    LogFlow(("pdmR3DevHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnQueueCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_QueueCreate(PPDMDEVINS pDevIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
+                                                 PFNPDMQUEUEDEV pfnCallback, bool fGCEnabled, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_QueueCreate: caller='%s'/%d: cbItem=%#x cItems=%#x cMilliesInterval=%u pfnCallback=%p fGCEnabled=%RTbool pszName=%p:{%s} ppQueue=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, fGCEnabled, pszName, pszName, ppQueue));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    if (pDevIns->iInstance > 0)
+    {
+        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s_%u", pszName, pDevIns->iInstance);
+        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
+    }
+
+    int rc = PDMR3QueueCreateDevice(pVM, pDevIns, cbItem, cItems, cMilliesInterval, pfnCallback, fGCEnabled, pszName, ppQueue);
+
+    LogFlow(("pdmR3DevHlp_QueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, rc, *ppQueue));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCritSectInit} */
+static DECLCALLBACK(int) pdmR3DevHlp_CritSectInit(PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL,
+                                                  const char *pszNameFmt, va_list va)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_CritSectInit: caller='%s'/%d: pCritSect=%p pszNameFmt=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pCritSect, pszNameFmt, pszNameFmt));
+
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    int rc = pdmR3CritSectInitDevice(pVM, pDevIns, pCritSect, RT_SRC_POS_ARGS, pszNameFmt, va);
+
+    LogFlow(("pdmR3DevHlp_CritSectInit: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnThreadCreate} */
+static DECLCALLBACK(int) pdmR3DevHlp_ThreadCreate(PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
+                                                  PFNPDMTHREADWAKEUPDEV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_ThreadCreate: caller='%s'/%d: ppThread=%p pvUser=%p pfnThread=%p pfnWakeup=%p cbStack=%#zx enmType=%d pszName=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName, pszName));
+
+    int rc = pdmR3ThreadCreateDevice(pDevIns->Internal.s.pVMR3, pDevIns, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName);
+
+    LogFlow(("pdmR3DevHlp_ThreadCreate: caller='%s'/%d: returns %Rrc *ppThread=%RTthrd\n", pDevIns->pReg->szName, pDevIns->iInstance,
+            rc, *ppThread));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnSetAsyncNotification} */
+static DECLCALLBACK(int) pdmR3DevHlp_SetAsyncNotification(PPDMDEVINS pDevIns, PFNPDMDEVASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT0(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pDevIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pDevIns->Internal.s.fIntFlags & (PDMDEVINSINT_FLAGS_SUSPENDED | PDMDEVINSINT_FLAGS_RESET), rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pDevIns->Internal.s.pVMR3);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_RESETTING
+               || enmVMState == VMSTATE_RESETTING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pDevIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3DevHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnAsyncNotificationCompleted} */
+static DECLCALLBACK(void) pdmR3DevHlp_AsyncNotificationCompleted(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (   enmVMState == VMSTATE_SUSPENDING
+        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+        || enmVMState == VMSTATE_SUSPENDING_LS
+        || enmVMState == VMSTATE_RESETTING
+        || enmVMState == VMSTATE_RESETTING_LS
+        || enmVMState == VMSTATE_POWERING_OFF
+        || enmVMState == VMSTATE_POWERING_OFF_LS)
+    {
+        LogFlow(("pdmR3DevHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
+    }
+    else
+        LogFlow(("pdmR3DevHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, enmVMState));
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnRTCRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_RTCRegister(PPDMDEVINS pDevIns, PCPDMRTCREG pRtcReg, PCPDMRTCHLP *ppRtcHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: pRtcReg=%p:{.u32Version=%#x, .pfnWrite=%p, .pfnRead=%p} ppRtcHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRtcReg, pRtcReg->u32Version, pRtcReg->pfnWrite,
+             pRtcReg->pfnWrite, ppRtcHlp));
+
+    /*
+     * Validate input.
+     */
+    if (pRtcReg->u32Version != PDM_RTCREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pRtcReg->u32Version,
+                         PDM_RTCREG_VERSION));
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (version)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    !pRtcReg->pfnWrite
+        ||  !pRtcReg->pfnRead)
+    {
+        Assert(pRtcReg->pfnWrite);
+        Assert(pRtcReg->pfnRead);
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (callbacks)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppRtcHlp)
+    {
+        Assert(ppRtcHlp);
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc (ppRtcHlp)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one DMA device.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (pVM->pdm.s.pRtc)
+    {
+        AssertMsgFailed(("Only one RTC device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate and initialize pci bus structure.
+     */
+    int rc = VINF_SUCCESS;
+    PPDMRTC pRtc = (PPDMRTC)MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE, sizeof(*pRtc));
+    if (pRtc)
+    {
+        pRtc->pDevIns   = pDevIns;
+        pRtc->Reg       = *pRtcReg;
+        pVM->pdm.s.pRtc = pRtc;
+
+        /* set the helper pointer. */
+        *ppRtcHlp = &g_pdmR3DevRtcHlp;
+        Log(("PDM: Registered RTC device '%s'/%d pDevIns=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    LogFlow(("pdmR3DevHlp_RTCRegister: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMARegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMARegister(PPDMDEVINS pDevIns, unsigned uChannel, PFNDMATRANSFERHANDLER pfnTransferHandler, void *pvUser)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMARegister: caller='%s'/%d: uChannel=%d pfnTransferHandler=%p pvUser=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pfnTransferHandler, pvUser));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+        pVM->pdm.s.pDmac->Reg.pfnRegister(pVM->pdm.s.pDmac->pDevIns, uChannel, pfnTransferHandler, pvUser);
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMARegister: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMAReadMemory} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMAReadMemory(PPDMDEVINS pDevIns, unsigned uChannel, void *pvBuffer, uint32_t off, uint32_t cbBlock, uint32_t *pcbRead)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMAReadMemory: caller='%s'/%d: uChannel=%d pvBuffer=%p off=%#x cbBlock=%#x pcbRead=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pvBuffer, off, cbBlock, pcbRead));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+    {
+        uint32_t cb = pVM->pdm.s.pDmac->Reg.pfnReadMemory(pVM->pdm.s.pDmac->pDevIns, uChannel, pvBuffer, off, cbBlock);
+        if (pcbRead)
+            *pcbRead = cb;
+    }
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMAReadMemory: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMAWriteMemory} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMAWriteMemory(PPDMDEVINS pDevIns, unsigned uChannel, const void *pvBuffer, uint32_t off, uint32_t cbBlock, uint32_t *pcbWritten)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMAWriteMemory: caller='%s'/%d: uChannel=%d pvBuffer=%p off=%#x cbBlock=%#x pcbWritten=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, pvBuffer, off, cbBlock, pcbWritten));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+    {
+        uint32_t cb = pVM->pdm.s.pDmac->Reg.pfnWriteMemory(pVM->pdm.s.pDmac->pDevIns, uChannel, pvBuffer, off, cbBlock);
+        if (pcbWritten)
+            *pcbWritten = cb;
+    }
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMAWriteMemory: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMASetDREQ} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMASetDREQ(PPDMDEVINS pDevIns, unsigned uChannel, unsigned uLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMASetDREQ: caller='%s'/%d: uChannel=%d uLevel=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel, uLevel));
+    int rc = VINF_SUCCESS;
+    if (pVM->pdm.s.pDmac)
+        pVM->pdm.s.pDmac->Reg.pfnSetDREQ(pVM->pdm.s.pDmac->pDevIns, uChannel, uLevel);
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        rc = VERR_PDM_NO_DMAC_INSTANCE;
+    }
+    LogFlow(("pdmR3DevHlp_DMASetDREQ: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMAGetChannelMode} */
+static DECLCALLBACK(uint8_t) pdmR3DevHlp_DMAGetChannelMode(PPDMDEVINS pDevIns, unsigned uChannel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMAGetChannelMode: caller='%s'/%d: uChannel=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uChannel));
+    uint8_t u8Mode;
+    if (pVM->pdm.s.pDmac)
+        u8Mode = pVM->pdm.s.pDmac->Reg.pfnGetChannelMode(pVM->pdm.s.pDmac->pDevIns, uChannel);
+    else
+    {
+        AssertMsgFailed(("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+        u8Mode = 3 << 2 /* illegal mode type */;
+    }
+    LogFlow(("pdmR3DevHlp_DMAGetChannelMode: caller='%s'/%d: returns %#04x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, u8Mode));
+    return u8Mode;
+}
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMASchedule} */
+static DECLCALLBACK(void) pdmR3DevHlp_DMASchedule(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_DMASchedule: caller='%s'/%d: VM_FF_PDM_DMA %d -> 1\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VM_FF_ISSET(pVM, VM_FF_PDM_DMA)));
+
+    AssertMsg(pVM->pdm.s.pDmac, ("Configuration error: No DMAC controller available. This could be related to init order too!\n"));
+    VM_FF_SET(pVM, VM_FF_PDM_DMA);
+    REMR3NotifyDmaPending(pVM);
+    VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCMOSWrite} */
+static DECLCALLBACK(int) pdmR3DevHlp_CMOSWrite(PPDMDEVINS pDevIns, unsigned iReg, uint8_t u8Value)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: iReg=%#04x u8Value=%#04x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iReg, u8Value));
+    int rc;
+    if (pVM->pdm.s.pRtc)
+        rc = pVM->pdm.s.pRtc->Reg.pfnWrite(pVM->pdm.s.pRtc->pDevIns, iReg, u8Value);
+    else
+        rc = VERR_PDM_NO_RTC_INSTANCE;
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: return %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnCMOSRead} */
+static DECLCALLBACK(int) pdmR3DevHlp_CMOSRead(PPDMDEVINS pDevIns, unsigned iReg, uint8_t *pu8Value)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: iReg=%#04x pu8Value=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iReg, pu8Value));
+    int rc;
+    if (pVM->pdm.s.pRtc)
+        rc = pVM->pdm.s.pRtc->Reg.pfnRead(pVM->pdm.s.pRtc->pDevIns, iReg, pu8Value);
+    else
+        rc = VERR_PDM_NO_RTC_INSTANCE;
+
+    LogFlow(("pdmR3DevHlp_CMOSWrite: caller='%s'/%d: return %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnAssertEMT} */
+static DECLCALLBACK(bool) pdmR3DevHlp_AssertEMT(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (VM_IS_EMT(pDevIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnAssertOther} */
+static DECLCALLBACK(bool) pdmR3DevHlp_AssertOther(PPDMDEVINS pDevIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    if (!VM_IS_EMT(pDevIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDEVHLP,pfnLdrGetRCInterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DevHlp_LdrGetRCInterfaceSymbols(PPDMDEVINS pDevIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_PDMLdrGetRCInterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "dev", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDevIns->pReg->szName) != NULL)
+    {
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+            rc = PDMR3LdrGetInterfaceSymbols(pDevIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDevIns->pReg->szRCMod, pDevIns->Internal.s.pDevR3->pszRCSearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             false /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a raw-mode enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'dev' and contain the driver name!\n",
+                         pszSymPrefix, pDevIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DevHlp_PDMLdrGetRCInterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
+             pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLP,pfnLdrGetR0InterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DevHlp_LdrGetR0InterfaceSymbols(PPDMDEVINS pDevIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_PDMLdrGetR0InterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "dev", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDevIns->pReg->szName) != NULL)
+    {
+        if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+            rc = PDMR3LdrGetInterfaceSymbols(pDevIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDevIns->pReg->szR0Mod, pDevIns->Internal.s.pDevR3->pszR0SearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             true /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a ring-0 enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'dev' and contain the driver name!\n",
+                         pszSymPrefix, pDevIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DevHlp_PDMLdrGetR0InterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
+             pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLP,pfnCallR0} */
+static DECLCALLBACK(int) pdmR3DevHlp_CallR0(PPDMDEVINS pDevIns, uint32_t uOperation, uint64_t u64Arg)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_CallR0: caller='%s'/%d: uOperation=%#x u64Arg=%#RX64\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, uOperation, u64Arg));
+
+    /*
+     * Resolve the ring-0 entry point.  There is not need to remember this like
+     * we do for drivers since this is mainly for construction time hacks and
+     * other things that aren't performance critical.
+     */
+    int rc;
+    if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+    {
+        char szSymbol[          sizeof("devR0") + sizeof(pDevIns->pReg->szName) + sizeof("ReqHandler")];
+        strcat(strcat(strcpy(szSymbol, "devR0"),         pDevIns->pReg->szName),         "ReqHandler");
+        szSymbol[sizeof("devR0") - 1] = RT_C_TO_UPPER(szSymbol[sizeof("devR0") - 1]);
+
+        PFNPDMDRVREQHANDLERR0 pfnReqHandlerR0;
+        rc = pdmR3DevGetSymbolR0Lazy(pDevIns, szSymbol, &pfnReqHandlerR0);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Make the ring-0 call.
+             */
+            PDMDEVICECALLREQHANDLERREQ Req;
+            Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+            Req.Hdr.cbReq       = sizeof(Req);
+            Req.pDevInsR0       = PDMDEVINS_2_R0PTR(pDevIns);
+            Req.pfnReqHandlerR0 = pfnReqHandlerR0;
+            Req.uOperation      = uOperation;
+            Req.u32Alignment    = 0;
+            Req.u64Arg          = u64Arg;
+            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_PDM_DEVICE_CALL_REQ_HANDLER, 0, &Req.Hdr);
+        }
+        else
+            pfnReqHandlerR0 = NIL_RTR0PTR;
+    }
+    else
+        rc = VERR_ACCESS_DENIED;
+    LogFlow(("pdmR3DevHlp_CallR0: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName,
+             pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVM} */
+static DECLCALLBACK(PVM) pdmR3DevHlp_GetVM(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3DevHlp_GetVM: caller='%s'/%d: returns %p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns->Internal.s.pVMR3));
+    return pDevIns->Internal.s.pVMR3;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVMCPU} */
+static DECLCALLBACK(PVMCPU) pdmR3DevHlp_GetVMCPU(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_GetVMCPU: caller='%s'/%d for CPU %u\n", pDevIns->pReg->szName, pDevIns->iInstance, VMMGetCpuId(pDevIns->Internal.s.pVMR3)));
+    return VMMGetCpu(pDevIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPCIBusRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PCIBusRegister(PPDMDEVINS pDevIns, PPDMPCIBUSREG pPciBusReg, PCPDMPCIHLPR3 *ppPciHlpR3)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: pPciBusReg=%p:{.u32Version=%#x, .pfnRegisterR3=%p, .pfnIORegionRegisterR3=%p, .pfnSetIrqR3=%p, "
+             ".pfnSaveExecR3=%p, .pfnLoadExecR3=%p, .pfnFakePCIBIOSR3=%p, .pszSetIrqRC=%p:{%s}, .pszSetIrqR0=%p:{%s}} ppPciHlpR3=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPciBusReg, pPciBusReg->u32Version, pPciBusReg->pfnRegisterR3,
+             pPciBusReg->pfnIORegionRegisterR3, pPciBusReg->pfnSetIrqR3, pPciBusReg->pfnSaveExecR3, pPciBusReg->pfnLoadExecR3,
+             pPciBusReg->pfnFakePCIBIOSR3, pPciBusReg->pszSetIrqRC, pPciBusReg->pszSetIrqRC, pPciBusReg->pszSetIrqR0, pPciBusReg->pszSetIrqR0, ppPciHlpR3));
+
+    /*
+     * Validate the structure.
+     */
+    if (pPciBusReg->u32Version != PDM_PCIBUSREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pPciBusReg->u32Version, PDM_PCIBUSREG_VERSION));
+        LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    !pPciBusReg->pfnRegisterR3
+        ||  !pPciBusReg->pfnIORegionRegisterR3
+        ||  !pPciBusReg->pfnSetIrqR3
+        ||  !pPciBusReg->pfnSaveExecR3
+        ||  !pPciBusReg->pfnLoadExecR3
+        ||  (!pPciBusReg->pfnFakePCIBIOSR3 && !pVM->pdm.s.aPciBuses[0].pDevInsR3)) /* Only the first bus needs to do the BIOS work. */
+    {
+        Assert(pPciBusReg->pfnRegisterR3);
+        Assert(pPciBusReg->pfnIORegionRegisterR3);
+        Assert(pPciBusReg->pfnSetIrqR3);
+        Assert(pPciBusReg->pfnSaveExecR3);
+        Assert(pPciBusReg->pfnLoadExecR3);
+        Assert(pPciBusReg->pfnFakePCIBIOSR3);
+        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pPciBusReg->pszSetIrqRC
+        &&  !VALID_PTR(pPciBusReg->pszSetIrqRC))
+    {
+        Assert(VALID_PTR(pPciBusReg->pszSetIrqRC));
+        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pPciBusReg->pszSetIrqR0
+        &&  !VALID_PTR(pPciBusReg->pszSetIrqR0))
+    {
+        Assert(VALID_PTR(pPciBusReg->pszSetIrqR0));
+        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+     if (!ppPciHlpR3)
+    {
+        Assert(ppPciHlpR3);
+        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (ppPciHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Find free PCI bus entry.
+     */
+    unsigned iBus = 0;
+    for (iBus = 0; iBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses); iBus++)
+        if (!pVM->pdm.s.aPciBuses[iBus].pDevInsR3)
+            break;
+    if (iBus >= RT_ELEMENTS(pVM->pdm.s.aPciBuses))
+    {
+        AssertMsgFailed(("Too many PCI buses. Max=%u\n", RT_ELEMENTS(pVM->pdm.s.aPciBuses)));
+        LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc (pci bus)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    PPDMPCIBUS pPciBus = &pVM->pdm.s.aPciBuses[iBus];
+
+    /*
+     * Resolve and init the RC bits.
+     */
+    if (pPciBusReg->pszSetIrqRC)
+    {
+        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pPciBusReg->pszSetIrqRC, &pPciBus->pfnSetIrqRC);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pPciBusReg->pszSetIrqRC, rc));
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pPciBus->pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
+    }
+    else
+    {
+        pPciBus->pfnSetIrqRC  = 0;
+        pPciBus->pDevInsRC    = 0;
+    }
+
+    /*
+     * Resolve and init the R0 bits.
+     */
+    if (pPciBusReg->pszSetIrqR0)
+    {
+        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pPciBusReg->pszSetIrqR0, &pPciBus->pfnSetIrqR0);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pPciBusReg->pszSetIrqR0, rc));
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_PCIRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pPciBus->pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
+    }
+    else
+    {
+        pPciBus->pfnSetIrqR0 = 0;
+        pPciBus->pDevInsR0   = 0;
+    }
+
+    /*
+     * Init the R3 bits.
+     */
+    pPciBus->iBus                    = iBus;
+    pPciBus->pDevInsR3               = pDevIns;
+    pPciBus->pfnRegisterR3           = pPciBusReg->pfnRegisterR3;
+    pPciBus->pfnRegisterMsiR3        = pPciBusReg->pfnRegisterMsiR3;
+    pPciBus->pfnIORegionRegisterR3   = pPciBusReg->pfnIORegionRegisterR3;
+    pPciBus->pfnSetConfigCallbacksR3 = pPciBusReg->pfnSetConfigCallbacksR3;
+    pPciBus->pfnSetIrqR3             = pPciBusReg->pfnSetIrqR3;
+    pPciBus->pfnSaveExecR3           = pPciBusReg->pfnSaveExecR3;
+    pPciBus->pfnLoadExecR3           = pPciBusReg->pfnLoadExecR3;
+    pPciBus->pfnFakePCIBIOSR3        = pPciBusReg->pfnFakePCIBIOSR3;
+
+    Log(("PDM: Registered PCI bus device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppPciHlpR3 = &g_pdmR3DevPciHlp;
+    LogFlow(("pdmR3DevHlp_PCIBusRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnPICRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_PICRegister(PPDMDEVINS pDevIns, PPDMPICREG pPicReg, PCPDMPICHLPR3 *ppPicHlpR3)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: pPicReg=%p:{.u32Version=%#x, .pfnSetIrqR3=%p, .pfnGetInterruptR3=%p, .pszGetIrqRC=%p:{%s}, .pszGetInterruptRC=%p:{%s}, .pszGetIrqR0=%p:{%s}, .pszGetInterruptR0=%p:{%s} } ppPicHlpR3=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pPicReg, pPicReg->u32Version, pPicReg->pfnSetIrqR3, pPicReg->pfnGetInterruptR3,
+             pPicReg->pszSetIrqRC, pPicReg->pszSetIrqRC, pPicReg->pszGetInterruptRC, pPicReg->pszGetInterruptRC,
+             pPicReg->pszSetIrqR0, pPicReg->pszSetIrqR0, pPicReg->pszGetInterruptR0, pPicReg->pszGetInterruptR0,
+             ppPicHlpR3));
+
+    /*
+     * Validate input.
+     */
+    if (pPicReg->u32Version != PDM_PICREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pPicReg->u32Version, PDM_PICREG_VERSION));
+        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    !pPicReg->pfnSetIrqR3
+        ||  !pPicReg->pfnGetInterruptR3)
+    {
+        Assert(pPicReg->pfnSetIrqR3);
+        Assert(pPicReg->pfnGetInterruptR3);
+        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    (   pPicReg->pszSetIrqRC
+             || pPicReg->pszGetInterruptRC)
+        &&  (   !VALID_PTR(pPicReg->pszSetIrqRC)
+             || !VALID_PTR(pPicReg->pszGetInterruptRC))
+       )
+    {
+        Assert(VALID_PTR(pPicReg->pszSetIrqRC));
+        Assert(VALID_PTR(pPicReg->pszGetInterruptRC));
+        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (RC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pPicReg->pszSetIrqRC
+        &&  !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC))
+    {
+        Assert(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC);
+        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (RC flag)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pPicReg->pszSetIrqR0
+        &&  !(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0))
+    {
+        Assert(pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0);
+        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (R0 flag)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!ppPicHlpR3)
+    {
+        Assert(ppPicHlpR3);
+        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc (ppPicHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one PIC device.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (pVM->pdm.s.Pic.pDevInsR3)
+    {
+        AssertMsgFailed(("Only one pic device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * RC stuff.
+     */
+    if (pPicReg->pszSetIrqRC)
+    {
+        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pPicReg->pszSetIrqRC, &pVM->pdm.s.Pic.pfnSetIrqRC);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pPicReg->pszSetIrqRC, rc));
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pPicReg->pszGetInterruptRC, &pVM->pdm.s.Pic.pfnGetInterruptRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pPicReg->pszGetInterruptRC, rc));
+        }
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pVM->pdm.s.Pic.pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
+    }
+    else
+    {
+        pVM->pdm.s.Pic.pDevInsRC = 0;
+        pVM->pdm.s.Pic.pfnSetIrqRC = 0;
+        pVM->pdm.s.Pic.pfnGetInterruptRC = 0;
+    }
+
+    /*
+     * R0 stuff.
+     */
+    if (pPicReg->pszSetIrqR0)
+    {
+        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pPicReg->pszSetIrqR0, &pVM->pdm.s.Pic.pfnSetIrqR0);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pPicReg->pszSetIrqR0, rc));
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pPicReg->pszGetInterruptR0, &pVM->pdm.s.Pic.pfnGetInterruptR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pPicReg->pszGetInterruptR0, rc));
+        }
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pVM->pdm.s.Pic.pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
+        Assert(pVM->pdm.s.Pic.pDevInsR0);
+    }
+    else
+    {
+        pVM->pdm.s.Pic.pfnSetIrqR0 = 0;
+        pVM->pdm.s.Pic.pfnGetInterruptR0 = 0;
+        pVM->pdm.s.Pic.pDevInsR0 = 0;
+    }
+
+    /*
+     * R3 stuff.
+     */
+    pVM->pdm.s.Pic.pDevInsR3 = pDevIns;
+    pVM->pdm.s.Pic.pfnSetIrqR3 = pPicReg->pfnSetIrqR3;
+    pVM->pdm.s.Pic.pfnGetInterruptR3 = pPicReg->pfnGetInterruptR3;
+    Log(("PDM: Registered PIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppPicHlpR3 = &g_pdmR3DevPicHlp;
+    LogFlow(("pdmR3DevHlp_PICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnAPICRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_APICRegister(PPDMDEVINS pDevIns, PPDMAPICREG pApicReg, PCPDMAPICHLPR3 *ppApicHlpR3)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: pApicReg=%p:{.u32Version=%#x, .pfnGetInterruptR3=%p, .pfnSetBaseR3=%p, .pfnGetBaseR3=%p, "
+             ".pfnSetTPRR3=%p, .pfnGetTPRR3=%p, .pfnWriteMSR3=%p, .pfnReadMSR3=%p, .pfnBusDeliverR3=%p, .pfnLocalInterruptR3=%p, pszGetInterruptRC=%p:{%s}, pszSetBaseRC=%p:{%s}, pszGetBaseRC=%p:{%s}, "
+             ".pszSetTPRRC=%p:{%s}, .pszGetTPRRC=%p:{%s}, .pszWriteMSRRC=%p:{%s}, .pszReadMSRRC=%p:{%s}, .pszBusDeliverRC=%p:{%s}, .pszLocalInterruptRC=%p:{%s}} ppApicHlpR3=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pApicReg, pApicReg->u32Version, pApicReg->pfnGetInterruptR3, pApicReg->pfnSetBaseR3,
+             pApicReg->pfnGetBaseR3, pApicReg->pfnSetTPRR3, pApicReg->pfnGetTPRR3, pApicReg->pfnWriteMSRR3, pApicReg->pfnReadMSRR3, pApicReg->pfnBusDeliverR3, pApicReg->pfnLocalInterruptR3, pApicReg->pszGetInterruptRC,
+             pApicReg->pszGetInterruptRC, pApicReg->pszSetBaseRC, pApicReg->pszSetBaseRC, pApicReg->pszGetBaseRC, pApicReg->pszGetBaseRC,
+             pApicReg->pszSetTPRRC, pApicReg->pszSetTPRRC, pApicReg->pszGetTPRRC, pApicReg->pszGetTPRRC, pApicReg->pszWriteMSRRC, pApicReg->pszWriteMSRRC, pApicReg->pszReadMSRRC, pApicReg->pszReadMSRRC, pApicReg->pszBusDeliverRC,
+             pApicReg->pszBusDeliverRC, pApicReg->pszLocalInterruptRC, pApicReg->pszLocalInterruptRC, ppApicHlpR3));
+
+    /*
+     * Validate input.
+     */
+    if (pApicReg->u32Version != PDM_APICREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pApicReg->u32Version, PDM_APICREG_VERSION));
+        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    !pApicReg->pfnGetInterruptR3
+        ||  !pApicReg->pfnHasPendingIrqR3
+        ||  !pApicReg->pfnSetBaseR3
+        ||  !pApicReg->pfnGetBaseR3
+        ||  !pApicReg->pfnSetTPRR3
+        ||  !pApicReg->pfnGetTPRR3
+        ||  !pApicReg->pfnWriteMSRR3
+        ||  !pApicReg->pfnReadMSRR3
+        ||  !pApicReg->pfnBusDeliverR3
+        ||  !pApicReg->pfnLocalInterruptR3)
+    {
+        Assert(pApicReg->pfnGetInterruptR3);
+        Assert(pApicReg->pfnHasPendingIrqR3);
+        Assert(pApicReg->pfnSetBaseR3);
+        Assert(pApicReg->pfnGetBaseR3);
+        Assert(pApicReg->pfnSetTPRR3);
+        Assert(pApicReg->pfnGetTPRR3);
+        Assert(pApicReg->pfnWriteMSRR3);
+        Assert(pApicReg->pfnReadMSRR3);
+        Assert(pApicReg->pfnBusDeliverR3);
+        Assert(pApicReg->pfnLocalInterruptR3);
+        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (   (    pApicReg->pszGetInterruptRC
+            ||  pApicReg->pszHasPendingIrqRC
+            ||  pApicReg->pszSetBaseRC
+            ||  pApicReg->pszGetBaseRC
+            ||  pApicReg->pszSetTPRRC
+            ||  pApicReg->pszGetTPRRC
+            ||  pApicReg->pszWriteMSRRC
+            ||  pApicReg->pszReadMSRRC
+            ||  pApicReg->pszBusDeliverRC
+            ||  pApicReg->pszLocalInterruptRC)
+        &&  (   !VALID_PTR(pApicReg->pszGetInterruptRC)
+            ||  !VALID_PTR(pApicReg->pszHasPendingIrqRC)
+            ||  !VALID_PTR(pApicReg->pszSetBaseRC)
+            ||  !VALID_PTR(pApicReg->pszGetBaseRC)
+            ||  !VALID_PTR(pApicReg->pszSetTPRRC)
+            ||  !VALID_PTR(pApicReg->pszGetTPRRC)
+            ||  !VALID_PTR(pApicReg->pszWriteMSRRC)
+            ||  !VALID_PTR(pApicReg->pszReadMSRRC)
+            ||  !VALID_PTR(pApicReg->pszBusDeliverRC)
+            ||  !VALID_PTR(pApicReg->pszLocalInterruptRC))
+       )
+    {
+        Assert(VALID_PTR(pApicReg->pszGetInterruptRC));
+        Assert(VALID_PTR(pApicReg->pszHasPendingIrqRC));
+        Assert(VALID_PTR(pApicReg->pszSetBaseRC));
+        Assert(VALID_PTR(pApicReg->pszGetBaseRC));
+        Assert(VALID_PTR(pApicReg->pszSetTPRRC));
+        Assert(VALID_PTR(pApicReg->pszGetTPRRC));
+        Assert(VALID_PTR(pApicReg->pszReadMSRRC));
+        Assert(VALID_PTR(pApicReg->pszWriteMSRRC));
+        Assert(VALID_PTR(pApicReg->pszBusDeliverRC));
+        Assert(VALID_PTR(pApicReg->pszLocalInterruptRC));
+        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (RC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (   (    pApicReg->pszGetInterruptR0
+            ||  pApicReg->pszHasPendingIrqR0
+            ||  pApicReg->pszSetBaseR0
+            ||  pApicReg->pszGetBaseR0
+            ||  pApicReg->pszSetTPRR0
+            ||  pApicReg->pszGetTPRR0
+            ||  pApicReg->pszWriteMSRR0
+            ||  pApicReg->pszReadMSRR0
+            ||  pApicReg->pszBusDeliverR0
+            ||  pApicReg->pszLocalInterruptR0)
+        &&  (   !VALID_PTR(pApicReg->pszGetInterruptR0)
+            ||  !VALID_PTR(pApicReg->pszHasPendingIrqR0)
+            ||  !VALID_PTR(pApicReg->pszSetBaseR0)
+            ||  !VALID_PTR(pApicReg->pszGetBaseR0)
+            ||  !VALID_PTR(pApicReg->pszSetTPRR0)
+            ||  !VALID_PTR(pApicReg->pszGetTPRR0)
+            ||  !VALID_PTR(pApicReg->pszReadMSRR0)
+            ||  !VALID_PTR(pApicReg->pszWriteMSRR0)
+            ||  !VALID_PTR(pApicReg->pszBusDeliverR0)
+            ||  !VALID_PTR(pApicReg->pszLocalInterruptR0))
+       )
+    {
+        Assert(VALID_PTR(pApicReg->pszGetInterruptR0));
+        Assert(VALID_PTR(pApicReg->pszHasPendingIrqR0));
+        Assert(VALID_PTR(pApicReg->pszSetBaseR0));
+        Assert(VALID_PTR(pApicReg->pszGetBaseR0));
+        Assert(VALID_PTR(pApicReg->pszSetTPRR0));
+        Assert(VALID_PTR(pApicReg->pszGetTPRR0));
+        Assert(VALID_PTR(pApicReg->pszReadMSRR0));
+        Assert(VALID_PTR(pApicReg->pszWriteMSRR0));
+        Assert(VALID_PTR(pApicReg->pszBusDeliverR0));
+        Assert(VALID_PTR(pApicReg->pszLocalInterruptR0));
+        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (R0 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!ppApicHlpR3)
+    {
+        Assert(ppApicHlpR3);
+        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc (ppApicHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one APIC device. On SMP we have single logical device covering all LAPICs,
+     * as they need to communicate and share state easily.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (pVM->pdm.s.Apic.pDevInsR3)
+    {
+        AssertMsgFailed(("Only one apic device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Resolve & initialize the RC bits.
+     */
+    if (pApicReg->pszGetInterruptRC)
+    {
+        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszGetInterruptRC, &pVM->pdm.s.Apic.pfnGetInterruptRC);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszGetInterruptRC, rc));
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszHasPendingIrqRC, &pVM->pdm.s.Apic.pfnHasPendingIrqRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszHasPendingIrqRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszSetBaseRC, &pVM->pdm.s.Apic.pfnSetBaseRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszSetBaseRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszGetBaseRC, &pVM->pdm.s.Apic.pfnGetBaseRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszGetBaseRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszSetTPRRC, &pVM->pdm.s.Apic.pfnSetTPRRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszSetTPRRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszGetTPRRC, &pVM->pdm.s.Apic.pfnGetTPRRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszGetTPRRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszWriteMSRRC, &pVM->pdm.s.Apic.pfnWriteMSRRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszWriteMSRRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszReadMSRRC, &pVM->pdm.s.Apic.pfnReadMSRRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszReadMSRRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszBusDeliverRC, &pVM->pdm.s.Apic.pfnBusDeliverRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszBusDeliverRC, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolRCLazy(pDevIns, pApicReg->pszLocalInterruptRC, &pVM->pdm.s.Apic.pfnLocalInterruptRC);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pApicReg->pszLocalInterruptRC, rc));
+        }
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pVM->pdm.s.Apic.pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
+    }
+    else
+    {
+        pVM->pdm.s.Apic.pDevInsRC           = 0;
+        pVM->pdm.s.Apic.pfnGetInterruptRC   = 0;
+        pVM->pdm.s.Apic.pfnHasPendingIrqRC  = 0;
+        pVM->pdm.s.Apic.pfnSetBaseRC        = 0;
+        pVM->pdm.s.Apic.pfnGetBaseRC        = 0;
+        pVM->pdm.s.Apic.pfnSetTPRRC         = 0;
+        pVM->pdm.s.Apic.pfnGetTPRRC         = 0;
+        pVM->pdm.s.Apic.pfnWriteMSRRC       = 0;
+        pVM->pdm.s.Apic.pfnReadMSRRC        = 0;
+        pVM->pdm.s.Apic.pfnBusDeliverRC     = 0;
+        pVM->pdm.s.Apic.pfnLocalInterruptRC = 0;
+    }
+
+    /*
+     * Resolve & initialize the R0 bits.
+     */
+    if (pApicReg->pszGetInterruptR0)
+    {
+        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszGetInterruptR0, &pVM->pdm.s.Apic.pfnGetInterruptR0);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszGetInterruptR0, rc));
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszHasPendingIrqR0, &pVM->pdm.s.Apic.pfnHasPendingIrqR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszHasPendingIrqR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszSetBaseR0, &pVM->pdm.s.Apic.pfnSetBaseR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszSetBaseR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszGetBaseR0, &pVM->pdm.s.Apic.pfnGetBaseR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszGetBaseR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszSetTPRR0, &pVM->pdm.s.Apic.pfnSetTPRR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszSetTPRR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszGetTPRR0, &pVM->pdm.s.Apic.pfnGetTPRR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszGetTPRR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszWriteMSRR0, &pVM->pdm.s.Apic.pfnWriteMSRR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszWriteMSRR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszReadMSRR0, &pVM->pdm.s.Apic.pfnReadMSRR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszReadMSRR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszBusDeliverR0, &pVM->pdm.s.Apic.pfnBusDeliverR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszBusDeliverR0, rc));
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pApicReg->pszLocalInterruptR0, &pVM->pdm.s.Apic.pfnLocalInterruptR0);
+            AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pApicReg->pszLocalInterruptR0, rc));
+        }
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pVM->pdm.s.Apic.pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
+        Assert(pVM->pdm.s.Apic.pDevInsR0);
+    }
+    else
+    {
+        pVM->pdm.s.Apic.pfnGetInterruptR0   = 0;
+        pVM->pdm.s.Apic.pfnHasPendingIrqR0  = 0;
+        pVM->pdm.s.Apic.pfnSetBaseR0        = 0;
+        pVM->pdm.s.Apic.pfnGetBaseR0        = 0;
+        pVM->pdm.s.Apic.pfnSetTPRR0         = 0;
+        pVM->pdm.s.Apic.pfnGetTPRR0         = 0;
+        pVM->pdm.s.Apic.pfnWriteMSRR0       = 0;
+        pVM->pdm.s.Apic.pfnReadMSRR0        = 0;
+        pVM->pdm.s.Apic.pfnBusDeliverR0     = 0;
+        pVM->pdm.s.Apic.pfnLocalInterruptR0 = 0;
+        pVM->pdm.s.Apic.pDevInsR0           = 0;
+    }
+
+    /*
+     * Initialize the HC bits.
+     */
+    pVM->pdm.s.Apic.pDevInsR3           = pDevIns;
+    pVM->pdm.s.Apic.pfnGetInterruptR3   = pApicReg->pfnGetInterruptR3;
+    pVM->pdm.s.Apic.pfnHasPendingIrqR3  = pApicReg->pfnHasPendingIrqR3;
+    pVM->pdm.s.Apic.pfnSetBaseR3        = pApicReg->pfnSetBaseR3;
+    pVM->pdm.s.Apic.pfnGetBaseR3        = pApicReg->pfnGetBaseR3;
+    pVM->pdm.s.Apic.pfnSetTPRR3         = pApicReg->pfnSetTPRR3;
+    pVM->pdm.s.Apic.pfnGetTPRR3         = pApicReg->pfnGetTPRR3;
+    pVM->pdm.s.Apic.pfnWriteMSRR3       = pApicReg->pfnWriteMSRR3;
+    pVM->pdm.s.Apic.pfnReadMSRR3        = pApicReg->pfnReadMSRR3;
+    pVM->pdm.s.Apic.pfnBusDeliverR3     = pApicReg->pfnBusDeliverR3;
+    pVM->pdm.s.Apic.pfnLocalInterruptR3 = pApicReg->pfnLocalInterruptR3;
+    Log(("PDM: Registered APIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppApicHlpR3 = &g_pdmR3DevApicHlp;
+    LogFlow(("pdmR3DevHlp_APICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnIOAPICRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_IOAPICRegister(PPDMDEVINS pDevIns, PPDMIOAPICREG pIoApicReg, PCPDMIOAPICHLPR3 *ppIoApicHlpR3)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: pIoApicReg=%p:{.u32Version=%#x, .pfnSetIrqR3=%p, .pszSetIrqRC=%p:{%s}, .pszSetIrqR0=%p:{%s}} ppIoApicHlpR3=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pIoApicReg, pIoApicReg->u32Version, pIoApicReg->pfnSetIrqR3,
+             pIoApicReg->pszSetIrqRC, pIoApicReg->pszSetIrqRC, pIoApicReg->pszSetIrqR0, pIoApicReg->pszSetIrqR0, ppIoApicHlpR3));
+
+    /*
+     * Validate input.
+     */
+    if (pIoApicReg->u32Version != PDM_IOAPICREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pIoApicReg->u32Version, PDM_IOAPICREG_VERSION));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!pIoApicReg->pfnSetIrqR3 || !pIoApicReg->pfnSendMsiR3)
+    {
+        Assert(pIoApicReg->pfnSetIrqR3);
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (R3 callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pIoApicReg->pszSetIrqRC
+        &&  !VALID_PTR(pIoApicReg->pszSetIrqRC))
+    {
+        Assert(VALID_PTR(pIoApicReg->pszSetIrqRC));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pIoApicReg->pszSendMsiRC
+        &&  !VALID_PTR(pIoApicReg->pszSendMsiRC))
+    {
+        Assert(VALID_PTR(pIoApicReg->pszSendMsiRC));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pIoApicReg->pszSetIrqR0
+        &&  !VALID_PTR(pIoApicReg->pszSetIrqR0))
+    {
+        Assert(VALID_PTR(pIoApicReg->pszSetIrqR0));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pIoApicReg->pszSendMsiR0
+        &&  !VALID_PTR(pIoApicReg->pszSendMsiR0))
+    {
+        Assert(VALID_PTR(pIoApicReg->pszSendMsiR0));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (GC callbacks)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!ppIoApicHlpR3)
+    {
+        Assert(ppIoApicHlpR3);
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (ppApicHlp)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * The I/O APIC requires the APIC to be present (hacks++).
+     * If the I/O APIC does GC stuff so must the APIC.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (!pVM->pdm.s.Apic.pDevInsR3)
+    {
+        AssertMsgFailed(("Configuration error / Init order error! No APIC!\n"));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (no APIC)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    pIoApicReg->pszSetIrqRC
+        &&  !pVM->pdm.s.Apic.pDevInsRC)
+    {
+        AssertMsgFailed(("Configuration error! APIC doesn't do GC, I/O APIC does!\n"));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (no GC APIC)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one I/O APIC device.
+     */
+    if (pVM->pdm.s.IoApic.pDevInsR3)
+    {
+        AssertMsgFailed(("Only one ioapic device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc (only one)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Resolve & initialize the GC bits.
+     */
+    if (pIoApicReg->pszSetIrqRC)
+    {
+        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pIoApicReg->pszSetIrqRC, &pVM->pdm.s.IoApic.pfnSetIrqRC);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pIoApicReg->pszSetIrqRC, rc));
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pVM->pdm.s.IoApic.pDevInsRC = PDMDEVINS_2_RCPTR(pDevIns);
+    }
+    else
+    {
+        pVM->pdm.s.IoApic.pDevInsRC   = 0;
+        pVM->pdm.s.IoApic.pfnSetIrqRC = 0;
+    }
+
+    if (pIoApicReg->pszSendMsiRC)
+    {
+        int rc = pdmR3DevGetSymbolRCLazy(pDevIns, pIoApicReg->pszSetIrqRC, &pVM->pdm.s.IoApic.pfnSendMsiRC);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szRCMod, pIoApicReg->pszSendMsiRC, rc));
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    else
+    {
+        pVM->pdm.s.IoApic.pfnSendMsiRC = 0;
+    }
+
+    /*
+     * Resolve & initialize the R0 bits.
+     */
+    if (pIoApicReg->pszSetIrqR0)
+    {
+        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pIoApicReg->pszSetIrqR0, &pVM->pdm.s.IoApic.pfnSetIrqR0);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pIoApicReg->pszSetIrqR0, rc));
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+        pVM->pdm.s.IoApic.pDevInsR0 = PDMDEVINS_2_R0PTR(pDevIns);
+        Assert(pVM->pdm.s.IoApic.pDevInsR0);
+    }
+    else
+    {
+        pVM->pdm.s.IoApic.pfnSetIrqR0 = 0;
+        pVM->pdm.s.IoApic.pDevInsR0   = 0;
+    }
+
+    if (pIoApicReg->pszSendMsiR0)
+    {
+        int rc = pdmR3DevGetSymbolR0Lazy(pDevIns, pIoApicReg->pszSetIrqR0, &pVM->pdm.s.IoApic.pfnSendMsiR0);
+        AssertMsgRC(rc, ("%s::%s rc=%Rrc\n", pDevIns->pReg->szR0Mod, pIoApicReg->pszSendMsiR0, rc));
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            return rc;
+        }
+    }
+    else
+    {
+        pVM->pdm.s.IoApic.pfnSendMsiR0 = 0;
+    }
+
+
+    /*
+     * Initialize the R3 bits.
+     */
+    pVM->pdm.s.IoApic.pDevInsR3   = pDevIns;
+    pVM->pdm.s.IoApic.pfnSetIrqR3 = pIoApicReg->pfnSetIrqR3;
+    pVM->pdm.s.IoApic.pfnSendMsiR3 = pIoApicReg->pfnSendMsiR3;
+    Log(("PDM: Registered I/O APIC device '%s'/%d pDevIns=%p\n", pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+
+    /* set the helper pointer and return. */
+    *ppIoApicHlpR3 = &g_pdmR3DevIoApicHlp;
+    LogFlow(("pdmR3DevHlp_IOAPICRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnHPETRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_HPETRegister(PPDMDEVINS pDevIns, PPDMHPETREG pHpetReg, PCPDMHPETHLPR3 *ppHpetHlpR3)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d:\n"));
+
+    /*
+     * Validate input.
+     */
+    if (pHpetReg->u32Version != PDM_HPETREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pHpetReg->u32Version, PDM_HPETREG_VERSION));
+        LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d: returns %Rrc (version)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppHpetHlpR3)
+    {
+        Assert(ppHpetHlpR3);
+        LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d: returns %Rrc (ppApicHlpR3)\n", pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /* set the helper pointer and return. */
+    *ppHpetHlpR3 = &g_pdmR3DevHpetHlp;
+    LogFlow(("pdmR3DevHlp_HPETRegister: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnDMACRegister} */
+static DECLCALLBACK(int) pdmR3DevHlp_DMACRegister(PPDMDEVINS pDevIns, PPDMDMACREG pDmacReg, PCPDMDMACHLP *ppDmacHlp)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: pDmacReg=%p:{.u32Version=%#x, .pfnRun=%p, .pfnRegister=%p, .pfnReadMemory=%p, .pfnWriteMemory=%p, .pfnSetDREQ=%p, .pfnGetChannelMode=%p} ppDmacHlp=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDmacReg, pDmacReg->u32Version, pDmacReg->pfnRun, pDmacReg->pfnRegister,
+             pDmacReg->pfnReadMemory, pDmacReg->pfnWriteMemory, pDmacReg->pfnSetDREQ, pDmacReg->pfnGetChannelMode, ppDmacHlp));
+
+    /*
+     * Validate input.
+     */
+    if (pDmacReg->u32Version != PDM_DMACREG_VERSION)
+    {
+        AssertMsgFailed(("u32Version=%#x expected %#x\n", pDmacReg->u32Version,
+                         PDM_DMACREG_VERSION));
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (version)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (    !pDmacReg->pfnRun
+        ||  !pDmacReg->pfnRegister
+        ||  !pDmacReg->pfnReadMemory
+        ||  !pDmacReg->pfnWriteMemory
+        ||  !pDmacReg->pfnSetDREQ
+        ||  !pDmacReg->pfnGetChannelMode)
+    {
+        Assert(pDmacReg->pfnRun);
+        Assert(pDmacReg->pfnRegister);
+        Assert(pDmacReg->pfnReadMemory);
+        Assert(pDmacReg->pfnWriteMemory);
+        Assert(pDmacReg->pfnSetDREQ);
+        Assert(pDmacReg->pfnGetChannelMode);
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (callbacks)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    if (!ppDmacHlp)
+    {
+        Assert(ppDmacHlp);
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc (ppDmacHlp)\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Only one DMA device.
+     */
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    if (pVM->pdm.s.pDmac)
+    {
+        AssertMsgFailed(("Only one DMA device is supported!\n"));
+        LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance, VERR_INVALID_PARAMETER));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate and initialize pci bus structure.
+     */
+    int rc = VINF_SUCCESS;
+    PPDMDMAC  pDmac = (PPDMDMAC)MMR3HeapAlloc(pDevIns->Internal.s.pVMR3, MM_TAG_PDM_DEVICE, sizeof(*pDmac));
+    if (pDmac)
+    {
+        pDmac->pDevIns   = pDevIns;
+        pDmac->Reg       = *pDmacReg;
+        pVM->pdm.s.pDmac = pDmac;
+
+        /* set the helper pointer. */
+        *ppDmacHlp = &g_pdmR3DevDmacHlp;
+        Log(("PDM: Registered DMAC device '%s'/%d pDevIns=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pDevIns));
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    LogFlow(("pdmR3DevHlp_DMACRegister: caller='%s'/%d: returns %Rrc\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnRegisterVMMDevHeap
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_RegisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTR3PTR pvHeap, unsigned cbSize)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+
+    int rc = PDMR3RegisterVMMDevHeap(pDevIns->Internal.s.pVMR3, GCPhys, pvHeap, cbSize);
+    return rc;
+}
+
+
+/**
+ * @copydoc PDMDEVHLPR3::pfnUnregisterVMMDevHeap
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_UnregisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+
+    int rc = PDMR3UnregisterVMMDevHeap(pDevIns->Internal.s.pVMR3, GCPhys);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMReset} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMReset(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMReset: caller='%s'/%d: VM_FF_RESET %d -> 1\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VM_FF_ISSET(pVM, VM_FF_RESET)));
+
+    /*
+     * We postpone this operation because we're likely to be inside a I/O instruction
+     * and the EIP will be updated when we return.
+     * We still return VINF_EM_RESET to break out of any execution loops and force FF evaluation.
+     */
+    bool fHaltOnReset;
+    int rc = CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "HaltOnReset", &fHaltOnReset);
+    if (RT_SUCCESS(rc) && fHaltOnReset)
+    {
+        Log(("pdmR3DevHlp_VMReset: Halt On Reset!\n"));
+        rc = VINF_EM_HALT;
+    }
+    else
+    {
+        VM_FF_SET(pVM, VM_FF_RESET);
+        rc = VINF_EM_RESET;
+    }
+
+    LogFlow(("pdmR3DevHlp_VMReset: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspend} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSuspend(PPDMDEVINS pDevIns)
+{
+    int rc;
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMSuspend: caller='%s'/%d:\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    /** @todo Always take the SMP path - fewer code paths. */
+    if (pVM->cCpus > 1)
+    {
+        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
+        rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)VMR3Suspend, 1, pVM);
+        AssertRC(rc);
+        rc = VINF_EM_SUSPEND;
+    }
+    else
+        rc = VMR3Suspend(pVM);
+
+    LogFlow(("pdmR3DevHlp_VMSuspend: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/**
+ * Worker for pdmR3DevHlp_VMSuspendSaveAndPowerOff that is invoked via a queued
+ * EMT request to avoid deadlocks.
+ *
+ * @returns VBox status code fit for scheduling.
+ * @param   pVM                 The VM handle.
+ * @param   pDevIns             The device that triggered this action.
+ */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSuspendSaveAndPowerOffWorker(PVM pVM, PPDMDEVINS pDevIns)
+{
+    /*
+     * Suspend the VM first then do the saving.
+     */
+    int rc = VMR3Suspend(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pVM->pUVM->pVmm2UserMethods->pfnSaveState(pVM->pUVM->pVmm2UserMethods, pVM);
+
+        /*
+         * On success, power off the VM, on failure we'll leave it suspended.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            rc = VMR3PowerOff(pVM);
+            if (RT_FAILURE(rc))
+                LogRel(("%s/SSP: VMR3PowerOff failed: %Rrc\n", pDevIns->pReg->szName, rc));
+        }
+        else
+            LogRel(("%s/SSP: pfnSaveState failed: %Rrc\n", pDevIns->pReg->szName, rc));
+    }
+    else
+        LogRel(("%s/SSP: Suspend failed: %Rrc\n", pDevIns->pReg->szName, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspendSaveAndPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMSuspendSaveAndPowerOff(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMSuspendSaveAndPowerOff: caller='%s'/%d:\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    int rc;
+    if (   pVM->pUVM->pVmm2UserMethods
+        && pVM->pUVM->pVmm2UserMethods->pfnSaveState)
+    {
+        rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)pdmR3DevHlp_VMSuspendSaveAndPowerOffWorker, 2, pVM, pDevIns);
+        if (RT_SUCCESS(rc))
+        {
+            LogRel(("%s: Suspending, Saving and Powering Off the VM\n", pDevIns->pReg->szName));
+            rc = VINF_EM_SUSPEND;
+        }
+    }
+    else
+        rc = VERR_NOT_SUPPORTED;
+
+    LogFlow(("pdmR3DevHlp_VMSuspendSaveAndPowerOff: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_VMPowerOff(PPDMDEVINS pDevIns)
+{
+    int rc;
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DevHlp_VMPowerOff: caller='%s'/%d:\n",
+             pDevIns->pReg->szName, pDevIns->iInstance));
+
+    /** @todo Always take the SMP path - fewer code paths. */
+    if (pVM->cCpus > 1)
+    {
+        /* We own the IOM lock here and could cause a deadlock by waiting for a VCPU that is blocking on the IOM lock. */
+        rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)VMR3PowerOff, 1, pVM);
+        AssertRC(rc);
+        /* Set the VCPU state to stopped here as well to make sure no
+         * inconsistency with the EM state occurs.
+         */
+        VMCPU_SET_STATE(VMMGetCpu(pVM), VMCPUSTATE_STOPPED);
+        rc = VINF_EM_OFF;
+    }
+    else
+        rc = VMR3PowerOff(pVM);
+
+    LogFlow(("pdmR3DevHlp_VMPowerOff: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20IsEnabled} */
+static DECLCALLBACK(bool) pdmR3DevHlp_A20IsEnabled(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+
+    bool fRc = PGMPhysIsA20Enabled(VMMGetCpu(pDevIns->Internal.s.pVMR3));
+
+    LogFlow(("pdmR3DevHlp_A20IsEnabled: caller='%s'/%d: returns %d\n", pDevIns->pReg->szName, pDevIns->iInstance, fRc));
+    return fRc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20Set} */
+static DECLCALLBACK(void) pdmR3DevHlp_A20Set(PPDMDEVINS pDevIns, bool fEnable)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DevHlp_A20Set: caller='%s'/%d: fEnable=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, fEnable));
+    PGMR3PhysSetA20(VMMGetCpu(pDevIns->Internal.s.pVMR3), fEnable);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetCpuId} */
+static DECLCALLBACK(void) pdmR3DevHlp_GetCpuId(PPDMDEVINS pDevIns, uint32_t iLeaf,
+                                               uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DevHlp_GetCpuId: caller='%s'/%d: iLeaf=%d pEax=%p pEbx=%p pEcx=%p pEdx=%p\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, iLeaf, pEax, pEbx, pEcx, pEdx));
+    AssertPtr(pEax); AssertPtr(pEbx); AssertPtr(pEcx); AssertPtr(pEdx);
+
+    CPUMGetGuestCpuId(VMMGetCpu(pDevIns->Internal.s.pVMR3), iLeaf, pEax, pEbx, pEcx, pEdx);
+
+    LogFlow(("pdmR3DevHlp_GetCpuId: caller='%s'/%d: returns void - *pEax=%#x *pEbx=%#x *pEcx=%#x *pEdx=%#x\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, *pEax, *pEbx, *pEcx, *pEdx));
+}
+
+
+/**
+ * The device helper structure for trusted devices.
+ */
+const PDMDEVHLPR3 g_pdmR3DevHlpTrusted =
+{
+    PDM_DEVHLPR3_VERSION,
+    pdmR3DevHlp_IOPortRegister,
+    pdmR3DevHlp_IOPortRegisterRC,
+    pdmR3DevHlp_IOPortRegisterR0,
+    pdmR3DevHlp_IOPortDeregister,
+    pdmR3DevHlp_MMIORegister,
+    pdmR3DevHlp_MMIORegisterRC,
+    pdmR3DevHlp_MMIORegisterR0,
+    pdmR3DevHlp_MMIODeregister,
+    pdmR3DevHlp_MMIO2Register,
+    pdmR3DevHlp_MMIO2Deregister,
+    pdmR3DevHlp_MMIO2Map,
+    pdmR3DevHlp_MMIO2Unmap,
+    pdmR3DevHlp_MMHyperMapMMIO2,
+    pdmR3DevHlp_MMIO2MapKernel,
+    pdmR3DevHlp_ROMRegister,
+    pdmR3DevHlp_ROMProtectShadow,
+    pdmR3DevHlp_SSMRegister,
+    pdmR3DevHlp_TMTimerCreate,
+    pdmR3DevHlp_TMUtcNow,
+    pdmR3DevHlp_PhysRead,
+    pdmR3DevHlp_PhysWrite,
+    pdmR3DevHlp_PhysGCPhys2CCPtr,
+    pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly,
+    pdmR3DevHlp_PhysReleasePageMappingLock,
+    pdmR3DevHlp_PhysReadGCVirt,
+    pdmR3DevHlp_PhysWriteGCVirt,
+    pdmR3DevHlp_PhysGCPtr2GCPhys,
+    pdmR3DevHlp_MMHeapAlloc,
+    pdmR3DevHlp_MMHeapAllocZ,
+    pdmR3DevHlp_MMHeapFree,
+    pdmR3DevHlp_VMState,
+    pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet,
+    pdmR3DevHlp_VMSetError,
+    pdmR3DevHlp_VMSetErrorV,
+    pdmR3DevHlp_VMSetRuntimeError,
+    pdmR3DevHlp_VMSetRuntimeErrorV,
+    pdmR3DevHlp_DBGFStopV,
+    pdmR3DevHlp_DBGFInfoRegister,
+    pdmR3DevHlp_STAMRegister,
+    pdmR3DevHlp_STAMRegisterF,
+    pdmR3DevHlp_STAMRegisterV,
+    pdmR3DevHlp_PCIRegister,
+    pdmR3DevHlp_PCIRegisterMsi,
+    pdmR3DevHlp_PCIIORegionRegister,
+    pdmR3DevHlp_PCISetConfigCallbacks,
+    pdmR3DevHlp_PCISetIrq,
+    pdmR3DevHlp_PCISetIrqNoWait,
+    pdmR3DevHlp_ISASetIrq,
+    pdmR3DevHlp_ISASetIrqNoWait,
+    pdmR3DevHlp_DriverAttach,
+    pdmR3DevHlp_QueueCreate,
+    pdmR3DevHlp_CritSectInit,
+    pdmR3DevHlp_ThreadCreate,
+    pdmR3DevHlp_SetAsyncNotification,
+    pdmR3DevHlp_AsyncNotificationCompleted,
+    pdmR3DevHlp_RTCRegister,
+    pdmR3DevHlp_PCIBusRegister,
+    pdmR3DevHlp_PICRegister,
+    pdmR3DevHlp_APICRegister,
+    pdmR3DevHlp_IOAPICRegister,
+    pdmR3DevHlp_HPETRegister,
+    pdmR3DevHlp_DMACRegister,
+    pdmR3DevHlp_DMARegister,
+    pdmR3DevHlp_DMAReadMemory,
+    pdmR3DevHlp_DMAWriteMemory,
+    pdmR3DevHlp_DMASetDREQ,
+    pdmR3DevHlp_DMAGetChannelMode,
+    pdmR3DevHlp_DMASchedule,
+    pdmR3DevHlp_CMOSWrite,
+    pdmR3DevHlp_CMOSRead,
+    pdmR3DevHlp_AssertEMT,
+    pdmR3DevHlp_AssertOther,
+    pdmR3DevHlp_LdrGetRCInterfaceSymbols,
+    pdmR3DevHlp_LdrGetR0InterfaceSymbols,
+    pdmR3DevHlp_CallR0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    pdmR3DevHlp_GetVM,
+    pdmR3DevHlp_GetVMCPU,
+    pdmR3DevHlp_RegisterVMMDevHeap,
+    pdmR3DevHlp_UnregisterVMMDevHeap,
+    pdmR3DevHlp_VMReset,
+    pdmR3DevHlp_VMSuspend,
+    pdmR3DevHlp_VMSuspendSaveAndPowerOff,
+    pdmR3DevHlp_VMPowerOff,
+    pdmR3DevHlp_A20IsEnabled,
+    pdmR3DevHlp_A20Set,
+    pdmR3DevHlp_GetCpuId,
+    pdmR3DevHlp_TMTimeVirtGet,
+    pdmR3DevHlp_TMTimeVirtGetFreq,
+    pdmR3DevHlp_TMTimeVirtGetNano,
+    PDM_DEVHLPR3_VERSION /* the end */
+};
+
+
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVM} */
+static DECLCALLBACK(PVM) pdmR3DevHlp_Untrusted_GetVM(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return NULL;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetVMCPU} */
+static DECLCALLBACK(PVMCPU) pdmR3DevHlp_Untrusted_GetVMCPU(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return NULL;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnRegisterVMMDevHeap} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_RegisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTR3PTR pvHeap, unsigned cbSize)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnUnregisterVMMDevHeap} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_UnregisterVMMDevHeap(PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMReset} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMReset(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspend} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMSuspend(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMSuspendSaveAndPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMSuspendSaveAndPowerOff(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMPowerOff} */
+static DECLCALLBACK(int) pdmR3DevHlp_Untrusted_VMPowerOff(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return VERR_ACCESS_DENIED;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20IsEnabled} */
+static DECLCALLBACK(bool) pdmR3DevHlp_Untrusted_A20IsEnabled(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnA20Set} */
+static DECLCALLBACK(void) pdmR3DevHlp_Untrusted_A20Set(PPDMDEVINS pDevIns, bool fEnable)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    NOREF(fEnable);
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetCpuId} */
+static DECLCALLBACK(void) pdmR3DevHlp_Untrusted_GetCpuId(PPDMDEVINS pDevIns, uint32_t iLeaf,
+                                                         uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    AssertReleaseMsgFailed(("Untrusted device called trusted helper! '%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+}
+
+
+/**
+ * The device helper structure for non-trusted devices.
+ */
+const PDMDEVHLPR3 g_pdmR3DevHlpUnTrusted =
+{
+    PDM_DEVHLPR3_VERSION,
+    pdmR3DevHlp_IOPortRegister,
+    pdmR3DevHlp_IOPortRegisterRC,
+    pdmR3DevHlp_IOPortRegisterR0,
+    pdmR3DevHlp_IOPortDeregister,
+    pdmR3DevHlp_MMIORegister,
+    pdmR3DevHlp_MMIORegisterRC,
+    pdmR3DevHlp_MMIORegisterR0,
+    pdmR3DevHlp_MMIODeregister,
+    pdmR3DevHlp_MMIO2Register,
+    pdmR3DevHlp_MMIO2Deregister,
+    pdmR3DevHlp_MMIO2Map,
+    pdmR3DevHlp_MMIO2Unmap,
+    pdmR3DevHlp_MMHyperMapMMIO2,
+    pdmR3DevHlp_MMIO2MapKernel,
+    pdmR3DevHlp_ROMRegister,
+    pdmR3DevHlp_ROMProtectShadow,
+    pdmR3DevHlp_SSMRegister,
+    pdmR3DevHlp_TMTimerCreate,
+    pdmR3DevHlp_TMUtcNow,
+    pdmR3DevHlp_PhysRead,
+    pdmR3DevHlp_PhysWrite,
+    pdmR3DevHlp_PhysGCPhys2CCPtr,
+    pdmR3DevHlp_PhysGCPhys2CCPtrReadOnly,
+    pdmR3DevHlp_PhysReleasePageMappingLock,
+    pdmR3DevHlp_PhysReadGCVirt,
+    pdmR3DevHlp_PhysWriteGCVirt,
+    pdmR3DevHlp_PhysGCPtr2GCPhys,
+    pdmR3DevHlp_MMHeapAlloc,
+    pdmR3DevHlp_MMHeapAllocZ,
+    pdmR3DevHlp_MMHeapFree,
+    pdmR3DevHlp_VMState,
+    pdmR3DevHlp_VMTeleportedAndNotFullyResumedYet,
+    pdmR3DevHlp_VMSetError,
+    pdmR3DevHlp_VMSetErrorV,
+    pdmR3DevHlp_VMSetRuntimeError,
+    pdmR3DevHlp_VMSetRuntimeErrorV,
+    pdmR3DevHlp_DBGFStopV,
+    pdmR3DevHlp_DBGFInfoRegister,
+    pdmR3DevHlp_STAMRegister,
+    pdmR3DevHlp_STAMRegisterF,
+    pdmR3DevHlp_STAMRegisterV,
+    pdmR3DevHlp_PCIRegister,
+    pdmR3DevHlp_PCIRegisterMsi,
+    pdmR3DevHlp_PCIIORegionRegister,
+    pdmR3DevHlp_PCISetConfigCallbacks,
+    pdmR3DevHlp_PCISetIrq,
+    pdmR3DevHlp_PCISetIrqNoWait,
+    pdmR3DevHlp_ISASetIrq,
+    pdmR3DevHlp_ISASetIrqNoWait,
+    pdmR3DevHlp_DriverAttach,
+    pdmR3DevHlp_QueueCreate,
+    pdmR3DevHlp_CritSectInit,
+    pdmR3DevHlp_ThreadCreate,
+    pdmR3DevHlp_SetAsyncNotification,
+    pdmR3DevHlp_AsyncNotificationCompleted,
+    pdmR3DevHlp_RTCRegister,
+    pdmR3DevHlp_PCIBusRegister,
+    pdmR3DevHlp_PICRegister,
+    pdmR3DevHlp_APICRegister,
+    pdmR3DevHlp_IOAPICRegister,
+    pdmR3DevHlp_HPETRegister,
+    pdmR3DevHlp_DMACRegister,
+    pdmR3DevHlp_DMARegister,
+    pdmR3DevHlp_DMAReadMemory,
+    pdmR3DevHlp_DMAWriteMemory,
+    pdmR3DevHlp_DMASetDREQ,
+    pdmR3DevHlp_DMAGetChannelMode,
+    pdmR3DevHlp_DMASchedule,
+    pdmR3DevHlp_CMOSWrite,
+    pdmR3DevHlp_CMOSRead,
+    pdmR3DevHlp_AssertEMT,
+    pdmR3DevHlp_AssertOther,
+    pdmR3DevHlp_LdrGetRCInterfaceSymbols,
+    pdmR3DevHlp_LdrGetR0InterfaceSymbols,
+    pdmR3DevHlp_CallR0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    pdmR3DevHlp_Untrusted_GetVM,
+    pdmR3DevHlp_Untrusted_GetVMCPU,
+    pdmR3DevHlp_Untrusted_RegisterVMMDevHeap,
+    pdmR3DevHlp_Untrusted_UnregisterVMMDevHeap,
+    pdmR3DevHlp_Untrusted_VMReset,
+    pdmR3DevHlp_Untrusted_VMSuspend,
+    pdmR3DevHlp_Untrusted_VMSuspendSaveAndPowerOff,
+    pdmR3DevHlp_Untrusted_VMPowerOff,
+    pdmR3DevHlp_Untrusted_A20IsEnabled,
+    pdmR3DevHlp_Untrusted_A20Set,
+    pdmR3DevHlp_Untrusted_GetCpuId,
+    pdmR3DevHlp_TMTimeVirtGet,
+    pdmR3DevHlp_TMTimeVirtGetFreq,
+    pdmR3DevHlp_TMTimeVirtGetNano,
+    PDM_DEVHLPR3_VERSION /* the end */
+};
+
+
+
+/**
+ * Queue consumer callback for internal component.
+ *
+ * @returns Success indicator.
+ *          If false the item will not be removed and the flushing will stop.
+ * @param   pVM         The VM handle.
+ * @param   pItem       The item to consume. Upon return this item will be freed.
+ */
+DECLCALLBACK(bool) pdmR3DevHlpQueueConsumer(PVM pVM, PPDMQUEUEITEMCORE pItem)
+{
+    PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)pItem;
+    LogFlow(("pdmR3DevHlpQueueConsumer: enmOp=%d pDevIns=%p\n", pTask->enmOp, pTask->pDevInsR3));
+    switch (pTask->enmOp)
+    {
+        case PDMDEVHLPTASKOP_ISA_SET_IRQ:
+            PDMIsaSetIrq(pVM, pTask->u.SetIRQ.iIrq, pTask->u.SetIRQ.iLevel);
+            break;
+
+        case PDMDEVHLPTASKOP_PCI_SET_IRQ:
+            pdmR3DevHlp_PCISetIrq(pTask->pDevInsR3, pTask->u.SetIRQ.iIrq, pTask->u.SetIRQ.iLevel);
+            break;
+
+        case PDMDEVHLPTASKOP_IOAPIC_SET_IRQ:
+            PDMIoApicSetIrq(pVM, pTask->u.SetIRQ.iIrq, pTask->u.SetIRQ.iLevel);
+            break;
+
+        default:
+            AssertReleaseMsgFailed(("Invalid operation %d\n", pTask->enmOp));
+            break;
+    }
+    return true;
+}
+
+/** @} */
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMDevMiscHlp.cpp	(revision 35333)
@@ -0,0 +1,676 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Misc. Device Helpers.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/rem.h>
+#include <VBox/vm.h>
+#include <VBox/vmm.h>
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+
+
+
+/** @name Ring-3 PIC Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMPICHLPR3,pfnSetInterruptFF} */
+static DECLCALLBACK(void) pdmR3PicHlp_SetInterruptFF(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+
+    if (pVM->pdm.s.Apic.pfnLocalInterruptR3)
+    {
+        LogFlow(("pdmR3PicHlp_SetInterruptFF: caller='%s'/%d: Setting local interrupt on LAPIC\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance));
+        /* Raise the LAPIC's LINT0 line instead of signaling the CPU directly. */
+        pVM->pdm.s.Apic.pfnLocalInterruptR3(pVM->pdm.s.Apic.pDevInsR3, 0, 1);
+        return;
+    }
+
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* for PIC we always deliver to CPU 0, MP use APIC */
+
+    LogFlow(("pdmR3PicHlp_SetInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT_PIC %d -> 1\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC)));
+
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+    REMR3NotifyInterruptSet(pVM, pVCpu);
+    VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE);
+}
+
+
+/** @interface_method_impl{PDMPICHLPR3,pfnClearInterruptFF} */
+static DECLCALLBACK(void) pdmR3PicHlp_ClearInterruptFF(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* for PIC we always deliver to CPU 0, MP use APIC */
+
+    if (pVM->pdm.s.Apic.pfnLocalInterruptR3)
+    {
+        /* Raise the LAPIC's LINT0 line instead of signaling the CPU directly. */
+        LogFlow(("pdmR3PicHlp_ClearInterruptFF: caller='%s'/%d: Clearing local interrupt on LAPIC\n",
+                 pDevIns->pReg->szName, pDevIns->iInstance));
+        /* Lower the LAPIC's LINT0 line instead of signaling the CPU directly. */
+        pVM->pdm.s.Apic.pfnLocalInterruptR3(pVM->pdm.s.Apic.pDevInsR3, 0, 0);
+        return;
+    }
+
+    LogFlow(("pdmR3PicHlp_ClearInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT_PIC %d -> 0\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_PIC)));
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+    REMR3NotifyInterruptClear(pVM, pVCpu);
+}
+
+
+/** @interface_method_impl{PDMPICHLPR3,pfnLock} */
+static DECLCALLBACK(int) pdmR3PicHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
+}
+
+
+/** @interface_method_impl{PDMPICHLPR3,pfnUnlock} */
+static DECLCALLBACK(void) pdmR3PicHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    pdmUnlock(pDevIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMPICHLPR3,pfnGetRCHelpers} */
+static DECLCALLBACK(PCPDMPICHLPRC) pdmR3PicHlp_GetRCHelpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    RTRCPTR pRCHelpers = 0;
+    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCPicHlp", &pRCHelpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pRCHelpers);
+    LogFlow(("pdmR3PicHlp_GetRCHelpers: caller='%s'/%d: returns %RRv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
+    return pRCHelpers;
+}
+
+
+/** @interface_method_impl{PDMPICHLPR3,pfnGetR0Helpers} */
+static DECLCALLBACK(PCPDMPICHLPR0) pdmR3PicHlp_GetR0Helpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    PCPDMPICHLPR0 pR0Helpers = 0;
+    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0PicHlp", &pR0Helpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pR0Helpers);
+    LogFlow(("pdmR3PicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
+    return pR0Helpers;
+}
+
+
+/**
+ * PIC Device Helpers.
+ */
+const PDMPICHLPR3 g_pdmR3DevPicHlp =
+{
+    PDM_PICHLPR3_VERSION,
+    pdmR3PicHlp_SetInterruptFF,
+    pdmR3PicHlp_ClearInterruptFF,
+    pdmR3PicHlp_Lock,
+    pdmR3PicHlp_Unlock,
+    pdmR3PicHlp_GetRCHelpers,
+    pdmR3PicHlp_GetR0Helpers,
+    PDM_PICHLPR3_VERSION /* the end */
+};
+
+/** @} */
+
+
+
+
+/** @name R3 APIC Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnSetInterruptFF} */
+static DECLCALLBACK(void) pdmR3ApicHlp_SetInterruptFF(PPDMDEVINS pDevIns, PDMAPICIRQ enmType, VMCPUID idCpu)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    PVMCPU pVCpu = &pVM->aCpus[idCpu];
+
+    AssertReturnVoid(idCpu < pVM->cCpus);
+
+    LogFlow(("pdmR3ApicHlp_SetInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT(%d) %d -> 1\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, idCpu, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC)));
+
+    switch (enmType)
+    {
+        case PDMAPICIRQ_HARDWARE:
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+            break;
+        case PDMAPICIRQ_NMI:
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+            break;
+        case PDMAPICIRQ_SMI:
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI);
+            break;
+        case PDMAPICIRQ_EXTINT:
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+            break;
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            break;
+    }
+    REMR3NotifyInterruptSet(pVM, pVCpu);
+    VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE);
+}
+
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnClearInterruptFF} */
+static DECLCALLBACK(void) pdmR3ApicHlp_ClearInterruptFF(PPDMDEVINS pDevIns, PDMAPICIRQ enmType, VMCPUID idCpu)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    PVMCPU pVCpu = &pVM->aCpus[idCpu];
+
+    AssertReturnVoid(idCpu < pVM->cCpus);
+
+    LogFlow(("pdmR3ApicHlp_ClearInterruptFF: caller='%s'/%d: VM_FF_INTERRUPT(%d) %d -> 0\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, idCpu, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INTERRUPT_APIC)));
+
+    /* Note: NMI/SMI can't be cleared. */
+    switch (enmType)
+    {
+        case PDMAPICIRQ_HARDWARE:
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+            break;
+        case PDMAPICIRQ_EXTINT:
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+            break;
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            break;
+    }
+    REMR3NotifyInterruptClear(pVM, pVCpu);
+}
+
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnChangeFeature} */
+static DECLCALLBACK(void) pdmR3ApicHlp_ChangeFeature(PPDMDEVINS pDevIns, PDMAPICVERSION enmVersion)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3ApicHlp_ChangeFeature: caller='%s'/%d: version=%d\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, (int)enmVersion));
+    switch (enmVersion)
+    {
+        case PDMAPICVERSION_NONE:
+            CPUMClearGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_APIC);
+            CPUMClearGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_X2APIC);
+            break;
+        case PDMAPICVERSION_APIC:
+            CPUMSetGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_APIC);
+            CPUMClearGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_X2APIC);
+            break;
+        case PDMAPICVERSION_X2APIC:
+            CPUMSetGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_X2APIC);
+            CPUMSetGuestCpuIdFeature(pDevIns->Internal.s.pVMR3, CPUMCPUIDFEATURE_APIC);
+            break;
+        default:
+            AssertMsgFailed(("Unknown APIC version: %d\n", (int)enmVersion));
+    }
+}
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnGetCpuId} */
+static DECLCALLBACK(VMCPUID) pdmR3ApicHlp_GetCpuId(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    return VMMGetCpuId(pDevIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnSendSipi} */
+static DECLCALLBACK(void) pdmR3ApicHlp_SendSipi(PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t uVector)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    VMMR3SendSipi(pDevIns->Internal.s.pVMR3, idCpu, uVector);
+}
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnSendInitIpi} */
+static DECLCALLBACK(void) pdmR3ApicHlp_SendInitIpi(PPDMDEVINS pDevIns, VMCPUID idCpu)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    VMMR3SendInitIpi(pDevIns->Internal.s.pVMR3, idCpu);
+}
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnGetRCHelpers} */
+static DECLCALLBACK(PCPDMAPICHLPRC) pdmR3ApicHlp_GetRCHelpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    RTRCPTR pRCHelpers = 0;
+    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCApicHlp", &pRCHelpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pRCHelpers);
+    LogFlow(("pdmR3ApicHlp_GetRCHelpers: caller='%s'/%d: returns %RRv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
+    return pRCHelpers;
+}
+
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnGetR0Helpers} */
+static DECLCALLBACK(PCPDMAPICHLPR0) pdmR3ApicHlp_GetR0Helpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    PCPDMAPICHLPR0 pR0Helpers = 0;
+    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0ApicHlp", &pR0Helpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pR0Helpers);
+    LogFlow(("pdmR3ApicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
+    return pR0Helpers;
+}
+
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnGetR3CritSect} */
+static DECLCALLBACK(R3PTRTYPE(PPDMCRITSECT)) pdmR3ApicHlp_GetR3CritSect(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3ApicHlp_Lock: caller='%s'/%d\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    return &pDevIns->Internal.s.pVMR3->pdm.s.CritSect;
+}
+
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnGetRCCritSect} */
+static DECLCALLBACK(RCPTRTYPE(PPDMCRITSECT)) pdmR3ApicHlp_GetRCCritSect(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    RTRCPTR RCPtr = MMHyperCCToRC(pVM, &pVM->pdm.s.CritSect);
+    LogFlow(("pdmR3ApicHlp_GetR0CritSect: caller='%s'/%d: return %RRv\n", pDevIns->pReg->szName, pDevIns->iInstance, RCPtr));
+    return RCPtr;
+}
+
+
+/** @interface_method_impl{PDMAPICHLPR3,pfnGetR3CritSect} */
+static DECLCALLBACK(R0PTRTYPE(PPDMCRITSECT)) pdmR3ApicHlp_GetR0CritSect(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    RTR0PTR R0Ptr = MMHyperCCToR0(pVM, &pVM->pdm.s.CritSect);
+    LogFlow(("pdmR3ApicHlp_GetR0CritSect: caller='%s'/%d: return %RHv\n", pDevIns->pReg->szName, pDevIns->iInstance, R0Ptr));
+    return R0Ptr;
+}
+
+
+
+/**
+ * APIC Device Helpers.
+ */
+const PDMAPICHLPR3 g_pdmR3DevApicHlp =
+{
+    PDM_APICHLPR3_VERSION,
+    pdmR3ApicHlp_SetInterruptFF,
+    pdmR3ApicHlp_ClearInterruptFF,
+    pdmR3ApicHlp_ChangeFeature,
+    pdmR3ApicHlp_GetCpuId,
+    pdmR3ApicHlp_SendSipi,
+    pdmR3ApicHlp_SendInitIpi,
+    pdmR3ApicHlp_GetRCHelpers,
+    pdmR3ApicHlp_GetR0Helpers,
+    pdmR3ApicHlp_GetR3CritSect,
+    pdmR3ApicHlp_GetRCCritSect,
+    pdmR3ApicHlp_GetR0CritSect,
+    PDM_APICHLPR3_VERSION /* the end */
+};
+
+/** @} */
+
+
+
+
+/** @name Ring-3 I/O APIC Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMIOAPICHLPR3,pfnApicBusDeliver} */
+static DECLCALLBACK(int) pdmR3IoApicHlp_ApicBusDeliver(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
+                                                        uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3IoApicHlp_ApicBusDeliver: caller='%s'/%d: u8Dest=%RX8 u8DestMode=%RX8 u8DeliveryMode=%RX8 iVector=%RX8 u8Polarity=%RX8 u8TriggerMode=%RX8\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, u8Dest, u8DestMode, u8DeliveryMode, iVector, u8Polarity, u8TriggerMode));
+    if (pVM->pdm.s.Apic.pfnBusDeliverR3)
+        return pVM->pdm.s.Apic.pfnBusDeliverR3(pVM->pdm.s.Apic.pDevInsR3, u8Dest, u8DestMode, u8DeliveryMode, iVector, u8Polarity, u8TriggerMode);
+    return VINF_SUCCESS;
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLPR3,pfnLock} */
+static DECLCALLBACK(int) pdmR3IoApicHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3IoApicHlp_Lock: caller='%s'/%d: rc=%Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLPR3,pfnUnlock} */
+static DECLCALLBACK(void) pdmR3IoApicHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3IoApicHlp_Unlock: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    pdmUnlock(pDevIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLPR3,pfnGetRCHelpers} */
+static DECLCALLBACK(PCPDMIOAPICHLPRC) pdmR3IoApicHlp_GetRCHelpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    RTRCPTR pRCHelpers = 0;
+    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCIoApicHlp", &pRCHelpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pRCHelpers);
+    LogFlow(("pdmR3IoApicHlp_GetRCHelpers: caller='%s'/%d: returns %RRv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
+    return pRCHelpers;
+}
+
+
+/** @interface_method_impl{PDMIOAPICHLPR3,pfnGetR0Helpers} */
+static DECLCALLBACK(PCPDMIOAPICHLPR0) pdmR3IoApicHlp_GetR0Helpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    PCPDMIOAPICHLPR0 pR0Helpers = 0;
+    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0IoApicHlp", &pR0Helpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pR0Helpers);
+    LogFlow(("pdmR3IoApicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
+    return pR0Helpers;
+}
+
+
+/**
+ * I/O APIC Device Helpers.
+ */
+const PDMIOAPICHLPR3 g_pdmR3DevIoApicHlp =
+{
+    PDM_IOAPICHLPR3_VERSION,
+    pdmR3IoApicHlp_ApicBusDeliver,
+    pdmR3IoApicHlp_Lock,
+    pdmR3IoApicHlp_Unlock,
+    pdmR3IoApicHlp_GetRCHelpers,
+    pdmR3IoApicHlp_GetR0Helpers,
+    PDM_IOAPICHLPR3_VERSION /* the end */
+};
+
+/** @} */
+
+
+
+
+/** @name Ring-3 PCI Bus Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnIsaSetIrq} */
+static DECLCALLBACK(void) pdmR3PciHlp_IsaSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR3PciHlp_IsaSetIrq: iIrq=%d iLevel=%d\n", iIrq, iLevel));
+    PDMIsaSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel);
+}
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnIoApicSetIrq} */
+static DECLCALLBACK(void) pdmR3PciHlp_IoApicSetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR3PciHlp_IoApicSetIrq: iIrq=%d iLevel=%d\n", iIrq, iLevel));
+    PDMIoApicSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel);
+}
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnIoApicSendMsi} */
+static DECLCALLBACK(void) pdmR3PciHlp_IoApicSendMsi(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    Log4(("pdmR3PciHlp_IoApicSendMsi: address=%p value=%x\n", GCAddr, uValue));
+    PDMIoApicSendMsi(pDevIns->Internal.s.pVMR3, GCAddr, uValue);
+}
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnIsMMIO2Base} */
+static DECLCALLBACK(bool) pdmR3PciHlp_IsMMIO2Base(PPDMDEVINS pDevIns, PPDMDEVINS pOwner, RTGCPHYS GCPhys)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    bool fRc = PGMR3PhysMMIO2IsBase(pDevIns->Internal.s.pVMR3, pOwner, GCPhys);
+    Log4(("pdmR3PciHlp_IsMMIO2Base: pOwner=%p GCPhys=%RGp -> %RTbool\n", pOwner, GCPhys, fRc));
+    return fRc;
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnLock} */
+static DECLCALLBACK(int) pdmR3PciHlp_Lock(PPDMDEVINS pDevIns, int rc)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3PciHlp_Lock: caller='%s'/%d: rc=%Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return pdmLockEx(pDevIns->Internal.s.pVMR3, rc);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnUnlock} */
+static DECLCALLBACK(void) pdmR3PciHlp_Unlock(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3PciHlp_Unlock: caller='%s'/%d:\n", pDevIns->pReg->szName, pDevIns->iInstance));
+    pdmUnlock(pDevIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnGetRCHelpers} */
+static DECLCALLBACK(PCPDMPCIHLPRC) pdmR3PciHlp_GetRCHelpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    RTRCPTR pRCHelpers = 0;
+    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCPciHlp", &pRCHelpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pRCHelpers);
+    LogFlow(("pdmR3IoApicHlp_GetGCHelpers: caller='%s'/%d: returns %RRv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
+    return pRCHelpers;
+}
+
+
+/** @interface_method_impl{PDMPCIHLPR3,pfnGetR0Helpers} */
+static DECLCALLBACK(PCPDMPCIHLPR0) pdmR3PciHlp_GetR0Helpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    PCPDMPCIHLPR0 pR0Helpers = 0;
+    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0PciHlp", &pR0Helpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pR0Helpers);
+    LogFlow(("pdmR3IoApicHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
+    return pR0Helpers;
+}
+
+
+/**
+ * PCI Bus Device Helpers.
+ */
+const PDMPCIHLPR3 g_pdmR3DevPciHlp =
+{
+    PDM_PCIHLPR3_VERSION,
+    pdmR3PciHlp_IsaSetIrq,
+    pdmR3PciHlp_IoApicSetIrq,
+    pdmR3PciHlp_IoApicSendMsi,
+    pdmR3PciHlp_IsMMIO2Base,
+    pdmR3PciHlp_GetRCHelpers,
+    pdmR3PciHlp_GetR0Helpers,
+    pdmR3PciHlp_Lock,
+    pdmR3PciHlp_Unlock,
+    PDM_PCIHLPR3_VERSION, /* the end */
+};
+
+/** @} */
+
+
+
+
+/** @name Ring-3 HPET Helpers
+ * {@
+ */
+
+/** @interface_method_impl{PDMHPETHLPR3,pfnSetLegacyMode} */
+static DECLCALLBACK(int) pdmR3HpetHlp_SetLegacyMode(PPDMDEVINS pDevIns, bool fActivated)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3HpetHlp_SetLegacyMode: caller='%s'/%d: fActivated=%RTbool\n", pDevIns->pReg->szName, pDevIns->iInstance, fActivated));
+
+    size_t                      i;
+    int                         rc = VINF_SUCCESS;
+    static const char * const   s_apszDevsToNotify[] =
+    {
+        "i8254",
+        "mc146818"
+    };
+    for (i = 0; i < RT_ELEMENTS(s_apszDevsToNotify); i++)
+    {
+        PPDMIBASE pBase;
+        rc = PDMR3QueryDevice(pDevIns->Internal.s.pVMR3, "i8254", 0, &pBase);
+        if (RT_SUCCESS(rc))
+        {
+            PPDMIHPETLEGACYNOTIFY pPort = PDMIBASE_QUERY_INTERFACE(pBase, PDMIHPETLEGACYNOTIFY);
+            AssertLogRelMsgBreakStmt(pPort, ("%s\n", s_apszDevsToNotify[i]), rc = VERR_INTERNAL_ERROR_3);
+            pPort->pfnModeChanged(pPort, fActivated);
+        }
+        else if (   rc == VERR_PDM_DEVICE_NOT_FOUND
+                 || rc == VERR_PDM_DEVICE_INSTANCE_NOT_FOUND)
+            rc = VINF_SUCCESS; /* the device isn't configured, ignore. */
+        else
+            AssertLogRelMsgFailedBreak(("%s -> %Rrc\n", s_apszDevsToNotify[i], rc));
+    }
+
+    /* Don't bother cleaning up, any failure here will cause a guru meditation. */
+
+    LogFlow(("pdmR3HpetHlp_SetLegacyMode: caller='%s'/%d: returns %Rrc\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMHPETHLPR3,pfnSetIrq} */
+static DECLCALLBACK(int) pdmR3HpetHlp_SetIrq(PPDMDEVINS pDevIns, int iIrq, int iLevel)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    LogFlow(("pdmR3HpetHlp_SetIrq: caller='%s'/%d: iIrq=%d iLevel=%d\n", pDevIns->pReg->szName, pDevIns->iInstance, iIrq, iLevel));
+    PDMIsaSetIrq(pDevIns->Internal.s.pVMR3, iIrq, iLevel);
+    return 0;
+}
+
+
+/** @interface_method_impl{PDMHPETHLPR3,pfnGetRCHelpers} */
+static DECLCALLBACK(PCPDMHPETHLPRC) pdmR3HpetHlp_GetRCHelpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    RTRCPTR pRCHelpers = 0;
+    int rc = PDMR3LdrGetSymbolRC(pDevIns->Internal.s.pVMR3, NULL, "g_pdmRCHpetHlp", &pRCHelpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pRCHelpers);
+    LogFlow(("pdmR3HpetHlp_GetGCHelpers: caller='%s'/%d: returns %RRv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pRCHelpers));
+    return pRCHelpers;
+}
+
+
+/** @interface_method_impl{PDMHPETHLPR3,pfnGetR0Helpers} */
+static DECLCALLBACK(PCPDMHPETHLPR0) pdmR3HpetHlp_GetR0Helpers(PPDMDEVINS pDevIns)
+{
+    PDMDEV_ASSERT_DEVINS(pDevIns);
+    VM_ASSERT_EMT(pDevIns->Internal.s.pVMR3);
+    PCPDMHPETHLPR0 pR0Helpers = 0;
+    int rc = PDMR3LdrGetSymbolR0(pDevIns->Internal.s.pVMR3, NULL, "g_pdmR0HpetHlp", &pR0Helpers);
+    AssertReleaseRC(rc);
+    AssertRelease(pR0Helpers);
+    LogFlow(("pdmR3HpetHlp_GetR0Helpers: caller='%s'/%d: returns %RHv\n",
+             pDevIns->pReg->szName, pDevIns->iInstance, pR0Helpers));
+    return pR0Helpers;
+}
+
+
+/**
+ * HPET Device Helpers.
+ */
+const PDMHPETHLPR3 g_pdmR3DevHpetHlp =
+{
+    PDM_HPETHLPR3_VERSION,
+    pdmR3HpetHlp_GetRCHelpers,
+    pdmR3HpetHlp_GetR0Helpers,
+    pdmR3HpetHlp_SetLegacyMode,
+    pdmR3HpetHlp_SetIrq,
+    PDM_HPETHLPR3_VERSION, /* the end */
+};
+
+/** @} */
+
+
+
+/* none yet */
+
+/**
+ * DMAC Device Helpers.
+ */
+const PDMDMACHLP g_pdmR3DevDmacHlp =
+{
+    PDM_DMACHLP_VERSION
+};
+
+
+
+
+/* none yet */
+
+/**
+ * RTC Device Helpers.
+ */
+const PDMRTCHLP g_pdmR3DevRtcHlp =
+{
+    PDM_RTCHLP_VERSION
+};
Index: /trunk/src/VBox/VMM/VMMR3/PDMDevice.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMDevice.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMDevice.cpp	(revision 35333)
@@ -0,0 +1,1042 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Device parts.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DEVICE
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/pgm.h>
+#include <VBox/iom.h>
+#include <VBox/cfgm.h>
+#include <VBox/rem.h>
+#include <VBox/dbgf.h>
+#include <VBox/vm.h>
+#include <VBox/vmm.h>
+
+#include <VBox/version.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/alloc.h>
+#include <iprt/alloca.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/path.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Internal callback structure pointer.
+ * The main purpose is to define the extra data we associate
+ * with PDMDEVREGCB so we can find the VM instance and so on.
+ */
+typedef struct PDMDEVREGCBINT
+{
+    /** The callback structure. */
+    PDMDEVREGCB     Core;
+    /** A bit of padding. */
+    uint32_t        u32[4];
+    /** VM Handle. */
+    PVM             pVM;
+    /** Pointer to the configuration node the registrations should be
+     * associated with.  Can be NULL. */
+    PCFGMNODE       pCfgNode;
+} PDMDEVREGCBINT;
+/** Pointer to a PDMDEVREGCBINT structure. */
+typedef PDMDEVREGCBINT *PPDMDEVREGCBINT;
+/** Pointer to a const PDMDEVREGCBINT structure. */
+typedef const PDMDEVREGCBINT *PCPDMDEVREGCBINT;
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int)    pdmR3DevReg_Register(PPDMDEVREGCB pCallbacks, PCPDMDEVREG pReg);
+static int                  pdmR3DevLoadModules(PVM pVM);
+static int                  pdmR3DevLoad(PVM pVM, PPDMDEVREGCBINT pRegCB, const char *pszFilename, const char *pszName);
+
+
+
+
+/**
+ * This function will initialize the devices for this VM instance.
+ *
+ *
+ * First of all this mean loading the builtin device and letting them
+ * register themselves. Beyond that any additional device modules are
+ * loaded and called for registration.
+ *
+ * Then the device configuration is enumerated, the instantiation order
+ * is determined, and finally they are instantiated.
+ *
+ * After all devices have been successfully instantiated the primary
+ * PCI Bus device is called to emulate the PCI BIOS, i.e. making the
+ * resource assignments. If there is no PCI device, this step is of course
+ * skipped.
+ *
+ * Finally the init completion routines of the instantiated devices
+ * are called.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM Handle.
+ */
+int pdmR3DevInit(PVM pVM)
+{
+    LogFlow(("pdmR3DevInit:\n"));
+
+    AssertRelease(!(RT_OFFSETOF(PDMDEVINS, achInstanceData) & 15));
+    AssertRelease(sizeof(pVM->pdm.s.pDevInstances->Internal.s) <= sizeof(pVM->pdm.s.pDevInstances->Internal.padding));
+
+    /*
+     * Load device modules.
+     */
+    int rc = pdmR3DevLoadModules(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifdef VBOX_WITH_USB
+    /* ditto for USB Devices. */
+    rc = pdmR3UsbLoadModules(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+#endif
+
+    /*
+     * Get the RC & R0 devhlps and create the devhlp R3 task queue.
+     */
+    PCPDMDEVHLPRC pHlpRC;
+    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDevHlp", &pHlpRC);
+    AssertReleaseRCReturn(rc, rc);
+
+    PCPDMDEVHLPR0 pHlpR0;
+    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "g_pdmR0DevHlp", &pHlpR0);
+    AssertReleaseRCReturn(rc, rc);
+
+    rc = PDMR3QueueCreateInternal(pVM, sizeof(PDMDEVHLPTASK), 8, 0, pdmR3DevHlpQueueConsumer, true, "DevHlp",
+                                  &pVM->pdm.s.pDevHlpQueueR3);
+    AssertRCReturn(rc, rc);
+    pVM->pdm.s.pDevHlpQueueR0 = PDMQueueR0Ptr(pVM->pdm.s.pDevHlpQueueR3);
+    pVM->pdm.s.pDevHlpQueueRC = PDMQueueRCPtr(pVM->pdm.s.pDevHlpQueueR3);
+
+
+    /*
+     *
+     * Enumerate the device instance configurations
+     * and come up with a instantiation order.
+     *
+     */
+    /* Switch to /Devices, which contains the device instantiations. */
+    PCFGMNODE pDevicesNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "Devices");
+
+    /*
+     * Count the device instances.
+     */
+    PCFGMNODE pCur;
+    PCFGMNODE pInstanceNode;
+    unsigned cDevs = 0;
+    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+            cDevs++;
+    if (!cDevs)
+    {
+        Log(("PDM: No devices were configured!\n"));
+        return VINF_SUCCESS;
+    }
+    Log2(("PDM: cDevs=%d!\n", cDevs));
+
+    /*
+     * Collect info on each device instance.
+     */
+    struct DEVORDER
+    {
+        /** Configuration node. */
+        PCFGMNODE   pNode;
+        /** Pointer to device. */
+        PPDMDEV     pDev;
+        /** Init order. */
+        uint32_t    u32Order;
+        /** VBox instance number. */
+        uint32_t    iInstance;
+    } *paDevs = (struct DEVORDER *)alloca(sizeof(paDevs[0]) * (cDevs + 1)); /* (One extra for swapping) */
+    Assert(paDevs);
+    unsigned i = 0;
+    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /* Get the device name. */
+        char szName[sizeof(paDevs[0].pDev->pReg->szName)];
+        rc = CFGMR3GetName(pCur, szName, sizeof(szName));
+        AssertMsgRCReturn(rc, ("Configuration error: device name is too long (or something)! rc=%Rrc\n", rc), rc);
+
+        /* Find the device. */
+        PPDMDEV pDev = pdmR3DevLookup(pVM, szName);
+        AssertMsgReturn(pDev, ("Configuration error: device '%s' not found!\n", szName), VERR_PDM_DEVICE_NOT_FOUND);
+
+        /* Configured priority or use default based on device class? */
+        uint32_t u32Order;
+        rc = CFGMR3QueryU32(pCur, "Priority", &u32Order);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        {
+            uint32_t u32 = pDev->pReg->fClass;
+            for (u32Order = 1; !(u32 & u32Order); u32Order <<= 1)
+                /* nop */;
+        }
+        else
+            AssertMsgRCReturn(rc, ("Configuration error: reading \"Priority\" for the '%s' device failed rc=%Rrc!\n", szName, rc), rc);
+
+        /* Enumerate the device instances. */
+        uint32_t const iStart = i;
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+        {
+            paDevs[i].pNode = pInstanceNode;
+            paDevs[i].pDev = pDev;
+            paDevs[i].u32Order = u32Order;
+
+            /* Get the instance number. */
+            char szInstance[32];
+            rc = CFGMR3GetName(pInstanceNode, szInstance, sizeof(szInstance));
+            AssertMsgRCReturn(rc, ("Configuration error: instance name is too long (or something)! rc=%Rrc\n", rc), rc);
+            char *pszNext = NULL;
+            rc = RTStrToUInt32Ex(szInstance, &pszNext, 0, &paDevs[i].iInstance);
+            AssertMsgRCReturn(rc, ("Configuration error: RTStrToInt32Ex failed on the instance name '%s'! rc=%Rrc\n", szInstance, rc), rc);
+            AssertMsgReturn(!*pszNext, ("Configuration error: the instance name '%s' isn't all digits. (%s)\n", szInstance, pszNext), VERR_INVALID_PARAMETER);
+
+            /* next instance */
+            i++;
+        }
+
+        /* check the number of instances */
+        if (i - iStart > pDev->pReg->cMaxInstances)
+            AssertLogRelMsgFailedReturn(("Configuration error: Too many instances of %s was configured: %u, max %u\n",
+                                         szName, i - iStart, pDev->pReg->cMaxInstances),
+                                        VERR_PDM_TOO_MANY_DEVICE_INSTANCES);
+    } /* devices */
+    Assert(i == cDevs);
+
+    /*
+     * Sort the device array ascending on u32Order. (bubble)
+     */
+    unsigned c = cDevs - 1;
+    while (c)
+    {
+        unsigned j = 0;
+        for (i = 0; i < c; i++)
+            if (paDevs[i].u32Order > paDevs[i + 1].u32Order)
+            {
+                paDevs[cDevs] = paDevs[i + 1];
+                paDevs[i + 1] = paDevs[i];
+                paDevs[i] = paDevs[cDevs];
+                j = i;
+            }
+        c = j;
+    }
+
+
+    /*
+     *
+     * Instantiate the devices.
+     *
+     */
+    for (i = 0; i < cDevs; i++)
+    {
+        /*
+         * Gather a bit of config.
+         */
+        /* trusted */
+        bool fTrusted;
+        rc = CFGMR3QueryBool(paDevs[i].pNode, "Trusted", &fTrusted);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            fTrusted = false;
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: failed to query boolean \"Trusted\", rc=%Rrc\n", rc));
+            return rc;
+        }
+        /* config node */
+        PCFGMNODE pConfigNode = CFGMR3GetChild(paDevs[i].pNode, "Config");
+        if (!pConfigNode)
+        {
+            rc = CFGMR3InsertNode(paDevs[i].pNode, "Config", &pConfigNode);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("Failed to create Config node! rc=%Rrc\n", rc));
+                return rc;
+            }
+        }
+        CFGMR3SetRestrictedRoot(pConfigNode);
+
+        /*
+         * Allocate the device instance.
+         */
+        AssertReturn(paDevs[i].pDev->cInstances < paDevs[i].pDev->pReg->cMaxInstances, VERR_PDM_TOO_MANY_DEVICE_INSTANCES);
+        size_t cb = RT_OFFSETOF(PDMDEVINS, achInstanceData[paDevs[i].pDev->pReg->cbInstance]);
+        cb = RT_ALIGN_Z(cb, 16);
+        PPDMDEVINS pDevIns;
+        if (paDevs[i].pDev->pReg->fFlags & (PDM_DEVREG_FLAGS_RC | PDM_DEVREG_FLAGS_R0))
+            rc = MMR3HyperAllocOnceNoRel(pVM, cb, 0, MM_TAG_PDM_DEVICE, (void **)&pDevIns);
+        else
+            rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_DEVICE, cb, (void **)&pDevIns);
+        if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("Failed to allocate %d bytes of instance data for device '%s'. rc=%Rrc\n",
+                             cb, paDevs[i].pDev->pReg->szName, rc));
+            return rc;
+        }
+
+        /*
+         * Initialize it.
+         */
+        pDevIns->u32Version                     = PDM_DEVINS_VERSION;
+        //pDevIns->Internal.s.pNextR3             = NULL;
+        //pDevIns->Internal.s.pPerDeviceNextR3    = NULL;
+        pDevIns->Internal.s.pDevR3              = paDevs[i].pDev;
+        pDevIns->Internal.s.pVMR3               = pVM;
+        pDevIns->Internal.s.pVMR0               = pVM->pVMR0;
+        pDevIns->Internal.s.pVMRC               = pVM->pVMRC;
+        //pDevIns->Internal.s.pLunsR3             = NULL;
+        pDevIns->Internal.s.pCfgHandle          = paDevs[i].pNode;
+        //pDevIns->Internal.s.pPciDeviceR3        = NULL;
+        //pDevIns->Internal.s.pPciBusR3           = NULL;
+        //pDevIns->Internal.s.pPciDeviceR0        = 0;
+        //pDevIns->Internal.s.pPciBusR0           = 0;
+        //pDevIns->Internal.s.pPciDeviceRC        = 0;
+        //pDevIns->Internal.s.pPciBusRC           = 0;
+        pDevIns->Internal.s.fIntFlags           = PDMDEVINSINT_FLAGS_SUSPENDED;
+        pDevIns->pHlpR3                         = fTrusted ? &g_pdmR3DevHlpTrusted : &g_pdmR3DevHlpUnTrusted;
+        pDevIns->pHlpRC                         = pHlpRC;
+        pDevIns->pHlpR0                         = pHlpR0;
+        pDevIns->pReg                           = paDevs[i].pDev->pReg;
+        pDevIns->pCfg                           = pConfigNode;
+        pDevIns->iInstance                      = paDevs[i].iInstance;
+        pDevIns->pvInstanceDataR3               = &pDevIns->achInstanceData[0];
+        pDevIns->pvInstanceDataRC               = pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC
+                                                ? MMHyperR3ToRC(pVM, pDevIns->pvInstanceDataR3) : NIL_RTRCPTR;
+        pDevIns->pvInstanceDataR0               = pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0
+                                                ? MMHyperR3ToR0(pVM, pDevIns->pvInstanceDataR3) : NIL_RTR0PTR;
+        //pDevIns->pCritSectR3                    = NULL;
+        //pDevIns->pCritSectR0                    = NIL_RTR0PTR;
+        //pDevIns->pCritSectRC                    = NIL_RTRCPTR;
+
+        /*
+         * Link it into all the lists.
+         */
+        /* The global instance FIFO. */
+        PPDMDEVINS pPrev1 = pVM->pdm.s.pDevInstances;
+        if (!pPrev1)
+            pVM->pdm.s.pDevInstances = pDevIns;
+        else
+        {
+            while (pPrev1->Internal.s.pNextR3)
+                pPrev1 = pPrev1->Internal.s.pNextR3;
+            pPrev1->Internal.s.pNextR3 = pDevIns;
+        }
+
+        /* The per device instance FIFO. */
+        PPDMDEVINS pPrev2 = paDevs[i].pDev->pInstances;
+        if (!pPrev2)
+            paDevs[i].pDev->pInstances = pDevIns;
+        else
+        {
+            while (pPrev2->Internal.s.pPerDeviceNextR3)
+                pPrev2 = pPrev2->Internal.s.pPerDeviceNextR3;
+            pPrev2->Internal.s.pPerDeviceNextR3 = pDevIns;
+        }
+
+        /*
+         * Call the constructor.
+         */
+        paDevs[i].pDev->cInstances++;
+        Log(("PDM: Constructing device '%s' instance %d...\n", pDevIns->pReg->szName, pDevIns->iInstance));
+        rc = pDevIns->pReg->pfnConstruct(pDevIns, pDevIns->iInstance, pDevIns->pCfg);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Per-device critsect fun.
+             */
+            if (pDevIns->pCritSectR3)
+            {
+                AssertStmt(PDMCritSectIsInitialized(pDevIns->pCritSectR3), rc = VERR_INTERNAL_ERROR_4);
+                if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+                {
+                    pDevIns->pCritSectR0 = MMHyperCCToR0(pVM, pDevIns->pCritSectR3);
+                    AssertStmt(pDevIns->pCritSectR0 != NIL_RTR0PTR, rc = VERR_INTERNAL_ERROR_3);
+                }
+                else
+                    AssertStmt(pDevIns->pCritSectR0 == NIL_RTRCPTR, rc = VERR_INTERNAL_ERROR_2);
+
+                if (pDevIns->pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+                {
+                    pDevIns->pCritSectRC = MMHyperCCToRC(pVM, pDevIns->pCritSectR3);
+                    AssertStmt(pDevIns->pCritSectRC != NIL_RTRCPTR, rc = VERR_INTERNAL_ERROR_3);
+                }
+                else
+                    AssertStmt(pDevIns->pCritSectRC == NIL_RTRCPTR, rc = VERR_INTERNAL_ERROR_2);
+            }
+            else
+                AssertStmt(   pDevIns->pCritSectRC == NIL_RTRCPTR
+                           && pDevIns->pCritSectR0 == NIL_RTR0PTR,
+                           rc = VERR_INTERNAL_ERROR_5);
+        }
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDM: Failed to construct '%s'/%d! %Rra\n", pDevIns->pReg->szName, pDevIns->iInstance, rc));
+            paDevs[i].pDev->cInstances--;
+            /* because we're damn lazy right now, we'll say that the destructor will be called even if the constructor fails. */
+            return rc;
+        }
+    } /* for device instances */
+
+#ifdef VBOX_WITH_USB
+    /* ditto for USB Devices. */
+    rc = pdmR3UsbInstantiateDevices(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+#endif
+
+
+    /*
+     *
+     * PCI BIOS Fake and Init Complete.
+     *
+     */
+    if (pVM->pdm.s.aPciBuses[0].pDevInsR3)
+    {
+        pdmLock(pVM);
+        rc = pVM->pdm.s.aPciBuses[0].pfnFakePCIBIOSR3(pVM->pdm.s.aPciBuses[0].pDevInsR3);
+        pdmUnlock(pVM);
+        if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("PCI BIOS fake failed rc=%Rrc\n", rc));
+            return rc;
+        }
+    }
+
+    for (PPDMDEVINS pDevIns = pVM->pdm.s.pDevInstances; pDevIns; pDevIns = pDevIns->Internal.s.pNextR3)
+    {
+        if (pDevIns->pReg->pfnInitComplete)
+        {
+            rc = pDevIns->pReg->pfnInitComplete(pDevIns);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("InitComplete on device '%s'/%d failed with rc=%Rrc\n",
+                                 pDevIns->pReg->szName, pDevIns->iInstance, rc));
+                return rc;
+            }
+        }
+    }
+
+#ifdef VBOX_WITH_USB
+    /* ditto for USB Devices. */
+    rc = pdmR3UsbVMInitComplete(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+#endif
+
+    LogFlow(("pdmR3DevInit: returns %Rrc\n", VINF_SUCCESS));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Lookups a device structure by name.
+ * @internal
+ */
+PPDMDEV pdmR3DevLookup(PVM pVM, const char *pszName)
+{
+    size_t cchName = strlen(pszName);
+    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
+        if (    pDev->cchName == cchName
+            &&  !strcmp(pDev->pReg->szName, pszName))
+            return pDev;
+    return NULL;
+}
+
+
+/**
+ * Loads the device modules.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+static int pdmR3DevLoadModules(PVM pVM)
+{
+    /*
+     * Initialize the callback structure.
+     */
+    PDMDEVREGCBINT RegCB;
+    RegCB.Core.u32Version  = PDM_DEVREG_CB_VERSION;
+    RegCB.Core.pfnRegister = pdmR3DevReg_Register;
+    RegCB.pVM              = pVM;
+    RegCB.pCfgNode         = NULL;
+
+    /*
+     * Load the builtin module
+     */
+    PCFGMNODE pDevicesNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/Devices");
+    bool fLoadBuiltin;
+    int rc = CFGMR3QueryBool(pDevicesNode, "LoadBuiltin", &fLoadBuiltin);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+        fLoadBuiltin = true;
+    else if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
+        return rc;
+    }
+    if (fLoadBuiltin)
+    {
+        /* make filename */
+        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3DevLoad(pVM, &RegCB, pszFilename, "VBoxDD");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* make filename */
+        pszFilename = pdmR3FileR3("VBoxDD2", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3DevLoad(pVM, &RegCB, pszFilename, "VBoxDD2");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Load additional device modules.
+     */
+    PCFGMNODE pCur;
+    for (pCur = CFGMR3GetFirstChild(pDevicesNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /*
+         * Get the name and path.
+         */
+        char szName[PDMMOD_NAME_LEN];
+        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
+        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
+        {
+            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
+            return VERR_PDM_MODULE_NAME_TOO_LONG;
+        }
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
+            return rc;
+        }
+
+        /* the path is optional, if no path the module name + path is used. */
+        char szFilename[RTPATH_MAX];
+        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            strcpy(szFilename, szName);
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
+            return rc;
+        }
+
+        /* prepend path? */
+        if (!RTPathHavePath(szFilename))
+        {
+            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
+            if (!psz)
+                return VERR_NO_TMP_MEMORY;
+            size_t cch = strlen(psz) + 1;
+            if (cch > sizeof(szFilename))
+            {
+                RTMemTmpFree(psz);
+                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
+                return VERR_FILENAME_TOO_LONG;
+            }
+            memcpy(szFilename, psz, cch);
+            RTMemTmpFree(psz);
+        }
+
+        /*
+         * Load the module and register it's devices.
+         */
+        RegCB.pCfgNode = pCur;
+        rc = pdmR3DevLoad(pVM, &RegCB, szFilename, szName);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads one device module and call the registration entry point.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pRegCB          The registration callback stuff.
+ * @param   pszFilename     Module filename.
+ * @param   pszName         Module name.
+ */
+static int pdmR3DevLoad(PVM pVM, PPDMDEVREGCBINT pRegCB, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Load it.
+     */
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Get the registration export and call it.
+         */
+        FNPDMVBOXDEVICESREGISTER *pfnVBoxDevicesRegister;
+        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxDevicesRegister", (void **)&pfnVBoxDevicesRegister);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("PDM: Calling VBoxDevicesRegister (%p) of %s (%s)\n", pfnVBoxDevicesRegister, pszName, pszFilename));
+            rc = pfnVBoxDevicesRegister(&pRegCB->Core, VBOX_VERSION);
+            if (RT_SUCCESS(rc))
+                Log(("PDM: Successfully loaded device module %s (%s).\n", pszName, pszFilename));
+            else
+                AssertMsgFailed(("VBoxDevicesRegister failed with rc=%Rrc for module %s (%s)\n", rc, pszName, pszFilename));
+        }
+        else
+        {
+            AssertMsgFailed(("Failed to locate 'VBoxDevicesRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
+            if (rc == VERR_SYMBOL_NOT_FOUND)
+                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to load %s %s!\n", pszFilename, pszName));
+    return rc;
+}
+
+
+/**
+ * @interface_method_impl{PDMDEVREGCB,pfnRegister}
+ */
+static DECLCALLBACK(int) pdmR3DevReg_Register(PPDMDEVREGCB pCallbacks, PCPDMDEVREG pReg)
+{
+    /*
+     * Validate the registration structure.
+     */
+    Assert(pReg);
+    AssertMsgReturn(pReg->u32Version == PDM_DEVREG_VERSION,
+                    ("Unknown struct version %#x!\n", pReg->u32Version),
+                    VERR_PDM_UNKNOWN_DEVREG_VERSION);
+
+    AssertMsgReturn(    pReg->szName[0]
+                    &&  strlen(pReg->szName) < sizeof(pReg->szName),
+                    ("Invalid name '%s'\n", pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(   !(pReg->fFlags & PDM_DEVREG_FLAGS_RC)
+                    || (   pReg->szRCMod[0]
+                        && strlen(pReg->szRCMod) < sizeof(pReg->szRCMod)),
+                    ("Invalid GC module name '%s' - (Device %s)\n", pReg->szRCMod, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(   !(pReg->fFlags & PDM_DEVREG_FLAGS_R0)
+                    || (   pReg->szR0Mod[0]
+                        && strlen(pReg->szR0Mod) < sizeof(pReg->szR0Mod)),
+                    ("Invalid R0 module name '%s' - (Device %s)\n", pReg->szR0Mod, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_HOST_BITS_MASK) == PDM_DEVREG_FLAGS_HOST_BITS_DEFAULT,
+                    ("Invalid host bits flags! fFlags=%#x (Device %s)\n", pReg->fFlags, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_HOST_BITS);
+    AssertMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_GUEST_BITS_MASK),
+                    ("Invalid guest bits flags! fFlags=%#x (Device %s)\n", pReg->fFlags, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->fClass,
+                    ("No class! (Device %s)\n", pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->cMaxInstances > 0,
+                    ("Max instances %u! (Device %s)\n", pReg->cMaxInstances, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->cbInstance <= (uint32_t)(pReg->fFlags & (PDM_DEVREG_FLAGS_RC | PDM_DEVREG_FLAGS_R0)  ? 96 * _1K : _1M),
+                    ("Instance size %d bytes! (Device %s)\n", pReg->cbInstance, pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertMsgReturn(pReg->pfnConstruct,
+                    ("No constructor! (Device %s)\n", pReg->szName),
+                    VERR_PDM_INVALID_DEVICE_REGISTRATION);
+    AssertLogRelMsgReturn((pReg->fFlags & PDM_DEVREG_FLAGS_GUEST_BITS_MASK) == PDM_DEVREG_FLAGS_GUEST_BITS_DEFAULT,
+                          ("PDM: Rejected device '%s' because it didn't match the guest bits.\n", pReg->szName),
+                          VERR_PDM_INVALID_DEVICE_GUEST_BITS);
+    AssertLogRelMsg(pReg->u32VersionEnd == PDM_DEVREG_VERSION,
+                    ("u32VersionEnd=%#x, expected %#x. (szName=%s)\n",
+                     pReg->u32VersionEnd, PDM_DEVREG_VERSION, pReg->szName));
+
+    /*
+     * Check for duplicate and find FIFO entry at the same time.
+     */
+    PCPDMDEVREGCBINT pRegCB = (PCPDMDEVREGCBINT)pCallbacks;
+    PPDMDEV pDevPrev = NULL;
+    PPDMDEV pDev = pRegCB->pVM->pdm.s.pDevs;
+    for (; pDev; pDevPrev = pDev, pDev = pDev->pNext)
+        AssertMsgReturn(strcmp(pDev->pReg->szName, pReg->szName),
+                        ("Device '%s' already exists\n", pReg->szName),
+                        VERR_PDM_DEVICE_NAME_CLASH);
+
+    /*
+     * Allocate new device structure, initialize and insert it into the list.
+     */
+    int rc;
+    pDev = (PPDMDEV)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DEVICE, sizeof(*pDev));
+    if (pDev)
+    {
+        pDev->pNext      = NULL;
+        pDev->cInstances = 0;
+        pDev->pInstances = NULL;
+        pDev->pReg       = pReg;
+        pDev->cchName    = (uint32_t)strlen(pReg->szName);
+        rc = CFGMR3QueryStringAllocDef(    pRegCB->pCfgNode, "RCSearchPath", &pDev->pszRCSearchPath, NULL);
+        if (RT_SUCCESS(rc))
+            rc = CFGMR3QueryStringAllocDef(pRegCB->pCfgNode, "R0SearchPath", &pDev->pszR0SearchPath, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            if (pDevPrev)
+                pDevPrev->pNext = pDev;
+            else
+                pRegCB->pVM->pdm.s.pDevs = pDev;
+            Log(("PDM: Registered device '%s'\n", pReg->szName));
+            return VINF_SUCCESS;
+        }
+
+        MMR3HeapFree(pDev);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Locates a LUN.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   ppLun           Where to store the pointer to the LUN if found.
+ * @thread  Try only do this in EMT...
+ */
+int pdmR3DevFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPPDMLUN ppLun)
+{
+    /*
+     * Iterate registered devices looking for the device.
+     */
+    size_t cchDevice = strlen(pszDevice);
+    for (PPDMDEV pDev = pVM->pdm.s.pDevs; pDev; pDev = pDev->pNext)
+    {
+        if (    pDev->cchName == cchDevice
+            &&  !memcmp(pDev->pReg->szName, pszDevice, cchDevice))
+        {
+            /*
+             * Iterate device instances.
+             */
+            for (PPDMDEVINS pDevIns = pDev->pInstances; pDevIns; pDevIns = pDevIns->Internal.s.pPerDeviceNextR3)
+            {
+                if (pDevIns->iInstance == iInstance)
+                {
+                    /*
+                     * Iterate luns.
+                     */
+                    for (PPDMLUN pLun = pDevIns->Internal.s.pLunsR3; pLun; pLun = pLun->pNext)
+                    {
+                        if (pLun->iLun == iLun)
+                        {
+                            *ppLun = pLun;
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    return VERR_PDM_LUN_NOT_FOUND;
+                }
+            }
+            return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND;
+        }
+    }
+    return VERR_PDM_DEVICE_NOT_FOUND;
+}
+
+
+/**
+ * Attaches a preconfigured driver to an existing device instance.
+ *
+ * This is used to change drivers and suchlike at runtime.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @param   ppBase          Where to store the base interface pointer. Optional.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DeviceAttach(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags, PPPDMIBASE ppBase)
+{
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("PDMR3DeviceAttach: pszDevice=%p:{%s} iInstance=%d iLun=%d fFlags=%#x ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, fFlags, ppBase));
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Can we attach anything at runtime?
+         */
+        PPDMDEVINS pDevIns = pLun->pDevIns;
+        if (pDevIns->pReg->pfnAttach)
+        {
+            if (!pLun->pTop)
+            {
+                rc = pDevIns->pReg->pfnAttach(pDevIns, iLun, fFlags);
+            }
+            else
+                rc = VERR_PDM_DRIVER_ALREADY_ATTACHED;
+        }
+        else
+            rc = VERR_PDM_DEVICE_NO_RT_ATTACH;
+
+        if (ppBase)
+            *ppBase = pLun->pTop ? &pLun->pTop->IBase : NULL;
+    }
+    else if (ppBase)
+        *ppBase = NULL;
+
+    if (ppBase)
+        LogFlow(("PDMR3DeviceAttach: returns %Rrc *ppBase=%p\n", rc, *ppBase));
+    else
+        LogFlow(("PDMR3DeviceAttach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Detaches a driver chain from an existing device instance.
+ *
+ * This is used to change drivers and suchlike at runtime.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DeviceDetach(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags)
+{
+    return PDMR3DriverDetach(pVM, pszDevice, iInstance, iLun, NULL, 0, fFlags);
+}
+
+
+/**
+ * Attaches a preconfigured driver to an existing device or driver instance.
+ *
+ * This is used to change drivers and suchlike at runtime.  The driver or device
+ * at the end of the chain will be told to attach to whatever is configured
+ * below it.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iInstance       Device instance.
+ * @param   iLun            The Logical Unit to obtain the interface of.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @param   ppBase          Where to store the base interface pointer. Optional.
+ *
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DriverAttach(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, uint32_t fFlags, PPPDMIBASE ppBase)
+{
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("PDMR3DriverAttach: pszDevice=%p:{%s} iInstance=%d iLun=%d fFlags=%#x ppBase=%p\n",
+             pszDevice, pszDevice, iInstance, iLun, fFlags, ppBase));
+
+    if (ppBase)
+        *ppBase = NULL;
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iInstance, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Anything attached to the LUN?
+         */
+        PPDMDRVINS pDrvIns = pLun->pTop;
+        if (!pDrvIns)
+        {
+            /* No, ask the device to attach to the new stuff. */
+            PPDMDEVINS pDevIns = pLun->pDevIns;
+            if (pDevIns->pReg->pfnAttach)
+            {
+                rc = pDevIns->pReg->pfnAttach(pDevIns, iLun, fFlags);
+                if (RT_SUCCESS(rc) && ppBase)
+                    *ppBase = pLun->pTop ? &pLun->pTop->IBase : NULL;
+            }
+            else
+                rc = VERR_PDM_DEVICE_NO_RT_ATTACH;
+        }
+        else
+        {
+            /* Yes, find the bottom most driver and ask it to attach to the new stuff. */
+            while (pDrvIns->Internal.s.pDown)
+                pDrvIns = pDrvIns->Internal.s.pDown;
+            if (pDrvIns->pReg->pfnAttach)
+            {
+                rc = pDrvIns->pReg->pfnAttach(pDrvIns, fFlags);
+                if (RT_SUCCESS(rc) && ppBase)
+                    *ppBase = pDrvIns->Internal.s.pDown
+                            ? &pDrvIns->Internal.s.pDown->IBase
+                            : NULL;
+            }
+            else
+                rc = VERR_PDM_DRIVER_NO_RT_ATTACH;
+        }
+    }
+
+    if (ppBase)
+        LogFlow(("PDMR3DriverAttach: returns %Rrc *ppBase=%p\n", rc, *ppBase));
+    else
+        LogFlow(("PDMR3DriverAttach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Detaches the specified driver instance.
+ *
+ * This is used to replumb drivers at runtime for simulating hot plugging and
+ * media changes.
+ *
+ * This is a superset of PDMR3DeviceDetach.  It allows detaching drivers from
+ * any driver or device by specifying the driver to start detaching at.  The
+ * only prerequisite is that the driver or device above implements the
+ * pfnDetach callback (PDMDRVREG / PDMDEVREG).
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iDevIns         Device instance.
+ * @param   iLun            The Logical Unit in which to look for the driver.
+ * @param   pszDriver       The name of the driver which to detach.  If NULL
+ *                          then the entire driver chain is detatched.
+ * @param   iOccurance      The occurrence of that driver in the chain.  This is
+ *                          usually 0.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3DriverDetach(PVM pVM, const char *pszDevice, unsigned iDevIns, unsigned iLun,
+                                 const char *pszDriver, unsigned iOccurance, uint32_t fFlags)
+{
+    LogFlow(("PDMR3DriverDetach: pszDevice=%p:{%s} iDevIns=%u iLun=%u pszDriver=%p:{%s} iOccurance=%u fFlags=%#x\n",
+             pszDevice, pszDevice, iDevIns, iLun, pszDriver, iOccurance, fFlags));
+    VM_ASSERT_EMT(pVM);
+    AssertPtr(pszDevice);
+    AssertPtrNull(pszDriver);
+    Assert(iOccurance == 0 || pszDriver);
+    Assert(!(fFlags & ~(PDM_TACH_FLAGS_NOT_HOT_PLUG)));
+
+    /*
+     * Find the LUN in question.
+     */
+    PPDMLUN pLun;
+    int rc = pdmR3DevFindLun(pVM, pszDevice, iDevIns, iLun, &pLun);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Locate the driver.
+         */
+        PPDMDRVINS pDrvIns = pLun->pTop;
+        if (pDrvIns)
+        {
+            if (pszDriver)
+            {
+                while (pDrvIns)
+                {
+                    if (!strcmp(pDrvIns->pReg->szName, pszDriver))
+                    {
+                        if (iOccurance == 0)
+                            break;
+                        iOccurance--;
+                    }
+                    pDrvIns = pDrvIns->Internal.s.pDown;
+                }
+            }
+            if (pDrvIns)
+                rc = pdmR3DrvDetach(pDrvIns, fFlags);
+            else
+                rc = VERR_PDM_DRIVER_INSTANCE_NOT_FOUND;
+        }
+        else
+            rc = VINF_PDM_NO_DRIVER_ATTACHED_TO_LUN;
+    }
+
+    LogFlow(("PDMR3DriverDetach: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Runtime detach and reattach of a new driver chain or sub chain.
+ *
+ * This is intended to be called on a non-EMT thread, this will instantiate the
+ * new driver (sub-)chain, and then the EMTs will do the actual replumbing.  The
+ * destruction of the old driver chain will be taken care of on the calling
+ * thread.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pszDevice       Device name.
+ * @param   iDevIns         Device instance.
+ * @param   iLun            The Logical Unit in which to look for the driver.
+ * @param   pszDriver       The name of the driver which to detach and replace.
+ *                          If NULL then the entire driver chain is to be
+ *                          reattached.
+ * @param   iOccurance      The occurrence of that driver in the chain.  This is
+ *                          usually 0.
+ * @param   fFlags          Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ * @param   pCfg            The configuration of the new driver chain that is
+ *                          going to be attached.  The subtree starts with the
+ *                          node containing a Driver key, a Config subtree and
+ *                          optionally an AttachedDriver subtree.
+ *                          If this parameter is NULL, then this call will work
+ *                          like at a non-pause version of PDMR3DriverDetach.
+ * @param   ppBase          Where to store the base interface pointer to the new
+ *                          driver.  Optional.
+ *
+ * @thread  Any thread. The EMTs will be involved at some point though.
+ */
+VMMR3DECL(int)  PDMR3DriverReattach(PVM pVM, const char *pszDevice, unsigned iDevIns, unsigned iLun,
+                                    const char *pszDriver, unsigned iOccurance, uint32_t fFlags,
+                                    PCFGMNODE pCfg, PPPDMIBASE ppBase)
+{
+    return VERR_NOT_IMPLEMENTED;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMDriver.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMDriver.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMDriver.cpp	(revision 35333)
@@ -0,0 +1,1476 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, Driver parts.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DRIVER
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/cfgm.h>
+#include <VBox/vmm.h>
+#include <VBox/sup.h>
+#include <VBox/vm.h>
+#include <VBox/version.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/ctype.h>
+#include <iprt/mem.h>
+#include <iprt/thread.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Internal callback structure pointer.
+ *
+ * The main purpose is to define the extra data we associate
+ * with PDMDRVREGCB so we can find the VM instance and so on.
+ */
+typedef struct PDMDRVREGCBINT
+{
+    /** The callback structure. */
+    PDMDRVREGCB     Core;
+    /** A bit of padding. */
+    uint32_t        u32[4];
+    /** VM Handle. */
+    PVM             pVM;
+    /** Pointer to the configuration node the registrations should be
+     * associated with.  Can be NULL. */
+    PCFGMNODE       pCfgNode;
+} PDMDRVREGCBINT, *PPDMDRVREGCBINT;
+typedef const PDMDRVREGCBINT *PCPDMDRVREGCBINT;
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) pdmR3DrvRegister(PCPDMDRVREGCB pCallbacks, PCPDMDRVREG pReg);
+static int pdmR3DrvLoad(PVM pVM, PPDMDRVREGCBINT pRegCB, const char *pszFilename, const char *pszName);
+
+
+
+/**
+ * Register external drivers
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pfnCallback Driver registration callback
+ */
+VMMR3DECL(int) PDMR3RegisterDrivers(PVM pVM, FNPDMVBOXDRIVERSREGISTER pfnCallback)
+{
+    /*
+     * The registration callbacks.
+     */
+    PDMDRVREGCBINT  RegCB;
+    RegCB.Core.u32Version   = PDM_DRVREG_CB_VERSION;
+    RegCB.Core.pfnRegister  = pdmR3DrvRegister;
+    RegCB.pVM               = pVM;
+
+    int rc = pfnCallback(&RegCB.Core, VBOX_VERSION);
+    if (RT_FAILURE(rc))
+        AssertMsgFailed(("VBoxDriversRegister failed with rc=%Rrc\n"));
+
+    return rc;
+}
+
+/**
+ * This function will initialize the drivers for this VM instance.
+ *
+ * First of all this mean loading the builtin drivers and letting them
+ * register themselves. Beyond that any additional driver modules are
+ * loaded and called for registration.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM Handle.
+ */
+int pdmR3DrvInit(PVM pVM)
+{
+    LogFlow(("pdmR3DrvInit:\n"));
+
+    AssertRelease(!(RT_OFFSETOF(PDMDRVINS, achInstanceData) & 15));
+    PPDMDRVINS pDrvInsAssert; NOREF(pDrvInsAssert);
+    AssertCompile(sizeof(pDrvInsAssert->Internal.s) <= sizeof(pDrvInsAssert->Internal.padding));
+    AssertRelease(sizeof(pDrvInsAssert->Internal.s) <= sizeof(pDrvInsAssert->Internal.padding));
+
+    /*
+     * The registration callbacks.
+     */
+    PDMDRVREGCBINT  RegCB;
+    RegCB.Core.u32Version   = PDM_DRVREG_CB_VERSION;
+    RegCB.Core.pfnRegister  = pdmR3DrvRegister;
+    RegCB.pVM               = pVM;
+    RegCB.pCfgNode          = NULL;
+
+    /*
+     * Load the builtin module
+     */
+    PCFGMNODE pDriversNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/Drivers");
+    bool fLoadBuiltin;
+    int rc = CFGMR3QueryBool(pDriversNode, "LoadBuiltin", &fLoadBuiltin);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+        fLoadBuiltin = true;
+    else if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
+        return rc;
+    }
+    if (fLoadBuiltin)
+    {
+        /* make filename */
+        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3DrvLoad(pVM, &RegCB, pszFilename, "VBoxDD");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Load additional driver modules.
+     */
+    for (PCFGMNODE pCur = CFGMR3GetFirstChild(pDriversNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /*
+         * Get the name and path.
+         */
+        char szName[PDMMOD_NAME_LEN];
+        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
+        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
+        {
+            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
+            return VERR_PDM_MODULE_NAME_TOO_LONG;
+        }
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
+            return rc;
+        }
+
+        /* the path is optional, if no path the module name + path is used. */
+        char szFilename[RTPATH_MAX];
+        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+            strcpy(szFilename, szName);
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
+            return rc;
+        }
+
+        /* prepend path? */
+        if (!RTPathHavePath(szFilename))
+        {
+            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
+            if (!psz)
+                return VERR_NO_TMP_MEMORY;
+            size_t cch = strlen(psz) + 1;
+            if (cch > sizeof(szFilename))
+            {
+                RTMemTmpFree(psz);
+                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
+                return VERR_FILENAME_TOO_LONG;
+            }
+            memcpy(szFilename, psz, cch);
+            RTMemTmpFree(psz);
+        }
+
+        /*
+         * Load the module and register it's drivers.
+         */
+        RegCB.pCfgNode = pCur;
+        rc = pdmR3DrvLoad(pVM, &RegCB, szFilename, szName);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    LogFlow(("pdmR3DrvInit: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads one driver module and call the registration entry point.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pRegCB          The registration callback stuff.
+ * @param   pszFilename     Module filename.
+ * @param   pszName         Module name.
+ */
+static int pdmR3DrvLoad(PVM pVM, PPDMDRVREGCBINT pRegCB, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Load it.
+     */
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Get the registration export and call it.
+         */
+        FNPDMVBOXDRIVERSREGISTER *pfnVBoxDriversRegister;
+        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxDriversRegister", (void **)&pfnVBoxDriversRegister);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("PDM: Calling VBoxDriversRegister (%p) of %s (%s)\n", pfnVBoxDriversRegister, pszName, pszFilename));
+            rc = pfnVBoxDriversRegister(&pRegCB->Core, VBOX_VERSION);
+            if (RT_SUCCESS(rc))
+                Log(("PDM: Successfully loaded driver module %s (%s).\n", pszName, pszFilename));
+            else
+                AssertMsgFailed(("VBoxDriversRegister failed with rc=%Rrc\n"));
+        }
+        else
+        {
+            AssertMsgFailed(("Failed to locate 'VBoxDriversRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
+            if (rc == VERR_SYMBOL_NOT_FOUND)
+                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to load %s (%s) rc=%Rrc!\n", pszName, pszFilename, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVREGCB,pfnRegister} */
+static DECLCALLBACK(int) pdmR3DrvRegister(PCPDMDRVREGCB pCallbacks, PCPDMDRVREG pReg)
+{
+    /*
+     * Validate the registration structure.
+     */
+    AssertPtrReturn(pReg, VERR_INVALID_POINTER);
+    AssertMsgReturn(pReg->u32Version == PDM_DRVREG_VERSION,
+                    ("%#x\n", pReg->u32Version),
+                    VERR_PDM_UNKNOWN_DRVREG_VERSION);
+    AssertReturn(pReg->szName[0], VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(RTStrEnd(pReg->szName, sizeof(pReg->szName)),
+                    (".*s\n", sizeof(pReg->szName), pReg->szName),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(    !(pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+                    ||  (   pReg->szR0Mod[0]
+                         && RTStrEnd(pReg->szR0Mod, sizeof(pReg->szR0Mod))),
+                    ("%s: %.*s\n", pReg->szName, sizeof(pReg->szR0Mod), pReg->szR0Mod),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(    !(pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+                    ||  (   pReg->szRCMod[0]
+                         && RTStrEnd(pReg->szRCMod, sizeof(pReg->szRCMod))),
+                    ("%s: %.*s\n", pReg->szName, sizeof(pReg->szRCMod), pReg->szRCMod),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(VALID_PTR(pReg->pszDescription),
+                    ("%s: %p\n", pReg->szName, pReg->pszDescription),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(!(pReg->fFlags & ~(PDM_DRVREG_FLAGS_HOST_BITS_MASK | PDM_DRVREG_FLAGS_R0 | PDM_DRVREG_FLAGS_RC)),
+                    ("%s: %#x\n", pReg->szName, pReg->fFlags),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn((pReg->fFlags & PDM_DRVREG_FLAGS_HOST_BITS_MASK) == PDM_DRVREG_FLAGS_HOST_BITS_DEFAULT,
+                    ("%s: %#x\n", pReg->szName, pReg->fFlags),
+                    VERR_PDM_INVALID_DRIVER_HOST_BITS);
+    AssertMsgReturn(pReg->cMaxInstances > 0,
+                    ("%s: %#x\n", pReg->szName, pReg->cMaxInstances),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(pReg->cbInstance <= _1M,
+                    ("%s: %#x\n", pReg->szName, pReg->cbInstance),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(VALID_PTR(pReg->pfnConstruct),
+                    ("%s: %p\n", pReg->szName, pReg->pfnConstruct),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(VALID_PTR(pReg->pfnRelocate) || !(pReg->fFlags & PDM_DRVREG_FLAGS_RC),
+                    ("%s: %#x\n", pReg->szName, pReg->cbInstance),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(pReg->pfnSoftReset == NULL,
+                    ("%s: %p\n", pReg->szName, pReg->pfnSoftReset),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+    AssertMsgReturn(pReg->u32VersionEnd == PDM_DRVREG_VERSION,
+                    ("%s: #x\n", pReg->szName, pReg->u32VersionEnd),
+                    VERR_PDM_INVALID_DRIVER_REGISTRATION);
+
+    /*
+     * Check for duplicate and find FIFO entry at the same time.
+     */
+    PCPDMDRVREGCBINT pRegCB = (PCPDMDRVREGCBINT)pCallbacks;
+    PPDMDRV pDrvPrev = NULL;
+    PPDMDRV pDrv = pRegCB->pVM->pdm.s.pDrvs;
+    for (; pDrv; pDrvPrev = pDrv, pDrv = pDrv->pNext)
+    {
+        if (!strcmp(pDrv->pReg->szName, pReg->szName))
+        {
+            AssertMsgFailed(("Driver '%s' already exists\n", pReg->szName));
+            return VERR_PDM_DRIVER_NAME_CLASH;
+        }
+    }
+
+    /*
+     * Allocate new driver structure and insert it into the list.
+     */
+    int rc;
+    pDrv = (PPDMDRV)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DRIVER, sizeof(*pDrv));
+    if (pDrv)
+    {
+        pDrv->pNext         = NULL;
+        pDrv->cInstances    = 0;
+        pDrv->iNextInstance = 0;
+        pDrv->pReg          = pReg;
+        rc = CFGMR3QueryStringAllocDef(    pRegCB->pCfgNode, "RCSearchPath", &pDrv->pszRCSearchPath, NULL);
+        if (RT_SUCCESS(rc))
+            rc = CFGMR3QueryStringAllocDef(pRegCB->pCfgNode, "R0SearchPath", &pDrv->pszR0SearchPath, NULL);
+        if (RT_SUCCESS(rc))
+        {
+            if (pDrvPrev)
+                pDrvPrev->pNext = pDrv;
+            else
+                pRegCB->pVM->pdm.s.pDrvs = pDrv;
+            Log(("PDM: Registered driver '%s'\n", pReg->szName));
+            return VINF_SUCCESS;
+        }
+        MMR3HeapFree(pDrv);
+    }
+    else
+        rc = VERR_NO_MEMORY;
+    return rc;
+}
+
+
+/**
+ * Lookups a driver structure by name.
+ * @internal
+ */
+PPDMDRV pdmR3DrvLookup(PVM pVM, const char *pszName)
+{
+    for (PPDMDRV pDrv = pVM->pdm.s.pDrvs; pDrv; pDrv = pDrv->pNext)
+        if (!strcmp(pDrv->pReg->szName, pszName))
+            return pDrv;
+    return NULL;
+}
+
+
+/**
+ * Instantiate a driver.
+ *
+ * @returns VBox status code, including informational statuses.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pNode               The CFGM node for the driver.
+ * @param   pBaseInterface      The base interface.
+ * @param   pDrvAbove           The driver above it.  NULL if it's the top-most
+ *                              driver.
+ * @param   pLun                The LUN the driver is being attached to.  NULL
+ *                              if we're instantiating a driver chain before
+ *                              attaching it - untested.
+ * @param   ppBaseInterface     Where to return the pointer to the base
+ *                              interface of the newly created driver.
+ *
+ * @remarks Recursive calls to this function is normal as the drivers will
+ *          attach to anything below them during the pfnContruct call.
+ */
+int pdmR3DrvInstantiate(PVM pVM, PCFGMNODE pNode, PPDMIBASE pBaseInterface, PPDMDRVINS pDrvAbove,
+                        PPDMLUN pLun, PPDMIBASE *ppBaseInterface)
+{
+    Assert(!pDrvAbove || !pDrvAbove->Internal.s.pDown);
+    Assert(!pDrvAbove || !pDrvAbove->pDownBase);
+
+    Assert(pBaseInterface->pfnQueryInterface(pBaseInterface, PDMIBASE_IID) == pBaseInterface);
+
+    /*
+     * Find the driver.
+     */
+    char *pszName;
+    int rc = CFGMR3QueryStringAlloc(pNode, "Driver", &pszName);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMDRV pDrv = pdmR3DrvLookup(pVM, pszName);
+        MMR3HeapFree(pszName);
+        if (    pDrv
+            &&  pDrv->cInstances < pDrv->pReg->cMaxInstances)
+        {
+            /* config node */
+            PCFGMNODE pConfigNode = CFGMR3GetChild(pNode, "Config");
+            if (!pConfigNode)
+                rc = CFGMR3InsertNode(pNode, "Config", &pConfigNode);
+            if (RT_SUCCESS(rc))
+            {
+                CFGMR3SetRestrictedRoot(pConfigNode);
+
+                /*
+                 * Allocate the driver instance.
+                 */
+                size_t cb = RT_OFFSETOF(PDMDRVINS, achInstanceData[pDrv->pReg->cbInstance]);
+                cb = RT_ALIGN_Z(cb, 16);
+                bool const fHyperHeap = !!(pDrv->pReg->fFlags & (PDM_DRVREG_FLAGS_R0 | PDM_DRVREG_FLAGS_RC));
+                PPDMDRVINS pNew;
+                if (fHyperHeap)
+                    rc = MMHyperAlloc(pVM, cb, 64, MM_TAG_PDM_DRIVER, (void **)&pNew);
+                else
+                    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_DRIVER, cb, (void **)&pNew);
+                if (pNew)
+                {
+                    /*
+                     * Initialize the instance structure (declaration order).
+                     */
+                    pNew->u32Version                = PDM_DRVINS_VERSION;
+                    pNew->Internal.s.pUp            = pDrvAbove ? pDrvAbove : NULL;
+                    //pNew->Internal.s.pDown          = NULL;
+                    pNew->Internal.s.pLun           = pLun;
+                    pNew->Internal.s.pDrv           = pDrv;
+                    pNew->Internal.s.pVMR3          = pVM;
+                    pNew->Internal.s.pVMR0          = pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_R0 ? pVM->pVMR0 : NIL_RTR0PTR;
+                    pNew->Internal.s.pVMRC          = pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_RC ? pVM->pVMRC : NIL_RTRCPTR;
+                    //pNew->Internal.s.fDetaching     = false;
+                    pNew->Internal.s.fVMSuspended   = true;
+                    //pNew->Internal.s.fVMReset       = false;
+                    pNew->Internal.s.fHyperHeap     = fHyperHeap;
+                    //pNew->Internal.s.pfnAsyncNotify = NULL;
+                    pNew->Internal.s.pCfgHandle     = pNode;
+                    pNew->pReg                      = pDrv->pReg;
+                    pNew->pCfg                      = pConfigNode;
+                    pNew->iInstance                 = pDrv->iNextInstance;
+                    pNew->pUpBase                   = pBaseInterface;
+                    Assert(!pDrvAbove || pBaseInterface == &pDrvAbove->IBase);
+                    //pNew->pDownBase                 = NULL;
+                    //pNew->IBase.pfnQueryInterface   = NULL;
+                    pNew->pHlpR3                    = &g_pdmR3DrvHlp;
+                    pNew->pvInstanceDataR3          = &pNew->achInstanceData[0];
+                    if (pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+                    {
+                        pNew->pvInstanceDataR0      = MMHyperR3ToR0(pVM, &pNew->achInstanceData[0]);
+                        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "g_pdmR0DrvHlp", &pNew->pHlpR0);
+                        AssertReleaseRCReturn(rc, rc);
+
+                    }
+                    if (pDrv->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+                    {
+                        pNew->pvInstanceDataR0      = MMHyperR3ToRC(pVM, &pNew->achInstanceData[0]);
+                        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_pdmRCDrvHlp", &pNew->pHlpRC);
+                        AssertReleaseRCReturn(rc, rc);
+                    }
+
+                    pDrv->iNextInstance++;
+                    pDrv->cInstances++;
+
+                    /*
+                     * Link with it with the driver above / LUN.
+                     */
+                    if (pDrvAbove)
+                    {
+                        pDrvAbove->pDownBase        = &pNew->IBase;
+                        pDrvAbove->Internal.s.pDown = pNew;
+                    }
+                    else if (pLun)
+                        pLun->pTop                  = pNew;
+                    if (pLun)
+                        pLun->pBottom               = pNew;
+
+                    /*
+                     * Invoke the constructor.
+                     */
+                    rc = pDrv->pReg->pfnConstruct(pNew, pNew->pCfg, 0 /*fFlags*/);
+                    if (RT_SUCCESS(rc))
+                    {
+                        AssertPtr(pNew->IBase.pfnQueryInterface);
+                        Assert(pNew->IBase.pfnQueryInterface(&pNew->IBase, PDMIBASE_IID) == &pNew->IBase);
+
+                        /* Success! */
+                        *ppBaseInterface = &pNew->IBase;
+                        if (pLun)
+                            Log(("PDM: Attached driver %p:'%s'/%d to LUN#%d on device '%s'/%d, pDrvAbove=%p:'%s'/%d\n",
+                                 pNew, pDrv->pReg->szName, pNew->iInstance,
+                                 pLun->iLun,
+                                 pLun->pDevIns ? pLun->pDevIns->pReg->szName : pLun->pUsbIns->pReg->szName,
+                                 pLun->pDevIns ? pLun->pDevIns->iInstance    : pLun->pUsbIns->iInstance,
+                                 pDrvAbove, pDrvAbove ? pDrvAbove->pReg->szName : "", pDrvAbove ? pDrvAbove->iInstance : UINT32_MAX));
+                        else
+                            Log(("PDM: Attached driver %p:'%s'/%d, pDrvAbove=%p:'%s'/%d\n",
+                                 pNew, pDrv->pReg->szName, pNew->iInstance,
+                                 pDrvAbove, pDrvAbove ? pDrvAbove->pReg->szName : "", pDrvAbove ? pDrvAbove->iInstance : UINT32_MAX));
+                    }
+                    else
+                        pdmR3DrvDestroyChain(pNew, PDM_TACH_FLAGS_NO_CALLBACKS);
+                }
+                else
+                {
+                    AssertMsgFailed(("Failed to allocate %d bytes for instantiating driver '%s'\n", cb, pszName));
+                    rc = VERR_NO_MEMORY;
+                }
+            }
+            else
+                AssertMsgFailed(("Failed to create Config node! rc=%Rrc\n", rc));
+        }
+        else if (pDrv)
+        {
+            AssertMsgFailed(("Too many instances of driver '%s', max is %u\n", pszName, pDrv->pReg->cMaxInstances));
+            rc = VERR_PDM_TOO_MANY_DRIVER_INSTANCES;
+        }
+        else
+        {
+            AssertMsgFailed(("Driver '%s' wasn't found!\n", pszName));
+            rc = VERR_PDM_DRIVER_NOT_FOUND;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Query for string value of \"Driver\" -> %Rrc\n", rc));
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            rc = VERR_PDM_CFG_MISSING_DRIVER_NAME;
+    }
+    return rc;
+}
+
+
+/**
+ * Detaches a driver from whatever it's attached to.
+ * This will of course lead to the destruction of the driver and all drivers below it in the chain.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pDrvIns     The driver instance to detach.
+ * @param   fFlags      Flags, combination of the PDMDEVATT_FLAGS_* \#defines.
+ */
+int pdmR3DrvDetach(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvDetach: pDrvIns=%p '%s'/%d\n", pDrvIns, pDrvIns->pReg->szName, pDrvIns->iInstance));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    /*
+     * Check that we're not doing this recursively, that could have unwanted sideeffects!
+     */
+    if (pDrvIns->Internal.s.fDetaching)
+    {
+        AssertMsgFailed(("Recursive detach! '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance));
+        return VINF_SUCCESS;           }
+
+    /*
+     * Check that we actually can detach this instance.
+     * The requirement is that the driver/device above has a detach method.
+     */
+    if (pDrvIns->Internal.s.pUp
+        ? !pDrvIns->Internal.s.pUp->pReg->pfnDetach
+        : !pDrvIns->Internal.s.pLun->pDevIns->pReg->pfnDetach)
+    {
+        AssertMsgFailed(("Cannot detach driver instance because the driver/device above doesn't support it!\n"));
+        return VERR_PDM_DRIVER_DETACH_NOT_POSSIBLE;
+    }
+
+    /*
+     * Join paths with pdmR3DrvDestroyChain.
+     */
+    pdmR3DrvDestroyChain(pDrvIns, fFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys a driver chain starting with the specified driver.
+ *
+ * This is used when unplugging a device at run time.
+ *
+ * @param   pDrvIns     Pointer to the driver instance to start with.
+ * @param   fFlags      PDM_TACH_FLAGS_NOT_HOT_PLUG, PDM_TACH_FLAGS_NO_CALLBACKS
+ *                      or 0.
+ */
+void pdmR3DrvDestroyChain(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Detach the bottommost driver until we've detached pDrvIns.
+     */
+    pDrvIns->Internal.s.fDetaching = true;
+    PPDMDRVINS pCur;
+    do
+    {
+        /* find the driver to detach. */
+        pCur = pDrvIns;
+        while (pCur->Internal.s.pDown)
+            pCur = pCur->Internal.s.pDown;
+        LogFlow(("pdmR3DrvDestroyChain: pCur=%p '%s'/%d\n", pCur, pCur->pReg->szName, pCur->iInstance));
+
+        /*
+         * Unlink it and notify parent.
+         */
+        pCur->Internal.s.fDetaching = true;
+
+        PPDMLUN pLun = pCur->Internal.s.pLun;
+        Assert(pLun->pBottom == pCur);
+        pLun->pBottom = pCur->Internal.s.pUp;
+
+        if (pCur->Internal.s.pUp)
+        {
+            /* driver parent */
+            PPDMDRVINS pParent = pCur->Internal.s.pUp;
+            pCur->Internal.s.pUp = NULL;
+            pParent->Internal.s.pDown = NULL;
+
+            if (!(fFlags & PDM_TACH_FLAGS_NO_CALLBACKS) && pParent->pReg->pfnDetach)
+                pParent->pReg->pfnDetach(pParent, fFlags);
+
+            pParent->pDownBase = NULL;
+        }
+        else
+        {
+            /* device parent */
+            Assert(pLun->pTop == pCur);
+            pLun->pTop = NULL;
+            if (!(fFlags & PDM_TACH_FLAGS_NO_CALLBACKS) && pLun->pDevIns->pReg->pfnDetach)
+                pLun->pDevIns->pReg->pfnDetach(pLun->pDevIns, pLun->iLun, fFlags);
+        }
+
+        /*
+         * Call destructor.
+         */
+        pCur->pUpBase = NULL;
+        if (pCur->pReg->pfnDestruct)
+            pCur->pReg->pfnDestruct(pCur);
+        pCur->Internal.s.pDrv->cInstances--;
+
+        /*
+         * Free all resources allocated by the driver.
+         */
+        /* Queues. */
+        int rc = PDMR3QueueDestroyDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* Timers. */
+        rc = TMR3TimerDestroyDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* SSM data units. */
+        rc = SSMR3DeregisterDriver(pVM, pCur, NULL, 0);
+        AssertRC(rc);
+
+        /* PDM threads. */
+        rc = pdmR3ThreadDestroyDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* Info handlers. */
+        rc = DBGFR3InfoDeregisterDriver(pVM, pCur, NULL);
+        AssertRC(rc);
+
+        /* PDM critsects. */
+        rc = pdmR3CritSectDeleteDriver(pVM, pCur);
+        AssertRC(rc);
+
+        /* Block caches. */
+        PDMR3BlkCacheReleaseDriver(pVM, pCur);
+
+        /* Finally, the driver it self. */
+        bool fHyperHeap = pCur->Internal.s.fHyperHeap;
+        ASMMemFill32(pCur, RT_OFFSETOF(PDMDRVINS, achInstanceData[pCur->pReg->cbInstance]), 0xdeadd0d0);
+        if (fHyperHeap)
+            MMHyperFree(pVM, pCur);
+        else
+            MMR3HeapFree(pCur);
+
+    } while (pCur != pDrvIns);
+}
+
+
+
+
+/** @name Driver Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMDRVHLP,pfnAttach} */
+static DECLCALLBACK(int) pdmR3DrvHlp_Attach(PPDMDRVINS pDrvIns, uint32_t fFlags, PPDMIBASE *ppBaseInterface)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DrvHlp_Attach: caller='%s'/%d: fFlags=%#x\n", pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
+    Assert(!(fFlags & ~(PDM_TACH_FLAGS_NOT_HOT_PLUG)));
+
+    /*
+     * Check that there isn't anything attached already.
+     */
+    int rc;
+    if (!pDrvIns->Internal.s.pDown)
+    {
+        Assert(pDrvIns->Internal.s.pLun->pBottom == pDrvIns);
+
+        /*
+         * Get the attached driver configuration.
+         */
+        PCFGMNODE pNode = CFGMR3GetChild(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver");
+        if (pNode)
+            rc = pdmR3DrvInstantiate(pVM, pNode, &pDrvIns->IBase, pDrvIns, pDrvIns->Internal.s.pLun, ppBaseInterface);
+        else
+            rc = VERR_PDM_NO_ATTACHED_DRIVER;
+    }
+    else
+    {
+        AssertMsgFailed(("Already got a driver attached. The driver should keep track of such things!\n"));
+        rc = VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+
+    LogFlow(("pdmR3DrvHlp_Attach: caller='%s'/%d: return %Rrc\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnDetach} */
+static DECLCALLBACK(int) pdmR3DrvHlp_Detach(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_Detach: caller='%s'/%d: fFlags=%#x\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    /*
+     * Anything attached?
+     */
+    int rc;
+    if (pDrvIns->Internal.s.pDown)
+        rc = pdmR3DrvDetach(pDrvIns->Internal.s.pDown, fFlags);
+    else
+    {
+        AssertMsgFailed(("Nothing attached!\n"));
+        rc = VERR_PDM_NO_DRIVER_ATTACHED;
+    }
+
+    LogFlow(("pdmR3DrvHlp_Detach: caller='%s'/%d: returns %Rrc\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnDetachSelf} */
+static DECLCALLBACK(int) pdmR3DrvHlp_DetachSelf(PPDMDRVINS pDrvIns, uint32_t fFlags)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_DetachSelf: caller='%s'/%d: fFlags=%#x\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, fFlags));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    int rc = pdmR3DrvDetach(pDrvIns, fFlags);
+
+    LogFlow(("pdmR3DrvHlp_Detach: returns %Rrc\n", rc)); /* pDrvIns is freed by now. */
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnMountPrepare} */
+static DECLCALLBACK(int) pdmR3DrvHlp_MountPrepare(PPDMDRVINS pDrvIns, const char *pszFilename, const char *pszCoreDriver)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: pszFilename=%p:{%s} pszCoreDriver=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszFilename, pszFilename, pszCoreDriver, pszCoreDriver));
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    /*
+     * Do the caller have anything attached below itself?
+     */
+    if (pDrvIns->Internal.s.pDown)
+    {
+        AssertMsgFailed(("Cannot prepare a mount when something's attached to you!\n"));
+        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+
+    /*
+     * We're asked to prepare, so we'll start off by nuking the
+     * attached configuration tree.
+     */
+    PCFGMNODE   pNode = CFGMR3GetChild(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver");
+    if (pNode)
+        CFGMR3RemoveNode(pNode);
+
+    /*
+     * If there is no core driver, we'll have to probe for it.
+     */
+    if (!pszCoreDriver)
+    {
+        /** @todo implement image probing. */
+        AssertReleaseMsgFailed(("Not implemented!\n"));
+        return VERR_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * Construct the basic attached driver configuration.
+     */
+    int rc = CFGMR3InsertNode(pDrvIns->Internal.s.pCfgHandle, "AttachedDriver", &pNode);
+    if (RT_SUCCESS(rc))
+    {
+        rc = CFGMR3InsertString(pNode, "Driver", pszCoreDriver);
+        if (RT_SUCCESS(rc))
+        {
+            PCFGMNODE pCfg;
+            rc = CFGMR3InsertNode(pNode, "Config", &pCfg);
+            if (RT_SUCCESS(rc))
+            {
+                rc = CFGMR3InsertString(pCfg, "Path", pszFilename);
+                if (RT_SUCCESS(rc))
+                {
+                    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: returns %Rrc (Driver=%s)\n",
+                             pDrvIns->pReg->szName, pDrvIns->iInstance, rc, pszCoreDriver));
+                    return rc;
+                }
+                else
+                    AssertMsgFailed(("Path string insert failed, rc=%Rrc\n", rc));
+            }
+            else
+                AssertMsgFailed(("Config node failed, rc=%Rrc\n", rc));
+        }
+        else
+            AssertMsgFailed(("Driver string insert failed, rc=%Rrc\n", rc));
+        CFGMR3RemoveNode(pNode);
+    }
+    else
+        AssertMsgFailed(("AttachedDriver node insert failed, rc=%Rrc\n", rc));
+
+    LogFlow(("pdmR3DrvHlp_MountPrepare: caller='%s'/%d: returns %Rrc\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnAssertEMT} */
+static DECLCALLBACK(bool) pdmR3DrvHlp_AssertEMT(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    if (VM_IS_EMT(pDrvIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnAssertOther} */
+static DECLCALLBACK(bool) pdmR3DrvHlp_AssertOther(PPDMDRVINS pDrvIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    if (!VM_IS_EMT(pDrvIns->Internal.s.pVMR3))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    return false;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnVMSetError} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetError(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc2 == rc); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetErrorV(PPDMDRVINS pDrvIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    int rc2 = VMSetErrorV(pDrvIns->Internal.s.pVMR3, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnVMSetRuntimeError} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetRuntimeError(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    va_list args;
+    va_start(args, pszFormat);
+    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, args);
+    va_end(args);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR3DrvHlp_VMSetRuntimeErrorV(PPDMDRVINS pDrvIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    int rc = VMSetRuntimeErrorV(pDrvIns->Internal.s.pVMR3, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMState} */
+static DECLCALLBACK(VMSTATE) pdmR3DrvHlp_VMState(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    VMSTATE enmVMState = VMR3GetState(pDrvIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DrvHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+             enmVMState, VMR3GetStateName(enmVMState)));
+    return enmVMState;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnVMTeleportedAndNotFullyResumedYet} */
+static DECLCALLBACK(bool) pdmR3DrvHlp_VMTeleportedAndNotFullyResumedYet(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    bool fRc = VMR3TeleportedAndNotFullyResumedYet(pDrvIns->Internal.s.pVMR3);
+
+    LogFlow(("pdmR3DrvHlp_VMState: caller='%s'/%d: returns %RTbool)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+             fRc));
+    return fRc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLPR3,pfnGetSupDrvSession} */
+static DECLCALLBACK(PSUPDRVSESSION) pdmR3DrvHlp_GetSupDrvSession(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    PSUPDRVSESSION pSession = pDrvIns->Internal.s.pVMR3->pSession;
+    LogFlow(("pdmR3DrvHlp_GetSupDrvSession: caller='%s'/%d: returns %p)\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+             pSession));
+    return pSession;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnQueueCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_QueueCreate(PPDMDRVINS pDrvIns, uint32_t cbItem, uint32_t cItems, uint32_t cMilliesInterval,
+                                                 PFNPDMQUEUEDRV pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_PDMQueueCreate: caller='%s'/%d: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%p:{%s} ppQueue=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, pszName, ppQueue, ppQueue));
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+
+    if (pDrvIns->iInstance > 0)
+    {
+        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DRIVER_DESC, "%s_%u", pszName, pDrvIns->iInstance);
+        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
+    }
+
+    int rc = PDMR3QueueCreateDriver(pVM, pDrvIns, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, ppQueue);
+
+    LogFlow(("pdmR3DrvHlp_PDMQueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc, *ppQueue));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnTMGetVirtualFreq} */
+static DECLCALLBACK(uint64_t) pdmR3DrvHlp_TMGetVirtualFreq(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    return TMVirtualGetFreq(pDrvIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnTMGetVirtualTime} */
+static DECLCALLBACK(uint64_t) pdmR3DrvHlp_TMGetVirtualTime(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+
+    return TMVirtualGet(pDrvIns->Internal.s.pVMR3);
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnTMTimerCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_TMTimerCreate(PPDMDRVINS pDrvIns, TMCLOCK enmClock, PFNTMTIMERDRV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
+
+    int rc = TMR3TimerCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer);
+
+    LogFlow(("pdmR3DrvHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc *ppTimer=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc, *ppTimer));
+    return rc;
+}
+
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSSMRegister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SSMRegister(PPDMDRVINS pDrvIns, uint32_t uVersion, size_t cbGuess,
+                                                 PFNSSMDRVLIVEPREP pfnLivePrep, PFNSSMDRVLIVEEXEC pfnLiveExec, PFNSSMDRVLIVEVOTE pfnLiveVote,
+                                                 PFNSSMDRVSAVEPREP pfnSavePrep, PFNSSMDRVSAVEEXEC pfnSaveExec, PFNSSMDRVSAVEDONE pfnSaveDone,
+                                                 PFNSSMDRVLOADPREP pfnLoadPrep, PFNSSMDRVLOADEXEC pfnLoadExec, PFNSSMDRVLOADDONE pfnLoadDone)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_SSMRegister: caller='%s'/%d: uVersion=#x cbGuess=%#x \n"
+             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p  pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoaddone=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, uVersion, cbGuess,
+             pfnLivePrep, pfnLiveExec, pfnLiveVote,
+             pfnSavePrep, pfnSaveExec, pfnSaveDone, pfnLoadPrep, pfnLoadExec, pfnLoadDone));
+
+    int rc = SSMR3RegisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pDrvIns->pReg->szName, pDrvIns->iInstance,
+                                 uVersion, cbGuess,
+                                 pfnLivePrep, pfnLiveExec, pfnLiveVote,
+                                 pfnSavePrep, pfnSaveExec, pfnSaveDone,
+                                 pfnLoadPrep, pfnLoadExec, pfnLoadDone);
+
+    LogFlow(("pdmR3DrvHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSSMDeregister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SSMDeregister(PPDMDRVINS pDrvIns, const char *pszName, uint32_t u32Instance)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_SSMDeregister: caller='%s'/%d: pszName=%p:{%s} u32Instance=%#x\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName, u32Instance));
+
+    int rc = SSMR3DeregisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pszName, u32Instance);
+
+    LogFlow(("pdmR3DrvHlp_SSMDeregister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLP,pfnDBGFInfoRegister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_DBGFInfoRegister(PPDMDRVINS pDrvIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERDRV pfnHandler)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    int rc = DBGFR3InfoRegisterDriver(pDrvIns->Internal.s.pVMR3, pszName, pszDesc, pfnHandler, pDrvIns);
+
+    LogFlow(("pdmR3DrvHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDEVHLP,pfnDBGFInfoDeregister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_DBGFInfoDeregister(PPDMDRVINS pDrvIns, const char *pszName)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_DBGFInfoDeregister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pszName));
+
+    int rc = DBGFR3InfoDeregisterDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, pszName);
+
+    LogFlow(("pdmR3DrvHlp_DBGFInfoDeregister: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSTAMRegister} */
+static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegister(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    STAM_REG(pDrvIns->Internal.s.pVMR3, pvSample, enmType, pszName, enmUnit, pszDesc);
+    /** @todo track the samples so they can be dumped & deregistered when the driver instance is destroyed.
+     * For now we just have to be careful not to use this call for drivers which can be unloaded. */
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSTAMRegisterF} */
+static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegisterF(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, ...)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterV(pDrvIns->Internal.s.pVMR3, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    AssertRC(rc);
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSTAMRegisterV} */
+static DECLCALLBACK(void) pdmR3DrvHlp_STAMRegisterV(PPDMDRVINS pDrvIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    int rc = STAMR3RegisterV(pDrvIns->Internal.s.pVMR3, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    AssertRC(rc);
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSTAMDeregister} */
+static DECLCALLBACK(int) pdmR3DrvHlp_STAMDeregister(PPDMDRVINS pDrvIns, void *pvSample)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+
+    int rc = STAMR3DeregisterU(pDrvIns->Internal.s.pVMR3->pUVM, pvSample);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSUPCallVMMR0Ex} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SUPCallVMMR0Ex(PPDMDRVINS pDrvIns, unsigned uOperation, void *pvArg, unsigned cbArg)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_SSMCallVMMR0Ex: caller='%s'/%d: uOperation=%u pvArg=%p cbArg=%d\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, uOperation, pvArg, cbArg));
+    int rc;
+    if (    uOperation >= VMMR0_DO_SRV_START
+        &&  uOperation <  VMMR0_DO_SRV_END)
+        rc = SUPR3CallVMMR0Ex(pDrvIns->Internal.s.pVMR3->pVMR0, NIL_VMCPUID, uOperation, 0, (PSUPVMMR0REQHDR)pvArg);
+    else
+    {
+        AssertMsgFailed(("Invalid uOperation=%u\n", uOperation));
+        rc = VERR_INVALID_PARAMETER;
+    }
+
+    LogFlow(("pdmR3DrvHlp_SUPCallVMMR0Ex: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnUSBRegisterHub} */
+static DECLCALLBACK(int) pdmR3DrvHlp_USBRegisterHub(PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_USBRegisterHub: caller='%s'/%d: fVersions=%#x cPorts=%#x pUsbHubReg=%p ppUsbHubHlp=%p\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, fVersions, cPorts, pUsbHubReg, ppUsbHubHlp));
+
+#ifdef VBOX_WITH_USB
+    int rc = pdmR3UsbRegisterHub(pDrvIns->Internal.s.pVMR3, pDrvIns, fVersions, cPorts, pUsbHubReg, ppUsbHubHlp);
+#else
+    int rc = VERR_NOT_SUPPORTED;
+#endif
+
+    LogFlow(("pdmR3DrvHlp_USBRegisterHub: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnSetAsyncNotification} */
+static DECLCALLBACK(int) pdmR3DrvHlp_SetAsyncNotification(PPDMDRVINS pDrvIns, PFNPDMDRVASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT0(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pDrvIns->pReg->szName, pDrvIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pDrvIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pDrvIns->Internal.s.fVMSuspended || pDrvIns->Internal.s.fVMReset, rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pDrvIns->Internal.s.pVMR3);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_RESETTING
+               || enmVMState == VMSTATE_RESETTING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pDrvIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3DrvHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName, pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnAsyncNotificationCompleted} */
+static DECLCALLBACK(void) pdmR3DrvHlp_AsyncNotificationCompleted(PPDMDRVINS pDrvIns)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (   enmVMState == VMSTATE_SUSPENDING
+        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+        || enmVMState == VMSTATE_SUSPENDING_LS
+        || enmVMState == VMSTATE_RESETTING
+        || enmVMState == VMSTATE_RESETTING_LS
+        || enmVMState == VMSTATE_POWERING_OFF
+        || enmVMState == VMSTATE_POWERING_OFF_LS)
+    {
+        LogFlow(("pdmR3DrvHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pDrvIns->pReg->szName, pDrvIns->iInstance));
+        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
+    }
+    else
+        LogFlow(("pdmR3DrvHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pDrvIns->pReg->szName, pDrvIns->iInstance, enmVMState));
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnThreadCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_ThreadCreate(PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
+                                                  PFNPDMTHREADWAKEUPDRV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_ThreadCreate: caller='%s'/%d: ppThread=%p pvUser=%p pfnThread=%p pfnWakeup=%p cbStack=%#zx enmType=%d pszName=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName, pszName));
+
+    int rc = pdmR3ThreadCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppThread, pvUser, pfnThread, pfnWakeup, cbStack, enmType, pszName);
+
+    LogFlow(("pdmR3DrvHlp_ThreadCreate: caller='%s'/%d: returns %Rrc *ppThread=%RTthrd\n", pDrvIns->pReg->szName, pDrvIns->iInstance,
+            rc, *ppThread));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnAsyncCompletionTemplateCreate} */
+static DECLCALLBACK(int) pdmR3DrvHlp_AsyncCompletionTemplateCreate(PPDMDRVINS pDrvIns, PPPDMASYNCCOMPLETIONTEMPLATE ppTemplate,
+                                                                   PFNPDMASYNCCOMPLETEDRV pfnCompleted, void *pvTemplateUser,
+                                                                   const char *pszDesc)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    LogFlow(("pdmR3DrvHlp_AsyncCompletionTemplateCreate: caller='%s'/%d: ppTemplate=%p pfnCompleted=%p pszDesc=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, ppTemplate, pfnCompleted, pszDesc, pszDesc));
+
+    int rc = PDMR3AsyncCompletionTemplateCreateDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppTemplate, pfnCompleted, pvTemplateUser, pszDesc);
+
+    LogFlow(("pdmR3DrvHlp_AsyncCompletionTemplateCreate: caller='%s'/%d: returns %Rrc *ppThread=%p\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc, *ppTemplate));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnLdrGetRCInterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DrvHlp_LdrGetRCInterfaceSymbols(PPDMDRVINS pDrvIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_LdrGetRCInterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "drv", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDrvIns->pReg->szName) != NULL)
+    {
+        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_RC)
+            rc = PDMR3LdrGetInterfaceSymbols(pDrvIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDrvIns->pReg->szRCMod, pDrvIns->Internal.s.pDrv->pszRCSearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             false /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a raw-mode enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'drv' and contain the driver name!\n",
+                         pszSymPrefix, pDrvIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DrvHlp_LdrGetRCInterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnLdrGetR0InterfaceSymbols} */
+static DECLCALLBACK(int) pdmR3DrvHlp_LdrGetR0InterfaceSymbols(PPDMDRVINS pDrvIns, void *pvInterface, size_t cbInterface,
+                                                              const char *pszSymPrefix, const char *pszSymList)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    VM_ASSERT_EMT(pDrvIns->Internal.s.pVMR3);
+    LogFlow(("pdmR3DrvHlp_LdrGetR0InterfaceSymbols: caller='%s'/%d: pvInterface=%p cbInterface=%zu pszSymPrefix=%p:{%s} pszSymList=%p:{%s}\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pvInterface, cbInterface, pszSymPrefix, pszSymPrefix, pszSymList, pszSymList));
+
+    int rc;
+    if (   strncmp(pszSymPrefix, "drv", 3) == 0
+        && RTStrIStr(pszSymPrefix + 3, pDrvIns->pReg->szName) != NULL)
+    {
+        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+            rc = PDMR3LdrGetInterfaceSymbols(pDrvIns->Internal.s.pVMR3,
+                                             pvInterface, cbInterface,
+                                             pDrvIns->pReg->szR0Mod, pDrvIns->Internal.s.pDrv->pszR0SearchPath,
+                                             pszSymPrefix, pszSymList,
+                                             true /*fRing0OrRC*/);
+        else
+        {
+            AssertMsgFailed(("Not a ring-0 enabled driver\n"));
+            rc = VERR_PERMISSION_DENIED;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Invalid prefix '%s' for '%s'; must start with 'drv' and contain the driver name!\n",
+                         pszSymPrefix, pDrvIns->pReg->szName));
+        rc = VERR_INVALID_NAME;
+    }
+
+    LogFlow(("pdmR3DrvHlp_LdrGetR0InterfaceSymbols: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnCritSectInit} */
+static DECLCALLBACK(int) pdmR3DrvHlp_CritSectInit(PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect,
+                                                  RT_SRC_POS_DECL, const char *pszName)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3DrvHlp_CritSectInit: caller='%s'/%d: pCritSect=%p pszName=%s\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, pCritSect, pszName));
+
+    int rc = pdmR3CritSectInitDriver(pVM, pDrvIns, pCritSect, RT_SRC_POS_ARGS, "%s_%u", pszName, pDrvIns->iInstance);
+
+    LogFlow(("pdmR3DrvHlp_CritSectInit: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnCallR0} */
+static DECLCALLBACK(int) pdmR3DrvHlp_CallR0(PPDMDRVINS pDrvIns, uint32_t uOperation, uint64_t u64Arg)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    PVM pVM = pDrvIns->Internal.s.pVMR3;
+    LogFlow(("pdmR3DrvHlp_CallR0: caller='%s'/%d: uOperation=%#x u64Arg=%#RX64\n",
+             pDrvIns->pReg->szName, pDrvIns->iInstance, uOperation, u64Arg));
+
+    /*
+     * Lazy resolve the ring-0 entry point.
+     */
+    int rc = VINF_SUCCESS;
+    PFNPDMDRVREQHANDLERR0 pfnReqHandlerR0 = pDrvIns->Internal.s.pfnReqHandlerR0;
+    if (RT_UNLIKELY(pfnReqHandlerR0 == NIL_RTR0PTR))
+    {
+        if (pDrvIns->pReg->fFlags & PDM_DRVREG_FLAGS_R0)
+        {
+            char szSymbol[          sizeof("drvR0") + sizeof(pDrvIns->pReg->szName) + sizeof("ReqHandler")];
+            strcat(strcat(strcpy(szSymbol, "drvR0"),         pDrvIns->pReg->szName),         "ReqHandler");
+            szSymbol[sizeof("drvR0") - 1] = RT_C_TO_UPPER(szSymbol[sizeof("drvR0") - 1]);
+
+            rc = PDMR3LdrGetSymbolR0Lazy(pVM, pDrvIns->pReg->szR0Mod, pDrvIns->Internal.s.pDrv->pszR0SearchPath, szSymbol,
+                                         &pfnReqHandlerR0);
+            if (RT_SUCCESS(rc))
+                pDrvIns->Internal.s.pfnReqHandlerR0 = pfnReqHandlerR0;
+            else
+                pfnReqHandlerR0 = NIL_RTR0PTR;
+        }
+        else
+            rc = VERR_ACCESS_DENIED;
+    }
+    if (RT_LIKELY(pfnReqHandlerR0 != NIL_RTR0PTR))
+    {
+        /*
+         * Make the ring-0 call.
+         */
+        PDMDRIVERCALLREQHANDLERREQ Req;
+        Req.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+        Req.Hdr.cbReq       = sizeof(Req);
+        Req.pDrvInsR0       = PDMDRVINS_2_R0PTR(pDrvIns);
+        Req.uOperation      = uOperation;
+        Req.u32Alignment    = 0;
+        Req.u64Arg          = u64Arg;
+        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_PDM_DRIVER_CALL_REQ_HANDLER, 0, &Req.Hdr);
+    }
+
+    LogFlow(("pdmR3DrvHlp_CallR0: caller='%s'/%d: returns %Rrc\n", pDrvIns->pReg->szName,
+             pDrvIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnFTSetCheckpoint} */
+static DECLCALLBACK(int) pdmR3DrvHlp_FTSetCheckpoint(PPDMDRVINS pDrvIns, FTMCHECKPOINTTYPE enmType)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    return FTMSetCheckpoint(pDrvIns->Internal.s.pVMR3, enmType);
+}
+
+
+/** @interface_method_impl{PDMDRVHLP,pfnBlkCacheRetain} */
+static DECLCALLBACK(int) pdmR3DrvHlp_BlkCacheRetain(PPDMDRVINS pDrvIns, PPPDMBLKCACHE ppBlkCache,
+                                                    PFNPDMBLKCACHEXFERCOMPLETEDRV pfnXferComplete,
+                                                    PFNPDMBLKCACHEXFERENQUEUEDRV pfnXferEnqueue,
+                                                    const char *pcszId)
+{
+    PDMDRV_ASSERT_DRVINS(pDrvIns);
+    return PDMR3BlkCacheRetainDriver(pDrvIns->Internal.s.pVMR3, pDrvIns, ppBlkCache,
+                                     pfnXferComplete, pfnXferEnqueue, pcszId);
+}
+
+
+/**
+ * The driver helper structure.
+ */
+const PDMDRVHLPR3 g_pdmR3DrvHlp =
+{
+    PDM_DRVHLPR3_VERSION,
+    pdmR3DrvHlp_Attach,
+    pdmR3DrvHlp_Detach,
+    pdmR3DrvHlp_DetachSelf,
+    pdmR3DrvHlp_MountPrepare,
+    pdmR3DrvHlp_AssertEMT,
+    pdmR3DrvHlp_AssertOther,
+    pdmR3DrvHlp_VMSetError,
+    pdmR3DrvHlp_VMSetErrorV,
+    pdmR3DrvHlp_VMSetRuntimeError,
+    pdmR3DrvHlp_VMSetRuntimeErrorV,
+    pdmR3DrvHlp_VMState,
+    pdmR3DrvHlp_VMTeleportedAndNotFullyResumedYet,
+    pdmR3DrvHlp_GetSupDrvSession,
+    pdmR3DrvHlp_QueueCreate,
+    pdmR3DrvHlp_TMGetVirtualFreq,
+    pdmR3DrvHlp_TMGetVirtualTime,
+    pdmR3DrvHlp_TMTimerCreate,
+    pdmR3DrvHlp_SSMRegister,
+    pdmR3DrvHlp_SSMDeregister,
+    pdmR3DrvHlp_DBGFInfoRegister,
+    pdmR3DrvHlp_DBGFInfoDeregister,
+    pdmR3DrvHlp_STAMRegister,
+    pdmR3DrvHlp_STAMRegisterF,
+    pdmR3DrvHlp_STAMRegisterV,
+    pdmR3DrvHlp_STAMDeregister,
+    pdmR3DrvHlp_SUPCallVMMR0Ex,
+    pdmR3DrvHlp_USBRegisterHub,
+    pdmR3DrvHlp_SetAsyncNotification,
+    pdmR3DrvHlp_AsyncNotificationCompleted,
+    pdmR3DrvHlp_ThreadCreate,
+    pdmR3DrvHlp_AsyncCompletionTemplateCreate,
+    pdmR3DrvHlp_LdrGetRCInterfaceSymbols,
+    pdmR3DrvHlp_LdrGetR0InterfaceSymbols,
+    pdmR3DrvHlp_CritSectInit,
+    pdmR3DrvHlp_CallR0,
+    pdmR3DrvHlp_FTSetCheckpoint,
+    pdmR3DrvHlp_BlkCacheRetain,
+    PDM_DRVHLPR3_VERSION /* u32TheEnd */
+};
+
+/** @} */
Index: /trunk/src/VBox/VMM/VMMR3/PDMLdr.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMLdr.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMLdr.cpp	(revision 35333)
@@ -0,0 +1,1630 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device Manager, module loader.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+//#define PDMLDR_FAKE_MODE
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_LDR
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/vmm.h>
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/sup.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/hwaccm.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/ctype.h>
+#include <iprt/file.h>
+#include <iprt/ldr.h>
+#include <iprt/mem.h>
+#include <iprt/path.h>
+#include <iprt/string.h>
+
+#include <limits.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Structure which the user argument of the RTLdrGetBits() callback points to.
+ * @internal
+ */
+typedef struct PDMGETIMPORTARGS
+{
+    PVM         pVM;
+    PPDMMOD     pModule;
+} PDMGETIMPORTARGS, *PPDMGETIMPORTARGS;
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) pdmR3GetImportRC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser);
+static int      pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName, const char *pszSearchPath);
+static char    *pdmR3FileRC(const char *pszFile, const char *pszSearchPath);
+static char    *pdmR3FileR0(const char *pszFile, const char *pszSearchPath);
+static char    *pdmR3File(const char *pszFile, const char *pszDefaultExt, const char *pszSearchPath, bool fShared);
+
+
+
+/**
+ * Loads the VMMR0.r0 module early in the init process.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+VMMR3DECL(int) PDMR3LdrLoadVMMR0U(PUVM pUVM)
+{
+    return pdmR3LoadR0U(pUVM, NULL, VMMR0_MAIN_MODULE_NAME, NULL);
+}
+
+
+/**
+ * Init the module loader part of PDM.
+ *
+ * This routine will load the Host Context Ring-0 and Guest
+ * Context VMM modules.
+ *
+ * @returns VBox status code.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvVMMR0Mod  The opaque returned by PDMR3LdrLoadVMMR0.
+ */
+int pdmR3LdrInitU(PUVM pUVM)
+{
+#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE)
+    return VINF_SUCCESS;
+
+#else
+
+    /*
+     * Load the mandatory GC module, the VMMR0.r0 is loaded before VM creation.
+     */
+    return PDMR3LdrLoadRC(pUVM->pVM, NULL, VMMGC_MAIN_MODULE_NAME);
+#endif
+}
+
+
+/**
+ * Terminate the module loader part of PDM.
+ *
+ * This will unload and free all modules.
+ *
+ * @param   pVM         The VM handle.
+ *
+ * @remarks This is normally called twice during termination.
+ */
+void pdmR3LdrTermU(PUVM pUVM)
+{
+    /*
+     * Free the modules.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD pModule = pUVM->pdm.s.pModules;
+    pUVM->pdm.s.pModules = NULL;
+    while (pModule)
+    {
+        /* free loader item. */
+        if (pModule->hLdrMod != NIL_RTLDRMOD)
+        {
+            int rc2 = RTLdrClose(pModule->hLdrMod);
+            AssertRC(rc2);
+            pModule->hLdrMod = NIL_RTLDRMOD;
+        }
+
+        /* free bits. */
+        switch (pModule->eType)
+        {
+            case PDMMOD_TYPE_R0:
+            {
+                Assert(pModule->ImageBase);
+                int rc2 = SUPR3FreeModule((void *)(uintptr_t)pModule->ImageBase);
+                AssertRC(rc2);
+                pModule->ImageBase = 0;
+                break;
+            }
+
+#ifdef VBOX_WITH_RAW_MODE
+            case PDMMOD_TYPE_RC:
+#endif
+            case PDMMOD_TYPE_R3:
+                /* MM will free this memory for us - it's alloc only memory. :-) */
+                break;
+
+            default:
+                AssertMsgFailed(("eType=%d\n", pModule->eType));
+                break;
+        }
+        pModule->pvBits = NULL;
+
+        void    *pvFree = pModule;
+        pModule = pModule->pNext;
+        RTMemFree(pvFree);
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Applies relocations to GC modules.
+ *
+ * This must be done very early in the relocation
+ * process so that components can resolve GC symbols during relocation.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void) PDMR3LdrRelocateU(PUVM pUVM, RTGCINTPTR offDelta)
+{
+#ifdef VBOX_WITH_RAW_MODE
+    LogFlow(("PDMR3LdrRelocate: offDelta=%RGv\n", offDelta));
+
+    /*
+     * GC Modules.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    if (pUVM->pdm.s.pModules)
+    {
+        /*
+         * The relocation have to be done in two passes so imports
+         * can be correctly resolved. The first pass will update
+         * the ImageBase saving the current value in OldImageBase.
+         * The second pass will do the actual relocation.
+         */
+        /* pass 1 */
+        PPDMMOD pCur;
+        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+        {
+            if (pCur->eType == PDMMOD_TYPE_RC)
+            {
+                pCur->OldImageBase = pCur->ImageBase;
+                pCur->ImageBase = MMHyperR3ToRC(pUVM->pVM, pCur->pvBits);
+            }
+        }
+
+        /* pass 2 */
+        for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+        {
+            if (pCur->eType == PDMMOD_TYPE_RC)
+            {
+                PDMGETIMPORTARGS Args;
+                Args.pVM = pUVM->pVM;
+                Args.pModule = pCur;
+                int rc = RTLdrRelocate(pCur->hLdrMod, pCur->pvBits, pCur->ImageBase, pCur->OldImageBase,
+                                       pdmR3GetImportRC, &Args);
+                AssertFatalMsgRC(rc, ("RTLdrRelocate failed, rc=%d\n", rc));
+                DBGFR3ModuleRelocate(pUVM->pVM, pCur->OldImageBase, pCur->ImageBase, RTLdrSize(pCur->hLdrMod),
+                                     pCur->szFilename, pCur->szName);
+            }
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+#endif
+}
+
+
+/**
+ * Loads a module into the host context ring-3.
+ *
+ * This is used by the driver and device init functions to load modules
+ * containing the drivers and devices. The function can be extended to
+ * load modules which are not native to the environment we're running in,
+ * but at the moment this is not required.
+ *
+ * No reference counting is kept, since we don't implement any facilities
+ * for unloading the module. But the module will naturally be released
+ * when the VM terminates.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszFilename     Filename of the module binary.
+ * @param   pszName         Module name. Case sensitive and the length is limited!
+ */
+int pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsg(pUVM->pVM->pdm.s.offVM, ("bad init order!\n"));
+    Assert(pszFilename);
+    size_t cchFilename = strlen(pszFilename);
+    Assert(pszName);
+    size_t cchName = strlen(pszName);
+    PPDMMOD  pCur;
+    if (cchName >= sizeof(pCur->szName))
+    {
+        AssertMsgFailed(("Name is too long, cchName=%d pszName='%s'\n", cchName, pszName));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Try lookup the name and see if the module exists.
+     */
+    int rc;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    {
+        if (!strcmp(pCur->szName, pszName))
+        {
+            if (pCur->eType == PDMMOD_TYPE_R3)
+                rc = VINF_PDM_ALREADY_LOADED;
+            else
+                rc = VERR_PDM_MODULE_NAME_CLASH;
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+            AssertMsgRC(rc, ("We've already got a module '%s' loaded!\n", pszName));
+            return rc;
+        }
+    }
+
+    /*
+     * Allocate the module list node and initialize it.
+     */
+    const char *pszSuff = RTLdrGetSuff();
+    size_t      cchSuff = RTPathHaveExt(pszFilename) ? 0 : strlen(pszSuff);
+    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(RT_OFFSETOF(PDMMOD, szFilename[cchFilename + cchSuff + 1]));
+    if (pModule)
+    {
+        pModule->eType = PDMMOD_TYPE_R3;
+        memcpy(pModule->szName, pszName, cchName); /* memory is zero'd, no need to copy terminator :-) */
+        memcpy(pModule->szFilename, pszFilename, cchFilename);
+        memcpy(&pModule->szFilename[cchFilename], pszSuff, cchSuff);
+
+        /*
+         * Load the loader item.
+         */
+        RTERRINFOSTATIC ErrInfo;
+        RTErrInfoInitStatic(&ErrInfo);
+        rc = SUPR3HardenedLdrLoadPlugIn(pModule->szFilename, &pModule->hLdrMod, &ErrInfo.Core);
+        if (RT_SUCCESS(rc))
+        {
+            pModule->pNext = pUVM->pdm.s.pModules;
+            pUVM->pdm.s.pModules = pModule;
+        }
+        else
+        {
+            /* Something went wrong, most likely module not found. Don't consider other unlikely errors */
+            rc = VMSetError(pUVM->pVM, rc, RT_SRC_POS,
+                            N_("Unable to load R3 module %s (%s): %s"), pModule->szFilename, pszName, ErrInfo.Core.pszMsg);
+            RTMemFree(pModule);
+        }
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+#ifdef VBOX_WITH_RAW_MODE
+/**
+ * Resolve an external symbol during RTLdrGetBits() of a RC module.
+ *
+ * @returns VBox status code.
+ * @param   hLdrMod         The loader module handle.
+ * @param   pszModule       Module name.
+ * @param   pszSymbol       Symbol name, NULL if uSymbol should be used.
+ * @param   uSymbol         Symbol ordinal, ~0 if pszSymbol should be used.
+ * @param   pValue          Where to store the symbol value (address).
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) pdmR3GetImportRC(RTLDRMOD hLdrMod, const char *pszModule, const char *pszSymbol, unsigned uSymbol, RTUINTPTR *pValue, void *pvUser)
+{
+    PVM         pVM     = ((PPDMGETIMPORTARGS)pvUser)->pVM;
+    PPDMMOD     pModule = ((PPDMGETIMPORTARGS)pvUser)->pModule;
+
+    /*
+     * Adjust input.
+     */
+    if (pszModule && !*pszModule)
+        pszModule = NULL;
+
+    /*
+     * Builtin module.
+     */
+    if (!pszModule || !strcmp(pszModule, "VMMGCBuiltin.gc"))
+    {
+        int rc = VINF_SUCCESS;
+        if (!strcmp(pszSymbol, "g_VM"))
+            *pValue = pVM->pVMRC;
+        else if (!strcmp(pszSymbol, "g_CPUM"))
+            *pValue = VM_RC_ADDR(pVM, &pVM->cpum);
+        else if (!strcmp(pszSymbol, "g_TRPM"))
+            *pValue = VM_RC_ADDR(pVM, &pVM->trpm);
+        else if (!strcmp(pszSymbol, "g_TRPMCPU"))
+            *pValue = VM_RC_ADDR(pVM, &pVM->aCpus[0].trpm);
+        else if (   !strncmp(pszSymbol, "VMM", 3)
+                 || !strcmp(pszSymbol, "g_Logger")
+                 || !strcmp(pszSymbol, "g_RelLogger"))
+        {
+            RTRCPTR RCPtr = 0;
+            rc = VMMR3GetImportRC(pVM, pszSymbol, &RCPtr);
+            if (RT_SUCCESS(rc))
+                *pValue = RCPtr;
+        }
+        else if (   !strncmp(pszSymbol, "TM", 2)
+                 || !strcmp(pszSymbol, "g_pSUPGlobalInfoPage"))
+        {
+            RTRCPTR RCPtr = 0;
+            rc = TMR3GetImportRC(pVM, pszSymbol, &RCPtr);
+            if (RT_SUCCESS(rc))
+                *pValue = RCPtr;
+        }
+        else
+        {
+            AssertMsg(!pszModule, ("Unknown builtin symbol '%s' for module '%s'!\n", pszSymbol, pModule->szName)); NOREF(pModule);
+            rc = VERR_SYMBOL_NOT_FOUND;
+        }
+        if (RT_SUCCESS(rc) || pszModule)
+        {
+            if (RT_FAILURE(rc))
+                LogRel(("PDMLdr: Couldn't find symbol '%s' in module '%s'!\n", pszSymbol, pszModule));
+            return rc;
+        }
+    }
+
+    /*
+     * Search for module.
+     */
+    PUVM     pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD  pCur = pUVM->pdm.s.pModules;
+    while (pCur)
+    {
+        if (    pCur->eType == PDMMOD_TYPE_RC
+            &&  (   !pszModule
+                 || !strcmp(pCur->szName, pszModule))
+           )
+        {
+            /* Search for the symbol. */
+            int rc = RTLdrGetSymbolEx(pCur->hLdrMod, pCur->pvBits, pCur->ImageBase, pszSymbol, pValue);
+            if (RT_SUCCESS(rc))
+            {
+                AssertMsg(*pValue - pCur->ImageBase < RTLdrSize(pCur->hLdrMod),
+                          ("%RRv-%RRv %s %RRv\n", (RTRCPTR)pCur->ImageBase,
+                           (RTRCPTR)(pCur->ImageBase + RTLdrSize(pCur->hLdrMod) - 1),
+                           pszSymbol, (RTRCPTR)*pValue));
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                return rc;
+            }
+            if (pszModule)
+            {
+                RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                AssertLogRelMsgFailed(("PDMLdr: Couldn't find symbol '%s' in module '%s'!\n", pszSymbol, pszModule));
+                return VERR_SYMBOL_NOT_FOUND;
+            }
+        }
+
+        /* next */
+        pCur = pCur->pNext;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertLogRelMsgFailed(("Couldn't find module '%s' for resolving symbol '%s'!\n", pszModule, pszSymbol));
+    return VERR_SYMBOL_NOT_FOUND;
+}
+
+
+/**
+ * Loads a module into the guest context (i.e. into the Hypervisor memory region).
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM to load it into.
+ * @param   pszFilename     Filename of the module binary.
+ * @param   pszName         Module name. Case sensitive and the length is limited!
+ */
+VMMR3DECL(int) PDMR3LdrLoadRC(PVM pVM, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+    PUVM     pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD  pCur = pUVM->pdm.s.pModules;
+    while (pCur)
+    {
+        if (!strcmp(pCur->szName, pszName))
+        {
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            AssertMsgFailed(("We've already got a module '%s' loaded!\n", pszName));
+            return VERR_PDM_MODULE_NAME_CLASH;
+        }
+        /* next */
+        pCur = pCur->pNext;
+    }
+
+    /*
+     * Find the file if not specified.
+     */
+    char *pszFile = NULL;
+    if (!pszFilename)
+        pszFilename = pszFile = pdmR3FileRC(pszName, NULL);
+
+    /*
+     * Allocate the module list node.
+     */
+    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(sizeof(*pModule) + strlen(pszFilename));
+    if (!pModule)
+    {
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        RTMemTmpFree(pszFile);
+        return VERR_NO_MEMORY;
+    }
+    AssertMsg(strlen(pszName) + 1 < sizeof(pModule->szName),
+              ("pazName is too long (%d chars) max is %d chars.\n", strlen(pszName), sizeof(pModule->szName) - 1));
+    strcpy(pModule->szName, pszName);
+    pModule->eType = PDMMOD_TYPE_RC;
+    strcpy(pModule->szFilename, pszFilename);
+
+
+    /*
+     * Open the loader item.
+     */
+    RTERRINFOSTATIC ErrInfo;
+    RTErrInfoInitStatic(&ErrInfo);
+    int rc = SUPR3HardenedVerifyPlugIn(pszFilename, &ErrInfo.Core);
+    if (RT_SUCCESS(rc))
+    {
+        RTErrInfoClear(&ErrInfo.Core);
+        rc = RTLdrOpen(pszFilename, 0, RTLDRARCH_X86_32, &pModule->hLdrMod);
+    }
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Allocate space in the hypervisor.
+         */
+        size_t          cb = RTLdrSize(pModule->hLdrMod);
+        cb = RT_ALIGN_Z(cb, PAGE_SIZE);
+        uint32_t        cPages = (uint32_t)(cb >> PAGE_SHIFT);
+        if (((size_t)cPages << PAGE_SHIFT) == cb)
+        {
+            PSUPPAGE    paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(paPages[0]));
+            if (paPages)
+            {
+                rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pModule->pvBits, NULL /*pR0Ptr*/, paPages);
+                if (RT_SUCCESS(rc))
+                {
+                    RTGCPTR GCPtr;
+                    rc = MMR3HyperMapPages(pVM, pModule->pvBits, NIL_RTR0PTR,
+                                           cPages, paPages, pModule->szName, &GCPtr);
+                    if (RT_SUCCESS(rc))
+                    {
+                        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+
+                        /*
+                         * Get relocated image bits.
+                         */
+                        Assert(MMHyperR3ToRC(pVM, pModule->pvBits) == GCPtr);
+                        pModule->ImageBase = GCPtr;
+                        PDMGETIMPORTARGS Args;
+                        Args.pVM = pVM;
+                        Args.pModule = pModule;
+                        rc = RTLdrGetBits(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pdmR3GetImportRC, &Args);
+                        if (RT_SUCCESS(rc))
+                        {
+                            /*
+                             * Insert the module.
+                             */
+                            if (pUVM->pdm.s.pModules)
+                            {
+                                /* we don't expect this list to be very long, so rather save the tail pointer. */
+                                pCur = pUVM->pdm.s.pModules;
+                                while (pCur->pNext)
+                                    pCur = pCur->pNext;
+                                pCur->pNext = pModule;
+                            }
+                            else
+                                pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
+                            Log(("PDM: RC Module at %RRv %s (%s)\n", (RTRCPTR)pModule->ImageBase, pszName, pszFilename));
+                            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+                            RTMemTmpFree(pszFile);
+                            RTMemTmpFree(paPages);
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    else
+                    {
+                        AssertRC(rc);
+                        SUPR3PageFreeEx(pModule->pvBits, cPages);
+                    }
+                }
+                else
+                    AssertMsgFailed(("SUPR3PageAlloc(%d,) -> %Rrc\n", cPages, rc));
+                RTMemTmpFree(paPages);
+            }
+            else
+                rc = VERR_NO_TMP_MEMORY;
+        }
+        else
+            rc = VERR_OUT_OF_RANGE;
+        int rc2 = RTLdrClose(pModule->hLdrMod);
+        AssertRC(rc2);
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+    /* Don't consider VERR_PDM_MODULE_NAME_CLASH and VERR_NO_MEMORY above as these are very unlikely. */
+    if (RT_FAILURE(rc) && RTErrInfoIsSet(&ErrInfo.Core))
+        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Cannot load RC module %s: %s"), pszFilename, ErrInfo.Core.pszMsg);
+    else if (RT_FAILURE(rc))
+        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Cannot load RC module %s"), pszFilename);
+
+    RTMemFree(pModule);
+    RTMemTmpFree(pszFile);
+    return rc;
+}
+#endif /* VBOX_WITH_RAW_MODE */
+
+
+/**
+ * Loads a module into the ring-0 context.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszFilename     Filename of the module binary.
+ * @param   pszName         Module name. Case sensitive and the length is limited!
+ * @param   pszSearchPath   List of directories to search if @a pszFilename is
+ *                          not specified.  Can be NULL, in which case the arch
+ *                          dependent install dir is searched.
+ */
+static int pdmR3LoadR0U(PUVM pUVM, const char *pszFilename, const char *pszName, const char *pszSearchPath)
+{
+    /*
+     * Validate input.
+     */
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMMOD pCur = pUVM->pdm.s.pModules;
+    while (pCur)
+    {
+        if (!strcmp(pCur->szName, pszName))
+        {
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            AssertMsgFailed(("We've already got a module '%s' loaded!\n", pszName));
+            return VERR_PDM_MODULE_NAME_CLASH;
+        }
+        /* next */
+        pCur = pCur->pNext;
+    }
+
+    /*
+     * Find the file if not specified.
+     */
+    char *pszFile = NULL;
+    if (!pszFilename)
+        pszFilename = pszFile = pdmR3FileR0(pszName, pszSearchPath);
+
+    /*
+     * Allocate the module list node.
+     */
+    PPDMMOD     pModule = (PPDMMOD)RTMemAllocZ(sizeof(*pModule) + strlen(pszFilename));
+    if (!pModule)
+    {
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        RTMemTmpFree(pszFile);
+        return VERR_NO_MEMORY;
+    }
+    AssertMsg(strlen(pszName) + 1 < sizeof(pModule->szName),
+              ("pazName is too long (%d chars) max is %d chars.\n", strlen(pszName), sizeof(pModule->szName) - 1));
+    strcpy(pModule->szName, pszName);
+    pModule->eType = PDMMOD_TYPE_R0;
+    strcpy(pModule->szFilename, pszFilename);
+
+    /*
+     * Ask the support library to load it.
+     */
+    void           *pvImageBase;
+    RTERRINFOSTATIC ErrInfo;
+    RTErrInfoInitStatic(&ErrInfo);
+    int rc = SUPR3LoadModule(pszFilename, pszName, &pvImageBase, &ErrInfo.Core);
+    if (RT_SUCCESS(rc))
+    {
+        pModule->hLdrMod = NIL_RTLDRMOD;
+        pModule->ImageBase = (uintptr_t)pvImageBase;
+
+        /*
+         * Insert the module.
+         */
+        if (pUVM->pdm.s.pModules)
+        {
+            /* we don't expect this list to be very long, so rather save the tail pointer. */
+            pCur = pUVM->pdm.s.pModules;
+            while (pCur->pNext)
+                pCur = pCur->pNext;
+            pCur->pNext = pModule;
+        }
+        else
+            pUVM->pdm.s.pModules = pModule; /* (pNext is zeroed by alloc) */
+        Log(("PDM: R0 Module at %RHv %s (%s)\n", (RTR0PTR)pModule->ImageBase, pszName, pszFilename));
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        RTMemTmpFree(pszFile);
+        return VINF_SUCCESS;
+    }
+
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    RTMemFree(pModule);
+    LogRel(("pdmR3LoadR0U: pszName=\"%s\" rc=%Rrc szErr=\"%s\"\n", pszName, rc, ErrInfo.Core.pszMsg));
+
+    /* Don't consider VERR_PDM_MODULE_NAME_CLASH and VERR_NO_MEMORY above as these are very unlikely. */
+    if (RT_FAILURE(rc) && pUVM->pVM) /** @todo VMR3SetErrorU. */
+        rc = VMSetError(pUVM->pVM, rc, RT_SRC_POS, N_("Cannot load R0 module %s: %s"), pszFilename, ErrInfo.Core.pszMsg);
+
+    RTMemTmpFree(pszFile); /* might be reference thru pszFilename in the above VMSetError call. */
+    return rc;
+}
+
+
+
+/**
+ * Get the address of a symbol in a given HC ring 3 module.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pszModule       Module name.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   ppvValue        Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolR3(PVM pVM, const char *pszModule, const char *pszSymbol, void **ppvValue)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+
+    /*
+     * Find the module.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    {
+        if (    pModule->eType == PDMMOD_TYPE_R3
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            RTUINTPTR Value = 0;
+            int rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pszSymbol, &Value);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                *ppvValue = (void *)(uintptr_t)Value;
+                Assert((uintptr_t)*ppvValue == Value);
+            }
+            else
+            {
+                if ((uintptr_t)pszSymbol < 0x10000)
+                    AssertMsg(rc, ("Couldn't symbol '%u' in module '%s'\n", (unsigned)(uintptr_t)pszSymbol, pszModule));
+                else
+                    AssertMsg(rc, ("Couldn't symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+            }
+            return rc;
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
+    return VERR_SYMBOL_NOT_FOUND;
+}
+
+
+/**
+ * Get the address of a symbol in a given HC ring 0 module.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pszModule       Module name. If NULL the main R0 module (VMMR0.r0) is assumes.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   ppvValue        Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolR0(PVM pVM, const char *pszModule, const char *pszSymbol, PRTR0PTR ppvValue)
+{
+#ifdef PDMLDR_FAKE_MODE
+    *ppvValue = 0xdeadbeef;
+    return VINF_SUCCESS;
+
+#else
+    /*
+     * Validate input.
+     */
+    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+    if (!pszModule)
+        pszModule = "VMMR0.r0";
+
+    /*
+     * Find the module.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    {
+        if (    pModule->eType == PDMMOD_TYPE_R0
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            int rc = SUPR3GetSymbolR0((void *)(uintptr_t)pModule->ImageBase, pszSymbol, (void **)ppvValue);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgRC(rc, ("Couldn't find symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+                LogRel(("PDMGetSymbol: Couldn't find symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+            }
+            return rc;
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
+    return VERR_SYMBOL_NOT_FOUND;
+#endif
+}
+
+
+/**
+ * Same as PDMR3LdrGetSymbolR0 except that the module will be attempted loaded if not found.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pszModule       Module name. If NULL the main R0 module (VMMR0.r0) is assumed.
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   ppvValue        Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolR0Lazy(PVM pVM, const char *pszModule, const char *pszSearchPath, const char *pszSymbol,
+                                       PRTR0PTR ppvValue)
+{
+#ifdef PDMLDR_FAKE_MODE
+    *ppvValue = 0xdeadbeef;
+    return VINF_SUCCESS;
+
+#else
+    /*
+     * Since we're lazy, we'll only check if the module is present
+     * and hand it over to PDMR3LdrGetSymbolR0 when that's done.
+     */
+    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+    if (pszModule)
+    {
+        AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
+        PUVM    pUVM = pVM->pUVM;
+        PPDMMOD pModule;
+        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+        for (pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+            if (    pModule->eType == PDMMOD_TYPE_R0
+                &&  !strcmp(pModule->szName, pszModule))
+                break;
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        if (!pModule)
+        {
+            int rc = pdmR3LoadR0U(pUVM, NULL, pszModule, pszSearchPath);
+            AssertMsgRCReturn(rc, ("pszModule=%s rc=%Rrc\n", pszModule, rc), VERR_MODULE_NOT_FOUND);
+        }
+    }
+    return PDMR3LdrGetSymbolR0(pVM, pszModule, pszSymbol, ppvValue);
+#endif
+}
+
+
+/**
+ * Get the address of a symbol in a given RC module.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pszModule       Module name. If NULL the main R0 module (VMMGC.gc) is assumes.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   pRCPtrValue     Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolRC(PVM pVM, const char *pszModule, const char *pszSymbol, PRTRCPTR pRCPtrValue)
+{
+#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE)
+    *pRCPtrValue = 0xfeedf00d;
+    return VINF_SUCCESS;
+
+#else
+    /*
+     * Validate input.
+     */
+    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+    if (!pszModule)
+        pszModule = "VMMGC.gc";
+
+    /*
+     * Find the module.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+    {
+        if (    pModule->eType == PDMMOD_TYPE_RC
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            RTUINTPTR Value;
+            int rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, pszSymbol, &Value);
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                *pRCPtrValue = (RTGCPTR)Value;
+                Assert(*pRCPtrValue == Value);
+            }
+            else
+            {
+                if ((uintptr_t)pszSymbol < 0x10000)
+                    AssertMsg(rc, ("Couldn't symbol '%u' in module '%s'\n", (unsigned)(uintptr_t)pszSymbol, pszModule));
+                else
+                    AssertMsg(rc, ("Couldn't symbol '%s' in module '%s'\n", pszSymbol, pszModule));
+            }
+            return rc;
+        }
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    AssertMsgFailed(("Couldn't locate module '%s'\n", pszModule));
+    return VERR_SYMBOL_NOT_FOUND;
+#endif
+}
+
+
+/**
+ * Same as PDMR3LdrGetSymbolRC except that the module will be attempted loaded if not found.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pszModule       Module name. If NULL the main R0 module (VMMGC.gc) is assumes.
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ * @param   pszSymbol       Symbol name. If it's value is less than 64k it's treated like a
+ *                          ordinal value rather than a string pointer.
+ * @param   pRCPtrValue     Where to store the symbol value.
+ */
+VMMR3DECL(int) PDMR3LdrGetSymbolRCLazy(PVM pVM, const char *pszModule, const char *pszSearchPath, const char *pszSymbol,
+                                       PRTRCPTR pRCPtrValue)
+{
+#if defined(PDMLDR_FAKE_MODE) || !defined(VBOX_WITH_RAW_MODE)
+    *pRCPtrValue = 0xfeedf00d;
+    return VINF_SUCCESS;
+
+#else
+    /*
+     * Since we're lazy, we'll only check if the module is present
+     * and hand it over to PDMR3LdrGetSymbolRC when that's done.
+     */
+    AssertMsg(pVM->pdm.s.offVM, ("bad init order!\n"));
+    if (pszModule)
+    {
+        AssertMsgReturn(!strpbrk(pszModule, "/\\:\n\r\t"), ("pszModule=%s\n", pszModule), VERR_INVALID_PARAMETER);
+        PUVM    pUVM = pVM->pUVM;
+        PPDMMOD pModule;
+        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+        for (pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+            if (    pModule->eType == PDMMOD_TYPE_RC
+                &&  !strcmp(pModule->szName, pszModule))
+                break;
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+        if (!pModule)
+        {
+            char *pszFilename = pdmR3FileRC(pszModule, pszSearchPath);
+            AssertMsgReturn(pszFilename, ("pszModule=%s\n", pszModule), VERR_MODULE_NOT_FOUND);
+            int rc = PDMR3LdrLoadRC(pVM, pszFilename, pszModule);
+            RTMemTmpFree(pszFilename);
+            AssertMsgRCReturn(rc, ("pszModule=%s rc=%Rrc\n", pszModule, rc), VERR_MODULE_NOT_FOUND);
+        }
+    }
+    return PDMR3LdrGetSymbolRC(pVM, pszModule, pszSymbol, pRCPtrValue);
+#endif
+}
+
+
+/**
+ * Constructs the full filename for a R3 image file.
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszFile     File name (no path).
+ */
+char *pdmR3FileR3(const char *pszFile, bool fShared)
+{
+    return pdmR3File(pszFile, NULL, NULL, fShared);
+}
+
+
+/**
+ * Constructs the full filename for a R0 image file.
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszFile         File name (no path).
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ */
+char *pdmR3FileR0(const char *pszFile, const char *pszSearchPath)
+{
+    return pdmR3File(pszFile, NULL, pszSearchPath, /*fShared=*/false);
+}
+
+
+/**
+ * Constructs the full filename for a RC image file.
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszFile         File name (no path).
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ */
+char *pdmR3FileRC(const char *pszFile, const char *pszSearchPath)
+{
+    return pdmR3File(pszFile, NULL, pszSearchPath, /*fShared=*/false);
+}
+
+
+/**
+ * Worker for pdmR3File().
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ *
+ * @param   pszDir        Directory part
+ * @param   pszFile       File name part
+ * @param   pszDefaultExt Extension part
+ */
+static char *pdmR3FileConstruct(const char *pszDir, const char *pszFile, const char *pszDefaultExt)
+{
+    /*
+     * Allocate temp memory for return buffer.
+     */
+    size_t cchDir  = strlen(pszDir);
+    size_t cchFile = strlen(pszFile);
+    size_t cchDefaultExt;
+
+    /*
+     * Default extention?
+     */
+    if (!pszDefaultExt || strchr(pszFile, '.'))
+        cchDefaultExt = 0;
+    else
+        cchDefaultExt = strlen(pszDefaultExt);
+
+    size_t cchPath = cchDir + 1 + cchFile + cchDefaultExt + 1;
+    AssertMsgReturn(cchPath <= RTPATH_MAX, ("Path too long!\n"), NULL);
+
+    char *pszRet = (char *)RTMemTmpAlloc(cchDir + 1 + cchFile + cchDefaultExt + 1);
+    AssertMsgReturn(pszRet, ("Out of temporary memory!\n"), NULL);
+
+    /*
+     * Construct the filename.
+     */
+    memcpy(pszRet, pszDir, cchDir);
+    pszRet[cchDir++] = '/';            /* this works everywhere */
+    memcpy(pszRet + cchDir, pszFile, cchFile + 1);
+    if (cchDefaultExt)
+        memcpy(pszRet + cchDir + cchFile, pszDefaultExt, cchDefaultExt + 1);
+
+    return pszRet;
+}
+
+
+/**
+ * Worker for pdmR3FileRC(), pdmR3FileR0() and pdmR3FileR3().
+ *
+ * @returns Pointer to temporary memory containing the filename.
+ *          Caller must free this using RTMemTmpFree().
+ * @returns NULL on failure.
+ * @param   pszFile         File name (no path).
+ * @param   pszDefaultExt   The default extention, NULL if none.
+ * @param   pszSearchPath   List of directories to search if @a pszFile is
+ *                          not qualified with a path.  Can be NULL, in which
+ *                          case the arch dependent install dir is searched.
+ * @param   fShared         If true, search in the shared directory (/usr/lib on Unix), else
+ *                          search in the private directory (/usr/lib/virtualbox on Unix).
+ *                          Ignored if VBOX_PATH_SHARED_LIBS is not defined.
+ * @todo    We'll have this elsewhere than in the root later!
+ * @todo    Remove the fShared hack again once we don't need to link against VBoxDD anymore!
+ */
+static char *pdmR3File(const char *pszFile, const char *pszDefaultExt, const char *pszSearchPath, bool fShared)
+{
+    char szPath[RTPATH_MAX];
+    int  rc;
+
+    AssertLogRelReturn(!fShared || !pszSearchPath, NULL);
+    Assert(!RTPathHavePath(pszFile));
+
+    /*
+     * If there is a path, search it.
+     */
+    if (   pszSearchPath
+        && *pszSearchPath)
+    {
+        /* Check the filename length. */
+        size_t const    cchFile = strlen(pszFile);
+        if (cchFile >= sizeof(szPath))
+            return NULL;
+
+        /*
+         * Walk the search path.
+         */
+        const char *psz = pszSearchPath;
+        while (*psz)
+        {
+            /* Skip leading blanks - no directories with leading spaces, thank you. */
+            while (RT_C_IS_BLANK(*psz))
+                psz++;
+
+            /* Find the end of this element. */
+            const char *pszNext;
+            const char *pszEnd = strchr(psz, ';');
+            if (!pszEnd)
+                pszEnd = pszNext = strchr(psz, '\0');
+            else
+                pszNext = pszEnd + 1;
+            if (pszEnd != psz)
+            {
+                rc = RTPathJoinEx(szPath, sizeof(szPath), psz, pszEnd - psz, pszFile, cchFile);
+                if (RT_SUCCESS(rc))
+                {
+                    if (RTFileExists(szPath))
+                    {
+                        size_t cchPath = strlen(szPath) + 1;
+                        char *pszRet = (char *)RTMemTmpAlloc(cchPath);
+                        if (pszRet)
+                            memcpy(pszRet, szPath, cchPath);
+                        return pszRet;
+                    }
+                }
+            }
+
+            /* advance */
+            psz = pszNext;
+        }
+    }
+
+    /*
+     * Use the default location.
+     */
+    rc = fShared
+       ? RTPathSharedLibs(    szPath, sizeof(szPath))
+       : RTPathAppPrivateArch(szPath, sizeof(szPath));
+    if (!RT_SUCCESS(rc))
+    {
+        AssertMsgFailed(("RTPath[SharedLibs|AppPrivateArch](,%d) failed rc=%d!\n", sizeof(szPath), rc));
+        return NULL;
+    }
+
+    return pdmR3FileConstruct(szPath, pszFile, pszDefaultExt);
+}
+
+
+/** @internal */
+typedef struct QMFEIPARG
+{
+    RTINTPTR    uPC;
+
+    char       *pszNearSym1;
+    size_t      cchNearSym1;
+    RTINTPTR    offNearSym1;
+
+    char       *pszNearSym2;
+    size_t      cchNearSym2;
+    RTINTPTR    offNearSym2;
+} QMFEIPARG, *PQMFEIPARG;
+
+
+/**
+ * Enumeration callback function used by RTLdrEnumSymbols().
+ *
+ * @returns VBox status code. Failure will stop the enumeration.
+ * @param   hLdrMod         The loader module handle.
+ * @param   pszSymbol       Symbol name. NULL if ordinal only.
+ * @param   uSymbol         Symbol ordinal, ~0 if not used.
+ * @param   Value           Symbol value.
+ * @param   pvUser          The user argument specified to RTLdrEnumSymbols().
+ */
+static DECLCALLBACK(int) pdmR3QueryModFromEIPEnumSymbols(RTLDRMOD hLdrMod, const char *pszSymbol, unsigned uSymbol, RTUINTPTR Value, void *pvUser)
+{
+    PQMFEIPARG pArgs = (PQMFEIPARG)pvUser;
+
+    RTINTPTR off = Value - pArgs->uPC;
+    if (off <= 0)   /* near1 is before or at same location. */
+    {
+        if (off > pArgs->offNearSym1)
+        {
+            pArgs->offNearSym1 = off;
+            if (pArgs->pszNearSym1 && pArgs->cchNearSym1)
+            {
+                *pArgs->pszNearSym1 = '\0';
+                if (pszSymbol)
+                    strncat(pArgs->pszNearSym1, pszSymbol, pArgs->cchNearSym1);
+                else
+                {
+                    char szOrd[32];
+                    RTStrPrintf(szOrd, sizeof(szOrd), "#%#x", uSymbol);
+                    strncat(pArgs->pszNearSym1, szOrd, pArgs->cchNearSym1);
+                }
+            }
+        }
+    }
+    else            /* near2 is after */
+    {
+        if (off < pArgs->offNearSym2)
+        {
+            pArgs->offNearSym2 = off;
+            if (pArgs->pszNearSym2 && pArgs->cchNearSym2)
+            {
+                *pArgs->pszNearSym2 = '\0';
+                if (pszSymbol)
+                    strncat(pArgs->pszNearSym2, pszSymbol, pArgs->cchNearSym2);
+                else
+                {
+                    char szOrd[32];
+                    RTStrPrintf(szOrd, sizeof(szOrd), "#%#x", uSymbol);
+                    strncat(pArgs->pszNearSym2, szOrd, pArgs->cchNearSym2);
+                }
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal worker for PDMR3LdrQueryRCModFromPC and PDMR3LdrQueryR0ModFromPC.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         VM handle
+ * @param   uPC         The program counter (eip/rip) to locate the module for.
+ * @param   enmType     The module type.
+ * @param   pszModName  Where to store the module name.
+ * @param   cchModName  Size of the module name buffer.
+ * @param   pMod        Base address of the module.
+ * @param   pszNearSym1 Name of the closes symbol from below.
+ * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
+ * @param   pNearSym1   The address of pszNearSym1.
+ * @param   pszNearSym2 Name of the closes symbol from below.
+ * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2.
+ * @param   pNearSym2   The address of pszNearSym2.
+ */
+static int pdmR3LdrQueryModFromPC(PVM pVM, RTUINTPTR uPC, PDMMODTYPE enmType,
+                                  char *pszModName,  size_t cchModName,  PRTUINTPTR pMod,
+                                  char *pszNearSym1, size_t cchNearSym1, PRTUINTPTR pNearSym1,
+                                  char *pszNearSym2, size_t cchNearSym2, PRTUINTPTR pNearSym2)
+{
+    PUVM    pUVM = pVM->pUVM;
+    int     rc   = VERR_MODULE_NOT_FOUND;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pCur= pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    {
+        if (pCur->eType != enmType)
+            continue;
+
+        /* The following RTLdrOpen call is a dirty hack to get ring-0 module information. */
+        RTLDRMOD hLdrMod = pCur->hLdrMod;
+        if (hLdrMod == NIL_RTLDRMOD && uPC >= pCur->ImageBase)
+        {
+            int rc2 = RTLdrOpen(pCur->szFilename, 0 /*fFlags*/, RTLDRARCH_HOST, &hLdrMod);
+            if (RT_FAILURE(rc2))
+                hLdrMod = NIL_RTLDRMOD;
+        }
+
+        if (   hLdrMod != NIL_RTLDRMOD
+            && uPC - pCur->ImageBase < RTLdrSize(hLdrMod))
+        {
+            if (pMod)
+                *pMod = pCur->ImageBase;
+            if (pszModName && cchModName)
+            {
+                *pszModName = '\0';
+                strncat(pszModName, pCur->szName, cchModName);
+            }
+            if (pNearSym1)   *pNearSym1   = 0;
+            if (pNearSym2)   *pNearSym2   = 0;
+            if (pszNearSym1) *pszNearSym1 = '\0';
+            if (pszNearSym2) *pszNearSym2 = '\0';
+
+            /*
+             * Locate the nearest symbols.
+             */
+            QMFEIPARG   Args;
+            Args.uPC         = uPC;
+            Args.pszNearSym1 = pszNearSym1;
+            Args.cchNearSym1 = cchNearSym1;
+            Args.offNearSym1 = RTINTPTR_MIN;
+            Args.pszNearSym2 = pszNearSym2;
+            Args.cchNearSym2 = cchNearSym2;
+            Args.offNearSym2 = RTINTPTR_MAX;
+
+            rc = RTLdrEnumSymbols(hLdrMod, RTLDR_ENUM_SYMBOL_FLAGS_ALL, pCur->pvBits, pCur->ImageBase,
+                                  pdmR3QueryModFromEIPEnumSymbols, &Args);
+            if (pNearSym1 && Args.offNearSym1 != RTINTPTR_MIN)
+                *pNearSym1 = Args.offNearSym1 + uPC;
+            if (pNearSym2 && Args.offNearSym2 != RTINTPTR_MAX)
+                *pNearSym2 = Args.offNearSym2 + uPC;
+
+            rc = VINF_SUCCESS;
+        }
+
+        if (hLdrMod != pCur->hLdrMod && hLdrMod != NIL_RTLDRMOD)
+            RTLdrClose(hLdrMod);
+
+        if (RT_SUCCESS(rc))
+            break;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Queries raw-mode context module information from an PC (eip/rip).
+ *
+ * This is typically used to locate a crash address.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         VM handle
+ * @param   uPC         The program counter (eip/rip) to locate the module for.
+ * @param   pszModName  Where to store the module name.
+ * @param   cchModName  Size of the module name buffer.
+ * @param   pMod        Base address of the module.
+ * @param   pszNearSym1 Name of the closes symbol from below.
+ * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
+ * @param   pNearSym1   The address of pszNearSym1.
+ * @param   pszNearSym2 Name of the closes symbol from below.
+ * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2.
+ * @param   pNearSym2   The address of pszNearSym2.
+ */
+VMMR3DECL(int) PDMR3LdrQueryRCModFromPC(PVM pVM, RTRCPTR uPC,
+                                        char *pszModName,  size_t cchModName,  PRTRCPTR pMod,
+                                        char *pszNearSym1, size_t cchNearSym1, PRTRCPTR pNearSym1,
+                                        char *pszNearSym2, size_t cchNearSym2, PRTRCPTR pNearSym2)
+{
+    RTUINTPTR AddrMod   = 0;
+    RTUINTPTR AddrNear1 = 0;
+    RTUINTPTR AddrNear2 = 0;
+    int rc = pdmR3LdrQueryModFromPC(pVM, uPC, PDMMOD_TYPE_RC,
+                                    pszModName,  cchModName,  &AddrMod,
+                                    pszNearSym1, cchNearSym1, &AddrNear1,
+                                    pszNearSym2, cchNearSym2, &AddrNear2);
+    if (RT_SUCCESS(rc))
+    {
+        if (pMod)
+            *pMod      = (RTRCPTR)AddrMod;
+        if (pNearSym1)
+            *pNearSym1 = (RTRCPTR)AddrNear1;
+        if (pNearSym2)
+            *pNearSym2 = (RTRCPTR)AddrNear2;
+    }
+    return rc;
+}
+
+
+/**
+ * Queries ring-0 context module information from an PC (eip/rip).
+ *
+ * This is typically used to locate a crash address.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         VM handle
+ * @param   uPC         The program counter (eip/rip) to locate the module for.
+ * @param   pszModName  Where to store the module name.
+ * @param   cchModName  Size of the module name buffer.
+ * @param   pMod        Base address of the module.
+ * @param   pszNearSym1 Name of the closes symbol from below.
+ * @param   cchNearSym1 Size of the buffer pointed to by pszNearSym1.
+ * @param   pNearSym1   The address of pszNearSym1.
+ * @param   pszNearSym2 Name of the closes symbol from below.
+ * @param   cchNearSym2 Size of the buffer pointed to by pszNearSym2. Optional.
+ * @param   pNearSym2   The address of pszNearSym2. Optional.
+ */
+VMMR3DECL(int) PDMR3LdrQueryR0ModFromPC(PVM pVM, RTR0PTR uPC,
+                                        char *pszModName,  size_t cchModName,  PRTR0PTR pMod,
+                                        char *pszNearSym1, size_t cchNearSym1, PRTR0PTR pNearSym1,
+                                        char *pszNearSym2, size_t cchNearSym2, PRTR0PTR pNearSym2)
+{
+    RTUINTPTR AddrMod   = 0;
+    RTUINTPTR AddrNear1 = 0;
+    RTUINTPTR AddrNear2 = 0;
+    int rc = pdmR3LdrQueryModFromPC(pVM, uPC, PDMMOD_TYPE_R0,
+                                    pszModName,  cchModName,  &AddrMod,
+                                    pszNearSym1, cchNearSym1, &AddrNear1,
+                                    pszNearSym2, cchNearSym2, &AddrNear2);
+    if (RT_SUCCESS(rc))
+    {
+        if (pMod)
+            *pMod      = (RTR0PTR)AddrMod;
+        if (pNearSym1)
+            *pNearSym1 = (RTR0PTR)AddrNear1;
+        if (pNearSym2)
+            *pNearSym2 = (RTR0PTR)AddrNear2;
+    }
+    return rc;
+}
+
+
+/**
+ * Enumerate all PDM modules.
+ *
+ * @returns VBox status.
+ * @param   pVM             VM Handle.
+ * @param   pfnCallback     Function to call back for each of the modules.
+ * @param   pvArg           User argument.
+ */
+VMMR3DECL(int)  PDMR3LdrEnumModules(PVM pVM, PFNPDMR3ENUM pfnCallback, void *pvArg)
+{
+    PUVM pUVM = pVM->pUVM;
+    int  rc   = VINF_SUCCESS;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pCur = pUVM->pdm.s.pModules; pCur; pCur = pCur->pNext)
+    {
+        rc = pfnCallback(pVM,
+                         pCur->szFilename,
+                         pCur->szName,
+                         pCur->ImageBase,
+                         pCur->eType == PDMMOD_TYPE_RC ? RTLdrSize(pCur->hLdrMod) : 0,
+                         pCur->eType == PDMMOD_TYPE_RC,
+                         pvArg);
+        if (RT_FAILURE(rc))
+            break;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Locates a module.
+ *
+ * @returns Pointer to the module if found.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszModule       The module name.
+ * @param   enmType         The module type.
+ * @param   fLazy           Lazy loading the module if set.
+ * @param   pszSearchPath   Search path for use when lazy loading.
+ */
+static PPDMMOD pdmR3LdrFindModule(PUVM pUVM, const char *pszModule, PDMMODTYPE enmType,
+                                  bool fLazy, const char *pszSearchPath)
+{
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMMOD pModule = pUVM->pdm.s.pModules; pModule; pModule = pModule->pNext)
+        if (    pModule->eType == enmType
+            &&  !strcmp(pModule->szName, pszModule))
+        {
+            RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+            return pModule;
+        }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    if (fLazy)
+    {
+        switch (enmType)
+        {
+#ifdef VBOX_WITH_RAW_MODE
+            case PDMMOD_TYPE_RC:
+            {
+                char *pszFilename = pdmR3FileRC(pszModule, pszSearchPath);
+                if (pszFilename)
+                {
+                    int rc = PDMR3LdrLoadRC(pUVM->pVM, pszFilename, pszModule);
+                    RTMemTmpFree(pszFilename);
+                    if (RT_SUCCESS(rc))
+                        return pdmR3LdrFindModule(pUVM, pszModule, enmType, false, NULL);
+                }
+                break;
+            }
+#endif
+
+            case PDMMOD_TYPE_R0:
+            {
+                int rc = pdmR3LoadR0U(pUVM, NULL, pszModule, pszSearchPath);
+                if (RT_SUCCESS(rc))
+                    return pdmR3LdrFindModule(pUVM, pszModule, enmType, false, NULL);
+                break;
+            }
+
+            default:
+                AssertFailed();
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * Resolves a ring-0 or raw-mode context interface.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pvInterface     Pointer to the interface structure.  The symbol list
+ *                          describes the layout.
+ * @param   cbInterface     The size of the structure pvInterface is pointing
+ *                          to.  For bounds checking.
+ * @param   pszModule       The module name.  If NULL we assume it's the default
+ *                          R0 or RC module (@a fRing0OrRC).  We'll attempt to
+ *                          load the module if it isn't found in the module
+ *                          list.
+ * @param   pszSearchPath   The module search path.  If NULL, search the
+ *                          architecture dependent install directory.
+ * @param   pszSymPrefix    What to prefix the symbols in the list with.  The
+ *                          idea is that you define a list that goes with an
+ *                          interface (INTERFACE_SYM_LIST) and reuse it with
+ *                          each implementation.
+ * @param   pszSymList      The symbol list for the interface.  This is a
+ *                          semi-colon separated list of symbol base names.  As
+ *                          mentioned above, each is prefixed with @a
+ *                          pszSymPrefix before resolving.  There are a couple
+ *                          of special symbol names that will cause us to skip
+ *                          ahead a little bit:
+ *                              - U8:whatever,
+ *                              - U16:whatever,
+ *                              - U32:whatever,
+ *                              - U64:whatever,
+ *                              - RCPTR:whatever,
+ *                              - R3PTR:whatever,
+ *                              - R0PTR:whatever,
+ *                              - GCPHYS:whatever,
+ *                              - HCPHYS:whatever.
+ * @param   fRing0          Set if it's a ring-0 context interface, clear if
+ *                          it's raw-mode context interface.
+ */
+VMMR3DECL(int) PDMR3LdrGetInterfaceSymbols(PVM pVM, void *pvInterface, size_t cbInterface,
+                                           const char *pszModule, const char *pszSearchPath,
+                                           const char *pszSymPrefix, const char *pszSymList,
+                                           bool fRing0)
+{
+    /*
+     * Find the module.
+     */
+    int     rc      = VINF_SUCCESS;
+    PPDMMOD pModule = pdmR3LdrFindModule(pVM->pUVM,
+                                         pszModule ? pszModule : fRing0 ? "VMMR0.r0" : "VMMGC.gc",
+                                         fRing0 ? PDMMOD_TYPE_R0 : PDMMOD_TYPE_RC,
+                                         true /*fLazy*/, pszSearchPath);
+    if (pModule)
+    {
+        /* Prep the symbol name. */
+        char            szSymbol[256];
+        size_t const    cchSymPrefix = strlen(pszSymPrefix);
+        AssertReturn(cchSymPrefix + 5 < sizeof(szSymbol), VERR_SYMBOL_NOT_FOUND);
+        memcpy(szSymbol, pszSymPrefix, cchSymPrefix);
+
+        /*
+         * Iterate the symbol list.
+         */
+        uint32_t        offInterface = 0;
+        const char     *pszCur       = pszSymList;
+        while (pszCur)
+        {
+            /*
+             * Find the end of the current symbol name.
+             */
+            size_t      cchSym;
+            const char *pszNext = strchr(pszCur, ';');
+            if (pszNext)
+            {
+                cchSym = pszNext - pszCur;
+                pszNext++;
+            }
+            else
+                cchSym = strlen(pszCur);
+            AssertBreakStmt(cchSym > 0, rc = VERR_INVALID_PARAMETER);
+
+            /* Is it a skip instruction? */
+            const char *pszColon = (const char *)memchr(pszCur, ':', cchSym);
+            if (pszColon)
+            {
+                /*
+                 * String switch on the instruction and execute it, checking
+                 * that we didn't overshoot the interface structure.
+                 */
+#define IS_SKIP_INSTR(szInstr) \
+                (   cchSkip == sizeof(szInstr) - 1 \
+                 && !memcmp(pszCur, szInstr, sizeof(szInstr) - 1) )
+
+                size_t const cchSkip = pszColon - pszCur;
+                if (IS_SKIP_INSTR("U8"))
+                    offInterface += sizeof(uint8_t);
+                else if (IS_SKIP_INSTR("U16"))
+                    offInterface += sizeof(uint16_t);
+                else if (IS_SKIP_INSTR("U32"))
+                    offInterface += sizeof(uint32_t);
+                else if (IS_SKIP_INSTR("U64"))
+                    offInterface += sizeof(uint64_t);
+                else if (IS_SKIP_INSTR("RCPTR"))
+                    offInterface += sizeof(RTRCPTR);
+                else if (IS_SKIP_INSTR("R3PTR"))
+                    offInterface += sizeof(RTR3PTR);
+                else if (IS_SKIP_INSTR("R0PTR"))
+                    offInterface += sizeof(RTR0PTR);
+                else if (IS_SKIP_INSTR("HCPHYS"))
+                    offInterface += sizeof(RTHCPHYS);
+                else if (IS_SKIP_INSTR("GCPHYS"))
+                    offInterface += sizeof(RTGCPHYS);
+                else
+                    AssertMsgFailedBreakStmt(("Invalid skip instruction %.*s (prefix=%s)\n", cchSym, pszCur, pszSymPrefix),
+                                             rc = VERR_INVALID_PARAMETER);
+                AssertMsgBreakStmt(offInterface <= cbInterface,
+                                   ("off=%#x cb=%#x (sym=%.*s prefix=%s)\n", offInterface, cbInterface, cchSym, pszCur, pszSymPrefix),
+                                   rc = VERR_BUFFER_OVERFLOW);
+#undef IS_SKIP_INSTR
+            }
+            else
+            {
+                /*
+                 * Construct the symbol name, get its value, store it and
+                 * advance the interface cursor.
+                 */
+                AssertReturn(cchSymPrefix + cchSym < sizeof(szSymbol), VERR_SYMBOL_NOT_FOUND);
+                memcpy(&szSymbol[cchSymPrefix], pszCur, cchSym);
+                szSymbol[cchSymPrefix + cchSym] = '\0';
+
+                if (fRing0)
+                {
+                    void *pvValue;
+                    rc = SUPR3GetSymbolR0((void *)(RTR0PTR)pModule->ImageBase, szSymbol, &pvValue);
+                    AssertMsgRCBreak(rc, ("Couldn't find symbol '%s' in module '%s'\n", szSymbol, pModule->szName));
+
+                    PRTR0PTR pValue = (PRTR0PTR)((uintptr_t)pvInterface + offInterface);
+                    AssertMsgBreakStmt(offInterface + sizeof(*pValue) <= cbInterface,
+                                       ("off=%#x cb=%#x sym=%s\n", offInterface, cbInterface, szSymbol),
+                                       rc = VERR_BUFFER_OVERFLOW);
+                    *pValue = (RTR0PTR)pvValue;
+                    Assert((void *)*pValue == pvValue);
+                    offInterface += sizeof(*pValue);
+                }
+                else
+                {
+                    RTUINTPTR Value;
+                    rc = RTLdrGetSymbolEx(pModule->hLdrMod, pModule->pvBits, pModule->ImageBase, szSymbol, &Value);
+                    AssertMsgRCBreak(rc, ("Couldn't find symbol '%s' in module '%s'\n", szSymbol, pModule->szName));
+
+                    PRTRCPTR pValue = (PRTRCPTR)((uintptr_t)pvInterface + offInterface);
+                    AssertMsgBreakStmt(offInterface + sizeof(*pValue) <= cbInterface,
+                                       ("off=%#x cb=%#x sym=%s\n", offInterface, cbInterface, szSymbol),
+                                       rc = VERR_BUFFER_OVERFLOW);
+                    *pValue = (RTRCPTR)Value;
+                    Assert(*pValue == Value);
+                    offInterface += sizeof(*pValue);
+                }
+            }
+
+            /* advance */
+            pszCur = pszNext;
+        }
+
+    }
+    else
+        rc = VERR_MODULE_NOT_FOUND;
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMQueue.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMQueue.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMQueue.cpp	(revision 35333)
@@ -0,0 +1,906 @@
+/* $Id$ */
+/** @file
+ * PDM Queue - Transport data and tasks to EMT and R3.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_QUEUE
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/rem.h>
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+DECLINLINE(void)            pdmR3QueueFree(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem);
+static bool                 pdmR3QueueFlush(PPDMQUEUE pQueue);
+static DECLCALLBACK(void)   pdmR3QueueTimer(PVM pVM, PTMTIMER pTimer, void *pvUser);
+
+
+
+/**
+ * Internal worker for the queue creation apis.
+ *
+ * @returns VBox status.
+ * @param   pVM                 VM handle.
+ * @param   cbItem              Item size.
+ * @param   cItems              Number of items.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   fRZEnabled          Set if the queue will be used from RC/R0 and need to be allocated from the hyper heap.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle.
+ */
+static int pdmR3QueueCreate(PVM pVM, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval, bool fRZEnabled,
+                            const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Validate input.
+     */
+    if (cbItem < sizeof(PDMQUEUEITEMCORE))
+    {
+        AssertMsgFailed(("cbItem=%d\n", cbItem));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (cItems < 1 || cItems >= 0x10000)
+    {
+        AssertMsgFailed(("cItems=%d valid:[1-65535]\n", cItems));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Align the item size and calculate the structure size.
+     */
+    cbItem = RT_ALIGN(cbItem, sizeof(RTUINTPTR));
+    size_t cb = cbItem * cItems + RT_ALIGN_Z(RT_OFFSETOF(PDMQUEUE, aFreeItems[cItems + PDMQUEUE_FREE_SLACK]), 16);
+    PPDMQUEUE pQueue;
+    int rc;
+    if (fRZEnabled)
+        rc = MMHyperAlloc(pVM, cb, 0, MM_TAG_PDM_QUEUE, (void **)&pQueue );
+    else
+        rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_QUEUE, cb, (void **)&pQueue);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize the data fields.
+     */
+    pQueue->pVMR3 = pVM;
+    pQueue->pVMR0 = fRZEnabled ? pVM->pVMR0 : NIL_RTR0PTR;
+    pQueue->pVMRC = fRZEnabled ? pVM->pVMRC : NIL_RTRCPTR;
+    pQueue->pszName = pszName;
+    pQueue->cMilliesInterval = cMilliesInterval;
+    //pQueue->pTimer = NULL;
+    pQueue->cbItem = cbItem;
+    pQueue->cItems = cItems;
+    //pQueue->pPendingR3 = NULL;
+    //pQueue->pPendingR0 = NULL;
+    //pQueue->pPendingRC = NULL;
+    pQueue->iFreeHead = cItems;
+    //pQueue->iFreeTail = 0;
+    PPDMQUEUEITEMCORE pItem = (PPDMQUEUEITEMCORE)((char *)pQueue + RT_ALIGN_Z(RT_OFFSETOF(PDMQUEUE, aFreeItems[cItems + PDMQUEUE_FREE_SLACK]), 16));
+    for (unsigned i = 0; i < cItems; i++, pItem = (PPDMQUEUEITEMCORE)((char *)pItem + cbItem))
+    {
+        pQueue->aFreeItems[i].pItemR3 = pItem;
+        if (fRZEnabled)
+        {
+            pQueue->aFreeItems[i].pItemR0 = MMHyperR3ToR0(pVM, pItem);
+            pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pVM, pItem);
+        }
+    }
+
+    /*
+     * Create timer?
+     */
+    if (cMilliesInterval)
+    {
+        rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, pdmR3QueueTimer, pQueue, "Queue timer", &pQueue->pTimer);
+        if (RT_SUCCESS(rc))
+        {
+            rc = TMTimerSetMillies(pQueue->pTimer, cMilliesInterval);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("TMTimerSetMillies failed rc=%Rrc\n", rc));
+                int rc2 = TMR3TimerDestroy(pQueue->pTimer); AssertRC(rc2);
+            }
+        }
+        else
+            AssertMsgFailed(("TMR3TimerCreateInternal failed rc=%Rrc\n", rc));
+        if (RT_FAILURE(rc))
+        {
+            if (fRZEnabled)
+                MMHyperFree(pVM, pQueue);
+            else
+                MMR3HeapFree(pQueue);
+            return rc;
+        }
+
+        /*
+         * Insert into the queue list for timer driven queues.
+         */
+        pdmLock(pVM);
+        pQueue->pNext = pUVM->pdm.s.pQueuesTimer;
+        pUVM->pdm.s.pQueuesTimer = pQueue;
+        pdmUnlock(pVM);
+    }
+    else
+    {
+        /*
+         * Insert into the queue list for forced action driven queues.
+         * This is a FIFO, so insert at the end.
+         */
+        /** @todo we should add a priority to the queues so we don't have to rely on
+         * the initialization order to deal with problems like #1605 (pgm/pcnet deadlock
+         * caused by the critsect queue to be last in the chain).
+         * - Update, the critical sections are no longer using queues, so this isn't a real
+         *   problem any longer. The priority might be a nice feature for later though.
+         */
+        pdmLock(pVM);
+        if (!pUVM->pdm.s.pQueuesForced)
+            pUVM->pdm.s.pQueuesForced = pQueue;
+        else
+        {
+            PPDMQUEUE pPrev = pUVM->pdm.s.pQueuesForced;
+            while (pPrev->pNext)
+                pPrev = pPrev->pNext;
+            pPrev->pNext = pQueue;
+        }
+        pdmUnlock(pVM);
+    }
+
+    /*
+     * Register the statistics.
+     */
+    STAMR3RegisterF(pVM, &pQueue->cbItem,               STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,        "Item size.",                       "/PDM/Queue/%s/cbItem",         pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->cItems,               STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,        "Queue size.",                      "/PDM/Queue/%s/cItems",         pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatAllocFailures,    STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,   "PDMQueueAlloc failures.",          "/PDM/Queue/%s/AllocFailures",  pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatInsert,           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Calls to PDMQueueInsert.",         "/PDM/Queue/%s/Insert",         pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatFlush,            STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Calls to pdmR3QueueFlush.",        "/PDM/Queue/%s/Flush",          pQueue->pszName);
+    STAMR3RegisterF(pVM, &pQueue->StatFlushLeftovers,   STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES,   "Left over items after flush.",     "/PDM/Queue/%s/FlushLeftovers", pQueue->pszName);
+#ifdef VBOX_WITH_STATISTICS
+    STAMR3RegisterF(pVM, &pQueue->StatFlushPrf,         STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_CALLS,        "Profiling pdmR3QueueFlush.",       "/PDM/Queue/%s/FlushPrf",       pQueue->pszName);
+    STAMR3RegisterF(pVM, (void *)&pQueue->cStatPending, STAMTYPE_U32,     STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT,        "Pending items.",                   "/PDM/Queue/%s/Pending",        pQueue->pszName);
+#endif
+
+    *ppQueue = pQueue;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Create a queue with a device owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle.
+ * @param   pDevIns             Device instance.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   fRZEnabled          Set if the queue must be usable from RC/R0.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(int) PDMR3QueueCreateDevice(PVM pVM, PPDMDEVINS pDevIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
+                                      PFNPDMQUEUEDEV pfnCallback, bool fRZEnabled, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateDevice: pDevIns=%p cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p fRZEnabled=%RTbool pszName=%s\n",
+             pDevIns, cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName));
+
+    /*
+     * Validate input.
+     */
+    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
+    if (!pfnCallback)
+    {
+        AssertMsgFailed(("No consumer callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, fRZEnabled, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_DEV;
+        pQueue->u.Dev.pDevIns = pDevIns;
+        pQueue->u.Dev.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created device queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pDevIns=%p\n",
+             cbItem, cItems, cMilliesInterval, pfnCallback, pDevIns));
+    }
+    return rc;
+}
+
+
+/**
+ * Create a queue with a driver owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle.
+ * @param   pDrvIns             Driver instance.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(int) PDMR3QueueCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
+                                      PFNPDMQUEUEDRV pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateDriver: pDrvIns=%p cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%s\n",
+             pDrvIns, cbItem, cItems, cMilliesInterval, pfnCallback, pszName));
+
+    /*
+     * Validate input.
+     */
+    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
+    if (!pfnCallback)
+    {
+        AssertMsgFailed(("No consumer callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, false, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_DRV;
+        pQueue->u.Drv.pDrvIns = pDrvIns;
+        pQueue->u.Drv.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created driver queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pDrvIns=%p\n",
+             cbItem, cItems, cMilliesInterval, pfnCallback, pDrvIns));
+    }
+    return rc;
+}
+
+
+/**
+ * Create a queue with an internal owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   fRZEnabled          Set if the queue must be usable from RC/R0.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(int) PDMR3QueueCreateInternal(PVM pVM, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
+                                        PFNPDMQUEUEINT pfnCallback, bool fRZEnabled, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateInternal: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p fRZEnabled=%RTbool pszName=%s\n",
+             cbItem, cItems, cMilliesInterval, pfnCallback, fRZEnabled, pszName));
+
+    /*
+     * Validate input.
+     */
+    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
+    if (!pfnCallback)
+    {
+        AssertMsgFailed(("No consumer callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, fRZEnabled, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_INTERNAL;
+        pQueue->u.Int.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created internal queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p\n",
+             cbItem, cItems, cMilliesInterval, pfnCallback));
+    }
+    return rc;
+}
+
+
+/**
+ * Create a queue with an external owner.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle.
+ * @param   cbItem              Size a queue item.
+ * @param   cItems              Number of items in the queue.
+ * @param   cMilliesInterval    Number of milliseconds between polling the queue.
+ *                              If 0 then the emulation thread will be notified whenever an item arrives.
+ * @param   pfnCallback         The consumer function.
+ * @param   pvUser              The user argument to the consumer function.
+ * @param   pszName             The queue name. Unique. Not copied.
+ * @param   ppQueue             Where to store the queue handle on success.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(int) PDMR3QueueCreateExternal(PVM pVM, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval, PFNPDMQUEUEEXT pfnCallback, void *pvUser, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    LogFlow(("PDMR3QueueCreateExternal: cbItem=%d cItems=%d cMilliesInterval=%d pfnCallback=%p pszName=%s\n", cbItem, cItems, cMilliesInterval, pfnCallback, pszName));
+
+    /*
+     * Validate input.
+     */
+    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
+    if (!pfnCallback)
+    {
+        AssertMsgFailed(("No consumer callback!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Create the queue.
+     */
+    PPDMQUEUE pQueue;
+    int rc = pdmR3QueueCreate(pVM, cbItem, cItems, cMilliesInterval, false, pszName, &pQueue);
+    if (RT_SUCCESS(rc))
+    {
+        pQueue->enmType = PDMQUEUETYPE_EXTERNAL;
+        pQueue->u.Ext.pvUser = pvUser;
+        pQueue->u.Ext.pfnCallback = pfnCallback;
+
+        *ppQueue = pQueue;
+        Log(("PDM: Created external queue %p; cbItem=%d cItems=%d cMillies=%d pfnCallback=%p pvUser=%p\n",
+             cbItem, cItems, cMilliesInterval, pfnCallback, pvUser));
+    }
+    return rc;
+}
+
+
+/**
+ * Destroy a queue.
+ *
+ * @returns VBox status code.
+ * @param   pQueue      Queue to destroy.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(int) PDMR3QueueDestroy(PPDMQUEUE pQueue)
+{
+    LogFlow(("PDMR3QueueDestroy: pQueue=%p\n", pQueue));
+
+    /*
+     * Validate input.
+     */
+    if (!pQueue)
+        return VERR_INVALID_PARAMETER;
+    Assert(pQueue && pQueue->pVMR3);
+    PVM     pVM  = pQueue->pVMR3;
+    PUVM    pUVM = pVM->pUVM;
+
+    pdmLock(pVM);
+
+    /*
+     * Unlink it.
+     */
+    if (pQueue->pTimer)
+    {
+        if (pUVM->pdm.s.pQueuesTimer != pQueue)
+        {
+            PPDMQUEUE pCur = pUVM->pdm.s.pQueuesTimer;
+            while (pCur)
+            {
+                if (pCur->pNext == pQueue)
+                {
+                    pCur->pNext = pQueue->pNext;
+                    break;
+                }
+                pCur = pCur->pNext;
+            }
+            AssertMsg(pCur, ("Didn't find the queue!\n"));
+        }
+        else
+            pUVM->pdm.s.pQueuesTimer = pQueue->pNext;
+    }
+    else
+    {
+        if (pUVM->pdm.s.pQueuesForced != pQueue)
+        {
+            PPDMQUEUE pCur = pUVM->pdm.s.pQueuesForced;
+            while (pCur)
+            {
+                if (pCur->pNext == pQueue)
+                {
+                    pCur->pNext = pQueue->pNext;
+                    break;
+                }
+                pCur = pCur->pNext;
+            }
+            AssertMsg(pCur, ("Didn't find the queue!\n"));
+        }
+        else
+            pUVM->pdm.s.pQueuesForced = pQueue->pNext;
+    }
+    pQueue->pNext = NULL;
+    pQueue->pVMR3 = NULL;
+    pdmUnlock(pVM);
+
+    /*
+     * Deregister statistics.
+     */
+    STAMR3Deregister(pVM, &pQueue->cbItem);
+    STAMR3Deregister(pVM, &pQueue->cbItem);
+    STAMR3Deregister(pVM, &pQueue->StatAllocFailures);
+    STAMR3Deregister(pVM, &pQueue->StatInsert);
+    STAMR3Deregister(pVM, &pQueue->StatFlush);
+    STAMR3Deregister(pVM, &pQueue->StatFlushLeftovers);
+#ifdef VBOX_WITH_STATISTICS
+    STAMR3Deregister(pVM, &pQueue->StatFlushPrf);
+    STAMR3Deregister(pVM, (void *)&pQueue->cStatPending);
+#endif
+
+    /*
+     * Destroy the timer and free it.
+     */
+    if (pQueue->pTimer)
+    {
+        TMR3TimerDestroy(pQueue->pTimer);
+        pQueue->pTimer = NULL;
+    }
+    if (pQueue->pVMRC)
+    {
+        pQueue->pVMRC = NIL_RTRCPTR;
+        pQueue->pVMR0 = NIL_RTR0PTR;
+        MMHyperFree(pVM, pQueue);
+    }
+    else
+        MMR3HeapFree(pQueue);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy a all queues owned by the specified device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pDevIns     Device instance.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(int) PDMR3QueueDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("PDMR3QueueDestroyDevice: pDevIns=%p\n", pDevIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+        return VERR_INVALID_PARAMETER;
+
+    PUVM pUVM = pVM->pUVM;
+    pdmLock(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
+    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
+    do
+    {
+        while (pQueue)
+        {
+            if (    pQueue->enmType == PDMQUEUETYPE_DEV
+                &&  pQueue->u.Dev.pDevIns == pDevIns)
+            {
+                PPDMQUEUE pQueueDestroy = pQueue;
+                pQueue = pQueue->pNext;
+                int rc = PDMR3QueueDestroy(pQueueDestroy);
+                AssertRC(rc);
+            }
+            else
+                pQueue = pQueue->pNext;
+        }
+
+        /* next queue list */
+        pQueue = pQueueNext;
+        pQueueNext = NULL;
+    } while (pQueue);
+
+    pdmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy a all queues owned by the specified driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pDrvIns     Driver instance.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(int) PDMR3QueueDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("PDMR3QueueDestroyDriver: pDrvIns=%p\n", pDrvIns));
+
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+        return VERR_INVALID_PARAMETER;
+
+    PUVM pUVM = pVM->pUVM;
+    pdmLock(pVM);
+
+    /*
+     * Unlink it.
+     */
+    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
+    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
+    do
+    {
+        while (pQueue)
+        {
+            if (    pQueue->enmType == PDMQUEUETYPE_DRV
+                &&  pQueue->u.Drv.pDrvIns == pDrvIns)
+            {
+                PPDMQUEUE pQueueDestroy = pQueue;
+                pQueue = pQueue->pNext;
+                int rc = PDMR3QueueDestroy(pQueueDestroy);
+                AssertRC(rc);
+            }
+            else
+                pQueue = pQueue->pNext;
+        }
+
+        /* next queue list */
+        pQueue = pQueueNext;
+        pQueueNext = NULL;
+    } while (pQueue);
+
+    pdmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Relocate the queues.
+ *
+ * @param   pVM             The VM handle.
+ * @param   offDelta        The relocation delta.
+ */
+void pdmR3QueueRelocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    /*
+     * Process the queues.
+     */
+    PUVM      pUVM       = pVM->pUVM;
+    PPDMQUEUE pQueueNext = pUVM->pdm.s.pQueuesTimer;
+    PPDMQUEUE pQueue     = pUVM->pdm.s.pQueuesForced;
+    do
+    {
+        while (pQueue)
+        {
+            if (pQueue->pVMRC)
+            {
+                pQueue->pVMRC = pVM->pVMRC;
+
+                /* Pending RC items. */
+                if (pQueue->pPendingRC)
+                {
+                    pQueue->pPendingRC += offDelta;
+                    PPDMQUEUEITEMCORE pCur = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pVM, pQueue->pPendingRC);
+                    while (pCur->pNextRC)
+                    {
+                        pCur->pNextRC += offDelta;
+                        pCur = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pVM, pCur->pNextRC);
+                    }
+                }
+
+                /* The free items. */
+                uint32_t i = pQueue->iFreeTail;
+                while (i != pQueue->iFreeHead)
+                {
+                    pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pVM, pQueue->aFreeItems[i].pItemR3);
+                    i = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
+                }
+            }
+
+            /* next queue */
+            pQueue = pQueue->pNext;
+        }
+
+        /* next queue list */
+        pQueue = pQueueNext;
+        pQueueNext = NULL;
+    } while (pQueue);
+}
+
+
+/**
+ * Flush pending queues.
+ * This is a forced action callback.
+ *
+ * @param   pVM     VM handle.
+ * @thread  Emulation thread only.
+ */
+VMMR3DECL(void) PDMR3QueueFlushAll(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("PDMR3QueuesFlush:\n"));
+
+    /*
+     * Only let one EMT flushing queues at any one time to preserve the order
+     * and to avoid wasting time. The FF is always cleared here, because it's
+     * only used to get someones attention. Queue inserts occurring during the
+     * flush are caught using the pending bit.
+     *
+     * Note! We must check the force action and pending flags after clearing
+     *       the active bit!
+     */
+    VM_FF_CLEAR(pVM, VM_FF_PDM_QUEUES);
+    while (!ASMAtomicBitTestAndSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT))
+    {
+        ASMAtomicBitClear(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT);
+
+        for (PPDMQUEUE pCur = pVM->pUVM->pdm.s.pQueuesForced; pCur; pCur = pCur->pNext)
+            if (    pCur->pPendingR3
+                ||  pCur->pPendingR0
+                ||  pCur->pPendingRC)
+                pdmR3QueueFlush(pCur);
+
+        ASMAtomicBitClear(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT);
+
+        /* We're done if there were no inserts while we were busy. */
+        if (   !ASMBitTest(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT)
+            && !VM_FF_ISPENDING(pVM, VM_FF_PDM_QUEUES))
+            break;
+        VM_FF_CLEAR(pVM, VM_FF_PDM_QUEUES);
+    }
+}
+
+
+/**
+ * Process pending items in one queue.
+ *
+ * @returns Success indicator.
+ *          If false the item the consumer said "enough!".
+ * @param   pQueue  The queue.
+ */
+static bool pdmR3QueueFlush(PPDMQUEUE pQueue)
+{
+    STAM_PROFILE_START(&pQueue->StatFlushPrf,p);
+
+    /*
+     * Get the lists.
+     */
+    PPDMQUEUEITEMCORE pItems   = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
+    RTRCPTR           pItemsRC = ASMAtomicXchgRCPtr(&pQueue->pPendingRC, NIL_RTRCPTR);
+    RTR0PTR           pItemsR0 = ASMAtomicXchgR0Ptr(&pQueue->pPendingR0, NIL_RTR0PTR);
+
+    AssertMsgReturn(   pItemsR0
+                    || pItemsRC
+                    || pItems,
+                    ("Someone is racing us? This shouldn't happen!\n"),
+                    true);
+
+    /*
+     * Reverse the list (it's inserted in LIFO order to avoid semaphores, remember).
+     */
+    PPDMQUEUEITEMCORE pCur = pItems;
+    pItems = NULL;
+    while (pCur)
+    {
+        PPDMQUEUEITEMCORE pInsert = pCur;
+        pCur = pCur->pNextR3;
+        pInsert->pNextR3 = pItems;
+        pItems = pInsert;
+    }
+
+    /*
+     * Do the same for any pending RC items.
+     */
+    while (pItemsRC)
+    {
+        PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperRCToR3(pQueue->pVMR3, pItemsRC);
+        pItemsRC = pInsert->pNextRC;
+        pInsert->pNextRC = NIL_RTRCPTR;
+        pInsert->pNextR3 = pItems;
+        pItems = pInsert;
+    }
+
+    /*
+     * Do the same for any pending R0 items.
+     */
+    while (pItemsR0)
+    {
+        PPDMQUEUEITEMCORE pInsert = (PPDMQUEUEITEMCORE)MMHyperR0ToR3(pQueue->pVMR3, pItemsR0);
+        pItemsR0 = pInsert->pNextR0;
+        pInsert->pNextR0 = NIL_RTR0PTR;
+        pInsert->pNextR3 = pItems;
+        pItems = pInsert;
+    }
+
+    /*
+     * Feed the items to the consumer function.
+     */
+    Log2(("pdmR3QueueFlush: pQueue=%p enmType=%d pItems=%p\n", pQueue, pQueue->enmType, pItems));
+    switch (pQueue->enmType)
+    {
+        case PDMQUEUETYPE_DEV:
+            while (pItems)
+            {
+                if (!pQueue->u.Dev.pfnCallback(pQueue->u.Dev.pDevIns, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFree(pQueue, pCur);
+            }
+            break;
+
+        case PDMQUEUETYPE_DRV:
+            while (pItems)
+            {
+                if (!pQueue->u.Drv.pfnCallback(pQueue->u.Drv.pDrvIns, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFree(pQueue, pCur);
+            }
+            break;
+
+        case PDMQUEUETYPE_INTERNAL:
+            while (pItems)
+            {
+                if (!pQueue->u.Int.pfnCallback(pQueue->pVMR3, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFree(pQueue, pCur);
+            }
+            break;
+
+        case PDMQUEUETYPE_EXTERNAL:
+            while (pItems)
+            {
+                if (!pQueue->u.Ext.pfnCallback(pQueue->u.Ext.pvUser, pItems))
+                    break;
+                pCur = pItems;
+                pItems = pItems->pNextR3;
+                pdmR3QueueFree(pQueue, pCur);
+            }
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid queue type %d\n", pQueue->enmType));
+            break;
+    }
+
+    /*
+     * Success?
+     */
+    if (pItems)
+    {
+        /*
+         * Reverse the list.
+         */
+        pCur = pItems;
+        pItems = NULL;
+        while (pCur)
+        {
+            PPDMQUEUEITEMCORE pInsert = pCur;
+            pCur = pInsert->pNextR3;
+            pInsert->pNextR3 = pItems;
+            pItems = pInsert;
+        }
+
+        /*
+         * Insert the list at the tail of the pending list.
+         */
+        for (;;)
+        {
+            if (ASMAtomicCmpXchgPtr(&pQueue->pPendingR3, pItems, NULL))
+                break;
+            PPDMQUEUEITEMCORE pPending = ASMAtomicXchgPtrT(&pQueue->pPendingR3, NULL, PPDMQUEUEITEMCORE);
+            if (pPending)
+            {
+                pCur = pPending;
+                while (pCur->pNextR3)
+                    pCur = pCur->pNextR3;
+                pCur->pNextR3 = pItems;
+                pItems = pPending;
+            }
+        }
+
+        STAM_REL_COUNTER_INC(&pQueue->StatFlushLeftovers);
+        STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
+        return false;
+    }
+
+    STAM_PROFILE_STOP(&pQueue->StatFlushPrf,p);
+    return true;
+}
+
+
+/**
+ * Free an item.
+ *
+ * @param   pQueue  The queue.
+ * @param   pItem   The item.
+ */
+DECLINLINE(void) pdmR3QueueFree(PPDMQUEUE pQueue, PPDMQUEUEITEMCORE pItem)
+{
+    VM_ASSERT_EMT(pQueue->pVMR3);
+
+    int i = pQueue->iFreeHead;
+    int iNext = (i + 1) % (pQueue->cItems + PDMQUEUE_FREE_SLACK);
+
+    pQueue->aFreeItems[i].pItemR3 = pItem;
+    if (pQueue->pVMRC)
+    {
+        pQueue->aFreeItems[i].pItemRC = MMHyperR3ToRC(pQueue->pVMR3, pItem);
+        pQueue->aFreeItems[i].pItemR0 = MMHyperR3ToR0(pQueue->pVMR3, pItem);
+    }
+
+    if (!ASMAtomicCmpXchgU32(&pQueue->iFreeHead, iNext, i))
+        AssertMsgFailed(("huh? i=%d iNext=%d iFreeHead=%d iFreeTail=%d\n", i, iNext, pQueue->iFreeHead, pQueue->iFreeTail));
+    STAM_STATS({ ASMAtomicDecU32(&pQueue->cStatPending); });
+}
+
+
+/**
+ * Timer handler for PDM queues.
+ * This is called by for a single queue.
+ *
+ * @param   pVM     VM handle.
+ * @param   pTimer  Pointer to timer.
+ * @param   pvUser  Pointer to the queue.
+ */
+static DECLCALLBACK(void) pdmR3QueueTimer(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    PPDMQUEUE pQueue = (PPDMQUEUE)pvUser;
+    Assert(pTimer == pQueue->pTimer); NOREF(pTimer);
+
+    if (   pQueue->pPendingR3
+        || pQueue->pPendingR0
+        || pQueue->pPendingRC)
+        pdmR3QueueFlush(pQueue);
+    int rc = TMTimerSetMillies(pQueue->pTimer, pQueue->cMilliesInterval);
+    AssertRC(rc);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMThread.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMThread.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMThread.cpp	(revision 35333)
@@ -0,0 +1,1080 @@
+/* $Id$ */
+/** @file
+ * PDM Thread - VM Thread Management.
+ */
+
+/*
+ * Copyright (C) 2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+///@todo #define LOG_GROUP LOG_GROUP_PDM_THREAD
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/mm.h>
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/semaphore.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) pdmR3ThreadMain(RTTHREAD Thread, void *pvUser);
+
+
+/**
+ * Wrapper around ASMAtomicCmpXchgSize.
+ */
+DECLINLINE(bool) pdmR3AtomicCmpXchgState(PPDMTHREAD pThread, PDMTHREADSTATE enmNewState, PDMTHREADSTATE enmOldState)
+{
+    bool fRc;
+    ASMAtomicCmpXchgSize(&pThread->enmState, enmNewState, enmOldState, fRc);
+    return fRc;
+}
+
+
+/**
+ * Does the wakeup call.
+ *
+ * @returns VBox status code. Already asserted on failure.
+ * @param   pThread     The PDM thread.
+ */
+static DECLCALLBACK(int) pdmR3ThreadWakeUp(PPDMTHREAD pThread)
+{
+    RTSemEventMultiSignal(pThread->Internal.s.SleepEvent);
+
+    int rc;
+    switch (pThread->Internal.s.enmType)
+    {
+        case PDMTHREADTYPE_DEVICE:
+            rc = pThread->u.Dev.pfnWakeUp(pThread->u.Dev.pDevIns, pThread);
+            break;
+
+        case PDMTHREADTYPE_USB:
+            rc = pThread->u.Usb.pfnWakeUp(pThread->u.Usb.pUsbIns, pThread);
+            break;
+
+        case PDMTHREADTYPE_DRIVER:
+            rc = pThread->u.Drv.pfnWakeUp(pThread->u.Drv.pDrvIns, pThread);
+            break;
+
+        case PDMTHREADTYPE_INTERNAL:
+            rc = pThread->u.Int.pfnWakeUp(pThread->Internal.s.pVM, pThread);
+            break;
+
+        case PDMTHREADTYPE_EXTERNAL:
+            rc = pThread->u.Ext.pfnWakeUp(pThread);
+            break;
+
+        default:
+            AssertMsgFailed(("%d\n", pThread->Internal.s.enmType));
+            rc = VERR_INTERNAL_ERROR;
+            break;
+    }
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Allocates new thread instance.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   ppThread    Where to store the pointer to the instance.
+ */
+static int pdmR3ThreadNew(PVM pVM, PPPDMTHREAD ppThread)
+{
+    PPDMTHREAD pThread;
+    int rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_THREAD, sizeof(*pThread), (void **)&pThread);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    pThread->u32Version     = PDMTHREAD_VERSION;
+    pThread->enmState       = PDMTHREADSTATE_INITIALIZING;
+    pThread->Thread         = NIL_RTTHREAD;
+    pThread->Internal.s.pVM = pVM;
+
+    *ppThread = pThread;
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Initialize a new thread, this actually creates the thread.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   ppThread    Where the thread instance data handle is.
+ * @param   cbStack     The stack size, see RTThreadCreate().
+ * @param   enmType     The thread type, see RTThreadCreate().
+ * @param   pszName     The thread name, see RTThreadCreate().
+ */
+static int pdmR3ThreadInit(PVM pVM, PPPDMTHREAD ppThread, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    PPDMTHREAD  pThread = *ppThread;
+    PUVM        pUVM    = pVM->pUVM;
+
+    /*
+     * Initialize the remainder of the structure.
+     */
+    pThread->Internal.s.pVM = pVM;
+
+    int rc = RTSemEventMultiCreate(&pThread->Internal.s.BlockEvent);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTSemEventMultiCreate(&pThread->Internal.s.SleepEvent);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Create the thread and wait for it to initialize.
+             * The newly created thread will set the PDMTHREAD::Thread member.
+             */
+            RTTHREAD Thread;
+            rc = RTThreadCreate(&Thread, pdmR3ThreadMain, pThread, cbStack, enmType, RTTHREADFLAGS_WAITABLE, pszName);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTThreadUserWait(Thread, 60*1000);
+                if (    RT_SUCCESS(rc)
+                    &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                    rc = VERR_INTERNAL_ERROR;
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Insert it into the thread list.
+                     */
+                    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+                    pThread->Internal.s.pNext = NULL;
+                    if (pUVM->pdm.s.pThreadsTail)
+                        pUVM->pdm.s.pThreadsTail->Internal.s.pNext = pThread;
+                    else
+                        pUVM->pdm.s.pThreads = pThread;
+                    pUVM->pdm.s.pThreadsTail = pThread;
+                    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+                    rc = RTThreadUserReset(Thread);
+                    AssertRC(rc);
+                    return rc;
+                }
+
+                /* bailout */
+                RTThreadWait(Thread, 60*1000, NULL);
+            }
+            RTSemEventMultiDestroy(pThread->Internal.s.SleepEvent);
+            pThread->Internal.s.SleepEvent = NIL_RTSEMEVENTMULTI;
+        }
+        RTSemEventMultiDestroy(pThread->Internal.s.BlockEvent);
+        pThread->Internal.s.BlockEvent = NIL_RTSEMEVENTMULTI;
+    }
+    MMHyperFree(pVM, pThread);
+    *ppThread = NULL;
+
+    return rc;
+}
+
+
+/**
+ * Device Helper for creating a thread associated with a device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pDevIns     The device instance.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+int pdmR3ThreadCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
+                            PFNPDMTHREADWAKEUPDEV pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_DEVICE;
+        pThread->u.Dev.pDevIns = pDevIns;
+        pThread->u.Dev.pfnThread = pfnThread;
+        pThread->u.Dev.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * USB Device Helper for creating a thread associated with an USB device.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pUsbIns     The USB device instance.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+int pdmR3ThreadCreateUsb(PVM pVM, PPDMUSBINS pUsbIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADUSB pfnThread,
+                         PFNPDMTHREADWAKEUPUSB pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_USB;
+        pThread->u.Usb.pUsbIns = pUsbIns;
+        pThread->u.Usb.pfnThread = pfnThread;
+        pThread->u.Usb.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Driver Helper for creating a thread associated with a driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pDrvIns     The driver instance.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+int pdmR3ThreadCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
+                            PFNPDMTHREADWAKEUPDRV pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_DRIVER;
+        pThread->u.Drv.pDrvIns = pDrvIns;
+        pThread->u.Drv.pfnThread = pfnThread;
+        pThread->u.Drv.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Creates a PDM thread for internal use in the VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+VMMR3DECL(int) PDMR3ThreadCreate(PVM pVM, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADINT pfnThread,
+                                 PFNPDMTHREADWAKEUPINT pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_INTERNAL;
+        pThread->u.Int.pfnThread = pfnThread;
+        pThread->u.Int.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Creates a PDM thread for VM use by some external party.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   ppThread    Where to store the thread 'handle'.
+ * @param   pvUser      The user argument to the thread function.
+ * @param   pfnThread   The thread function.
+ * @param   pfnWakeUp   The wakup callback. This is called on the EMT thread when
+ *                      a state change is pending.
+ * @param   cbStack     See RTThreadCreate.
+ * @param   enmType     See RTThreadCreate.
+ * @param   pszName     See RTThreadCreate.
+ */
+VMMR3DECL(int) PDMR3ThreadCreateExternal(PVM pVM, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADEXT pfnThread,
+                                         PFNPDMTHREADWAKEUPEXT pfnWakeUp, size_t cbStack, RTTHREADTYPE enmType, const char *pszName)
+{
+    int rc = pdmR3ThreadNew(pVM, ppThread);
+    if (RT_SUCCESS(rc))
+    {
+        PPDMTHREAD pThread = *ppThread;
+        pThread->pvUser = pvUser;
+        pThread->Internal.s.enmType = PDMTHREADTYPE_EXTERNAL;
+        pThread->u.Ext.pfnThread = pfnThread;
+        pThread->u.Ext.pfnWakeUp = pfnWakeUp;
+        rc = pdmR3ThreadInit(pVM, ppThread, cbStack, enmType, pszName);
+    }
+    return rc;
+}
+
+
+/**
+ * Destroys a PDM thread.
+ *
+ * This will wakeup the thread, tell it to terminate, and wait for it terminate.
+ *
+ * @returns VBox status code.
+ *          This reflects the success off destroying the thread and not the exit code
+ *          of the thread as this is stored in *pRcThread.
+ * @param   pThread         The thread to destroy.
+ * @param   pRcThread       Where to store the thread exit code. Optional.
+ * @thread  The emulation thread (EMT).
+ */
+VMMR3DECL(int) PDMR3ThreadDestroy(PPDMTHREAD pThread, int *pRcThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread != RTThreadSelf());
+    AssertPtrNullReturn(pRcThread, VERR_INVALID_POINTER);
+    PVM pVM = pThread->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    PUVM pUVM = pVM->pUVM;
+
+    /*
+     * Advance the thread to the terminating state.
+     */
+    int rc = VINF_SUCCESS;
+    if (pThread->enmState <= PDMTHREADSTATE_TERMINATING)
+    {
+        for (;;)
+        {
+            PDMTHREADSTATE enmState = pThread->enmState;
+            switch (enmState)
+            {
+                case PDMTHREADSTATE_RUNNING:
+                    if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                        continue;
+                    rc = pdmR3ThreadWakeUp(pThread);
+                    break;
+
+                case PDMTHREADSTATE_SUSPENDED:
+                case PDMTHREADSTATE_SUSPENDING:
+                case PDMTHREADSTATE_RESUMING:
+                case PDMTHREADSTATE_INITIALIZING:
+                    if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                        continue;
+                    break;
+
+                case PDMTHREADSTATE_TERMINATING:
+                case PDMTHREADSTATE_TERMINATED:
+                    break;
+
+                default:
+                    AssertMsgFailed(("enmState=%d\n", enmState));
+                    rc = VERR_INTERNAL_ERROR;
+                    break;
+            }
+            break;
+        }
+    }
+    int rc2 = RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
+    AssertRC(rc2);
+
+    /*
+     * Wait for it to terminate and the do cleanups.
+     */
+    rc2 = RTThreadWait(pThread->Thread, RT_SUCCESS(rc) ? 60*1000 : 150, pRcThread);
+    if (RT_SUCCESS(rc2))
+    {
+        /* make it invalid. */
+        pThread->u32Version = 0xffffffff;
+        pThread->enmState = PDMTHREADSTATE_INVALID;
+        pThread->Thread = NIL_RTTHREAD;
+
+        /* unlink */
+        RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+        if (pUVM->pdm.s.pThreads == pThread)
+        {
+            pUVM->pdm.s.pThreads = pThread->Internal.s.pNext;
+            if (!pThread->Internal.s.pNext)
+                pUVM->pdm.s.pThreadsTail = NULL;
+        }
+        else
+        {
+            PPDMTHREAD pPrev = pUVM->pdm.s.pThreads;
+            while (pPrev && pPrev->Internal.s.pNext != pThread)
+                pPrev = pPrev->Internal.s.pNext;
+            Assert(pPrev);
+            if (pPrev)
+                pPrev->Internal.s.pNext = pThread->Internal.s.pNext;
+            if (!pThread->Internal.s.pNext)
+                pUVM->pdm.s.pThreadsTail = pPrev;
+        }
+        pThread->Internal.s.pNext = NULL;
+        RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+
+        /* free the resources */
+        RTSemEventMultiDestroy(pThread->Internal.s.BlockEvent);
+        pThread->Internal.s.BlockEvent = NIL_RTSEMEVENTMULTI;
+
+        RTSemEventMultiDestroy(pThread->Internal.s.SleepEvent);
+        pThread->Internal.s.SleepEvent = NIL_RTSEMEVENTMULTI;
+
+        MMR3HeapFree(pThread);
+    }
+    else if (RT_SUCCESS(rc))
+        rc = rc2;
+
+    return rc;
+}
+
+
+/**
+ * Destroys all threads associated with a device.
+ *
+ * This function is called by PDMDevice when a device is
+ * destroyed (not currently implemented).
+ *
+ * @returns VBox status code of the first failure.
+ * @param   pVM         The VM handle.
+ * @param   pDevIns     the device instance.
+ */
+int pdmR3ThreadDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    int     rc   = VINF_SUCCESS;
+    PUVM    pUVM = pVM->pUVM;
+
+    AssertPtr(pDevIns);
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DEVICE
+            &&  pThread->u.Dev.pDevIns == pDevIns)
+        {
+            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        pThread = pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Destroys all threads associated with an USB device.
+ *
+ * This function is called by PDMUsb when a device is destroyed.
+ *
+ * @returns VBox status code of the first failure.
+ * @param   pVM         The VM handle.
+ * @param   pUsbIns     The USB device instance.
+ */
+int pdmR3ThreadDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    int     rc   = VINF_SUCCESS;
+    PUVM    pUVM = pVM->pUVM;
+
+    AssertPtr(pUsbIns);
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DEVICE
+            &&  pThread->u.Usb.pUsbIns == pUsbIns)
+        {
+            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        pThread = pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Destroys all threads associated with a driver.
+ *
+ * This function is called by PDMDriver when a driver is destroyed.
+ *
+ * @returns VBox status code of the first failure.
+ * @param   pVM         The VM handle.
+ * @param   pDrvIns     The driver instance.
+ */
+int pdmR3ThreadDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    int     rc   = VINF_SUCCESS;
+    PUVM    pUVM = pVM->pUVM;
+
+    AssertPtr(pDrvIns);
+
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        if (    pThread->Internal.s.enmType == PDMTHREADTYPE_DRIVER
+            &&  pThread->u.Drv.pDrvIns == pDrvIns)
+        {
+            int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+        }
+        pThread = pNext;
+    }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return rc;
+}
+
+
+/**
+ * Called For VM power off.
+ *
+ * @param   pVM         The VM handle.
+ */
+void pdmR3ThreadDestroyAll(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    PPDMTHREAD pThread = pUVM->pdm.s.pThreads;
+    while (pThread)
+    {
+        PPDMTHREAD pNext = pThread->Internal.s.pNext;
+        int rc2 = PDMR3ThreadDestroy(pThread, NULL);
+        AssertRC(rc2);
+        pThread = pNext;
+    }
+    Assert(!pUVM->pdm.s.pThreads && !pUVM->pdm.s.pThreadsTail);
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+}
+
+
+/**
+ * Initiate termination of the thread (self) because something failed in a bad way.
+ *
+ * @param   pThread         The PDM thread.
+ */
+static void pdmR3ThreadBailMeOut(PPDMTHREAD pThread)
+{
+    for (;;)
+    {
+        PDMTHREADSTATE enmState = pThread->enmState;
+        switch (enmState)
+        {
+            case PDMTHREADSTATE_SUSPENDING:
+            case PDMTHREADSTATE_SUSPENDED:
+            case PDMTHREADSTATE_RESUMING:
+            case PDMTHREADSTATE_RUNNING:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                break;
+
+            case PDMTHREADSTATE_TERMINATING:
+            case PDMTHREADSTATE_TERMINATED:
+                break;
+
+            case PDMTHREADSTATE_INITIALIZING:
+            default:
+                AssertMsgFailed(("enmState=%d\n", enmState));
+                break;
+        }
+        break;
+    }
+}
+
+
+/**
+ * Called by the PDM thread in response to a wakeup call with
+ * suspending as the new state.
+ *
+ * The thread will block in side this call until the state is changed in
+ * response to a VM state change or to the device/driver/whatever calling the
+ * PDMR3ThreadResume API.
+ *
+ * @returns VBox status code.
+ *          On failure, terminate the thread.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadIAmSuspending(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtr(pThread);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread == RTThreadSelf() || pThread->enmState == PDMTHREADSTATE_INITIALIZING);
+    PDMTHREADSTATE enmState = pThread->enmState;
+    Assert(     enmState == PDMTHREADSTATE_SUSPENDING
+           ||   enmState == PDMTHREADSTATE_INITIALIZING);
+
+    /*
+     * Update the state, notify the control thread (the API caller) and go to sleep.
+     */
+    int rc = VERR_WRONG_ORDER;
+    if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_SUSPENDED, enmState))
+    {
+        rc = RTThreadUserSignal(pThread->Thread);
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTSemEventMultiWait(pThread->Internal.s.BlockEvent, RT_INDEFINITE_WAIT);
+            if (    RT_SUCCESS(rc)
+                &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                return rc;
+
+            if (RT_SUCCESS(rc))
+                rc = VERR_INTERNAL_ERROR;
+        }
+    }
+
+    AssertMsgFailed(("rc=%d enmState=%d\n", rc, pThread->enmState));
+    pdmR3ThreadBailMeOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Called by the PDM thread in response to a resuming state.
+ *
+ * The purpose of this API is to tell the PDMR3ThreadResume caller that
+ * the PDM thread has successfully resumed. It will also do the
+ * state transition from the resuming to the running state.
+ *
+ * @returns VBox status code.
+ *          On failure, terminate the thread.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadIAmRunning(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    Assert(pThread->enmState == PDMTHREADSTATE_RESUMING);
+    Assert(pThread->Thread == RTThreadSelf());
+
+    /*
+     * Update the state and tell the control thread (the guy calling the resume API).
+     */
+    int rc = VERR_WRONG_ORDER;
+    if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_RUNNING, PDMTHREADSTATE_RESUMING))
+    {
+        rc = RTThreadUserSignal(pThread->Thread);
+        if (RT_SUCCESS(rc))
+            return rc;
+    }
+
+    AssertMsgFailed(("rc=%d enmState=%d\n", rc, pThread->enmState));
+    pdmR3ThreadBailMeOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Called by the PDM thread instead of RTThreadSleep.
+ *
+ * The difference is that the sleep will be interrupted on state change. The
+ * thread must be in the running state, otherwise it will return immediately.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success or state change.
+ * @retval  VERR_INTERRUPTED on signal or APC.
+ *
+ * @param   pThread     The PDM thread.
+ * @param   cMillies    The number of milliseconds to sleep.
+ */
+VMMR3DECL(int) PDMR3ThreadSleep(PPDMTHREAD pThread, RTMSINTERVAL cMillies)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertReturn(pThread->enmState > PDMTHREADSTATE_INVALID && pThread->enmState < PDMTHREADSTATE_TERMINATED, VERR_INTERNAL_ERROR);
+    AssertReturn(pThread->Thread == RTThreadSelf(), VERR_INTERNAL_ERROR);
+
+    /*
+     * Reset the event semaphore, check the state and sleep.
+     */
+    RTSemEventMultiReset(pThread->Internal.s.SleepEvent);
+    if (pThread->enmState != PDMTHREADSTATE_RUNNING)
+        return VINF_SUCCESS;
+    return RTSemEventMultiWaitNoResume(pThread->Internal.s.SleepEvent, cMillies);
+}
+
+
+/**
+ * The PDM thread function.
+ *
+ * @returns return from pfnThread.
+ *
+ * @param   Thread  The thread handle.
+ * @param   pvUser  Pointer to the PDMTHREAD structure.
+ */
+static DECLCALLBACK(int) pdmR3ThreadMain(RTTHREAD Thread, void *pvUser)
+{
+    PPDMTHREAD pThread = (PPDMTHREAD)pvUser;
+    Log(("PDMThread: Initializing thread %RTthrd / %p / '%s'...\n", Thread, pThread, RTThreadGetName(Thread)));
+    pThread->Thread = Thread;
+
+    /*
+     * The run loop.
+     *
+     * It handles simple thread functions which returns when they see a suspending
+     * request and leaves the PDMR3ThreadIAmSuspending and PDMR3ThreadIAmRunning
+     * parts to us.
+     */
+    int rc;
+    for (;;)
+    {
+        switch (pThread->Internal.s.enmType)
+        {
+            case PDMTHREADTYPE_DEVICE:
+                rc = pThread->u.Dev.pfnThread(pThread->u.Dev.pDevIns, pThread);
+                break;
+
+            case PDMTHREADTYPE_USB:
+                rc = pThread->u.Usb.pfnThread(pThread->u.Usb.pUsbIns, pThread);
+                break;
+
+            case PDMTHREADTYPE_DRIVER:
+                rc = pThread->u.Drv.pfnThread(pThread->u.Drv.pDrvIns, pThread);
+                break;
+
+            case PDMTHREADTYPE_INTERNAL:
+                rc = pThread->u.Int.pfnThread(pThread->Internal.s.pVM, pThread);
+                break;
+
+            case PDMTHREADTYPE_EXTERNAL:
+                rc = pThread->u.Ext.pfnThread(pThread);
+                break;
+
+            default:
+                AssertMsgFailed(("%d\n", pThread->Internal.s.enmType));
+                rc = VERR_INTERNAL_ERROR;
+                break;
+        }
+        if (RT_FAILURE(rc))
+            break;
+
+        /*
+         * If this is a simple thread function, the state will be suspending
+         * or initializing now. If it isn't we're supposed to terminate.
+         */
+        if (    pThread->enmState != PDMTHREADSTATE_SUSPENDING
+            &&  pThread->enmState != PDMTHREADSTATE_INITIALIZING)
+        {
+            Assert(pThread->enmState == PDMTHREADSTATE_TERMINATING);
+            break;
+        }
+        rc = PDMR3ThreadIAmSuspending(pThread);
+        if (RT_FAILURE(rc))
+            break;
+        if (pThread->enmState != PDMTHREADSTATE_RESUMING)
+        {
+            Assert(pThread->enmState == PDMTHREADSTATE_TERMINATING);
+            break;
+        }
+
+        rc = PDMR3ThreadIAmRunning(pThread);
+        if (RT_FAILURE(rc))
+            break;
+    }
+
+    if (RT_FAILURE(rc))
+        LogRel(("PDMThread: Thread '%s' (%RTthrd) quit unexpectedly with rc=%Rrc.\n", RTThreadGetName(Thread), Thread, rc));
+
+    /*
+     * Advance the state to terminating and then on to terminated.
+     */
+    for (;;)
+    {
+        PDMTHREADSTATE enmState = pThread->enmState;
+        if (    enmState == PDMTHREADSTATE_TERMINATING
+            ||  pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+            break;
+    }
+
+    ASMAtomicXchgSize(&pThread->enmState, PDMTHREADSTATE_TERMINATED);
+    int rc2 = RTThreadUserSignal(Thread); AssertRC(rc2);
+
+    Log(("PDMThread: Terminating thread %RTthrd / %p / '%s': %Rrc\n", Thread, pThread, RTThreadGetName(Thread), rc));
+    return rc;
+}
+
+
+/**
+ * Initiate termination of the thread because something failed in a bad way.
+ *
+ * @param   pThread         The PDM thread.
+ */
+static void pdmR3ThreadBailOut(PPDMTHREAD pThread)
+{
+    for (;;)
+    {
+        PDMTHREADSTATE enmState = pThread->enmState;
+        switch (enmState)
+        {
+            case PDMTHREADSTATE_SUSPENDING:
+            case PDMTHREADSTATE_SUSPENDED:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
+                break;
+
+            case PDMTHREADSTATE_RESUMING:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                break;
+
+            case PDMTHREADSTATE_RUNNING:
+                if (!pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_TERMINATING, enmState))
+                    continue;
+                pdmR3ThreadWakeUp(pThread);
+                break;
+
+            case PDMTHREADSTATE_TERMINATING:
+            case PDMTHREADSTATE_TERMINATED:
+                break;
+
+            case PDMTHREADSTATE_INITIALIZING:
+            default:
+                AssertMsgFailed(("enmState=%d\n", enmState));
+                break;
+        }
+        break;
+    }
+}
+
+
+/**
+ * Suspends the thread.
+ *
+ * This can be called at the power off / suspend notifications to suspend the
+ * PDM thread a bit early. The thread will be automatically suspend upon
+ * completion of the device/driver notification cycle.
+ *
+ * The caller is responsible for serializing the control operations on the
+ * thread. That basically means, always do these calls from the EMT.
+ *
+ * @returns VBox status code.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadSuspend(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread != RTThreadSelf());
+
+    /*
+     * This is a noop if the thread is already suspended.
+     */
+    if (pThread->enmState == PDMTHREADSTATE_SUSPENDED)
+        return VINF_SUCCESS;
+
+    /*
+     * Change the state to resuming and kick the thread.
+     */
+    int rc = RTSemEventMultiReset(pThread->Internal.s.BlockEvent);
+    if (RT_SUCCESS(rc))
+    {
+        rc = RTThreadUserReset(pThread->Thread);
+        if (RT_SUCCESS(rc))
+        {
+            rc = VERR_WRONG_ORDER;
+            if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_SUSPENDING, PDMTHREADSTATE_RUNNING))
+            {
+                rc = pdmR3ThreadWakeUp(pThread);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Wait for the thread to reach the suspended state.
+                     */
+                    if (pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                        rc = RTThreadUserWait(pThread->Thread, 60*1000);
+                    if (    RT_SUCCESS(rc)
+                        &&  pThread->enmState != PDMTHREADSTATE_SUSPENDED)
+                        rc = VERR_INTERNAL_ERROR;
+                    if (RT_SUCCESS(rc))
+                        return rc;
+                }
+            }
+        }
+    }
+
+    /*
+     * Something failed, initialize termination.
+     */
+    AssertMsgFailed(("PDMR3ThreadSuspend -> rc=%Rrc enmState=%d\n", rc, pThread->enmState));
+    pdmR3ThreadBailOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Suspend all running threads.
+ *
+ * This is called by PDMR3Suspend() and PDMR3PowerOff() after all the devices
+ * and drivers have been notified about the suspend / power off.
+ *
+ * @return VBox status code.
+ * @param   pVM         The VM handle.
+ */
+int pdmR3ThreadSuspendAll(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect); /* This may cause deadlocks later... */
+    for (PPDMTHREAD pThread = pUVM->pdm.s.pThreads; pThread; pThread = pThread->Internal.s.pNext)
+        switch (pThread->enmState)
+        {
+            case PDMTHREADSTATE_RUNNING:
+            {
+                int rc = PDMR3ThreadSuspend(pThread);
+                AssertRCReturn(rc, rc);
+                break;
+            }
+
+            /* suspend -> power off; voluntary suspend. */
+            case PDMTHREADSTATE_SUSPENDED:
+                break;
+
+            default:
+                AssertMsgFailed(("pThread=%p enmState=%d\n", pThread, pThread->enmState));
+                break;
+        }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resumes the thread.
+ *
+ * This can be called the power on / resume notifications to resume the
+ * PDM thread a bit early. The thread will be automatically resumed upon
+ * return from these two notification callbacks (devices/drivers).
+ *
+ * The caller is responsible for serializing the control operations on the
+ * thread. That basically means, always do these calls from the EMT.
+ *
+ * @returns VBox status code.
+ * @param   pThread     The PDM thread.
+ */
+VMMR3DECL(int) PDMR3ThreadResume(PPDMTHREAD pThread)
+{
+    /*
+     * Assert sanity.
+     */
+    AssertPtrReturn(pThread, VERR_INVALID_POINTER);
+    AssertReturn(pThread->u32Version == PDMTHREAD_VERSION, VERR_INVALID_MAGIC);
+    Assert(pThread->Thread != RTThreadSelf());
+
+    /*
+     * Change the state to resuming and kick the thread.
+     */
+    int rc = RTThreadUserReset(pThread->Thread);
+    if (RT_SUCCESS(rc))
+    {
+        rc = VERR_WRONG_ORDER;
+        if (pdmR3AtomicCmpXchgState(pThread, PDMTHREADSTATE_RESUMING, PDMTHREADSTATE_SUSPENDED))
+        {
+            rc = RTSemEventMultiSignal(pThread->Internal.s.BlockEvent);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Wait for the thread to reach the running state.
+                 */
+                rc = RTThreadUserWait(pThread->Thread, 60*1000);
+                if (    RT_SUCCESS(rc)
+                    &&  pThread->enmState != PDMTHREADSTATE_RUNNING)
+                    rc = VERR_INTERNAL_ERROR;
+                if (RT_SUCCESS(rc))
+                    return rc;
+            }
+        }
+    }
+
+    /*
+     * Something failed, initialize termination.
+     */
+    AssertMsgFailed(("PDMR3ThreadResume -> rc=%Rrc enmState=%d\n", rc, pThread->enmState));
+    pdmR3ThreadBailOut(pThread);
+    return rc;
+}
+
+
+/**
+ * Resumes all threads not running.
+ *
+ * This is called by PDMR3Resume() and PDMR3PowerOn() after all the devices
+ * and drivers have been notified about the resume / power on .
+ *
+ * @return VBox status code.
+ * @param   pVM         The VM handle.
+ */
+int pdmR3ThreadResumeAll(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->pdm.s.ListCritSect);
+    for (PPDMTHREAD pThread = pUVM->pdm.s.pThreads; pThread; pThread = pThread->Internal.s.pNext)
+        switch (pThread->enmState)
+        {
+            case PDMTHREADSTATE_SUSPENDED:
+            {
+                int rc = PDMR3ThreadResume(pThread);
+                AssertRCReturn(rc, rc);
+                break;
+            }
+
+            default:
+                AssertMsgFailed(("pThread=%p enmState=%d\n", pThread, pThread->enmState));
+                break;
+        }
+    RTCritSectLeave(&pUVM->pdm.s.ListCritSect);
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PDMUsb.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PDMUsb.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PDMUsb.cpp	(revision 35333)
@@ -0,0 +1,1444 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device and Driver Manager, USB part.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_DRIVER
+#include "PDMInternal.h"
+#include <VBox/pdm.h>
+#include <VBox/vusb.h>
+#include <VBox/mm.h>
+#include <VBox/cfgm.h>
+#include <VBox/vmm.h>
+#include <VBox/sup.h>
+#include <VBox/vm.h>
+#include <VBox/version.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <iprt/asm.h>
+#include <iprt/alloc.h>
+#include <iprt/alloca.h>
+#include <iprt/path.h>
+#include <iprt/uuid.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Internal callback structure pointer.
+ *
+ * The main purpose is to define the extra data we associate
+ * with PDMUSBREGCB so we can find the VM instance and so on.
+ */
+typedef struct PDMUSBREGCBINT
+{
+    /** The callback structure. */
+    PDMUSBREGCB     Core;
+    /** A bit of padding. */
+    uint32_t        u32[4];
+    /** VM Handle. */
+    PVM             pVM;
+} PDMUSBREGCBINT, *PPDMUSBREGCBINT;
+typedef const PDMUSBREGCBINT *PCPDMUSBREGCBINT;
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** @def PDMUSB_ASSERT_USBINS
+ * Asserts the validity of the USB device instance.
+ */
+#ifdef VBOX_STRICT
+# define PDMUSB_ASSERT_USBINS(pUsbIns) \
+    do { \
+        AssertPtr(pUsbIns); \
+        Assert(pUsbIns->u32Version == PDM_USBINS_VERSION); \
+        Assert(pUsbIns->pvInstanceDataR3 == (void *)&pUsbIns->achInstanceData[0]); \
+    } while (0)
+#else
+# define PDMUSB_ASSERT_USBINS(pUsbIns)   do { } while (0)
+#endif
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static void pdmR3UsbDestroyDevice(PVM pVM, PPDMUSBINS pUsbIns);
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+extern const PDMUSBHLP g_pdmR3UsbHlp;
+
+
+AssertCompile(sizeof(PDMUSBINSINT) <= RT_SIZEOFMEMB(PDMUSBINS, Internal.padding));
+
+
+/**
+ * Registers a USB hub driver.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pDrvIns         The driver instance of the hub.
+ * @param   fVersions       Indicates the kinds of USB devices that can be attached to this HUB.
+ * @param   cPorts          The number of ports.
+ * @param   pUsbHubReg      The hub callback structure that PDMUsb uses to interact with it.
+ * @param   ppUsbHubHlp     The helper callback structure that the hub uses to talk to PDMUsb.
+ * @thread  EMT
+ */
+int pdmR3UsbRegisterHub(PVM pVM, PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp)
+{
+    /*
+     * Validate input.
+     */
+    /* The driver must be in the USB class. */
+    if (!(pDrvIns->pReg->fClass & PDM_DRVREG_CLASS_USB))
+    {
+        LogRel(("pdmR3UsbRegisterHub: fClass=%#x expected %#x to be set\n", pDrvIns->pReg->fClass, PDM_DRVREG_CLASS_USB));
+        return VERR_INVALID_PARAMETER;
+    }
+    AssertMsgReturn(!(fVersions & ~(VUSB_STDVER_11 | VUSB_STDVER_20)), ("%#x\n", fVersions), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(ppUsbHubHlp, VERR_INVALID_POINTER);
+    AssertPtrReturn(pUsbHubReg, VERR_INVALID_POINTER);
+    AssertReturn(pUsbHubReg->u32Version == PDM_USBHUBREG_VERSION, VERR_INVALID_MAGIC);
+    AssertReturn(pUsbHubReg->u32TheEnd == PDM_USBHUBREG_VERSION, VERR_INVALID_MAGIC);
+    AssertPtrReturn(pUsbHubReg->pfnAttachDevice, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pUsbHubReg->pfnDetachDevice, VERR_INVALID_PARAMETER);
+
+    /*
+     * Check for duplicate registration and find the last hub for FIFO registration.
+     */
+    PPDMUSBHUB pPrev = NULL;
+    for (PPDMUSBHUB pCur = pVM->pdm.s.pUsbHubs; pCur; pCur = pCur->pNext)
+    {
+        if (pCur->pDrvIns == pDrvIns)
+            return VERR_PDM_USB_HUB_EXISTS;
+        pPrev = pCur;
+    }
+
+    /*
+     * Create an internal USB hub structure.
+     */
+    PPDMUSBHUB pHub = (PPDMUSBHUB)MMR3HeapAlloc(pVM, MM_TAG_PDM_DRIVER, sizeof(*pHub));
+    if (!pHub)
+        return VERR_NO_MEMORY;
+
+    pHub->fVersions = fVersions;
+    pHub->cPorts = cPorts;
+    pHub->cAvailablePorts = cPorts;
+    pHub->pDrvIns = pDrvIns;
+    pHub->Reg = *pUsbHubReg;
+    pHub->pNext = NULL;
+
+    /* link it */
+    if (pPrev)
+        pPrev->pNext = pHub;
+    else
+        pVM->pdm.s.pUsbHubs = pHub;
+
+    Log(("PDM: Registered USB hub %p/%s\n", pDrvIns, pDrvIns->pReg->szName));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads one device module and call the registration entry point.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pRegCB          The registration callback stuff.
+ * @param   pszFilename     Module filename.
+ * @param   pszName         Module name.
+ */
+static int pdmR3UsbLoad(PVM pVM, PCPDMUSBREGCBINT pRegCB, const char *pszFilename, const char *pszName)
+{
+    /*
+     * Load it.
+     */
+    int rc = pdmR3LoadR3U(pVM->pUVM, pszFilename, pszName);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Get the registration export and call it.
+         */
+        FNPDMVBOXUSBREGISTER *pfnVBoxUsbRegister;
+        rc = PDMR3LdrGetSymbolR3(pVM, pszName, "VBoxUsbRegister", (void **)&pfnVBoxUsbRegister);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("PDM: Calling VBoxUsbRegister (%p) of %s (%s)\n", pfnVBoxUsbRegister, pszName, pszFilename));
+            rc = pfnVBoxUsbRegister(&pRegCB->Core, VBOX_VERSION);
+            if (RT_SUCCESS(rc))
+                Log(("PDM: Successfully loaded device module %s (%s).\n", pszName, pszFilename));
+            else
+                AssertMsgFailed(("VBoxDevicesRegister failed with rc=%Rrc for module %s (%s)\n", rc, pszName, pszFilename));
+        }
+        else
+        {
+            AssertMsgFailed(("Failed to locate 'VBoxUsbRegister' in %s (%s) rc=%Rrc\n", pszName, pszFilename, rc));
+            if (rc == VERR_SYMBOL_NOT_FOUND)
+                rc = VERR_PDM_NO_REGISTRATION_EXPORT;
+        }
+    }
+    else
+        AssertMsgFailed(("Failed to load VBoxDD!\n"));
+    return rc;
+}
+
+
+
+/**
+ * @interface_method_impl{PDMUSBREGCB,pfnRegister}
+ */
+static DECLCALLBACK(int) pdmR3UsbReg_Register(PCPDMUSBREGCB pCallbacks, PCPDMUSBREG pReg)
+{
+    /*
+     * Validate the registration structure.
+     */
+    Assert(pReg);
+    AssertMsgReturn(pReg->u32Version == PDM_USBREG_VERSION,
+                    ("Unknown struct version %#x!\n", pReg->u32Version),
+                    VERR_PDM_UNKNOWN_USBREG_VERSION);
+    AssertMsgReturn(    pReg->szName[0]
+                    &&  strlen(pReg->szName) < sizeof(pReg->szName),
+                    ("Invalid name '%s'\n", pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn(pReg->fFlags == 0, ("fFlags=%#x\n", pReg->fFlags), VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn(pReg->cMaxInstances > 0,
+                    ("Max instances %u! (USB Device %s)\n", pReg->cMaxInstances, pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn(pReg->cbInstance <= _1M,
+                    ("Instance size %d bytes! (USB Device %s)\n", pReg->cbInstance, pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+    AssertMsgReturn(pReg->pfnConstruct, ("No constructor! (USB Device %s)\n", pReg->szName),
+                    VERR_PDM_INVALID_USB_REGISTRATION);
+
+    /*
+     * Check for duplicate and find FIFO entry at the same time.
+     */
+    PCPDMUSBREGCBINT pRegCB = (PCPDMUSBREGCBINT)pCallbacks;
+    PPDMUSB pUsbPrev = NULL;
+    PPDMUSB pUsb = pRegCB->pVM->pdm.s.pUsbDevs;
+    for (; pUsb; pUsbPrev = pUsb, pUsb = pUsb->pNext)
+        AssertMsgReturn(strcmp(pUsb->pReg->szName, pReg->szName),
+                        ("USB Device '%s' already exists\n", pReg->szName),
+                        VERR_PDM_USB_NAME_CLASH);
+
+    /*
+     * Allocate new device structure and insert it into the list.
+     */
+    pUsb = (PPDMUSB)MMR3HeapAlloc(pRegCB->pVM, MM_TAG_PDM_DEVICE, sizeof(*pUsb));
+    if (pUsb)
+    {
+        pUsb->pNext = NULL;
+        pUsb->iNextInstance = 0;
+        pUsb->pInstances = NULL;
+        pUsb->pReg = pReg;
+        pUsb->cchName = (RTUINT)strlen(pReg->szName);
+
+        if (pUsbPrev)
+            pUsbPrev->pNext = pUsb;
+        else
+            pRegCB->pVM->pdm.s.pUsbDevs = pUsb;
+        Log(("PDM: Registered USB device '%s'\n", pReg->szName));
+        return VINF_SUCCESS;
+    }
+    return VERR_NO_MEMORY;
+}
+
+
+/**
+ * Load USB Device modules.
+ *
+ * This is called by pdmR3DevInit() after it has loaded it's device modules.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+int pdmR3UsbLoadModules(PVM pVM)
+{
+    LogFlow(("pdmR3UsbLoadModules:\n"));
+
+    AssertRelease(!(RT_OFFSETOF(PDMUSBINS, achInstanceData) & 15));
+    AssertRelease(sizeof(pVM->pdm.s.pUsbInstances->Internal.s) <= sizeof(pVM->pdm.s.pUsbInstances->Internal.padding));
+
+    /*
+     * Initialize the callback structure.
+     */
+    PDMUSBREGCBINT RegCB;
+    RegCB.Core.u32Version = PDM_USBREG_CB_VERSION;
+    RegCB.Core.pfnRegister = pdmR3UsbReg_Register;
+    RegCB.pVM = pVM;
+
+    /*
+     * Load the builtin module
+     */
+    PCFGMNODE pUsbNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM/USB/");
+    bool fLoadBuiltin;
+    int rc = CFGMR3QueryBool(pUsbNode, "LoadBuiltin", &fLoadBuiltin);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_NO_PARENT)
+        fLoadBuiltin = true;
+    else if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Configuration error: Querying boolean \"LoadBuiltin\" failed with %Rrc\n", rc));
+        return rc;
+    }
+    if (fLoadBuiltin)
+    {
+        /* make filename */
+        char *pszFilename = pdmR3FileR3("VBoxDD", true /*fShared*/);
+        if (!pszFilename)
+            return VERR_NO_TMP_MEMORY;
+        rc = pdmR3UsbLoad(pVM, &RegCB, pszFilename, "VBoxDD");
+        RTMemTmpFree(pszFilename);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Load additional device modules.
+     */
+    PCFGMNODE pCur;
+    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /*
+         * Get the name and path.
+         */
+        char szName[PDMMOD_NAME_LEN];
+        rc = CFGMR3GetName(pCur, &szName[0], sizeof(szName));
+        if (rc == VERR_CFGM_NOT_ENOUGH_SPACE)
+        {
+            AssertMsgFailed(("configuration error: The module name is too long, cchName=%zu.\n", CFGMR3GetNameLen(pCur)));
+            return VERR_PDM_MODULE_NAME_TOO_LONG;
+        }
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("CFGMR3GetName -> %Rrc.\n", rc));
+            return rc;
+        }
+
+        /* the path is optional, if no path the module name + path is used. */
+        char szFilename[RTPATH_MAX];
+        rc = CFGMR3QueryString(pCur, "Path", &szFilename[0], sizeof(szFilename));
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            strcpy(szFilename, szName);
+        else if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("configuration error: Failure to query the module path, rc=%Rrc.\n", rc));
+            return rc;
+        }
+
+        /* prepend path? */
+        if (!RTPathHavePath(szFilename))
+        {
+            char *psz = pdmR3FileR3(szFilename, false /*fShared*/);
+            if (!psz)
+                return VERR_NO_TMP_MEMORY;
+            size_t cch = strlen(psz) + 1;
+            if (cch > sizeof(szFilename))
+            {
+                RTMemTmpFree(psz);
+                AssertMsgFailed(("Filename too long! cch=%d '%s'\n", cch, psz));
+                return VERR_FILENAME_TOO_LONG;
+            }
+            memcpy(szFilename, psz, cch);
+            RTMemTmpFree(psz);
+        }
+
+        /*
+         * Load the module and register it's devices.
+         */
+        rc = pdmR3UsbLoad(pVM, &RegCB, szFilename, szName);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Send the init-complete notification to all the USB devices.
+ *
+ * This is called from pdmR3DevInit() after it has do its notification round.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+int pdmR3UsbVMInitComplete(PVM pVM)
+{
+    for (PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+    {
+        if (pUsbIns->pReg->pfnVMInitComplete)
+        {
+            int rc = pUsbIns->pReg->pfnVMInitComplete(pUsbIns);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("InitComplete on USB device '%s'/%d failed with rc=%Rrc\n",
+                                 pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+                return rc;
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Lookups a device structure by name.
+ * @internal
+ */
+PPDMUSB pdmR3UsbLookup(PVM pVM, const char *pszName)
+{
+    size_t cchName = strlen(pszName);
+    for (PPDMUSB pUsb = pVM->pdm.s.pUsbDevs; pUsb; pUsb = pUsb->pNext)
+        if (    pUsb->cchName == cchName
+            &&  !strcmp(pUsb->pReg->szName, pszName))
+            return pUsb;
+    return NULL;
+}
+
+
+/**
+ * Locates a suitable hub for the specified kind of device.
+ *
+ * @returns VINF_SUCCESS and *ppHub on success.
+ *          VERR_PDM_NO_USB_HUBS or VERR_PDM_NO_USB_PORTS on failure.
+ * @param   pVM             The VM handle.
+ * @param   iUsbVersion     The USB device version.
+ * @param   ppHub           Where to store the pointer to the USB hub.
+ */
+static int pdmR3UsbFindHub(PVM pVM, uint32_t iUsbVersion, PPDMUSBHUB *ppHub)
+{
+    *ppHub = NULL;
+    if (!pVM->pdm.s.pUsbHubs)
+        return VERR_PDM_NO_USB_HUBS;
+
+    for (PPDMUSBHUB pCur = pVM->pdm.s.pUsbHubs; pCur; pCur = pCur->pNext)
+        if (    pCur->cAvailablePorts > 0
+            &&  (   (pCur->fVersions & iUsbVersion)
+                 || pCur->fVersions == VUSB_STDVER_11))
+        {
+            *ppHub = pCur;
+            if (pCur->fVersions & iUsbVersion)
+                break;
+        }
+    if (*ppHub)
+        return VINF_SUCCESS;
+    return VERR_PDM_NO_USB_PORTS;
+}
+
+
+/**
+ * Creates the device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pUsbDev         The USB device emulation.
+ * @param   iInstance       -1 if not called by pdmR3UsbInstantiateDevices().
+ * @param   pUuid           The UUID for this device.
+ * @param   pInstanceNode   The instance CFGM node. NULL if not called by pdmR3UsbInstantiateDevices().
+ * @param   ppConfig        Pointer to the device configuration pointer. This is set to NULL if inserted
+ *                          into the tree or cleaned up.
+ *
+ *                          In the pdmR3UsbInstantiateDevices() case (pInstanceNode != NULL) this is
+ *                          the actual config node and will not be cleaned up.
+ *
+ * @parma   iUsbVersion     The USB version preferred by the device.
+ */
+static int pdmR3UsbCreateDevice(PVM pVM, PPDMUSBHUB pHub, PPDMUSB pUsbDev, int iInstance, PCRTUUID pUuid, PCFGMNODE pInstanceNode, PCFGMNODE *ppConfig, uint32_t iUsbVersion)
+{
+    const bool fAtRuntime = pInstanceNode == NULL;
+    int rc;
+
+    /*
+     * If not called by pdmR3UsbInstantiateDevices(), we'll have to fix
+     * the configuration now.
+     */
+    /* USB device node. */
+    PCFGMNODE pDevNode = CFGMR3GetChildF(CFGMR3GetRoot(pVM), "USB/%s/", pUsbDev->pReg->szName);
+    if (!pDevNode)
+    {
+        rc = CFGMR3InsertNodeF(CFGMR3GetRoot(pVM), &pDevNode, "USB/%s/", pUsbDev->pReg->szName);
+        AssertRCReturn(rc, rc);
+    }
+
+    /* The instance node and number. */
+    if (!pInstanceNode)
+    {
+        for (unsigned c = 0; c < _2M; c++)
+        {
+            iInstance = pUsbDev->iNextInstance++;
+            rc = CFGMR3InsertNodeF(pDevNode, &pInstanceNode, "%d/", iInstance);
+            if (rc != VERR_CFGM_NODE_EXISTS)
+                break;
+        }
+        AssertRCReturn(rc, rc);
+    }
+    else
+    {
+        Assert(iInstance >= 0);
+        if (iInstance >= (int)pUsbDev->iNextInstance)
+            pUsbDev->iNextInstance = iInstance + 1;
+    }
+
+    /* The instance config node. */
+    PCFGMNODE pConfigToDelete = NULL;
+    PCFGMNODE pConfig = NULL;
+    if (!ppConfig || !*ppConfig)
+    {
+        rc = CFGMR3InsertNode(pInstanceNode, "Config", &pConfig);
+        AssertRCReturn(rc, rc);
+    }
+    else if (fAtRuntime)
+    {
+        rc = CFGMR3InsertSubTree(pInstanceNode, "Config", *ppConfig, &pConfig);
+        AssertRCReturn(rc, rc);
+        *ppConfig = NULL;
+        pConfigToDelete = pConfig;
+    }
+    else
+        pConfig = *ppConfig;
+    Assert(CFGMR3GetChild(pInstanceNode, "Config") == pConfig);
+
+    /* The global device config node. */
+    PCFGMNODE pGlobalConfig = CFGMR3GetChild(pDevNode, "GlobalConfig");
+    if (!pGlobalConfig)
+    {
+        rc = CFGMR3InsertNode(pDevNode, "GlobalConfig", &pGlobalConfig);
+        if (RT_FAILURE(rc))
+        {
+            CFGMR3RemoveNode(pConfigToDelete);
+            AssertRCReturn(rc, rc);
+        }
+    }
+
+    /*
+     * Allocate the device instance.
+     */
+    size_t cb = RT_OFFSETOF(PDMUSBINS, achInstanceData[pUsbDev->pReg->cbInstance]);
+    cb = RT_ALIGN_Z(cb, 16);
+    PPDMUSBINS pUsbIns;
+    rc = MMR3HeapAllocZEx(pVM, MM_TAG_PDM_USB, cb, (void **)&pUsbIns);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to allocate %d bytes of instance data for USB device '%s'. rc=%Rrc\n",
+                         cb, pUsbDev->pReg->szName, rc));
+        CFGMR3RemoveNode(pConfigToDelete);
+        return rc;
+    }
+
+    /*
+     * Initialize it.
+     */
+    pUsbIns->u32Version                     = PDM_USBINS_VERSION;
+    //pUsbIns->Internal.s.pNext               = NULL;
+    //pUsbIns->Internal.s.pPerDeviceNext      = NULL;
+    pUsbIns->Internal.s.pUsbDev             = pUsbDev;
+    pUsbIns->Internal.s.pVM                 = pVM;
+    //pUsbIns->Internal.s.pLuns               = NULL;
+    pUsbIns->Internal.s.pCfg                = pInstanceNode;
+    pUsbIns->Internal.s.pCfgDelete          = pConfigToDelete;
+    pUsbIns->Internal.s.pCfgGlobal          = pGlobalConfig;
+    pUsbIns->Internal.s.Uuid                = *pUuid;
+    //pUsbIns->Internal.s.pHub                = NULL;
+    pUsbIns->Internal.s.iPort               = UINT32_MAX; /* to be determined. */
+    pUsbIns->Internal.s.fVMSuspended        = true;
+    //pUsbIns->Internal.s.pfnAsyncNotify      = NULL;
+    pUsbIns->pHlpR3                         = &g_pdmR3UsbHlp;
+    pUsbIns->pReg                           = pUsbDev->pReg;
+    pUsbIns->pCfg                           = pConfig;
+    pUsbIns->pCfgGlobal                     = pGlobalConfig;
+    pUsbIns->iInstance                      = iInstance;
+    pUsbIns->pvInstanceDataR3               = &pUsbIns->achInstanceData[0];
+    pUsbIns->pszName                        = RTStrDup(pUsbDev->pReg->szName);
+
+    /*
+     * Link it into all the lists.
+     */
+    /* The global instance FIFO. */
+    PPDMUSBINS pPrev1 = pVM->pdm.s.pUsbInstances;
+    if (!pPrev1)
+        pVM->pdm.s.pUsbInstances = pUsbIns;
+    else
+    {
+        while (pPrev1->Internal.s.pNext)
+        {
+            Assert(pPrev1->u32Version == PDM_USBINS_VERSION);
+            pPrev1 = pPrev1->Internal.s.pNext;
+        }
+        pPrev1->Internal.s.pNext = pUsbIns;
+    }
+
+    /* The per device instance FIFO. */
+    PPDMUSBINS pPrev2 = pUsbDev->pInstances;
+    if (!pPrev2)
+        pUsbDev->pInstances = pUsbIns;
+    else
+    {
+        while (pPrev2->Internal.s.pPerDeviceNext)
+        {
+            Assert(pPrev2->u32Version == PDM_USBINS_VERSION);
+            pPrev2 = pPrev2->Internal.s.pPerDeviceNext;
+        }
+        pPrev2->Internal.s.pPerDeviceNext = pUsbIns;
+    }
+
+    /*
+     * Call the constructor.
+     */
+    Log(("PDM: Constructing USB device '%s' instance %d...\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+    rc = pUsbIns->pReg->pfnConstruct(pUsbIns, pUsbIns->iInstance, pUsbIns->pCfg, pUsbIns->pCfgGlobal);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Attach it to the hub.
+         */
+        Log(("PDM: Attaching it...\n"));
+        rc = pHub->Reg.pfnAttachDevice(pHub->pDrvIns, pUsbIns, &pUsbIns->Internal.s.iPort);
+        if (RT_SUCCESS(rc))
+        {
+            pHub->cAvailablePorts--;
+            Assert((int32_t)pHub->cAvailablePorts >= 0 && pHub->cAvailablePorts < pHub->cPorts);
+            pUsbIns->Internal.s.pHub = pHub;
+
+            /* Send the hot-plugged notification if applicable. */
+            if (fAtRuntime && pUsbIns->pReg->pfnHotPlugged)
+                pUsbIns->pReg->pfnHotPlugged(pUsbIns);
+
+            Log(("PDM: Successfully attached USB device '%s' instance %d to hub %p\n",
+                 pUsbIns->pReg->szName, pUsbIns->iInstance, pHub));
+            return VINF_SUCCESS;
+        }
+
+        LogRel(("PDM: Failed to attach USB device '%s' instance %d to hub %p: %Rrc\n",
+                pUsbIns->pReg->szName, pUsbIns->iInstance, pHub, rc));
+    }
+    else
+        AssertMsgFailed(("Failed to construct '%s'/%d! %Rra\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    if (fAtRuntime)
+        pdmR3UsbDestroyDevice(pVM, pUsbIns);
+    /* else: destructors are invoked later. */
+    return rc;
+}
+
+
+/**
+ * Instantiate USB devices.
+ *
+ * This is called by pdmR3DevInit() after it has instantiated the
+ * other devices and their drivers. If there aren't any hubs
+ * around, we'll silently skip the USB devices.
+ *
+ * @returns VBox status code.
+ * @param   pVM
+ */
+int pdmR3UsbInstantiateDevices(PVM pVM)
+{
+    /*
+     * Any hubs?
+     */
+    if (!pVM->pdm.s.pUsbHubs)
+    {
+        Log(("PDM: No USB hubs, skipping USB device instantiation.\n"));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Count the device instances.
+     */
+    PCFGMNODE pCur;
+    PCFGMNODE pUsbNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "USB/");
+    PCFGMNODE pInstanceNode;
+    unsigned cUsbDevs = 0;
+    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        PCFGMNODE pGlobal = CFGMR3GetChild(pCur, "GlobalConfig/");
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+            if (pInstanceNode != pGlobal)
+                cUsbDevs++;
+    }
+    if (!cUsbDevs)
+    {
+        Log(("PDM: No USB devices were configured!\n"));
+        return VINF_SUCCESS;
+    }
+    Log2(("PDM: cUsbDevs=%d!\n", cUsbDevs));
+
+    /*
+     * Collect info on each USB device instance.
+     */
+    struct USBDEVORDER
+    {
+        /** Configuration node. */
+        PCFGMNODE   pNode;
+        /** Pointer to the USB device. */
+        PPDMUSB     pUsbDev;
+        /** Init order. */
+        uint32_t    u32Order;
+        /** VBox instance number. */
+        uint32_t    iInstance;
+    } *paUsbDevs = (struct USBDEVORDER *)alloca(sizeof(paUsbDevs[0]) * (cUsbDevs + 1)); /* (One extra for swapping) */
+    Assert(paUsbDevs);
+    int rc;
+    unsigned i = 0;
+    for (pCur = CFGMR3GetFirstChild(pUsbNode); pCur; pCur = CFGMR3GetNextChild(pCur))
+    {
+        /* Get the device name. */
+        char szName[sizeof(paUsbDevs[0].pUsbDev->pReg->szName)];
+        rc = CFGMR3GetName(pCur, szName, sizeof(szName));
+        AssertMsgRCReturn(rc, ("Configuration error: device name is too long (or something)! rc=%Rrc\n", rc), rc);
+
+        /* Find the device. */
+        PPDMUSB pUsbDev = pdmR3UsbLookup(pVM, szName);
+        AssertMsgReturn(pUsbDev, ("Configuration error: device '%s' not found!\n", szName), VERR_PDM_DEVICE_NOT_FOUND);
+
+        /* Configured priority or use default? */
+        uint32_t u32Order;
+        rc = CFGMR3QueryU32(pCur, "Priority", &u32Order);
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+            u32Order = i << 4;
+        else
+            AssertMsgRCReturn(rc, ("Configuration error: reading \"Priority\" for the '%s' USB device failed rc=%Rrc!\n", szName, rc), rc);
+
+        /* Global config. */
+        PCFGMNODE pGlobal = CFGMR3GetChild(pCur, "GlobalConfig/");
+        if (!pGlobal)
+        {
+            rc = CFGMR3InsertNode(pCur, "GlobalConfig/", &pGlobal);
+            AssertMsgRCReturn(rc, ("Failed to create GlobalConfig node! rc=%Rrc\n", rc), rc);
+            CFGMR3SetRestrictedRoot(pGlobal);
+        }
+
+        /* Enumerate the device instances. */
+        for (pInstanceNode = CFGMR3GetFirstChild(pCur); pInstanceNode; pInstanceNode = CFGMR3GetNextChild(pInstanceNode))
+        {
+            if (pInstanceNode == pGlobal)
+                continue;
+
+            paUsbDevs[i].pNode = pInstanceNode;
+            paUsbDevs[i].pUsbDev = pUsbDev;
+            paUsbDevs[i].u32Order = u32Order;
+
+            /* Get the instance number. */
+            char szInstance[32];
+            rc = CFGMR3GetName(pInstanceNode, szInstance, sizeof(szInstance));
+            AssertMsgRCReturn(rc, ("Configuration error: instance name is too long (or something)! rc=%Rrc\n", rc), rc);
+            char *pszNext = NULL;
+            rc = RTStrToUInt32Ex(szInstance, &pszNext, 0, &paUsbDevs[i].iInstance);
+            AssertMsgRCReturn(rc, ("Configuration error: RTStrToInt32Ex failed on the instance name '%s'! rc=%Rrc\n", szInstance, rc), rc);
+            AssertMsgReturn(!*pszNext, ("Configuration error: the instance name '%s' isn't all digits. (%s)\n", szInstance, pszNext), VERR_INVALID_PARAMETER);
+
+            /* next instance */
+            i++;
+        }
+    } /* devices */
+    Assert(i == cUsbDevs);
+
+    /*
+     * Sort the device array ascending on u32Order. (bubble)
+     */
+    unsigned c = cUsbDevs - 1;
+    while (c)
+    {
+        unsigned j = 0;
+        for (i = 0; i < c; i++)
+            if (paUsbDevs[i].u32Order > paUsbDevs[i + 1].u32Order)
+            {
+                paUsbDevs[cUsbDevs] = paUsbDevs[i + 1];
+                paUsbDevs[i + 1] = paUsbDevs[i];
+                paUsbDevs[i] = paUsbDevs[cUsbDevs];
+                j = i;
+            }
+        c = j;
+    }
+
+    /*
+     * Instantiate the devices.
+     */
+    for (i = 0; i < cUsbDevs; i++)
+    {
+        /*
+         * Make sure there is a config node and mark it as restricted.
+         */
+        PCFGMNODE pConfigNode = CFGMR3GetChild(paUsbDevs[i].pNode, "Config/");
+        if (!pConfigNode)
+        {
+            rc = CFGMR3InsertNode(paUsbDevs[i].pNode, "Config", &pConfigNode);
+            AssertMsgRCReturn(rc, ("Failed to create Config node! rc=%Rrc\n", rc), rc);
+        }
+        CFGMR3SetRestrictedRoot(pConfigNode);
+
+        /** @todo
+         * Figure out the USB version from the USB device registration and the configuration.
+         */
+        uint32_t iUsbVersion = VUSB_STDVER_11;
+
+        /*
+         * Find a suitable hub with free ports.
+         */
+        PPDMUSBHUB pHub;
+        rc = pdmR3UsbFindHub(pVM, iUsbVersion, &pHub);
+        if (RT_FAILURE(rc))
+        {
+            Log(("pdmR3UsbFindHub failed %Rrc\n", rc));
+            return rc;
+        }
+
+        /*
+         * Create and attach the device.
+         */
+        RTUUID Uuid;
+        rc = RTUuidCreate(&Uuid);
+        AssertRCReturn(rc, rc);
+        rc = pdmR3UsbCreateDevice(pVM, pHub, paUsbDevs[i].pUsbDev, paUsbDevs[i].iInstance, &Uuid, paUsbDevs[i].pNode, &pConfigNode, iUsbVersion);
+        if (RT_FAILURE(rc))
+            return rc;
+    } /* for device instances */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a USB proxy device instance.
+ *
+ * This will find an appropriate HUB for the USB device, create the necessary CFGM stuff
+ * and try instantiate the proxy device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pUuid           The UUID to be associated with the device.
+ * @param   fRemote         Whether it's a remove or local device.
+ * @param   pszAddress      The address string.
+ * @param   pvBackend       Pointer to the backend.
+ * @param   iUsbVersion     The preferred USB version.
+ * @param   fMaskedIfs      The interfaces to hide from the guest.
+ */
+VMMR3DECL(int) PDMR3USBCreateProxyDevice(PVM pVM, PCRTUUID pUuid, bool fRemote, const char *pszAddress, void *pvBackend,
+                                         uint32_t iUsbVersion, uint32_t fMaskedIfs)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    AssertPtrReturn(pUuid, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszAddress, VERR_INVALID_POINTER);
+    AssertReturn(    iUsbVersion == VUSB_STDVER_20
+                 ||  iUsbVersion == VUSB_STDVER_11, VERR_INVALID_PARAMETER);
+
+    /*
+     * Find the USBProxy driver.
+     */
+    PPDMUSB pUsbDev = pdmR3UsbLookup(pVM, "USBProxy");
+    if (!pUsbDev)
+    {
+        LogRel(("PDMR3USBCreateProxyDevice: The USBProxy device class wasn't found\n"));
+        return VERR_PDM_NO_USBPROXY;
+    }
+
+    /*
+     * Find a suitable hub with free ports.
+     */
+    PPDMUSBHUB pHub;
+    int rc = pdmR3UsbFindHub(pVM, iUsbVersion, &pHub);
+    if (RT_FAILURE(rc))
+    {
+        Log(("pdmR3UsbFindHub: failed %Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Create the CFGM configuration node.
+     */
+    PCFGMNODE pConfig = CFGMR3CreateTree(pVM);
+    AssertReturn(pConfig, VERR_NO_MEMORY);
+    do /* break loop */
+    {
+        rc = CFGMR3InsertString(pConfig,  "Address", pszAddress);               AssertRCBreak(rc);
+        char szUuid[RTUUID_STR_LENGTH];
+        rc = RTUuidToStr(pUuid, &szUuid[0], sizeof(szUuid));                    AssertRCBreak(rc);
+        rc = CFGMR3InsertString(pConfig,  "UUID", szUuid);                      AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "Remote", fRemote);                   AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "USBVersion", iUsbVersion);           AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "pvBackend", (uintptr_t)pvBackend);   AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "MaskedIfs", fMaskedIfs);             AssertRCBreak(rc);
+        rc = CFGMR3InsertInteger(pConfig, "Force11Device", !(pHub->fVersions & iUsbVersion)); AssertRCBreak(rc);
+    } while (0); /* break loop */
+    if (RT_FAILURE(rc))
+    {
+        CFGMR3RemoveNode(pConfig);
+        LogRel(("PDMR3USBCreateProxyDevice: failed to setup CFGM config, rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Finally, try create it.
+     */
+    rc = pdmR3UsbCreateDevice(pVM, pHub, pUsbDev, -1, pUuid, NULL, &pConfig, iUsbVersion);
+    if (RT_FAILURE(rc) && pConfig)
+        CFGMR3RemoveNode(pConfig);
+    return rc;
+}
+
+
+/**
+ * Destroys a hot-plugged USB device.
+ *
+ * The device must be detached from the HUB at this point.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pUsbIns         The USB device instance to destroy.
+ * @thread  EMT
+ */
+static void pdmR3UsbDestroyDevice(PVM pVM, PPDMUSBINS pUsbIns)
+{
+    Assert(!pUsbIns->Internal.s.pHub);
+
+    /*
+     * Do the unplug notification.
+     */
+    /** @todo what about the drivers? */
+    if (pUsbIns->pReg->pfnHotUnplugged)
+        pUsbIns->pReg->pfnHotUnplugged(pUsbIns);
+
+    /*
+     * Destroy the luns with their driver chains and call the device destructor.
+     */
+    while (pUsbIns->Internal.s.pLuns)
+    {
+        PPDMLUN pLun = pUsbIns->Internal.s.pLuns;
+        pUsbIns->Internal.s.pLuns = pLun->pNext;
+        if (pLun->pTop)
+            pdmR3DrvDestroyChain(pLun->pTop, PDM_TACH_FLAGS_NOT_HOT_PLUG); /* Hotplugging is handled differently here atm. */
+        MMR3HeapFree(pLun);
+    }
+
+    /* finally, the device. */
+    if (pUsbIns->pReg->pfnDestruct)
+    {
+        Log(("PDM: Destructing USB device '%s' instance %d...\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        pUsbIns->pReg->pfnDestruct(pUsbIns);
+    }
+    //TMR3TimerDestroyUsb(pVM, pUsbIns);
+    //SSMR3DeregisterUsb(pVM, pUsbIns, NULL, 0);
+    pdmR3ThreadDestroyUsb(pVM, pUsbIns);
+
+    /*
+     * Unlink it.
+     */
+    /* The global instance FIFO. */
+    if (pVM->pdm.s.pUsbInstances == pUsbIns)
+        pVM->pdm.s.pUsbInstances = pUsbIns->Internal.s.pNext;
+    else
+    {
+        PPDMUSBINS pPrev = pVM->pdm.s.pUsbInstances;
+        while (pPrev && pPrev->Internal.s.pNext != pUsbIns)
+        {
+            Assert(pPrev->u32Version == PDM_USBINS_VERSION);
+            pPrev = pPrev->Internal.s.pNext;
+        }
+        Assert(pPrev); Assert(pPrev != pUsbIns);
+        if (pPrev)
+            pPrev->Internal.s.pNext = pUsbIns->Internal.s.pNext;
+    }
+
+    /* The per device instance FIFO. */
+    PPDMUSB pUsbDev = pUsbIns->Internal.s.pUsbDev;
+    if (pUsbDev->pInstances == pUsbIns)
+        pUsbDev->pInstances = pUsbIns->Internal.s.pPerDeviceNext;
+    else
+    {
+        PPDMUSBINS pPrev = pUsbDev->pInstances;
+        while (pPrev && pPrev->Internal.s.pPerDeviceNext != pUsbIns)
+        {
+            Assert(pPrev->u32Version == PDM_USBINS_VERSION);
+            pPrev = pPrev->Internal.s.pPerDeviceNext;
+        }
+        Assert(pPrev); Assert(pPrev != pUsbIns);
+        if (pPrev)
+            pPrev->Internal.s.pPerDeviceNext = pUsbIns->Internal.s.pPerDeviceNext;
+    }
+
+    /*
+     * Trash it.
+     */
+    pUsbIns->u32Version = 0;
+    pUsbIns->pReg = NULL;
+    if (pUsbIns->pszName)
+    {
+        RTStrFree(pUsbIns->pszName);
+        pUsbIns->pszName = NULL;
+    }
+    CFGMR3RemoveNode(pUsbIns->Internal.s.pCfgDelete);
+    MMR3HeapFree(pUsbIns);
+}
+
+
+/**
+ * Detaches and destroys a USB device.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pUuid           The UUID associated with the device to detach.
+ * @thread  EMT
+ */
+VMMR3DECL(int) PDMR3USBDetachDevice(PVM pVM, PCRTUUID pUuid)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT(pVM);
+    AssertPtrReturn(pUuid, VERR_INVALID_POINTER);
+    AssertPtrReturn(pVM, VERR_INVALID_POINTER);
+
+    /*
+     * Search the global list for it.
+     */
+    PPDMUSBINS pUsbIns = pVM->pdm.s.pUsbInstances;
+    for ( ; pUsbIns; pUsbIns = pUsbIns->Internal.s.pNext)
+        if (!RTUuidCompare(&pUsbIns->Internal.s.Uuid, pUuid))
+            break;
+    if (!pUsbIns)
+        return VERR_PDM_DEVICE_INSTANCE_NOT_FOUND; /** @todo VERR_PDM_USB_INSTANCE_NOT_FOUND */
+
+    /*
+     * Detach it from the HUB (if it's actually attached to one).
+     */
+    PPDMUSBHUB pHub = pUsbIns->Internal.s.pHub;
+    if (pHub)
+    {
+        int rc = pHub->Reg.pfnDetachDevice(pHub->pDrvIns, pUsbIns, pUsbIns->Internal.s.iPort);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("PDM: Failed to detach USB device '%s' instance %d from %p: %Rrc\n",
+                    pUsbIns->pReg->szName, pUsbIns->iInstance, pHub, rc));
+            return rc;
+        }
+
+        pHub->cAvailablePorts++;
+        Assert(pHub->cAvailablePorts > 0 && pHub->cAvailablePorts <= pHub->cPorts);
+        pUsbIns->Internal.s.pHub = NULL;
+    }
+
+    /*
+     * Notify about unplugging and destroy the device with it's drivers.
+     */
+    pdmR3UsbDestroyDevice(pVM, pUsbIns);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if there are any USB hubs attached.
+ *
+ * @returns true / false accordingly.
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+VMMR3DECL(bool) PDMR3USBHasHub(PVM pVM)
+{
+    return pVM->pdm.s.pUsbHubs != NULL;
+}
+
+
+/** @name USB Device Helpers
+ * @{
+ */
+
+/** @interface_method_impl{PDMUSBHLPR3,pfnDriverAttach} */
+static DECLCALLBACK(int) pdmR3UsbHlp_DriverAttach(PPDMUSBINS pUsbIns, RTUINT iLun, PPDMIBASE pBaseInterface, PPDMIBASE *ppBaseInterface, const char *pszDesc)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: iLun=%d pBaseInterface=%p ppBaseInterface=%p pszDesc=%p:{%s}\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, iLun, pBaseInterface, ppBaseInterface, pszDesc, pszDesc));
+
+    /*
+     * Lookup the LUN, it might already be registered.
+     */
+    PPDMLUN pLunPrev = NULL;
+    PPDMLUN pLun = pUsbIns->Internal.s.pLuns;
+    for (; pLun; pLunPrev = pLun, pLun = pLun->pNext)
+        if (pLun->iLun == iLun)
+            break;
+
+    /*
+     * Create the LUN if if wasn't found, else check if driver is already attached to it.
+     */
+    if (!pLun)
+    {
+        if (    !pBaseInterface
+            ||  !pszDesc
+            ||  !*pszDesc)
+        {
+            Assert(pBaseInterface);
+            Assert(pszDesc || *pszDesc);
+            return VERR_INVALID_PARAMETER;
+        }
+
+        pLun = (PPDMLUN)MMR3HeapAlloc(pVM, MM_TAG_PDM_LUN, sizeof(*pLun));
+        if (!pLun)
+            return VERR_NO_MEMORY;
+
+        pLun->iLun      = iLun;
+        pLun->pNext     = pLunPrev ? pLunPrev->pNext : NULL;
+        pLun->pTop      = NULL;
+        pLun->pBottom   = NULL;
+        pLun->pDevIns   = NULL;
+        pLun->pUsbIns   = pUsbIns;
+        pLun->pszDesc   = pszDesc;
+        pLun->pBase     = pBaseInterface;
+        if (!pLunPrev)
+            pUsbIns->Internal.s.pLuns = pLun;
+        else
+            pLunPrev->pNext = pLun;
+        Log(("pdmR3UsbHlp_DriverAttach: Registered LUN#%d '%s' with device '%s'/%d.\n",
+             iLun, pszDesc, pUsbIns->pReg->szName, pUsbIns->iInstance));
+    }
+    else if (pLun->pTop)
+    {
+        AssertMsgFailed(("Already attached! The device should keep track of such things!\n"));
+        LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, VERR_PDM_DRIVER_ALREADY_ATTACHED));
+        return VERR_PDM_DRIVER_ALREADY_ATTACHED;
+    }
+    Assert(pLun->pBase == pBaseInterface);
+
+
+    /*
+     * Get the attached driver configuration.
+     */
+    int rc;
+    PCFGMNODE pNode = CFGMR3GetChildF(pUsbIns->Internal.s.pCfg, "LUN#%u", iLun);
+    if (pNode)
+        rc = pdmR3DrvInstantiate(pVM, pNode, pBaseInterface, NULL /*pDrvAbove*/, pLun, ppBaseInterface);
+    else
+        rc = VERR_PDM_NO_ATTACHED_DRIVER;
+
+
+    LogFlow(("pdmR3UsbHlp_DriverAttach: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnAssertEMT} */
+static DECLCALLBACK(bool) pdmR3UsbHlp_AssertEMT(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    if (VM_IS_EMT(pUsbIns->Internal.s.pVM))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertEMT '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnAssertOther} */
+static DECLCALLBACK(bool) pdmR3UsbHlp_AssertOther(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    if (!VM_IS_EMT(pUsbIns->Internal.s.pVM))
+        return true;
+
+    char szMsg[100];
+    RTStrPrintf(szMsg, sizeof(szMsg), "AssertOther '%s'/%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance);
+    RTAssertMsg1Weak(szMsg, iLine, pszFile, pszFunction);
+    AssertBreakpoint();
+    return false;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnDBGFStopV} */
+static DECLCALLBACK(int) pdmR3UsbHlp_DBGFStopV(PPDMUSBINS pUsbIns, const char *pszFile, unsigned iLine, const char *pszFunction, const char *pszFormat, va_list args)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+#ifdef LOG_ENABLED
+    va_list va2;
+    va_copy(va2, args);
+    LogFlow(("pdmR3UsbHlp_DBGFStopV: caller='%s'/%d: pszFile=%p:{%s} iLine=%d pszFunction=%p:{%s} pszFormat=%p:{%s} (%N)\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, pszFile, pszFile, iLine, pszFunction, pszFunction, pszFormat, pszFormat, pszFormat, &va2));
+    va_end(va2);
+#endif
+
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    int rc = DBGFR3EventSrcV(pVM, DBGFEVENT_DEV_STOP, pszFile, iLine, pszFunction, pszFormat, args);
+    if (rc == VERR_DBGF_NOT_ATTACHED)
+        rc = VINF_SUCCESS;
+
+    LogFlow(("pdmR3UsbHlp_DBGFStopV: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnDBGFInfoRegister} */
+static DECLCALLBACK(int) pdmR3UsbHlp_DBGFInfoRegister(PPDMUSBINS pUsbIns, const char *pszName, const char *pszDesc, PFNDBGFHANDLERUSB pfnHandler)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_DBGFInfoRegister: caller='%s'/%d: pszName=%p:{%s} pszDesc=%p:{%s} pfnHandler=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, pszName, pszName, pszDesc, pszDesc, pfnHandler));
+
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    /** @todo int rc = DBGFR3InfoRegisterUsb(pVM, pszName, pszDesc, pfnHandler, pUsbIns); */
+    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
+
+    LogFlow(("pdmR3UsbHlp_DBGFInfoRegister: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnMMHeapAlloc} */
+static DECLCALLBACK(void *) pdmR3UsbHlp_MMHeapAlloc(PPDMUSBINS pUsbIns, size_t cb)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_MMHeapAlloc: caller='%s'/%d: cb=%#x\n", pUsbIns->pReg->szName, pUsbIns->iInstance, cb));
+
+    void *pv = MMR3HeapAlloc(pUsbIns->Internal.s.pVM, MM_TAG_PDM_USB_USER, cb);
+
+    LogFlow(("pdmR3UsbHlp_MMHeapAlloc: caller='%s'/%d: returns %p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnMMHeapAllocZ} */
+static DECLCALLBACK(void *) pdmR3UsbHlp_MMHeapAllocZ(PPDMUSBINS pUsbIns, size_t cb)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_MMHeapAllocZ: caller='%s'/%d: cb=%#x\n", pUsbIns->pReg->szName, pUsbIns->iInstance, cb));
+
+    void *pv = MMR3HeapAllocZ(pUsbIns->Internal.s.pVM, MM_TAG_PDM_USB_USER, cb);
+
+    LogFlow(("pdmR3UsbHlp_MMHeapAllocZ: caller='%s'/%d: returns %p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pv));
+    return pv;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnPDMQueueCreate} */
+static DECLCALLBACK(int) pdmR3UsbHlp_PDMQueueCreate(PPDMUSBINS pUsbIns, RTUINT cbItem, RTUINT cItems, uint32_t cMilliesInterval,
+                                                    PFNPDMQUEUEUSB pfnCallback, const char *pszName, PPDMQUEUE *ppQueue)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    LogFlow(("pdmR3UsbHlp_PDMQueueCreate: caller='%s'/%d: cbItem=%#x cItems=%#x cMilliesInterval=%u pfnCallback=%p pszName=%p:{%s} ppQueue=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, cbItem, cItems, cMilliesInterval, pfnCallback, pszName, pszName, ppQueue));
+
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+
+    if (pUsbIns->iInstance > 0)
+    {
+        pszName = MMR3HeapAPrintf(pVM, MM_TAG_PDM_DEVICE_DESC, "%s_%u", pszName, pUsbIns->iInstance);
+        AssertLogRelReturn(pszName, VERR_NO_MEMORY);
+    }
+
+    /** @todo int rc = PDMR3QueueCreateUsb(pVM, pUsbIns, cbItem, cItems, cMilliesInterval, pfnCallback, fGCEnabled, pszName, ppQueue); */
+    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
+
+    LogFlow(("pdmR3UsbHlp_PDMQueueCreate: caller='%s'/%d: returns %Rrc *ppQueue=%p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc, *ppQueue));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnSSMRegister} */
+static DECLCALLBACK(int) pdmR3UsbHlp_SSMRegister(PPDMUSBINS pUsbIns, uint32_t uVersion, size_t cbGuess,
+                                                 PFNSSMUSBLIVEPREP pfnLivePrep, PFNSSMUSBLIVEEXEC pfnLiveExec, PFNSSMUSBLIVEVOTE pfnLiveVote,
+                                                 PFNSSMUSBSAVEPREP pfnSavePrep, PFNSSMUSBSAVEEXEC pfnSaveExec, PFNSSMUSBSAVEDONE pfnSaveDone,
+                                                 PFNSSMUSBLOADPREP pfnLoadPrep, PFNSSMUSBLOADEXEC pfnLoadExec, PFNSSMUSBLOADDONE pfnLoadDone)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    VM_ASSERT_EMT(pUsbIns->Internal.s.pVM);
+    LogFlow(("pdmR3UsbHlp_SSMRegister: caller='%s'/%d: uVersion=#x cbGuess=%#x\n"
+             "    pfnLivePrep=%p pfnLiveExec=%p pfnLiveVote=%p pfnSavePrep=%p pfnSaveExec=%p pfnSaveDone=%p pszLoadPrep=%p pfnLoadExec=%p pfnLoadDone=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, uVersion, cbGuess,
+             pfnLivePrep, pfnLiveExec, pfnLiveVote,
+             pfnSavePrep, pfnSaveExec, pfnSaveDone,
+             pfnLoadPrep, pfnLoadExec, pfnLoadDone));
+
+    /** @todo
+    int rc = SSMR3RegisterUsb(pUsbIns->Internal.s.pVM, pUsbIns, pUsbIns->pReg->szName, pUsbIns->iInstance,
+                              uVersion, cbGuess,
+                              pfnLivePrep, pfnLiveExec, pfnLiveVote,
+                              pfnSavePrep, pfnSaveExec, pfnSaveDone,
+                              pfnLoadPrep, pfnLoadExec, pfnLoadDone); */
+    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
+
+    LogFlow(("pdmR3UsbHlp_SSMRegister: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnSTAMRegisterV} */
+static DECLCALLBACK(void) pdmR3UsbHlp_STAMRegisterV(PPDMUSBINS pUsbIns, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility,
+                                                    STAMUNIT enmUnit, const char *pszDesc, const char *pszName, va_list args)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+
+    int rc = STAMR3RegisterV(pVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    AssertRC(rc);
+
+    NOREF(pVM);
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnTMTimerCreate} */
+static DECLCALLBACK(int) pdmR3UsbHlp_TMTimerCreate(PPDMUSBINS pUsbIns, TMCLOCK enmClock, PFNTMTIMERUSB pfnCallback, void *pvUser,
+                                                   uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+    VM_ASSERT_EMT(pVM);
+    LogFlow(("pdmR3UsbHlp_TMTimerCreate: caller='%s'/%d: enmClock=%d pfnCallback=%p pvUser=%p fFlags=%#x pszDesc=%p:{%s} ppTimer=%p\n",
+             pUsbIns->pReg->szName, pUsbIns->iInstance, enmClock, pfnCallback, pvUser, fFlags, pszDesc, pszDesc, ppTimer));
+
+    if (pUsbIns->iInstance > 0) /** @todo use a string cache here later. */
+    {
+         char *pszDesc2 = MMR3HeapAPrintf(pVM, MM_TAG_PDM_USB_DESC, "%s [%u]", pszDesc, pUsbIns->iInstance);
+         if (pszDesc2)
+             pszDesc = pszDesc2;
+    }
+
+    /** @todo
+    int rc = TMR3TimerCreateUsb(pVM, pUsbIns, enmClock, pfnCallback, pvUser, fFlags, pszDesc, ppTimer); */
+    int rc = VERR_NOT_IMPLEMENTED; AssertFailed();
+
+    LogFlow(("pdmR3UsbHlp_TMTimerCreate: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMSetErrorV} */
+static DECLCALLBACK(int) pdmR3UsbHlp_VMSetErrorV(PPDMUSBINS pUsbIns, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    int rc2 = VMSetErrorV(pUsbIns->Internal.s.pVM, rc, RT_SRC_POS_ARGS, pszFormat, va); Assert(rc2 == rc); NOREF(rc2);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMSetRuntimeErrorV} */
+static DECLCALLBACK(int) pdmR3UsbHlp_VMSetRuntimeErrorV(PPDMUSBINS pUsbIns, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    int rc = VMSetRuntimeErrorV(pUsbIns->Internal.s.pVM, fFlags, pszErrorId, pszFormat, va);
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnVMState} */
+static DECLCALLBACK(VMSTATE) pdmR3UsbHlp_VMState(PPDMUSBINS pUsbIns)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+
+    VMSTATE enmVMState = VMR3GetState(pUsbIns->Internal.s.pVM);
+
+    LogFlow(("pdmR3UsbHlp_VMState: caller='%s'/%d: returns %d (%s)\n", pUsbIns->pReg->szName, pUsbIns->iInstance,
+             enmVMState, VMR3GetStateName(enmVMState)));
+    return enmVMState;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnSetAsyncNotification} */
+static DECLCALLBACK(int) pdmR3UsbHlp_SetAsyncNotification(PPDMUSBINS pUsbIns, PFNPDMUSBASYNCNOTIFY pfnAsyncNotify)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    VM_ASSERT_EMT0(pUsbIns->Internal.s.pVM);
+    LogFlow(("pdmR3UsbHlp_SetAsyncNotification: caller='%s'/%d: pfnAsyncNotify=%p\n", pUsbIns->pReg->szName, pUsbIns->iInstance, pfnAsyncNotify));
+
+    int rc = VINF_SUCCESS;
+    AssertStmt(pfnAsyncNotify, rc = VERR_INVALID_PARAMETER);
+    AssertStmt(!pUsbIns->Internal.s.pfnAsyncNotify, rc = VERR_WRONG_ORDER);
+    AssertStmt(pUsbIns->Internal.s.fVMSuspended || pUsbIns->Internal.s.fVMReset, rc = VERR_WRONG_ORDER);
+    VMSTATE enmVMState = VMR3GetState(pUsbIns->Internal.s.pVM);
+    AssertStmt(   enmVMState == VMSTATE_SUSPENDING
+               || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+               || enmVMState == VMSTATE_SUSPENDING_LS
+               || enmVMState == VMSTATE_RESETTING
+               || enmVMState == VMSTATE_RESETTING_LS
+               || enmVMState == VMSTATE_POWERING_OFF
+               || enmVMState == VMSTATE_POWERING_OFF_LS,
+               rc = VERR_INVALID_STATE);
+
+    if (RT_SUCCESS(rc))
+        pUsbIns->Internal.s.pfnAsyncNotify = pfnAsyncNotify;
+
+    LogFlow(("pdmR3UsbHlp_SetAsyncNotification: caller='%s'/%d: returns %Rrc\n", pUsbIns->pReg->szName, pUsbIns->iInstance, rc));
+    return rc;
+}
+
+
+/** @interface_method_impl{PDMUSBHLP,pfnAsyncNotificationCompleted} */
+static DECLCALLBACK(void) pdmR3UsbHlp_AsyncNotificationCompleted(PPDMUSBINS pUsbIns)
+{
+    PDMUSB_ASSERT_USBINS(pUsbIns);
+    PVM pVM = pUsbIns->Internal.s.pVM;
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (   enmVMState == VMSTATE_SUSPENDING
+        || enmVMState == VMSTATE_SUSPENDING_EXT_LS
+        || enmVMState == VMSTATE_SUSPENDING_LS
+        || enmVMState == VMSTATE_RESETTING
+        || enmVMState == VMSTATE_RESETTING_LS
+        || enmVMState == VMSTATE_POWERING_OFF
+        || enmVMState == VMSTATE_POWERING_OFF_LS)
+    {
+        LogFlow(("pdmR3UsbHlp_AsyncNotificationCompleted: caller='%s'/%d:\n", pUsbIns->pReg->szName, pUsbIns->iInstance));
+        VMR3AsyncPdmNotificationWakeupU(pVM->pUVM);
+    }
+    else
+        LogFlow(("pdmR3UsbHlp_AsyncNotificationCompleted: caller='%s'/%d: enmVMState=%d\n", pUsbIns->pReg->szName, pUsbIns->iInstance, enmVMState));
+}
+
+
+/**
+ * The USB device helper structure.
+ */
+const PDMUSBHLP g_pdmR3UsbHlp =
+{
+    PDM_USBHLP_VERSION,
+    pdmR3UsbHlp_DriverAttach,
+    pdmR3UsbHlp_AssertEMT,
+    pdmR3UsbHlp_AssertOther,
+    pdmR3UsbHlp_DBGFStopV,
+    pdmR3UsbHlp_DBGFInfoRegister,
+    pdmR3UsbHlp_MMHeapAlloc,
+    pdmR3UsbHlp_MMHeapAllocZ,
+    pdmR3UsbHlp_PDMQueueCreate,
+    pdmR3UsbHlp_SSMRegister,
+    pdmR3UsbHlp_STAMRegisterV,
+    pdmR3UsbHlp_TMTimerCreate,
+    pdmR3UsbHlp_VMSetErrorV,
+    pdmR3UsbHlp_VMSetRuntimeErrorV,
+    pdmR3UsbHlp_VMState,
+    pdmR3UsbHlp_SetAsyncNotification,
+    pdmR3UsbHlp_AsyncNotificationCompleted,
+    PDM_USBHLP_VERSION
+};
+
+/** @}  */
Index: /trunk/src/VBox/VMM/VMMR3/PGM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGM.cpp	(revision 35333)
@@ -0,0 +1,4011 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager and Monitor. (Mixing stuff here, not good?)
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_pgm PGM - The Page Manager and Monitor
+ *
+ * @see grp_pgm,
+ * @ref pg_pgm_pool,
+ * @ref pg_pgm_phys.
+ *
+ *
+ * @section         sec_pgm_modes           Paging Modes
+ *
+ * There are three memory contexts: Host Context (HC), Guest Context (GC)
+ * and intermediate context. When talking about paging HC can also be referred to
+ * as "host paging", and GC referred to as "shadow paging".
+ *
+ * We define three basic paging modes: 32-bit, PAE and AMD64. The host paging mode
+ * is defined by the host operating system. The mode used in the shadow paging mode
+ * depends on the host paging mode and what the mode the guest is currently in. The
+ * following relation between the two is defined:
+ *
+ * @verbatim
+     Host > 32-bit |  PAE   | AMD64  |
+   Guest  |        |        |        |
+   ==v================================
+   32-bit   32-bit    PAE     PAE
+   -------|--------|--------|--------|
+   PAE       PAE      PAE     PAE
+   -------|--------|--------|--------|
+   AMD64    AMD64    AMD64    AMD64
+   -------|--------|--------|--------| @endverbatim
+ *
+ * All configuration except those in the diagonal (upper left) are expected to
+ * require special effort from the switcher (i.e. a bit slower).
+ *
+ *
+ *
+ *
+ * @section         sec_pgm_shw             The Shadow Memory Context
+ *
+ *
+ *  [..]
+ *
+ * Because of guest context mappings requires PDPT and PML4 entries to allow
+ * writing on AMD64, the two upper levels will have fixed flags whatever the
+ * guest is thinking of using there. So, when shadowing the PD level we will
+ * calculate the effective flags of PD and all the higher levels. In legacy
+ * PAE mode this only applies to the PWT and PCD bits (the rest are
+ * ignored/reserved/MBZ). We will ignore those bits for the present.
+ *
+ *
+ *
+ * @section         sec_pgm_int             The Intermediate Memory Context
+ *
+ * The world switch goes thru an intermediate memory context which purpose it is
+ * to provide different mappings of the switcher code. All guest mappings are also
+ * present in this context.
+ *
+ * The switcher code is mapped at the same location as on the host, at an
+ * identity mapped location (physical equals virtual address), and at the
+ * hypervisor location. The identity mapped location is for when the world
+ * switches that involves disabling paging.
+ *
+ * PGM maintain page tables for 32-bit, PAE and AMD64 paging modes. This
+ * simplifies switching guest CPU mode and consistency at the cost of more
+ * code to do the work. All memory use for those page tables is located below
+ * 4GB (this includes page tables for guest context mappings).
+ *
+ *
+ * @subsection      subsec_pgm_int_gc       Guest Context Mappings
+ *
+ * During assignment and relocation of a guest context mapping the intermediate
+ * memory context is used to verify the new location.
+ *
+ * Guest context mappings are currently restricted to below 4GB, for reasons
+ * of simplicity. This may change when we implement AMD64 support.
+ *
+ *
+ *
+ *
+ * @section         sec_pgm_misc            Misc
+ *
+ * @subsection      subsec_pgm_misc_diff    Differences Between Legacy PAE and Long Mode PAE
+ *
+ * The differences between legacy PAE and long mode PAE are:
+ *      -# PDPE bits 1, 2, 5 and 6 are defined differently. In leagcy mode they are
+ *         all marked down as must-be-zero, while in long mode 1, 2 and 5 have the
+ *         usual meanings while 6 is ignored (AMD). This means that upon switching to
+ *         legacy PAE mode we'll have to clear these bits and when going to long mode
+ *         they must be set. This applies to both intermediate and shadow contexts,
+ *         however we don't need to do it for the intermediate one since we're
+ *         executing with CR0.WP at that time.
+ *      -# CR3 allows a 32-byte aligned address in legacy mode, while in long mode
+ *         a page aligned one is required.
+ *
+ *
+ * @section         sec_pgm_handlers        Access Handlers
+ *
+ * Placeholder.
+ *
+ *
+ * @subsection      sec_pgm_handlers_virt   Virtual Access Handlers
+ *
+ * Placeholder.
+ *
+ *
+ * @subsection      sec_pgm_handlers_virt   Virtual Access Handlers
+ *
+ * We currently implement three types of virtual access handlers:  ALL, WRITE
+ * and HYPERVISOR (WRITE). See PGMVIRTHANDLERTYPE for some more details.
+ *
+ * The HYPERVISOR access handlers is kept in a separate tree since it doesn't apply
+ * to physical pages (PGMTREES::HyperVirtHandlers) and only needs to be consulted in
+ * a special \#PF case. The ALL and WRITE are in the PGMTREES::VirtHandlers tree, the
+ * rest of this section is going to be about these handlers.
+ *
+ * We'll go thru the life cycle of a handler and try make sense of it all, don't know
+ * how successful this is gonna be...
+ *
+ * 1. A handler is registered thru the PGMR3HandlerVirtualRegister and
+ * PGMHandlerVirtualRegisterEx APIs. We check for conflicting virtual handlers
+ * and create a new node that is inserted into the AVL tree (range key). Then
+ * a full PGM resync is flagged (clear pool, sync cr3, update virtual bit of PGMPAGE).
+ *
+ * 2. The following PGMSyncCR3/SyncCR3 operation will first make invoke HandlerVirtualUpdate.
+ *
+ * 2a. HandlerVirtualUpdate will will lookup all the pages covered by virtual handlers
+ * via the current guest CR3 and update the physical page -> virtual handler
+ * translation. Needless to say, this doesn't exactly scale very well. If any changes
+ * are detected, it will flag a virtual bit update just like we did on registration.
+ * PGMPHYS pages with changes will have their virtual handler state reset to NONE.
+ *
+ * 2b. The virtual bit update process will iterate all the pages covered by all the
+ * virtual handlers and update the PGMPAGE virtual handler state to the max of all
+ * virtual handlers on that page.
+ *
+ * 2c. Back in SyncCR3 we will now flush the entire shadow page cache to make sure
+ * we don't miss any alias mappings of the monitored pages.
+ *
+ * 2d. SyncCR3 will then proceed with syncing the CR3 table.
+ *
+ * 3. \#PF(np,read) on a page in the range. This will cause it to be synced
+ * read-only and resumed if it's a WRITE handler. If it's an ALL handler we
+ * will call the handlers like in the next step. If the physical mapping has
+ * changed we will - some time in the future - perform a handler callback
+ * (optional) and update the physical -> virtual handler cache.
+ *
+ * 4. \#PF(,write) on a page in the range. This will cause the handler to
+ * be invoked.
+ *
+ * 5. The guest invalidates the page and changes the physical backing or
+ * unmaps it. This should cause the invalidation callback to be invoked
+ * (it might not yet be 100% perfect). Exactly what happens next... is
+ * this where we mess up and end up out of sync for a while?
+ *
+ * 6. The handler is deregistered by the client via PGMHandlerVirtualDeregister.
+ * We will then set all PGMPAGEs in the physical -> virtual handler cache for
+ * this handler to NONE and trigger a full PGM resync (basically the same
+ * as int step 1). Which means 2 is executed again.
+ *
+ *
+ * @subsubsection   sub_sec_pgm_handler_virt_todo   TODOs
+ *
+ * There is a bunch of things that needs to be done to make the virtual handlers
+ * work 100% correctly and work more efficiently.
+ *
+ * The first bit hasn't been implemented yet because it's going to slow the
+ * whole mess down even more, and besides it seems to be working reliably for
+ * our current uses. OTOH, some of the optimizations might end up more or less
+ * implementing the missing bits, so we'll see.
+ *
+ * On the optimization side, the first thing to do is to try avoid unnecessary
+ * cache flushing. Then try team up with the shadowing code to track changes
+ * in mappings by means of access to them (shadow in), updates to shadows pages,
+ * invlpg, and shadow PT discarding (perhaps).
+ *
+ * Some idea that have popped up for optimization for current and new features:
+ *    - bitmap indicating where there are virtual handlers installed.
+ *      (4KB => 2**20 pages, page 2**12 => covers 32-bit address space 1:1!)
+ *    - Further optimize this by min/max (needs min/max avl getters).
+ *    - Shadow page table entry bit (if any left)?
+ *
+ */
+
+
+/** @page pg_pgm_phys   PGM Physical Guest Memory Management
+ *
+ *
+ * Objectives:
+ *      - Guest RAM over-commitment using memory ballooning,
+ *        zero pages and general page sharing.
+ *      - Moving or mirroring a VM onto a different physical machine.
+ *
+ *
+ * @subsection subsec_pgmPhys_Definitions       Definitions
+ *
+ * Allocation chunk - A RTR0MemObjAllocPhysNC object and the tracking
+ * machinery associated with it.
+ *
+ *
+ *
+ *
+ * @subsection subsec_pgmPhys_AllocPage         Allocating a page.
+ *
+ * Initially we map *all* guest memory to the (per VM) zero page, which
+ * means that none of the read functions will cause pages to be allocated.
+ *
+ * Exception, access bit in page tables that have been shared. This must
+ * be handled, but we must also make sure PGMGst*Modify doesn't make
+ * unnecessary modifications.
+ *
+ * Allocation points:
+ *      - PGMPhysSimpleWriteGCPhys and PGMPhysWrite.
+ *      - Replacing a zero page mapping at \#PF.
+ *      - Replacing a shared page mapping at \#PF.
+ *      - ROM registration (currently MMR3RomRegister).
+ *      - VM restore (pgmR3Load).
+ *
+ * For the first three it would make sense to keep a few pages handy
+ * until we've reached the max memory commitment for the VM.
+ *
+ * For the ROM registration, we know exactly how many pages we need
+ * and will request these from ring-0. For restore, we will save
+ * the number of non-zero pages in the saved state and allocate
+ * them up front. This would allow the ring-0 component to refuse
+ * the request if the isn't sufficient memory available for VM use.
+ *
+ * Btw. for both ROM and restore allocations we won't be requiring
+ * zeroed pages as they are going to be filled instantly.
+ *
+ *
+ * @subsection subsec_pgmPhys_FreePage          Freeing a page
+ *
+ * There are a few points where a page can be freed:
+ *      - After being replaced by the zero page.
+ *      - After being replaced by a shared page.
+ *      - After being ballooned by the guest additions.
+ *      - At reset.
+ *      - At restore.
+ *
+ * When freeing one or more pages they will be returned to the ring-0
+ * component and replaced by the zero page.
+ *
+ * The reasoning for clearing out all the pages on reset is that it will
+ * return us to the exact same state as on power on, and may thereby help
+ * us reduce the memory load on the system. Further it might have a
+ * (temporary) positive influence on memory fragmentation (@see subsec_pgmPhys_Fragmentation).
+ *
+ * On restore, as mention under the allocation topic, pages should be
+ * freed / allocated depending on how many is actually required by the
+ * new VM state. The simplest approach is to do like on reset, and free
+ * all non-ROM pages and then allocate what we need.
+ *
+ * A measure to prevent some fragmentation, would be to let each allocation
+ * chunk have some affinity towards the VM having allocated the most pages
+ * from it. Also, try make sure to allocate from allocation chunks that
+ * are almost full. Admittedly, both these measures might work counter to
+ * our intentions and its probably not worth putting a lot of effort,
+ * cpu time or memory into this.
+ *
+ *
+ * @subsection subsec_pgmPhys_SharePage         Sharing a page
+ *
+ * The basic idea is that there there will be a idle priority kernel
+ * thread walking the non-shared VM pages hashing them and looking for
+ * pages with the same checksum. If such pages are found, it will compare
+ * them byte-by-byte to see if they actually are identical. If found to be
+ * identical it will allocate a shared page, copy the content, check that
+ * the page didn't change while doing this, and finally request both the
+ * VMs to use the shared page instead. If the page is all zeros (special
+ * checksum and byte-by-byte check) it will request the VM that owns it
+ * to replace it with the zero page.
+ *
+ * To make this efficient, we will have to make sure not to try share a page
+ * that will change its contents soon. This part requires the most work.
+ * A simple idea would be to request the VM to write monitor the page for
+ * a while to make sure it isn't modified any time soon. Also, it may
+ * make sense to skip pages that are being write monitored since this
+ * information is readily available to the thread if it works on the
+ * per-VM guest memory structures (presently called PGMRAMRANGE).
+ *
+ *
+ * @subsection subsec_pgmPhys_Fragmentation     Fragmentation Concerns and Counter Measures
+ *
+ * The pages are organized in allocation chunks in ring-0, this is a necessity
+ * if we wish to have an OS agnostic approach to this whole thing. (On Linux we
+ * could easily work on a page-by-page basis if we liked. Whether this is possible
+ * or efficient on NT I don't quite know.) Fragmentation within these chunks may
+ * become a problem as part of the idea here is that we wish to return memory to
+ * the host system.
+ *
+ * For instance, starting two VMs at the same time, they will both allocate the
+ * guest memory on-demand and if permitted their page allocations will be
+ * intermixed. Shut down one of the two VMs and it will be difficult to return
+ * any memory to the host system because the page allocation for the two VMs are
+ * mixed up in the same allocation chunks.
+ *
+ * To further complicate matters, when pages are freed because they have been
+ * ballooned or become shared/zero the whole idea is that the page is supposed
+ * to be reused by another VM or returned to the host system. This will cause
+ * allocation chunks to contain pages belonging to different VMs and prevent
+ * returning memory to the host when one of those VM shuts down.
+ *
+ * The only way to really deal with this problem is to move pages. This can
+ * either be done at VM shutdown and or by the idle priority worker thread
+ * that will be responsible for finding sharable/zero pages. The mechanisms
+ * involved for coercing a VM to move a page (or to do it for it) will be
+ * the same as when telling it to share/zero a page.
+ *
+ *
+ * @subsection subsec_pgmPhys_Tracking      Tracking Structures And Their Cost
+ *
+ * There's a difficult balance between keeping the per-page tracking structures
+ * (global and guest page) easy to use and keeping them from eating too much
+ * memory. We have limited virtual memory resources available when operating in
+ * 32-bit kernel space (on 64-bit there'll it's quite a different story). The
+ * tracking structures will be attempted designed such that we can deal with up
+ * to 32GB of memory on a 32-bit system and essentially unlimited on 64-bit ones.
+ *
+ *
+ * @subsubsection subsubsec_pgmPhys_Tracking_Kernel     Kernel Space
+ *
+ * @see pg_GMM
+ *
+ * @subsubsection subsubsec_pgmPhys_Tracking_PerVM      Per-VM
+ *
+ * Fixed info is the physical address of the page (HCPhys) and the page id
+ * (described above). Theoretically we'll need 48(-12) bits for the HCPhys part.
+ * Today we've restricting ourselves to 40(-12) bits because this is the current
+ * restrictions of all AMD64 implementations (I think Barcelona will up this
+ * to 48(-12) bits, not that it really matters) and I needed the bits for
+ * tracking mappings of a page. 48-12 = 36. That leaves 28 bits, which means a
+ * decent range for the page id: 2^(28+12) = 1024TB.
+ *
+ * In additions to these, we'll have to keep maintaining the page flags as we
+ * currently do. Although it wouldn't harm to optimize these quite a bit, like
+ * for instance the ROM shouldn't depend on having a write handler installed
+ * in order for it to become read-only. A RO/RW bit should be considered so
+ * that the page syncing code doesn't have to mess about checking multiple
+ * flag combinations (ROM || RW handler || write monitored) in order to
+ * figure out how to setup a shadow PTE. But this of course, is second
+ * priority at present. Current this requires 12 bits, but could probably
+ * be optimized to ~8.
+ *
+ * Then there's the 24 bits used to track which shadow page tables are
+ * currently mapping a page for the purpose of speeding up physical
+ * access handlers, and thereby the page pool cache. More bit for this
+ * purpose wouldn't hurt IIRC.
+ *
+ * Then there is a new bit in which we need to record what kind of page
+ * this is, shared, zero, normal or write-monitored-normal. This'll
+ * require 2 bits. One bit might be needed for indicating whether a
+ * write monitored page has been written to. And yet another one or
+ * two for tracking migration status. 3-4 bits total then.
+ *
+ * Whatever is left will can be used to record the sharabilitiy of a
+ * page. The page checksum will not be stored in the per-VM table as
+ * the idle thread will not be permitted to do modifications to it.
+ * It will instead have to keep its own working set of potentially
+ * shareable pages and their check sums and stuff.
+ *
+ * For the present we'll keep the current packing of the
+ * PGMRAMRANGE::aHCPhys to keep the changes simple, only of course,
+ * we'll have to change it to a struct with a total of 128-bits at
+ * our disposal.
+ *
+ * The initial layout will be like this:
+ * @verbatim
+    RTHCPHYS HCPhys;            The current stuff.
+        63:40                   Current shadow PT tracking stuff.
+        39:12                   The physical page frame number.
+        11:0                    The current flags.
+    uint32_t u28PageId : 28;    The page id.
+    uint32_t u2State : 2;       The page state { zero, shared, normal, write monitored }.
+    uint32_t fWrittenTo : 1;    Whether a write monitored page was written to.
+    uint32_t u1Reserved : 1;    Reserved for later.
+    uint32_t u32Reserved;       Reserved for later, mostly sharing stats.
+ @endverbatim
+ *
+ * The final layout will be something like this:
+ * @verbatim
+    RTHCPHYS HCPhys;            The current stuff.
+        63:48                   High page id (12+).
+        47:12                   The physical page frame number.
+        11:0                    Low page id.
+    uint32_t fReadOnly : 1;     Whether it's readonly page (rom or monitored in some way).
+    uint32_t u3Type : 3;        The page type {RESERVED, MMIO, MMIO2, ROM, shadowed ROM, RAM}.
+    uint32_t u2PhysMon : 2;     Physical access handler type {none, read, write, all}.
+    uint32_t u2VirtMon : 2;     Virtual access handler type {none, read, write, all}..
+    uint32_t u2State : 2;       The page state { zero, shared, normal, write monitored }.
+    uint32_t fWrittenTo : 1;    Whether a write monitored page was written to.
+    uint32_t u20Reserved : 20;  Reserved for later, mostly sharing stats.
+    uint32_t u32Tracking;       The shadow PT tracking stuff, roughly.
+ @endverbatim
+ *
+ * Cost wise, this means we'll double the cost for guest memory. There isn't anyway
+ * around that I'm afraid. It means that the cost of dealing out 32GB of memory
+ * to one or more VMs is: (32GB >> PAGE_SHIFT) * 16 bytes, or 128MBs. Or another
+ * example, the VM heap cost when assigning 1GB to a VM will be: 4MB.
+ *
+ * A couple of cost examples for the total cost per-VM + kernel.
+ * 32-bit Windows and 32-bit linux:
+ *      1GB guest ram, 256K pages:  4MB +  2MB(+) =   6MB
+ *      4GB guest ram, 1M pages:   16MB +  8MB(+) =  24MB
+ *     32GB guest ram, 8M pages:  128MB + 64MB(+) = 192MB
+ * 64-bit Windows and 64-bit linux:
+ *      1GB guest ram, 256K pages:  4MB +  3MB(+) =   7MB
+ *      4GB guest ram, 1M pages:   16MB + 12MB(+) =  28MB
+ *     32GB guest ram, 8M pages:  128MB + 96MB(+) = 224MB
+ *
+ * UPDATE - 2007-09-27:
+ * Will need a ballooned flag/state too because we cannot
+ * trust the guest 100% and reporting the same page as ballooned more
+ * than once will put the GMM off balance.
+ *
+ *
+ * @subsection subsec_pgmPhys_Serializing       Serializing Access
+ *
+ * Initially, we'll try a simple scheme:
+ *
+ *      - The per-VM RAM tracking structures (PGMRAMRANGE) is only modified
+ *        by the EMT thread of that VM while in the pgm critsect.
+ *      - Other threads in the VM process that needs to make reliable use of
+ *        the per-VM RAM tracking structures will enter the critsect.
+ *      - No process external thread or kernel thread will ever try enter
+ *        the pgm critical section, as that just won't work.
+ *      - The idle thread (and similar threads) doesn't not need 100% reliable
+ *        data when performing it tasks as the EMT thread will be the one to
+ *        do the actual changes later anyway. So, as long as it only accesses
+ *        the main ram range, it can do so by somehow preventing the VM from
+ *        being destroyed while it works on it...
+ *
+ *      - The over-commitment management, including the allocating/freeing
+ *        chunks, is serialized by a ring-0 mutex lock (a fast one since the
+ *        more mundane mutex implementation is broken on Linux).
+ *      - A separate mutex is protecting the set of allocation chunks so
+ *        that pages can be shared or/and freed up while some other VM is
+ *        allocating more chunks. This mutex can be take from under the other
+ *        one, but not the other way around.
+ *
+ *
+ * @subsection subsec_pgmPhys_Request           VM Request interface
+ *
+ * When in ring-0 it will become necessary to send requests to a VM so it can
+ * for instance move a page while defragmenting during VM destroy. The idle
+ * thread will make use of this interface to request VMs to setup shared
+ * pages and to perform write monitoring of pages.
+ *
+ * I would propose an interface similar to the current VMReq interface, similar
+ * in that it doesn't require locking and that the one sending the request may
+ * wait for completion if it wishes to. This shouldn't be very difficult to
+ * realize.
+ *
+ * The requests themselves are also pretty simple. They are basically:
+ *      -# Check that some precondition is still true.
+ *      -# Do the update.
+ *      -# Update all shadow page tables involved with the page.
+ *
+ * The 3rd step is identical to what we're already doing when updating a
+ * physical handler, see pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs.
+ *
+ *
+ *
+ * @section sec_pgmPhys_MappingCaches   Mapping Caches
+ *
+ * In order to be able to map in and out memory and to be able to support
+ * guest with more RAM than we've got virtual address space, we'll employing
+ * a mapping cache.  Normally ring-0 and ring-3 can share the same cache,
+ * however on 32-bit darwin the ring-0 code is running in a different memory
+ * context and therefore needs a separate cache.  In raw-mode context we also
+ * need a separate cache.  The 32-bit darwin mapping cache and the one for
+ * raw-mode context share a lot of code, see PGMRZDYNMAP.
+ *
+ *
+ * @subsection subsec_pgmPhys_MappingCaches_R3  Ring-3
+ *
+ * We've considered implementing the ring-3 mapping cache page based but found
+ * that this was bother some when one had to take into account TLBs+SMP and
+ * portability (missing the necessary APIs on several platforms). There were
+ * also some performance concerns with this approach which hadn't quite been
+ * worked out.
+ *
+ * Instead, we'll be mapping allocation chunks into the VM process. This simplifies
+ * matters greatly quite a bit since we don't need to invent any new ring-0 stuff,
+ * only some minor RTR0MEMOBJ mapping stuff. The main concern here is that mapping
+ * compared to the previous idea is that mapping or unmapping a 1MB chunk is more
+ * costly than a single page, although how much more costly is uncertain. We'll
+ * try address this by using a very big cache, preferably bigger than the actual
+ * VM RAM size if possible. The current VM RAM sizes should give some idea for
+ * 32-bit boxes, while on 64-bit we can probably get away with employing an
+ * unlimited cache.
+ *
+ * The cache have to parts, as already indicated, the ring-3 side and the
+ * ring-0 side.
+ *
+ * The ring-0 will be tied to the page allocator since it will operate on the
+ * memory objects it contains. It will therefore require the first ring-0 mutex
+ * discussed in @ref subsec_pgmPhys_Serializing. We
+ * some double house keeping wrt to who has mapped what I think, since both
+ * VMMR0.r0 and RTR0MemObj will keep track of mapping relations
+ *
+ * The ring-3 part will be protected by the pgm critsect. For simplicity, we'll
+ * require anyone that desires to do changes to the mapping cache to do that
+ * from within this critsect. Alternatively, we could employ a separate critsect
+ * for serializing changes to the mapping cache as this would reduce potential
+ * contention with other threads accessing mappings unrelated to the changes
+ * that are in process. We can see about this later, contention will show
+ * up in the statistics anyway, so it'll be simple to tell.
+ *
+ * The organization of the ring-3 part will be very much like how the allocation
+ * chunks are organized in ring-0, that is in an AVL tree by chunk id. To avoid
+ * having to walk the tree all the time, we'll have a couple of lookaside entries
+ * like in we do for I/O ports and MMIO in IOM.
+ *
+ * The simplified flow of a PGMPhysRead/Write function:
+ *      -# Enter the PGM critsect.
+ *      -# Lookup GCPhys in the ram ranges and get the Page ID.
+ *      -# Calc the Allocation Chunk ID from the Page ID.
+ *      -# Check the lookaside entries and then the AVL tree for the Chunk ID.
+ *         If not found in cache:
+ *              -# Call ring-0 and request it to be mapped and supply
+ *                 a chunk to be unmapped if the cache is maxed out already.
+ *              -# Insert the new mapping into the AVL tree (id + R3 address).
+ *      -# Update the relevant lookaside entry and return the mapping address.
+ *      -# Do the read/write according to monitoring flags and everything.
+ *      -# Leave the critsect.
+ *
+ *
+ * @section sec_pgmPhys_Fallback            Fallback
+ *
+ * Current all the "second tier" hosts will not support the RTR0MemObjAllocPhysNC
+ * API and thus require a fallback.
+ *
+ * So, when RTR0MemObjAllocPhysNC returns VERR_NOT_SUPPORTED the page allocator
+ * will return to the ring-3 caller (and later ring-0) and asking it to seed
+ * the page allocator with some fresh pages (VERR_GMM_SEED_ME). Ring-3 will
+ * then perform an SUPR3PageAlloc(cbChunk >> PAGE_SHIFT) call and make a
+ * "SeededAllocPages" call to ring-0.
+ *
+ * The first time ring-0 sees the VERR_NOT_SUPPORTED failure it will disable
+ * all page sharing (zero page detection will continue). It will also force
+ * all allocations to come from the VM which seeded the page. Both these
+ * measures are taken to make sure that there will never be any need for
+ * mapping anything into ring-3 - everything will be mapped already.
+ *
+ * Whether we'll continue to use the current MM locked memory management
+ * for this I don't quite know (I'd prefer not to and just ditch that all
+ * together), we'll see what's simplest to do.
+ *
+ *
+ *
+ * @section sec_pgmPhys_Changes             Changes
+ *
+ * Breakdown of the changes involved?
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/cpum.h>
+#include <VBox/iom.h>
+#include <VBox/sup.h>
+#include <VBox/mm.h>
+#include <VBox/em.h>
+#include <VBox/stam.h>
+#include <VBox/rem.h>
+#include <VBox/selm.h>
+#include <VBox/ssm.h>
+#include <VBox/hwaccm.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include "PGMInline.h"
+
+#include <VBox/dbg.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#include <iprt/env.h>
+#include <iprt/mem.h>
+#include <iprt/file.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int                pgmR3InitPaging(PVM pVM);
+static int                pgmR3InitStats(PVM pVM);
+static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(int)  pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int)  pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int)  pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser);
+#ifdef VBOX_STRICT
+static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser);
+#endif
+static int                pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0);
+static void               pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst);
+static PGMMODE            pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher);
+
+#ifdef VBOX_WITH_DEBUGGER
+/** @todo Convert the first two commands to 'info' items. */
+static DECLCALLBACK(int)  pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+static DECLCALLBACK(int)  pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+static DECLCALLBACK(int)  pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+static DECLCALLBACK(int)  pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+# ifdef VBOX_STRICT
+static DECLCALLBACK(int)  pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+# endif
+static DECLCALLBACK(int)  pgmR3CmdPhysToFile(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+#endif
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+/** Argument descriptors for '.pgmerror' and '.pgmerroroff'. */
+static const DBGCVARDESC g_aPgmErrorArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "where",        "Error injection location." },
+};
+
+static const DBGCVARDESC g_aPgmPhysToFileArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "file",         "The file name." },
+    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "nozero",       "If present, zero pages are skipped." },
+};
+
+# ifdef DEBUG_sandervl
+static const DBGCVARDESC g_aPgmCountPhysWritesArgs[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  1,           1,          DBGCVAR_CAT_STRING,     0,                              "enabled",      "on/off." },
+    {  1,           1,          DBGCVAR_CAT_NUMBER_NO_RANGE, 0,                         "interval",     "Interval in ms." },
+};
+# endif
+
+/** Command descriptors. */
+static const DBGCCMD    g_aCmds[] =
+{
+    /* pszCmd,  cArgsMin, cArgsMax, paArgDesc,                cArgDescs,    pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
+    { "pgmram",        0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdRam,        "",                     "Display the ram ranges." },
+    { "pgmsync",       0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdSync,       "",                     "Sync the CR3 page." },
+    { "pgmerror",      0, 1,        &g_aPgmErrorArgs[0],      1,            NULL,               0,          pgmR3CmdError,      "",                     "Enables inject runtime of errors into parts of PGM." },
+    { "pgmerroroff",   0, 1,        &g_aPgmErrorArgs[0],      1,            NULL,               0,          pgmR3CmdError,      "",                     "Disables inject runtime errors into parts of PGM." },
+# ifdef VBOX_STRICT
+    { "pgmassertcr3",  0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdAssertCR3,  "",                     "Check the shadow CR3 mapping." },
+#  if HC_ARCH_BITS == 64
+    { "pgmcheckduppages", 0, 0,     NULL,                     0,            NULL,               0,          pgmR3CmdCheckDuplicatePages,  "",           "Check for duplicate pages in all running VMs." },
+    { "pgmsharedmodules", 0, 0,     NULL,                     0,            NULL,               0,          pgmR3CmdShowSharedModules,  "",             "Print shared modules info." },
+#  endif
+# endif
+    { "pgmsyncalways", 0, 0,        NULL,                     0,            NULL,               0,          pgmR3CmdSyncAlways, "",                     "Toggle permanent CR3 syncing." },
+    { "pgmphystofile", 1, 2,        &g_aPgmPhysToFileArgs[0], 2,            NULL,               0,          pgmR3CmdPhysToFile, "",                     "Save the physical memory to file." },
+};
+#endif
+
+
+
+
+/*
+ * Shadow - 32-bit mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_32BIT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_32BIT(name)
+#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_32BIT_STR(name)
+#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_32BIT_STR(name)
+#include "PGMShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_REAL(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_32BIT_REAL_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_32BIT_REAL_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
+#include "PGMBth.h"
+#include "PGMGstDefs.h"
+#include "PGMGst.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_PROT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_32BIT_PROT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_32BIT_PROT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
+#include "PGMBth.h"
+#include "PGMGstDefs.h"
+#include "PGMGst.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_32BIT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_32BIT_32BIT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_32BIT_32BIT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD
+#include "PGMBth.h"
+#include "PGMGstDefs.h"
+#include "PGMGst.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+#undef PGM_SHW_NAME_RC_STR
+#undef PGM_SHW_NAME_R0_STR
+
+
+/*
+ * Shadow - PAE mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_PAE
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_PAE(name)
+#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_PAE_STR(name)
+#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_PAE_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
+#include "PGMShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_REAL_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_REAL_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PROT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_PROT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_PROT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_32BIT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_32BIT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_32BIT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_FOR_32BIT
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PAE_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PAE_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PAE(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_PAE_PAE_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_PAE_PAE_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT
+#include "PGMBth.h"
+#include "PGMGstDefs.h"
+#include "PGMGst.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+#undef PGM_SHW_NAME_RC_STR
+#undef PGM_SHW_NAME_R0_STR
+
+
+/*
+ * Shadow - AMD64 mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_AMD64
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_AMD64(name)
+#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_AMD64_STR(name)
+#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_AMD64_STR(name)
+#include "PGMShw.h"
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_GST_NAME_RC_STR(name)  PGM_GST_NAME_RC_AMD64_STR(name)
+# define PGM_GST_NAME_R0_STR(name)  PGM_GST_NAME_R0_AMD64_STR(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_AMD64_AMD64(name)
+# define PGM_BTH_NAME_RC_STR(name)  PGM_BTH_NAME_RC_AMD64_AMD64_STR(name)
+# define PGM_BTH_NAME_R0_STR(name)  PGM_BTH_NAME_R0_AMD64_AMD64_STR(name)
+# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+# define BTH_PGMPOOLKIND_ROOT       PGMPOOLKIND_64BIT_PML4
+# include "PGMBth.h"
+# include "PGMGstDefs.h"
+# include "PGMGst.h"
+# undef BTH_PGMPOOLKIND_PT_FOR_BIG
+# undef BTH_PGMPOOLKIND_PT_FOR_PT
+# undef BTH_PGMPOOLKIND_ROOT
+# undef PGM_BTH_NAME
+# undef PGM_BTH_NAME_RC_STR
+# undef PGM_BTH_NAME_R0_STR
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+# undef PGM_GST_NAME_RC_STR
+# undef PGM_GST_NAME_R0_STR
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+#undef PGM_SHW_NAME_RC_STR
+#undef PGM_SHW_NAME_R0_STR
+
+
+/*
+ * Shadow - Nested paging mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NESTED
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED(name)
+#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_NESTED_STR(name)
+#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_NESTED_STR(name)
+#include "PGMShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_REAL(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_REAL_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_REAL_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PROT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_PROT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_PROT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_32BIT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_32BIT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PAE_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PAE_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_NESTED_PAE_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_NESTED_PAE_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_GST_NAME_RC_STR(name)  PGM_GST_NAME_RC_AMD64_STR(name)
+# define PGM_GST_NAME_R0_STR(name)  PGM_GST_NAME_R0_AMD64_STR(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_AMD64(name)
+# define PGM_BTH_NAME_RC_STR(name)  PGM_BTH_NAME_RC_NESTED_AMD64_STR(name)
+# define PGM_BTH_NAME_R0_STR(name)  PGM_BTH_NAME_R0_NESTED_AMD64_STR(name)
+# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+# include "PGMGstDefs.h"
+# include "PGMBth.h"
+# undef BTH_PGMPOOLKIND_PT_FOR_BIG
+# undef BTH_PGMPOOLKIND_PT_FOR_PT
+# undef PGM_BTH_NAME
+# undef PGM_BTH_NAME_RC_STR
+# undef PGM_BTH_NAME_R0_STR
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+# undef PGM_GST_NAME_RC_STR
+# undef PGM_GST_NAME_R0_STR
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+#undef PGM_SHW_NAME_RC_STR
+#undef PGM_SHW_NAME_R0_STR
+
+
+/*
+ * Shadow - EPT
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_EPT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_EPT(name)
+#define PGM_SHW_NAME_RC_STR(name)   PGM_SHW_NAME_RC_EPT_STR(name)
+#define PGM_SHW_NAME_R0_STR(name)   PGM_SHW_NAME_R0_EPT_STR(name)
+#include "PGMShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_REAL_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_REAL_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_REAL(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_REAL_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_REAL_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PROT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PROT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PROT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_PROT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_PROT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_32BIT_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_32BIT_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_32BIT(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_32BIT_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_32BIT_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_GST_NAME_RC_STR(name)   PGM_GST_NAME_RC_PAE_STR(name)
+#define PGM_GST_NAME_R0_STR(name)   PGM_GST_NAME_R0_PAE_STR(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PAE(name)
+#define PGM_BTH_NAME_RC_STR(name)   PGM_BTH_NAME_RC_EPT_PAE_STR(name)
+#define PGM_BTH_NAME_R0_STR(name)   PGM_BTH_NAME_R0_EPT_PAE_STR(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+#include "PGMGstDefs.h"
+#include "PGMBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_BTH_NAME_RC_STR
+#undef PGM_BTH_NAME_R0_STR
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+#undef PGM_GST_NAME_RC_STR
+#undef PGM_GST_NAME_R0_STR
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_GST_NAME_RC_STR(name)  PGM_GST_NAME_RC_AMD64_STR(name)
+# define PGM_GST_NAME_R0_STR(name)  PGM_GST_NAME_R0_AMD64_STR(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_EPT_AMD64(name)
+# define PGM_BTH_NAME_RC_STR(name)  PGM_BTH_NAME_RC_EPT_AMD64_STR(name)
+# define PGM_BTH_NAME_R0_STR(name)  PGM_BTH_NAME_R0_EPT_AMD64_STR(name)
+# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+# include "PGMGstDefs.h"
+# include "PGMBth.h"
+# undef BTH_PGMPOOLKIND_PT_FOR_BIG
+# undef BTH_PGMPOOLKIND_PT_FOR_PT
+# undef PGM_BTH_NAME
+# undef PGM_BTH_NAME_RC_STR
+# undef PGM_BTH_NAME_R0_STR
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+# undef PGM_GST_NAME_RC_STR
+# undef PGM_GST_NAME_R0_STR
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+#undef PGM_SHW_NAME_RC_STR
+#undef PGM_SHW_NAME_R0_STR
+
+
+
+/**
+ * Initiates the paging of VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to VM structure.
+ */
+VMMR3DECL(int) PGMR3Init(PVM pVM)
+{
+    LogFlow(("PGMR3Init:\n"));
+    PCFGMNODE pCfgPGM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM");
+    int rc;
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompile(sizeof(pVM->pgm.s) <= sizeof(pVM->pgm.padding));
+    AssertCompile(sizeof(pVM->aCpus[0].pgm.s) <= sizeof(pVM->aCpus[0].pgm.padding));
+    AssertCompileMemberAlignment(PGM, CritSect, sizeof(uintptr_t));
+
+    /*
+     * Init the structure.
+     */
+#ifdef PGM_WITHOUT_MAPPINGS
+    pVM->pgm.s.fMappingsDisabled = true;
+#endif
+    pVM->pgm.s.offVM       = RT_OFFSETOF(VM, pgm.s);
+    pVM->pgm.s.offVCpuPGM  = RT_OFFSETOF(VMCPU, pgm.s);
+
+    /* Init the per-CPU part. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[idCpu];
+        PPGMCPU pPGM = &pVCpu->pgm.s;
+
+        pPGM->offVM      = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)pVM;
+        pPGM->offVCpu    = RT_OFFSETOF(VMCPU, pgm.s);
+        pPGM->offPGM     = (uintptr_t)&pVCpu->pgm.s - (uintptr_t)&pVM->pgm.s;
+
+        pPGM->enmShadowMode    = PGMMODE_INVALID;
+        pPGM->enmGuestMode     = PGMMODE_INVALID;
+
+        pPGM->GCPhysCR3        = NIL_RTGCPHYS;
+
+        pPGM->pGst32BitPdR3    = NULL;
+        pPGM->pGstPaePdptR3    = NULL;
+        pPGM->pGstAmd64Pml4R3  = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+        pPGM->pGst32BitPdR0    = NIL_RTR0PTR;
+        pPGM->pGstPaePdptR0    = NIL_RTR0PTR;
+        pPGM->pGstAmd64Pml4R0  = NIL_RTR0PTR;
+#endif
+        pPGM->pGst32BitPdRC    = NIL_RTRCPTR;
+        pPGM->pGstPaePdptRC    = NIL_RTRCPTR;
+        for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.apGstPaePDsR3); i++)
+        {
+            pPGM->apGstPaePDsR3[i]             = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+            pPGM->apGstPaePDsR0[i]             = NIL_RTR0PTR;
+#endif
+            pPGM->apGstPaePDsRC[i]             = NIL_RTRCPTR;
+            pPGM->aGCPhysGstPaePDs[i]          = NIL_RTGCPHYS;
+            pPGM->aGCPhysGstPaePDsMonitored[i] = NIL_RTGCPHYS;
+        }
+
+        pPGM->fA20Enabled      = true;
+    }
+
+    pVM->pgm.s.enmHostMode      = SUPPAGINGMODE_INVALID;
+    pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1; /* default; checked later */
+    pVM->pgm.s.GCPtrPrevRamRangeMapping = MM_HYPER_AREA_ADDRESS;
+
+    rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "RamPreAlloc", &pVM->pgm.s.fRamPreAlloc,
+#ifdef VBOX_WITH_PREALLOC_RAM_BY_DEFAULT
+                            true
+#else
+                            false
+#endif
+                           );
+    AssertLogRelRCReturn(rc, rc);
+
+#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
+    rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, _1G / GMM_CHUNK_SIZE);
+#else
+    rc = CFGMR3QueryU32Def(pCfgPGM, "MaxRing3Chunks", &pVM->pgm.s.ChunkR3Map.cMax, UINT32_MAX);
+#endif
+    AssertLogRelRCReturn(rc, rc);
+    for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
+        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
+
+    /*
+     * Get the configured RAM size - to estimate saved state size.
+     */
+    uint64_t    cbRam;
+    rc = CFGMR3QueryU64(CFGMR3GetRoot(pVM), "RamSize", &cbRam);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        cbRam = 0;
+    else if (RT_SUCCESS(rc))
+    {
+        if (cbRam < PAGE_SIZE)
+            cbRam = 0;
+        cbRam = RT_ALIGN_64(cbRam, PAGE_SIZE);
+    }
+    else
+    {
+        AssertMsgFailed(("Configuration error: Failed to query integer \"RamSize\", rc=%Rrc.\n", rc));
+        return rc;
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Allocate memory for the statistics before someone tries to use them.
+     */
+    size_t cbTotalStats = RT_ALIGN_Z(sizeof(PGMSTATS), 64) + RT_ALIGN_Z(sizeof(PGMCPUSTATS), 64) * pVM->cCpus;
+    void *pv;
+    rc = MMHyperAlloc(pVM, RT_ALIGN_Z(cbTotalStats, PAGE_SIZE), PAGE_SIZE, MM_TAG_PGM, &pv);
+    AssertRCReturn(rc, rc);
+
+    pVM->pgm.s.pStatsR3 = (PGMSTATS *)pv;
+    pVM->pgm.s.pStatsR0 = MMHyperCCToR0(pVM, pv);
+    pVM->pgm.s.pStatsRC = MMHyperCCToRC(pVM, pv);
+    pv = (uint8_t *)pv + RT_ALIGN_Z(sizeof(PGMSTATS), 64);
+
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+    {
+        pVM->aCpus[iCpu].pgm.s.pStatsR3 = (PGMCPUSTATS *)pv;
+        pVM->aCpus[iCpu].pgm.s.pStatsR0 = MMHyperCCToR0(pVM, pv);
+        pVM->aCpus[iCpu].pgm.s.pStatsRC = MMHyperCCToRC(pVM, pv);
+
+        pv = (uint8_t *)pv + RT_ALIGN_Z(sizeof(PGMCPUSTATS), 64);
+    }
+#endif /* VBOX_WITH_STATISTICS */
+
+    /*
+     * Register callbacks, string formatters and the saved state data unit.
+     */
+#ifdef VBOX_STRICT
+    VMR3AtStateRegister(pVM, pgmR3ResetNoMorePhysWritesFlag, NULL);
+#endif
+    PGMRegisterStringFormatTypes();
+
+    rc = pgmR3InitSavedState(pVM, cbRam);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize the PGM critical section and flush the phys TLBs
+     */
+    rc = PDMR3CritSectInit(pVM, &pVM->pgm.s.CritSect, RT_SRC_POS, "PGM");
+    AssertRCReturn(rc, rc);
+
+    PGMR3PhysChunkInvalidateTLB(pVM);
+    PGMPhysInvalidatePageMapTLB(pVM);
+
+    /*
+     * For the time being we sport a full set of handy pages in addition to the base
+     * memory to simplify things.
+     */
+    rc = MMR3ReserveHandyPages(pVM, RT_ELEMENTS(pVM->pgm.s.aHandyPages)); /** @todo this should be changed to PGM_HANDY_PAGES_MIN but this needs proper testing... */
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Trees
+     */
+    rc = MMHyperAlloc(pVM, sizeof(PGMTREES), 0, MM_TAG_PGM, (void **)&pVM->pgm.s.pTreesR3);
+    if (RT_SUCCESS(rc))
+    {
+        pVM->pgm.s.pTreesR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pTreesR3);
+        pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
+
+        /*
+         * Allocate the zero page.
+         */
+        rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvZeroPgR3);
+    }
+    if (RT_SUCCESS(rc))
+    {
+        pVM->pgm.s.pvZeroPgRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pvZeroPgR3);
+        pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
+        pVM->pgm.s.HCPhysZeroPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvZeroPgR3);
+        AssertRelease(pVM->pgm.s.HCPhysZeroPg != NIL_RTHCPHYS);
+
+        /*
+         * Allocate the invalid MMIO page.
+         * (The invalid bits in HCPhysInvMmioPg are set later on init complete.)
+         */
+        rc = MMHyperAlloc(pVM, PAGE_SIZE, PAGE_SIZE, MM_TAG_PGM, &pVM->pgm.s.pvMmioPgR3);
+    }
+    if (RT_SUCCESS(rc))
+    {
+        ASMMemFill32(pVM->pgm.s.pvMmioPgR3, PAGE_SIZE, 0xfeedface);
+        pVM->pgm.s.HCPhysMmioPg = MMR3HyperHCVirt2HCPhys(pVM, pVM->pgm.s.pvMmioPgR3);
+        AssertRelease(pVM->pgm.s.HCPhysMmioPg != NIL_RTHCPHYS);
+        pVM->pgm.s.HCPhysInvMmioPg = pVM->pgm.s.HCPhysMmioPg;
+
+        /*
+         * Init the paging.
+         */
+        rc = pgmR3InitPaging(pVM);
+    }
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Init the page pool.
+         */
+        rc = pgmR3PoolInit(pVM);
+    }
+    if (RT_SUCCESS(rc))
+    {
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PVMCPU pVCpu = &pVM->aCpus[i];
+            rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
+            if (RT_FAILURE(rc))
+                break;
+        }
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Info & statistics
+         */
+        DBGFR3InfoRegisterInternal(pVM, "mode",
+                                   "Shows the current paging mode. "
+                                   "Recognizes 'all', 'guest', 'shadow' and 'host' as arguments, defaulting to 'all' if nothing is given.",
+                                   pgmR3InfoMode);
+        DBGFR3InfoRegisterInternal(pVM, "pgmcr3",
+                                   "Dumps all the entries in the top level paging table. No arguments.",
+                                   pgmR3InfoCr3);
+        DBGFR3InfoRegisterInternal(pVM, "phys",
+                                   "Dumps all the physical address ranges. No arguments.",
+                                   pgmR3PhysInfo);
+        DBGFR3InfoRegisterInternal(pVM, "handlers",
+                                   "Dumps physical, virtual and hyper virtual handlers. "
+                                   "Pass 'phys', 'virt', 'hyper' as argument if only one kind is wanted."
+                                   "Add 'nost' if the statistics are unwanted, use together with 'all' or explicit selection.",
+                                   pgmR3InfoHandlers);
+        DBGFR3InfoRegisterInternal(pVM, "mappings",
+                                   "Dumps guest mappings.",
+                                   pgmR3MapInfo);
+
+        pgmR3InitStats(pVM);
+
+#ifdef VBOX_WITH_DEBUGGER
+        /*
+         * Debugger commands.
+         */
+        static bool s_fRegisteredCmds = false;
+        if (!s_fRegisteredCmds)
+        {
+            int rc2 = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+            if (RT_SUCCESS(rc2))
+                s_fRegisteredCmds = true;
+        }
+#endif
+        return VINF_SUCCESS;
+    }
+
+    /* Almost no cleanup necessary, MM frees all memory. */
+    PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
+
+    return rc;
+}
+
+
+/**
+ * Init paging.
+ *
+ * Since we need to check what mode the host is operating in before we can choose
+ * the right paging functions for the host we have to delay this until R0 has
+ * been initialized.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle.
+ */
+static int pgmR3InitPaging(PVM pVM)
+{
+    /*
+     * Force a recalculation of modes and switcher so everyone gets notified.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
+        pVCpu->pgm.s.enmGuestMode  = PGMMODE_INVALID;
+    }
+
+    pVM->pgm.s.enmHostMode   = SUPPAGINGMODE_INVALID;
+
+    /*
+     * Allocate static mapping space for whatever the cr3 register
+     * points to and in the case of PAE mode to the 4 PDs.
+     */
+    int rc = MMR3HyperReserve(pVM, PAGE_SIZE * 5, "CR3 mapping", &pVM->pgm.s.GCPtrCR3Mapping);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to reserve two pages for cr mapping in HMA, rc=%Rrc\n", rc));
+        return rc;
+    }
+    MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+
+    /*
+     * Allocate pages for the three possible intermediate contexts
+     * (AMD64, PAE and plain 32-Bit). We maintain all three contexts
+     * for the sake of simplicity. The AMD64 uses the PAE for the
+     * lower levels, making the total number of pages 11 (3 + 7 + 1).
+     *
+     * We assume that two page tables will be enought for the core code
+     * mappings (HC virtual and identity).
+     */
+    pVM->pgm.s.pInterPD         = (PX86PD)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.pInterPD,         VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPTs[0]    = (PX86PT)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.apInterPTs[0],    VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPTs[1]    = (PX86PT)MMR3PageAllocLow(pVM);    AssertReturn(pVM->pgm.s.apInterPTs[1],    VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePTs[0] = (PX86PTPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePTs[0], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePTs[1] = (PX86PTPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePTs[1], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[0] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[0], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[1] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[1], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[2] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[2], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.apInterPaePDs[3] = (PX86PDPAE)MMR3PageAlloc(pVM);    AssertReturn(pVM->pgm.s.apInterPaePDs[3], VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.pInterPaePDPT    = (PX86PDPT)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePDPT,    VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.pInterPaePDPT64  = (PX86PDPT)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePDPT64,  VERR_NO_PAGE_MEMORY);
+    pVM->pgm.s.pInterPaePML4    = (PX86PML4)MMR3PageAllocLow(pVM);  AssertReturn(pVM->pgm.s.pInterPaePML4,    VERR_NO_PAGE_MEMORY);
+
+    pVM->pgm.s.HCPhysInterPD = MMPage2Phys(pVM, pVM->pgm.s.pInterPD);
+    AssertRelease(pVM->pgm.s.HCPhysInterPD != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPD & PAGE_OFFSET_MASK));
+    pVM->pgm.s.HCPhysInterPaePDPT = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT);
+    AssertRelease(pVM->pgm.s.HCPhysInterPaePDPT != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePDPT & PAGE_OFFSET_MASK));
+    pVM->pgm.s.HCPhysInterPaePML4 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePML4);
+    AssertRelease(pVM->pgm.s.HCPhysInterPaePML4 != NIL_RTHCPHYS && !(pVM->pgm.s.HCPhysInterPaePML4 & PAGE_OFFSET_MASK) && pVM->pgm.s.HCPhysInterPaePML4 < 0xffffffff);
+
+    /*
+     * Initialize the pages, setting up the PML4 and PDPT for repetitive 4GB action.
+     */
+    ASMMemZeroPage(pVM->pgm.s.pInterPD);
+    ASMMemZeroPage(pVM->pgm.s.apInterPTs[0]);
+    ASMMemZeroPage(pVM->pgm.s.apInterPTs[1]);
+
+    ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[0]);
+    ASMMemZeroPage(pVM->pgm.s.apInterPaePTs[1]);
+
+    ASMMemZeroPage(pVM->pgm.s.pInterPaePDPT);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.apInterPaePDs); i++)
+    {
+        ASMMemZeroPage(pVM->pgm.s.apInterPaePDs[i]);
+        pVM->pgm.s.pInterPaePDPT->a[i].u = X86_PDPE_P | PGM_PLXFLAGS_PERMANENT
+                                          | MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[i]);
+    }
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePDPT64->a); i++)
+    {
+        const unsigned iPD = i % RT_ELEMENTS(pVM->pgm.s.apInterPaePDs);
+        pVM->pgm.s.pInterPaePDPT64->a[i].u = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A | PGM_PLXFLAGS_PERMANENT
+                                            | MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[iPD]);
+    }
+
+    RTHCPHYS HCPhysInterPaePDPT64 = MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.pInterPaePML4->a); i++)
+        pVM->pgm.s.pInterPaePML4->a[i].u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A | PGM_PLXFLAGS_PERMANENT
+                                         | HCPhysInterPaePDPT64;
+
+    /*
+     * Initialize paging workers and mode from current host mode
+     * and the guest running in real mode.
+     */
+    pVM->pgm.s.enmHostMode = SUPR3GetPagingMode();
+    switch (pVM->pgm.s.enmHostMode)
+    {
+        case SUPPAGINGMODE_32_BIT:
+        case SUPPAGINGMODE_32_BIT_GLOBAL:
+        case SUPPAGINGMODE_PAE:
+        case SUPPAGINGMODE_PAE_GLOBAL:
+        case SUPPAGINGMODE_PAE_NX:
+        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+            break;
+
+        case SUPPAGINGMODE_AMD64:
+        case SUPPAGINGMODE_AMD64_GLOBAL:
+        case SUPPAGINGMODE_AMD64_NX:
+        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+#ifndef VBOX_WITH_HYBRID_32BIT_KERNEL
+            if (ARCH_BITS != 64)
+            {
+                AssertMsgFailed(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
+                LogRel(("Host mode %d (64-bit) is not supported by non-64bit builds\n", pVM->pgm.s.enmHostMode));
+                return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
+            }
+#endif
+            break;
+        default:
+            AssertMsgFailed(("Host mode %d is not supported\n", pVM->pgm.s.enmHostMode));
+            return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
+    }
+    rc = pgmR3ModeDataInit(pVM, false /* don't resolve GC and R0 syms yet */);
+    if (RT_SUCCESS(rc))
+    {
+        LogFlow(("pgmR3InitPaging: returns successfully\n"));
+#if HC_ARCH_BITS == 64
+        LogRel(("Debug: HCPhysInterPD=%RHp HCPhysInterPaePDPT=%RHp HCPhysInterPaePML4=%RHp\n",
+                pVM->pgm.s.HCPhysInterPD, pVM->pgm.s.HCPhysInterPaePDPT, pVM->pgm.s.HCPhysInterPaePML4));
+        LogRel(("Debug: apInterPTs={%RHp,%RHp} apInterPaePTs={%RHp,%RHp} apInterPaePDs={%RHp,%RHp,%RHp,%RHp} pInterPaePDPT64=%RHp\n",
+                MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]),    MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]),
+                MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[1]),
+                MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[0]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[1]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[2]), MMPage2Phys(pVM, pVM->pgm.s.apInterPaePDs[3]),
+                MMPage2Phys(pVM, pVM->pgm.s.pInterPaePDPT64)));
+#endif
+        return VINF_SUCCESS;
+    }
+
+    LogFlow(("pgmR3InitPaging: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Init statistics
+ * @returns VBox status code.
+ */
+static int pgmR3InitStats(PVM pVM)
+{
+    PPGM pPGM = &pVM->pgm.s;
+    int  rc;
+
+    /*
+     * Release statistics.
+     */
+    /* Common - misc variables */
+    STAM_REL_REG(pVM, &pPGM->cAllPages,                          STAMTYPE_U32,     "/PGM/Page/cAllPages",                STAMUNIT_COUNT,     "The total number of pages.");
+    STAM_REL_REG(pVM, &pPGM->cPrivatePages,                      STAMTYPE_U32,     "/PGM/Page/cPrivatePages",            STAMUNIT_COUNT,     "The number of private pages.");
+    STAM_REL_REG(pVM, &pPGM->cSharedPages,                       STAMTYPE_U32,     "/PGM/Page/cSharedPages",             STAMUNIT_COUNT,     "The number of shared pages.");
+    STAM_REL_REG(pVM, &pPGM->cReusedSharedPages,                 STAMTYPE_U32,     "/PGM/Page/cReusedSharedPages",       STAMUNIT_COUNT,     "The number of reused shared pages.");
+    STAM_REL_REG(pVM, &pPGM->cZeroPages,                         STAMTYPE_U32,     "/PGM/Page/cZeroPages",               STAMUNIT_COUNT,     "The number of zero backed pages.");
+    STAM_REL_REG(pVM, &pPGM->cPureMmioPages,                     STAMTYPE_U32,     "/PGM/Page/cPureMmioPages",           STAMUNIT_COUNT,     "The number of pure MMIO pages.");
+    STAM_REL_REG(pVM, &pPGM->cMonitoredPages,                    STAMTYPE_U32,     "/PGM/Page/cMonitoredPages",          STAMUNIT_COUNT,     "The number of write monitored pages.");
+    STAM_REL_REG(pVM, &pPGM->cWrittenToPages,                    STAMTYPE_U32,     "/PGM/Page/cWrittenToPages",          STAMUNIT_COUNT,     "The number of previously write monitored pages that have been written to.");
+    STAM_REL_REG(pVM, &pPGM->cWriteLockedPages,                  STAMTYPE_U32,     "/PGM/Page/cWriteLockedPages",        STAMUNIT_COUNT,     "The number of write(/read) locked pages.");
+    STAM_REL_REG(pVM, &pPGM->cReadLockedPages,                   STAMTYPE_U32,     "/PGM/Page/cReadLockedPages",         STAMUNIT_COUNT,     "The number of read (only) locked pages.");
+    STAM_REL_REG(pVM, &pPGM->cBalloonedPages,                    STAMTYPE_U32,     "/PGM/Page/cBalloonedPages",          STAMUNIT_COUNT,     "The number of ballooned pages.");
+    STAM_REL_REG(pVM, &pPGM->cHandyPages,                        STAMTYPE_U32,     "/PGM/Page/cHandyPages",              STAMUNIT_COUNT,     "The number of handy pages (not included in cAllPages).");
+    STAM_REL_REG(pVM, &pPGM->cRelocations,                       STAMTYPE_COUNTER, "/PGM/cRelocations",                  STAMUNIT_OCCURENCES,"Number of hypervisor relocations.");
+    STAM_REL_REG(pVM, &pPGM->ChunkR3Map.c,                       STAMTYPE_U32,     "/PGM/ChunkR3Map/c",                  STAMUNIT_COUNT,     "Number of mapped chunks.");
+    STAM_REL_REG(pVM, &pPGM->ChunkR3Map.cMax,                    STAMTYPE_U32,     "/PGM/ChunkR3Map/cMax",               STAMUNIT_COUNT,     "Maximum number of mapped chunks.");
+    STAM_REL_REG(pVM, &pPGM->cMappedChunks,                      STAMTYPE_U32,     "/PGM/ChunkR3Map/Mapped",             STAMUNIT_COUNT,     "Number of times we mapped a chunk.");
+    STAM_REL_REG(pVM, &pPGM->cUnmappedChunks,                    STAMTYPE_U32,     "/PGM/ChunkR3Map/Unmapped",           STAMUNIT_COUNT,     "Number of times we unmapped a chunk.");
+
+    STAM_REL_REG(pVM, &pPGM->StatLargePageAlloc,                 STAMTYPE_COUNTER, "/PGM/LargePage/Alloc",               STAMUNIT_OCCURENCES, "The number of large pages we've used.");
+    STAM_REL_REG(pVM, &pPGM->StatLargePageReused,                STAMTYPE_COUNTER, "/PGM/LargePage/Reused",              STAMUNIT_OCCURENCES, "The number of times we've reused a large page.");
+    STAM_REL_REG(pVM, &pPGM->StatLargePageRefused,               STAMTYPE_COUNTER, "/PGM/LargePage/Refused",             STAMUNIT_OCCURENCES, "The number of times we couldn't use a large page.");
+    STAM_REL_REG(pVM, &pPGM->StatLargePageRecheck,               STAMTYPE_COUNTER, "/PGM/LargePage/Recheck",             STAMUNIT_OCCURENCES, "The number of times we've rechecked a disabled large page.");
+
+    /* Live save */
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.fActive,              STAMTYPE_U8,      "/PGM/LiveSave/fActive",              STAMUNIT_COUNT,     "Active or not.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cIgnoredPages,        STAMTYPE_U32,     "/PGM/LiveSave/cIgnoredPages",        STAMUNIT_COUNT,     "The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM).");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesLong,      STAMTYPE_U32,     "/PGM/LiveSave/cDirtyPagesLong",      STAMUNIT_COUNT,     "Longer term dirty page average.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cDirtyPagesShort,     STAMTYPE_U32,     "/PGM/LiveSave/cDirtyPagesShort",     STAMUNIT_COUNT,     "Short term dirty page average.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cPagesPerSecond,      STAMTYPE_U32,     "/PGM/LiveSave/cPagesPerSecond",      STAMUNIT_COUNT,     "Pages per second.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.cSavedPages,          STAMTYPE_U64,     "/PGM/LiveSave/cSavedPages",          STAMUNIT_COUNT,     "The total number of saved pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cReadyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Ram/cReadPages",       STAMUNIT_COUNT,     "RAM: Ready pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cDirtyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Ram/cDirtyPages",      STAMUNIT_COUNT,     "RAM: Dirty pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cZeroPages,       STAMTYPE_U32,     "/PGM/LiveSave/Ram/cZeroPages",       STAMUNIT_COUNT,     "RAM: Ready zero pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Ram.cMonitoredPages,  STAMTYPE_U32,     "/PGM/LiveSave/Ram/cMonitoredPages",  STAMUNIT_COUNT,     "RAM: Write monitored pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cReadyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Rom/cReadPages",       STAMUNIT_COUNT,     "ROM: Ready pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cDirtyPages,      STAMTYPE_U32,     "/PGM/LiveSave/Rom/cDirtyPages",      STAMUNIT_COUNT,     "ROM: Dirty pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cZeroPages,       STAMTYPE_U32,     "/PGM/LiveSave/Rom/cZeroPages",       STAMUNIT_COUNT,     "ROM: Ready zero pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Rom.cMonitoredPages,  STAMTYPE_U32,     "/PGM/LiveSave/Rom/cMonitoredPages",  STAMUNIT_COUNT,     "ROM: Write monitored pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cReadyPages,    STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cReadPages",     STAMUNIT_COUNT,     "MMIO2: Ready pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cDirtyPages,    STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cDirtyPages",    STAMUNIT_COUNT,     "MMIO2: Dirty pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cZeroPages,     STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cZeroPages",     STAMUNIT_COUNT,     "MMIO2: Ready zero pages.");
+    STAM_REL_REG_USED(pVM, &pPGM->LiveSave.Mmio2.cMonitoredPages,STAMTYPE_U32,     "/PGM/LiveSave/Mmio2/cMonitoredPages",STAMUNIT_COUNT,     "MMIO2: Write monitored pages.");
+
+#ifdef VBOX_WITH_STATISTICS
+
+# define PGM_REG_COUNTER(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b); \
+        AssertRC(rc);
+
+# define PGM_REG_COUNTER_BYTES(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES, c, b); \
+        AssertRC(rc);
+
+# define PGM_REG_PROFILE(a, b, c) \
+        rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b); \
+        AssertRC(rc);
+
+    PGMSTATS *pStats = pVM->pgm.s.pStatsR3;
+
+    PGM_REG_PROFILE(&pStats->StatAllocLargePage,                "/PGM/LargePage/Prof/Alloc",          "Time spent by the host OS for large page allocation.");
+    PGM_REG_PROFILE(&pStats->StatClearLargePage,                "/PGM/LargePage/Prof/Clear",          "Time spent clearing the newly allocated large pages.");
+    PGM_REG_COUNTER(&pStats->StatLargePageOverflow,             "/PGM/LargePage/Overflow",            "The number of times allocating a large page took too long.");
+    PGM_REG_PROFILE(&pStats->StatR3IsValidLargePage,            "/PGM/LargePage/Prof/R3/IsValid",     "pgmPhysIsValidLargePage profiling - R3.");
+    PGM_REG_PROFILE(&pStats->StatRZIsValidLargePage,            "/PGM/LargePage/Prof/RZ/IsValid",     "pgmPhysIsValidLargePage profiling - RZ.");
+
+    PGM_REG_COUNTER(&pStats->StatR3DetectedConflicts,           "/PGM/R3/DetectedConflicts",          "The number of times PGMR3CheckMappingConflicts() detected a conflict.");
+    PGM_REG_PROFILE(&pStats->StatR3ResolveConflict,             "/PGM/R3/ResolveConflict",            "pgmR3SyncPTResolveConflict() profiling (includes the entire relocation).");
+    PGM_REG_COUNTER(&pStats->StatR3PhysRead,                    "/PGM/R3/Phys/Read",                  "The number of times PGMPhysRead was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysReadBytes,         "/PGM/R3/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysWrite,                   "/PGM/R3/Phys/Write",                 "The number of times PGMPhysWrite was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysWriteBytes,        "/PGM/R3/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysSimpleRead,              "/PGM/R3/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysSimpleReadBytes,   "/PGM/R3/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysSimpleWrite,             "/PGM/R3/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatR3PhysSimpleWriteBytes,  "/PGM/R3/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
+
+    PGM_REG_COUNTER(&pStats->StatRZChunkR3MapTlbHits,           "/PGM/ChunkR3Map/TlbHitsRZ",          "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatRZChunkR3MapTlbMisses,         "/PGM/ChunkR3Map/TlbMissesRZ",        "TLB misses.");
+    PGM_REG_PROFILE(&pStats->StatChunkAging,                    "/PGM/ChunkR3Map/Map/Aging",          "Chunk aging profiling.");
+    PGM_REG_PROFILE(&pStats->StatChunkFindCandidate,            "/PGM/ChunkR3Map/Map/Find",           "Chunk unmap find profiling.");
+    PGM_REG_PROFILE(&pStats->StatChunkUnmap,                    "/PGM/ChunkR3Map/Map/Unmap",          "Chunk unmap of address space profiling.");
+    PGM_REG_PROFILE(&pStats->StatChunkMap,                      "/PGM/ChunkR3Map/Map/Map",            "Chunk map of address space profiling.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPageMapTlbHits,              "/PGM/RZ/Page/MapTlbHits",            "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatRZPageMapTlbMisses,            "/PGM/RZ/Page/MapTlbMisses",          "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatR3ChunkR3MapTlbHits,           "/PGM/ChunkR3Map/TlbHitsR3",          "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatR3ChunkR3MapTlbMisses,         "/PGM/ChunkR3Map/TlbMissesR3",        "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatR3PageMapTlbHits,              "/PGM/R3/Page/MapTlbHits",            "TLB hits.");
+    PGM_REG_COUNTER(&pStats->StatR3PageMapTlbMisses,            "/PGM/R3/Page/MapTlbMisses",          "TLB misses.");
+    PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushes,             "/PGM/R3/Page/MapTlbFlushes",         "TLB flushes (all contexts).");
+    PGM_REG_COUNTER(&pStats->StatPageMapTlbFlushEntry,          "/PGM/R3/Page/MapTlbFlushEntry",      "TLB entry flushes (all contexts).");
+
+    PGM_REG_PROFILE(&pStats->StatRZSyncCR3HandlerVirtualUpdate, "/PGM/RZ/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
+    PGM_REG_PROFILE(&pStats->StatRZSyncCR3HandlerVirtualReset,  "/PGM/RZ/SyncCR3/Handlers/VirtualReset",  "Profiling of the virtual handler resets.");
+    PGM_REG_PROFILE(&pStats->StatR3SyncCR3HandlerVirtualUpdate, "/PGM/R3/SyncCR3/Handlers/VirtualUpdate", "Profiling of the virtual handler updates.");
+    PGM_REG_PROFILE(&pStats->StatR3SyncCR3HandlerVirtualReset,  "/PGM/R3/SyncCR3/Handlers/VirtualReset",  "Profiling of the virtual handler resets.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerReset,            "/PGM/RZ/PhysHandlerReset",           "The number of times PGMHandlerPhysicalReset is called.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerReset,            "/PGM/R3/PhysHandlerReset",           "The number of times PGMHandlerPhysicalReset is called.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerLookupHits,       "/PGM/RZ/PhysHandlerLookupHits",      "The number of cache hits when looking up physical handlers.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerLookupHits,       "/PGM/R3/PhysHandlerLookupHits",      "The number of cache hits when looking up physical handlers.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysHandlerLookupMisses,     "/PGM/RZ/PhysHandlerLookupMisses",    "The number of cache misses when looking up physical handlers.");
+    PGM_REG_COUNTER(&pStats->StatR3PhysHandlerLookupMisses,     "/PGM/R3/PhysHandlerLookupMisses",    "The number of cache misses when looking up physical handlers.");
+    PGM_REG_PROFILE(&pStats->StatRZVirtHandlerSearchByPhys,     "/PGM/RZ/VirtHandlerSearchByPhys",    "Profiling of pgmHandlerVirtualFindByPhysAddr.");
+    PGM_REG_PROFILE(&pStats->StatR3VirtHandlerSearchByPhys,     "/PGM/R3/VirtHandlerSearchByPhys",    "Profiling of pgmHandlerVirtualFindByPhysAddr.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPageReplaceShared,           "/PGM/RZ/Page/ReplacedShared",        "Times a shared page was replaced.");
+    PGM_REG_COUNTER(&pStats->StatRZPageReplaceZero,             "/PGM/RZ/Page/ReplacedZero",          "Times the zero page was replaced.");
+/// @todo    PGM_REG_COUNTER(&pStats->StatRZPageHandyAllocs,             "/PGM/RZ/Page/HandyAllocs",               "Number of times we've allocated more handy pages.");
+    PGM_REG_COUNTER(&pStats->StatR3PageReplaceShared,           "/PGM/R3/Page/ReplacedShared",        "Times a shared page was replaced.");
+    PGM_REG_COUNTER(&pStats->StatR3PageReplaceZero,             "/PGM/R3/Page/ReplacedZero",          "Times the zero page was replaced.");
+/// @todo    PGM_REG_COUNTER(&pStats->StatR3PageHandyAllocs,             "/PGM/R3/Page/HandyAllocs",               "Number of times we've allocated more handy pages.");
+
+    PGM_REG_COUNTER(&pStats->StatRZPhysRead,                    "/PGM/RZ/Phys/Read",                  "The number of times PGMPhysRead was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysReadBytes,         "/PGM/RZ/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysWrite,                   "/PGM/RZ/Phys/Write",                 "The number of times PGMPhysWrite was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysWriteBytes,        "/PGM/RZ/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysSimpleRead,              "/PGM/RZ/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysSimpleReadBytes,   "/PGM/RZ/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
+    PGM_REG_COUNTER(&pStats->StatRZPhysSimpleWrite,             "/PGM/RZ/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRZPhysSimpleWriteBytes,  "/PGM/RZ/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
+
+    /* GC only: */
+    PGM_REG_COUNTER(&pStats->StatRCInvlPgConflict,              "/PGM/RC/InvlPgConflict",             "Number of times PGMInvalidatePage() detected a mapping conflict.");
+    PGM_REG_COUNTER(&pStats->StatRCInvlPgSyncMonCR3,            "/PGM/RC/InvlPgSyncMonitorCR3",       "Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3.");
+
+    PGM_REG_COUNTER(&pStats->StatRCPhysRead,                    "/PGM/RC/Phys/Read",                  "The number of times PGMPhysRead was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysReadBytes,         "/PGM/RC/Phys/Read/Bytes",            "The number of bytes read by PGMPhysRead.");
+    PGM_REG_COUNTER(&pStats->StatRCPhysWrite,                   "/PGM/RC/Phys/Write",                 "The number of times PGMPhysWrite was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysWriteBytes,        "/PGM/RC/Phys/Write/Bytes",           "The number of bytes written by PGMPhysWrite.");
+    PGM_REG_COUNTER(&pStats->StatRCPhysSimpleRead,              "/PGM/RC/Phys/Simple/Read",           "The number of times PGMPhysSimpleReadGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysSimpleReadBytes,   "/PGM/RC/Phys/Simple/Read/Bytes",     "The number of bytes read by PGMPhysSimpleReadGCPtr.");
+    PGM_REG_COUNTER(&pStats->StatRCPhysSimpleWrite,             "/PGM/RC/Phys/Simple/Write",          "The number of times PGMPhysSimpleWriteGCPtr was called.");
+    PGM_REG_COUNTER_BYTES(&pStats->StatRCPhysSimpleWriteBytes,  "/PGM/RC/Phys/Simple/Write/Bytes",    "The number of bytes written by PGMPhysSimpleWriteGCPtr.");
+
+    PGM_REG_COUNTER(&pStats->StatTrackVirgin,                   "/PGM/Track/Virgin",                  "The number of first time shadowings");
+    PGM_REG_COUNTER(&pStats->StatTrackAliased,                  "/PGM/Track/Aliased",                 "The number of times switching to cRef2, i.e. the page is being shadowed by two PTs.");
+    PGM_REG_COUNTER(&pStats->StatTrackAliasedMany,              "/PGM/Track/AliasedMany",             "The number of times we're tracking using cRef2.");
+    PGM_REG_COUNTER(&pStats->StatTrackAliasedLots,              "/PGM/Track/AliasedLots",             "The number of times we're hitting pages which has overflowed cRef2");
+    PGM_REG_COUNTER(&pStats->StatTrackOverflows,                "/PGM/Track/Overflows",               "The number of times the extent list grows too long.");
+    PGM_REG_COUNTER(&pStats->StatTrackNoExtentsLeft,            "/PGM/Track/NoExtentLeft",            "The number of times the extent list was exhausted.");
+    PGM_REG_PROFILE(&pStats->StatTrackDeref,                    "/PGM/Track/Deref",                   "Profiling of SyncPageWorkerTrackDeref (expensive).");
+
+# undef PGM_REG_COUNTER
+# undef PGM_REG_PROFILE
+#endif
+
+    /*
+     * Note! The layout below matches the member layout exactly!
+     */
+
+    /*
+     * Common - stats
+     */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PPGMCPU pPgmCpu = &pVM->aCpus[idCpu].pgm.s;
+
+#define PGM_REG_COUNTER(a, b, c) \
+    rc = STAMR3RegisterF(pVM, a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, c, b, idCpu); \
+    AssertRC(rc);
+#define PGM_REG_PROFILE(a, b, c) \
+    rc = STAMR3RegisterF(pVM, a, STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS_PER_CALL, c, b, idCpu); \
+    AssertRC(rc);
+
+        PGM_REG_COUNTER(&pPgmCpu->cGuestModeChanges, "/PGM/CPU%u/cGuestModeChanges",  "Number of guest mode changes.");
+
+#ifdef VBOX_WITH_STATISTICS
+        PGMCPUSTATS *pCpuStats = pVM->aCpus[idCpu].pgm.s.pStatsR3;
+
+# if 0 /* rarely useful; leave for debugging. */
+        for (unsigned j = 0; j < RT_ELEMENTS(pPgmCpu->StatSyncPtPD); j++)
+            STAMR3RegisterF(pVM, &pCpuStats->StatSyncPtPD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "The number of SyncPT per PD n.", "/PGM/CPU%u/PDSyncPT/%04X", i, j);
+        for (unsigned j = 0; j < RT_ELEMENTS(pCpuStats->StatSyncPagePD); j++)
+            STAMR3RegisterF(pVM, &pCpuStats->StatSyncPagePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "The number of SyncPage per PD n.", "/PGM/CPU%u/PDSyncPage/%04X", i, j);
+# endif
+        /* R0 only: */
+        PGM_REG_PROFILE(&pCpuStats->StatR0NpMiscfg,                      "/PGM/CPU%u/R0/NpMiscfg",                     "PGMR0Trap0eHandlerNPMisconfig() profiling.");
+        PGM_REG_COUNTER(&pCpuStats->StatR0NpMiscfgSyncPage,              "/PGM/CPU%u/R0/NpMiscfgSyncPage",             "SyncPage calls from PGMR0Trap0eHandlerNPMisconfig().");
+
+        /* RZ only: */
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0e,                      "/PGM/CPU%u/RZ/Trap0e",                     "Profiling of the PGMTrap0eHandler() body.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Ballooned,        "/PGM/CPU%u/RZ/Trap0e/Time2/Ballooned",         "Profiling of the Trap0eHandler body when the cause is read access to a ballooned page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2CSAM,             "/PGM/CPU%u/RZ/Trap0e/Time2/CSAM",              "Profiling of the Trap0eHandler body when the cause is CSAM.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2DirtyAndAccessed, "/PGM/CPU%u/RZ/Trap0e/Time2/DirtyAndAccessedBits", "Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2GuestTrap,        "/PGM/CPU%u/RZ/Trap0e/Time2/GuestTrap",         "Profiling of the Trap0eHandler body when the cause is a guest trap.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndPhys,          "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerPhysical",   "Profiling of the Trap0eHandler body when the cause is a physical handler.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndVirt,          "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerVirtual",    "Profiling of the Trap0eHandler body when the cause is a virtual handler.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2HndUnhandled,     "/PGM/CPU%u/RZ/Trap0e/Time2/HandlerUnhandled",  "Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2InvalidPhys,      "/PGM/CPU%u/RZ/Trap0e/Time2/InvalidPhys",       "Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2MakeWritable,     "/PGM/CPU%u/RZ/Trap0e/Time2/MakeWritable",      "Profiling of the Trap0eHandler body when the cause is that a page needed to be made writeable.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Mapping,          "/PGM/CPU%u/RZ/Trap0e/Time2/Mapping",           "Profiling of the Trap0eHandler body when the cause is related to the guest mappings.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2Misc,             "/PGM/CPU%u/RZ/Trap0e/Time2/Misc",              "Profiling of the Trap0eHandler body when the cause is not known.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSync,        "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSync",         "Profiling of the Trap0eHandler body when the cause is an out-of-sync page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndPhys, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncHndPhys",  "Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndVirt, "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncHndVirt",  "Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2OutOfSyncHndObs,  "/PGM/CPU%u/RZ/Trap0e/Time2/OutOfSyncObsHnd",   "Profiling of the Trap0eHandler body when the cause is an obsolete handler page.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2SyncPT,           "/PGM/CPU%u/RZ/Trap0e/Time2/SyncPT",            "Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZTrap0eTime2WPEmulation,      "/PGM/CPU%u/RZ/Trap0e/Time2/WPEmulation",       "Profiling of the Trap0eHandler body when the cause is CR0.WP emulation.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eConflicts,             "/PGM/CPU%u/RZ/Trap0e/Conflicts",               "The number of times #PF was caused by an undetected conflict.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersMapping,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Mapping",        "Number of traps due to access handlers in mappings.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersOutOfSync,     "/PGM/CPU%u/RZ/Trap0e/Handlers/OutOfSync",      "Number of traps due to out-of-sync handled pages.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysAll,       "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysAll",        "Number of traps due to physical all-access handlers.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysAllOpt,    "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysAllOpt",     "Number of the physical all-access handler traps using the optimization.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersPhysWrite,     "/PGM/CPU%u/RZ/Trap0e/Handlers/PhysWrite",      "Number of traps due to physical write-access handlers.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersVirtual,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Virtual",        "Number of traps due to virtual access handlers.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersVirtualByPhys, "/PGM/CPU%u/RZ/Trap0e/Handlers/VirtualByPhys",  "Number of traps due to virtual access handlers by physical address.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersVirtualUnmarked,"/PGM/CPU%u/RZ/Trap0e/Handlers/VirtualUnmarked","Number of traps due to virtual access handlers by virtual address (without proper physical flags).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersUnhandled,     "/PGM/CPU%u/RZ/Trap0e/Handlers/Unhandled",      "Number of traps due to access outside range of monitored page(s).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eHandlersInvalid,       "/PGM/CPU%u/RZ/Trap0e/Handlers/Invalid",        "Number of traps due to access to invalid physical memory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNotPresentRead,      "/PGM/CPU%u/RZ/Trap0e/Err/User/NPRead",         "Number of user mode not present read page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNotPresentWrite,     "/PGM/CPU%u/RZ/Trap0e/Err/User/NPWrite",        "Number of user mode not present write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSWrite,               "/PGM/CPU%u/RZ/Trap0e/Err/User/Write",          "Number of user mode write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSReserved,            "/PGM/CPU%u/RZ/Trap0e/Err/User/Reserved",       "Number of user mode reserved bit page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSNXE,                 "/PGM/CPU%u/RZ/Trap0e/Err/User/NXE",            "Number of user mode NXE page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eUSRead,                "/PGM/CPU%u/RZ/Trap0e/Err/User/Read",           "Number of user mode read page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVNotPresentRead,      "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPRead",   "Number of supervisor mode not present read page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVNotPresentWrite,     "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NPWrite",  "Number of supervisor mode not present write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVWrite,               "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Write",    "Number of supervisor mode write page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSVReserved,            "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/Reserved", "Number of supervisor mode reserved bit page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eSNXE,                  "/PGM/CPU%u/RZ/Trap0e/Err/Supervisor/NXE",      "Number of supervisor mode NXE page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eGuestPF,               "/PGM/CPU%u/RZ/Trap0e/GuestPF",                 "Number of real guest page faults.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eGuestPFMapping,        "/PGM/CPU%u/RZ/Trap0e/GuestPF/InMapping",       "Number of real guest page faults in a mapping.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eWPEmulInRZ,            "/PGM/CPU%u/RZ/Trap0e/WP/InRZ",                 "Number of guest page faults due to X86_CR0_WP emulation.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZTrap0eWPEmulToR3,            "/PGM/CPU%u/RZ/Trap0e/WP/ToR3",                 "Number of guest page faults due to X86_CR0_WP emulation (forward to R3 for emulation).");
+#if 0 /* rarely useful; leave for debugging. */
+        for (unsigned j = 0; j < RT_ELEMENTS(pCpuStats->StatRZTrap0ePD); j++)
+            STAMR3RegisterF(pVM, &pCpuStats->StatRZTrap0ePD[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES,
+                            "The number of traps in page directory n.", "/PGM/CPU%u/RZ/Trap0e/PD/%04X", i, j);
+#endif
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteHandled,        "/PGM/CPU%u/RZ/CR3WriteHandled",                "The number of times the Guest CR3 change was successfully handled.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteUnhandled,      "/PGM/CPU%u/RZ/CR3WriteUnhandled",              "The number of times the Guest CR3 change was passed back to the recompiler.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestCR3WriteConflict,       "/PGM/CPU%u/RZ/CR3WriteConflict",               "The number of times the Guest CR3 monitoring detected a conflict.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestROMWriteHandled,        "/PGM/CPU%u/RZ/ROMWriteHandled",                "The number of times the Guest ROM change was successfully handled.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZGuestROMWriteUnhandled,      "/PGM/CPU%u/RZ/ROMWriteUnhandled",              "The number of times the Guest ROM change was passed back to the recompiler.");
+
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapMigrateInvlPg,         "/PGM/CPU%u/RZ/DynMap/MigrateInvlPg",            "invlpg count in PGMR0DynMapMigrateAutoSet.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapGCPageInl,             "/PGM/CPU%u/RZ/DynMap/PageGCPageInl",            "Calls to pgmR0DynMapGCPageInlined.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlHits,         "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/Hits",       "Hash table lookup hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlMisses,       "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/Misses",     "Misses that falls back to the code common.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlRamHits,      "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/RamHits",    "1st ram range hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapGCPageInlRamMisses,    "/PGM/CPU%u/RZ/DynMap/PageGCPageInl/RamMisses",  "1st ram range misses, takes slow path.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapHCPageInl,             "/PGM/CPU%u/RZ/DynMap/PageHCPageInl",            "Calls to pgmRZDynMapHCPageInlined.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapHCPageInlHits,         "/PGM/CPU%u/RZ/DynMap/PageHCPageInl/Hits",       "Hash table lookup hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapHCPageInlMisses,       "/PGM/CPU%u/RZ/DynMap/PageHCPageInl/Misses",     "Misses that falls back to the code common.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPage,                  "/PGM/CPU%u/RZ/DynMap/Page",                     "Calls to pgmR0DynMapPage");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetOptimize,           "/PGM/CPU%u/RZ/DynMap/Page/SetOptimize",         "Calls to pgmRZDynMapOptimizeAutoSet.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchFlushes,      "/PGM/CPU%u/RZ/DynMap/Page/SetSearchFlushes",    "Set search restoring to subset flushes.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchHits,         "/PGM/CPU%u/RZ/DynMap/Page/SetSearchHits",       "Set search hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSetSearchMisses,       "/PGM/CPU%u/RZ/DynMap/Page/SetSearchMisses",     "Set search misses.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDynMapHCPage,                "/PGM/CPU%u/RZ/DynMap/Page/HCPage",              "Calls to pgmRZDynMapHCPageCommon (ring-0).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits0,             "/PGM/CPU%u/RZ/DynMap/Page/Hits0",               "Hits at iPage+0");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits1,             "/PGM/CPU%u/RZ/DynMap/Page/Hits1",               "Hits at iPage+1");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageHits2,             "/PGM/CPU%u/RZ/DynMap/Page/Hits2",               "Hits at iPage+2");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageInvlPg,            "/PGM/CPU%u/RZ/DynMap/Page/InvlPg",              "invlpg count in pgmR0DynMapPageSlow.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlow,              "/PGM/CPU%u/RZ/DynMap/Page/Slow",                "Calls to pgmR0DynMapPageSlow - subtract this from pgmR0DynMapPage to get 1st level hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLoopHits,      "/PGM/CPU%u/RZ/DynMap/Page/SlowLoopHits" ,       "Hits in the loop path.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLoopMisses,    "/PGM/CPU%u/RZ/DynMap/Page/SlowLoopMisses",      "Misses in the loop path. NonLoopMisses = Slow - SlowLoopHit - SlowLoopMisses");
+        //PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPageSlowLostHits,      "/PGM/CPU%u/R0/DynMap/Page/SlowLostHits",        "Lost hits.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapSubsets,               "/PGM/CPU%u/RZ/DynMap/Subsets",                  "Times PGMRZDynMapPushAutoSubset was called.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDynMapPopFlushes,            "/PGM/CPU%u/RZ/DynMap/SubsetPopFlushes",         "Times PGMRZDynMapPopAutoSubset flushes the subset.");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[0],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct000..09",      "00-09% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[1],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct010..19",      "10-19% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[2],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct020..29",      "20-29% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[3],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct030..39",      "30-39% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[4],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct040..49",      "40-49% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[5],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct050..59",      "50-59% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[6],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct060..69",      "60-69% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[7],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct070..79",      "70-79% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[8],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct080..89",      "80-89% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[9],      "/PGM/CPU%u/RZ/DynMap/SetFilledPct090..99",      "90-99% filled (RC: min(set-size, dynmap-size))");
+        PGM_REG_COUNTER(&pCpuStats->aStatRZDynMapSetFilledPct[10],     "/PGM/CPU%u/RZ/DynMap/SetFilledPct100",          "100% filled (RC: min(set-size, dynmap-size))");
+
+        /* HC only: */
+
+        /* RZ & R3: */
+        PGM_REG_PROFILE(&pCpuStats->StatRZSyncCR3,                     "/PGM/CPU%u/RZ/SyncCR3",                        "Profiling of the PGMSyncCR3() body.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZSyncCR3Handlers,             "/PGM/CPU%u/RZ/SyncCR3/Handlers",               "Profiling of the PGMSyncCR3() update handler section.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3Global,               "/PGM/CPU%u/RZ/SyncCR3/Global",                 "The number of global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3NotGlobal,            "/PGM/CPU%u/RZ/SyncCR3/NotGlobal",              "The number of non-global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstCacheHit,          "/PGM/CPU%u/RZ/SyncCR3/DstChacheHit",           "The number of times we got some kind of a cache hit.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstFreed,             "/PGM/CPU%u/RZ/SyncCR3/DstFreed",               "The number of times we've had to free a shadow entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstFreedSrcNP,        "/PGM/CPU%u/RZ/SyncCR3/DstFreedSrcNP",          "The number of times we've had to free a shadow entry for which the source entry was not present.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstNotPresent,        "/PGM/CPU%u/RZ/SyncCR3/DstNotPresent",          "The number of times we've encountered a not present shadow entry for a present guest entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstSkippedGlobalPD,   "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPD",     "The number of times a global page directory wasn't flushed.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncCR3DstSkippedGlobalPT,   "/PGM/CPU%u/RZ/SyncCR3/DstSkippedGlobalPT",     "The number of times a page table with only global entries wasn't flushed.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZSyncPT,                      "/PGM/CPU%u/RZ/SyncPT",                         "Profiling of the pfnSyncPT() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPTFailed,                "/PGM/CPU%u/RZ/SyncPT/Failed",                  "The number of times pfnSyncPT() failed.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPT4K,                    "/PGM/CPU%u/RZ/SyncPT/4K",                      "Nr of 4K PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPT4M,                    "/PGM/CPU%u/RZ/SyncPT/4M",                      "Nr of 4M PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPagePDNAs,               "/PGM/CPU%u/RZ/SyncPagePDNAs",                  "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZSyncPagePDOutOfSync,         "/PGM/CPU%u/RZ/SyncPagePDOutOfSync",            "The number of time we've encountered an out-of-sync PD in SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZAccessedPage,                "/PGM/CPU%u/RZ/AccessedPage",               "The number of pages marked not present for accessed bit emulation.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZDirtyBitTracking,            "/PGM/CPU%u/RZ/DirtyPage",                  "Profiling the dirty bit tracking in CheckPageFault().");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPage,                   "/PGM/CPU%u/RZ/DirtyPage/Mark",             "The number of pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageBig,                "/PGM/CPU%u/RZ/DirtyPage/MarkBig",          "The number of 4MB pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageSkipped,            "/PGM/CPU%u/RZ/DirtyPage/Skipped",          "The number of pages already dirty or readonly.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageTrap,               "/PGM/CPU%u/RZ/DirtyPage/Trap",             "The number of traps generated for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyPageStale,              "/PGM/CPU%u/RZ/DirtyPage/Stale",            "The number of traps generated for dirty bit tracking (stale tlb entries).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtiedPage,                 "/PGM/CPU%u/RZ/DirtyPage/SetDirty",         "The number of pages marked dirty because of write accesses.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZDirtyTrackRealPF,            "/PGM/CPU%u/RZ/DirtyPage/RealPF",           "The number of real pages faults during dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageAlreadyDirty,            "/PGM/CPU%u/RZ/DirtyPage/AlreadySet",       "The number of pages already marked dirty because of write accesses.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZInvalidatePage,              "/PGM/CPU%u/RZ/InvalidatePage",             "PGMInvalidatePage() profiling.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4KBPages,      "/PGM/CPU%u/RZ/InvalidatePage/4KBPages",    "The number of times PGMInvalidatePage() was called for a 4KB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4MBPages,      "/PGM/CPU%u/RZ/InvalidatePage/4MBPages",    "The number of times PGMInvalidatePage() was called for a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePage4MBPagesSkip,  "/PGM/CPU%u/RZ/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDMappings,    "/PGM/CPU%u/RZ/InvalidatePage/PDMappings",  "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDNAs,         "/PGM/CPU%u/RZ/InvalidatePage/PDNAs",       "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDNPs,         "/PGM/CPU%u/RZ/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePagePDOutOfSync,   "/PGM/CPU%u/RZ/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZInvalidatePageSkipped,       "/PGM/CPU%u/RZ/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncSupervisor,     "/PGM/CPU%u/RZ/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncUser,           "/PGM/CPU%u/RZ/OutOfSync/User",             "Number of traps due to pages out of sync (P) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncSupervisorWrite,"/PGM/CPU%u/RZ/OutOfSync/SuperVisorWrite",  "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncUserWrite,      "/PGM/CPU%u/RZ/OutOfSync/UserWrite",        "Number of traps due to pages out of sync (RW) and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZPageOutOfSyncBallloon,       "/PGM/CPU%u/RZ/OutOfSync/Balloon",          "The number of times a ballooned page was accessed (read).");
+        PGM_REG_PROFILE(&pCpuStats->StatRZPrefetch,                    "/PGM/CPU%u/RZ/Prefetch",                   "PGMPrefetchPage profiling.");
+        PGM_REG_PROFILE(&pCpuStats->StatRZFlushTLB,                    "/PGM/CPU%u/RZ/FlushTLB",                   "Profiling of the PGMFlushTLB() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBNewCR3,              "/PGM/CPU%u/RZ/FlushTLB/NewCR3",            "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBNewCR3Global,        "/PGM/CPU%u/RZ/FlushTLB/NewCR3Global",      "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBSameCR3,             "/PGM/CPU%u/RZ/FlushTLB/SameCR3",           "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
+        PGM_REG_COUNTER(&pCpuStats->StatRZFlushTLBSameCR3Global,       "/PGM/CPU%u/RZ/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
+        PGM_REG_PROFILE(&pCpuStats->StatRZGstModifyPage,               "/PGM/CPU%u/RZ/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
+
+        PGM_REG_PROFILE(&pCpuStats->StatR3SyncCR3,                     "/PGM/CPU%u/R3/SyncCR3",                        "Profiling of the PGMSyncCR3() body.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3SyncCR3Handlers,             "/PGM/CPU%u/R3/SyncCR3/Handlers",               "Profiling of the PGMSyncCR3() update handler section.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3Global,               "/PGM/CPU%u/R3/SyncCR3/Global",                 "The number of global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3NotGlobal,            "/PGM/CPU%u/R3/SyncCR3/NotGlobal",              "The number of non-global CR3 syncs.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstCacheHit,          "/PGM/CPU%u/R3/SyncCR3/DstChacheHit",           "The number of times we got some kind of a cache hit.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstFreed,             "/PGM/CPU%u/R3/SyncCR3/DstFreed",               "The number of times we've had to free a shadow entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstFreedSrcNP,        "/PGM/CPU%u/R3/SyncCR3/DstFreedSrcNP",          "The number of times we've had to free a shadow entry for which the source entry was not present.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstNotPresent,        "/PGM/CPU%u/R3/SyncCR3/DstNotPresent",          "The number of times we've encountered a not present shadow entry for a present guest entry.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstSkippedGlobalPD,   "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPD",     "The number of times a global page directory wasn't flushed.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncCR3DstSkippedGlobalPT,   "/PGM/CPU%u/R3/SyncCR3/DstSkippedGlobalPT",     "The number of times a page table with only global entries wasn't flushed.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3SyncPT,                      "/PGM/CPU%u/R3/SyncPT",                         "Profiling of the pfnSyncPT() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPTFailed,                "/PGM/CPU%u/R3/SyncPT/Failed",                  "The number of times pfnSyncPT() failed.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPT4K,                    "/PGM/CPU%u/R3/SyncPT/4K",                      "Nr of 4K PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPT4M,                    "/PGM/CPU%u/R3/SyncPT/4M",                      "Nr of 4M PT syncs");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPagePDNAs,               "/PGM/CPU%u/R3/SyncPagePDNAs",                  "The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3SyncPagePDOutOfSync,         "/PGM/CPU%u/R3/SyncPagePDOutOfSync",            "The number of time we've encountered an out-of-sync PD in SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3AccessedPage,                "/PGM/CPU%u/R3/AccessedPage",               "The number of pages marked not present for accessed bit emulation.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3DirtyBitTracking,            "/PGM/CPU%u/R3/DirtyPage",                  "Profiling the dirty bit tracking in CheckPageFault().");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPage,                   "/PGM/CPU%u/R3/DirtyPage/Mark",             "The number of pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageBig,                "/PGM/CPU%u/R3/DirtyPage/MarkBig",          "The number of 4MB pages marked read-only for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageSkipped,            "/PGM/CPU%u/R3/DirtyPage/Skipped",          "The number of pages already dirty or readonly.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyPageTrap,               "/PGM/CPU%u/R3/DirtyPage/Trap",             "The number of traps generated for dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtiedPage,                 "/PGM/CPU%u/R3/DirtyPage/SetDirty",         "The number of pages marked dirty because of write accesses.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3DirtyTrackRealPF,            "/PGM/CPU%u/R3/DirtyPage/RealPF",           "The number of real pages faults during dirty bit tracking.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageAlreadyDirty,            "/PGM/CPU%u/R3/DirtyPage/AlreadySet",       "The number of pages already marked dirty because of write accesses.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3InvalidatePage,              "/PGM/CPU%u/R3/InvalidatePage",             "PGMInvalidatePage() profiling.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4KBPages,      "/PGM/CPU%u/R3/InvalidatePage/4KBPages",    "The number of times PGMInvalidatePage() was called for a 4KB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4MBPages,      "/PGM/CPU%u/R3/InvalidatePage/4MBPages",    "The number of times PGMInvalidatePage() was called for a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePage4MBPagesSkip,  "/PGM/CPU%u/R3/InvalidatePage/4MBPagesSkip","The number of times PGMInvalidatePage() skipped a 4MB page.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDMappings,    "/PGM/CPU%u/R3/InvalidatePage/PDMappings",  "The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict).");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDNAs,         "/PGM/CPU%u/R3/InvalidatePage/PDNAs",       "The number of times PGMInvalidatePage() was called for a not accessed page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDNPs,         "/PGM/CPU%u/R3/InvalidatePage/PDNPs",       "The number of times PGMInvalidatePage() was called for a not present page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePagePDOutOfSync,   "/PGM/CPU%u/R3/InvalidatePage/PDOutOfSync", "The number of times PGMInvalidatePage() was called for an out of sync page directory.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3InvalidatePageSkipped,       "/PGM/CPU%u/R3/InvalidatePage/Skipped",     "The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncSupervisor,     "/PGM/CPU%u/R3/OutOfSync/SuperVisor",       "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncUser,           "/PGM/CPU%u/R3/OutOfSync/User",             "Number of traps due to pages out of sync and times VerifyAccessSyncPage calls SyncPage.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3PageOutOfSyncBallloon,       "/PGM/CPU%u/R3/OutOfSync/Balloon",          "The number of times a ballooned page was accessed (read).");
+        PGM_REG_PROFILE(&pCpuStats->StatR3Prefetch,                    "/PGM/CPU%u/R3/Prefetch",                   "PGMPrefetchPage profiling.");
+        PGM_REG_PROFILE(&pCpuStats->StatR3FlushTLB,                    "/PGM/CPU%u/R3/FlushTLB",                   "Profiling of the PGMFlushTLB() body.");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBNewCR3,              "/PGM/CPU%u/R3/FlushTLB/NewCR3",            "The number of times PGMFlushTLB was called with a new CR3, non-global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBNewCR3Global,        "/PGM/CPU%u/R3/FlushTLB/NewCR3Global",      "The number of times PGMFlushTLB was called with a new CR3, global. (switch)");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBSameCR3,             "/PGM/CPU%u/R3/FlushTLB/SameCR3",           "The number of times PGMFlushTLB was called with the same CR3, non-global. (flush)");
+        PGM_REG_COUNTER(&pCpuStats->StatR3FlushTLBSameCR3Global,       "/PGM/CPU%u/R3/FlushTLB/SameCR3Global",     "The number of times PGMFlushTLB was called with the same CR3, global. (flush)");
+        PGM_REG_PROFILE(&pCpuStats->StatR3GstModifyPage,               "/PGM/CPU%u/R3/GstModifyPage",              "Profiling of the PGMGstModifyPage() body.");
+#endif /* VBOX_WITH_STATISTICS */
+
+#undef PGM_REG_PROFILE
+#undef PGM_REG_COUNTER
+
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Init the PGM bits that rely on VMMR0 and MM to be fully initialized.
+ *
+ * The dynamic mapping area will also be allocated and initialized at this
+ * time. We could allocate it during PGMR3Init of course, but the mapping
+ * wouldn't be allocated at that time preventing us from setting up the
+ * page table entries with the dummy page.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(int) PGMR3InitDynMap(PVM pVM)
+{
+    RTGCPTR GCPtr;
+    int     rc;
+
+    /*
+     * Reserve space for the dynamic mappings.
+     */
+    rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping", &GCPtr);
+    if (RT_SUCCESS(rc))
+        pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
+
+    if (    RT_SUCCESS(rc)
+        &&  (pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) != ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT))
+    {
+        rc = MMR3HyperReserve(pVM, MM_HYPER_DYNAMIC_SIZE, "Dynamic mapping not crossing", &GCPtr);
+        if (RT_SUCCESS(rc))
+            pVM->pgm.s.pbDynPageMapBaseGC = GCPtr;
+    }
+    if (RT_SUCCESS(rc))
+    {
+        AssertRelease((pVM->pgm.s.pbDynPageMapBaseGC >> X86_PD_PAE_SHIFT) == ((pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE - 1) >> X86_PD_PAE_SHIFT));
+        MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+    }
+    return rc;
+}
+
+
+/**
+ * Ring-3 init finalizing.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(int) PGMR3InitFinalize(PVM pVM)
+{
+    int rc;
+
+    /*
+     * Reserve space for the dynamic mappings.
+     * Initialize the dynamic mapping pages with dummy pages to simply the cache.
+     */
+    /* get the pointer to the page table entries. */
+    PPGMMAPPING pMapping = pgmGetMapping(pVM, pVM->pgm.s.pbDynPageMapBaseGC);
+    AssertRelease(pMapping);
+    const uintptr_t off = pVM->pgm.s.pbDynPageMapBaseGC - pMapping->GCPtr;
+    const unsigned iPT =  off >> X86_PD_SHIFT;
+    const unsigned iPG = (off >> X86_PT_SHIFT) & X86_PT_MASK;
+    pVM->pgm.s.paDynPageMap32BitPTEsGC = pMapping->aPTs[iPT].pPTRC      + iPG * sizeof(pMapping->aPTs[0].pPTR3->a[0]);
+    pVM->pgm.s.paDynPageMapPaePTEsGC   = pMapping->aPTs[iPT].paPaePTsRC + iPG * sizeof(pMapping->aPTs[0].paPaePTsR3->a[0]);
+
+    /* init cache area */
+    RTHCPHYS HCPhysDummy = MMR3PageDummyHCPhys(pVM);
+    for (uint32_t offDynMap = 0; offDynMap < MM_HYPER_DYNAMIC_SIZE; offDynMap += PAGE_SIZE)
+    {
+        rc = PGMMap(pVM, pVM->pgm.s.pbDynPageMapBaseGC + offDynMap, HCPhysDummy, PAGE_SIZE, 0);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Determine the max physical address width (MAXPHYADDR) and apply it to
+     * all the mask members and stuff.
+     */
+    uint32_t cMaxPhysAddrWidth;
+    uint32_t uMaxExtLeaf = ASMCpuId_EAX(0x80000000);
+    if (   uMaxExtLeaf >= 0x80000008
+        && uMaxExtLeaf <= 0x80000fff)
+    {
+        cMaxPhysAddrWidth = ASMCpuId_EAX(0x80000008) & 0xff;
+        LogRel(("PGM: The CPU physical address width is %u bits\n", cMaxPhysAddrWidth));
+        cMaxPhysAddrWidth = RT_MIN(52, cMaxPhysAddrWidth);
+        pVM->pgm.s.fLessThan52PhysicalAddressBits = cMaxPhysAddrWidth < 52;
+        for (uint32_t iBit = cMaxPhysAddrWidth; iBit < 52; iBit++)
+            pVM->pgm.s.HCPhysInvMmioPg |= RT_BIT_64(iBit);
+    }
+    else
+    {
+        LogRel(("PGM: ASSUMING CPU physical address width of 48 bits (uMaxExtLeaf=%#x)\n", uMaxExtLeaf));
+        cMaxPhysAddrWidth = 48;
+        pVM->pgm.s.fLessThan52PhysicalAddressBits = true;
+        pVM->pgm.s.HCPhysInvMmioPg |= UINT64_C(0x000f0000000000);
+    }
+
+    pVM->pgm.s.GCPhysInvAddrMask = 0;
+    for (uint32_t iBit = cMaxPhysAddrWidth; iBit < 64; iBit++)
+        pVM->pgm.s.GCPhysInvAddrMask |= RT_BIT_64(iBit);
+
+    /*
+     * Initialize the invalid paging entry masks, assuming NX is disabled.
+     */
+    uint64_t fMbzPageFrameMask = pVM->pgm.s.GCPhysInvAddrMask & UINT64_C(0x000ffffffffff000);
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[iCpu];
+
+        /** @todo The manuals are not entirely clear whether the physical
+         *        address width is relevant.  See table 5-9 in the intel
+         *        manual vs the PDE4M descriptions.  Write testcase (NP). */
+        pVCpu->pgm.s.fGst32BitMbzBigPdeMask   = ((uint32_t)(fMbzPageFrameMask >> (32 - 13)) & X86_PDE4M_PG_HIGH_MASK)
+                                              | X86_PDE4M_MBZ_MASK;
+
+        pVCpu->pgm.s.fGstPaeMbzPteMask        = fMbzPageFrameMask | X86_PTE_PAE_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdeMask        = fMbzPageFrameMask | X86_PDE_PAE_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstPaeMbzBigPdeMask     = fMbzPageFrameMask | X86_PDE2M_PAE_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdpeMask       = fMbzPageFrameMask | X86_PDPE_PAE_MBZ_MASK;
+
+        pVCpu->pgm.s.fGstAmd64MbzPteMask      = fMbzPageFrameMask | X86_PTE_LM_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdeMask      = fMbzPageFrameMask | X86_PDE_LM_MBZ_MASK_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdeMask   = fMbzPageFrameMask | X86_PDE2M_LM_MBZ_MASK_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdpeMask     = fMbzPageFrameMask | X86_PDPE_LM_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask  = fMbzPageFrameMask | X86_PDPE1G_LM_MBZ_MASK_NO_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPml4eMask    = fMbzPageFrameMask | X86_PML4E_MBZ_MASK_NO_NX;
+
+        pVCpu->pgm.s.fGst64ShadowedPteMask    = X86_PTE_P   | X86_PTE_RW   | X86_PTE_US   | X86_PTE_G | X86_PTE_A | X86_PTE_D;
+        pVCpu->pgm.s.fGst64ShadowedPdeMask    = X86_PDE_P   | X86_PDE_RW   | X86_PDE_US   | X86_PDE_A;
+        pVCpu->pgm.s.fGst64ShadowedBigPdeMask = X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A;
+        pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask =
+            X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_G | X86_PDE4M_A | X86_PDE4M_D;
+        pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask  = X86_PDPE_P  | X86_PDPE_RW  | X86_PDPE_US  | X86_PDPE_A;
+        pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
+    }
+
+    /*
+     * Note that AMD uses all the 8 reserved bits for the address (so 40 bits in total);
+     * Intel only goes up to 36 bits, so we stick to 36 as well.
+     * Update: More recent intel manuals specifies 40 bits just like AMD.
+     */
+    uint32_t u32Dummy, u32Features;
+    CPUMGetGuestCpuId(VMMGetCpu(pVM), 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
+    if (u32Features & X86_CPUID_FEATURE_EDX_PSE36)
+        pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(RT_MAX(36, cMaxPhysAddrWidth)) - 1;
+    else
+        pVM->pgm.s.GCPhys4MBPSEMask = RT_BIT_64(32) - 1;
+
+    /*
+     * Allocate memory if we're supposed to do that.
+     */
+    if (pVM->pgm.s.fRamPreAlloc)
+        rc = pgmR3PhysRamPreAllocate(pVM);
+
+    LogRel(("PGMR3InitFinalize: 4 MB PSE mask %RGp\n", pVM->pgm.s.GCPhys4MBPSEMask));
+    return rc;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this component.
+ *
+ * This function will be called at init and whenever the VMM need to relocate it
+ * self inside the GC.
+ *
+ * @param   pVM     The VM.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void) PGMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("PGMR3Relocate %RGv to %RGv\n", pVM->pgm.s.GCPtrCR3Mapping, pVM->pgm.s.GCPtrCR3Mapping + offDelta));
+
+    /*
+     * Paging stuff.
+     */
+    pVM->pgm.s.GCPtrCR3Mapping += offDelta;
+
+    pgmR3ModeDataInit(pVM, true /* resolve GC/R0 symbols */);
+
+    /* Shadow, guest and both mode switch & relocation for each VCPU. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU  pVCpu = &pVM->aCpus[i];
+
+        pgmR3ModeDataSwitch(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
+
+        PGM_SHW_PFN(Relocate, pVCpu)(pVCpu, offDelta);
+        PGM_GST_PFN(Relocate, pVCpu)(pVCpu, offDelta);
+        PGM_BTH_PFN(Relocate, pVCpu)(pVCpu, offDelta);
+    }
+
+    /*
+     * Trees.
+     */
+    pVM->pgm.s.pTreesRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pTreesR3);
+
+    /*
+     * Ram ranges.
+     */
+    if (pVM->pgm.s.pRamRangesR3)
+    {
+        /* Update the pSelfRC pointers and relink them. */
+        for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+            if (!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
+                pCur->pSelfRC = MMHyperCCToRC(pVM, pCur);
+        pgmR3PhysRelinkRamRanges(pVM);
+    }
+
+    /*
+     * Update the pSelfRC pointer of the MMIO2 ram ranges since they might not
+     * be mapped and thus not included in the above exercise.
+     */
+    for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
+        if (!(pCur->RamRange.fFlags & PGM_RAM_RANGE_FLAGS_FLOATING))
+            pCur->RamRange.pSelfRC = MMHyperCCToRC(pVM, &pCur->RamRange);
+
+    /*
+     * Update the two page directories with all page table mappings.
+     * (One or more of them have changed, that's why we're here.)
+     */
+    pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pMappingsR3);
+    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur->pNextR3; pCur = pCur->pNextR3)
+        pCur->pNextRC = MMHyperR3ToRC(pVM, pCur->pNextR3);
+
+    /* Relocate GC addresses of Page Tables. */
+    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
+    {
+        for (RTHCUINT i = 0; i < pCur->cPTs; i++)
+        {
+            pCur->aPTs[i].pPTRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].pPTR3);
+            pCur->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pCur->aPTs[i].paPaePTsR3);
+        }
+    }
+
+    /*
+     * Dynamic page mapping area.
+     */
+    pVM->pgm.s.paDynPageMap32BitPTEsGC += offDelta;
+    pVM->pgm.s.paDynPageMapPaePTEsGC   += offDelta;
+    pVM->pgm.s.pbDynPageMapBaseGC      += offDelta;
+
+    if (pVM->pgm.s.pRCDynMap)
+    {
+        pVM->pgm.s.pRCDynMap += offDelta;
+        PPGMRCDYNMAP pDynMap = (PPGMRCDYNMAP)MMHyperRCToCC(pVM, pVM->pgm.s.pRCDynMap);
+
+        pDynMap->paPages     += offDelta;
+        PPGMRCDYNMAPENTRY paPages = (PPGMRCDYNMAPENTRY)MMHyperRCToCC(pVM, pDynMap->paPages);
+
+        for (uint32_t iPage = 0; iPage < pDynMap->cPages; iPage++)
+        {
+            paPages[iPage].pvPage  += offDelta;
+            paPages[iPage].uPte.pv += offDelta;
+        }
+    }
+
+    /*
+     * The Zero page.
+     */
+    pVM->pgm.s.pvZeroPgR0 = MMHyperR3ToR0(pVM, pVM->pgm.s.pvZeroPgR3);
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR || !VMMIsHwVirtExtForced(pVM));
+#else
+    AssertRelease(pVM->pgm.s.pvZeroPgR0 != NIL_RTR0PTR);
+#endif
+
+    /*
+     * Physical and virtual handlers.
+     */
+    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,     true, pgmR3RelocatePhysHandler,      &offDelta);
+    pVM->pgm.s.pLastPhysHandlerRC = NIL_RTRCPTR;
+    RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->VirtHandlers,      true, pgmR3RelocateVirtHandler,      &offDelta);
+    RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3RelocateHyperVirtHandler, &offDelta);
+
+    /*
+     * The page pool.
+     */
+    pgmR3PoolRelocate(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Statistics.
+     */
+    pVM->pgm.s.pStatsRC = MMHyperCCToRC(pVM, pVM->pgm.s.pStatsR3);
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+        pVM->aCpus[iCpu].pgm.s.pStatsRC = MMHyperCCToRC(pVM, pVM->aCpus[iCpu].pgm.s.pStatsR3);
+#endif
+}
+
+
+/**
+ * Callback function for relocating a physical access handler.
+ *
+ * @returns 0 (continue enum)
+ * @param   pNode       Pointer to a PGMPHYSHANDLER node.
+ * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
+ *                      not certain the delta will fit in a void pointer for all possible configs.
+ */
+static DECLCALLBACK(int) pgmR3RelocatePhysHandler(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMPHYSHANDLER pHandler = (PPGMPHYSHANDLER)pNode;
+    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
+    if (pHandler->pfnHandlerRC)
+        pHandler->pfnHandlerRC += offDelta;
+    if (pHandler->pvUserRC >= 0x10000)
+        pHandler->pvUserRC += offDelta;
+    return 0;
+}
+
+
+/**
+ * Callback function for relocating a virtual access handler.
+ *
+ * @returns 0 (continue enum)
+ * @param   pNode       Pointer to a PGMVIRTHANDLER node.
+ * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
+ *                      not certain the delta will fit in a void pointer for all possible configs.
+ */
+static DECLCALLBACK(int) pgmR3RelocateVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
+{
+    PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
+    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
+    Assert(     pHandler->enmType == PGMVIRTHANDLERTYPE_ALL
+           ||   pHandler->enmType == PGMVIRTHANDLERTYPE_WRITE);
+    Assert(pHandler->pfnHandlerRC);
+    pHandler->pfnHandlerRC += offDelta;
+    return 0;
+}
+
+
+/**
+ * Callback function for relocating a virtual access handler for the hypervisor mapping.
+ *
+ * @returns 0 (continue enum)
+ * @param   pNode       Pointer to a PGMVIRTHANDLER node.
+ * @param   pvUser      Pointer to the offDelta. This is a pointer to the delta since we're
+ *                      not certain the delta will fit in a void pointer for all possible configs.
+ */
+static DECLCALLBACK(int) pgmR3RelocateHyperVirtHandler(PAVLROGCPTRNODECORE pNode, void *pvUser)
+{
+    PPGMVIRTHANDLER pHandler = (PPGMVIRTHANDLER)pNode;
+    RTGCINTPTR      offDelta = *(PRTGCINTPTR)pvUser;
+    Assert(pHandler->enmType == PGMVIRTHANDLERTYPE_HYPERVISOR);
+    Assert(pHandler->pfnHandlerRC);
+    pHandler->pfnHandlerRC  += offDelta;
+    return 0;
+}
+
+
+/**
+ * Resets a virtual CPU when unplugged.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pVCpu               The virtual CPU handle.
+ */
+VMMR3DECL(void) PGMR3ResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
+{
+    int rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
+    AssertRC(rc);
+
+    rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
+    AssertRC(rc);
+
+    STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
+
+    pgmR3PoolResetUnpluggedCpu(pVM, pVCpu);
+
+    /*
+     * Re-init other members.
+     */
+    pVCpu->pgm.s.fA20Enabled = true;
+
+    /*
+     * Clear the FFs PGM owns.
+     */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the PGM component this means that any PD write monitors
+ * needs to be removed.
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) PGMR3Reset(PVM pVM)
+{
+    int rc;
+
+    LogFlow(("PGMR3Reset:\n"));
+    VM_ASSERT_EMT(pVM);
+
+    pgmLock(pVM);
+
+    /*
+     * Unfix any fixed mappings and disable CR3 monitoring.
+     */
+    pVM->pgm.s.fMappingsFixed         = false;
+    pVM->pgm.s.fMappingsFixedRestored = false;
+    pVM->pgm.s.GCPtrMappingFixed      = NIL_RTGCPTR;
+    pVM->pgm.s.cbMappingFixed         = 0;
+
+    /*
+     * Exit the guest paging mode before the pgm pool gets reset.
+     * Important to clean up the amd64 case.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU  pVCpu = &pVM->aCpus[i];
+        rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
+        AssertRC(rc);
+    }
+
+#ifdef DEBUG
+    DBGFR3InfoLog(pVM, "mappings", NULL);
+    DBGFR3InfoLog(pVM, "handlers", "all nostat");
+#endif
+
+    /*
+     * Switch mode back to real mode. (before resetting the pgm pool!)
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU  pVCpu = &pVM->aCpus[i];
+
+        rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
+        AssertRC(rc);
+
+        STAM_REL_COUNTER_RESET(&pVCpu->pgm.s.cGuestModeChanges);
+    }
+
+    /*
+     * Reset the shadow page pool.
+     */
+    pgmR3PoolReset(pVM);
+
+    /*
+     * Re-init various other members and clear the FFs that PGM owns.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        pVCpu->pgm.s.fA20Enabled = true;
+        pVCpu->pgm.s.fGst32BitPageSizeExtension = false;
+        PGMNotifyNxeChanged(pVCpu, false);
+
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+    }
+
+    /*
+     * Reset (zero) RAM and shadow ROM pages.
+     */
+    rc = pgmR3PhysRamReset(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3PhysRomReset(pVM);
+
+
+    pgmUnlock(pVM);
+    AssertReleaseRC(rc);
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * VM state change callback for clearing fNoMorePhysWrites after
+ * a snapshot has been created.
+ */
+static DECLCALLBACK(void) pgmR3ResetNoMorePhysWritesFlag(PVM pVM, VMSTATE enmState, VMSTATE enmOldState, void *pvUser)
+{
+    if (   enmState == VMSTATE_RUNNING
+        || enmState == VMSTATE_RESUMING)
+        pVM->pgm.s.fNoMorePhysWrites = false;
+}
+#endif
+
+/**
+ * Private API to reset fNoMorePhysWrites.
+ */
+VMMR3DECL(void) PGMR3ResetNoMorePhysWritesFlag(PVM pVM)
+{
+    pVM->pgm.s.fNoMorePhysWrites = false;
+}
+
+/**
+ * Terminates the PGM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to VM structure.
+ */
+VMMR3DECL(int) PGMR3Term(PVM pVM)
+{
+    /* Must free shared pages here. */
+    pgmLock(pVM);
+    pgmR3PhysRamTerm(pVM);
+    pgmR3PhysRomTerm(pVM);
+    pgmUnlock(pVM);
+
+    PGMDeregisterStringFormatTypes();
+    return PDMR3CritSectDelete(&pVM->pgm.s.CritSect);
+}
+
+
+/**
+ * Show paging mode.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     "all" (default), "guest", "shadow" or "host".
+ */
+static DECLCALLBACK(void) pgmR3InfoMode(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /* digest argument. */
+    bool fGuest, fShadow, fHost;
+    if (pszArgs)
+        pszArgs = RTStrStripL(pszArgs);
+    if (!pszArgs || !*pszArgs || strstr(pszArgs, "all"))
+        fShadow = fHost = fGuest = true;
+    else
+    {
+        fShadow = fHost = fGuest = false;
+        if (strstr(pszArgs, "guest"))
+            fGuest = true;
+        if (strstr(pszArgs, "shadow"))
+            fShadow = true;
+        if (strstr(pszArgs, "host"))
+            fHost = true;
+    }
+
+    /** @todo SMP support! */
+    /* print info. */
+    if (fGuest)
+        pHlp->pfnPrintf(pHlp, "Guest paging mode:  %s, changed %RU64 times, A20 %s\n",
+                        PGMGetModeName(pVM->aCpus[0].pgm.s.enmGuestMode), pVM->aCpus[0].pgm.s.cGuestModeChanges.c,
+                        pVM->aCpus[0].pgm.s.fA20Enabled ? "enabled" : "disabled");
+    if (fShadow)
+        pHlp->pfnPrintf(pHlp, "Shadow paging mode: %s\n", PGMGetModeName(pVM->aCpus[0].pgm.s.enmShadowMode));
+    if (fHost)
+    {
+        const char *psz;
+        switch (pVM->pgm.s.enmHostMode)
+        {
+            case SUPPAGINGMODE_INVALID:             psz = "invalid"; break;
+            case SUPPAGINGMODE_32_BIT:              psz = "32-bit"; break;
+            case SUPPAGINGMODE_32_BIT_GLOBAL:       psz = "32-bit+G"; break;
+            case SUPPAGINGMODE_PAE:                 psz = "PAE"; break;
+            case SUPPAGINGMODE_PAE_GLOBAL:          psz = "PAE+G"; break;
+            case SUPPAGINGMODE_PAE_NX:              psz = "PAE+NX"; break;
+            case SUPPAGINGMODE_PAE_GLOBAL_NX:       psz = "PAE+G+NX"; break;
+            case SUPPAGINGMODE_AMD64:               psz = "AMD64"; break;
+            case SUPPAGINGMODE_AMD64_GLOBAL:        psz = "AMD64+G"; break;
+            case SUPPAGINGMODE_AMD64_NX:            psz = "AMD64+NX"; break;
+            case SUPPAGINGMODE_AMD64_GLOBAL_NX:     psz = "AMD64+G+NX"; break;
+            default:                                psz = "unknown"; break;
+        }
+        pHlp->pfnPrintf(pHlp, "Host paging mode:   %s\n", psz);
+    }
+}
+
+
+/**
+ * Dump registered MMIO ranges to the log.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) pgmR3PhysInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    pHlp->pfnPrintf(pHlp,
+                    "RAM ranges (pVM=%p)\n"
+                    "%.*s %.*s\n",
+                    pVM,
+                    sizeof(RTGCPHYS) * 4 + 1, "GC Phys Range                    ",
+                    sizeof(RTHCPTR) * 2,      "pvHC            ");
+
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        pHlp->pfnPrintf(pHlp,
+                        "%RGp-%RGp %RHv %s\n",
+                        pCur->GCPhys,
+                        pCur->GCPhysLast,
+                        pCur->pvR3,
+                        pCur->pszDesc);
+}
+
+/**
+ * Dump the page directory to the log.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) pgmR3InfoCr3(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /** @todo SMP support!! */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+/** @todo fix this! Convert the PGMR3DumpHierarchyHC functions to do guest stuff. */
+    /* Big pages supported? */
+    const bool fPSE  = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PSE);
+
+    /* Global pages supported? */
+    const bool fPGE = !!(CPUMGetGuestCR4(pVCpu) & X86_CR4_PGE);
+
+    NOREF(pszArgs);
+
+    /*
+     * Get page directory addresses.
+     */
+    PX86PD     pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+    Assert(pPDSrc);
+    Assert(PGMPhysGCPhys2R3PtrAssert(pVM, (RTGCPHYS)(CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK), sizeof(*pPDSrc)) == pPDSrc);
+
+    /*
+     * Iterate the page directory.
+     */
+    for (unsigned iPD = 0; iPD < RT_ELEMENTS(pPDSrc->a); iPD++)
+    {
+        X86PDE PdeSrc = pPDSrc->a[iPD];
+        if (PdeSrc.n.u1Present)
+        {
+            if (PdeSrc.b.u1Size && fPSE)
+                pHlp->pfnPrintf(pHlp,
+                                "%04X - %RGp P=%d U=%d RW=%d G=%d - BIG\n",
+                                iPD,
+                                pgmGstGet4MBPhysPage(&pVM->pgm.s, PdeSrc),
+                                PdeSrc.b.u1Present, PdeSrc.b.u1User, PdeSrc.b.u1Write, PdeSrc.b.u1Global && fPGE);
+            else
+                pHlp->pfnPrintf(pHlp,
+                                "%04X - %RGp P=%d U=%d RW=%d [G=%d]\n",
+                                iPD,
+                                (RTGCPHYS)(PdeSrc.u & X86_PDE_PG_MASK),
+                                PdeSrc.n.u1Present, PdeSrc.n.u1User, PdeSrc.n.u1Write, PdeSrc.b.u1Global && fPGE);
+        }
+    }
+}
+
+
+/**
+ * Service a VMMCALLRING3_PGM_LOCK call.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+VMMR3DECL(int) PGMR3LockCall(PVM pVM)
+{
+    int rc = PDMR3CritSectEnterEx(&pVM->pgm.s.CritSect, true /* fHostCall */);
+    AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Converts a PGMMODE value to a PGM_TYPE_* \#define.
+ *
+ * @returns PGM_TYPE_*.
+ * @param   pgmMode     The mode value to convert.
+ */
+DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
+{
+    switch (pgmMode)
+    {
+        case PGMMODE_REAL:      return PGM_TYPE_REAL;
+        case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
+        case PGMMODE_32_BIT:    return PGM_TYPE_32BIT;
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:    return PGM_TYPE_PAE;
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:  return PGM_TYPE_AMD64;
+        case PGMMODE_NESTED:    return PGM_TYPE_NESTED;
+        case PGMMODE_EPT:       return PGM_TYPE_EPT;
+        default:
+            AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
+    }
+}
+
+
+/**
+ * Gets the index into the paging mode data array of a SHW+GST mode.
+ *
+ * @returns PGM::paPagingData index.
+ * @param   uShwType      The shadow paging mode type.
+ * @param   uGstType      The guest paging mode type.
+ */
+DECLINLINE(unsigned) pgmModeDataIndex(unsigned uShwType, unsigned uGstType)
+{
+    Assert(uShwType >= PGM_TYPE_32BIT && uShwType <= PGM_TYPE_MAX);
+    Assert(uGstType >= PGM_TYPE_REAL  && uGstType <= PGM_TYPE_AMD64);
+    return (uShwType - PGM_TYPE_32BIT) * (PGM_TYPE_AMD64 - PGM_TYPE_REAL + 1)
+         + (uGstType - PGM_TYPE_REAL);
+}
+
+
+/**
+ * Gets the index into the paging mode data array of a SHW+GST mode.
+ *
+ * @returns PGM::paPagingData index.
+ * @param   enmShw      The shadow paging mode.
+ * @param   enmGst      The guest paging mode.
+ */
+DECLINLINE(unsigned) pgmModeDataIndexByMode(PGMMODE enmShw, PGMMODE enmGst)
+{
+    Assert(enmShw >= PGMMODE_32_BIT && enmShw <= PGMMODE_MAX);
+    Assert(enmGst > PGMMODE_INVALID && enmGst < PGMMODE_MAX);
+    return pgmModeDataIndex(pgmModeToType(enmShw), pgmModeToType(enmGst));
+}
+
+
+/**
+ * Calculates the max data index.
+ * @returns The number of entries in the paging data array.
+ */
+DECLINLINE(unsigned) pgmModeDataMaxIndex(void)
+{
+    return pgmModeDataIndex(PGM_TYPE_MAX, PGM_TYPE_AMD64) + 1;
+}
+
+
+/**
+ * Initializes the paging mode data kept in PGM::paModeData.
+ *
+ * @param   pVM             The VM handle.
+ * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
+ *                          This is used early in the init process to avoid trouble with PDM
+ *                          not being initialized yet.
+ */
+static int pgmR3ModeDataInit(PVM pVM, bool fResolveGCAndR0)
+{
+    PPGMMODEDATA pModeData;
+    int rc;
+
+    /*
+     * Allocate the array on the first call.
+     */
+    if (!pVM->pgm.s.paModeData)
+    {
+        pVM->pgm.s.paModeData = (PPGMMODEDATA)MMR3HeapAllocZ(pVM, MM_TAG_PGM, sizeof(PGMMODEDATA) * pgmModeDataMaxIndex());
+        AssertReturn(pVM->pgm.s.paModeData, VERR_NO_MEMORY);
+    }
+
+    /*
+     * Initialize the array entries.
+     */
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_REAL)];
+    pModeData->uShwType = PGM_TYPE_32BIT;
+    pModeData->uGstType = PGM_TYPE_REAL;
+    rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_REAL(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_32BIT_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGMMODE_PROTECTED)];
+    pModeData->uShwType = PGM_TYPE_32BIT;
+    pModeData->uGstType = PGM_TYPE_PROT;
+    rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_PROT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_32BIT_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_32BIT, PGM_TYPE_32BIT)];
+    pModeData->uShwType = PGM_TYPE_32BIT;
+    pModeData->uGstType = PGM_TYPE_32BIT;
+    rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_32BIT_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_REAL)];
+    pModeData->uShwType = PGM_TYPE_PAE;
+    pModeData->uGstType = PGM_TYPE_REAL;
+    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_REAL(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_PAE_REAL(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PROT)];
+    pModeData->uShwType = PGM_TYPE_PAE;
+    pModeData->uGstType = PGM_TYPE_PROT;
+    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_PROT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_PAE_PROT(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_32BIT)];
+    pModeData->uShwType = PGM_TYPE_PAE;
+    pModeData->uGstType = PGM_TYPE_32BIT;
+    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_PAE_32BIT(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_PAE, PGM_TYPE_PAE)];
+    pModeData->uShwType = PGM_TYPE_PAE;
+    pModeData->uGstType = PGM_TYPE_PAE;
+    rc = PGM_SHW_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_PAE_PAE(InitData)(    pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_AMD64, PGM_TYPE_AMD64)];
+    pModeData->uShwType = PGM_TYPE_AMD64;
+    pModeData->uGstType = PGM_TYPE_AMD64;
+    rc = PGM_SHW_NAME_AMD64(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_AMD64(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_AMD64_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+#endif
+
+    /* The nested paging mode. */
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_REAL)];
+    pModeData->uShwType = PGM_TYPE_NESTED;
+    pModeData->uGstType = PGM_TYPE_REAL;
+    rc = PGM_GST_NAME_REAL(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_NESTED_REAL(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGMMODE_PROTECTED)];
+    pModeData->uShwType = PGM_TYPE_NESTED;
+    pModeData->uGstType = PGM_TYPE_PROT;
+    rc = PGM_GST_NAME_PROT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_NESTED_PROT(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_32BIT)];
+    pModeData->uShwType = PGM_TYPE_NESTED;
+    pModeData->uGstType = PGM_TYPE_32BIT;
+    rc = PGM_GST_NAME_32BIT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_NESTED_32BIT(InitData)(pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_PAE)];
+    pModeData->uShwType = PGM_TYPE_NESTED;
+    pModeData->uGstType = PGM_TYPE_PAE;
+    rc = PGM_GST_NAME_PAE(InitData)(         pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_NESTED_PAE(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
+    pModeData->uShwType = PGM_TYPE_NESTED;
+    pModeData->uGstType = PGM_TYPE_AMD64;
+    rc = PGM_GST_NAME_AMD64(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_NESTED_AMD64(InitData)( pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+#endif
+
+    /* The shadow part of the nested callback mode depends on the host paging mode (AMD-V only). */
+    switch (pVM->pgm.s.enmHostMode)
+    {
+#if HC_ARCH_BITS == 32
+    case SUPPAGINGMODE_32_BIT:
+    case SUPPAGINGMODE_32_BIT_GLOBAL:
+        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
+        {
+            pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
+            rc = PGM_SHW_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+        }
+# ifdef VBOX_WITH_64_BITS_GUESTS
+        pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
+        rc = PGM_SHW_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+# endif
+        break;
+
+    case SUPPAGINGMODE_PAE:
+    case SUPPAGINGMODE_PAE_NX:
+    case SUPPAGINGMODE_PAE_GLOBAL:
+    case SUPPAGINGMODE_PAE_GLOBAL_NX:
+        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
+        {
+            pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
+            rc = PGM_SHW_NAME_PAE(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+        }
+# ifdef VBOX_WITH_64_BITS_GUESTS
+        pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, PGM_TYPE_AMD64)];
+        rc = PGM_SHW_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+# endif
+        break;
+#endif /* HC_ARCH_BITS == 32 */
+
+#if HC_ARCH_BITS == 64 || defined(RT_OS_DARWIN)
+    case SUPPAGINGMODE_AMD64:
+    case SUPPAGINGMODE_AMD64_GLOBAL:
+    case SUPPAGINGMODE_AMD64_NX:
+    case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+# ifdef VBOX_WITH_64_BITS_GUESTS
+        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_AMD64; i++)
+# else
+        for (unsigned i = PGM_TYPE_REAL; i <= PGM_TYPE_PAE; i++)
+# endif
+        {
+            pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_NESTED, i)];
+            rc = PGM_SHW_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+        }
+        break;
+#endif /* HC_ARCH_BITS == 64 || RT_OS_DARWIN */
+
+    default:
+        AssertFailed();
+        break;
+    }
+
+    /* Extended paging (EPT) / Intel VT-x */
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_REAL)];
+    pModeData->uShwType = PGM_TYPE_EPT;
+    pModeData->uGstType = PGM_TYPE_REAL;
+    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_REAL(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_EPT_REAL(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PROT)];
+    pModeData->uShwType = PGM_TYPE_EPT;
+    pModeData->uGstType = PGM_TYPE_PROT;
+    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_PROT(InitData)(       pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_EPT_PROT(InitData)(   pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_32BIT)];
+    pModeData->uShwType = PGM_TYPE_EPT;
+    pModeData->uGstType = PGM_TYPE_32BIT;
+    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_32BIT(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_EPT_32BIT(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_PAE)];
+    pModeData->uShwType = PGM_TYPE_EPT;
+    pModeData->uGstType = PGM_TYPE_PAE;
+    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_PAE(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_EPT_PAE(InitData)(    pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndex(PGM_TYPE_EPT, PGM_TYPE_AMD64)];
+    pModeData->uShwType = PGM_TYPE_EPT;
+    pModeData->uGstType = PGM_TYPE_AMD64;
+    rc = PGM_SHW_NAME_EPT(InitData)(        pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_GST_NAME_AMD64(InitData)(      pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+    rc = PGM_BTH_NAME_EPT_AMD64(InitData)(  pVM, pModeData, fResolveGCAndR0); AssertRCReturn(rc, rc);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Switch to different (or relocated in the relocate case) mode data.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   enmShw      The the shadow paging mode.
+ * @param   enmGst      The the guest paging mode.
+ */
+static void pgmR3ModeDataSwitch(PVM pVM, PVMCPU pVCpu, PGMMODE enmShw, PGMMODE enmGst)
+{
+    PPGMMODEDATA pModeData = &pVM->pgm.s.paModeData[pgmModeDataIndexByMode(enmShw, enmGst)];
+
+    Assert(pModeData->uGstType == pgmModeToType(enmGst));
+    Assert(pModeData->uShwType == pgmModeToType(enmShw));
+
+    /* shadow */
+    pVCpu->pgm.s.pfnR3ShwRelocate             = pModeData->pfnR3ShwRelocate;
+    pVCpu->pgm.s.pfnR3ShwExit                 = pModeData->pfnR3ShwExit;
+    pVCpu->pgm.s.pfnR3ShwGetPage              = pModeData->pfnR3ShwGetPage;
+    Assert(pVCpu->pgm.s.pfnR3ShwGetPage);
+    pVCpu->pgm.s.pfnR3ShwModifyPage           = pModeData->pfnR3ShwModifyPage;
+
+    pVCpu->pgm.s.pfnRCShwGetPage              = pModeData->pfnRCShwGetPage;
+    pVCpu->pgm.s.pfnRCShwModifyPage           = pModeData->pfnRCShwModifyPage;
+
+    pVCpu->pgm.s.pfnR0ShwGetPage              = pModeData->pfnR0ShwGetPage;
+    pVCpu->pgm.s.pfnR0ShwModifyPage           = pModeData->pfnR0ShwModifyPage;
+
+
+    /* guest */
+    pVCpu->pgm.s.pfnR3GstRelocate             = pModeData->pfnR3GstRelocate;
+    pVCpu->pgm.s.pfnR3GstExit                 = pModeData->pfnR3GstExit;
+    pVCpu->pgm.s.pfnR3GstGetPage              = pModeData->pfnR3GstGetPage;
+    Assert(pVCpu->pgm.s.pfnR3GstGetPage);
+    pVCpu->pgm.s.pfnR3GstModifyPage           = pModeData->pfnR3GstModifyPage;
+    pVCpu->pgm.s.pfnR3GstGetPDE               = pModeData->pfnR3GstGetPDE;
+    pVCpu->pgm.s.pfnRCGstGetPage              = pModeData->pfnRCGstGetPage;
+    pVCpu->pgm.s.pfnRCGstModifyPage           = pModeData->pfnRCGstModifyPage;
+    pVCpu->pgm.s.pfnRCGstGetPDE               = pModeData->pfnRCGstGetPDE;
+    pVCpu->pgm.s.pfnR0GstGetPage              = pModeData->pfnR0GstGetPage;
+    pVCpu->pgm.s.pfnR0GstModifyPage           = pModeData->pfnR0GstModifyPage;
+    pVCpu->pgm.s.pfnR0GstGetPDE               = pModeData->pfnR0GstGetPDE;
+
+    /* both */
+    pVCpu->pgm.s.pfnR3BthRelocate             = pModeData->pfnR3BthRelocate;
+    pVCpu->pgm.s.pfnR3BthInvalidatePage       = pModeData->pfnR3BthInvalidatePage;
+    pVCpu->pgm.s.pfnR3BthSyncCR3              = pModeData->pfnR3BthSyncCR3;
+    Assert(pVCpu->pgm.s.pfnR3BthSyncCR3);
+    pVCpu->pgm.s.pfnR3BthPrefetchPage         = pModeData->pfnR3BthPrefetchPage;
+    pVCpu->pgm.s.pfnR3BthVerifyAccessSyncPage = pModeData->pfnR3BthVerifyAccessSyncPage;
+#ifdef VBOX_STRICT
+    pVCpu->pgm.s.pfnR3BthAssertCR3            = pModeData->pfnR3BthAssertCR3;
+#endif
+    pVCpu->pgm.s.pfnR3BthMapCR3               = pModeData->pfnR3BthMapCR3;
+    pVCpu->pgm.s.pfnR3BthUnmapCR3             = pModeData->pfnR3BthUnmapCR3;
+
+    pVCpu->pgm.s.pfnRCBthTrap0eHandler        = pModeData->pfnRCBthTrap0eHandler;
+    pVCpu->pgm.s.pfnRCBthInvalidatePage       = pModeData->pfnRCBthInvalidatePage;
+    pVCpu->pgm.s.pfnRCBthSyncCR3              = pModeData->pfnRCBthSyncCR3;
+    pVCpu->pgm.s.pfnRCBthPrefetchPage         = pModeData->pfnRCBthPrefetchPage;
+    pVCpu->pgm.s.pfnRCBthVerifyAccessSyncPage = pModeData->pfnRCBthVerifyAccessSyncPage;
+#ifdef VBOX_STRICT
+    pVCpu->pgm.s.pfnRCBthAssertCR3            = pModeData->pfnRCBthAssertCR3;
+#endif
+    pVCpu->pgm.s.pfnRCBthMapCR3               = pModeData->pfnRCBthMapCR3;
+    pVCpu->pgm.s.pfnRCBthUnmapCR3             = pModeData->pfnRCBthUnmapCR3;
+
+    pVCpu->pgm.s.pfnR0BthTrap0eHandler        = pModeData->pfnR0BthTrap0eHandler;
+    pVCpu->pgm.s.pfnR0BthInvalidatePage       = pModeData->pfnR0BthInvalidatePage;
+    pVCpu->pgm.s.pfnR0BthSyncCR3              = pModeData->pfnR0BthSyncCR3;
+    pVCpu->pgm.s.pfnR0BthPrefetchPage         = pModeData->pfnR0BthPrefetchPage;
+    pVCpu->pgm.s.pfnR0BthVerifyAccessSyncPage = pModeData->pfnR0BthVerifyAccessSyncPage;
+#ifdef VBOX_STRICT
+    pVCpu->pgm.s.pfnR0BthAssertCR3            = pModeData->pfnR0BthAssertCR3;
+#endif
+    pVCpu->pgm.s.pfnR0BthMapCR3               = pModeData->pfnR0BthMapCR3;
+    pVCpu->pgm.s.pfnR0BthUnmapCR3             = pModeData->pfnR0BthUnmapCR3;
+}
+
+
+/**
+ * Calculates the shadow paging mode.
+ *
+ * @returns The shadow paging mode.
+ * @param   pVM             VM handle.
+ * @param   enmGuestMode    The guest mode.
+ * @param   enmHostMode     The host mode.
+ * @param   enmShadowMode   The current shadow mode.
+ * @param   penmSwitcher    Where to store the switcher to use.
+ *                          VMMSWITCHER_INVALID means no change.
+ */
+static PGMMODE pgmR3CalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode, VMMSWITCHER *penmSwitcher)
+{
+    VMMSWITCHER enmSwitcher = VMMSWITCHER_INVALID;
+    switch (enmGuestMode)
+    {
+        /*
+         * When switching to real or protected mode we don't change
+         * anything since it's likely that we'll switch back pretty soon.
+         *
+         * During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
+         * and is supposed to determine which shadow paging and switcher to
+         * use during init.
+         */
+        case PGMMODE_REAL:
+        case PGMMODE_PROTECTED:
+            if (    enmShadowMode != PGMMODE_INVALID
+                && !HWACCMIsEnabled(pVM) /* always switch in hwaccm mode! */)
+                break; /* (no change) */
+
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_32_BIT;
+                    enmSwitcher = VMMSWITCHER_32_TO_32;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
+#ifdef DEBUG_bird
+                    if (RTEnvExist("VBOX_32BIT"))
+                    {
+                        enmShadowMode = PGMMODE_32_BIT;
+                        enmSwitcher = VMMSWITCHER_PAE_TO_32;
+                    }
+#endif
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
+#ifdef DEBUG_bird
+                    if (RTEnvExist("VBOX_32BIT"))
+                    {
+                        enmShadowMode = PGMMODE_32_BIT;
+                        enmSwitcher = VMMSWITCHER_AMD64_TO_32;
+                    }
+#endif
+                    break;
+
+                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
+            }
+            break;
+
+        case PGMMODE_32_BIT:
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_32_BIT;
+                    enmSwitcher = VMMSWITCHER_32_TO_32;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
+#ifdef DEBUG_bird
+                    if (RTEnvExist("VBOX_32BIT"))
+                    {
+                        enmShadowMode = PGMMODE_32_BIT;
+                        enmSwitcher = VMMSWITCHER_PAE_TO_32;
+                    }
+#endif
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
+#ifdef DEBUG_bird
+                    if (RTEnvExist("VBOX_32BIT"))
+                    {
+                        enmShadowMode = PGMMODE_32_BIT;
+                        enmSwitcher = VMMSWITCHER_AMD64_TO_32;
+                    }
+#endif
+                    break;
+
+                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
+            }
+            break;
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_PAE;
+                    enmSwitcher = VMMSWITCHER_32_TO_PAE;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    enmSwitcher = VMMSWITCHER_PAE_TO_PAE;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    enmSwitcher = VMMSWITCHER_AMD64_TO_PAE;
+                    break;
+
+                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
+            }
+            break;
+
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_AMD64;
+                    enmSwitcher = VMMSWITCHER_32_TO_AMD64;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_AMD64;
+                    enmSwitcher = VMMSWITCHER_PAE_TO_AMD64;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_AMD64;
+                    enmSwitcher = VMMSWITCHER_AMD64_TO_AMD64;
+                    break;
+
+                default: AssertMsgFailed(("enmHostMode=%d\n", enmHostMode)); break;
+            }
+            break;
+
+
+        default:
+            AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
+            *penmSwitcher = VMMSWITCHER_INVALID;
+            return PGMMODE_INVALID;
+    }
+    /* Override the shadow mode is nested paging is active. */
+    pVM->pgm.s.fNestedPaging = HWACCMIsNestedPagingActive(pVM);
+    if (pVM->pgm.s.fNestedPaging)
+        enmShadowMode = HWACCMGetShwPagingMode(pVM);
+
+    *penmSwitcher = enmSwitcher;
+    return enmShadowMode;
+}
+
+
+/**
+ * Performs the actual mode change.
+ * This is called by PGMChangeMode and pgmR3InitPaging().
+ *
+ * @returns VBox status code. May suspend or power off the VM on error, but this
+ *          will trigger using FFs and not status codes.
+ *
+ * @param   pVM             VM handle.
+ * @param   pVCpu           The VMCPU to operate on.
+ * @param   enmGuestMode    The new guest mode. This is assumed to be different from
+ *                          the current mode.
+ */
+VMMR3DECL(int) PGMR3ChangeMode(PVM pVM, PVMCPU pVCpu, PGMMODE enmGuestMode)
+{
+    bool fIsOldGuestPagingMode64Bits = (pVCpu->pgm.s.enmGuestMode >= PGMMODE_AMD64);
+    bool fIsNewGuestPagingMode64Bits = (enmGuestMode >= PGMMODE_AMD64);
+
+    Log(("PGMR3ChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
+    STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
+
+    /*
+     * Calc the shadow mode and switcher.
+     */
+    VMMSWITCHER enmSwitcher;
+    PGMMODE     enmShadowMode = pgmR3CalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode, &enmSwitcher);
+
+#ifdef VBOX_WITH_RAW_MODE
+    if (enmSwitcher != VMMSWITCHER_INVALID)
+    {
+        /*
+         * Select new switcher.
+         */
+        int rc = VMMR3SelectSwitcher(pVM, enmSwitcher);
+        if (RT_FAILURE(rc))
+        {
+            AssertReleaseMsgFailed(("VMMR3SelectSwitcher(%d) -> %Rrc\n", enmSwitcher, rc));
+            return rc;
+        }
+    }
+#endif
+
+    /*
+     * Exit old mode(s).
+     */
+#if HC_ARCH_BITS == 32
+    /* The nested shadow paging mode for AMD-V does change when running 64 bits guests on 32 bits hosts; typically PAE <-> AMD64 */
+    const bool fForceShwEnterExit = (    fIsOldGuestPagingMode64Bits != fIsNewGuestPagingMode64Bits
+                                     &&  enmShadowMode == PGMMODE_NESTED);
+#else
+    const bool fForceShwEnterExit = false;
+#endif
+    /* shadow */
+    if (    enmShadowMode != pVCpu->pgm.s.enmShadowMode
+        ||  fForceShwEnterExit)
+    {
+        LogFlow(("PGMR3ChangeMode: Shadow mode: %s -> %s\n",  PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
+        if (PGM_SHW_PFN(Exit, pVCpu))
+        {
+            int rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsgFailed(("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc));
+                return rc;
+            }
+        }
+
+    }
+    else
+        LogFlow(("PGMR3ChangeMode: Shadow mode remains: %s\n",  PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
+
+    /* guest */
+    if (PGM_GST_PFN(Exit, pVCpu))
+    {
+        int rc = PGM_GST_PFN(Exit, pVCpu)(pVCpu);
+        if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc));
+            return rc;
+        }
+    }
+
+    /*
+     * Load new paging mode data.
+     */
+    pgmR3ModeDataSwitch(pVM, pVCpu, enmShadowMode, enmGuestMode);
+
+    /*
+     * Enter new shadow mode (if changed).
+     */
+    if (    enmShadowMode != pVCpu->pgm.s.enmShadowMode
+        ||  fForceShwEnterExit)
+    {
+        int rc;
+        pVCpu->pgm.s.enmShadowMode = enmShadowMode;
+        switch (enmShadowMode)
+        {
+            case PGMMODE_32_BIT:
+                rc = PGM_SHW_NAME_32BIT(Enter)(pVCpu, false);
+                break;
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+                rc = PGM_SHW_NAME_PAE(Enter)(pVCpu, false);
+                break;
+            case PGMMODE_AMD64:
+            case PGMMODE_AMD64_NX:
+                rc = PGM_SHW_NAME_AMD64(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
+                break;
+            case PGMMODE_NESTED:
+                rc = PGM_SHW_NAME_NESTED(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
+                break;
+            case PGMMODE_EPT:
+                rc = PGM_SHW_NAME_EPT(Enter)(pVCpu, fIsNewGuestPagingMode64Bits);
+                break;
+            case PGMMODE_REAL:
+            case PGMMODE_PROTECTED:
+            default:
+                AssertReleaseMsgFailed(("enmShadowMode=%d\n", enmShadowMode));
+                return VERR_INTERNAL_ERROR;
+        }
+        if (RT_FAILURE(rc))
+        {
+            AssertReleaseMsgFailed(("Entering enmShadowMode=%d failed: %Rrc\n", enmShadowMode, rc));
+            pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
+            return rc;
+        }
+    }
+
+    /*
+     * Always flag the necessary updates
+     */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    /*
+     * Enter the new guest and shadow+guest modes.
+     */
+    int rc = -1;
+    int rc2 = -1;
+    RTGCPHYS GCPhysCR3 = NIL_RTGCPHYS;
+    pVCpu->pgm.s.enmGuestMode = enmGuestMode;
+    switch (enmGuestMode)
+    {
+        case PGMMODE_REAL:
+            rc = PGM_GST_NAME_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
+            switch (pVCpu->pgm.s.enmShadowMode)
+            {
+                case PGMMODE_32_BIT:
+                    rc2 = PGM_BTH_NAME_32BIT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_PAE:
+                case PGMMODE_PAE_NX:
+                    rc2 = PGM_BTH_NAME_PAE_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_NESTED:
+                    rc2 = PGM_BTH_NAME_NESTED_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_EPT:
+                    rc2 = PGM_BTH_NAME_EPT_REAL(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_AMD64:
+                case PGMMODE_AMD64_NX:
+                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
+                default: AssertFailed(); break;
+            }
+            break;
+
+        case PGMMODE_PROTECTED:
+            rc = PGM_GST_NAME_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
+            switch (pVCpu->pgm.s.enmShadowMode)
+            {
+                case PGMMODE_32_BIT:
+                    rc2 = PGM_BTH_NAME_32BIT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_PAE:
+                case PGMMODE_PAE_NX:
+                    rc2 = PGM_BTH_NAME_PAE_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_NESTED:
+                    rc2 = PGM_BTH_NAME_NESTED_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_EPT:
+                    rc2 = PGM_BTH_NAME_EPT_PROT(Enter)(pVCpu, NIL_RTGCPHYS);
+                    break;
+                case PGMMODE_AMD64:
+                case PGMMODE_AMD64_NX:
+                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
+                default: AssertFailed(); break;
+            }
+            break;
+
+        case PGMMODE_32_BIT:
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
+            rc = PGM_GST_NAME_32BIT(Enter)(pVCpu, GCPhysCR3);
+            switch (pVCpu->pgm.s.enmShadowMode)
+            {
+                case PGMMODE_32_BIT:
+                    rc2 = PGM_BTH_NAME_32BIT_32BIT(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_PAE:
+                case PGMMODE_PAE_NX:
+                    rc2 = PGM_BTH_NAME_PAE_32BIT(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_NESTED:
+                    rc2 = PGM_BTH_NAME_NESTED_32BIT(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_EPT:
+                    rc2 = PGM_BTH_NAME_EPT_32BIT(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_AMD64:
+                case PGMMODE_AMD64_NX:
+                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
+                default: AssertFailed(); break;
+            }
+            break;
+
+        case PGMMODE_PAE_NX:
+        case PGMMODE_PAE:
+        {
+            uint32_t u32Dummy, u32Features;
+
+            CPUMGetGuestCpuId(pVCpu, 1, &u32Dummy, &u32Dummy, &u32Dummy, &u32Features);
+            if (!(u32Features & X86_CPUID_FEATURE_EDX_PAE))
+                return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
+                                         N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (General/Advanced)"));
+
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
+            rc = PGM_GST_NAME_PAE(Enter)(pVCpu, GCPhysCR3);
+            switch (pVCpu->pgm.s.enmShadowMode)
+            {
+                case PGMMODE_PAE:
+                case PGMMODE_PAE_NX:
+                    rc2 = PGM_BTH_NAME_PAE_PAE(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_NESTED:
+                    rc2 = PGM_BTH_NAME_NESTED_PAE(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_EPT:
+                    rc2 = PGM_BTH_NAME_EPT_PAE(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_32_BIT:
+                case PGMMODE_AMD64:
+                case PGMMODE_AMD64_NX:
+                    AssertMsgFailed(("Should use PAE shadow mode!\n"));
+                default: AssertFailed(); break;
+            }
+            break;
+        }
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+        case PGMMODE_AMD64_NX:
+        case PGMMODE_AMD64:
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & UINT64_C(0xfffffffffffff000); /** @todo define this mask! */
+            rc = PGM_GST_NAME_AMD64(Enter)(pVCpu, GCPhysCR3);
+            switch (pVCpu->pgm.s.enmShadowMode)
+            {
+                case PGMMODE_AMD64:
+                case PGMMODE_AMD64_NX:
+                    rc2 = PGM_BTH_NAME_AMD64_AMD64(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_NESTED:
+                    rc2 = PGM_BTH_NAME_NESTED_AMD64(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_EPT:
+                    rc2 = PGM_BTH_NAME_EPT_AMD64(Enter)(pVCpu, GCPhysCR3);
+                    break;
+                case PGMMODE_32_BIT:
+                case PGMMODE_PAE:
+                case PGMMODE_PAE_NX:
+                    AssertMsgFailed(("Should use AMD64 shadow mode!\n"));
+                default: AssertFailed(); break;
+            }
+            break;
+#endif
+
+        default:
+            AssertReleaseMsgFailed(("enmGuestMode=%d\n", enmGuestMode));
+            rc = VERR_NOT_IMPLEMENTED;
+            break;
+    }
+
+    /* status codes. */
+    AssertRC(rc);
+    AssertRC(rc2);
+    if (RT_SUCCESS(rc))
+    {
+        rc = rc2;
+        if (RT_SUCCESS(rc)) /* no informational status codes. */
+            rc = VINF_SUCCESS;
+    }
+
+    /* Notify HWACCM as well. */
+    HWACCMR3PagingModeChanged(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
+    return rc;
+}
+
+
+/**
+ * Called by pgmPoolFlushAllInt prior to flushing the pool.
+ *
+ * @returns VBox status code, fully asserted.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU to operate on.
+ */
+int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu)
+{
+    /* Unmap the old CR3 value before flushing everything. */
+    int rc = PGM_BTH_PFN(UnmapCR3, pVCpu)(pVCpu);
+    AssertRC(rc);
+
+    /* Exit the current shadow paging mode as well; nested paging and EPT use a root CR3 which will get flushed here. */
+    rc = PGM_SHW_PFN(Exit, pVCpu)(pVCpu);
+    AssertRC(rc);
+    Assert(pVCpu->pgm.s.pShwPageCR3R3 == NULL);
+    return rc;
+}
+
+
+/**
+ * Called by pgmPoolFlushAllInt after flushing the pool.
+ *
+ * @returns VBox status code, fully asserted.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU to operate on.
+ */
+int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu)
+{
+    pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID;
+    int rc = PGMR3ChangeMode(pVM, pVCpu, PGMGetGuestMode(pVCpu));
+    Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+    AssertRCReturn(rc, rc);
+    AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+    Assert(pVCpu->pgm.s.pShwPageCR3R3 != NULL);
+    AssertMsg(   pVCpu->pgm.s.enmShadowMode >= PGMMODE_NESTED
+              || CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu),
+              ("%RHp != %RHp %s\n", (RTHCPHYS)CPUMGetHyperCR3(pVCpu), PGMGetHyperCR3(pVCpu), PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
+    return rc;
+}
+
+#ifdef VBOX_WITH_DEBUGGER
+
+/**
+ * The '.pgmram' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) pgmR3CmdRam(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
+    if (!pVM->pgm.s.pRamRangesRC)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no Ram is registered.\n");
+
+    /*
+     * Dump the ranges.
+     */
+    int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "From     - To (incl) pvHC\n");
+    PPGMRAMRANGE pRam;
+    for (pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    {
+        rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL,
+            "%RGp - %RGp  %p\n",
+            pRam->GCPhys, pRam->GCPhysLast, pRam->pvR3);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The '.pgmerror' and '.pgmerroroff' commands.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int)  pgmR3CmdError(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
+    AssertReturn(cArgs == 0 || (cArgs == 1 && paArgs[0].enmType == DBGCVAR_TYPE_STRING),
+                 pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Hit bug in the parser.\n"));
+
+    if (!cArgs)
+    {
+        /*
+         * Print the list of error injection locations with status.
+         */
+        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "PGM error inject locations:\n");
+        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "  handy - %RTbool\n", pVM->pgm.s.fErrInjHandyPages);
+    }
+    else
+    {
+
+        /*
+         * String switch on where to inject the error.
+         */
+        bool const  fNewState = !strcmp(pCmd->pszCmd, "pgmerror");
+        const char *pszWhere = paArgs[0].u.pszString;
+        if (!strcmp(pszWhere, "handy"))
+            ASMAtomicWriteBool(&pVM->pgm.s.fErrInjHandyPages, fNewState);
+        else
+            return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Invalid 'where' value: %s.\n", pszWhere);
+        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "done\n");
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The '.pgmsync' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) pgmR3CmdSync(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /** @todo SMP support */
+
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
+
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    /*
+     * Force page directory sync.
+     */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Forcing page directory sync.\n");
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * The '.pgmassertcr3' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) pgmR3CmdAssertCR3(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /** @todo SMP support!! */
+
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
+
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    int rc = pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Checking shadow CR3 page tables for consistency.\n");
+    if (RT_FAILURE(rc))
+        return rc;
+
+    PGMAssertCR3(pVM, pVCpu, CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu));
+
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * The '.pgmsyncalways' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) pgmR3CmdSyncAlways(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /** @todo SMP support!! */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
+
+    /*
+     * Force page directory sync.
+     */
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS)
+    {
+        ASMAtomicAndU32(&pVCpu->pgm.s.fSyncFlags, ~PGM_SYNC_ALWAYS);
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Disabled permanent forced page directory syncing.\n");
+    }
+    else
+    {
+        ASMAtomicOrU32(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_ALWAYS);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Enabled permanent forced page directory syncing.\n");
+    }
+}
+
+
+/**
+ * The '.pgmphystofile' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) pgmR3CmdPhysToFile(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
+    if (    cArgs < 1
+        ||  cArgs > 2
+        ||  paArgs[0].enmType != DBGCVAR_TYPE_STRING
+        ||  (   cArgs > 1
+             && paArgs[1].enmType != DBGCVAR_TYPE_STRING))
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: parser error, invalid arguments.\n");
+    if (    cArgs >= 2
+        &&  strcmp(paArgs[1].u.pszString, "nozero"))
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: Invalid 2nd argument '%s', must be 'nozero'.\n", paArgs[1].u.pszString);
+    bool fIncZeroPgs = cArgs < 2;
+
+    /*
+     * Open the output file and get the ram parameters.
+     */
+    RTFILE hFile;
+    int rc = RTFileOpen(&hFile, paArgs[0].u.pszString, RTFILE_O_WRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE);
+    if (RT_FAILURE(rc))
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileOpen(,'%s',) -> %Rrc.\n", paArgs[0].u.pszString, rc);
+
+    uint32_t cbRamHole = 0;
+    CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
+    uint64_t cbRam     = 0;
+    CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
+    RTGCPHYS GCPhysEnd = cbRam + cbRamHole;
+
+    /*
+     * Dump the physical memory, page by page.
+     */
+    RTGCPHYS    GCPhys = 0;
+    char        abZeroPg[PAGE_SIZE];
+    RT_ZERO(abZeroPg);
+
+    pgmLock(pVM);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+          pRam && pRam->GCPhys < GCPhysEnd && RT_SUCCESS(rc);
+          pRam = pRam->pNextR3)
+    {
+        /* fill the gap */
+        if (pRam->GCPhys > GCPhys && fIncZeroPgs)
+        {
+            while (pRam->GCPhys > GCPhys && RT_SUCCESS(rc))
+            {
+                rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
+                GCPhys += PAGE_SIZE;
+            }
+        }
+
+        PCPGMPAGE pPage = &pRam->aPages[0];
+        while (GCPhys < pRam->GCPhysLast && RT_SUCCESS(rc))
+        {
+            if (    PGM_PAGE_IS_ZERO(pPage)
+                ||  PGM_PAGE_IS_BALLOONED(pPage))
+            {
+                if (fIncZeroPgs)
+                {
+                    rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
+                    if (RT_FAILURE(rc))
+                        pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                }
+            }
+            else
+            {
+                switch (PGM_PAGE_GET_TYPE(pPage))
+                {
+                    case PGMPAGETYPE_RAM:
+                    case PGMPAGETYPE_ROM_SHADOW: /* trouble?? */
+                    case PGMPAGETYPE_ROM:
+                    case PGMPAGETYPE_MMIO2:
+                    {
+                        void const     *pvPage;
+                        PGMPAGEMAPLOCK  Lock;
+                        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = RTFileWrite(hFile, pvPage, PAGE_SIZE, NULL);
+                            PGMPhysReleasePageMappingLock(pVM, &Lock);
+                            if (RT_FAILURE(rc))
+                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                        }
+                        else
+                            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: PGMPhysGCPhys2CCPtrReadOnly -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                        break;
+                    }
+
+                    default:
+                        AssertFailed();
+                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                    case PGMPAGETYPE_MMIO:
+                        if (fIncZeroPgs)
+                        {
+                            rc = RTFileWrite(hFile, abZeroPg, PAGE_SIZE, NULL);
+                            if (RT_FAILURE(rc))
+                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: RTFileWrite -> %Rrc at GCPhys=%RGp.\n", rc, GCPhys);
+                        }
+                        break;
+                }
+            }
+
+
+            /* advance */
+            GCPhys += PAGE_SIZE;
+            pPage++;
+        }
+    }
+    pgmUnlock(pVM);
+
+    RTFileClose(hFile);
+    if (RT_SUCCESS(rc))
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Successfully saved physical memory to '%s'.\n", paArgs[0].u.pszString);
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_DEBUGGER */
+
+/**
+ * pvUser argument of the pgmR3CheckIntegrity*Node callbacks.
+ */
+typedef struct PGMCHECKINTARGS
+{
+    bool                    fLeftToRight;    /**< true: left-to-right; false: right-to-left. */
+    PPGMPHYSHANDLER         pPrevPhys;
+    PPGMVIRTHANDLER         pPrevVirt;
+    PPGMPHYS2VIRTHANDLER    pPrevPhys2Virt;
+    PVM                     pVM;
+} PGMCHECKINTARGS, *PPGMCHECKINTARGS;
+
+/**
+ * Validate a node in the physical handler tree.
+ *
+ * @returns 0 on if ok, other wise 1.
+ * @param   pNode       The handler node.
+ * @param   pvUser      pVM.
+ */
+static DECLCALLBACK(int) pgmR3CheckIntegrityPhysHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
+    AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
+    AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+    AssertReleaseMsg(   !pArgs->pPrevPhys
+                     || (pArgs->fLeftToRight ? pArgs->pPrevPhys->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys->Core.KeyLast > pCur->Core.Key),
+                     ("pPrevPhys=%p %RGp-%RGp %s\n"
+                      "     pCur=%p %RGp-%RGp %s\n",
+                      pArgs->pPrevPhys, pArgs->pPrevPhys->Core.Key, pArgs->pPrevPhys->Core.KeyLast, pArgs->pPrevPhys->pszDesc,
+                      pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+    pArgs->pPrevPhys = pCur;
+    return 0;
+}
+
+
+/**
+ * Validate a node in the virtual handler tree.
+ *
+ * @returns 0 on if ok, other wise 1.
+ * @param   pNode       The handler node.
+ * @param   pvUser      pVM.
+ */
+static DECLCALLBACK(int) pgmR3CheckIntegrityVirtHandlerNode(PAVLROGCPTRNODECORE pNode, void *pvUser)
+{
+    PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
+    PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)pNode;
+    AssertReleaseReturn(!((uintptr_t)pCur & 7), 1);
+    AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGv-%RGv %s\n", pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+    AssertReleaseMsg(   !pArgs->pPrevVirt
+                     || (pArgs->fLeftToRight ? pArgs->pPrevVirt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevVirt->Core.KeyLast > pCur->Core.Key),
+                     ("pPrevVirt=%p %RGv-%RGv %s\n"
+                      "     pCur=%p %RGv-%RGv %s\n",
+                      pArgs->pPrevVirt, pArgs->pPrevVirt->Core.Key, pArgs->pPrevVirt->Core.KeyLast, pArgs->pPrevVirt->pszDesc,
+                      pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+    for (unsigned iPage = 0; iPage < pCur->cPages; iPage++)
+    {
+        AssertReleaseMsg(pCur->aPhysToVirt[iPage].offVirtHandler == -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage]),
+                         ("pCur=%p %RGv-%RGv %s\n"
+                          "iPage=%d offVirtHandle=%#x expected %#x\n",
+                          pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc,
+                          iPage, pCur->aPhysToVirt[iPage].offVirtHandler, -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[iPage])));
+    }
+    pArgs->pPrevVirt = pCur;
+    return 0;
+}
+
+
+/**
+ * Validate a node in the virtual handler tree.
+ *
+ * @returns 0 on if ok, other wise 1.
+ * @param   pNode       The handler node.
+ * @param   pvUser      pVM.
+ */
+static DECLCALLBACK(int) pgmR3CheckIntegrityPhysToVirtHandlerNode(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMCHECKINTARGS pArgs = (PPGMCHECKINTARGS)pvUser;
+    PPGMPHYS2VIRTHANDLER pCur = (PPGMPHYS2VIRTHANDLER)pNode;
+    AssertReleaseMsgReturn(!((uintptr_t)pCur & 3),      ("\n"), 1);
+    AssertReleaseMsgReturn(!(pCur->offVirtHandler & 3), ("\n"), 1);
+    AssertReleaseMsg(pCur->Core.Key <= pCur->Core.KeyLast,("pCur=%p %RGp-%RGp\n", pCur, pCur->Core.Key, pCur->Core.KeyLast));
+    AssertReleaseMsg(   !pArgs->pPrevPhys2Virt
+                     || (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
+                     ("pPrevPhys2Virt=%p %RGp-%RGp\n"
+                      "          pCur=%p %RGp-%RGp\n",
+                      pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
+                      pCur, pCur->Core.Key, pCur->Core.KeyLast));
+    AssertReleaseMsg(   !pArgs->pPrevPhys2Virt
+                     || (pArgs->fLeftToRight ? pArgs->pPrevPhys2Virt->Core.KeyLast < pCur->Core.Key : pArgs->pPrevPhys2Virt->Core.KeyLast > pCur->Core.Key),
+                     ("pPrevPhys2Virt=%p %RGp-%RGp\n"
+                      "          pCur=%p %RGp-%RGp\n",
+                      pArgs->pPrevPhys2Virt, pArgs->pPrevPhys2Virt->Core.Key, pArgs->pPrevPhys2Virt->Core.KeyLast,
+                      pCur, pCur->Core.Key, pCur->Core.KeyLast));
+    AssertReleaseMsg((pCur->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE | PGMPHYS2VIRTHANDLER_IS_HEAD)) == (PGMPHYS2VIRTHANDLER_IN_TREE | PGMPHYS2VIRTHANDLER_IS_HEAD),
+                     ("pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                      pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
+    if (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
+    {
+        PPGMPHYS2VIRTHANDLER pCur2 = pCur;
+        for (;;)
+        {
+            pCur2 = (PPGMPHYS2VIRTHANDLER)((intptr_t)pCur + (pCur->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
+            AssertReleaseMsg(pCur2 != pCur,
+                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias));
+            AssertReleaseMsg((pCur2->offNextAlias & (PGMPHYS2VIRTHANDLER_IN_TREE | PGMPHYS2VIRTHANDLER_IS_HEAD)) == PGMPHYS2VIRTHANDLER_IN_TREE,
+                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
+                              "pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
+                              pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
+            AssertReleaseMsg((pCur2->Core.Key ^ pCur->Core.Key) < PAGE_SIZE,
+                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
+                              "pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
+                              pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
+            AssertReleaseMsg((pCur2->Core.KeyLast ^ pCur->Core.KeyLast) < PAGE_SIZE,
+                             (" pCur=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
+                              "pCur2=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                              pCur, pCur->Core.Key, pCur->Core.KeyLast, pCur->offVirtHandler, pCur->offNextAlias,
+                              pCur2, pCur2->Core.Key, pCur2->Core.KeyLast, pCur2->offVirtHandler, pCur2->offNextAlias));
+            if (!(pCur2->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
+                break;
+        }
+    }
+
+    pArgs->pPrevPhys2Virt = pCur;
+    return 0;
+}
+
+
+/**
+ * Perform an integrity check on the PGM component.
+ *
+ * @returns VINF_SUCCESS if everything is fine.
+ * @returns VBox error status after asserting on integrity breach.
+ * @param   pVM     The VM handle.
+ */
+VMMR3DECL(int) PGMR3CheckIntegrity(PVM pVM)
+{
+    AssertReleaseReturn(pVM->pgm.s.offVM, VERR_INTERNAL_ERROR);
+
+    /*
+     * Check the trees.
+     */
+    int cErrors = 0;
+    const static PGMCHECKINTARGS s_LeftToRight = {  true, NULL, NULL, NULL, pVM };
+    const static PGMCHECKINTARGS s_RightToLeft = { false, NULL, NULL, NULL, pVM };
+    PGMCHECKINTARGS Args = s_LeftToRight;
+    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,       true,  pgmR3CheckIntegrityPhysHandlerNode, &Args);
+    Args = s_RightToLeft;
+    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers,       false, pgmR3CheckIntegrityPhysHandlerNode, &Args);
+    Args = s_LeftToRight;
+    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers,       true,  pgmR3CheckIntegrityVirtHandlerNode, &Args);
+    Args = s_RightToLeft;
+    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->VirtHandlers,       false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
+    Args = s_LeftToRight;
+    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers,  true,  pgmR3CheckIntegrityVirtHandlerNode, &Args);
+    Args = s_RightToLeft;
+    cErrors += RTAvlroGCPtrDoWithAll( &pVM->pgm.s.pTreesR3->HyperVirtHandlers,  false, pgmR3CheckIntegrityVirtHandlerNode, &Args);
+    Args = s_LeftToRight;
+    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, true,  pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
+    Args = s_RightToLeft;
+    cErrors += RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysToVirtHandlers, false, pgmR3CheckIntegrityPhysToVirtHandlerNode, &Args);
+
+    return !cErrors ? VINF_SUCCESS : VERR_INTERNAL_ERROR;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMBth.h
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMBth.h	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMBth.h	(revision 35333)
@@ -0,0 +1,208 @@
+/* $Id$ */
+/** @file
+ * VBox - Page Manager / Monitor, Shadow+Guest Paging Template.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+RT_C_DECLS_BEGIN
+PGM_BTH_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0);
+PGM_BTH_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
+PGM_BTH_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta);
+
+PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
+PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
+PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uError);
+PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
+PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
+PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
+PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
+PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
+RT_C_DECLS_END
+
+
+/**
+ * Initializes the both bit of the paging mode data.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
+ *                          This is used early in the init process to avoid trouble with PDM
+ *                          not being initialized yet.
+ */
+PGM_BTH_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0)
+{
+    Assert(pModeData->uShwType == PGM_SHW_TYPE); Assert(pModeData->uGstType == PGM_GST_TYPE);
+
+    /* Ring 3 */
+    pModeData->pfnR3BthRelocate             = PGM_BTH_NAME(Relocate);
+    pModeData->pfnR3BthSyncCR3              = PGM_BTH_NAME(SyncCR3);
+    pModeData->pfnR3BthInvalidatePage       = PGM_BTH_NAME(InvalidatePage);
+    pModeData->pfnR3BthPrefetchPage         = PGM_BTH_NAME(PrefetchPage);
+    pModeData->pfnR3BthVerifyAccessSyncPage = PGM_BTH_NAME(VerifyAccessSyncPage);
+#ifdef VBOX_STRICT
+    pModeData->pfnR3BthAssertCR3            = PGM_BTH_NAME(AssertCR3);
+#endif
+    pModeData->pfnR3BthMapCR3               = PGM_BTH_NAME(MapCR3);
+    pModeData->pfnR3BthUnmapCR3             = PGM_BTH_NAME(UnmapCR3);
+
+    if (fResolveGCAndR0)
+    {
+        int rc;
+
+#if PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT /* No AMD64 for traditional virtualization, only VT-x and AMD-V. */
+        /* GC */
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(Trap0eHandler),       &pModeData->pfnRCBthTrap0eHandler);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(Trap0eHandler),  rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(InvalidatePage),      &pModeData->pfnRCBthInvalidatePage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(InvalidatePage), rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(SyncCR3),             &pModeData->pfnRCBthSyncCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(SyncCR3), rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(PrefetchPage),        &pModeData->pfnRCBthPrefetchPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(PrefetchPage), rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(VerifyAccessSyncPage),&pModeData->pfnRCBthVerifyAccessSyncPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(VerifyAccessSyncPage), rc), rc);
+# ifdef VBOX_STRICT
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(AssertCR3),           &pModeData->pfnRCBthAssertCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(AssertCR3), rc), rc);
+# endif
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(MapCR3),              &pModeData->pfnRCBthMapCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(MapCR3), rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_BTH_NAME_RC_STR(UnmapCR3),            &pModeData->pfnRCBthUnmapCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_RC_STR(UnmapCR3), rc), rc);
+#endif /* Not AMD64 shadow paging. */
+
+        /* Ring 0 */
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(Trap0eHandler),       &pModeData->pfnR0BthTrap0eHandler);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(Trap0eHandler),  rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(InvalidatePage),      &pModeData->pfnR0BthInvalidatePage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(InvalidatePage), rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(SyncCR3),             &pModeData->pfnR0BthSyncCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(SyncCR3), rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(PrefetchPage),        &pModeData->pfnR0BthPrefetchPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(PrefetchPage), rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(VerifyAccessSyncPage),&pModeData->pfnR0BthVerifyAccessSyncPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(VerifyAccessSyncPage), rc), rc);
+#ifdef VBOX_STRICT
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(AssertCR3),           &pModeData->pfnR0BthAssertCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(AssertCR3), rc), rc);
+#endif
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(MapCR3),              &pModeData->pfnR0BthMapCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(MapCR3), rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_BTH_NAME_R0_STR(UnmapCR3),            &pModeData->pfnR0BthUnmapCR3);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_BTH_NAME_R0_STR(UnmapCR3), rc), rc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enters the shadow+guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   GCPhysCR3   The physical address from the CR3 register.
+ */
+PGM_BTH_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
+{
+    /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
+     * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
+     */
+#if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
+          || PGM_SHW_TYPE == PGM_TYPE_PAE    \
+          || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
+      && (   PGM_GST_TYPE == PGM_TYPE_REAL   \
+          || PGM_GST_TYPE == PGM_TYPE_PROT))
+
+    PVM pVM = pVCpu->pVMR3;
+
+    Assert(HWACCMIsNestedPagingActive(pVM) == pVM->pgm.s.fNestedPaging);
+    Assert(!pVM->pgm.s.fNestedPaging);
+
+    pgmLock(pVM);
+    /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
+     *       but any calls to GC need a proper shadow page setup as well.
+     */
+    /* Free the previous root mapping if still active. */
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+    {
+        Assert(pVCpu->pgm.s.pShwPageCR3R3->enmKind != PGMPOOLKIND_FREE);
+
+        /* Mark the page as unlocked; allow flushing again. */
+        pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+        /* Remove the hypervisor mappings from the shadow page table. */
+        pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.pShwPageCR3R3, pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
+        pVCpu->pgm.s.pShwPageCR3R3 = 0;
+        pVCpu->pgm.s.pShwPageCR3RC = 0;
+        pVCpu->pgm.s.pShwPageCR3R0 = 0;
+        pVCpu->pgm.s.iShwUser      = 0;
+        pVCpu->pgm.s.iShwUserTable = 0;
+    }
+
+    /* construct a fake address. */
+    GCPhysCR3 = RT_BIT_64(63);
+    pVCpu->pgm.s.iShwUser      = SHW_POOL_ROOT_IDX;
+    pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
+    int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable,
+                          &pVCpu->pgm.s.pShwPageCR3R3);
+    if (rc == VERR_PGM_POOL_FLUSHED)
+    {
+        Log(("Bth-Enter: PGM pool flushed -> signal sync cr3\n"));
+        Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+        pgmUnlock(pVM);
+        return VINF_PGM_SYNC_CR3;
+    }
+    AssertRCReturn(rc, rc);
+
+    /* Mark the page as locked; disallow flushing. */
+    pgmPoolLockPage(pPool, pVCpu->pgm.s.pShwPageCR3R3);
+
+    pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.pShwPageCR3R3);
+    pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.pShwPageCR3R3);
+
+    /* Set the current hypervisor CR3. */
+    CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
+
+    /* Apply all hypervisor mappings to the new CR3. */
+    rc = pgmMapActivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+    pgmUnlock(pVM);
+    return rc;
+#else
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Relocate any GC pointers related to shadow mode paging.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   offDelta    The relocation offset.
+ */
+PGM_BTH_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta)
+{
+    /* nothing special to do here - InitData does the job. */
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMDbg.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMDbg.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMDbg.cpp	(revision 35333)
@@ -0,0 +1,2329 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager and Monitor - Debugger & Debugging APIs.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/pgm.h>
+#include <VBox/stam.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include "PGMInline.h"
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The max needle size that we will bother searching for
+ * This must not be more than half a page! */
+#define MAX_NEEDLE_SIZE     256
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * State structure for the paging hierarchy dumpers.
+ */
+typedef struct PGMR3DUMPHIERARCHYSTATE
+{
+    /** The VM handle. */
+    PVM             pVM;
+    /** Output helpers. */
+    PCDBGFINFOHLP   pHlp;
+    /** Set if PSE, PAE or long mode is enabled. */
+    bool            fPse;
+    /** Set if PAE or long mode is enabled. */
+    bool            fPae;
+    /** Set if long mode is enabled. */
+    bool            fLme;
+    /** Set if nested paging. */
+    bool            fNp;
+    /** Set if EPT. */
+    bool            fEpt;
+    /** Set if NXE is enabled. */
+    bool            fNxe;
+    /** The number or chars the address needs. */
+    uint8_t         cchAddress;
+    /** The last reserved bit. */
+    uint8_t         uLastRsvdBit;
+    /** Dump the page info as well (shadow page summary / guest physical
+     *  page summary). */
+    bool            fDumpPageInfo;
+    /** Whether or not to print the header. */
+    bool            fPrintHeader;
+    /** Whether to print the CR3 value */
+    bool            fPrintCr3;
+    /** Padding*/
+    bool            afReserved[5];
+    /** The current address.  */
+    uint64_t        u64Address;
+    /** The last address to dump structures for. */
+    uint64_t        u64FirstAddress;
+    /** The last address to dump structures for. */
+    uint64_t        u64LastAddress;
+    /** Mask with the high reserved bits set. */
+    uint64_t        u64HighReservedBits;
+    /** The number of leaf entries that we've printed. */
+    uint64_t        cLeaves;
+} PGMR3DUMPHIERARCHYSTATE;
+/** Pointer to the paging hierarchy dumper state. */
+typedef PGMR3DUMPHIERARCHYSTATE *PPGMR3DUMPHIERARCHYSTATE;
+
+
+
+/**
+ * Converts a R3 pointer to a GC physical address.
+ *
+ * Only for the debugger.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success, *pGCPhys is set.
+ * @retval  VERR_INVALID_POINTER if the pointer is not within the GC physical memory.
+ *
+ * @param   pVM         The VM handle.
+ * @param   R3Ptr       The R3 pointer to convert.
+ * @param   pGCPhys     Where to store the GC physical address on success.
+ */
+VMMR3DECL(int) PGMR3DbgR3Ptr2GCPhys(PVM pVM, RTR3PTR R3Ptr, PRTGCPHYS pGCPhys)
+{
+    *pGCPhys = NIL_RTGCPHYS;
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Converts a R3 pointer to a HC physical address.
+ *
+ * Only for the debugger.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success, *pHCPhys is set.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical page but has no physical backing.
+ * @retval  VERR_INVALID_POINTER if the pointer is not within the GC physical memory.
+ *
+ * @param   pVM         The VM handle.
+ * @param   R3Ptr       The R3 pointer to convert.
+ * @param   pHCPhys     Where to store the HC physical address on success.
+ */
+VMMR3DECL(int) PGMR3DbgR3Ptr2HCPhys(PVM pVM, RTR3PTR R3Ptr, PRTHCPHYS pHCPhys)
+{
+    *pHCPhys = NIL_RTHCPHYS;
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Converts a HC physical address to a GC physical address.
+ *
+ * Only for the debugger.
+ *
+ * @returns VBox status code
+ * @retval  VINF_SUCCESS on success, *pGCPhys is set.
+ * @retval  VERR_INVALID_POINTER if the HC physical address is not within the GC physical memory.
+ *
+ * @param   pVM     The VM handle.
+ * @param   HCPhys  The HC physical address to convert.
+ * @param   pGCPhys Where to store the GC physical address on success.
+ */
+VMMR3DECL(int) PGMR3DbgHCPhys2GCPhys(PVM pVM, RTHCPHYS HCPhys, PRTGCPHYS pGCPhys)
+{
+    /*
+     * Validate and adjust the input a bit.
+     */
+    if (HCPhys == NIL_RTHCPHYS)
+        return VERR_INVALID_POINTER;
+    unsigned off = HCPhys & PAGE_OFFSET_MASK;
+    HCPhys &= X86_PTE_PAE_PG_MASK;
+    if (HCPhys == 0)
+        return VERR_INVALID_POINTER;
+
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
+            {
+                *pGCPhys = pRam->GCPhys + (iPage << PAGE_SHIFT) + off;
+                return VINF_SUCCESS;
+            }
+    }
+    return VERR_INVALID_POINTER;
+}
+
+
+/**
+ * Read physical memory API for the debugger, similar to
+ * PGMPhysSimpleReadGCPhys.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pvDst       Where to store what's read.
+ * @param   GCPhysDst   Where to start reading from.
+ * @param   cb          The number of bytes to attempt reading.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbRead     For store the actual number of bytes read, pass NULL if
+ *                      partial reads are unwanted.
+ * @todo    Unused?
+ */
+VMMR3DECL(int) PGMR3DbgReadGCPhys(PVM pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb, uint32_t fFlags, size_t *pcbRead)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* try simple first. */
+    int rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbRead)
+        return rc;
+
+    /* partial read that failed, chop it up in pages. */
+    *pcbRead = 0;
+    size_t const cbReq = cb;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPhysSrc & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbRead  += cbChunk;
+        cb        -= cbChunk;
+        GCPhysSrc += cbChunk;
+        pvDst = (uint8_t *)pvDst + cbChunk;
+    }
+
+    return *pcbRead && RT_FAILURE(rc) ? -rc : rc;
+}
+
+
+/**
+ * Write physical memory API for the debugger, similar to
+ * PGMPhysSimpleWriteGCPhys.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCPhysDst   Where to start writing.
+ * @param   pvSrc       What to write.
+ * @param   cb          The number of bytes to attempt writing.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbWritten  For store the actual number of bytes written, pass NULL
+ *                      if partial writes are unwanted.
+ * @todo    Unused?
+ */
+VMMR3DECL(int) PGMR3DbgWriteGCPhys(PVM pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb, uint32_t fFlags, size_t *pcbWritten)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* try simple first. */
+    int rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysDst, pvSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbWritten)
+        return rc;
+
+    /* partial write that failed, chop it up in pages. */
+    *pcbWritten = 0;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPhysDst & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleWriteGCPhys(pVM, GCPhysDst, pvSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbWritten += cbChunk;
+        cb          -= cbChunk;
+        GCPhysDst   += cbChunk;
+        pvSrc = (uint8_t const *)pvSrc + cbChunk;
+    }
+
+    return *pcbWritten && RT_FAILURE(rc) ? -rc : rc;
+
+}
+
+
+/**
+ * Read virtual memory API for the debugger, similar to PGMPhysSimpleReadGCPtr.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pvDst       Where to store what's read.
+ * @param   GCPtrDst    Where to start reading from.
+ * @param   cb          The number of bytes to attempt reading.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbRead     For store the actual number of bytes read, pass NULL if
+ *                      partial reads are unwanted.
+ * @todo    Unused?
+ */
+VMMR3DECL(int) PGMR3DbgReadGCPtr(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, uint32_t fFlags, size_t *pcbRead)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* @todo SMP support! */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+/** @todo deal with HMA */
+    /* try simple first. */
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCPtrSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbRead)
+        return rc;
+
+    /* partial read that failed, chop it up in pages. */
+    *pcbRead = 0;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPtrSrc & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pvDst, GCPtrSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbRead  += cbChunk;
+        cb        -= cbChunk;
+        GCPtrSrc  += cbChunk;
+        pvDst = (uint8_t *)pvDst + cbChunk;
+    }
+
+    return *pcbRead && RT_FAILURE(rc) ? -rc : rc;
+
+}
+
+
+/**
+ * Write virtual memory API for the debugger, similar to
+ * PGMPhysSimpleWriteGCPtr.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCPtrDst    Where to start writing.
+ * @param   pvSrc       What to write.
+ * @param   cb          The number of bytes to attempt writing.
+ * @param   fFlags      Flags, MBZ.
+ * @param   pcbWritten  For store the actual number of bytes written, pass NULL
+ *                      if partial writes are unwanted.
+ * @todo    Unused?
+ */
+VMMR3DECL(int) PGMR3DbgWriteGCPtr(PVM pVM, RTGCPTR GCPtrDst, void const *pvSrc, size_t cb, uint32_t fFlags, size_t *pcbWritten)
+{
+    /* validate */
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(pVM, VERR_INVALID_PARAMETER);
+
+    /* @todo SMP support! */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+/** @todo deal with HMA */
+    /* try simple first. */
+    int rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCPtrDst, pvSrc, cb);
+    if (RT_SUCCESS(rc) || !pcbWritten)
+        return rc;
+
+    /* partial write that failed, chop it up in pages. */
+    *pcbWritten = 0;
+    rc = VINF_SUCCESS;
+    while (cb > 0)
+    {
+        size_t cbChunk = PAGE_SIZE;
+        cbChunk -= GCPtrDst & PAGE_OFFSET_MASK;
+        if (cbChunk > cb)
+            cbChunk = cb;
+
+        rc = PGMPhysSimpleWriteGCPtr(pVCpu, GCPtrDst, pvSrc, cbChunk);
+
+        /* advance */
+        if (RT_FAILURE(rc))
+            break;
+        *pcbWritten += cbChunk;
+        cb          -= cbChunk;
+        GCPtrDst    += cbChunk;
+        pvSrc = (uint8_t const *)pvSrc + cbChunk;
+    }
+
+    return *pcbWritten && RT_FAILURE(rc) ? -rc : rc;
+
+}
+
+
+/**
+ * memchr() with alignment considerations.
+ *
+ * @returns Pointer to matching byte, NULL if none found.
+ * @param   pb                  Where to search. Aligned.
+ * @param   b                   What to search for.
+ * @param   cb                  How much to search .
+ * @param   uAlign              The alignment restriction of the result.
+ */
+static const uint8_t *pgmR3DbgAlignedMemChr(const uint8_t *pb, uint8_t b, size_t cb, uint32_t uAlign)
+{
+    const uint8_t *pbRet;
+    if (uAlign <= 32)
+    {
+        pbRet = (const uint8_t *)memchr(pb, b, cb);
+        if ((uintptr_t)pbRet & (uAlign - 1))
+        {
+            do
+            {
+                pbRet++;
+                size_t cbLeft = cb - (pbRet - pb);
+                if (!cbLeft)
+                {
+                    pbRet = NULL;
+                    break;
+                }
+                pbRet = (const uint8_t *)memchr(pbRet, b, cbLeft);
+            } while ((uintptr_t)pbRet & (uAlign - 1));
+        }
+    }
+    else
+    {
+        pbRet = NULL;
+        if (cb)
+        {
+            for (;;)
+            {
+                if (*pb == b)
+                {
+                    pbRet = pb;
+                    break;
+                }
+                if (cb <= uAlign)
+                    break;
+                cb -= uAlign;
+                pb += uAlign;
+            }
+        }
+    }
+    return pbRet;
+}
+
+
+/**
+ * Scans a page for a byte string, keeping track of potential
+ * cross page matches.
+ *
+ * @returns true and *poff on match.
+ *          false on mismatch.
+ * @param   pbPage          Pointer to the current page.
+ * @param   poff            Input: The offset into the page (aligned).
+ *                          Output: The page offset of the match on success.
+ * @param   cb              The number of bytes to search, starting of *poff.
+ * @param   uAlign          The needle alignment. This is of course less than a page.
+ * @param   pabNeedle       The byte string to search for.
+ * @param   cbNeedle        The length of the byte string.
+ * @param   pabPrev         The buffer that keeps track of a partial match that we
+ *                          bring over from the previous page. This buffer must be
+ *                          at least cbNeedle - 1 big.
+ * @param   pcbPrev         Input: The number of partial matching bytes from the previous page.
+ *                          Output: The number of partial matching bytes from this page.
+ *                          Initialize to 0 before the first call to this function.
+ */
+static bool pgmR3DbgScanPage(const uint8_t *pbPage, int32_t *poff, uint32_t cb, uint32_t uAlign,
+                             const uint8_t *pabNeedle, size_t cbNeedle,
+                             uint8_t *pabPrev, size_t *pcbPrev)
+{
+    /*
+     * Try complete any partial match from the previous page.
+     */
+    if (*pcbPrev > 0)
+    {
+        size_t cbPrev = *pcbPrev;
+        Assert(!*poff);
+        Assert(cbPrev < cbNeedle);
+        if (!memcmp(pbPage, pabNeedle + cbPrev, cbNeedle - cbPrev))
+        {
+            if (cbNeedle - cbPrev > cb)
+                return false;
+            *poff = -(int32_t)cbPrev;
+            return true;
+        }
+
+        /* check out the remainder of the previous page. */
+        const uint8_t *pb = pabPrev;
+        for (;;)
+        {
+            if (cbPrev <= uAlign)
+                break;
+            cbPrev -= uAlign;
+            pb = pgmR3DbgAlignedMemChr(pb + uAlign, *pabNeedle, cbPrev, uAlign);
+            if (!pb)
+                break;
+            cbPrev = *pcbPrev - (pb - pabPrev);
+            if (    !memcmp(pb + 1, &pabNeedle[1], cbPrev - 1)
+                &&  !memcmp(pbPage, pabNeedle + cbPrev, cbNeedle - cbPrev))
+            {
+                if (cbNeedle - cbPrev > cb)
+                    return false;
+                *poff = -(int32_t)cbPrev;
+                return true;
+            }
+        }
+
+        *pcbPrev = 0;
+    }
+
+    /*
+     * Match the body of the page.
+     */
+    const uint8_t *pb = pbPage + *poff;
+    const uint8_t *pbEnd = pb + cb;
+    for (;;)
+    {
+        pb = pgmR3DbgAlignedMemChr(pb, *pabNeedle, cb, uAlign);
+        if (!pb)
+            break;
+        cb = pbEnd - pb;
+        if (cb >= cbNeedle)
+        {
+            /* match? */
+            if (!memcmp(pb + 1, &pabNeedle[1], cbNeedle - 1))
+            {
+                *poff = pb - pbPage;
+                return true;
+            }
+        }
+        else
+        {
+            /* partial match at the end of the page? */
+            if (!memcmp(pb + 1, &pabNeedle[1], cb - 1))
+            {
+                /* We're copying one byte more that we really need here, but wtf. */
+                memcpy(pabPrev, pb, cb);
+                *pcbPrev = cb;
+                return false;
+            }
+        }
+
+        /* no match, skip ahead. */
+        if (cb <= uAlign)
+            break;
+        pb += uAlign;
+        cb -= uAlign;
+    }
+
+    return false;
+}
+
+
+/**
+ * Scans guest physical memory for a byte string.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS and *pGCPtrHit on success.
+ * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
+ * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
+ * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   GCPhys          Where to start searching.
+ * @param   cbRange         The number of bytes to search.
+ * @param   GCPhysAlign     The alignment of the needle. Must be a power of two
+ *                          and less or equal to 4GB.
+ * @param   pabNeedle       The byte string to search for.
+ * @param   cbNeedle        The length of the byte string. Max 256 bytes.
+ * @param   pGCPhysHit      Where to store the address of the first occurrence on success.
+ */
+VMMR3DECL(int) PGMR3DbgScanPhysical(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cbRange, RTGCPHYS GCPhysAlign,
+                                    const uint8_t *pabNeedle, size_t cbNeedle, PRTGCPHYS pGCPhysHit)
+{
+    /*
+     * Validate and adjust the input a bit.
+     */
+    if (!VALID_PTR(pGCPhysHit))
+        return VERR_INVALID_POINTER;
+    *pGCPhysHit = NIL_RTGCPHYS;
+
+    if (    !VALID_PTR(pabNeedle)
+        ||  GCPhys == NIL_RTGCPHYS)
+        return VERR_INVALID_POINTER;
+    if (!cbNeedle)
+        return VERR_INVALID_PARAMETER;
+    if (cbNeedle > MAX_NEEDLE_SIZE)
+        return VERR_INVALID_PARAMETER;
+
+    if (!cbRange)
+        return VERR_DBGF_MEM_NOT_FOUND;
+    if (GCPhys + cbNeedle - 1 < GCPhys)
+        return VERR_DBGF_MEM_NOT_FOUND;
+
+    if (!GCPhysAlign)
+        return VERR_INVALID_PARAMETER;
+    if (GCPhysAlign > UINT32_MAX)
+        return VERR_NOT_POWER_OF_TWO;
+    if (GCPhysAlign & (GCPhysAlign - 1))
+        return VERR_INVALID_PARAMETER;
+
+    if (GCPhys & (GCPhysAlign - 1))
+    {
+        RTGCPHYS Adj = GCPhysAlign - (GCPhys & (GCPhysAlign - 1));
+        if (    cbRange <= Adj
+            ||  GCPhys + Adj < GCPhys)
+            return VERR_DBGF_MEM_NOT_FOUND;
+        GCPhys  += Adj;
+        cbRange -= Adj;
+    }
+
+    const bool      fAllZero   = ASMMemIsAll8(pabNeedle, cbNeedle, 0) == NULL;
+    const uint32_t  cIncPages  = GCPhysAlign <= PAGE_SIZE
+                               ? 1
+                               : GCPhysAlign >> PAGE_SHIFT;
+    const RTGCPHYS  GCPhysLast = GCPhys + cbRange - 1 >= GCPhys
+                               ? GCPhys + cbRange - 1
+                               : ~(RTGCPHYS)0;
+
+    /*
+     * Search the memory - ignore MMIO and zero pages, also don't
+     * bother to match across ranges.
+     */
+    pgmLock(pVM);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        /*
+         * If the search range starts prior to the current ram range record,
+         * adjust the search range and possibly conclude the search.
+         */
+        RTGCPHYS off;
+        if (GCPhys < pRam->GCPhys)
+        {
+            if (GCPhysLast < pRam->GCPhys)
+                break;
+            GCPhys = pRam->GCPhys;
+            off = 0;
+        }
+        else
+            off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            /*
+             * Iterate the relevant pages.
+             */
+            uint8_t         abPrev[MAX_NEEDLE_SIZE];
+            size_t          cbPrev   = 0;
+            const uint32_t  cPages   = pRam->cb >> PAGE_SHIFT;
+            uint32_t        iPage    = off >> PAGE_SHIFT;
+            uint32_t        offPage  = GCPhys & PAGE_OFFSET_MASK;
+            GCPhys &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
+            for (;; offPage = 0)
+            {
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                if (    (   !PGM_PAGE_IS_ZERO(pPage)
+                         || fAllZero)
+                    &&  !PGM_PAGE_IS_BALLOONED(pPage)
+                    &&  !PGM_PAGE_IS_MMIO(pPage))
+                {
+                    void const     *pvPage;
+                    PGMPAGEMAPLOCK  Lock;
+                    int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
+                    if (RT_SUCCESS(rc))
+                    {
+                        int32_t     offHit = offPage;
+                        bool        fRc;
+                        if (GCPhysAlign < PAGE_SIZE)
+                        {
+                            uint32_t cbSearch = (GCPhys ^ GCPhysLast) & ~(RTGCPHYS)PAGE_OFFSET_MASK
+                                              ? PAGE_SIZE                           - (uint32_t)offPage
+                                              : (GCPhysLast & PAGE_OFFSET_MASK) + 1 - (uint32_t)offPage;
+                            fRc = pgmR3DbgScanPage((uint8_t const *)pvPage, &offHit, cbSearch, (uint32_t)GCPhysAlign,
+                                                   pabNeedle, cbNeedle, &abPrev[0], &cbPrev);
+                        }
+                        else
+                            fRc = memcmp(pvPage, pabNeedle, cbNeedle) == 0
+                               && (GCPhysLast - GCPhys) >= cbNeedle;
+                        PGMPhysReleasePageMappingLock(pVM, &Lock);
+                        if (fRc)
+                        {
+                            *pGCPhysHit = GCPhys + offHit;
+                            pgmUnlock(pVM);
+                            return VINF_SUCCESS;
+                        }
+                    }
+                    else
+                        cbPrev = 0; /* ignore error. */
+                }
+                else
+                    cbPrev = 0;
+
+                /* advance to the next page. */
+                GCPhys += (RTGCPHYS)cIncPages << PAGE_SHIFT;
+                if (GCPhys >= GCPhysLast) /* (may not always hit, but we're run out of ranges.) */
+                {
+                    pgmUnlock(pVM);
+                    return VERR_DBGF_MEM_NOT_FOUND;
+                }
+                iPage += cIncPages;
+                if (    iPage < cIncPages
+                    ||  iPage >= cPages)
+                    break;
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return VERR_DBGF_MEM_NOT_FOUND;
+}
+
+
+/**
+ * Scans (guest) virtual memory for a byte string.
+ *
+ * @returns VBox status codes:
+ * @retval  VINF_SUCCESS and *pGCPtrHit on success.
+ * @retval  VERR_DBGF_MEM_NOT_FOUND if not found.
+ * @retval  VERR_INVALID_POINTER if any of the pointer arguments are invalid.
+ * @retval  VERR_INVALID_ARGUMENT if any other arguments are invalid.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pVCpu           The CPU context to search in.
+ * @param   GCPtr           Where to start searching.
+ * @param   GCPtrAlign      The alignment of the needle. Must be a power of two
+ *                          and less or equal to 4GB.
+ * @param   cbRange         The number of bytes to search. Max 256 bytes.
+ * @param   pabNeedle       The byte string to search for.
+ * @param   cbNeedle        The length of the byte string.
+ * @param   pGCPtrHit       Where to store the address of the first occurrence on success.
+ */
+VMMR3DECL(int) PGMR3DbgScanVirtual(PVM pVM, PVMCPU pVCpu, RTGCPTR GCPtr, RTGCPTR cbRange, RTGCPTR GCPtrAlign,
+                                   const uint8_t *pabNeedle, size_t cbNeedle, PRTGCUINTPTR pGCPtrHit)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Validate and adjust the input a bit.
+     */
+    if (!VALID_PTR(pGCPtrHit))
+        return VERR_INVALID_POINTER;
+    *pGCPtrHit = 0;
+
+    if (!VALID_PTR(pabNeedle))
+        return VERR_INVALID_POINTER;
+    if (!cbNeedle)
+        return VERR_INVALID_PARAMETER;
+    if (cbNeedle > MAX_NEEDLE_SIZE)
+        return VERR_INVALID_PARAMETER;
+
+    if (!cbRange)
+        return VERR_DBGF_MEM_NOT_FOUND;
+    if (GCPtr + cbNeedle - 1 < GCPtr)
+        return VERR_DBGF_MEM_NOT_FOUND;
+
+    if (!GCPtrAlign)
+        return VERR_INVALID_PARAMETER;
+    if (GCPtrAlign > UINT32_MAX)
+        return VERR_NOT_POWER_OF_TWO;
+    if (GCPtrAlign & (GCPtrAlign - 1))
+        return VERR_INVALID_PARAMETER;
+
+    if (GCPtr & (GCPtrAlign - 1))
+    {
+        RTGCPTR Adj = GCPtrAlign - (GCPtr & (GCPtrAlign - 1));
+        if (    cbRange <= Adj
+            ||  GCPtr + Adj < GCPtr)
+            return VERR_DBGF_MEM_NOT_FOUND;
+        GCPtr   += Adj;
+        cbRange -= Adj;
+    }
+
+    /*
+     * Search the memory - ignore MMIO, zero and not-present pages.
+     */
+    const bool      fAllZero  = ASMMemIsAll8(pabNeedle, cbNeedle, 0) == NULL;
+    PGMMODE         enmMode   = PGMGetGuestMode(pVCpu);
+    RTGCPTR         GCPtrMask = PGMMODE_IS_LONG_MODE(enmMode) ? UINT64_MAX : UINT32_MAX;
+    uint8_t         abPrev[MAX_NEEDLE_SIZE];
+    size_t          cbPrev    = 0;
+    const uint32_t  cIncPages = GCPtrAlign <= PAGE_SIZE
+                              ? 1
+                              : GCPtrAlign >> PAGE_SHIFT;
+    const RTGCPTR   GCPtrLast = GCPtr + cbRange - 1 >= GCPtr
+                              ? (GCPtr + cbRange - 1) & GCPtrMask
+                              : GCPtrMask;
+    RTGCPTR         cPages    = (((GCPtrLast - GCPtr) + (GCPtr & PAGE_OFFSET_MASK)) >> PAGE_SHIFT) + 1;
+    uint32_t        offPage   = GCPtr & PAGE_OFFSET_MASK;
+    GCPtr &= ~(RTGCPTR)PAGE_OFFSET_MASK;
+    for (;; offPage = 0)
+    {
+        RTGCPHYS GCPhys;
+        int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
+        if (RT_SUCCESS(rc))
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+            if (    pPage
+                &&  (   !PGM_PAGE_IS_ZERO(pPage)
+                     || fAllZero)
+                &&  !PGM_PAGE_IS_BALLOONED(pPage)
+                &&  !PGM_PAGE_IS_MMIO(pPage))
+            {
+                void const *pvPage;
+                PGMPAGEMAPLOCK Lock;
+                rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, &pvPage, &Lock);
+                if (RT_SUCCESS(rc))
+                {
+                    int32_t offHit = offPage;
+                    bool    fRc;
+                    if (GCPtrAlign < PAGE_SIZE)
+                    {
+                        uint32_t cbSearch = cPages > 0
+                                          ? PAGE_SIZE                          - (uint32_t)offPage
+                                          : (GCPtrLast & PAGE_OFFSET_MASK) + 1 - (uint32_t)offPage;
+                        fRc = pgmR3DbgScanPage((uint8_t const *)pvPage, &offHit, cbSearch, (uint32_t)GCPtrAlign,
+                                               pabNeedle, cbNeedle, &abPrev[0], &cbPrev);
+                    }
+                    else
+                        fRc = memcmp(pvPage, pabNeedle, cbNeedle) == 0
+                           && (GCPtrLast - GCPtr) >= cbNeedle;
+                    PGMPhysReleasePageMappingLock(pVM, &Lock);
+                    if (fRc)
+                    {
+                        *pGCPtrHit = GCPtr + offHit;
+                        return VINF_SUCCESS;
+                    }
+                }
+                else
+                    cbPrev = 0; /* ignore error. */
+            }
+            else
+                cbPrev = 0;
+        }
+        else
+            cbPrev = 0; /* ignore error. */
+
+        /* advance to the next page. */
+        if (cPages <= cIncPages)
+            break;
+        cPages -= cIncPages;
+        GCPtr += (RTGCPTR)cIncPages << PAGE_SHIFT;
+    }
+    return VERR_DBGF_MEM_NOT_FOUND;
+}
+
+
+/**
+ * Initializes the dumper state.
+ *
+ * @param   pState          The state to initialize.
+ * @param   pVM             The VM handle.
+ * @param   fFlags          The flags.
+ * @param   u64FirstAddr    The first address.
+ * @param   u64LastAddr     The last address.
+ * @param   pHlp            The output helpers.
+ */
+static void pgmR3DumpHierarchyInitState(PPGMR3DUMPHIERARCHYSTATE pState, PVM pVM, uint32_t fFlags,
+                                        uint64_t u64FirstAddr, uint64_t u64LastAddr, PCDBGFINFOHLP pHlp)
+{
+    pState->pVM                 = pVM;
+    pState->pHlp                = pHlp ? pHlp : DBGFR3InfoLogHlp();
+    pState->fPse                = !!(fFlags & (DBGFPGDMP_FLAGS_PSE | DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME));
+    pState->fPae                = !!(fFlags & (DBGFPGDMP_FLAGS_PAE | DBGFPGDMP_FLAGS_LME));
+    pState->fLme                = !!(fFlags & DBGFPGDMP_FLAGS_LME);
+    pState->fNp                 = !!(fFlags & DBGFPGDMP_FLAGS_NP);
+    pState->fEpt                = !!(fFlags & DBGFPGDMP_FLAGS_EPT);
+    pState->fNxe                = !!(fFlags & DBGFPGDMP_FLAGS_NXE);
+    pState->cchAddress          = pState->fLme ? 16 : 8;
+    pState->uLastRsvdBit        = pState->fNxe ? 62 : 63;
+    pState->fDumpPageInfo       = !!(fFlags & DBGFPGDMP_FLAGS_PAGE_INFO);
+    pState->fPrintHeader        = !!(fFlags & DBGFPGDMP_FLAGS_HEADER);
+    pState->fPrintCr3           = !!(fFlags & DBGFPGDMP_FLAGS_PRINT_CR3);
+    pState->afReserved[0]       = false;
+    pState->afReserved[1]       = false;
+    pState->afReserved[2]       = false;
+    pState->afReserved[3]       = false;
+    pState->afReserved[4]       = false;
+    pState->u64Address          = u64FirstAddr;
+    pState->u64FirstAddress     = u64FirstAddr;
+    pState->u64LastAddress      = u64LastAddr;
+    pState->u64HighReservedBits = pState->uLastRsvdBit == 62 ? UINT64_C(0x7ff) << 52 : UINT64_C(0xfff) << 52;
+    pState->cLeaves             = 0;
+}
+
+
+/**
+ * The simple way out, too tired to think of a more elegant solution.
+ *
+ * @returns The base address of this page table/directory/whatever.
+ * @param   pState              The state where we get the current address.
+ * @param   cShift              The shift count for the table entries.
+ * @param   cEntries            The number of table entries.
+ * @param   piFirst             Where to return the table index of the first
+ *                              entry to dump.
+ * @param   piLast              Where to return the table index of the last
+ *                              entry.
+ */
+static uint64_t pgmR3DumpHierarchyCalcRange(PPGMR3DUMPHIERARCHYSTATE pState, uint32_t cShift, uint32_t cEntries,
+                                            uint32_t *piFirst, uint32_t *piLast)
+{
+    const uint64_t iBase  = (pState->u64Address     >> cShift) & ~(uint64_t)(cEntries - 1);
+    const uint64_t iFirst = pState->u64FirstAddress >> cShift;
+    const uint64_t iLast  = pState->u64LastAddress  >> cShift;
+
+    if (   iBase                >= iFirst
+        && iBase + cEntries - 1 <= iLast)
+    {
+        /* full range. */
+        *piFirst = 0;
+        *piLast  = cEntries - 1;
+    }
+    else if (   iBase + cEntries - 1 < iFirst
+             || iBase                > iLast)
+    {
+        /* no match */
+        *piFirst = cEntries;
+        *piLast  = 0;
+    }
+    else
+    {
+        /* partial overlap */
+        *piFirst = iBase <= iFirst
+                 ? iFirst - iBase
+                 : 0;
+        *piLast  = iBase + cEntries - 1 <= iLast
+                 ? cEntries - 1
+                 : iLast - iBase;
+    }
+
+    return iBase << cShift;
+}
+
+
+/**
+ * Maps/finds the shadow page.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The physical address of the shadow page.
+ * @param   pszDesc             The description.
+ * @param   fIsMapping          Set if it's a mapping.
+ * @param   ppv                 Where to return the pointer.
+ */
+static int pgmR3DumpHierarchyShwMapPage(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, const char *pszDesc,
+                                        bool fIsMapping, void const **ppv)
+{
+    void *pvPage;
+    if (!fIsMapping)
+    {
+        int rc = MMPagePhys2PageTry(pState->pVM, HCPhys, &pvPage);
+        if (RT_FAILURE(rc))
+        {
+            pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! %s at HCPhys=%RHp was not found in the page pool!\n",
+                                    pState->cchAddress, pState->u64Address, pszDesc, HCPhys);
+            return rc;
+        }
+    }
+    else
+    {
+        pvPage = NULL;
+        for (PPGMMAPPING pMap = pState->pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
+        {
+            uint64_t off = pState->u64Address - pMap->GCPtr;
+            if (off < pMap->cb)
+            {
+                const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
+                const int iSub = (int)((off >> X86_PD_PAE_SHIFT) & 1); /* MSC is a pain sometimes */
+                if ((iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0) != HCPhys)
+                    pState->pHlp->pfnPrintf(pState->pHlp,
+                                            "%0*llx error! Mapping error! PT %d has HCPhysPT=%RHp not %RHp is in the PD.\n",
+                                            pState->cchAddress, pState->u64Address, iPDE,
+                                            iSub ? pMap->aPTs[iPDE].HCPhysPaePT1 : pMap->aPTs[iPDE].HCPhysPaePT0, HCPhys);
+                pvPage = &pMap->aPTs[iPDE].paPaePTsR3[iSub];
+                break;
+            }
+        }
+        if (!pvPage)
+        {
+            pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! PT mapping %s at HCPhys=%RHp was not found in the page pool!\n",
+                                    pState->cchAddress, pState->u64Address, HCPhys);
+            return VERR_INVALID_PARAMETER;
+        }
+    }
+    *ppv = pvPage;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps the a shadow page summary or smth.
+ *
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The page address.
+ */
+static void pgmR3DumpHierarchyShwTablePageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys)
+{
+    pgmLock(pState->pVM);
+    char            szPage[80];
+    PPGMPOOLPAGE    pPage = pgmPoolQueryPageForDbg(pState->pVM->pgm.s.CTX_SUFF(pPool), HCPhys);
+    if (pPage)
+        RTStrPrintf(szPage, sizeof(szPage), " idx=0i%u", pPage->idx);
+    else
+    {
+        /* probably a mapping */
+        strcpy(szPage, " not found");
+        for (PPGMMAPPING pMap = pState->pVM->pgm.s.pMappingsR3; pMap; pMap = pMap->pNextR3)
+        {
+            uint64_t off = pState->u64Address - pMap->GCPtr;
+            if (off < pMap->cb)
+            {
+                const int iPDE = (uint32_t)(off >> X86_PD_SHIFT);
+                if (pMap->aPTs[iPDE].HCPhysPT == HCPhys)
+                    RTStrPrintf(szPage, sizeof(szPage), " #%u: %s", iPDE, pMap->pszDesc);
+                else if (pMap->aPTs[iPDE].HCPhysPaePT0 == HCPhys)
+                    RTStrPrintf(szPage, sizeof(szPage), " #%u/0: %s", iPDE, pMap->pszDesc);
+                else if (pMap->aPTs[iPDE].HCPhysPaePT1 == HCPhys)
+                    RTStrPrintf(szPage, sizeof(szPage), " #%u/1: %s", iPDE, pMap->pszDesc);
+                else
+                    continue;
+                break;
+            }
+        }
+    }
+    pgmUnlock(pState->pVM);
+    pState->pHlp->pfnPrintf(pState->pHlp, "%s", szPage);
+}
+
+
+/**
+ * Figures out which guest page this is and dumps a summary.
+ *
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The page address.
+ * @param   cbPage              The page size.
+ */
+static void pgmR3DumpHierarchyShwGuestPageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, uint32_t cbPage)
+{
+    char        szPage[80];
+    RTGCPHYS    GCPhys;
+    int rc = PGMR3DbgHCPhys2GCPhys(pState->pVM, HCPhys, &GCPhys);
+    if (RT_SUCCESS(rc))
+    {
+        pgmLock(pState->pVM);
+        PCPGMPAGE pPage = pgmPhysGetPage(&pState->pVM->pgm.s, GCPhys);
+        if (pPage)
+            RTStrPrintf(szPage, sizeof(szPage), "%R[pgmpage]", pPage);
+        else
+            strcpy(szPage, "not found");
+        pgmUnlock(pState->pVM);
+        pState->pHlp->pfnPrintf(pState->pHlp, " -> %RGp %s", GCPhys, szPage);
+    }
+    else
+    {
+        /* check the heap */
+        uint32_t cbAlloc;
+        rc = MMR3HyperQueryInfoFromHCPhys(pState->pVM, HCPhys, szPage, sizeof(szPage), &cbAlloc);
+        if (RT_SUCCESS(rc))
+            pState->pHlp->pfnPrintf(pState->pHlp, " %s %#x bytes", szPage, cbAlloc);
+        else
+            pState->pHlp->pfnPrintf(pState->pHlp, " not found");
+    }
+}
+
+
+/**
+ * Dumps a PAE shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState              The dumper state.
+ * @param   HCPhys              The page table address.
+ * @param   fIsMapping          Whether it is a mapping.
+ */
+static int pgmR3DumpHierarchyShwPaePT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, bool fIsMapping)
+{
+    PCPGMSHWPTPAE pPT;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page table", fIsMapping, (void const **)&pPT);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+        if (PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_P)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PT_PAE_SHIFT);
+            if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
+            {
+                X86PTEPAE Pte;
+                Pte.u = PGMSHWPTEPAE_GET_U(pPT->a[i]);
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme  /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                        ? "%016llx 3    | P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx"
+                                        :  "%08llx 2   |  P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pte.n.u1Write       ? 'W'  : 'R',
+                                        Pte.n.u1User        ? 'U'  : 'S',
+                                        Pte.n.u1Accessed    ? 'A'  : '-',
+                                        Pte.n.u1Dirty       ? 'D'  : '-',
+                                        Pte.n.u1Global      ? 'G'  : '-',
+                                        Pte.n.u1WriteThru   ? "WT" : "--",
+                                        Pte.n.u1CacheDisable? "CD" : "--",
+                                        Pte.n.u1PAT         ? "AT" : "--",
+                                        Pte.n.u1NoExecute   ? "NX" : "--",
+                                        Pte.u & PGM_PTFLAGS_TRACK_DIRTY   ? 'd' : '-',
+                                        Pte.u & RT_BIT(10)                ? '1' : '0',
+                                        Pte.u & PGM_PTFLAGS_CSAM_VALIDATED? 'v' : '-',
+                                        Pte.u & X86_PTE_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwGuestPageInfo(pState, Pte.u & X86_PTE_PAE_PG_MASK, _4K);
+                if ((Pte.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx%s", (Pte.u >> 52) & 0x7ff, pState->fLme ? "" : "!");
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+            }
+            else if (   (PGMSHWPTEPAE_GET_U(pPT->a[i]) & (pState->pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX))
+                     ==                                  (pState->pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX))
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme
+                                        ? "%016llx 3    | invalid / MMIO optimization\n"
+                                        :  "%08llx 2   |  invalid / MMIO optimization\n",
+                                        pState->u64Address);
+            else
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme
+                                        ? "%016llx 3    | invalid: %RX64\n"
+                                        :  "%08llx 2   |  invalid: %RX64\n",
+                                        pState->u64Address, PGMSHWPTEPAE_GET_U(pPT->a[i]));
+            pState->cLeaves++;
+        }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the page directory table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyShwPaePD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    PCX86PDPAE pPD;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory", false, (void const **)&pPD);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDEPAE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PD_PAE_SHIFT);
+            if (Pde.b.u1Size)
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 2M a  p ?  phys*/
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write       ? 'W'  : 'R',
+                                        Pde.b.u1User        ? 'U'  : 'S',
+                                        Pde.b.u1Accessed    ? 'A'  : '-',
+                                        Pde.b.u1Dirty       ? 'D'  : '-',
+                                        Pde.b.u1Global      ? 'G'  : '-',
+                                        Pde.b.u1WriteThru   ? "WT" : "--",
+                                        Pde.b.u1CacheDisable? "CD" : "--",
+                                        Pde.b.u1PAT         ? "AT" : "--",
+                                        Pde.b.u1NoExecute   ? "NX" : "--",
+                                        Pde.u & RT_BIT_64(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        Pde.u & X86_PDE2M_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwGuestPageInfo(pState, Pde.u & X86_PDE2M_PAE_PG_MASK, _2M);
+                if ((Pde.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx%s", (Pde.u >> 52) & 0x7ff, pState->fLme ? "" : "!");
+                if ((Pde.u >> 13) & 0xff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 20:13=%02llx%s", (Pde.u >> 13) & 0x0ff, pState->fLme ? "" : "!");
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 4M a  p ?  phys */
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.n.u1Write       ? 'W'  : 'R',
+                                        Pde.n.u1User        ? 'U'  : 'S',
+                                        Pde.n.u1Accessed    ? 'A'  : '-',
+                                        Pde.n.u1Reserved0   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru   ? "WT" : "--",
+                                        Pde.n.u1CacheDisable? "CD" : "--",
+                                        Pde.n.u1NoExecute   ? "NX" : "--",
+                                        Pde.u & RT_BIT_64(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        Pde.u & X86_PDE_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pde.u & X86_PDE_PAE_PG_MASK);
+                if ((Pde.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx!", (Pde.u >> 52) & 0x7ff);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyShwPaePT(pState, Pde.u & X86_PDE_PAE_PG_MASK, !!(Pde.u & PGM_PDFLAGS_MAPPING));
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory pointer table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the page directory pointer table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyShwPaePDPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    /* Fend of addresses that are out of range in PAE mode - simplifies the code below. */
+    if (!pState->fLme && pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PDPT pPDPT;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory pointer table", false, (void const **)&pPDPT);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PDPT_SHIFT,
+                                                          pState->fLme ? X86_PG_AMD64_PDPE_ENTRIES : X86_PG_PAE_PDPE_ENTRIES,
+                                                          &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDPE Pdpe = pPDPT->a[i];
+        if (Pdpe.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PDPT_SHIFT);
+            if (pState->fLme)
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%016llx 1  |   P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.lm.u1Write       ? 'W'  : 'R',
+                                        Pdpe.lm.u1User        ? 'U'  : 'S',
+                                        Pdpe.lm.u1Accessed    ? 'A'  : '-',
+                                        Pdpe.lm.u3Reserved & 1? '?'  : '.', /* ignored */
+                                        Pdpe.lm.u3Reserved & 4? '!'  : '.', /* mbz */
+                                        Pdpe.lm.u1WriteThru   ? "WT" : "--",
+                                        Pdpe.lm.u1CacheDisable? "CD" : "--",
+                                        Pdpe.lm.u3Reserved & 2? "!"  : "..",/* mbz */
+                                        Pdpe.lm.u1NoExecute   ? "NX" : "--",
+                                        Pdpe.u & RT_BIT(9)                ? '1' : '0',
+                                        Pdpe.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
+                                        Pdpe.u & RT_BIT(11)               ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK);
+                if ((Pdpe.u >> 52) & 0x7ff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx", (Pdpe.u >> 52) & 0x7ff);
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.n.u2Reserved & 1? '!'  : '.', /* mbz */
+                                        Pdpe.n.u2Reserved & 2? '!'  : '.', /* mbz */
+                                        Pdpe.n.u4Reserved & 1? '!'  : '.', /* mbz */
+                                        Pdpe.n.u4Reserved & 2? '!'  : '.', /* mbz */
+                                        Pdpe.n.u4Reserved & 8? '!'  : '.', /* mbz */
+                                        Pdpe.n.u1WriteThru   ? "WT" : "--",
+                                        Pdpe.n.u1CacheDisable? "CD" : "--",
+                                        Pdpe.n.u4Reserved & 2? "!"  : "..",/* mbz */
+                                        Pdpe.lm.u1NoExecute  ? "!!"  : "..",/* mbz */
+                                        Pdpe.u & RT_BIT(9)                ? '1' : '0',
+                                        Pdpe.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
+                                        Pdpe.u & RT_BIT(11)               ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK);
+                if ((Pdpe.u >> 52) & 0xfff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 63:52=%03llx!", (Pdpe.u >> 52) & 0xfff);
+            }
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyShwPaePD(pState, Pdpe.u & X86_PDPE_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pVM         The VM handle.
+ * @param   HCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyShwPaePML4(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    PCX86PML4 pPML4;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page map level 4", false, (void const **)&pPML4);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth);
+    cMaxDepth--;
+
+    /*
+     * This is a bit tricky as we're working on unsigned addresses while the
+     * AMD64 spec uses signed tricks.
+     */
+    uint32_t iFirst = (pState->u64FirstAddress >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    uint32_t iLast  = (pState->u64LastAddress  >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (   pState->u64LastAddress  <= UINT64_C(0x00007fffffffffff)
+        || pState->u64FirstAddress >= UINT64_C(0xffff800000000000))
+    { /* Simple, nothing to adjust */ }
+    else if (pState->u64FirstAddress <= UINT64_C(0x00007fffffffffff))
+        iLast = X86_PG_AMD64_ENTRIES / 2 - 1;
+    else if (pState->u64LastAddress  >= UINT64_C(0xffff800000000000))
+        iFirst = X86_PG_AMD64_ENTRIES / 2;
+    else
+        iFirst = X86_PG_AMD64_ENTRIES; /* neither address is canonical */
+
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PML4E Pml4e = pPML4->a[i];
+        if (Pml4e.n.u1Present)
+        {
+            pState->u64Address = ((uint64_t)i << X86_PML4_SHIFT)
+                               | (i >= RT_ELEMENTS(pPML4->a) / 2 ? UINT64_C(0xffff000000000000) : 0);
+            pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                    "%016llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                    pState->u64Address,
+                                    Pml4e.n.u1Write       ? 'W'  : 'R',
+                                    Pml4e.n.u1User        ? 'U'  : 'S',
+                                    Pml4e.n.u1Accessed    ? 'A'  : '-',
+                                    Pml4e.n.u3Reserved & 1? '?'  : '.', /* ignored */
+                                    Pml4e.n.u3Reserved & 4? '!'  : '.', /* mbz */
+                                    Pml4e.n.u1WriteThru   ? "WT" : "--",
+                                    Pml4e.n.u1CacheDisable? "CD" : "--",
+                                    Pml4e.n.u3Reserved & 2? "!"  : "..",/* mbz */
+                                    Pml4e.n.u1NoExecute   ? "NX" : "--",
+                                    Pml4e.u & RT_BIT(9)                ? '1' : '0',
+                                    Pml4e.u & PGM_PLXFLAGS_PERMANENT   ? 'p' : '-',
+                                    Pml4e.u & RT_BIT(11)               ? '1' : '0',
+                                    Pml4e.u & X86_PML4E_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyShwTablePageInfo(pState, Pml4e.u & X86_PML4E_PG_MASK);
+            if ((Pml4e.u >> 52) & 0x7ff)
+                pState->pHlp->pfnPrintf(pState->pHlp, " 62:52=%03llx!", (Pml4e.u >> 52) & 0x7ff);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyShwPaePDPT(pState, Pml4e.u & X86_PML4E_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pVM         The VM handle.
+ * @param   pPT         Pointer to the page table.
+ * @param   fMapping    Set if it's a guest mapping.
+ */
+static int pgmR3DumpHierarchyShw32BitPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, bool fMapping)
+{
+    PCX86PT pPT;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page table", fMapping, (void const **)&pPT);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PTE Pte = pPT->a[i];
+        if (Pte.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + (i << X86_PT_SHIFT);
+            pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d  */
+                                    "%08llx 1  |   P %c %c %c %c %c %s %s %s .. 4K %c%c%c  %08x",
+                                    pState->u64Address,
+                                    Pte.n.u1Write       ? 'W'  : 'R',
+                                    Pte.n.u1User        ? 'U'  : 'S',
+                                    Pte.n.u1Accessed    ? 'A'  : '-',
+                                    Pte.n.u1Dirty       ? 'D'  : '-',
+                                    Pte.n.u1Global      ? 'G'  : '-',
+                                    Pte.n.u1WriteThru   ? "WT" : "--",
+                                    Pte.n.u1CacheDisable? "CD" : "--",
+                                    Pte.n.u1PAT         ? "AT" : "--",
+                                    Pte.u & PGM_PTFLAGS_TRACK_DIRTY     ? 'd' : '-',
+                                    Pte.u & RT_BIT(10)                  ? '1' : '0',
+                                    Pte.u & PGM_PTFLAGS_CSAM_VALIDATED  ? 'v' : '-',
+                                    Pte.u & X86_PDE_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyShwGuestPageInfo(pState, Pte.u & X86_PDE_PG_MASK, _4K);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page directory and page tables.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   HCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyShw32BitPD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS HCPhys, unsigned cMaxDepth)
+{
+    if (pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PD pPD;
+    int rc = pgmR3DumpHierarchyShwMapPage(pState, HCPhys, "Page directory", false, (void const **)&pPD);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = (uint32_t)i << X86_PD_SHIFT;
+            if (Pde.b.u1Size && pState->fPse)
+            {
+                uint64_t u64Phys = ((uint64_t)(Pde.u & X86_PDE4M_PG_HIGH_MASK) << X86_PDE4M_PG_HIGH_SHIFT)
+                                 | (Pde.u & X86_PDE4M_PG_MASK);
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s .. 4M %c%c%c  %08llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write       ? 'W'  : 'R',
+                                        Pde.b.u1User        ? 'U'  : 'S',
+                                        Pde.b.u1Accessed    ? 'A'  : '-',
+                                        Pde.b.u1Dirty       ? 'D'  : '-',
+                                        Pde.b.u1Global      ? 'G'  : '-',
+                                        Pde.b.u1WriteThru   ? "WT" : "--",
+                                        Pde.b.u1CacheDisable? "CD" : "--",
+                                        Pde.b.u1PAT         ? "AT" : "--",
+                                        Pde.u & RT_BIT_32(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        u64Phys);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwGuestPageInfo(pState, u64Phys, _4M);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s .. .. 4K %c%c%c  %08x",
+                                        pState->u64Address,
+                                        Pde.n.u1Write       ? 'W'  : 'R',
+                                        Pde.n.u1User        ? 'U'  : 'S',
+                                        Pde.n.u1Accessed    ? 'A'  : '-',
+                                        Pde.n.u1Reserved0   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1   ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru   ? "WT" : "--",
+                                        Pde.n.u1CacheDisable? "CD" : "--",
+                                        Pde.u & RT_BIT_32(9)            ? '1' : '0',
+                                        Pde.u & PGM_PDFLAGS_MAPPING     ? 'm' : '-',
+                                        Pde.u & PGM_PDFLAGS_TRACK_DIRTY ? 'd' : '-',
+                                        Pde.u & X86_PDE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyShwTablePageInfo(pState, Pde.u & X86_PDE_PG_MASK);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyShw32BitPT(pState, Pde.u & X86_PDE_PG_MASK, !!(Pde.u & PGM_PDFLAGS_MAPPING));
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Internal worker that initiates the actual dump.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   cr3                 The CR3 value.
+ * @param   cMaxDepth           The max depth.
+ */
+static int pgmR3DumpHierarchyShwDoIt(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t cr3, unsigned cMaxDepth)
+{
+    int             rc;
+    unsigned const  cch     = pState->cchAddress;
+    uint64_t const  cr3Mask = pState->fEpt ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fLme ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fPae ? X86_CR3_PAE_PAGE_MASK
+                            :                X86_CR3_PAGE_MASK;
+    if (pState->fPrintCr3)
+    {
+        const char * const  pszMode = pState->fEpt ? "Extended Page Tables"
+                                    : pState->fLme ? "Long Mode"
+                                    : pState->fPae ? "PAE Mode"
+                                    : pState->fPse ? "32-bit w/ PSE"
+                                    :                "32-bit";
+        pState->pHlp->pfnPrintf(pState->pHlp, "cr3=%0*llx", cch, cr3);
+        if (pState->fDumpPageInfo)
+            pgmR3DumpHierarchyShwTablePageInfo(pState, cr3 & X86_CR3_AMD64_PAGE_MASK);
+        pState->pHlp->pfnPrintf(pState->pHlp, " %s%s\n",
+                                pszMode,
+                                pState->fNp  ? " + Nested Paging" : "",
+                                pState->fNxe ? " + NX" : "");
+    }
+
+
+    if (pState->fEpt)
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        R - Readable\n"
+                                    "%-*s        | W - Writeable\n"
+                                    "%-*s        | | X - Executable\n"
+                                    "%-*s        | | | EMT - EPT memory type\n"
+                                    "%-*s        | | | |   PAT - Ignored PAT?\n"
+                                    "%-*s        | | | |   |    AVL1 - 4 available bits\n"
+                                    "%-*s        | | | |   |    |   AVL2 - 12 available bits\n"
+                                    "%-*s Level  | | | |   |    |    |     page  \n"
+                                  /* xxxx n **** R W X EMT PAT AVL1 AVL2   xxxxxxxxxxxxx
+                                                 R W X  7   0   f   fff    0123456701234567 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+
+        pState->pHlp->pfnPrintf(pState->pHlp, "EPT dumping is not yet implemented, sorry.\n");
+        /** @todo implemented EPT dumping. */
+        rc = VERR_NOT_IMPLEMENTED;
+    }
+    else
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        P - Present\n"
+                                    "%-*s        | R/W - Read (0) / Write (1)\n"
+                                    "%-*s        | | U/S - User (1) / Supervisor (0)\n"
+                                    "%-*s        | | | A - Accessed\n"
+                                    "%-*s        | | | | D - Dirty\n"
+                                    "%-*s        | | | | | G - Global\n"
+                                    "%-*s        | | | | | | WT - Write thru\n"
+                                    "%-*s        | | | | | | |  CD - Cache disable\n"
+                                    "%-*s        | | | | | | |  |  AT - Attribute table (PAT)\n"
+                                    "%-*s        | | | | | | |  |  |  NX - No execute (K8)\n"
+                                    "%-*s        | | | | | | |  |  |  |  4K/4M/2M - Page size.\n"
+                                    "%-*s        | | | | | | |  |  |  |  |  AVL - a=allocated; m=mapping; d=track dirty;\n"
+                                    "%-*s        | | | | | | |  |  |  |  |  |     p=permanent; v=validated;\n"
+                                    "%-*s Level  | | | | | | |  |  |  |  |  |    Page\n"
+                                  /* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
+                                                 - W U - - - -- -- -- -- -- 010 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+        if (pState->fLme)
+            rc = pgmR3DumpHierarchyShwPaePML4(pState, cr3 & cr3Mask, cMaxDepth);
+        else if (pState->fPae)
+            rc = pgmR3DumpHierarchyShwPaePDPT(pState, cr3 & cr3Mask, cMaxDepth);
+        else
+            rc = pgmR3DumpHierarchyShw32BitPD(pState, cr3 & cr3Mask, cMaxDepth);
+    }
+
+    if (!pState->cLeaves)
+        pState->pHlp->pfnPrintf(pState->pHlp, "not present\n");
+    return rc;
+}
+
+
+/**
+ * dbgfR3PagingDumpEx worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   cr3             The CR3 register value.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
+ * @param   u64FirstAddr    The start address.
+ * @param   u64LastAddr     The address to stop after.
+ * @param   cMaxDepth       The max depth.
+ * @param   pHlp            The output callbacks.  Defaults to log if NULL.
+ *
+ * @internal
+ */
+VMMR3_INT_DECL(int) PGMR3DumpHierarchyShw(PVM pVM, uint64_t cr3, uint32_t fFlags, uint64_t u64FirstAddr, uint64_t u64LastAddr,
+                                          uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /* Minimal validation as we're only supposed to service DBGF. */
+    AssertReturn(~(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & (DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3)), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags & DBGFPGDMP_FLAGS_SHADOW, VERR_INVALID_PARAMETER);
+
+    PGMR3DUMPHIERARCHYSTATE State;
+    pgmR3DumpHierarchyInitState(&State, pVM, fFlags, u64FirstAddr, u64LastAddr, pHlp);
+    return pgmR3DumpHierarchyShwDoIt(&State, cr3, cMaxDepth);
+}
+
+
+/**
+ * Dumps a page table hierarchy use only physical addresses and cr4/lm flags.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pVM         The VM handle.
+ * @param   cr3         The root of the hierarchy.
+ * @param   cr4         The cr4, only PAE and PSE is currently used.
+ * @param   fLongMode   Set if long mode, false if not long mode.
+ * @param   cMaxDepth   Number of levels to dump.
+ * @param   pHlp        Pointer to the output functions.
+ *
+ * @deprecated Use DBGFR3PagingDumpEx.
+ */
+VMMR3DECL(int) PGMR3DumpHierarchyHC(PVM pVM, uint64_t cr3, uint64_t cr4, bool fLongMode, unsigned cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    if (!cMaxDepth)
+        return VINF_SUCCESS;
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        pVCpu = &pVM->aCpus[0];
+
+    uint32_t fFlags = DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3 | DBGFPGDMP_FLAGS_PAGE_INFO | DBGFPGDMP_FLAGS_SHADOW;
+    fFlags |= cr4 & (X86_CR4_PAE | X86_CR4_PSE);
+    if (fLongMode)
+        fFlags |= DBGFPGDMP_FLAGS_LME;
+
+    return DBGFR3PagingDumpEx(pVM, pVCpu->idCpu, fFlags, cr3, 0, fLongMode ? UINT64_MAX : UINT32_MAX, cMaxDepth, pHlp);
+}
+
+
+/**
+ * Maps the guest page.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   GCPhys              The physical address of the guest page.
+ * @param   pszDesc             The description.
+ * @param   ppv                 Where to return the pointer.
+ * @param   pLock               Where to return the mapping lock.  Hand this to
+ *                              PGMPhysReleasePageMappingLock when done.
+ */
+static int pgmR3DumpHierarchyGstMapPage(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, const char *pszDesc,
+                                        void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = PGMPhysGCPhys2CCPtrReadOnly(pState->pVM, GCPhys, ppv, pLock);
+    if (RT_FAILURE(rc))
+    {
+        pState->pHlp->pfnPrintf(pState->pHlp, "%0*llx error! Failed to map %s at GCPhys=%RGp: %Rrc!\n",
+                                pState->cchAddress, pState->u64Address, pszDesc, GCPhys, rc);
+        return rc;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Figures out which guest page this is and dumps a summary.
+ *
+ * @param   pState              The dumper state.
+ * @param   GCPhys              The page address.
+ * @param   cbPage              The page size.
+ */
+static void pgmR3DumpHierarchyGstPageInfo(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, uint32_t cbPage)
+{
+    char szPage[80];
+    pgmLock(pState->pVM);
+    PCPGMPAGE pPage = pgmPhysGetPage(&pState->pVM->pgm.s, GCPhys);
+    if (pPage)
+        RTStrPrintf(szPage, sizeof(szPage), " %R[pgmpage]", pPage);
+    else
+        strcpy(szPage, " not found");
+    pgmUnlock(pState->pVM);
+    pState->pHlp->pfnPrintf(pState->pHlp, "%s", szPage);
+}
+
+
+/**
+ * Checks the entry for reserved bits.
+ *
+ * @param   pState              The dumper state.
+ * @param   u64Entry            The entry to check.
+ */
+static void pgmR3DumpHierarchyGstCheckReservedHighBits(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t u64Entry)
+{
+    uint32_t uRsvd = (u64Entry & pState->u64HighReservedBits) >> 52;
+    if (uRsvd)
+        pState->pHlp->pfnPrintf(pState->pHlp, " %u:52=%03x%s",
+                                pState->uLastRsvdBit, uRsvd, pState->fLme ? "" : "!");
+    /** @todo check the valid physical bits as well. */
+}
+
+
+/**
+ * Dumps a PAE shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState              The dumper state.
+ * @param   GCPhys              The page table address.
+ */
+static int pgmR3DumpHierarchyGstPaePT(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys)
+{
+    PCX86PTPAE      pPT;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page table", (void const **)&pPT, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PTEPAE Pte = pPT->a[i];
+        if (Pte.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PT_PAE_SHIFT);
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    pState->fLme  /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                    ? "%016llx 3    | P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx"
+                                    :  "%08llx 2   |  P %c %c %c %c %c %s %s %s %s 4K %c%c%c  %016llx",
+                                    pState->u64Address,
+                                    Pte.n.u1Write       ? 'W'  : 'R',
+                                    Pte.n.u1User        ? 'U'  : 'S',
+                                    Pte.n.u1Accessed    ? 'A'  : '-',
+                                    Pte.n.u1Dirty       ? 'D'  : '-',
+                                    Pte.n.u1Global      ? 'G'  : '-',
+                                    Pte.n.u1WriteThru   ? "WT" : "--",
+                                    Pte.n.u1CacheDisable? "CD" : "--",
+                                    Pte.n.u1PAT         ? "AT" : "--",
+                                    Pte.n.u1NoExecute   ? "NX" : "--",
+                                    Pte.u & RT_BIT(9)   ? '1' : '0',
+                                    Pte.u & RT_BIT(10)  ? '1' : '0',
+                                    Pte.u & RT_BIT(11)  ? '1' : '0',
+                                    Pte.u & X86_PTE_PAE_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyGstPageInfo(pState, Pte.u & X86_PTE_PAE_PG_MASK, _4K);
+            pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pte.u);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+            pState->cLeaves++;
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyGstPaePD(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, unsigned cMaxDepth)
+{
+    PCX86PDPAE      pPD;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory", (void const **)&pPD, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_PAE_SHIFT, X86_PG_PAE_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDEPAE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PD_PAE_SHIFT);
+            if (Pde.b.u1Size)
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 2M a  p ?  phys*/
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s %s %s 2M %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write         ? 'W'  : 'R',
+                                        Pde.b.u1User          ? 'U'  : 'S',
+                                        Pde.b.u1Accessed      ? 'A'  : '-',
+                                        Pde.b.u1Dirty         ? 'D'  : '-',
+                                        Pde.b.u1Global        ? 'G'  : '-',
+                                        Pde.b.u1WriteThru     ? "WT" : "--",
+                                        Pde.b.u1CacheDisable  ? "CD" : "--",
+                                        Pde.b.u1PAT           ? "AT" : "--",
+                                        Pde.b.u1NoExecute     ? "NX" : "--",
+                                        Pde.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pde.u & X86_PDE2M_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE2M_PAE_PG_MASK, _2M);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pde.u);
+                if ((Pde.u >> 13) & 0xff)
+                    pState->pHlp->pfnPrintf(pState->pHlp, " 20:13=%02llx%s", (Pde.u >> 13) & 0x0ff, pState->fLme ? "" : "!");
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,
+                                        pState->fLme    /*P R  S  A  D  G  WT CD AT NX 4M a  p ?  phys */
+                                        ? "%016llx 2   |  P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx"
+                                        :  "%08llx 1  |   P %c %c %c %c %c %s %s .. %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pde.n.u1Write         ? 'W'  : 'R',
+                                        Pde.n.u1User          ? 'U'  : 'S',
+                                        Pde.n.u1Accessed      ? 'A'  : '-',
+                                        Pde.n.u1Reserved0     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru     ? "WT" : "--",
+                                        Pde.n.u1CacheDisable  ? "CD" : "--",
+                                        Pde.n.u1NoExecute     ? "NX" : "--",
+                                        Pde.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pde.u & X86_PDE_PAE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE_PAE_PG_MASK, _4K);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pde.u);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyGstPaePT(pState, Pde.u & X86_PDE_PAE_PG_MASK);
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Dumps a PAE shadow page directory pointer table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int  pgmR3DumpHierarchyGstPaePDPT(PPGMR3DUMPHIERARCHYSTATE pState, RTGCPHYS GCPhys, unsigned cMaxDepth)
+{
+    /* Fend of addresses that are out of range in PAE mode - simplifies the code below. */
+    if (!pState->fLme && pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PDPT       pPDPT;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory pointer table", (void const **)&pPDPT, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PDPT_SHIFT,
+                                                          pState->fLme ? X86_PG_AMD64_PDPE_ENTRIES : X86_PG_PAE_PDPE_ENTRIES,
+                                                          &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDPE Pdpe = pPDPT->a[i];
+        if (Pdpe.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + ((uint64_t)i << X86_PDPT_SHIFT);
+            if (pState->fLme)
+            {
+                /** @todo Do 1G pages.  */
+                pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%016llx 1  |   P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.lm.u1Write        ? 'W'  : 'R',
+                                        Pdpe.lm.u1User         ? 'U'  : 'S',
+                                        Pdpe.lm.u1Accessed     ? 'A'  : '-',
+                                        Pdpe.lm.u3Reserved & 1 ? '?'  : '.', /* ignored */
+                                        Pdpe.lm.u3Reserved & 4 ? '!'  : '.', /* mbz */
+                                        Pdpe.lm.u1WriteThru    ? "WT" : "--",
+                                        Pdpe.lm.u1CacheDisable ? "CD" : "--",
+                                        Pdpe.lm.u3Reserved & 2 ? "!"  : "..",/* mbz */
+                                        Pdpe.lm.u1NoExecute    ? "NX" : "--",
+                                        Pdpe.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK, _4K);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pdpe.u);
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX .. a p ?  */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                        pState->u64Address,
+                                        Pdpe.n.u2Reserved & 1  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u2Reserved & 2  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u4Reserved & 1  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u4Reserved & 2  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u4Reserved & 8  ? '!'  : '.',  /* mbz */
+                                        Pdpe.n.u1WriteThru     ? "WT" : "--",
+                                        Pdpe.n.u1CacheDisable  ? "CD" : "--",
+                                        Pdpe.n.u4Reserved & 2  ? "!"  : "..", /* mbz */
+                                        Pdpe.lm.u1NoExecute    ? "!!"  : "..",/* mbz */
+                                        Pdpe.u & RT_BIT_64(9)  ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(10) ? '1' : '0',
+                                        Pdpe.u & RT_BIT_64(11) ? '1' : '0',
+                                        Pdpe.u & X86_PDPE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pdpe.u & X86_PDPE_PG_MASK, _4K);
+                pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pdpe.u);
+            }
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyGstPaePD(pState, Pdpe.u & X86_PDPE_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pVM         The VM handle.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyGstPaePML4(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys, unsigned cMaxDepth)
+{
+    PCX86PML4       pPML4;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page map level 4", (void const **)&pPML4, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth);
+    cMaxDepth--;
+
+    /*
+     * This is a bit tricky as we're working on unsigned addresses while the
+     * AMD64 spec uses signed tricks.
+     */
+    uint32_t iFirst = (pState->u64FirstAddress >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    uint32_t iLast  = (pState->u64LastAddress  >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (   pState->u64LastAddress  <= UINT64_C(0x00007fffffffffff)
+        || pState->u64FirstAddress >= UINT64_C(0xffff800000000000))
+    { /* Simple, nothing to adjust */ }
+    else if (pState->u64FirstAddress <= UINT64_C(0x00007fffffffffff))
+        iLast = X86_PG_AMD64_ENTRIES / 2 - 1;
+    else if (pState->u64LastAddress  >= UINT64_C(0xffff800000000000))
+        iFirst = X86_PG_AMD64_ENTRIES / 2;
+    else
+        iFirst = X86_PG_AMD64_ENTRIES; /* neither address is canonical */
+
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PML4E Pml4e = pPML4->a[i];
+        if (Pml4e.n.u1Present)
+        {
+            pState->u64Address = ((uint64_t)i << X86_PML4_SHIFT)
+                               | (i >= RT_ELEMENTS(pPML4->a) / 2 ? UINT64_C(0xffff000000000000) : 0);
+            pState->pHlp->pfnPrintf(pState->pHlp, /*P R  S  A  D  G  WT CD AT NX 4M a p ?  */
+                                    "%016llx 0 |    P %c %c %c %c %c %s %s %s %s .. %c%c%c  %016llx",
+                                    pState->u64Address,
+                                    Pml4e.n.u1Write         ? 'W'  : 'R',
+                                    Pml4e.n.u1User          ? 'U'  : 'S',
+                                    Pml4e.n.u1Accessed      ? 'A'  : '-',
+                                    Pml4e.n.u3Reserved & 1  ? '?'  : '.', /* ignored */
+                                    Pml4e.n.u3Reserved & 4  ? '!'  : '.', /* mbz */
+                                    Pml4e.n.u1WriteThru     ? "WT" : "--",
+                                    Pml4e.n.u1CacheDisable  ? "CD" : "--",
+                                    Pml4e.n.u3Reserved & 2  ? "!"  : "..",/* mbz */
+                                    Pml4e.n.u1NoExecute     ? "NX" : "--",
+                                    Pml4e.u & RT_BIT_64(9)  ? '1' : '0',
+                                    Pml4e.u & RT_BIT_64(10) ? '1' : '0',
+                                    Pml4e.u & RT_BIT_64(11) ? '1' : '0',
+                                    Pml4e.u & X86_PML4E_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyGstPageInfo(pState, Pml4e.u & X86_PML4E_PG_MASK, _4K);
+            pgmR3DumpHierarchyGstCheckReservedHighBits(pState, Pml4e.u);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+            if (cMaxDepth)
+            {
+                int rc2 = pgmR3DumpHierarchyGstPaePDPT(pState, Pml4e.u & X86_PML4E_PG_MASK, cMaxDepth);
+                if (rc2 < rc && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+            else
+                pState->cLeaves++;
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page table.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ */
+static int pgmR3DumpHierarchyGst32BitPT(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys)
+{
+    PCX86PT         pPT;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page table", (void const **)&pPT, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PT_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PTE Pte = pPT->a[i];
+        if (Pte.n.u1Present)
+        {
+            pState->u64Address = u64BaseAddress + (i << X86_PT_SHIFT);
+            pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d  */
+                                    "%08llx 1  |   P %c %c %c %c %c %s %s %s .. 4K %c%c%c  %08x",
+                                    pState->u64Address,
+                                    Pte.n.u1Write         ? 'W'  : 'R',
+                                    Pte.n.u1User          ? 'U'  : 'S',
+                                    Pte.n.u1Accessed      ? 'A'  : '-',
+                                    Pte.n.u1Dirty         ? 'D'  : '-',
+                                    Pte.n.u1Global        ? 'G'  : '-',
+                                    Pte.n.u1WriteThru     ? "WT" : "--",
+                                    Pte.n.u1CacheDisable  ? "CD" : "--",
+                                    Pte.n.u1PAT           ? "AT" : "--",
+                                    Pte.u & RT_BIT_32(9)  ? '1' : '0',
+                                    Pte.u & RT_BIT_32(10) ? '1' : '0',
+                                    Pte.u & RT_BIT_32(11) ? '1' : '0',
+                                    Pte.u & X86_PDE_PG_MASK);
+            if (pState->fDumpPageInfo)
+                pgmR3DumpHierarchyGstPageInfo(pState, Pte.u & X86_PDE_PG_MASK, _4K);
+            pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps a 32-bit shadow page directory and page tables.
+ *
+ * @returns VBox status code (VINF_SUCCESS).
+ * @param   pState      The dumper state.
+ * @param   GCPhys      The physical address of the table.
+ * @param   cMaxDepth   The maximum depth.
+ */
+static int pgmR3DumpHierarchyGst32BitPD(PPGMR3DUMPHIERARCHYSTATE pState, RTHCPHYS GCPhys, unsigned cMaxDepth)
+{
+    if (pState->u64Address >= _4G)
+        return VINF_SUCCESS;
+
+    PCX86PD         pPD;
+    PGMPAGEMAPLOCK  Lock;
+    int rc = pgmR3DumpHierarchyGstMapPage(pState, GCPhys, "Page directory", (void const **)&pPD, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(cMaxDepth > 0);
+    cMaxDepth--;
+
+    uint32_t iFirst, iLast;
+    uint64_t u64BaseAddress = pgmR3DumpHierarchyCalcRange(pState, X86_PD_SHIFT, X86_PG_ENTRIES, &iFirst, &iLast);
+    for (uint32_t i = iFirst; i <= iLast; i++)
+    {
+        X86PDE Pde = pPD->a[i];
+        if (Pde.n.u1Present)
+        {
+            pState->u64Address = (uint32_t)i << X86_PD_SHIFT;
+            if (Pde.b.u1Size && pState->fPse)
+            {
+                uint64_t u64Phys = ((uint64_t)(Pde.u & X86_PDE4M_PG_HIGH_MASK) << X86_PDE4M_PG_HIGH_SHIFT)
+                                 | (Pde.u & X86_PDE4M_PG_MASK);
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s %s .. 4M %c%c%c  %08llx",
+                                        pState->u64Address,
+                                        Pde.b.u1Write         ? 'W'  : 'R',
+                                        Pde.b.u1User          ? 'U'  : 'S',
+                                        Pde.b.u1Accessed      ? 'A'  : '-',
+                                        Pde.b.u1Dirty         ? 'D'  : '-',
+                                        Pde.b.u1Global        ? 'G'  : '-',
+                                        Pde.b.u1WriteThru     ? "WT" : "--",
+                                        Pde.b.u1CacheDisable  ? "CD" : "--",
+                                        Pde.b.u1PAT           ? "AT" : "--",
+                                        Pde.u & RT_BIT_32(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_32(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_32(11) ? '1' : '0',
+                                        u64Phys);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, u64Phys, _4M);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+                pState->cLeaves++;
+            }
+            else
+            {
+                pState->pHlp->pfnPrintf(pState->pHlp,/*P R  S  A  D  G  WT CD AT NX 4M a m d   phys */
+                                        "%08llx 0 |    P %c %c %c %c %c %s %s .. .. .. %c%c%c  %08x",
+                                        pState->u64Address,
+                                        Pde.n.u1Write         ? 'W'  : 'R',
+                                        Pde.n.u1User          ? 'U'  : 'S',
+                                        Pde.n.u1Accessed      ? 'A'  : '-',
+                                        Pde.n.u1Reserved0     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1Reserved1     ? '?'  : '.', /* ignored */
+                                        Pde.n.u1WriteThru     ? "WT" : "--",
+                                        Pde.n.u1CacheDisable  ? "CD" : "--",
+                                        Pde.u & RT_BIT_32(9)  ? '1' : '0',
+                                        Pde.u & RT_BIT_32(10) ? '1' : '0',
+                                        Pde.u & RT_BIT_32(11) ? '1' : '0',
+                                        Pde.u & X86_PDE_PG_MASK);
+                if (pState->fDumpPageInfo)
+                    pgmR3DumpHierarchyGstPageInfo(pState, Pde.u & X86_PDE_PG_MASK, _4K);
+                pState->pHlp->pfnPrintf(pState->pHlp, "\n");
+
+                if (cMaxDepth)
+                {
+                    int rc2 = pgmR3DumpHierarchyGst32BitPT(pState, Pde.u & X86_PDE_PG_MASK);
+                    if (rc2 < rc && RT_SUCCESS(rc))
+                        rc = rc2;
+                }
+                else
+                    pState->cLeaves++;
+            }
+        }
+    }
+
+    PGMPhysReleasePageMappingLock(pState->pVM, &Lock);
+    return rc;
+}
+
+
+/**
+ * Internal worker that initiates the actual dump.
+ *
+ * @returns VBox status code.
+ * @param   pState              The dumper state.
+ * @param   cr3                 The CR3 value.
+ * @param   cMaxDepth           The max depth.
+ */
+static int pgmR3DumpHierarchyGstDoIt(PPGMR3DUMPHIERARCHYSTATE pState, uint64_t cr3, unsigned cMaxDepth)
+{
+    int             rc;
+    unsigned const  cch     = pState->cchAddress;
+    uint64_t const  cr3Mask = pState->fEpt ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fLme ? X86_CR3_AMD64_PAGE_MASK
+                            : pState->fPae ? X86_CR3_PAE_PAGE_MASK
+                            :                X86_CR3_PAGE_MASK;
+    if (pState->fPrintCr3)
+    {
+        const char * const  pszMode = pState->fEpt ? "Extended Page Tables"
+                                    : pState->fLme ? "Long Mode"
+                                    : pState->fPae ? "PAE Mode"
+                                    : pState->fPse ? "32-bit w/ PSE"
+                                    :                "32-bit";
+        pState->pHlp->pfnPrintf(pState->pHlp, "cr3=%0*llx", cch, cr3);
+        if (pState->fDumpPageInfo)
+            pgmR3DumpHierarchyGstPageInfo(pState, cr3 & X86_CR3_AMD64_PAGE_MASK, _4K);
+        pState->pHlp->pfnPrintf(pState->pHlp, " %s%s%s\n",
+                                pszMode,
+                                pState->fNp  ? " + Nested Paging" : "",
+                                pState->fNxe ? " + NX" : "");
+    }
+
+
+    if (pState->fEpt)
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        R - Readable\n"
+                                    "%-*s        | W - Writeable\n"
+                                    "%-*s        | | X - Executable\n"
+                                    "%-*s        | | | EMT - EPT memory type\n"
+                                    "%-*s        | | | |   PAT - Ignored PAT?\n"
+                                    "%-*s        | | | |   |    AVL1 - 4 available bits\n"
+                                    "%-*s        | | | |   |    |   AVL2 - 12 available bits\n"
+                                    "%-*s Level  | | | |   |    |    |     page  \n"
+                                  /* xxxx n **** R W X EMT PAT AVL1 AVL2   xxxxxxxxxxxxx
+                                                 R W X  7   0   f   fff    0123456701234567 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+
+        pState->pHlp->pfnPrintf(pState->pHlp, "EPT dumping is not yet implemented, sorry.\n");
+        /** @todo implemented EPT dumping. */
+        rc = VERR_NOT_IMPLEMENTED;
+    }
+    else
+    {
+        if (pState->fPrintHeader)
+            pState->pHlp->pfnPrintf(pState->pHlp,
+                                    "%-*s        P - Present\n"
+                                    "%-*s        | R/W - Read (0) / Write (1)\n"
+                                    "%-*s        | | U/S - User (1) / Supervisor (0)\n"
+                                    "%-*s        | | | A - Accessed\n"
+                                    "%-*s        | | | | D - Dirty\n"
+                                    "%-*s        | | | | | G - Global\n"
+                                    "%-*s        | | | | | | WT - Write thru\n"
+                                    "%-*s        | | | | | | |  CD - Cache disable\n"
+                                    "%-*s        | | | | | | |  |  AT - Attribute table (PAT)\n"
+                                    "%-*s        | | | | | | |  |  |  NX - No execute (K8)\n"
+                                    "%-*s        | | | | | | |  |  |  |  4K/4M/2M - Page size.\n"
+                                    "%-*s        | | | | | | |  |  |  |  |  AVL - 3 available bits.\n"
+                                    "%-*s Level  | | | | | | |  |  |  |  |  |    Page\n"
+                                  /* xxxx n **** P R S A D G WT CD AT NX 4M AVL xxxxxxxxxxxxx
+                                                 - W U - - - -- -- -- -- -- 010 */
+                                    ,
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "",
+                                    cch, "", cch, "", cch, "", cch, "", cch, "", cch, "", cch, "Address");
+        if (pState->fLme)
+            rc = pgmR3DumpHierarchyGstPaePML4(pState, cr3 & cr3Mask, cMaxDepth);
+        else if (pState->fPae)
+            rc = pgmR3DumpHierarchyGstPaePDPT(pState, cr3 & cr3Mask, cMaxDepth);
+        else
+            rc = pgmR3DumpHierarchyGst32BitPD(pState, cr3 & cr3Mask, cMaxDepth);
+    }
+
+    if (!pState->cLeaves)
+        pState->pHlp->pfnPrintf(pState->pHlp, "not present\n");
+    return rc;
+}
+
+
+/**
+ * dbgfR3PagingDumpEx worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   cr3             The CR3 register value.
+ * @param   fFlags          The flags, DBGFPGDMP_FLAGS_XXX.
+ * @param   FirstAddr       The start address.
+ * @param   LastAddr        The address to stop after.
+ * @param   cMaxDepth       The max depth.
+ * @param   pHlp            The output callbacks.  Defaults to log if NULL.
+ *
+ * @internal
+ */
+VMMR3_INT_DECL(int) PGMR3DumpHierarchyGst(PVM pVM, uint64_t cr3, uint32_t fFlags, RTGCPTR FirstAddr, RTGCPTR LastAddr,
+                                          uint32_t cMaxDepth, PCDBGFINFOHLP pHlp)
+{
+    /* Minimal validation as we're only supposed to service DBGF. */
+    AssertReturn(~(fFlags & ~DBGFPGDMP_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & (DBGFPGDMP_FLAGS_CURRENT_MODE | DBGFPGDMP_FLAGS_CURRENT_CR3)), VERR_INVALID_PARAMETER);
+    AssertReturn(fFlags & DBGFPGDMP_FLAGS_GUEST, VERR_INVALID_PARAMETER);
+
+    PGMR3DUMPHIERARCHYSTATE State;
+    pgmR3DumpHierarchyInitState(&State, pVM, fFlags, FirstAddr, LastAddr, pHlp);
+    return pgmR3DumpHierarchyGstDoIt(&State, cr3, cMaxDepth);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMGst.h
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMGst.h	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMGst.h	(revision 35333)
@@ -0,0 +1,134 @@
+/* $Id$ */
+/** @file
+ * VBox - Page Manager / Monitor, Guest Paging Template.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+RT_C_DECLS_BEGIN
+/* r3 */
+PGM_GST_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0);
+PGM_GST_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
+PGM_GST_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta);
+PGM_GST_DECL(int, Exit)(PVMCPU pVCpu);
+
+/* all */
+PGM_GST_DECL(int, GetPage)(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys);
+PGM_GST_DECL(int, ModifyPage)(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask);
+PGM_GST_DECL(int, GetPDE)(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPDE);
+RT_C_DECLS_END
+
+
+/**
+ * Initializes the guest bit of the paging mode data.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
+ *                          This is used early in the init process to avoid trouble with PDM
+ *                          not being initialized yet.
+ */
+PGM_GST_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0)
+{
+    Assert(pModeData->uGstType == PGM_GST_TYPE);
+
+    /* Ring-3 */
+    pModeData->pfnR3GstRelocate           = PGM_GST_NAME(Relocate);
+    pModeData->pfnR3GstExit               = PGM_GST_NAME(Exit);
+    pModeData->pfnR3GstGetPDE             = PGM_GST_NAME(GetPDE);
+    pModeData->pfnR3GstGetPage            = PGM_GST_NAME(GetPage);
+    pModeData->pfnR3GstModifyPage         = PGM_GST_NAME(ModifyPage);
+
+    if (fResolveGCAndR0)
+    {
+        int rc;
+
+#if PGM_SHW_TYPE != PGM_TYPE_AMD64 /* No AMD64 for traditional virtualization, only VT-x and AMD-V. */
+        /* GC */
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_GST_NAME_RC_STR(GetPage),          &pModeData->pfnRCGstGetPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_RC_STR(GetPage),  rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_GST_NAME_RC_STR(ModifyPage),       &pModeData->pfnRCGstModifyPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_RC_STR(ModifyPage),  rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_GST_NAME_RC_STR(GetPDE),           &pModeData->pfnRCGstGetPDE);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_RC_STR(GetPDE), rc), rc);
+#endif /* Not AMD64 shadow paging. */
+
+        /* Ring-0 */
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_GST_NAME_R0_STR(GetPage),          &pModeData->pfnR0GstGetPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_R0_STR(GetPage),  rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_GST_NAME_R0_STR(ModifyPage),       &pModeData->pfnR0GstModifyPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_R0_STR(ModifyPage),  rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_GST_NAME_R0_STR(GetPDE),           &pModeData->pfnR0GstGetPDE);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_GST_NAME_R0_STR(GetPDE), rc), rc);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enters the guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   GCPhysCR3   The physical address from the CR3 register.
+ */
+PGM_GST_DECL(int, Enter)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
+{
+    /*
+     * Map and monitor CR3
+     */
+    int rc = PGM_BTH_PFN(MapCR3, pVCpu)(pVCpu, GCPhysCR3);
+    return rc;
+}
+
+
+/**
+ * Relocate any GC pointers related to guest mode paging.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   offDelta    The relocation offset.
+ */
+PGM_GST_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta)
+{
+    pVCpu->pgm.s.pGst32BitPdRC += offDelta;
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.apGstPaePDsRC); i++)
+    {
+        pVCpu->pgm.s.apGstPaePDsRC[i] += offDelta;
+    }
+    pVCpu->pgm.s.pGstPaePdptRC += offDelta;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exits the guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The VMCPU to operate on.
+ */
+PGM_GST_DECL(int, Exit)(PVMCPU pVCpu)
+{
+    int rc;
+
+    rc = PGM_BTH_PFN(UnmapCR3, pVCpu)(pVCpu);
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMHandler.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMHandler.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMHandler.cpp	(revision 35333)
@@ -0,0 +1,640 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager / Monitor, Access Handlers.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/cpum.h>
+#include <VBox/iom.h>
+#include <VBox/sup.h>
+#include <VBox/mm.h>
+#include <VBox/em.h>
+#include <VBox/stam.h>
+#include <VBox/csam.h>
+#include <VBox/rem.h>
+#include <VBox/dbgf.h>
+#include <VBox/rem.h>
+#include <VBox/selm.h>
+#include <VBox/ssm.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include "PGMInline.h"
+#include <VBox/dbg.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/hwaccm.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneClear(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneSet(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int) pgmR3InfoHandlersPhysicalOne(PAVLROGCPHYSNODECORE pNode, void *pvUser);
+static DECLCALLBACK(int) pgmR3InfoHandlersVirtualOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
+
+
+
+/**
+ * Register a access handler for a physical range.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   enmType         Handler type. Any of the PGMPHYSHANDLERTYPE_PHYSICAL* enums.
+ * @param   GCPhys          Start physical address.
+ * @param   GCPhysLast      Last physical address. (inclusive)
+ * @param   pfnHandlerR3    The R3 handler.
+ * @param   pvUserR3        User argument to the R3 handler.
+ * @param   pszModR0        The R0 handler module. NULL means the default R0 module.
+ * @param   pszHandlerR0    The R0 handler symbol name.
+ * @param   pvUserR0        User argument to the R0 handler.
+ * @param   pszModRC        The RC handler module. NULL means the default RC
+ *                          module.
+ * @param   pszHandlerRC    The RC handler symbol name.
+ * @param   pvUserRC        User argument to the RC handler. Values less than
+ *                          0x10000 will not be relocated.
+ * @param   pszDesc         Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int) PGMR3HandlerPhysicalRegister(PVM pVM, PGMPHYSHANDLERTYPE enmType, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
+                                            PFNPGMR3PHYSHANDLER pfnHandlerR3, void *pvUserR3,
+                                            const char *pszModR0, const char *pszHandlerR0, RTR0PTR pvUserR0,
+                                            const char *pszModRC, const char *pszHandlerRC, RTRCPTR pvUserRC, const char *pszDesc)
+{
+    LogFlow(("PGMR3HandlerPhysicalRegister: enmType=%d GCPhys=%RGp GCPhysLast=%RGp pfnHandlerR3=%RHv pvUserHC=%RHv pszModR0=%s pszHandlerR0=%s pvUserR0=%RHv pszModRC=%s pszHandlerRC=%s pvUser=%RRv pszDesc=%s\n",
+             enmType, GCPhys, GCPhysLast, pfnHandlerR3, pvUserR3, pszModR0, pszHandlerR0, pvUserR0, pszHandlerRC, pszModRC, pvUserRC, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    if (!pszModRC)
+        pszModRC = VMMGC_MAIN_MODULE_NAME;
+    if (!pszModR0)
+        pszModR0 = VMMR0_MAIN_MODULE_NAME;
+    if (!pszHandlerR0)
+        pszHandlerR0 = "pgmPhysHandlerRedirectToHC";
+    if (!pszHandlerRC)
+        pszHandlerRC = "pgmPhysHandlerRedirectToHC";
+    AssertPtrReturn(pfnHandlerR3, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszHandlerR0, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszHandlerRC, VERR_INVALID_POINTER);
+
+    /*
+     * Resolve the R0 handler.
+     */
+    R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0 = NIL_RTR0PTR;
+    int rc = VINF_SUCCESS;
+    rc = PDMR3LdrGetSymbolR0Lazy(pVM, pszModR0, NULL /*pszSearchPath*/, pszHandlerR0, &pfnHandlerR0);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Resolve the GC handler.
+         */
+        RTRCPTR pfnHandlerRC = NIL_RTRCPTR;
+        rc = PDMR3LdrGetSymbolRCLazy(pVM, pszModRC, NULL /*pszSearchPath*/, pszHandlerRC, &pfnHandlerRC);
+        if (RT_SUCCESS(rc))
+            return PGMHandlerPhysicalRegisterEx(pVM, enmType, GCPhys, GCPhysLast, pfnHandlerR3, pvUserR3,
+                                                pfnHandlerR0, pvUserR0, pfnHandlerRC, pvUserRC, pszDesc);
+
+        AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszModRC, pszHandlerRC, rc));
+    }
+    else
+        AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszModR0, pszHandlerR0, rc));
+
+    return rc;
+}
+
+
+/**
+ * Updates the physical page access handlers.
+ *
+ * @param   pVM     VM handle.
+ * @remark  Only used when restoring a saved state.
+ */
+void pgmR3HandlerPhysicalUpdateAll(PVM pVM)
+{
+    LogFlow(("pgmHandlerPhysicalUpdateAll:\n"));
+
+    /*
+     * Clear and set.
+     * (the right -> left on the setting pass is just bird speculating on cache hits)
+     */
+    pgmLock(pVM);
+    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers,  true, pgmR3HandlerPhysicalOneClear, pVM);
+    RTAvlroGCPhysDoWithAll(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, false, pgmR3HandlerPhysicalOneSet, pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Clears all the page level flags for one physical handler range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to a PGMPHYSHANDLER.
+ * @param   pvUser  VM handle.
+ */
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneClear(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
+    PPGMRAMRANGE    pRamHint = NULL;
+    RTGCPHYS        GCPhys = pCur->Core.Key;
+    RTUINT          cPages = pCur->cPages;
+    PPGM            pPGM = &((PVM)pvUser)->pgm.s;
+    for (;;)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pPGM, GCPhys, &pPage, &pRamHint);
+        if (RT_SUCCESS(rc))
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE);
+        else
+            AssertRC(rc);
+
+        if (--cPages == 0)
+            return 0;
+        GCPhys += PAGE_SIZE;
+    }
+}
+
+
+/**
+ * Sets all the page level flags for one physical handler range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to a PGMPHYSHANDLER.
+ * @param   pvUser  VM handle.
+ */
+static DECLCALLBACK(int) pgmR3HandlerPhysicalOneSet(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)pNode;
+    unsigned        uState = pgmHandlerPhysicalCalcState(pCur);
+    PPGMRAMRANGE    pRamHint = NULL;
+    RTGCPHYS        GCPhys = pCur->Core.Key;
+    RTUINT          cPages = pCur->cPages;
+    PPGM            pPGM = &((PVM)pvUser)->pgm.s;
+    for (;;)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pPGM, GCPhys, &pPage, &pRamHint);
+        if (RT_SUCCESS(rc))
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState);
+        else
+            AssertRC(rc);
+
+        if (--cPages == 0)
+            return 0;
+        GCPhys += PAGE_SIZE;
+    }
+}
+
+
+/**
+ * Register a access handler for a virtual range.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   enmType         Handler type. Any of the PGMVIRTHANDLERTYPE_* enums.
+ * @param   GCPtr           Start address.
+ * @param   GCPtrLast       Last address (inclusive).
+ * @param   pfnInvalidateR3 The R3 invalidate callback (can be 0)
+ * @param   pfnHandlerR3    The R3 handler.
+ * @param   pszHandlerRC    The RC handler symbol name.
+ * @param   pszModRC        The RC handler module.
+ * @param   pszDesc         Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int) PGMR3HandlerVirtualRegister(PVM pVM, PGMVIRTHANDLERTYPE enmType, RTGCPTR GCPtr, RTGCPTR GCPtrLast,
+                                           PFNPGMR3VIRTINVALIDATE pfnInvalidateR3,
+                                           PFNPGMR3VIRTHANDLER pfnHandlerR3,
+                                           const char *pszHandlerRC, const char *pszModRC,
+                                           const char *pszDesc)
+{
+    LogFlow(("PGMR3HandlerVirtualRegisterEx: enmType=%d GCPtr=%RGv GCPtrLast=%RGv pszHandlerRC=%p:{%s} pszModRC=%p:{%s} pszDesc=%s\n",
+             enmType, GCPtr, GCPtrLast, pszHandlerRC, pszHandlerRC, pszModRC, pszModRC, pszDesc));
+
+    /* Not supported/relevant for VT-x and AMD-V. */
+    if (HWACCMIsEnabled(pVM))
+        return VERR_NOT_IMPLEMENTED;
+
+    /*
+     * Validate input.
+     */
+    if (!pszModRC)
+        pszModRC = VMMGC_MAIN_MODULE_NAME;
+    if (!pszModRC || !*pszModRC || !pszHandlerRC || !*pszHandlerRC)
+    {
+        AssertMsgFailed(("pfnHandlerGC or/and pszModRC is missing\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Resolve the GC handler.
+     */
+    RTRCPTR pfnHandlerRC;
+    int rc = PDMR3LdrGetSymbolRCLazy(pVM, pszModRC, NULL /*pszSearchPath*/, pszHandlerRC, &pfnHandlerRC);
+    if (RT_SUCCESS(rc))
+        return PGMR3HandlerVirtualRegisterEx(pVM, enmType, GCPtr, GCPtrLast, pfnInvalidateR3, pfnHandlerR3, pfnHandlerRC, pszDesc);
+
+    AssertMsgFailed(("Failed to resolve %s.%s, rc=%Rrc.\n", pszModRC, pszHandlerRC, rc));
+    return rc;
+}
+
+
+/**
+ * Register an access handler for a virtual range.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   enmType         Handler type. Any of the PGMVIRTHANDLERTYPE_* enums.
+ * @param   GCPtr           Start address.
+ * @param   GCPtrLast       Last address (inclusive).
+ * @param   pfnInvalidateR3 The R3 invalidate callback (can be 0)
+ * @param   pfnHandlerR3    The R3 handler.
+ * @param   pfnHandlerRC    The RC handler.
+ * @param   pszDesc         Pointer to description string. This must not be freed.
+ * @thread  EMT
+ */
+/** @todo create a template for virtual handlers (see async i/o), we're wasting space
+ * duplicating the function pointers now. (Or we will once we add the missing callbacks.) */
+VMMDECL(int) PGMR3HandlerVirtualRegisterEx(PVM pVM, PGMVIRTHANDLERTYPE enmType, RTGCPTR GCPtr, RTGCPTR GCPtrLast,
+                                           R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3,
+                                           R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3,
+                                           RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC,
+                                           R3PTRTYPE(const char *) pszDesc)
+{
+    Log(("PGMR3HandlerVirtualRegister: enmType=%d GCPtr=%RGv GCPtrLast=%RGv pfnInvalidateR3=%RHv pfnHandlerR3=%RHv pfnHandlerRC=%RRv pszDesc=%s\n",
+         enmType, GCPtr, GCPtrLast, pfnInvalidateR3, pfnHandlerR3, pfnHandlerRC, pszDesc));
+
+    /* Not supported/relevant for VT-x and AMD-V. */
+    if (HWACCMIsEnabled(pVM))
+        return VERR_NOT_IMPLEMENTED;
+
+    /*
+     * Validate input.
+     */
+    switch (enmType)
+    {
+        case PGMVIRTHANDLERTYPE_ALL:
+            AssertReleaseMsgReturn(   (GCPtr     & PAGE_OFFSET_MASK) == 0
+                                   && (GCPtrLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK,
+                                   ("PGMVIRTHANDLERTYPE_ALL: GCPtr=%RGv GCPtrLast=%RGv\n", GCPtr, GCPtrLast),
+                                   VERR_NOT_IMPLEMENTED);
+            break;
+        case PGMVIRTHANDLERTYPE_WRITE:
+            if (!pfnHandlerR3)
+            {
+                AssertMsgFailed(("No HC handler specified!!\n"));
+                return VERR_INVALID_PARAMETER;
+            }
+            break;
+
+        case PGMVIRTHANDLERTYPE_HYPERVISOR:
+            if (pfnHandlerR3)
+            {
+                AssertMsgFailed(("R3 handler specified for hypervisor range!?!\n"));
+                return VERR_INVALID_PARAMETER;
+            }
+            break;
+        default:
+            AssertMsgFailed(("Invalid enmType! enmType=%d\n", enmType));
+            return VERR_INVALID_PARAMETER;
+    }
+    if (GCPtrLast < GCPtr)
+    {
+        AssertMsgFailed(("GCPtrLast < GCPtr (%#x < %#x)\n", GCPtrLast, GCPtr));
+        return VERR_INVALID_PARAMETER;
+    }
+    if (!pfnHandlerRC)
+    {
+        AssertMsgFailed(("pfnHandlerRC is missing\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Allocate and initialize a new entry.
+     */
+    unsigned cPages = (RT_ALIGN(GCPtrLast + 1, PAGE_SIZE) - (GCPtr & PAGE_BASE_GC_MASK)) >> PAGE_SHIFT;
+    PPGMVIRTHANDLER pNew;
+    int rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[cPages]), 0, MM_TAG_PGM_HANDLERS, (void **)&pNew); /** @todo r=bird: incorrect member name PhysToVirt? */
+    if (RT_FAILURE(rc))
+        return rc;
+
+    pNew->Core.Key      = GCPtr;
+    pNew->Core.KeyLast  = GCPtrLast;
+
+    pNew->enmType       = enmType;
+    pNew->pfnInvalidateR3 = pfnInvalidateR3;
+    pNew->pfnHandlerRC  = pfnHandlerRC;
+    pNew->pfnHandlerR3  = pfnHandlerR3;
+    pNew->pszDesc       = pszDesc;
+    pNew->cb            = GCPtrLast - GCPtr + 1;
+    pNew->cPages        = cPages;
+    /* Will be synced at next guest execution attempt. */
+    while (cPages-- > 0)
+    {
+        pNew->aPhysToVirt[cPages].Core.Key = NIL_RTGCPHYS;
+        pNew->aPhysToVirt[cPages].Core.KeyLast = NIL_RTGCPHYS;
+        pNew->aPhysToVirt[cPages].offVirtHandler = -RT_OFFSETOF(PGMVIRTHANDLER, aPhysToVirt[cPages]);
+        pNew->aPhysToVirt[cPages].offNextAlias = 0;
+    }
+
+    /*
+     * Try to insert it into the tree.
+     *
+     * The current implementation doesn't allow multiple handlers for
+     * the same range this makes everything much simpler and faster.
+     */
+    AVLROGCPTRTREE *pRoot = enmType != PGMVIRTHANDLERTYPE_HYPERVISOR
+                          ? &pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers
+                          : &pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers;
+    pgmLock(pVM);
+    if (*pRoot != 0)
+    {
+        PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrGetBestFit(pRoot, pNew->Core.Key, true);
+        if (    !pCur
+            ||  GCPtr     > pCur->Core.KeyLast
+            ||  GCPtrLast < pCur->Core.Key)
+            pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrGetBestFit(pRoot, pNew->Core.Key, false);
+        if (    pCur
+            &&  GCPtr     <= pCur->Core.KeyLast
+            &&  GCPtrLast >= pCur->Core.Key)
+        {
+            /*
+             * The LDT sometimes conflicts with the IDT and LDT ranges while being
+             * updated on linux. So, we don't assert simply log it.
+             */
+            Log(("PGMR3HandlerVirtualRegister: Conflict with existing range %RGv-%RGv (%s), req. %RGv-%RGv (%s)\n",
+                 pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc, GCPtr, GCPtrLast, pszDesc));
+            MMHyperFree(pVM, pNew);
+            pgmUnlock(pVM);
+            return VERR_PGM_HANDLER_VIRTUAL_CONFLICT;
+        }
+    }
+    if (RTAvlroGCPtrInsert(pRoot, &pNew->Core))
+    {
+        if (enmType != PGMVIRTHANDLERTYPE_HYPERVISOR)
+        {
+            PVMCPU pVCpu = VMMGetCpu(pVM);
+
+            pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL | PGM_SYNC_CLEAR_PGM_POOL;
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        }
+        pgmUnlock(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+        char szPath[256];
+        RTStrPrintf(szPath, sizeof(szPath), "/PGM/VirtHandler/Calls/%RGv-%RGv", pNew->Core.Key, pNew->Core.KeyLast);
+        rc = STAMR3Register(pVM, &pNew->Stat, STAMTYPE_PROFILE, STAMVISIBILITY_USED, szPath, STAMUNIT_TICKS_PER_CALL, pszDesc);
+        AssertRC(rc);
+#endif
+        return VINF_SUCCESS;
+    }
+
+    pgmUnlock(pVM);
+    AssertFailed();
+    MMHyperFree(pVM, pNew);
+    return VERR_PGM_HANDLER_VIRTUAL_CONFLICT;
+}
+
+
+/**
+ * Modify the page invalidation callback handler for a registered virtual range.
+ * (add more when needed)
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   GCPtr           Start address.
+ * @param   pfnInvalidateR3 The R3 invalidate callback (can be 0)
+ * @remarks Doesn't work with the hypervisor access handler type.
+ */
+VMMDECL(int) PGMHandlerVirtualChangeInvalidateCallback(PVM pVM, RTGCPTR GCPtr, PFNPGMR3VIRTINVALIDATE pfnInvalidateR3)
+{
+    pgmLock(pVM);
+    PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrGet(&pVM->pgm.s.pTreesR3->VirtHandlers, GCPtr);
+    if (pCur)
+    {
+        pCur->pfnInvalidateR3 = pfnInvalidateR3;
+        pgmUnlock(pVM);
+        return VINF_SUCCESS;
+    }
+    pgmUnlock(pVM);
+    AssertMsgFailed(("Range %#x not found!\n", GCPtr));
+    return VERR_INVALID_PARAMETER;
+}
+
+/**
+ * Deregister an access handler for a virtual range.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   GCPtr       Start address.
+ * @thread  EMT
+ */
+VMMDECL(int) PGMHandlerVirtualDeregister(PVM pVM, RTGCPTR GCPtr)
+{
+    pgmLock(pVM);
+
+    /*
+     * Find the handler.
+     * We naturally assume GCPtr is a unique specification.
+     */
+    PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtr);
+    if (RT_LIKELY(pCur))
+    {
+        Log(("PGMHandlerVirtualDeregister: Removing Virtual (%d) Range %RGv-%RGv %s\n", pCur->enmType,
+             pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+        Assert(pCur->enmType != PGMVIRTHANDLERTYPE_HYPERVISOR);
+
+        /*
+         * Reset the flags and remove phys2virt nodes.
+         */
+        PPGM pPGM = &pVM->pgm.s;
+        for (unsigned iPage = 0; iPage < pCur->cPages; iPage++)
+            if (pCur->aPhysToVirt[iPage].offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE)
+                pgmHandlerVirtualClearPage(pPGM, pCur, iPage);
+
+        /*
+         * Schedule CR3 sync.
+         */
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL | PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+    else
+    {
+        /* must be a hypervisor one then. */
+        pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers, GCPtr);
+        if (RT_UNLIKELY(!pCur))
+        {
+            pgmUnlock(pVM);
+            AssertMsgFailed(("Range %#x not found!\n", GCPtr));
+            return VERR_INVALID_PARAMETER;
+        }
+
+        Log(("PGMHandlerVirtualDeregister: Removing Hyper Virtual (%d) Range %RGv-%RGv %s\n", pCur->enmType,
+             pCur->Core.Key, pCur->Core.KeyLast, pCur->pszDesc));
+        Assert(pCur->enmType == PGMVIRTHANDLERTYPE_HYPERVISOR);
+    }
+
+    pgmUnlock(pVM);
+
+    STAM_DEREG(pVM, &pCur->Stat);
+    MMHyperFree(pVM, pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Arguments for pgmR3InfoHandlersPhysicalOne and pgmR3InfoHandlersVirtualOne.
+ */
+typedef struct PGMHANDLERINFOARG
+{
+    /** The output helpers.*/
+    PCDBGFINFOHLP   pHlp;
+    /** Set if statistics should be dumped. */
+    bool            fStats;
+} PGMHANDLERINFOARG, *PPGMHANDLERINFOARG;
+
+
+/**
+ * Info callback for 'pgmhandlers'.
+ *
+ * @param   pHlp        The output helpers.
+ * @param   pszArgs     The arguments. phys or virt.
+ */
+DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /*
+     * Test input.
+     */
+    PGMHANDLERINFOARG Args = { pHlp, /* .fStats = */ true };
+    bool fPhysical = !pszArgs  || !*pszArgs;
+    bool fVirtual = fPhysical;
+    bool fHyper = fPhysical;
+    if (!fPhysical)
+    {
+        bool fAll = strstr(pszArgs, "all") != NULL;
+        fPhysical   = fAll || strstr(pszArgs, "phys") != NULL;
+        fVirtual    = fAll || strstr(pszArgs, "virt") != NULL;
+        fHyper      = fAll || strstr(pszArgs, "hyper")!= NULL;
+        Args.fStats = strstr(pszArgs, "nost") == NULL;
+    }
+
+    /*
+     * Dump the handlers.
+     */
+    if (fPhysical)
+    {
+        pHlp->pfnPrintf(pHlp,
+            "Physical handlers: (PhysHandlers=%d (%#x))\n"
+            "From     - To (incl) HandlerHC UserHC    HandlerGC UserGC    Type     Description\n",
+            pVM->pgm.s.pTreesR3->PhysHandlers, pVM->pgm.s.pTreesR3->PhysHandlers);
+        RTAvlroGCPhysDoWithAll(&pVM->pgm.s.pTreesR3->PhysHandlers, true, pgmR3InfoHandlersPhysicalOne, &Args);
+    }
+
+    if (fVirtual)
+    {
+        pHlp->pfnPrintf(pHlp,
+            "Virtual handlers:\n"
+            "From     - To (excl) HandlerHC HandlerGC Type     Description\n");
+        RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->VirtHandlers, true, pgmR3InfoHandlersVirtualOne, &Args);
+    }
+
+    if (fHyper)
+    {
+        pHlp->pfnPrintf(pHlp,
+            "Hypervisor Virtual handlers:\n"
+            "From     - To (excl) HandlerHC HandlerGC Type     Description\n");
+        RTAvlroGCPtrDoWithAll(&pVM->pgm.s.pTreesR3->HyperVirtHandlers, true, pgmR3InfoHandlersVirtualOne, &Args);
+    }
+}
+
+
+/**
+ * Displays one physical handler range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to a PGMPHYSHANDLER.
+ * @param   pvUser  Pointer to command helper functions.
+ */
+static DECLCALLBACK(int) pgmR3InfoHandlersPhysicalOne(PAVLROGCPHYSNODECORE pNode, void *pvUser)
+{
+    PPGMPHYSHANDLER     pCur = (PPGMPHYSHANDLER)pNode;
+    PPGMHANDLERINFOARG  pArgs= (PPGMHANDLERINFOARG)pvUser;
+    PCDBGFINFOHLP       pHlp = pArgs->pHlp;
+    const char *pszType;
+    switch (pCur->enmType)
+    {
+        case PGMPHYSHANDLERTYPE_MMIO:           pszType = "MMIO   "; break;
+        case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE: pszType = "Write  "; break;
+        case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:   pszType = "All    "; break;
+        default:                                pszType = "????"; break;
+    }
+    pHlp->pfnPrintf(pHlp,
+        "%RGp - %RGp  %RHv  %RHv  %RRv  %RRv  %s  %s\n",
+        pCur->Core.Key, pCur->Core.KeyLast, pCur->pfnHandlerR3, pCur->pvUserR3, pCur->pfnHandlerRC, pCur->pvUserRC, pszType, pCur->pszDesc);
+#ifdef VBOX_WITH_STATISTICS
+    if (pArgs->fStats)
+        pHlp->pfnPrintf(pHlp, "   cPeriods: %9RU64  cTicks: %11RU64  Min: %11RU64  Avg: %11RU64 Max: %11RU64\n",
+                        pCur->Stat.cPeriods, pCur->Stat.cTicks, pCur->Stat.cTicksMin,
+                        pCur->Stat.cPeriods ? pCur->Stat.cTicks / pCur->Stat.cPeriods : 0, pCur->Stat.cTicksMax);
+#endif
+    return 0;
+}
+
+
+/**
+ * Displays one virtual handler range.
+ *
+ * @returns 0
+ * @param   pNode   Pointer to a PGMVIRTHANDLER.
+ * @param   pvUser  Pointer to command helper functions.
+ */
+static DECLCALLBACK(int) pgmR3InfoHandlersVirtualOne(PAVLROGCPTRNODECORE pNode, void *pvUser)
+{
+    PPGMVIRTHANDLER     pCur = (PPGMVIRTHANDLER)pNode;
+    PPGMHANDLERINFOARG  pArgs= (PPGMHANDLERINFOARG)pvUser;
+    PCDBGFINFOHLP       pHlp = pArgs->pHlp;
+    const char *pszType;
+    switch (pCur->enmType)
+    {
+        case PGMVIRTHANDLERTYPE_WRITE:      pszType = "Write  "; break;
+        case PGMVIRTHANDLERTYPE_ALL:        pszType = "All    "; break;
+        case PGMVIRTHANDLERTYPE_HYPERVISOR: pszType = "WriteHyp "; break;
+        default:                            pszType = "????"; break;
+    }
+    pHlp->pfnPrintf(pHlp, "%RGv - %RGv  %RHv  %RRv  %s  %s\n",
+        pCur->Core.Key, pCur->Core.KeyLast, pCur->pfnHandlerR3, pCur->pfnHandlerRC, pszType, pCur->pszDesc);
+#ifdef VBOX_WITH_STATISTICS
+    if (pArgs->fStats)
+        pHlp->pfnPrintf(pHlp, "   cPeriods: %9RU64  cTicks: %11RU64  Min: %11RU64  Avg: %11RU64 Max: %11RU64\n",
+                        pCur->Stat.cPeriods, pCur->Stat.cTicks, pCur->Stat.cTicksMin,
+                        pCur->Stat.cPeriods ? pCur->Stat.cTicks / pCur->Stat.cPeriods : 0, pCur->Stat.cTicksMax);
+#endif
+    return 0;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMMap.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMMap.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMMap.cpp	(revision 35333)
@@ -0,0 +1,1469 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager, Guest Context Mappings.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include "PGMInline.h"
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static void pgmR3MapClearPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iOldPDE);
+static void pgmR3MapSetPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
+static int  pgmR3MapIntermediateCheckOne(PVM pVM, uintptr_t uAddress, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault);
+static void pgmR3MapIntermediateDoOne(PVM pVM, uintptr_t uAddress, RTHCPHYS HCPhys, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault);
+
+
+/**
+ * Creates a page table based mapping in GC.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   GCPtr           Virtual Address. (Page table aligned!)
+ * @param   cb              Size of the range. Must be a 4MB aligned!
+ * @param   fFlags          PGMR3MAPPT_FLAGS_UNMAPPABLE or 0.
+ * @param   pfnRelocate     Relocation callback function.
+ * @param   pvUser          User argument to the callback.
+ * @param   pszDesc         Pointer to description string. This must not be freed.
+ */
+VMMR3DECL(int) PGMR3MapPT(PVM pVM, RTGCPTR GCPtr, uint32_t cb, uint32_t fFlags, PFNPGMRELOCATE pfnRelocate, void *pvUser, const char *pszDesc)
+{
+    LogFlow(("PGMR3MapPT: GCPtr=%#x cb=%d fFlags=%#x pfnRelocate=%p pvUser=%p pszDesc=%s\n", GCPtr, cb, fFlags, pfnRelocate, pvUser, pszDesc));
+    AssertMsg(pVM->pgm.s.pInterPD, ("Paging isn't initialized, init order problems!\n"));
+
+    /*
+     * Validate input.
+     */
+    Assert(!fFlags || fFlags == PGMR3MAPPT_FLAGS_UNMAPPABLE);
+    if (cb < _2M || cb > 64 * _1M)
+    {
+        AssertMsgFailed(("Serious? cb=%d\n", cb));
+        return VERR_INVALID_PARAMETER;
+    }
+    cb = RT_ALIGN_32(cb, _4M);
+    RTGCPTR GCPtrLast = GCPtr + cb - 1;
+
+    AssertMsgReturn(GCPtrLast >= GCPtr, ("Range wraps! GCPtr=%x GCPtrLast=%x\n", GCPtr, GCPtrLast),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!pVM->pgm.s.fMappingsFixed, ("Mappings are fixed! It's not possible to add new mappings at this time!\n"),
+                    VERR_PGM_MAPPINGS_FIXED);
+    AssertPtrReturn(pfnRelocate, VERR_INVALID_PARAMETER);
+
+    /*
+     * Find list location.
+     */
+    PPGMMAPPING pPrev = NULL;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        if (pCur->GCPtrLast >= GCPtr && pCur->GCPtr <= GCPtrLast)
+        {
+            AssertMsgFailed(("Address is already in use by %s. req %#x-%#x take %#x-%#x\n",
+                             pCur->pszDesc, GCPtr, GCPtrLast, pCur->GCPtr, pCur->GCPtrLast));
+            LogRel(("VERR_PGM_MAPPING_CONFLICT: Address is already in use by %s. req %#x-%#x take %#x-%#x\n",
+                    pCur->pszDesc, GCPtr, GCPtrLast, pCur->GCPtr, pCur->GCPtrLast));
+            return VERR_PGM_MAPPING_CONFLICT;
+        }
+        if (pCur->GCPtr > GCPtr)
+            break;
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+
+    /*
+     * Check for conflicts with intermediate mappings.
+     */
+    const unsigned iPageDir = GCPtr >> X86_PD_SHIFT;
+    const unsigned cPTs     = cb >> X86_PD_SHIFT;
+    if (pVM->pgm.s.fFinalizedMappings)
+    {
+        for (unsigned i = 0; i < cPTs; i++)
+            if (pVM->pgm.s.pInterPD->a[iPageDir + i].n.u1Present)
+            {
+                AssertMsgFailed(("Address %#x is already in use by an intermediate mapping.\n", GCPtr + (i << PAGE_SHIFT)));
+                LogRel(("VERR_PGM_MAPPING_CONFLICT: Address %#x is already in use by an intermediate mapping.\n", GCPtr + (i << PAGE_SHIFT)));
+                return VERR_PGM_MAPPING_CONFLICT;
+            }
+        /** @todo AMD64: add check in PAE structures too, so we can remove all the 32-Bit paging stuff there. */
+    }
+
+    /*
+     * Allocate and initialize the new list node.
+     */
+    PPGMMAPPING pNew;
+    int rc;
+    if (fFlags & PGMR3MAPPT_FLAGS_UNMAPPABLE)
+        rc = MMHyperAlloc(           pVM, RT_OFFSETOF(PGMMAPPING, aPTs[cPTs]), 0, MM_TAG_PGM_MAPPINGS, (void **)&pNew);
+    else
+        rc = MMR3HyperAllocOnceNoRel(pVM, RT_OFFSETOF(PGMMAPPING, aPTs[cPTs]), 0, MM_TAG_PGM_MAPPINGS, (void **)&pNew);
+    if (RT_FAILURE(rc))
+        return rc;
+    pNew->GCPtr         = GCPtr;
+    pNew->GCPtrLast     = GCPtrLast;
+    pNew->cb            = cb;
+    pNew->pfnRelocate   = pfnRelocate;
+    pNew->pvUser        = pvUser;
+    pNew->pszDesc       = pszDesc;
+    pNew->cPTs          = cPTs;
+
+    /*
+     * Allocate page tables and insert them into the page directories.
+     * (One 32-bit PT and two PAE PTs.)
+     */
+    uint8_t *pbPTs;
+    if (fFlags & PGMR3MAPPT_FLAGS_UNMAPPABLE)
+        rc = MMHyperAlloc(           pVM, PAGE_SIZE * 3 * cPTs, PAGE_SIZE, MM_TAG_PGM_MAPPINGS, (void **)&pbPTs);
+    else
+        rc = MMR3HyperAllocOnceNoRel(pVM, PAGE_SIZE * 3 * cPTs, PAGE_SIZE, MM_TAG_PGM_MAPPINGS, (void **)&pbPTs);
+    if (RT_FAILURE(rc))
+    {
+        MMHyperFree(pVM, pNew);
+        return VERR_NO_MEMORY;
+    }
+
+    /*
+     * Init the page tables and insert them into the page directories.
+     */
+    Log4(("PGMR3MapPT: GCPtr=%RGv cPTs=%u pbPTs=%p\n", GCPtr, cPTs, pbPTs));
+    for (unsigned i = 0; i < cPTs; i++)
+    {
+        /*
+         * 32-bit.
+         */
+        pNew->aPTs[i].pPTR3    = (PX86PT)pbPTs;
+        pNew->aPTs[i].pPTRC    = MMHyperR3ToRC(pVM, pNew->aPTs[i].pPTR3);
+        pNew->aPTs[i].pPTR0    = MMHyperR3ToR0(pVM, pNew->aPTs[i].pPTR3);
+        pNew->aPTs[i].HCPhysPT = MMR3HyperHCVirt2HCPhys(pVM, pNew->aPTs[i].pPTR3);
+        pbPTs += PAGE_SIZE;
+        Log4(("PGMR3MapPT: i=%d: pPTR3=%RHv pPTRC=%RRv pPRTR0=%RHv HCPhysPT=%RHp\n",
+              i, pNew->aPTs[i].pPTR3, pNew->aPTs[i].pPTRC, pNew->aPTs[i].pPTR0, pNew->aPTs[i].HCPhysPT));
+
+        /*
+         * PAE.
+         */
+        pNew->aPTs[i].HCPhysPaePT0 = MMR3HyperHCVirt2HCPhys(pVM, pbPTs);
+        pNew->aPTs[i].HCPhysPaePT1 = MMR3HyperHCVirt2HCPhys(pVM, pbPTs + PAGE_SIZE);
+        pNew->aPTs[i].paPaePTsR3 = (PPGMSHWPTPAE)pbPTs;
+        pNew->aPTs[i].paPaePTsRC = MMHyperR3ToRC(pVM, pbPTs);
+        pNew->aPTs[i].paPaePTsR0 = MMHyperR3ToR0(pVM, pbPTs);
+        pbPTs += PAGE_SIZE * 2;
+        Log4(("PGMR3MapPT: i=%d: paPaePTsR#=%RHv paPaePTsRC=%RRv paPaePTsR#=%RHv HCPhysPaePT0=%RHp HCPhysPaePT1=%RHp\n",
+              i, pNew->aPTs[i].paPaePTsR3, pNew->aPTs[i].paPaePTsRC, pNew->aPTs[i].paPaePTsR0, pNew->aPTs[i].HCPhysPaePT0, pNew->aPTs[i].HCPhysPaePT1));
+    }
+    if (pVM->pgm.s.fFinalizedMappings)
+        pgmR3MapSetPDEs(pVM, pNew, iPageDir);
+    /* else PGMR3FinalizeMappings() */
+
+    /*
+     * Insert the new mapping.
+     */
+    pNew->pNextR3 = pCur;
+    pNew->pNextRC = pCur ? MMHyperR3ToRC(pVM, pCur) : NIL_RTRCPTR;
+    pNew->pNextR0 = pCur ? MMHyperR3ToR0(pVM, pCur) : NIL_RTR0PTR;
+    if (pPrev)
+    {
+        pPrev->pNextR3 = pNew;
+        pPrev->pNextRC = MMHyperR3ToRC(pVM, pNew);
+        pPrev->pNextR0 = MMHyperR3ToR0(pVM, pNew);
+    }
+    else
+    {
+        pVM->pgm.s.pMappingsR3 = pNew;
+        pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pNew);
+        pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pNew);
+    }
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Removes a page table based mapping.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM Handle.
+ * @param   GCPtr   Virtual Address. (Page table aligned!)
+ *
+ * @remarks Don't call this without passing PGMR3MAPPT_FLAGS_UNMAPPABLE to
+ *          PGMR3MapPT or you'll burn in the heap.
+ */
+VMMR3DECL(int)  PGMR3UnmapPT(PVM pVM, RTGCPTR GCPtr)
+{
+    LogFlow(("PGMR3UnmapPT: GCPtr=%#x\n", GCPtr));
+    AssertReturn(pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
+
+    /*
+     * Find it.
+     */
+    PPGMMAPPING pPrev = NULL;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        if (pCur->GCPtr == GCPtr)
+        {
+            /*
+             * Unlink it.
+             */
+            if (pPrev)
+            {
+                pPrev->pNextR3 = pCur->pNextR3;
+                pPrev->pNextRC = pCur->pNextRC;
+                pPrev->pNextR0 = pCur->pNextR0;
+            }
+            else
+            {
+                pVM->pgm.s.pMappingsR3 = pCur->pNextR3;
+                pVM->pgm.s.pMappingsRC = pCur->pNextRC;
+                pVM->pgm.s.pMappingsR0 = pCur->pNextR0;
+            }
+
+            /*
+             * Free the page table memory, clear page directory entries
+             * and free the page tables and node memory.
+             */
+            MMHyperFree(pVM, pCur->aPTs[0].pPTR3);
+            if (pCur->GCPtr != NIL_RTGCPTR)
+                pgmR3MapClearPDEs(pVM, pCur, pCur->GCPtr >> X86_PD_SHIFT);
+            MMHyperFree(pVM, pCur);
+
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                PVMCPU pVCpu = &pVM->aCpus[i];
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+            }
+            return VINF_SUCCESS;
+        }
+
+        /* done? */
+        if (pCur->GCPtr > GCPtr)
+            break;
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+
+    AssertMsgFailed(("No mapping for %#x found!\n", GCPtr));
+    return VERR_INVALID_PARAMETER;
+}
+
+
+/**
+ * Checks whether a range of PDEs in the intermediate
+ * memory context are unused.
+ *
+ * We're talking 32-bit PDEs here.
+ *
+ * @returns true/false.
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   iPD         The first PDE in the range.
+ * @param   cPTs        The number of PDEs in the range.
+ */
+DECLINLINE(bool) pgmR3AreIntermediatePDEsUnused(PVM pVM, unsigned iPD, unsigned cPTs)
+{
+    if (pVM->pgm.s.pInterPD->a[iPD].n.u1Present)
+        return false;
+    while (cPTs > 1)
+    {
+        iPD++;
+        if (pVM->pgm.s.pInterPD->a[iPD].n.u1Present)
+            return false;
+        cPTs--;
+    }
+    return true;
+}
+
+
+/**
+ * Unlinks the mapping.
+ *
+ * The mapping *must* be in the list.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pMapping        The mapping to unlink.
+ */
+static void pgmR3MapUnlink(PVM pVM, PPGMMAPPING pMapping)
+{
+    PPGMMAPPING pAfterThis = pVM->pgm.s.pMappingsR3;
+    if (pAfterThis == pMapping)
+    {
+        /* head */
+        pVM->pgm.s.pMappingsR3 = pMapping->pNextR3;
+        pVM->pgm.s.pMappingsRC = pMapping->pNextRC;
+        pVM->pgm.s.pMappingsR0 = pMapping->pNextR0;
+    }
+    else
+    {
+        /* in the list */
+        while (pAfterThis->pNextR3 != pMapping)
+        {
+            pAfterThis = pAfterThis->pNextR3;
+            AssertReleaseReturnVoid(pAfterThis);
+        }
+
+        pAfterThis->pNextR3 = pMapping->pNextR3;
+        pAfterThis->pNextRC = pMapping->pNextRC;
+        pAfterThis->pNextR0 = pMapping->pNextR0;
+    }
+}
+
+
+/**
+ * Links the mapping.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pMapping        The mapping to linked.
+ */
+static void pgmR3MapLink(PVM pVM, PPGMMAPPING pMapping)
+{
+    /*
+     * Find the list location (it's sorted by GCPhys) and link it in.
+     */
+    if (    !pVM->pgm.s.pMappingsR3
+        ||  pVM->pgm.s.pMappingsR3->GCPtr > pMapping->GCPtr)
+    {
+        /* head */
+        pMapping->pNextR3 = pVM->pgm.s.pMappingsR3;
+        pMapping->pNextRC = pVM->pgm.s.pMappingsRC;
+        pMapping->pNextR0 = pVM->pgm.s.pMappingsR0;
+        pVM->pgm.s.pMappingsR3 = pMapping;
+        pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pMapping);
+        pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pMapping);
+    }
+    else
+    {
+        /* in the list */
+        PPGMMAPPING pAfterThis  = pVM->pgm.s.pMappingsR3;
+        PPGMMAPPING pBeforeThis = pAfterThis->pNextR3;
+        while (pBeforeThis && pBeforeThis->GCPtr <= pMapping->GCPtr)
+        {
+            pAfterThis = pBeforeThis;
+            pBeforeThis = pBeforeThis->pNextR3;
+        }
+
+        pMapping->pNextR3 = pAfterThis->pNextR3;
+        pMapping->pNextRC = pAfterThis->pNextRC;
+        pMapping->pNextR0 = pAfterThis->pNextR0;
+        pAfterThis->pNextR3 = pMapping;
+        pAfterThis->pNextRC = MMHyperR3ToRC(pVM, pMapping);
+        pAfterThis->pNextR0 = MMHyperR3ToR0(pVM, pMapping);
+    }
+}
+
+
+/**
+ * Finalizes the intermediate context.
+ *
+ * This is called at the end of the ring-3 init and will construct the
+ * intermediate paging structures, relocating all the mappings in the process.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ * @thread  EMT(0)
+ */
+VMMR3DECL(int) PGMR3FinalizeMappings(PVM pVM)
+{
+    AssertReturn(!pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
+    pVM->pgm.s.fFinalizedMappings = true;
+
+    /*
+     * Loop until all mappings have been finalized.
+     */
+#if 0
+    unsigned    iPDNext = UINT32_C(0xc0000000) >> X86_PD_SHIFT; /* makes CSAM/PATM freak out booting linux. :-/ */
+#elif 0
+    unsigned    iPDNext = MM_HYPER_AREA_ADDRESS >> X86_PD_SHIFT;
+#else
+    unsigned    iPDNext = 1 << X86_PD_SHIFT; /* no hint, map them from the top. */
+#endif
+
+    PPGMMAPPING pCur;
+    do
+    {
+        pCur = pVM->pgm.s.pMappingsR3;
+        while (pCur)
+        {
+            if (!pCur->fFinalized)
+            {
+                /*
+                 * Find a suitable location.
+                 */
+                RTGCPTR const   GCPtrOld = pCur->GCPtr;
+                const unsigned  cPTs     = pCur->cPTs;
+                unsigned        iPDNew   = iPDNext;
+                if (    iPDNew + cPTs >= X86_PG_ENTRIES /* exclude the last PD */
+                    ||  !pgmR3AreIntermediatePDEsUnused(pVM, iPDNew, cPTs)
+                    ||  !pCur->pfnRelocate(pVM, GCPtrOld, (RTGCPTR)iPDNew << X86_PD_SHIFT, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
+                {
+                    /* No luck, just scan down from 4GB-4MB, giving up at 4MB. */
+                    iPDNew = X86_PG_ENTRIES - cPTs - 1;
+                    while (     iPDNew > 0
+                           &&   (   !pgmR3AreIntermediatePDEsUnused(pVM, iPDNew, cPTs)
+                                 || !pCur->pfnRelocate(pVM, GCPtrOld, (RTGCPTR)iPDNew << X86_PD_SHIFT, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
+                           )
+                        iPDNew--;
+                    AssertLogRelReturn(iPDNew != 0, VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+                }
+
+                /*
+                 * Relocate it (something akin to pgmR3MapRelocate).
+                 */
+                pgmR3MapSetPDEs(pVM, pCur, iPDNew);
+
+                /* unlink the mapping, update the entry and relink it. */
+                pgmR3MapUnlink(pVM, pCur);
+
+                RTGCPTR const GCPtrNew = (RTGCPTR)iPDNew << X86_PD_SHIFT;
+                pCur->GCPtr      = GCPtrNew;
+                pCur->GCPtrLast  = GCPtrNew + pCur->cb - 1;
+                pCur->fFinalized = true;
+
+                pgmR3MapLink(pVM, pCur);
+
+                /* Finally work the callback. */
+                pCur->pfnRelocate(pVM, GCPtrOld, GCPtrNew, PGMRELOCATECALL_RELOCATE, pCur->pvUser);
+
+                /*
+                 * The list order might have changed, start from the beginning again.
+                 */
+                iPDNext = iPDNew + cPTs;
+                break;
+            }
+
+            /* next */
+            pCur = pCur->pNextR3;
+        }
+    } while (pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the size of the current guest mappings if they were to be
+ * put next to one another.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM.
+ * @param   pcb     Where to store the size.
+ */
+VMMR3DECL(int) PGMR3MappingsSize(PVM pVM, uint32_t *pcb)
+{
+    RTGCPTR cb = 0;
+    for (PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
+        cb += pCur->cb;
+
+    *pcb = cb;
+    AssertReturn(*pcb == cb, VERR_NUMBER_TOO_BIG);
+    Log(("PGMR3MappingsSize: return %d (%#x) bytes\n", cb, cb));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Fixates the guest context mappings in a range reserved from the Guest OS.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM.
+ * @param   GCPtrBase   The address of the reserved range of guest memory.
+ * @param   cb          The size of the range starting at GCPtrBase.
+ */
+VMMR3DECL(int) PGMR3MappingsFix(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb)
+{
+    Log(("PGMR3MappingsFix: GCPtrBase=%RGv cb=%#x (fMappingsFixed=%RTbool fMappingsDisabled=%RTbool)\n",
+         GCPtrBase, cb, pVM->pgm.s.fMappingsFixed, pVM->pgm.s.fMappingsDisabled));
+
+    /*
+     * Ignore the additions mapping fix call if disabled.
+     */
+    if (!pgmMapAreMappingsEnabled(&pVM->pgm.s))
+    {
+        Assert(HWACCMIsEnabled(pVM));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Only applies to VCPU 0 as we don't support SMP guests with raw mode.
+     */
+    Assert(pVM->cCpus == 1);
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    /*
+     * Before we do anything we'll do a forced PD sync to try make sure any
+     * pending relocations because of these mappings have been resolved.
+     */
+    PGMSyncCR3(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR3(pVCpu), CPUMGetGuestCR4(pVCpu), true);
+
+    return pgmR3MappingsFixInternal(pVM, GCPtrBase, cb);
+}
+
+
+/**
+ * Internal worker for PGMR3MappingsFix and pgmR3Load.
+ *
+ * (This does not perform a SyncCR3 before the fixation like PGMR3MappingsFix.)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM.
+ * @param   GCPtrBase   The address of the reserved range of guest memory.
+ * @param   cb          The size of the range starting at GCPtrBase.
+ */
+int pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb)
+{
+    /*
+     * Check input arguments and pre-conditions.
+     */
+    AssertMsgReturn(!(GCPtrBase & X86_PAGE_4M_OFFSET_MASK), ("GCPtrBase (%#x) has to be aligned on a 4MB address!\n", GCPtrBase),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(cb && !(cb & X86_PAGE_4M_OFFSET_MASK), ("cb (%#x) is 0 or not aligned on a 4MB address!\n", cb),
+                    VERR_INVALID_PARAMETER);
+    AssertReturn(pgmMapAreMappingsEnabled(&pVM->pgm.s), VERR_INTERNAL_ERROR_3);
+    AssertReturn(pVM->cCpus == 1, VERR_INTERNAL_ERROR_4);
+
+    /*
+     * Check that it's not conflicting with a core code mapping in the intermediate page table.
+     */
+    unsigned    iPDNew = GCPtrBase >> X86_PD_SHIFT;
+    unsigned    i = cb >> X86_PD_SHIFT;
+    while (i-- > 0)
+    {
+        if (pVM->pgm.s.pInterPD->a[iPDNew + i].n.u1Present)
+        {
+            /* Check that it's not one or our mappings. */
+            PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+            while (pCur)
+            {
+                if (iPDNew + i - (pCur->GCPtr >> X86_PD_SHIFT) < (pCur->cb >> X86_PD_SHIFT))
+                    break;
+                pCur = pCur->pNextR3;
+            }
+            if (!pCur)
+            {
+                LogRel(("PGMR3MappingsFix: Conflicts with intermediate PDE %#x (GCPtrBase=%RGv cb=%#zx). The guest should retry.\n",
+                        iPDNew + i, GCPtrBase, cb));
+                return VERR_PGM_MAPPINGS_FIX_CONFLICT;
+            }
+        }
+    }
+
+    /*
+     * In PAE / PAE mode, make sure we don't cross page directories.
+     */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+    if (    (   pVCpu->pgm.s.enmGuestMode  == PGMMODE_PAE
+             || pVCpu->pgm.s.enmGuestMode  == PGMMODE_PAE_NX)
+        &&  (   pVCpu->pgm.s.enmShadowMode == PGMMODE_PAE
+             || pVCpu->pgm.s.enmShadowMode == PGMMODE_PAE_NX))
+    {
+        unsigned iPdptBase = GCPtrBase >> X86_PDPT_SHIFT;
+        unsigned iPdptLast = (GCPtrBase + cb - 1) >> X86_PDPT_SHIFT;
+        if (iPdptBase != iPdptLast)
+        {
+            LogRel(("PGMR3MappingsFix: Crosses PD boundary; iPdptBase=%#x iPdptLast=%#x (GCPtrBase=%RGv cb=%#zx). The guest should retry.\n",
+                    iPdptBase, iPdptLast, GCPtrBase, cb));
+            return VERR_PGM_MAPPINGS_FIX_CONFLICT;
+        }
+    }
+
+    /*
+     * Loop the mappings and check that they all agree on their new locations.
+     */
+    RTGCPTR     GCPtrCur = GCPtrBase;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        if (!pCur->pfnRelocate(pVM, pCur->GCPtr, GCPtrCur, PGMRELOCATECALL_SUGGEST, pCur->pvUser))
+        {
+            AssertMsgFailed(("The suggested fixed address %#x was rejected by '%s'!\n", GCPtrCur, pCur->pszDesc));
+            return VERR_PGM_MAPPINGS_FIX_REJECTED;
+        }
+        /* next */
+        GCPtrCur += pCur->cb;
+        pCur = pCur->pNextR3;
+    }
+    if (GCPtrCur > GCPtrBase + cb)
+    {
+        AssertMsgFailed(("cb (%#x) is less than the required range %#x!\n", cb, GCPtrCur - GCPtrBase));
+        return VERR_PGM_MAPPINGS_FIX_TOO_SMALL;
+    }
+
+    /*
+     * Loop the table assigning the mappings to the passed in memory
+     * and call their relocator callback.
+     */
+    GCPtrCur = GCPtrBase;
+    pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur)
+    {
+        RTGCPTR const GCPtrOld = pCur->GCPtr;
+
+        /*
+         * Relocate the page table(s).
+         */
+        if (pCur->GCPtr != NIL_RTGCPTR)
+            pgmR3MapClearPDEs(pVM, pCur, GCPtrOld >> X86_PD_SHIFT);
+        pgmR3MapSetPDEs(pVM, pCur, GCPtrCur >> X86_PD_SHIFT);
+
+        /*
+         * Update the entry.
+         */
+        pCur->GCPtr = GCPtrCur;
+        pCur->GCPtrLast = GCPtrCur + pCur->cb - 1;
+
+        /*
+         * Callback to execute the relocation.
+         */
+        pCur->pfnRelocate(pVM, GCPtrOld, GCPtrCur, PGMRELOCATECALL_RELOCATE, pCur->pvUser);
+
+        /*
+         * Advance.
+         */
+        GCPtrCur += pCur->cb;
+        pCur = pCur->pNextR3;
+    }
+
+    /*
+     * Mark the mappings as fixed at this new location and return.
+     */
+    pVM->pgm.s.fMappingsFixed           = true;
+    pVM->pgm.s.fMappingsFixedRestored   = false;
+    pVM->pgm.s.GCPtrMappingFixed        = GCPtrBase;
+    pVM->pgm.s.cbMappingFixed           = cb;
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
+        VMCPU_FF_SET(&pVM->aCpus[idCpu], VMCPU_FF_PGM_SYNC_CR3);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Interface for disabling the guest mappings when switching to HWACCM mode
+ * during VM creation and VM reset.
+ *
+ * (This doesn't touch the intermediate table!)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM.
+ */
+VMMR3DECL(int) PGMR3MappingsDisable(PVM pVM)
+{
+    AssertReturn(!pVM->pgm.s.fMappingsFixed,            VERR_INTERNAL_ERROR_4);
+    AssertReturn(!pVM->pgm.s.fMappingsFixedRestored,    VERR_INTERNAL_ERROR_4);
+    if (pVM->pgm.s.fMappingsDisabled)
+        return VINF_SUCCESS;
+
+    /*
+     * Deactivate (only applies to Virtual CPU #0).
+     */
+    if (pVM->aCpus[0].pgm.s.pShwPageCR3R3)
+    {
+        pgmLock(pVM);                           /* to avoid assertions */
+        int rc = pgmMapDeactivateCR3(pVM, pVM->aCpus[0].pgm.s.pShwPageCR3R3);
+        pgmUnlock(pVM);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Mark the mappings as disabled and trigger a CR3 re-sync.
+     */
+    pVM->pgm.s.fMappingsDisabled = true;
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
+        VMCPU_FF_SET(&pVM->aCpus[idCpu], VMCPU_FF_PGM_SYNC_CR3);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unfixes the mappings.
+ *
+ * Unless PGMR3MappingsDisable is in effect, mapping conflict detection will be
+ * enabled after this call.  If the mappings are fixed, a full CR3 resync will
+ * take place afterwards.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM.
+ */
+VMMR3DECL(int) PGMR3MappingsUnfix(PVM pVM)
+{
+    Log(("PGMR3MappingsUnfix: fMappingsFixed=%RTbool fMappingsDisabled=%RTbool\n", pVM->pgm.s.fMappingsFixed, pVM->pgm.s.fMappingsDisabled));
+    if (   pgmMapAreMappingsEnabled(&pVM->pgm.s)
+        && (    pVM->pgm.s.fMappingsFixed
+            ||  pVM->pgm.s.fMappingsFixedRestored)
+       )
+    {
+        bool const fResyncCR3 = pVM->pgm.s.fMappingsFixed;
+
+        pVM->pgm.s.fMappingsFixed           = false;
+        pVM->pgm.s.fMappingsFixedRestored   = false;
+        pVM->pgm.s.GCPtrMappingFixed        = 0;
+        pVM->pgm.s.cbMappingFixed           = 0;
+
+        if (fResyncCR3)
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+                VMCPU_FF_SET(&pVM->aCpus[i], VMCPU_FF_PGM_SYNC_CR3);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the mappings needs re-fixing after a restore.
+ *
+ * @returns true if they need, false if not.
+ * @param   pVM                 The VM handle.
+ */
+VMMR3DECL(bool) PGMR3MappingsNeedReFixing(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->pgm.s.fMappingsFixedRestored;
+}
+
+
+/**
+ * Map pages into the intermediate context (switcher code).
+ *
+ * These pages are mapped at both the give virtual address and at the physical
+ * address (for identity mapping).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The virtual machine.
+ * @param   Addr        Intermediate context address of the mapping.
+ * @param   HCPhys      Start of the range of physical pages. This must be entriely below 4GB!
+ * @param   cbPages     Number of bytes to map.
+ *
+ * @remark  This API shall not be used to anything but mapping the switcher code.
+ */
+VMMR3DECL(int) PGMR3MapIntermediate(PVM pVM, RTUINTPTR Addr, RTHCPHYS HCPhys, unsigned cbPages)
+{
+    LogFlow(("PGMR3MapIntermediate: Addr=%RTptr HCPhys=%RHp cbPages=%#x\n", Addr, HCPhys, cbPages));
+
+    /*
+     * Adjust input.
+     */
+    cbPages += (uint32_t)HCPhys & PAGE_OFFSET_MASK;
+    cbPages  = RT_ALIGN(cbPages, PAGE_SIZE);
+    HCPhys  &= X86_PTE_PAE_PG_MASK;
+    Addr    &= PAGE_BASE_MASK;
+    /* We only care about the first 4GB, because on AMD64 we'll be repeating them all over the address space. */
+    uint32_t uAddress = (uint32_t)Addr;
+
+    /*
+     * Assert input and state.
+     */
+    AssertMsg(pVM->pgm.s.offVM, ("Bad init order\n"));
+    AssertMsg(pVM->pgm.s.pInterPD, ("Bad init order, paging.\n"));
+    AssertMsg(cbPages <= (512 << PAGE_SHIFT), ("The mapping is too big %d bytes\n", cbPages));
+    AssertMsg(HCPhys < _4G && HCPhys + cbPages < _4G, ("Addr=%RTptr HCPhys=%RHp cbPages=%d\n", Addr, HCPhys, cbPages));
+    AssertReturn(!pVM->pgm.s.fFinalizedMappings, VERR_WRONG_ORDER);
+
+    /*
+     * Check for internal conflicts between the virtual address and the physical address.
+     * A 1:1 mapping is fine, but partial overlapping is a no-no.
+     */
+    if (    uAddress != HCPhys
+        &&  (   uAddress < HCPhys
+                ? HCPhys - uAddress < cbPages
+                : uAddress - HCPhys < cbPages
+            )
+       )
+        AssertLogRelMsgFailedReturn(("Addr=%RTptr HCPhys=%RHp cbPages=%d\n", Addr, HCPhys, cbPages),
+                                    VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+
+    const unsigned cPages = cbPages >> PAGE_SHIFT;
+    int rc = pgmR3MapIntermediateCheckOne(pVM, uAddress, cPages, pVM->pgm.s.apInterPTs[0], pVM->pgm.s.apInterPaePTs[0]);
+    if (RT_FAILURE(rc))
+        return rc;
+    rc = pgmR3MapIntermediateCheckOne(pVM, (uintptr_t)HCPhys, cPages, pVM->pgm.s.apInterPTs[1], pVM->pgm.s.apInterPaePTs[1]);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Everythings fine, do the mapping.
+     */
+    pgmR3MapIntermediateDoOne(pVM, uAddress, HCPhys, cPages, pVM->pgm.s.apInterPTs[0], pVM->pgm.s.apInterPaePTs[0]);
+    pgmR3MapIntermediateDoOne(pVM, (uintptr_t)HCPhys, HCPhys, cPages, pVM->pgm.s.apInterPTs[1], pVM->pgm.s.apInterPaePTs[1]);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates that there are no conflicts for this mapping into the intermediate context.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   uAddress    Address of the mapping.
+ * @param   cPages      Number of pages.
+ * @param   pPTDefault      Pointer to the default page table for this mapping.
+ * @param   pPTPaeDefault   Pointer to the default page table for this mapping.
+ */
+static int pgmR3MapIntermediateCheckOne(PVM pVM, uintptr_t uAddress, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault)
+{
+    AssertMsg((uAddress >> X86_PD_SHIFT) + cPages <= 1024, ("64-bit fixme\n"));
+
+    /*
+     * Check that the ranges are available.
+     * (This code doesn't have to be fast.)
+     */
+    while (cPages > 0)
+    {
+        /*
+         * 32-Bit.
+         */
+        unsigned iPDE = (uAddress >> X86_PD_SHIFT) & X86_PD_MASK;
+        unsigned iPTE = (uAddress >> X86_PT_SHIFT) & X86_PT_MASK;
+        PX86PT pPT = pPTDefault;
+        if (pVM->pgm.s.pInterPD->a[iPDE].u)
+        {
+            RTHCPHYS HCPhysPT = pVM->pgm.s.pInterPD->a[iPDE].u & X86_PDE_PG_MASK;
+            if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[0]))
+                pPT = pVM->pgm.s.apInterPTs[0];
+            else if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPTs[1]))
+                pPT = pVM->pgm.s.apInterPTs[1];
+            else
+            {
+                /** @todo this must be handled with a relocation of the conflicting mapping!
+                 * Which of course cannot be done because we're in the middle of the initialization. bad design! */
+                AssertLogRelMsgFailedReturn(("Conflict between core code and PGMR3Mapping(). uAddress=%RHv\n", uAddress),
+                                            VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+            }
+        }
+        if (pPT->a[iPTE].u)
+            AssertLogRelMsgFailedReturn(("Conflict iPTE=%#x iPDE=%#x uAddress=%RHv pPT->a[iPTE].u=%RX32\n", iPTE, iPDE, uAddress, pPT->a[iPTE].u),
+                                        VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+
+        /*
+         * PAE.
+         */
+        const unsigned iPDPE= (uAddress >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+        iPDE = (uAddress >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+        iPTE = (uAddress >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK;
+        Assert(iPDPE < 4);
+        Assert(pVM->pgm.s.apInterPaePDs[iPDPE]);
+        PX86PTPAE pPTPae = pPTPaeDefault;
+        if (pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u)
+        {
+            RTHCPHYS HCPhysPT = pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u & X86_PDE_PAE_PG_MASK;
+            if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]))
+                pPTPae = pVM->pgm.s.apInterPaePTs[0];
+            else if (HCPhysPT == MMPage2Phys(pVM, pVM->pgm.s.apInterPaePTs[0]))
+                pPTPae = pVM->pgm.s.apInterPaePTs[1];
+            else
+            {
+                /** @todo this must be handled with a relocation of the conflicting mapping!
+                 * Which of course cannot be done because we're in the middle of the initialization. bad design! */
+                AssertLogRelMsgFailedReturn(("Conflict between core code and PGMR3Mapping(). uAddress=%RHv\n", uAddress),
+                                            VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+            }
+        }
+        if (pPTPae->a[iPTE].u)
+            AssertLogRelMsgFailedReturn(("Conflict iPTE=%#x iPDE=%#x uAddress=%RHv pPTPae->a[iPTE].u=%#RX64\n", iPTE, iPDE, uAddress, pPTPae->a[iPTE].u),
+                                        VERR_PGM_INTERMEDIATE_PAGING_CONFLICT);
+
+        /* next */
+        uAddress += PAGE_SIZE;
+        cPages--;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Sets up the intermediate page tables for a verified mapping.
+ *
+ * @param   pVM             VM handle.
+ * @param   uAddress        Address of the mapping.
+ * @param   HCPhys          The physical address of the page range.
+ * @param   cPages          Number of pages.
+ * @param   pPTDefault      Pointer to the default page table for this mapping.
+ * @param   pPTPaeDefault   Pointer to the default page table for this mapping.
+ */
+static void pgmR3MapIntermediateDoOne(PVM pVM, uintptr_t uAddress, RTHCPHYS HCPhys, unsigned cPages, PX86PT pPTDefault, PX86PTPAE pPTPaeDefault)
+{
+    while (cPages > 0)
+    {
+        /*
+         * 32-Bit.
+         */
+        unsigned iPDE = (uAddress >> X86_PD_SHIFT) & X86_PD_MASK;
+        unsigned iPTE = (uAddress >> X86_PT_SHIFT) & X86_PT_MASK;
+        PX86PT pPT;
+        if (pVM->pgm.s.pInterPD->a[iPDE].u)
+            pPT = (PX86PT)MMPagePhys2Page(pVM, pVM->pgm.s.pInterPD->a[iPDE].u & X86_PDE_PG_MASK);
+        else
+        {
+            pVM->pgm.s.pInterPD->a[iPDE].u = X86_PDE_P | X86_PDE_A | X86_PDE_RW
+                                           | (uint32_t)MMPage2Phys(pVM, pPTDefault);
+            pPT = pPTDefault;
+        }
+        pPT->a[iPTE].u = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D | (uint32_t)HCPhys;
+
+        /*
+         * PAE
+         */
+        const unsigned iPDPE= (uAddress >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+        iPDE = (uAddress >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+        iPTE = (uAddress >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK;
+        Assert(iPDPE < 4);
+        Assert(pVM->pgm.s.apInterPaePDs[iPDPE]);
+        PX86PTPAE pPTPae;
+        if (pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u)
+            pPTPae = (PX86PTPAE)MMPagePhys2Page(pVM, pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u & X86_PDE_PAE_PG_MASK);
+        else
+        {
+            pPTPae = pPTPaeDefault;
+            pVM->pgm.s.apInterPaePDs[iPDPE]->a[iPDE].u = X86_PDE_P | X86_PDE_A | X86_PDE_RW
+                                                       | MMPage2Phys(pVM, pPTPaeDefault);
+        }
+        pPTPae->a[iPTE].u = X86_PTE_P | X86_PTE_RW | X86_PTE_A | X86_PTE_D | HCPhys;
+
+        /* next */
+        cPages--;
+        HCPhys += PAGE_SIZE;
+        uAddress += PAGE_SIZE;
+    }
+}
+
+
+/**
+ * Clears all PDEs involved with the mapping in the shadow and intermediate page tables.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pMap        Pointer to the mapping in question.
+ * @param   iOldPDE     The index of the 32-bit PDE corresponding to the base of the mapping.
+ */
+static void pgmR3MapClearPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iOldPDE)
+{
+    unsigned i     = pMap->cPTs;
+    PVMCPU   pVCpu = VMMGetCpu(pVM);
+    pgmLock(pVM);                           /* to avoid assertions */
+
+    pgmMapClearShadowPDEs(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pMap, iOldPDE, false /*fDeactivateCR3*/);
+
+    iOldPDE += i;
+    while (i-- > 0)
+    {
+        iOldPDE--;
+
+        /*
+         * 32-bit.
+         */
+        pVM->pgm.s.pInterPD->a[iOldPDE].u        = 0;
+
+        /*
+         * PAE.
+         */
+        const unsigned iPD = iOldPDE / 256;         /* iOldPDE * 2 / 512; iOldPDE is in 4 MB pages */
+        unsigned iPDE = iOldPDE * 2 % 512;
+        pVM->pgm.s.apInterPaePDs[iPD]->a[iPDE].u = 0;
+        iPDE++;
+        AssertFatal(iPDE < 512);
+        pVM->pgm.s.apInterPaePDs[iPD]->a[iPDE].u = 0;
+    }
+
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Sets all PDEs involved with the mapping in the shadow and intermediate page tables.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pMap        Pointer to the mapping in question.
+ * @param   iNewPDE     The index of the 32-bit PDE corresponding to the base of the mapping.
+ */
+static void pgmR3MapSetPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE)
+{
+    PPGM   pPGM  = &pVM->pgm.s;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    pgmLock(pVM);                           /* to avoid assertions */
+
+    Assert(!pgmMapAreMappingsEnabled(&pVM->pgm.s) || PGMGetGuestMode(pVCpu) <= PGMMODE_PAE_NX);
+
+    pgmMapSetShadowPDEs(pVM, pMap, iNewPDE);
+
+    /*
+     * Init the page tables and insert them into the page directories.
+     */
+    unsigned i = pMap->cPTs;
+    iNewPDE += i;
+    while (i-- > 0)
+    {
+        iNewPDE--;
+
+        /*
+         * 32-bit.
+         */
+        X86PDE Pde;
+        /* Default mapping page directory flags are read/write and supervisor; individual page attributes determine the final flags */
+        Pde.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | (uint32_t)pMap->aPTs[i].HCPhysPT;
+        pPGM->pInterPD->a[iNewPDE]        = Pde;
+        /*
+         * PAE.
+         */
+        const unsigned iPD = iNewPDE / 256;
+        unsigned iPDE = iNewPDE * 2 % 512;
+        X86PDEPAE PdePae0;
+        PdePae0.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT0;
+        pPGM->apInterPaePDs[iPD]->a[iPDE] = PdePae0;
+        iPDE++;
+        AssertFatal(iPDE < 512);
+        X86PDEPAE PdePae1;
+        PdePae1.u = PGM_PDFLAGS_MAPPING | X86_PDE_P | X86_PDE_A | X86_PDE_RW | X86_PDE_US | pMap->aPTs[i].HCPhysPaePT1;
+        pPGM->apInterPaePDs[iPD]->a[iPDE] = PdePae1;
+    }
+
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Relocates a mapping to a new address.
+ *
+ * @param   pVM                 VM handle.
+ * @param   pMapping            The mapping to relocate.
+ * @param   GCPtrOldMapping     The address of the start of the old mapping.
+ *                              NIL_RTGCPTR if not currently mapped.
+ * @param   GCPtrNewMapping     The address of the start of the new mapping.
+ */
+static void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping)
+{
+    Log(("PGM: Relocating %s from %RGv to %RGv\n", pMapping->pszDesc, GCPtrOldMapping, GCPtrNewMapping));
+    AssertMsg(GCPtrOldMapping == pMapping->GCPtr, ("%RGv vs %RGv\n", GCPtrOldMapping, pMapping->GCPtr));
+    AssertMsg((GCPtrOldMapping >> X86_PD_SHIFT) < X86_PG_ENTRIES, ("%RGv\n", GCPtrOldMapping));
+    AssertMsg((GCPtrNewMapping >> X86_PD_SHIFT) < X86_PG_ENTRIES, ("%RGv\n", GCPtrOldMapping));
+
+    /*
+     * Relocate the page table(s).
+     */
+    if (GCPtrOldMapping != NIL_RTGCPTR)
+        pgmR3MapClearPDEs(pVM, pMapping, GCPtrOldMapping >> X86_PD_SHIFT);
+    pgmR3MapSetPDEs(pVM, pMapping, GCPtrNewMapping >> X86_PD_SHIFT);
+
+    /*
+     * Update and resort the mapping list.
+     */
+
+    /* Find previous mapping for pMapping, put result into pPrevMap. */
+    PPGMMAPPING pPrevMap = NULL;
+    PPGMMAPPING pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur && pCur != pMapping)
+    {
+        /* next */
+        pPrevMap = pCur;
+        pCur = pCur->pNextR3;
+    }
+    Assert(pCur);
+
+    /* Find mapping which >= than pMapping. */
+    RTGCPTR     GCPtrNew = GCPtrNewMapping;
+    PPGMMAPPING pPrev = NULL;
+    pCur = pVM->pgm.s.pMappingsR3;
+    while (pCur && pCur->GCPtr < GCPtrNew)
+    {
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+
+    if (pCur != pMapping && pPrev != pMapping)
+    {
+        /*
+         * Unlink.
+         */
+        if (pPrevMap)
+        {
+            pPrevMap->pNextR3 = pMapping->pNextR3;
+            pPrevMap->pNextRC = pMapping->pNextRC;
+            pPrevMap->pNextR0 = pMapping->pNextR0;
+        }
+        else
+        {
+            pVM->pgm.s.pMappingsR3 = pMapping->pNextR3;
+            pVM->pgm.s.pMappingsRC = pMapping->pNextRC;
+            pVM->pgm.s.pMappingsR0 = pMapping->pNextR0;
+        }
+
+        /*
+         * Link
+         */
+        pMapping->pNextR3 = pCur;
+        if (pPrev)
+        {
+            pMapping->pNextRC = pPrev->pNextRC;
+            pMapping->pNextR0 = pPrev->pNextR0;
+            pPrev->pNextR3 = pMapping;
+            pPrev->pNextRC = MMHyperR3ToRC(pVM, pMapping);
+            pPrev->pNextR0 = MMHyperR3ToR0(pVM, pMapping);
+        }
+        else
+        {
+            pMapping->pNextRC = pVM->pgm.s.pMappingsRC;
+            pMapping->pNextR0 = pVM->pgm.s.pMappingsR0;
+            pVM->pgm.s.pMappingsR3 = pMapping;
+            pVM->pgm.s.pMappingsRC = MMHyperR3ToRC(pVM, pMapping);
+            pVM->pgm.s.pMappingsR0 = MMHyperR3ToR0(pVM, pMapping);
+        }
+    }
+
+    /*
+     * Update the entry.
+     */
+    pMapping->GCPtr = GCPtrNew;
+    pMapping->GCPtrLast = GCPtrNew + pMapping->cb - 1;
+
+    /*
+     * Callback to execute the relocation.
+     */
+    pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_RELOCATE, pMapping->pvUser);
+}
+
+
+/**
+ * Checks if a new mapping address wasn't previously used and caused a clash with guest mappings.
+ *
+ * @returns VBox status code.
+ * @param   pMapping            The mapping which conflicts.
+ * @param   GCPtr               New mapping address to try
+ */
+bool pgmR3MapIsKnownConflictAddress(PPGMMAPPING pMapping, RTGCPTR GCPtr)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pMapping->aGCPtrConflicts); i++)
+    {
+        if (GCPtr == pMapping->aGCPtrConflicts[i])
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Resolves a conflict between a page table based GC mapping and
+ * the Guest OS page tables. (32 bits version)
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM Handle.
+ * @param   pMapping            The mapping which conflicts.
+ * @param   pPDSrc              The page directory of the guest OS.
+ * @param   GCPtrOldMapping     The address of the start of the current mapping.
+ */
+int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping)
+{
+    STAM_REL_COUNTER_INC(&pVM->pgm.s.cRelocations);
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+
+    /* Raw mode only which implies one VCPU. */
+    Assert(pVM->cCpus == 1);
+
+    pMapping->aGCPtrConflicts[pMapping->cConflicts & (PGMMAPPING_CONFLICT_MAX-1)] = GCPtrOldMapping;
+    pMapping->cConflicts++;
+
+    /*
+     * Scan for free page directory entries.
+     *
+     * Note that we do not support mappings at the very end of the
+     * address space since that will break our GCPtrEnd assumptions.
+     */
+    const unsigned  cPTs = pMapping->cPTs;
+    unsigned        iPDNew = RT_ELEMENTS(pPDSrc->a) - cPTs; /* (+ 1 - 1) */
+    while (iPDNew-- > 0)
+    {
+        if (pPDSrc->a[iPDNew].n.u1Present)
+            continue;
+
+        if (pgmR3MapIsKnownConflictAddress(pMapping, iPDNew << X86_PD_SHIFT))
+            continue;
+
+        if (cPTs > 1)
+        {
+            bool fOk = true;
+            for (unsigned i = 1; fOk && i < cPTs; i++)
+                if (pPDSrc->a[iPDNew + i].n.u1Present)
+                    fOk = false;
+            if (!fOk)
+                continue;
+        }
+
+        /*
+         * Check that it's not conflicting with an intermediate page table mapping.
+         */
+        bool        fOk = true;
+        unsigned    i   = cPTs;
+        while (fOk && i-- > 0)
+            fOk = !pVM->pgm.s.pInterPD->a[iPDNew + i].n.u1Present;
+        if (!fOk)
+            continue;
+        /** @todo AMD64 should check the PAE directories and skip the 32bit stuff. */
+
+        /*
+         * Ask for the mapping.
+         */
+        RTGCPTR GCPtrNewMapping = (RTGCPTR32)iPDNew << X86_PD_SHIFT;
+
+        if (pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_SUGGEST, pMapping->pvUser))
+        {
+            pgmR3MapRelocate(pVM, pMapping, GCPtrOldMapping, GCPtrNewMapping);
+            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+            return VINF_SUCCESS;
+        }
+    }
+
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+    AssertMsgFailed(("Failed to relocate page table mapping '%s' from %#x! (cPTs=%d)\n", pMapping->pszDesc, GCPtrOldMapping, cPTs));
+    return VERR_PGM_NO_HYPERVISOR_ADDRESS;
+}
+
+
+/**
+ * Resolves a conflict between a page table based GC mapping and
+ * the Guest OS page tables. (PAE bits version)
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM Handle.
+ * @param   pMapping            The mapping which conflicts.
+ * @param   GCPtrOldMapping     The address of the start of the current mapping.
+ */
+int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping)
+{
+    STAM_REL_COUNTER_INC(&pVM->pgm.s.cRelocations);
+    STAM_PROFILE_START(&pVM->pgm.s.StatR3ResolveConflict, a);
+
+    /* Raw mode only which implies one VCPU. */
+    Assert(pVM->cCpus == 1);
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    pMapping->aGCPtrConflicts[pMapping->cConflicts & (PGMMAPPING_CONFLICT_MAX-1)] = GCPtrOldMapping;
+    pMapping->cConflicts++;
+
+    for (int iPDPTE = X86_PG_PAE_PDPE_ENTRIES - 1; iPDPTE >= 0; iPDPTE--)
+    {
+        unsigned  iPDSrc;
+        PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, (RTGCPTR32)iPDPTE << X86_PDPT_SHIFT, &iPDSrc, NULL);
+
+        /*
+         * Scan for free page directory entries.
+         *
+         * Note that we do not support mappings at the very end of the
+         * address space since that will break our GCPtrEnd assumptions.
+         * Nor do we support mappings crossing page directories.
+         */
+        const unsigned  cPTs = pMapping->cb >> X86_PD_PAE_SHIFT;
+        unsigned        iPDNew = RT_ELEMENTS(pPDSrc->a) - cPTs; /* (+ 1 - 1) */
+
+        while (iPDNew-- > 0)
+        {
+            /* Ugly assumption that mappings start on a 4 MB boundary. */
+            if (iPDNew & 1)
+                continue;
+
+            if (pgmR3MapIsKnownConflictAddress(pMapping, ((RTGCPTR32)iPDPTE << X86_PDPT_SHIFT) + (iPDNew << X86_PD_PAE_SHIFT)))
+                continue;
+
+            if (pPDSrc)
+            {
+                if (pPDSrc->a[iPDNew].n.u1Present)
+                    continue;
+                if (cPTs > 1)
+                {
+                    bool fOk = true;
+                    for (unsigned i = 1; fOk && i < cPTs; i++)
+                        if (pPDSrc->a[iPDNew + i].n.u1Present)
+                            fOk = false;
+                    if (!fOk)
+                        continue;
+                }
+            }
+            /*
+             * Check that it's not conflicting with an intermediate page table mapping.
+             */
+            bool        fOk = true;
+            unsigned    i   = cPTs;
+            while (fOk && i-- > 0)
+                fOk = !pVM->pgm.s.apInterPaePDs[iPDPTE]->a[iPDNew + i].n.u1Present;
+            if (!fOk)
+                continue;
+
+            /*
+             * Ask for the mapping.
+             */
+            RTGCPTR GCPtrNewMapping = ((RTGCPTR32)iPDPTE << X86_PDPT_SHIFT) + ((RTGCPTR32)iPDNew << X86_PD_PAE_SHIFT);
+
+            if (pMapping->pfnRelocate(pVM, GCPtrOldMapping, GCPtrNewMapping, PGMRELOCATECALL_SUGGEST, pMapping->pvUser))
+            {
+                pgmR3MapRelocate(pVM, pMapping, GCPtrOldMapping, GCPtrNewMapping);
+                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+                return VINF_SUCCESS;
+            }
+        }
+    }
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatR3ResolveConflict, a);
+    AssertMsgFailed(("Failed to relocate page table mapping '%s' from %#x! (cPTs=%d)\n", pMapping->pszDesc, GCPtrOldMapping, pMapping->cb >> X86_PD_PAE_SHIFT));
+    return VERR_PGM_NO_HYPERVISOR_ADDRESS;
+}
+
+
+/**
+ * Read memory from the guest mappings.
+ *
+ * This will use the page tables associated with the mappings to
+ * read the memory. This means that not all kind of memory is readable
+ * since we don't necessarily know how to convert that physical address
+ * to a HC virtual one.
+ *
+ * @returns VBox status.
+ * @param   pVM         VM handle.
+ * @param   pvDst       The destination address (HC of course).
+ * @param   GCPtrSrc    The source address (GC virtual address).
+ * @param   cb          Number of bytes to read.
+ *
+ * @remarks The is indirectly for DBGF only.
+ * @todo    Consider renaming it to indicate it's special usage, or just
+ *          reimplement it in MMR3HyperReadGCVirt.
+ */
+VMMR3DECL(int) PGMR3MapRead(PVM pVM, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
+{
+    /*
+     * Simplicity over speed... Chop the request up into chunks
+     * which don't cross pages.
+     */
+    if (cb + (GCPtrSrc & PAGE_OFFSET_MASK) > PAGE_SIZE)
+    {
+        for (;;)
+        {
+            size_t cbRead = RT_MIN(cb, PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK));
+            int rc = PGMR3MapRead(pVM, pvDst, GCPtrSrc, cbRead);
+            if (RT_FAILURE(rc))
+                return rc;
+            cb -= cbRead;
+            if (!cb)
+                break;
+            pvDst = (char *)pvDst + cbRead;
+            GCPtrSrc += cbRead;
+        }
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Find the mapping.
+     */
+    PPGMMAPPING pCur = pVM->pgm.s.CTX_SUFF(pMappings);
+    while (pCur)
+    {
+        RTGCPTR off = GCPtrSrc - pCur->GCPtr;
+        if (off < pCur->cb)
+        {
+            if (off + cb > pCur->cb)
+            {
+                AssertMsgFailed(("Invalid page range %RGv LB%#x. mapping '%s' %RGv to %RGv\n",
+                                 GCPtrSrc, cb, pCur->pszDesc, pCur->GCPtr, pCur->GCPtrLast));
+                return VERR_INVALID_PARAMETER;
+            }
+
+            unsigned iPT  = off >> X86_PD_SHIFT;
+            unsigned iPTE = (off >> PAGE_SHIFT) & X86_PT_MASK;
+            while (cb > 0 && iPTE < RT_ELEMENTS(CTXALLSUFF(pCur->aPTs[iPT].pPT)->a))
+            {
+                PCPGMSHWPTEPAE pPte = &pCur->aPTs[iPT].CTXALLSUFF(paPaePTs)[iPTE / 512].a[iPTE % 512];
+                if (!PGMSHWPTEPAE_IS_P(*pPte))
+                    return VERR_PAGE_NOT_PRESENT;
+                RTHCPHYS HCPhys = PGMSHWPTEPAE_GET_HCPHYS(*pPte);
+
+                /*
+                 * Get the virtual page from the physical one.
+                 */
+                void *pvPage;
+                int rc = MMR3HCPhys2HCVirt(pVM, HCPhys, &pvPage);
+                if (RT_FAILURE(rc))
+                    return rc;
+
+                memcpy(pvDst, (char *)pvPage + (GCPtrSrc & PAGE_OFFSET_MASK), cb);
+                return VINF_SUCCESS;
+            }
+        }
+
+        /* next */
+        pCur = CTXALLSUFF(pCur->pNext);
+    }
+
+    return VERR_INVALID_POINTER;
+}
+
+
+/**
+ * Info callback for 'pgmhandlers'.
+ *
+ * @param   pHlp        The output helpers.
+ * @param   pszArgs     The arguments. phys or virt.
+ */
+DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    if (pVM->pgm.s.fMappingsDisabled)
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are DISABLED.\n");
+    else if (pVM->pgm.s.fMappingsFixed)
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are FIXED: %RGv-%RGv\n",
+                        pVM->pgm.s.GCPtrMappingFixed, pVM->pgm.s.GCPtrMappingFixed + pVM->pgm.s.cbMappingFixed - 1);
+    else if (pVM->pgm.s.fMappingsFixedRestored)
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are FLOATING-RESTORED-FIXED: %RGv-%RGv\n",
+                        pVM->pgm.s.GCPtrMappingFixed, pVM->pgm.s.GCPtrMappingFixed + pVM->pgm.s.cbMappingFixed - 1);
+    else
+        pHlp->pfnPrintf(pHlp, "\nThe mappings are FLOATING.\n");
+
+    PPGMMAPPING pCur;
+    for (pCur = pVM->pgm.s.pMappingsR3; pCur; pCur = pCur->pNextR3)
+    {
+        pHlp->pfnPrintf(pHlp, "%RGv - %RGv  %s\n", pCur->GCPtr, pCur->GCPtrLast, pCur->pszDesc);
+        if (pCur->cConflicts > 0)
+        {
+            pHlp->pfnPrintf(pHlp, "  %u conflict%s: ", pCur->cConflicts, pCur->cConflicts == 1 ? "" : "s");
+            uint32_t cLeft = RT_MIN(pCur->cConflicts, RT_ELEMENTS(pCur->aGCPtrConflicts));
+            uint32_t i     = pCur->cConflicts;
+            while (cLeft-- > 0)
+            {
+                i = (i - 1) & (PGMMAPPING_CONFLICT_MAX - 1);
+                pHlp->pfnPrintf(pHlp, cLeft ? "%RGv, " : "%RGv\n", pCur->aGCPtrConflicts[i]);
+            }
+        }
+    }
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp	(revision 35333)
@@ -0,0 +1,4324 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager and Monitor, Physical Memory Addressing.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_PHYS
+#include <VBox/pgm.h>
+#include <VBox/iom.h>
+#include <VBox/mm.h>
+#include <VBox/stam.h>
+#include <VBox/rem.h>
+#include <VBox/pdmdev.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include "PGMInline.h"
+#include <VBox/sup.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+#include <iprt/system.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The number of pages to free in one batch. */
+#define PGMPHYS_FREE_PAGE_BATCH_SIZE    128
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
+
+
+/*
+ * PGMR3PhysReadU8-64
+ * PGMR3PhysWriteU8-64
+ */
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU8
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
+#define PGMPHYS_DATASIZE    1
+#define PGMPHYS_DATATYPE    uint8_t
+#include "PGMPhysRWTmpl.h"
+
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU16
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
+#define PGMPHYS_DATASIZE    2
+#define PGMPHYS_DATATYPE    uint16_t
+#include "PGMPhysRWTmpl.h"
+
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU32
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
+#define PGMPHYS_DATASIZE    4
+#define PGMPHYS_DATATYPE    uint32_t
+#include "PGMPhysRWTmpl.h"
+
+#define PGMPHYSFN_READNAME  PGMR3PhysReadU64
+#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
+#define PGMPHYS_DATASIZE    8
+#define PGMPHYS_DATATYPE    uint64_t
+#include "PGMPhysRWTmpl.h"
+
+
+/**
+ * EMT worker for PGMR3PhysReadExternal.
+ */
+static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead)
+{
+    PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Read from physical memory, external users.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS.
+ *
+ * @param   pVM             VM Handle.
+ * @param   GCPhys          Physical address to read from.
+ * @param   pvBuf           Where to read into.
+ * @param   cbRead          How many bytes to read.
+ *
+ * @thread  Any but EMTs.
+ */
+VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead)
+{
+    VM_ASSERT_OTHER_THREAD(pVM);
+
+    AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
+
+    pgmLock(pVM);
+
+    /*
+     * Copy loop on ram ranges.
+     */
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (;;)
+    {
+        /* Find range. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPHYS off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                unsigned iPage = off >> PAGE_SHIFT;
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+
+                /*
+                 * If the page has an ALL access handler, we'll have to
+                 * delegate the job to EMT.
+                 */
+                if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+                {
+                    pgmUnlock(pVM);
+
+                    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 4,
+                                           pVM, &GCPhys, pvBuf, cbRead);
+                }
+                Assert(!PGM_PAGE_IS_MMIO(pPage));
+
+                /*
+                 * Simple stuff, go ahead.
+                 */
+                size_t   cb    = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
+                if (cb > cbRead)
+                    cb = cbRead;
+                const void *pvSrc;
+                int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc);
+                if (RT_SUCCESS(rc))
+                    memcpy(pvBuf, pvSrc, cb);
+                else
+                {
+                    AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                           pRam->GCPhys + off, pPage, rc));
+                    memset(pvBuf, 0xff, cb);
+                }
+
+                /* next page */
+                if (cb >= cbRead)
+                {
+                    pgmUnlock(pVM);
+                    return VINF_SUCCESS;
+                }
+                cbRead -= cb;
+                off    += cb;
+                GCPhys += cb;
+                pvBuf   = (char *)pvBuf + cb;
+            } /* walk pages in ram range. */
+        }
+        else
+        {
+            LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
+
+            /*
+             * Unassigned address space.
+             */
+            if (!pRam)
+                break;
+            size_t cb = pRam->GCPhys - GCPhys;
+            if (cb >= cbRead)
+            {
+                memset(pvBuf, 0xff, cbRead);
+                break;
+            }
+            memset(pvBuf, 0xff, cb);
+
+            cbRead -= cb;
+            pvBuf   = (char *)pvBuf + cb;
+            GCPhys += cb;
+        }
+    } /* Ram range walk */
+
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * EMT worker for PGMR3PhysWriteExternal.
+ */
+static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite)
+{
+    /** @todo VERR_EM_NO_MEMORY */
+    PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Write to physical memory, external users.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS.
+ * @retval  VERR_EM_NO_MEMORY.
+ *
+ * @param   pVM             VM Handle.
+ * @param   GCPhys          Physical address to write to.
+ * @param   pvBuf           What to write.
+ * @param   cbWrite         How many bytes to write.
+ * @param   pszWho          Who is writing.  For tracking down who is writing
+ *                          after we've saved the state.
+ *
+ * @thread  Any but EMTs.
+ */
+VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, const char *pszWho)
+{
+    VM_ASSERT_OTHER_THREAD(pVM);
+
+    AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
+              ("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x pszWho=%s\n",
+               GCPhys, cbWrite, pszWho));
+    AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
+
+    pgmLock(pVM);
+
+    /*
+     * Copy loop on ram ranges, stop when we hit something difficult.
+     */
+    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    for (;;)
+    {
+        /* Find range. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPTR off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                RTGCPTR     iPage = off >> PAGE_SHIFT;
+                PPGMPAGE    pPage = &pRam->aPages[iPage];
+
+                /*
+                 * Is the page problematic, we have to do the work on the EMT.
+                 *
+                 * Allocating writable pages and access handlers are
+                 * problematic, write monitored pages are simple and can be
+                 * dealt with here.
+                 */
+                if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+                    ||  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+                {
+                    if (    PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
+                        && !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                        pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
+                    else
+                    {
+                        pgmUnlock(pVM);
+
+                        return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 4,
+                                               pVM, &GCPhys, pvBuf, cbWrite);
+                    }
+                }
+                Assert(!PGM_PAGE_IS_MMIO(pPage));
+
+                /*
+                 * Simple stuff, go ahead.
+                 */
+                size_t      cb    = PAGE_SIZE - (off & PAGE_OFFSET_MASK);
+                if (cb > cbWrite)
+                    cb = cbWrite;
+                void *pvDst;
+                int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst);
+                if (RT_SUCCESS(rc))
+                    memcpy(pvDst, pvBuf, cb);
+                else
+                    AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                           pRam->GCPhys + off, pPage, rc));
+
+                /* next page */
+                if (cb >= cbWrite)
+                {
+                    pgmUnlock(pVM);
+                    return VINF_SUCCESS;
+                }
+
+                cbWrite -= cb;
+                off     += cb;
+                GCPhys  += cb;
+                pvBuf    = (const char *)pvBuf + cb;
+            } /* walk pages in ram range */
+        }
+        else
+        {
+            /*
+             * Unassigned address space, skip it.
+             */
+            if (!pRam)
+                break;
+            size_t cb = pRam->GCPhys - GCPhys;
+            if (cb >= cbWrite)
+                break;
+            cbWrite -= cb;
+            pvBuf   = (const char *)pvBuf + cb;
+            GCPhys += cb;
+        }
+    } /* Ram range walk */
+
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
+ *
+ * @returns see PGMR3PhysGCPhys2CCPtrExternal
+ * @param   pVM         The VM handle.
+ * @param   pGCPhys     Pointer to the guest physical address.
+ * @param   ppv         Where to store the mapping address.
+ * @param   pLock       Where to store the lock.
+ */
+static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    /*
+     * Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
+     * an access handler after it succeeds.
+     */
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMPAGEMAPTLBE pTlbe;
+        int rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, *pGCPhys, &pTlbe);
+        AssertFatalRC(rc2);
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (PGM_PAGE_IS_MMIO(pPage))
+        {
+            PGMPhysReleasePageMappingLock(pVM, pLock);
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+        }
+        else if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                 ||  pgmPoolIsDirtyPage(pVM, *pGCPhys)
+#endif
+                )
+        {
+            /* We *must* flush any corresponding pgm pool page here, otherwise we'll
+             * not be informed about writes and keep bogus gst->shw mappings around.
+             */
+            pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
+            Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
+            /** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
+             *        active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
+        }
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page into ring-3, external threads.
+ *
+ * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
+ * release it.
+ *
+ * This API will assume your intention is to write to the page, and will
+ * therefore replace shared and zero pages. If you do not intend to modify the
+ * page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
+ *          backing or if the page has any active access handlers. The caller
+ *          must fall back on using PGMR3PhysWriteExternal.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than the
+ *          PGM one) because of the deadlock risk when we have to delegating the
+ *          task to an EMT.
+ * @thread  Any.
+ */
+VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    AssertPtr(ppv);
+    AssertPtr(pLock);
+
+    Assert(VM_IS_EMT(pVM) || !PGMIsLockOwner(pVM));
+
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (PGM_PAGE_IS_MMIO(pPage))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+        else
+        {
+            /*
+             * If the page is shared, the zero page, or being write monitored
+             * it must be converted to an page that's writable if possible.
+             * We can only deal with write monitored pages here, the rest have
+             * to be on an EMT.
+             */
+            if (    PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+                ||  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                ||  pgmPoolIsDirtyPage(pVM, GCPhys)
+#endif
+               )
+            {
+                if (    PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
+                    &&  !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                    &&  !pgmPoolIsDirtyPage(pVM, GCPhys)
+#endif
+                   )
+                    pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage);
+                else
+                {
+                    pgmUnlock(pVM);
+
+                    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
+                                           pVM, &GCPhys, ppv, pLock);
+                }
+            }
+
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            PPGMPAGEMAP pMap = pTlbe->pMap;
+            if (pMap)
+                pMap->cRefs++;
+
+            unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+            if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+            {
+                if (cLocks == 0)
+                    pVM->pgm.s.cWriteLockedPages++;
+                PGM_PAGE_INC_WRITE_LOCKS(pPage);
+            }
+            else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
+            {
+                PGM_PAGE_INC_WRITE_LOCKS(pPage);
+                AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
+                if (pMap)
+                    pMap->cRefs++; /* Extra ref to prevent it from going away. */
+            }
+
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+            pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
+            pLock->pvMap = pMap;
+        }
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page into ring-3, external threads.
+ *
+ * When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
+ * release it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
+ *          backing or if the page as an active ALL access handler. The caller
+ *          must fall back on using PGMPhysRead.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @thread  Any.
+ */
+VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = pgmLock(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMPAGE pPage = pTlbe->pPage;
+#if 1
+        /* MMIO pages doesn't have any readable backing. */
+        if (PGM_PAGE_IS_MMIO(pPage))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+#else
+        if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+#endif
+        else
+        {
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            PPGMPAGEMAP pMap = pTlbe->pMap;
+            if (pMap)
+                pMap->cRefs++;
+
+            unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+            if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+            {
+                if (cLocks == 0)
+                    pVM->pgm.s.cReadLockedPages++;
+                PGM_PAGE_INC_READ_LOCKS(pPage);
+            }
+            else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
+            {
+                PGM_PAGE_INC_READ_LOCKS(pPage);
+                AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
+                if (pMap)
+                    pMap->cRefs++; /* Extra ref to prevent it from going away. */
+            }
+
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+            pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
+            pLock->pvMap = pMap;
+        }
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Relinks the RAM ranges using the pSelfRC and pSelfR0 pointers.
+ *
+ * Called when anything was relocated.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ */
+void pgmR3PhysRelinkRamRanges(PVM pVM)
+{
+    PPGMRAMRANGE pCur;
+
+#ifdef VBOX_STRICT
+    for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+    {
+        Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfR0 == MMHyperCCToR0(pVM, pCur));
+        Assert((pCur->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pCur->pSelfRC == MMHyperCCToRC(pVM, pCur));
+        Assert((pCur->GCPhys     & PAGE_OFFSET_MASK) == 0);
+        Assert((pCur->GCPhysLast & PAGE_OFFSET_MASK) == PAGE_OFFSET_MASK);
+        Assert((pCur->cb         & PAGE_OFFSET_MASK) == 0);
+        Assert(pCur->cb == pCur->GCPhysLast - pCur->GCPhys + 1);
+        for (PPGMRAMRANGE pCur2 = pVM->pgm.s.pRamRangesR3; pCur2; pCur2 = pCur2->pNextR3)
+            Assert(   pCur2 == pCur
+                   || strcmp(pCur2->pszDesc, pCur->pszDesc)); /** @todo fix MMIO ranges!! */
+    }
+#endif
+
+    pCur = pVM->pgm.s.pRamRangesR3;
+    if (pCur)
+    {
+        pVM->pgm.s.pRamRangesR0 = pCur->pSelfR0;
+        pVM->pgm.s.pRamRangesRC = pCur->pSelfRC;
+
+        for (; pCur->pNextR3; pCur = pCur->pNextR3)
+        {
+            pCur->pNextR0 = pCur->pNextR3->pSelfR0;
+            pCur->pNextRC = pCur->pNextR3->pSelfRC;
+        }
+
+        Assert(pCur->pNextR0 == NIL_RTR0PTR);
+        Assert(pCur->pNextRC == NIL_RTRCPTR);
+    }
+    else
+    {
+        Assert(pVM->pgm.s.pRamRangesR0 == NIL_RTR0PTR);
+        Assert(pVM->pgm.s.pRamRangesRC == NIL_RTRCPTR);
+    }
+    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
+}
+
+
+/**
+ * Links a new RAM range into the list.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pNew        Pointer to the new list entry.
+ * @param   pPrev       Pointer to the previous list entry. If NULL, insert as head.
+ */
+static void pgmR3PhysLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, PPGMRAMRANGE pPrev)
+{
+    AssertMsg(pNew->pszDesc, ("%RGp-%RGp\n", pNew->GCPhys, pNew->GCPhysLast));
+    Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfR0 == MMHyperCCToR0(pVM, pNew));
+    Assert((pNew->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pNew->pSelfRC == MMHyperCCToRC(pVM, pNew));
+
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pRam = pPrev ? pPrev->pNextR3 : pVM->pgm.s.pRamRangesR3;
+    pNew->pNextR3 = pRam;
+    pNew->pNextR0 = pRam ? pRam->pSelfR0 : NIL_RTR0PTR;
+    pNew->pNextRC = pRam ? pRam->pSelfRC : NIL_RTRCPTR;
+
+    if (pPrev)
+    {
+        pPrev->pNextR3 = pNew;
+        pPrev->pNextR0 = pNew->pSelfR0;
+        pPrev->pNextRC = pNew->pSelfRC;
+    }
+    else
+    {
+        pVM->pgm.s.pRamRangesR3 = pNew;
+        pVM->pgm.s.pRamRangesR0 = pNew->pSelfR0;
+        pVM->pgm.s.pRamRangesRC = pNew->pSelfRC;
+    }
+    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Unlink an existing RAM range from the list.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pRam        Pointer to the new list entry.
+ * @param   pPrev       Pointer to the previous list entry. If NULL, insert as head.
+ */
+static void pgmR3PhysUnlinkRamRange2(PVM pVM, PPGMRAMRANGE pRam, PPGMRAMRANGE pPrev)
+{
+    Assert(pPrev ? pPrev->pNextR3 == pRam : pVM->pgm.s.pRamRangesR3 == pRam);
+    Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfR0 == MMHyperCCToR0(pVM, pRam));
+    Assert((pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING) || pRam->pSelfRC == MMHyperCCToRC(pVM, pRam));
+
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pNext = pRam->pNextR3;
+    if (pPrev)
+    {
+        pPrev->pNextR3 = pNext;
+        pPrev->pNextR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
+        pPrev->pNextRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
+    }
+    else
+    {
+        Assert(pVM->pgm.s.pRamRangesR3 == pRam);
+        pVM->pgm.s.pRamRangesR3 = pNext;
+        pVM->pgm.s.pRamRangesR0 = pNext ? pNext->pSelfR0 : NIL_RTR0PTR;
+        pVM->pgm.s.pRamRangesRC = pNext ? pNext->pSelfRC : NIL_RTRCPTR;
+    }
+    ASMAtomicIncU32(&pVM->pgm.s.idRamRangesGen);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Unlink an existing RAM range from the list.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pRam        Pointer to the new list entry.
+ */
+static void pgmR3PhysUnlinkRamRange(PVM pVM, PPGMRAMRANGE pRam)
+{
+    pgmLock(pVM);
+
+    /* find prev. */
+    PPGMRAMRANGE pPrev = NULL;
+    PPGMRAMRANGE pCur = pVM->pgm.s.pRamRangesR3;
+    while (pCur != pRam)
+    {
+        pPrev = pCur;
+        pCur = pCur->pNextR3;
+    }
+    AssertFatal(pCur);
+
+    pgmR3PhysUnlinkRamRange2(pVM, pRam, pPrev);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Frees a range of pages, replacing them with ZERO pages of the specified type.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pRam        The RAM range in which the pages resides.
+ * @param   GCPhys      The address of the first page.
+ * @param   GCPhysLast  The address of the last page.
+ * @param   uType       The page type to replace then with.
+ */
+static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint8_t uType)
+{
+    Assert(PGMIsLockOwner(pVM));
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    /* Iterate the pages. */
+    PPGMPAGE pPageDst   = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+    uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> PAGE_SHIFT) + 1;
+    while (cPagesLeft-- > 0)
+    {
+        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
+        AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
+
+        PGM_PAGE_SET_TYPE(pPageDst, uType);
+
+        GCPhys += PAGE_SIZE;
+        pPageDst++;
+    }
+
+    if (cPendingPages)
+    {
+        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    GMMR3FreePagesCleanup(pReq);
+
+    return rc;
+}
+
+#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
+/**
+ * Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
+ * @param   pvUser      User parameter
+ */
+static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    uintptr_t          *paUser          = (uintptr_t *)pvUser;
+    bool                fInflate        = !!paUser[0];
+    unsigned            cPages          = paUser[1];
+    RTGCPHYS           *paPhysPage      = (RTGCPHYS *)paUser[2];
+    uint32_t            cPendingPages   = 0;
+    PGMMFREEPAGESREQ    pReq;
+    int                 rc;
+
+    Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
+    pgmLock(pVM);
+
+    if (fInflate)
+    {
+        /* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
+        pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
+
+        /* Replace pages with ZERO pages. */
+        rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+        if (RT_FAILURE(rc))
+        {
+            pgmUnlock(pVM);
+            AssertLogRelRC(rc);
+            return rc;
+        }
+
+        /* Iterate the pages. */
+        for (unsigned i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
+            if (    pPage == NULL
+                ||  pPage->uTypeY != PGMPAGETYPE_RAM)
+            {
+                Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], (pPage) ? pPage->uTypeY : 0));
+                break;
+            }
+
+            LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
+
+            /* Flush the shadow PT if this page was previously used as a guest page table. */
+            pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
+
+            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i]);
+            if (RT_FAILURE(rc))
+            {
+                pgmUnlock(pVM);
+                AssertLogRelRC(rc);
+                return rc;
+            }
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+            PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_BALLOONED);
+        }
+
+        if (cPendingPages)
+        {
+            rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+            if (RT_FAILURE(rc))
+            {
+                pgmUnlock(pVM);
+                AssertLogRelRC(rc);
+                return rc;
+            }
+        }
+        GMMR3FreePagesCleanup(pReq);
+    }
+    else
+    {
+        /* Iterate the pages. */
+        for (unsigned i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, paPhysPage[i]);
+            AssertBreak(pPage && pPage->uTypeY == PGMPAGETYPE_RAM);
+
+            LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
+
+            Assert(PGM_PAGE_IS_BALLOONED(pPage));
+
+            /* Change back to zero page. */
+            PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
+        }
+
+        /* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
+    }
+
+    /* Notify GMM about the balloon change. */
+    rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
+    if (RT_SUCCESS(rc))
+    {
+        if (!fInflate)
+        {
+            Assert(pVM->pgm.s.cBalloonedPages >= cPages);
+            pVM->pgm.s.cBalloonedPages -= cPages;
+        }
+        else
+            pVM->pgm.s.cBalloonedPages += cPages;
+    }
+
+    pgmUnlock(pVM);
+
+    /* Flush the recompiler's TLB as well. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        CPUMSetChangedFlags(&pVM->aCpus[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    AssertLogRelRC(rc);
+    return rc;
+}
+
+/**
+ * Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   fInflate    Inflate or deflate memory balloon
+ * @param   cPages      Number of pages to free
+ * @param   paPhysPage  Array of guest physical addresses
+ */
+static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
+{
+    uintptr_t paUser[3];
+
+    paUser[0] = fInflate;
+    paUser[1] = cPages;
+    paUser[2] = (uintptr_t)paPhysPage;
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
+    AssertRC(rc);
+
+    /* Made a copy in PGMR3PhysFreeRamPages; free it here. */
+    RTMemFree(paPhysPage);
+}
+#endif
+
+/**
+ * Inflate or deflate a memory balloon
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   fInflate    Inflate or deflate memory balloon
+ * @param   cPages      Number of pages to free
+ * @param   paPhysPage  Array of guest physical addresses
+ */
+VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
+{
+    /* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
+#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) || defined(RT_OS_SOLARIS) || defined(RT_OS_LINUX) || defined(RT_OS_FREEBSD))
+    int rc;
+
+    /* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
+    AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
+
+    /* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
+     * In the SMP case we post a request packet to postpone the job.
+     */
+    if (pVM->cCpus > 1)
+    {
+        unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
+        RTGCPHYS *paPhysPageCopy = (RTGCPHYS *)RTMemAlloc(cbPhysPage);
+        AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
+
+        memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
+
+        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
+        AssertRC(rc);
+    }
+    else
+    {
+        uintptr_t paUser[3];
+
+        paUser[0] = fInflate;
+        paUser[1] = cPages;
+        paUser[2] = (uintptr_t)paPhysPage;
+        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
+        AssertRC(rc);
+    }
+    return rc;
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+/**
+ * Rendezvous callback used by PGMR3WriteProtectRAM that write protects all physical RAM
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
+ * @param   pvUser      User parameter
+ */
+static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+
+    pgmLock(pVM);
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    pgmPoolResetDirtyPages(pVM);
+#endif
+
+    /** @todo pointless to write protect the physical page pointed to by RSP. */
+
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        unsigned cPages = pRam->cb >> PAGE_SHIFT;
+        for (unsigned iPage = 0; iPage < cPages; iPage++)
+        {
+            PPGMPAGE    pPage = &pRam->aPages[iPage];
+            PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+
+            if (    RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
+                ||  enmPageType == PGMPAGETYPE_MMIO2)
+            {
+                /*
+                    * A RAM page.
+                    */
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                case PGM_PAGE_STATE_ALLOCATED:
+                    /** @todo Optimize this: Don't always re-enable write
+                        * monitoring if the page is known to be very busy. */
+                    if (PGM_PAGE_IS_WRITTEN_TO(pPage))
+                    {
+                        PGM_PAGE_CLEAR_WRITTEN_TO(pPage);
+                        /* Remember this dirty page for the next (memory) sync. */
+                        PGM_PAGE_SET_FT_DIRTY(pPage);
+                    }
+
+                    PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_WRITE_MONITORED);
+                    pVM->pgm.s.cMonitoredPages++;
+                    break;
+
+                case PGM_PAGE_STATE_SHARED:
+                    AssertFailed();
+                    break;
+
+                case PGM_PAGE_STATE_WRITE_MONITORED:    /* nothing to change. */
+                default:
+                    break;
+                }
+            }
+        }
+    }
+    pgmR3PoolWriteProtectPages(pVM);
+    PGM_INVL_ALL_VCPU_TLBS(pVM);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+/**
+ * Protect all physical RAM to monitor writes
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
+{
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
+    AssertRC(rc);
+    return rc;
+}
+
+/**
+ * Enumerate all dirty FT pages
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pfnEnum     Enumerate callback handler
+ * @param   pvUser      Enumerate callback handler parameter
+ */
+VMMR3DECL(int) PGMR3PhysEnumDirtyFTPages(PVM pVM, PFNPGMENUMDIRTYFTPAGES pfnEnum, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+
+    pgmLock(pVM);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        unsigned cPages = pRam->cb >> PAGE_SHIFT;
+        for (unsigned iPage = 0; iPage < cPages; iPage++)
+        {
+            PPGMPAGE    pPage = &pRam->aPages[iPage];
+            PGMPAGETYPE enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+
+            if (    RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
+                ||  enmPageType == PGMPAGETYPE_MMIO2)
+            {
+                /*
+                    * A RAM page.
+                    */
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                case PGM_PAGE_STATE_ALLOCATED:
+                case PGM_PAGE_STATE_WRITE_MONITORED:
+                    if (    !PGM_PAGE_IS_WRITTEN_TO(pPage)  /* not very recently updated? */
+                        &&  PGM_PAGE_IS_FT_DIRTY(pPage))
+                    {
+                        unsigned       cbPageRange = PAGE_SIZE;
+                        unsigned       iPageClean  = iPage + 1;
+                        RTGCPHYS       GCPhysPage  = pRam->GCPhys + iPage * PAGE_SIZE;
+                        uint8_t       *pu8Page     = NULL;
+                        PGMPAGEMAPLOCK Lock;
+
+                        /* Find the next clean page, so we can merge adjacent dirty pages. */
+                        for (; iPageClean < cPages; iPageClean++)
+                        {
+                            PPGMPAGE pPageNext = &pRam->aPages[iPageClean];
+                            if (    RT_UNLIKELY(PGM_PAGE_GET_TYPE(pPageNext) != PGMPAGETYPE_RAM)
+                                ||  PGM_PAGE_GET_STATE(pPageNext) != PGM_PAGE_STATE_ALLOCATED
+                                ||  PGM_PAGE_IS_WRITTEN_TO(pPageNext)
+                                ||  !PGM_PAGE_IS_FT_DIRTY(pPageNext)
+                                /* Crossing a chunk boundary? */
+                                ||  (GCPhysPage & GMM_PAGEID_IDX_MASK) != ((GCPhysPage + cbPageRange) & GMM_PAGEID_IDX_MASK)
+                                )
+                                break;
+
+                            cbPageRange += PAGE_SIZE;
+                        }
+
+                        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysPage, (const void **)&pu8Page, &Lock);
+                        if (RT_SUCCESS(rc))
+                        {
+                            /** @todo this is risky; the range might be changed, but little choice as the sync costs a lot of time */
+                            pgmUnlock(pVM);
+                            pfnEnum(pVM, GCPhysPage, pu8Page, cbPageRange, pvUser);
+                            pgmLock(pVM);
+                            PGMPhysReleasePageMappingLock(pVM, &Lock);
+                        }
+
+                        for (iPage; iPage < iPageClean; iPage++)
+                            PGM_PAGE_CLEAR_FT_DIRTY(&pRam->aPages[iPage]);
+
+                        iPage = iPageClean - 1;
+                    }
+                    break;
+                }
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Gets the number of ram ranges.
+ *
+ * @returns Number of ram ranges.  Returns UINT32_MAX if @a pVM is invalid.
+ * @param   pVM             The VM handle.
+ */
+VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+
+    pgmLock(pVM);
+    uint32_t cRamRanges = 0;
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRanges); pCur; pCur = pCur->CTX_SUFF(pNext))
+        cRamRanges++;
+    pgmUnlock(pVM);
+    return cRamRanges;
+}
+
+
+/**
+ * Get information about a range.
+ *
+ * @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
+ * @param   pVM             The VM handle
+ * @param   iRange          The ordinal of the range.
+ * @param   pGCPhysStart    Where to return the start of the range. Optional.
+ * @param   pGCPhysLast     Where to return the address of the last byte in the
+ *                          range. Optional.
+ * @param   pfIsMmio        Where to indicate that this is a pure MMIO range.
+ *                          Optional.
+ */
+VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
+                                 const char **ppszDesc, bool *pfIsMmio)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    pgmLock(pVM);
+    uint32_t iCurRange = 0;
+    for (PPGMRAMRANGE pCur = pVM->pgm.s.CTX_SUFF(pRamRanges); pCur; pCur = pCur->CTX_SUFF(pNext), iCurRange++)
+        if (iCurRange == iRange)
+        {
+            if (pGCPhysStart)
+                *pGCPhysStart = pCur->GCPhys;
+            if (pGCPhysLast)
+                *pGCPhysLast = pCur->GCPhysLast;
+            if (pfIsMmio)
+                *pfIsMmio = !!(pCur->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
+
+            pgmUnlock(pVM);
+            return VINF_SUCCESS;
+        }
+    pgmUnlock(pVM);
+    return VERR_OUT_OF_RANGE;
+}
+
+
+/**
+ * Query the amount of free memory inside VMMR0
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   puTotalAllocSize    Pointer to total allocated  memory inside VMMR0 (in bytes)
+ * @param   puTotalFreeSize     Pointer to total free (allocated but not used yet) memory inside VMMR0 (in bytes)
+ * @param   puTotalBalloonSize  Pointer to total ballooned memory inside VMMR0 (in bytes)
+ * @param   puTotalSharedSize   Pointer to total shared memory inside VMMR0 (in bytes)
+ */
+VMMR3DECL(int) PGMR3QueryVMMMemoryStats(PVM pVM, uint64_t *puTotalAllocSize, uint64_t *puTotalFreeSize, uint64_t *puTotalBalloonSize, uint64_t *puTotalSharedSize)
+{
+    int rc;
+
+    uint64_t cAllocPages = 0, cFreePages = 0, cBalloonPages = 0, cSharedPages = 0;
+    rc = GMMR3QueryHypervisorMemoryStats(pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
+    AssertRCReturn(rc, rc);
+
+    if (puTotalAllocSize)
+        *puTotalAllocSize = cAllocPages * _4K;
+
+    if (puTotalFreeSize)
+        *puTotalFreeSize = cFreePages * _4K;
+
+    if (puTotalBalloonSize)
+        *puTotalBalloonSize = cBalloonPages * _4K;
+
+    if (puTotalSharedSize)
+        *puTotalSharedSize = cSharedPages * _4K;
+
+    Log(("PGMR3QueryVMMMemoryStats: all=%x free=%x ballooned=%x shared=%x\n", cAllocPages, cFreePages, cBalloonPages, cSharedPages));
+    return VINF_SUCCESS;
+}
+
+/**
+ * Query memory stats for the VM
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   puTotalAllocSize    Pointer to total allocated  memory inside the VM (in bytes)
+ * @param   puTotalFreeSize     Pointer to total free (allocated but not used yet) memory inside the VM (in bytes)
+ * @param   puTotalBalloonSize  Pointer to total ballooned memory inside the VM (in bytes)
+ * @param   puTotalSharedSize   Pointer to total shared memory inside the VM (in bytes)
+ */
+VMMR3DECL(int) PGMR3QueryMemoryStats(PVM pVM, uint64_t *pulTotalMem, uint64_t *pulPrivateMem, uint64_t *puTotalSharedMem, uint64_t *puTotalZeroMem)
+{
+    if (pulTotalMem)
+        *pulTotalMem = (uint64_t)pVM->pgm.s.cAllPages * _4K;
+
+    if (pulPrivateMem)
+        *pulPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages * _4K;
+
+    if (puTotalSharedMem)
+        *puTotalSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages * _4K;
+
+    if (puTotalZeroMem)
+        *puTotalZeroMem = (uint64_t)pVM->pgm.s.cZeroPages * _4K;
+
+    Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
+    return VINF_SUCCESS;
+}
+
+/**
+ * PGMR3PhysRegisterRam worker that initializes and links a RAM range.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pNew            The new RAM range.
+ * @param   GCPhys          The address of the RAM range.
+ * @param   GCPhysLast      The last address of the RAM range.
+ * @param   RCPtrNew        The RC address if the range is floating. NIL_RTRCPTR
+ *                          if in HMA.
+ * @param   R0PtrNew        Ditto for R0.
+ * @param   pszDesc         The description.
+ * @param   pPrev           The previous RAM range (for linking).
+ */
+static void pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
+                                         RTRCPTR RCPtrNew, RTR0PTR R0PtrNew, const char *pszDesc, PPGMRAMRANGE pPrev)
+{
+    /*
+     * Initialize the range.
+     */
+    pNew->pSelfR0       = R0PtrNew != NIL_RTR0PTR ? R0PtrNew : MMHyperCCToR0(pVM, pNew);
+    pNew->pSelfRC       = RCPtrNew != NIL_RTRCPTR ? RCPtrNew : MMHyperCCToRC(pVM, pNew);
+    pNew->GCPhys        = GCPhys;
+    pNew->GCPhysLast    = GCPhysLast;
+    pNew->cb            = GCPhysLast - GCPhys + 1;
+    pNew->pszDesc       = pszDesc;
+    pNew->fFlags        = RCPtrNew != NIL_RTRCPTR ? PGM_RAM_RANGE_FLAGS_FLOATING : 0;
+    pNew->pvR3          = NULL;
+    pNew->paLSPages     = NULL;
+
+    uint32_t const cPages = pNew->cb >> PAGE_SHIFT;
+    RTGCPHYS iPage = cPages;
+    while (iPage-- > 0)
+        PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
+
+    /* Update the page count stats. */
+    pVM->pgm.s.cZeroPages += cPages;
+    pVM->pgm.s.cAllPages  += cPages;
+
+    /*
+     * Link it.
+     */
+    pgmR3PhysLinkRamRange(pVM, pNew, pPrev);
+}
+
+
+/**
+ * Relocate a floating RAM range.
+ *
+ * @copydoc FNPGMRELOCATE.
+ */
+static DECLCALLBACK(bool) pgmR3PhysRamRangeRelocate(PVM pVM, RTGCPTR GCPtrOld, RTGCPTR GCPtrNew, PGMRELOCATECALL enmMode, void *pvUser)
+{
+    PPGMRAMRANGE pRam = (PPGMRAMRANGE)pvUser;
+    Assert(pRam->fFlags & PGM_RAM_RANGE_FLAGS_FLOATING);
+    Assert(pRam->pSelfRC == GCPtrOld + PAGE_SIZE);
+
+    switch (enmMode)
+    {
+        case PGMRELOCATECALL_SUGGEST:
+            return true;
+        case PGMRELOCATECALL_RELOCATE:
+        {
+            /* Update myself and then relink all the ranges. */
+            pgmLock(pVM);
+            pRam->pSelfRC = (RTRCPTR)(GCPtrNew + PAGE_SIZE);
+            pgmR3PhysRelinkRamRanges(pVM);
+            pgmUnlock(pVM);
+            return true;
+        }
+
+        default:
+            AssertFailedReturn(false);
+    }
+}
+
+
+/**
+ * PGMR3PhysRegisterRam worker that registers a high chunk.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   GCPhys          The address of the RAM.
+ * @param   cRamPages       The number of RAM pages to register.
+ * @param   cbChunk         The size of the PGMRAMRANGE guest mapping.
+ * @param   iChunk          The chunk number.
+ * @param   pszDesc         The RAM range description.
+ * @param   ppPrev          Previous RAM range pointer. In/Out.
+ */
+static int pgmR3PhysRegisterHighRamChunk(PVM pVM, RTGCPHYS GCPhys, uint32_t cRamPages,
+                                         uint32_t cbChunk, uint32_t iChunk, const char *pszDesc,
+                                         PPGMRAMRANGE *ppPrev)
+{
+    const char *pszDescChunk = iChunk == 0
+                             ? pszDesc
+                             : MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, iChunk + 1);
+    AssertReturn(pszDescChunk, VERR_NO_MEMORY);
+
+    /*
+     * Allocate memory for the new chunk.
+     */
+    size_t const cChunkPages  = RT_ALIGN_Z(RT_UOFFSETOF(PGMRAMRANGE, aPages[cRamPages]), PAGE_SIZE) >> PAGE_SHIFT;
+    PSUPPAGE     paChunkPages = (PSUPPAGE)RTMemTmpAllocZ(sizeof(SUPPAGE) * cChunkPages);
+    AssertReturn(paChunkPages, VERR_NO_TMP_MEMORY);
+    RTR0PTR      R0PtrChunk   = NIL_RTR0PTR;
+    void        *pvChunk      = NULL;
+    int rc = SUPR3PageAllocEx(cChunkPages, 0 /*fFlags*/, &pvChunk,
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+                              VMMIsHwVirtExtForced(pVM) ? &R0PtrChunk : NULL,
+#else
+                              NULL,
+#endif
+                              paChunkPages);
+    if (RT_SUCCESS(rc))
+    {
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        if (!VMMIsHwVirtExtForced(pVM))
+            R0PtrChunk = NIL_RTR0PTR;
+#else
+        R0PtrChunk = (uintptr_t)pvChunk;
+#endif
+        memset(pvChunk, 0, cChunkPages << PAGE_SHIFT);
+
+        PPGMRAMRANGE pNew = (PPGMRAMRANGE)pvChunk;
+
+        /*
+         * Create a mapping and map the pages into it.
+         * We push these in below the HMA.
+         */
+        RTGCPTR GCPtrChunkMap = pVM->pgm.s.GCPtrPrevRamRangeMapping - cbChunk;
+        rc = PGMR3MapPT(pVM, GCPtrChunkMap, cbChunk, 0 /*fFlags*/, pgmR3PhysRamRangeRelocate, pNew, pszDescChunk);
+        if (RT_SUCCESS(rc))
+        {
+            pVM->pgm.s.GCPtrPrevRamRangeMapping = GCPtrChunkMap;
+
+            RTGCPTR const   GCPtrChunk = GCPtrChunkMap + PAGE_SIZE;
+            RTGCPTR         GCPtrPage  = GCPtrChunk;
+            for (uint32_t iPage = 0; iPage < cChunkPages && RT_SUCCESS(rc); iPage++, GCPtrPage += PAGE_SIZE)
+                rc = PGMMap(pVM, GCPtrPage, paChunkPages[iPage].Phys, PAGE_SIZE, 0);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Ok, init and link the range.
+                 */
+                pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhys + ((RTGCPHYS)cRamPages << PAGE_SHIFT) - 1,
+                                             (RTRCPTR)GCPtrChunk, R0PtrChunk, pszDescChunk, *ppPrev);
+                *ppPrev = pNew;
+            }
+        }
+
+        if (RT_FAILURE(rc))
+            SUPR3PageFreeEx(pvChunk, cChunkPages);
+    }
+
+    RTMemTmpFree(paChunkPages);
+    return rc;
+}
+
+
+/**
+ * Sets up a range RAM.
+ *
+ * This will check for conflicting registrations, make a resource
+ * reservation for the memory (with GMM), and setup the per-page
+ * tracking structures (PGMPAGE).
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   GCPhys          The physical address of the RAM.
+ * @param   cb              The size of the RAM.
+ * @param   pszDesc         The description - not copied, so, don't free or change it.
+ */
+VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
+{
+   /*
+     * Validate input.
+     */
+    Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
+    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
+    AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+
+    pgmLock(pVM);
+
+    /*
+     * Find range location and check for conflicts.
+     * (We don't lock here because the locking by EMT is only required on update.)
+     */
+    PPGMRAMRANGE    pPrev = NULL;
+    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhysLast >= pRam->GCPhys
+            &&  GCPhys     <= pRam->GCPhysLast)
+            AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
+                                         GCPhys, GCPhysLast, pszDesc,
+                                         pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                        VERR_PGM_RAM_CONFLICT);
+
+        /* next */
+        pPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+
+    /*
+     * Register it with GMM (the API bitches).
+     */
+    const RTGCPHYS cPages = cb >> PAGE_SHIFT;
+    int rc = MMR3IncreaseBaseReservation(pVM, cPages);
+    if (RT_FAILURE(rc))
+    {
+        pgmUnlock(pVM);
+        return rc;
+    }
+
+    if (    GCPhys >= _4G
+        &&  cPages > 256)
+    {
+        /*
+         * The PGMRAMRANGE structures for the high memory can get very big.
+         * In order to avoid SUPR3PageAllocEx allocation failures due to the
+         * allocation size limit there and also to avoid being unable to find
+         * guest mapping space for them, we split this memory up into 4MB in
+         * (potential) raw-mode configs and 16MB chunks in forced AMD-V/VT-x
+         * mode.
+         *
+         * The first and last page of each mapping are guard pages and marked
+         * not-present. So, we've got 4186112 and 16769024 bytes available for
+         * the PGMRAMRANGE structure.
+         *
+         * Note! The sizes used here will influence the saved state.
+         */
+        uint32_t cbChunk;
+        uint32_t cPagesPerChunk;
+        if (VMMIsHwVirtExtForced(pVM))
+        {
+            cbChunk = 16U*_1M;
+            cPagesPerChunk = 1048048; /* max ~1048059 */
+            AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 1048048 < 16U*_1M - PAGE_SIZE * 2);
+        }
+        else
+        {
+            cbChunk = 4U*_1M;
+            cPagesPerChunk = 261616; /* max ~261627 */
+            AssertCompile(sizeof(PGMRAMRANGE) + sizeof(PGMPAGE) * 261616  <  4U*_1M - PAGE_SIZE * 2);
+        }
+        AssertRelease(RT_UOFFSETOF(PGMRAMRANGE, aPages[cPagesPerChunk]) + PAGE_SIZE * 2 <= cbChunk);
+
+        RTGCPHYS cPagesLeft  = cPages;
+        RTGCPHYS GCPhysChunk = GCPhys;
+        uint32_t iChunk      = 0;
+        while (cPagesLeft > 0)
+        {
+            uint32_t cPagesInChunk = cPagesLeft;
+            if (cPagesInChunk > cPagesPerChunk)
+                cPagesInChunk = cPagesPerChunk;
+
+            rc = pgmR3PhysRegisterHighRamChunk(pVM, GCPhysChunk, cPagesInChunk, cbChunk, iChunk, pszDesc, &pPrev);
+            AssertRCReturn(rc, rc);
+
+            /* advance */
+            GCPhysChunk += (RTGCPHYS)cPagesInChunk << PAGE_SHIFT;
+            cPagesLeft  -= cPagesInChunk;
+            iChunk++;
+        }
+    }
+    else
+    {
+        /*
+         * Allocate, initialize and link the new RAM range.
+         */
+        const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
+        PPGMRAMRANGE pNew;
+        rc = MMR3HyperAllocOnceNoRel(pVM, cbRamRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
+        AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
+
+        pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhys, GCPhysLast, NIL_RTRCPTR, NIL_RTR0PTR, pszDesc, pPrev);
+    }
+    PGMPhysInvalidatePageMapTLB(pVM);
+    pgmUnlock(pVM);
+
+    /*
+     * Notify REM.
+     */
+    REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_RAM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
+ *
+ * We do this late in the init process so that all the ROM and MMIO ranges have
+ * been registered already and we don't go wasting memory on them.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+int pgmR3PhysRamPreAllocate(PVM pVM)
+{
+    Assert(pVM->pgm.s.fRamPreAlloc);
+    Log(("pgmR3PhysRamPreAllocate: enter\n"));
+
+    /*
+     * Walk the RAM ranges and allocate all RAM pages, halt at
+     * the first allocation error.
+     */
+    uint64_t cPages = 0;
+    uint64_t NanoTS = RTTimeNanoTS();
+    pgmLock(pVM);
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    {
+        PPGMPAGE    pPage  = &pRam->aPages[0];
+        RTGCPHYS    GCPhys = pRam->GCPhys;
+        uint32_t    cLeft  = pRam->cb >> PAGE_SHIFT;
+        while (cLeft-- > 0)
+        {
+            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
+            {
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ZERO:
+                    {
+                        int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
+                        if (RT_FAILURE(rc))
+                        {
+                            LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
+                            pgmUnlock(pVM);
+                            return rc;
+                        }
+                        cPages++;
+                        break;
+                    }
+
+                    case PGM_PAGE_STATE_BALLOONED:
+                    case PGM_PAGE_STATE_ALLOCATED:
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                    case PGM_PAGE_STATE_SHARED:
+                        /* nothing to do here. */
+                        break;
+                }
+            }
+
+            /* next */
+            pPage++;
+            GCPhys += PAGE_SIZE;
+        }
+    }
+    pgmUnlock(pVM);
+    NanoTS = RTTimeNanoTS() - NanoTS;
+
+    LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
+    Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resets (zeros) the RAM.
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+int pgmR3PhysRamReset(PVM pVM)
+{
+    Assert(PGMIsLockOwner(pVM));
+
+    /* Reset the memory balloon. */
+    int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
+    AssertRC(rc);
+
+#ifdef VBOX_WITH_PAGE_SHARING
+    /* Clear all registered shared modules. */
+    rc = GMMR3ResetSharedModules(pVM);
+    AssertRC(rc);
+#endif
+    /* Reset counters. */
+    pVM->pgm.s.cReusedSharedPages = 0;
+    pVM->pgm.s.cBalloonedPages    = 0;
+
+    /*
+     * We batch up pages that should be freed instead of calling GMM for
+     * each and every one of them.
+     */
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Walk the ram ranges.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    {
+        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
+        AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
+
+        if (!pVM->pgm.s.fRamPreAlloc)
+        {
+            /* Replace all RAM pages by ZERO pages. */
+            while (iPage-- > 0)
+            {
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                switch (PGM_PAGE_GET_TYPE(pPage))
+                {
+                    case PGMPAGETYPE_RAM:
+                        /* Do not replace pages part of a 2 MB continuous range with zero pages, but zero them instead. */
+                        if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
+                        {
+                            void *pvPage;
+                            rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
+                            AssertLogRelRCReturn(rc, rc);
+                            ASMMemZeroPage(pvPage);
+                        }
+                        else
+                        if (PGM_PAGE_IS_BALLOONED(pPage))
+                        {
+                            /* Turn into a zero page; the balloon status is lost when the VM reboots. */
+                            PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
+                        }
+                        else
+                        if (!PGM_PAGE_IS_ZERO(pPage))
+                        {
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                            AssertLogRelRCReturn(rc, rc);
+                        }
+                        break;
+
+                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                        pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                                           true /*fDoAccounting*/);
+                        break;
+
+                    case PGMPAGETYPE_MMIO2:
+                    case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
+                    case PGMPAGETYPE_ROM:
+                    case PGMPAGETYPE_MMIO:
+                        break;
+                    default:
+                        AssertFailed();
+                }
+            } /* for each page */
+        }
+        else
+        {
+            /* Zero the memory. */
+            while (iPage-- > 0)
+            {
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                switch (PGM_PAGE_GET_TYPE(pPage))
+                {
+                    case PGMPAGETYPE_RAM:
+                        switch (PGM_PAGE_GET_STATE(pPage))
+                        {
+                            case PGM_PAGE_STATE_ZERO:
+                                break;
+
+                            case PGM_PAGE_STATE_BALLOONED:
+                                /* Turn into a zero page; the balloon status is lost when the VM reboots. */
+                                PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
+                                break;
+
+                            case PGM_PAGE_STATE_SHARED:
+                            case PGM_PAGE_STATE_WRITE_MONITORED:
+                                rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                                AssertLogRelRCReturn(rc, rc);
+                                /* no break */
+
+                            case PGM_PAGE_STATE_ALLOCATED:
+                            {
+                                void *pvPage;
+                                rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pvPage);
+                                AssertLogRelRCReturn(rc, rc);
+                                ASMMemZeroPage(pvPage);
+                                break;
+                            }
+                        }
+                        break;
+
+                    case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                        pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT),
+                                                           true /*fDoAccounting*/);
+                        break;
+
+                    case PGMPAGETYPE_MMIO2:
+                    case PGMPAGETYPE_ROM_SHADOW:
+                    case PGMPAGETYPE_ROM:
+                    case PGMPAGETYPE_MMIO:
+                        break;
+                    default:
+                        AssertFailed();
+
+                }
+            } /* for each page */
+        }
+
+    }
+
+    /*
+     * Finish off any pages pending freeing.
+     */
+    if (cPendingPages)
+    {
+        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    GMMR3FreePagesCleanup(pReq);
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Frees all RAM during VM termination
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+int pgmR3PhysRamTerm(PVM pVM)
+{
+    Assert(PGMIsLockOwner(pVM));
+
+    /* Reset the memory balloon. */
+    int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
+    AssertRC(rc);
+
+#ifdef VBOX_WITH_PAGE_SHARING
+    /* Clear all registered shared modules. */
+    rc = GMMR3ResetSharedModules(pVM);
+    AssertRC(rc);
+#endif
+
+    /*
+     * We batch up pages that should be freed instead of calling GMM for
+     * each and every one of them.
+     */
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    /*
+     * Walk the ram ranges.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    {
+        uint32_t iPage = pRam->cb >> PAGE_SHIFT;
+        AssertMsg(((RTGCPHYS)iPage << PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << PAGE_SHIFT, pRam->cb));
+
+        /* Replace all RAM pages by ZERO pages. */
+        while (iPage-- > 0)
+        {
+            PPGMPAGE pPage = &pRam->aPages[iPage];
+            switch (PGM_PAGE_GET_TYPE(pPage))
+            {
+                case PGMPAGETYPE_RAM:
+                    /* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
+                    if (PGM_PAGE_IS_SHARED(pPage))
+                    {
+                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT));
+                        AssertLogRelRCReturn(rc, rc);
+                    }
+                    break;
+
+                case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                case PGMPAGETYPE_MMIO2:
+                case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
+                case PGMPAGETYPE_ROM:
+                case PGMPAGETYPE_MMIO:
+                    break;
+                default:
+                    AssertFailed();
+            }
+        } /* for each page */
+    }
+
+    /*
+     * Finish off any pages pending freeing.
+     */
+    if (cPendingPages)
+    {
+        rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+        AssertLogRelRCReturn(rc, rc);
+    }
+    GMMR3FreePagesCleanup(pReq);
+    return VINF_SUCCESS;
+}
+
+/**
+ * This is the interface IOM is using to register an MMIO region.
+ *
+ * It will check for conflicts and ensure that a RAM range structure
+ * is present before calling the PGMR3HandlerPhysicalRegister API to
+ * register the callbacks.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   GCPhys          The start of the MMIO region.
+ * @param   cb              The size of the MMIO region.
+ * @param   pfnHandlerR3    The address of the ring-3 handler. (IOMR3MMIOHandler)
+ * @param   pvUserR3        The user argument for R3.
+ * @param   pfnHandlerR0    The address of the ring-0 handler. (IOMMMIOHandler)
+ * @param   pvUserR0        The user argument for R0.
+ * @param   pfnHandlerRC    The address of the RC handler. (IOMMMIOHandler)
+ * @param   pvUserRC        The user argument for RC.
+ * @param   pszDesc         The description of the MMIO region.
+ */
+VMMR3DECL(int) PGMR3PhysMMIORegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                     R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3, RTR3PTR pvUserR3,
+                                     R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0, RTR0PTR pvUserR0,
+                                     RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC, RTRCPTR pvUserRC,
+                                     R3PTRTYPE(const char *) pszDesc)
+{
+    /*
+     * Assert on some assumption.
+     */
+    VM_ASSERT_EMT(pVM);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
+
+    /*
+     * Make sure there's a RAM range structure for the region.
+     */
+    int rc;
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    bool fRamExists = false;
+    PPGMRAMRANGE pRamPrev = NULL;
+    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhysLast >= pRam->GCPhys
+            &&  GCPhys     <= pRam->GCPhysLast)
+        {
+            /* Simplification: all within the same range. */
+            AssertLogRelMsgReturn(   GCPhys     >= pRam->GCPhys
+                                  && GCPhysLast <= pRam->GCPhysLast,
+                                  ("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
+                                   GCPhys, GCPhysLast, pszDesc,
+                                   pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                  VERR_PGM_RAM_CONFLICT);
+
+            /* Check that it's all RAM or MMIO pages. */
+            PCPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+            uint32_t cLeft = cb >> PAGE_SHIFT;
+            while (cLeft-- > 0)
+            {
+                AssertLogRelMsgReturn(   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
+                                      || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO,
+                                      ("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
+                                       GCPhys, GCPhysLast, pszDesc, PGM_PAGE_GET_TYPE(pPage), pRam->pszDesc),
+                                      VERR_PGM_RAM_CONFLICT);
+                pPage++;
+            }
+
+            /* Looks good. */
+            fRamExists = true;
+            break;
+        }
+
+        /* next */
+        pRamPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+    PPGMRAMRANGE pNew;
+    if (fRamExists)
+    {
+        pNew = NULL;
+
+        /*
+         * Make all the pages in the range MMIO/ZERO pages, freeing any
+         * RAM pages currently mapped here. This might not be 100% correct
+         * for PCI memory, but we're doing the same thing for MMIO2 pages.
+         */
+        rc = pgmLock(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = pgmR3PhysFreePageRange(pVM, pRam, GCPhys, GCPhysLast, PGMPAGETYPE_MMIO);
+            pgmUnlock(pVM);
+        }
+        AssertRCReturn(rc, rc);
+
+        /* Force a PGM pool flush as guest ram references have been changed. */
+        /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+    else
+    {
+        pgmLock(pVM);
+
+        /*
+         * No RAM range, insert an ad hoc one.
+         *
+         * Note that we don't have to tell REM about this range because
+         * PGMHandlerPhysicalRegisterEx will do that for us.
+         */
+        Log(("PGMR3PhysMMIORegister: Adding ad hoc MMIO range for %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc));
+
+        const uint32_t cPages = cb >> PAGE_SHIFT;
+        const size_t cbRamRange = RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]);
+        rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), 16, MM_TAG_PGM_PHYS, (void **)&pNew);
+        AssertLogRelMsgRCReturn(rc, ("cbRamRange=%zu\n", cbRamRange), rc);
+
+        /* Initialize the range. */
+        pNew->pSelfR0       = MMHyperCCToR0(pVM, pNew);
+        pNew->pSelfRC       = MMHyperCCToRC(pVM, pNew);
+        pNew->GCPhys        = GCPhys;
+        pNew->GCPhysLast    = GCPhysLast;
+        pNew->cb            = cb;
+        pNew->pszDesc       = pszDesc;
+        pNew->fFlags        = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO;
+        pNew->pvR3          = NULL;
+        pNew->paLSPages     = NULL;
+
+        uint32_t iPage = cPages;
+        while (iPage-- > 0)
+            PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
+        Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
+
+        /* update the page count stats. */
+        pVM->pgm.s.cPureMmioPages += cPages;
+        pVM->pgm.s.cAllPages      += cPages;
+
+        /* link it */
+        pgmR3PhysLinkRamRange(pVM, pNew, pRamPrev);
+
+        pgmUnlock(pVM);
+    }
+
+    /*
+     * Register the access handler.
+     */
+    rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_MMIO, GCPhys, GCPhysLast,
+                                      pfnHandlerR3, pvUserR3,
+                                      pfnHandlerR0, pvUserR0,
+                                      pfnHandlerRC, pvUserRC, pszDesc);
+    if (    RT_FAILURE(rc)
+        &&  !fRamExists)
+    {
+        pVM->pgm.s.cPureMmioPages -= cb >> PAGE_SHIFT;
+        pVM->pgm.s.cAllPages      -= cb >> PAGE_SHIFT;
+
+        /* remove the ad hoc range. */
+        pgmR3PhysUnlinkRamRange2(pVM, pNew, pRamPrev);
+        pNew->cb = pNew->GCPhys = pNew->GCPhysLast = NIL_RTGCPHYS;
+        MMHyperFree(pVM, pRam);
+    }
+    PGMPhysInvalidatePageMapTLB(pVM);
+
+    return rc;
+}
+
+
+/**
+ * This is the interface IOM is using to register an MMIO region.
+ *
+ * It will take care of calling PGMHandlerPhysicalDeregister and clean up
+ * any ad hoc PGMRAMRANGE left behind.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   GCPhys          The start of the MMIO region.
+ * @param   cb              The size of the MMIO region.
+ */
+VMMR3DECL(int) PGMR3PhysMMIODeregister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * First deregister the handler, then check if we should remove the ram range.
+     */
+    int rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
+    if (RT_SUCCESS(rc))
+    {
+        RTGCPHYS        GCPhysLast  = GCPhys + (cb - 1);
+        PPGMRAMRANGE    pRamPrev    = NULL;
+        PPGMRAMRANGE    pRam        = pVM->pgm.s.pRamRangesR3;
+        while (pRam && GCPhysLast >= pRam->GCPhys)
+        {
+            /** @todo We're being a bit too careful here. rewrite. */
+            if (    GCPhysLast == pRam->GCPhysLast
+                &&  GCPhys     == pRam->GCPhys)
+            {
+                Assert(pRam->cb == cb);
+
+                /*
+                 * See if all the pages are dead MMIO pages.
+                 */
+                uint32_t const  cPages   = cb >> PAGE_SHIFT;
+                bool            fAllMMIO = true;
+                uint32_t        iPage    = 0;
+                uint32_t        cLeft    = cPages;
+                while (cLeft-- > 0)
+                {
+                    PPGMPAGE    pPage    = &pRam->aPages[iPage];
+                    if (    PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO
+                        /*|| not-out-of-action later */)
+                    {
+                        fAllMMIO = false;
+                        Assert(PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO2_ALIAS_MMIO);
+                        AssertMsgFailed(("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
+                        break;
+                    }
+                    Assert(PGM_PAGE_IS_ZERO(pPage));
+                    pPage++;
+                }
+                if (fAllMMIO)
+                {
+                    /*
+                     * Ad-hoc range, unlink and free it.
+                     */
+                    Log(("PGMR3PhysMMIODeregister: Freeing ad hoc MMIO range for %RGp-%RGp %s\n",
+                         GCPhys, GCPhysLast, pRam->pszDesc));
+
+                    pVM->pgm.s.cAllPages      -= cPages;
+                    pVM->pgm.s.cPureMmioPages -= cPages;
+
+                    pgmR3PhysUnlinkRamRange2(pVM, pRam, pRamPrev);
+                    pRam->cb = pRam->GCPhys = pRam->GCPhysLast = NIL_RTGCPHYS;
+                    MMHyperFree(pVM, pRam);
+                    break;
+                }
+            }
+
+            /*
+             * Range match? It will all be within one range (see PGMAllHandler.cpp).
+             */
+            if (    GCPhysLast >= pRam->GCPhys
+                &&  GCPhys     <= pRam->GCPhysLast)
+            {
+                Assert(GCPhys     >= pRam->GCPhys);
+                Assert(GCPhysLast <= pRam->GCPhysLast);
+
+                /*
+                 * Turn the pages back into RAM pages.
+                 */
+                uint32_t iPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
+                uint32_t cLeft = cb >> PAGE_SHIFT;
+                while (cLeft--)
+                {
+                    PPGMPAGE pPage = &pRam->aPages[iPage];
+                    AssertMsg(PGM_PAGE_IS_MMIO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
+                    AssertMsg(PGM_PAGE_IS_ZERO(pPage), ("%RGp %R[pgmpage]\n", pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), pPage));
+                    if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
+                        PGM_PAGE_SET_TYPE(pPage, PGMPAGETYPE_RAM);
+                }
+                break;
+            }
+
+            /* next */
+            pRamPrev = pRam;
+            pRam = pRam->pNextR3;
+        }
+    }
+
+    /* Force a PGM pool flush as guest ram references have been changed. */
+    /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    PGMPhysInvalidatePageMapTLB(pVM);
+    return rc;
+}
+
+
+/**
+ * Locate a MMIO2 range.
+ *
+ * @returns Pointer to the MMIO2 range.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   iRegion         The region.
+ */
+DECLINLINE(PPGMMMIO2RANGE) pgmR3PhysMMIO2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
+{
+    /*
+     * Search the list.
+     */
+    for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
+        if (   pCur->pDevInsR3 == pDevIns
+            && pCur->iRegion == iRegion)
+            return pCur;
+    return NULL;
+}
+
+
+/**
+ * Allocate and register an MMIO2 region.
+ *
+ * As mentioned elsewhere, MMIO2 is just RAM spelled differently.  It's RAM
+ * associated with a device. It is also non-shared memory with a permanent
+ * ring-3 mapping and page backing (presently).
+ *
+ * A MMIO2 range may overlap with base memory if a lot of RAM is configured for
+ * the VM, in which case we'll drop the base memory pages.  Presently we will
+ * make no attempt to preserve anything that happens to be present in the base
+ * memory that is replaced, this is of course incorrectly but it's too much
+ * effort.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
+ *          memory.
+ * @retval  VERR_ALREADY_EXISTS if the region already exists.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   iRegion         The region number.  If the MMIO2 memory is a PCI
+ *                          I/O region this number has to be the number of that
+ *                          region. Otherwise it can be any number safe
+ *                          UINT8_MAX.
+ * @param   cb              The size of the region.  Must be page aligned.
+ * @param   fFlags          Reserved for future use, must be zero.
+ * @param   ppv             Where to store the pointer to the ring-3 mapping of
+ *                          the memory.
+ * @param   pszDesc         The description.
+ */
+VMMR3DECL(int) PGMR3PhysMMIO2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS cb, uint32_t fFlags, void **ppv, const char *pszDesc)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(ppv, VERR_INVALID_POINTER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
+    AssertReturn(pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion) == NULL, VERR_ALREADY_EXISTS);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(cb, VERR_INVALID_PARAMETER);
+    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+
+    const uint32_t cPages = cb >> PAGE_SHIFT;
+    AssertLogRelReturn(((RTGCPHYS)cPages << PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
+    AssertLogRelReturn(cPages <= INT32_MAX / 2, VERR_NO_MEMORY);
+
+    /*
+     * For the 2nd+ instance, mangle the description string so it's unique.
+     */
+    if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
+    {
+        pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
+        if (!pszDesc)
+            return VERR_NO_MEMORY;
+    }
+
+    /*
+     * Try reserve and allocate the backing memory first as this is what is
+     * most likely to fail.
+     */
+    int rc = MMR3AdjustFixedReservation(pVM, cPages, pszDesc);
+    if (RT_SUCCESS(rc))
+    {
+        void *pvPages;
+        PSUPPAGE paPages = (PSUPPAGE)RTMemTmpAlloc(cPages * sizeof(SUPPAGE));
+        if (RT_SUCCESS(rc))
+            rc = SUPR3PageAllocEx(cPages, 0 /*fFlags*/, &pvPages, NULL /*pR0Ptr*/, paPages);
+        if (RT_SUCCESS(rc))
+        {
+            memset(pvPages, 0, cPages * PAGE_SIZE);
+
+            /*
+             * Create the MMIO2 range record for it.
+             */
+            const size_t cbRange = RT_OFFSETOF(PGMMMIO2RANGE, RamRange.aPages[cPages]);
+            PPGMMMIO2RANGE pNew;
+            rc = MMR3HyperAllocOnceNoRel(pVM, cbRange, 0, MM_TAG_PGM_PHYS, (void **)&pNew);
+            AssertLogRelMsgRC(rc, ("cbRamRange=%zu\n", cbRange));
+            if (RT_SUCCESS(rc))
+            {
+                pNew->pDevInsR3             = pDevIns;
+                pNew->pvR3                  = pvPages;
+                //pNew->pNext               = NULL;
+                //pNew->fMapped             = false;
+                //pNew->fOverlapping        = false;
+                pNew->iRegion               = iRegion;
+                pNew->idSavedState          = UINT8_MAX;
+                pNew->RamRange.pSelfR0      = MMHyperCCToR0(pVM, &pNew->RamRange);
+                pNew->RamRange.pSelfRC      = MMHyperCCToRC(pVM, &pNew->RamRange);
+                pNew->RamRange.GCPhys       = NIL_RTGCPHYS;
+                pNew->RamRange.GCPhysLast   = NIL_RTGCPHYS;
+                pNew->RamRange.pszDesc      = pszDesc;
+                pNew->RamRange.cb           = cb;
+                pNew->RamRange.fFlags       = PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2;
+                pNew->RamRange.pvR3         = pvPages;
+                //pNew->RamRange.paLSPages    = NULL;
+
+                uint32_t iPage = cPages;
+                while (iPage-- > 0)
+                {
+                    PGM_PAGE_INIT(&pNew->RamRange.aPages[iPage],
+                                  paPages[iPage].Phys, NIL_GMM_PAGEID,
+                                  PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
+                }
+
+                /* update page count stats */
+                pVM->pgm.s.cAllPages     += cPages;
+                pVM->pgm.s.cPrivatePages += cPages;
+
+                /*
+                 * Link it into the list.
+                 * Since there is no particular order, just push it.
+                 */
+                pgmLock(pVM);
+                pNew->pNextR3 = pVM->pgm.s.pMmio2RangesR3;
+                pVM->pgm.s.pMmio2RangesR3 = pNew;
+                pgmUnlock(pVM);
+
+                *ppv = pvPages;
+                RTMemTmpFree(paPages);
+                PGMPhysInvalidatePageMapTLB(pVM);
+                return VINF_SUCCESS;
+            }
+
+            SUPR3PageFreeEx(pvPages, cPages);
+        }
+        RTMemTmpFree(paPages);
+        MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pszDesc);
+    }
+    if (pDevIns->iInstance > 0)
+        MMR3HeapFree((void *)pszDesc);
+    return rc;
+}
+
+
+/**
+ * Deregisters and frees an MMIO2 region.
+ *
+ * Any physical (and virtual) access handlers registered for the region must
+ * be deregistered before calling this function.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The device instance owning the region.
+ * @param   iRegion         The region. If it's UINT32_MAX it'll be a wildcard match.
+ */
+VMMR3DECL(int) PGMR3PhysMMIO2Deregister(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX || iRegion == UINT32_MAX, VERR_INVALID_PARAMETER);
+
+    pgmLock(pVM);
+    int rc = VINF_SUCCESS;
+    unsigned cFound = 0;
+    PPGMMMIO2RANGE pPrev = NULL;
+    PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3;
+    while (pCur)
+    {
+        if (    pCur->pDevInsR3 == pDevIns
+            &&  (   iRegion == UINT32_MAX
+                 || pCur->iRegion == iRegion))
+        {
+            cFound++;
+
+            /*
+             * Unmap it if it's mapped.
+             */
+            if (pCur->fMapped)
+            {
+                int rc2 = PGMR3PhysMMIO2Unmap(pVM, pCur->pDevInsR3, pCur->iRegion, pCur->RamRange.GCPhys);
+                AssertRC(rc2);
+                if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                    rc = rc2;
+            }
+
+            /*
+             * Unlink it
+             */
+            PPGMMMIO2RANGE pNext = pCur->pNextR3;
+            if (pPrev)
+                pPrev->pNextR3 = pNext;
+            else
+                pVM->pgm.s.pMmio2RangesR3 = pNext;
+            pCur->pNextR3 = NULL;
+
+            /*
+             * Free the memory.
+             */
+            int rc2 = SUPR3PageFreeEx(pCur->pvR3, pCur->RamRange.cb >> PAGE_SHIFT);
+            AssertRC(rc2);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+
+            uint32_t const cPages = pCur->RamRange.cb >> PAGE_SHIFT;
+            rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cPages, pCur->RamRange.pszDesc);
+            AssertRC(rc2);
+            if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
+                rc = rc2;
+
+            /* we're leaking hyper memory here if done at runtime. */
+#ifdef VBOX_STRICT
+            VMSTATE const enmState = VMR3GetState(pVM);
+            AssertMsg(   enmState == VMSTATE_POWERING_OFF
+                      || enmState == VMSTATE_POWERING_OFF_LS
+                      || enmState == VMSTATE_OFF
+                      || enmState == VMSTATE_OFF_LS
+                      || enmState == VMSTATE_DESTROYING
+                      || enmState == VMSTATE_TERMINATED
+                      || enmState == VMSTATE_CREATING
+                      , ("%s\n", VMR3GetStateName(enmState)));
+#endif
+            /*rc = MMHyperFree(pVM, pCur);
+            AssertRCReturn(rc, rc); - not safe, see the alloc call. */
+
+
+            /* update page count stats */
+            pVM->pgm.s.cAllPages     -= cPages;
+            pVM->pgm.s.cPrivatePages -= cPages;
+
+            /* next */
+            pCur = pNext;
+        }
+        else
+        {
+            pPrev = pCur;
+            pCur = pCur->pNextR3;
+        }
+    }
+    PGMPhysInvalidatePageMapTLB(pVM);
+    pgmUnlock(pVM);
+    return !cFound && iRegion != UINT32_MAX ? VERR_NOT_FOUND : rc;
+}
+
+
+/**
+ * Maps a MMIO2 region.
+ *
+ * This is done when a guest / the bios / state loading changes the
+ * PCI config. The replacing of base memory has the same restrictions
+ * as during registration, of course.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The
+ */
+VMMR3DECL(int) PGMR3PhysMMIO2Map(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+    AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
+    AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+
+    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
+    AssertReturn(pCur, VERR_NOT_FOUND);
+    AssertReturn(!pCur->fMapped, VERR_WRONG_ORDER);
+    Assert(pCur->RamRange.GCPhys == NIL_RTGCPHYS);
+    Assert(pCur->RamRange.GCPhysLast == NIL_RTGCPHYS);
+
+    const RTGCPHYS GCPhysLast = GCPhys + pCur->RamRange.cb - 1;
+    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
+
+    /*
+     * Find our location in the ram range list, checking for
+     * restriction we don't bother implementing yet (partially overlapping).
+     */
+    bool fRamExists = false;
+    PPGMRAMRANGE pRamPrev = NULL;
+    PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhys     <= pRam->GCPhysLast
+            &&  GCPhysLast >= pRam->GCPhys)
+        {
+            /* completely within? */
+            AssertLogRelMsgReturn(   GCPhys     >= pRam->GCPhys
+                                  && GCPhysLast <= pRam->GCPhysLast,
+                                  ("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
+                                   GCPhys, GCPhysLast, pCur->RamRange.pszDesc,
+                                   pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                  VERR_PGM_RAM_CONFLICT);
+            fRamExists = true;
+            break;
+        }
+
+        /* next */
+        pRamPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+    if (fRamExists)
+    {
+        PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
+        while (cPagesLeft-- > 0)
+        {
+            AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
+                                  ("%RGp isn't a RAM page (%d) - mapping %RGp-%RGp (MMIO2/%s).\n",
+                                   GCPhys, PGM_PAGE_GET_TYPE(pPage), GCPhys, GCPhysLast, pCur->RamRange.pszDesc),
+                                  VERR_PGM_RAM_CONFLICT);
+            pPage++;
+        }
+    }
+    Log(("PGMR3PhysMMIO2Map: %RGp-%RGp fRamExists=%RTbool %s\n",
+         GCPhys, GCPhysLast, fRamExists, pCur->RamRange.pszDesc));
+
+    /*
+     * Make the changes.
+     */
+    pgmLock(pVM);
+
+    pCur->RamRange.GCPhys = GCPhys;
+    pCur->RamRange.GCPhysLast = GCPhysLast;
+    pCur->fMapped = true;
+    pCur->fOverlapping = fRamExists;
+
+    if (fRamExists)
+    {
+/** @todo use pgmR3PhysFreePageRange here. */
+        uint32_t            cPendingPages = 0;
+        PGMMFREEPAGESREQ    pReq;
+        int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+        AssertLogRelRCReturn(rc, rc);
+
+        /* replace the pages, freeing all present RAM pages. */
+        PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
+        PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
+        while (cPagesLeft-- > 0)
+        {
+            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys);
+            AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
+
+            RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
+            PGM_PAGE_SET_HCPHYS(pPageDst, HCPhys);
+            PGM_PAGE_SET_TYPE(pPageDst, PGMPAGETYPE_MMIO2);
+            PGM_PAGE_SET_STATE(pPageDst, PGM_PAGE_STATE_ALLOCATED);
+            PGM_PAGE_SET_PDE_TYPE(pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
+            PGM_PAGE_SET_PTE_INDEX(pPageDst, 0);
+            PGM_PAGE_SET_TRACKING(pPageDst, 0);
+
+            pVM->pgm.s.cZeroPages--;
+            GCPhys += PAGE_SIZE;
+            pPageSrc++;
+            pPageDst++;
+        }
+
+        /* Flush physical page map TLB. */
+        PGMPhysInvalidatePageMapTLB(pVM);
+
+        if (cPendingPages)
+        {
+            rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+            AssertLogRelRCReturn(rc, rc);
+        }
+        GMMR3FreePagesCleanup(pReq);
+
+        /* Force a PGM pool flush as guest ram references have been changed. */
+        /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+        pgmUnlock(pVM);
+    }
+    else
+    {
+        RTGCPHYS cb = pCur->RamRange.cb;
+
+        /* Clear the tracking data of pages we're going to reactivate. */
+        PPGMPAGE pPageSrc = &pCur->RamRange.aPages[0];
+        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
+        while (cPagesLeft-- > 0)
+        {
+            PGM_PAGE_SET_TRACKING(pPageSrc, 0);
+            PGM_PAGE_SET_PTE_INDEX(pPageSrc, 0);
+            pPageSrc++;
+        }
+
+        /* link in the ram range */
+        pgmR3PhysLinkRamRange(pVM, &pCur->RamRange, pRamPrev);
+        pgmUnlock(pVM);
+
+        REMR3NotifyPhysRamRegister(pVM, GCPhys, cb, REM_NOTIFY_PHYS_RAM_FLAGS_MMIO2);
+    }
+
+    PGMPhysInvalidatePageMapTLB(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unmaps a MMIO2 region.
+ *
+ * This is done when a guest / the bios / state loading changes the
+ * PCI config. The replacing of base memory has the same restrictions
+ * as during registration, of course.
+ */
+VMMR3DECL(int) PGMR3PhysMMIO2Unmap(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS GCPhys)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+    AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
+    AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+
+    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
+    AssertReturn(pCur, VERR_NOT_FOUND);
+    AssertReturn(pCur->fMapped, VERR_WRONG_ORDER);
+    AssertReturn(pCur->RamRange.GCPhys == GCPhys, VERR_INVALID_PARAMETER);
+    Assert(pCur->RamRange.GCPhysLast != NIL_RTGCPHYS);
+
+    Log(("PGMR3PhysMMIO2Unmap: %RGp-%RGp %s\n",
+         pCur->RamRange.GCPhys, pCur->RamRange.GCPhysLast, pCur->RamRange.pszDesc));
+
+    /*
+     * Unmap it.
+     */
+    pgmLock(pVM);
+
+    RTGCPHYS    GCPhysRangeREM;
+    RTGCPHYS    cbRangeREM;
+    bool        fInformREM;
+    if (pCur->fOverlapping)
+    {
+        /* Restore the RAM pages we've replaced. */
+        PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+        while (pRam->GCPhys > pCur->RamRange.GCPhysLast)
+            pRam = pRam->pNextR3;
+
+        PPGMPAGE pPageDst = &pRam->aPages[(pCur->RamRange.GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+        uint32_t cPagesLeft = pCur->RamRange.cb >> PAGE_SHIFT;
+        while (cPagesLeft-- > 0)
+        {
+            PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
+            pVM->pgm.s.cZeroPages++;
+            pPageDst++;
+        }
+
+        /* Flush physical page map TLB. */
+        PGMPhysInvalidatePageMapTLB(pVM);
+
+        GCPhysRangeREM = NIL_RTGCPHYS;  /* shuts up gcc */
+        cbRangeREM     = RTGCPHYS_MAX;  /* ditto */
+        fInformREM     = false;
+    }
+    else
+    {
+        GCPhysRangeREM = pCur->RamRange.GCPhys;
+        cbRangeREM     = pCur->RamRange.cb;
+        fInformREM     = true;
+
+        pgmR3PhysUnlinkRamRange(pVM, &pCur->RamRange);
+    }
+
+    pCur->RamRange.GCPhys = NIL_RTGCPHYS;
+    pCur->RamRange.GCPhysLast = NIL_RTGCPHYS;
+    pCur->fOverlapping = false;
+    pCur->fMapped = false;
+
+    /* Force a PGM pool flush as guest ram references have been changed. */
+    /** todo; not entirely SMP safe; assuming for now the guest takes care of this internally (not touch mapped mmio while changing the mapping). */
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    PGMPhysInvalidatePageMapTLB(pVM);
+    pgmUnlock(pVM);
+
+    if (fInformREM)
+        REMR3NotifyPhysRamDeregister(pVM, GCPhysRangeREM, cbRangeREM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the given address is an MMIO2 base address or not.
+ *
+ * @returns true/false accordingly.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The owner of the memory, optional.
+ * @param   GCPhys          The address to check.
+ */
+VMMR3DECL(bool) PGMR3PhysMMIO2IsBase(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, false);
+    AssertPtrReturn(pDevIns, false);
+    AssertReturn(GCPhys != NIL_RTGCPHYS, false);
+    AssertReturn(GCPhys != 0, false);
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), false);
+
+    /*
+     * Search the list.
+     */
+    pgmLock(pVM);
+    for (PPGMMMIO2RANGE pCur = pVM->pgm.s.pMmio2RangesR3; pCur; pCur = pCur->pNextR3)
+        if (pCur->RamRange.GCPhys == GCPhys)
+        {
+            Assert(pCur->fMapped);
+            pgmUnlock(pVM);
+            return true;
+        }
+    pgmUnlock(pVM);
+    return false;
+}
+
+
+/**
+ * Gets the HC physical address of a page in the MMIO2 region.
+ *
+ * This is API is intended for MMHyper and shouldn't be called
+ * by anyone else...
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pDevIns         The owner of the memory, optional.
+ * @param   iRegion         The region.
+ * @param   off             The page expressed an offset into the MMIO2 region.
+ * @param   pHCPhys         Where to store the result.
+ */
+VMMR3DECL(int) PGMR3PhysMMIO2GetHCPhys(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, PRTHCPHYS pHCPhys)
+{
+    /*
+     * Validate input
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+
+    pgmLock(pVM);
+    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
+    AssertReturn(pCur, VERR_NOT_FOUND);
+    AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
+
+    PCPGMPAGE pPage = &pCur->RamRange.aPages[off >> PAGE_SHIFT];
+    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Maps a portion of an MMIO2 region into kernel space (host).
+ *
+ * The kernel mapping will become invalid when the MMIO2 memory is deregistered
+ * or the VM is terminated.
+ *
+ * @return VBox status code.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   pDevIns     The device owning the MMIO2 memory.
+ * @param   iRegion     The region.
+ * @param   off         The offset into the region. Must be page aligned.
+ * @param   cb          The number of bytes to map. Must be page aligned.
+ * @param   pszDesc     Mapping description.
+ * @param   pR0Ptr      Where to store the R0 address.
+ */
+VMMR3DECL(int) PGMR3PhysMMIO2MapKernel(PVM pVM, PPDMDEVINS pDevIns, uint32_t iRegion, RTGCPHYS off, RTGCPHYS cb,
+                                       const char *pszDesc, PRTR0PTR pR0Ptr)
+{
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
+
+    PPGMMMIO2RANGE pCur = pgmR3PhysMMIO2Find(pVM, pDevIns, iRegion);
+    AssertReturn(pCur, VERR_NOT_FOUND);
+    AssertReturn(off < pCur->RamRange.cb, VERR_INVALID_PARAMETER);
+    AssertReturn(cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
+    AssertReturn(off + cb <= pCur->RamRange.cb, VERR_INVALID_PARAMETER);
+
+    /*
+     * Pass the request on to the support library/driver.
+     */
+    int rc = SUPR3PageMapKernel(pCur->pvR3, off, cb, 0, pR0Ptr);
+
+    return rc;
+}
+
+
+/**
+ * Registers a ROM image.
+ *
+ * Shadowed ROM images requires double the amount of backing memory, so,
+ * don't use that unless you have to. Shadowing of ROM images is process
+ * where we can select where the reads go and where the writes go. On real
+ * hardware the chipset provides means to configure this. We provide
+ * PGMR3PhysProtectROM() for this purpose.
+ *
+ * A read-only copy of the ROM image will always be kept around while we
+ * will allocate RAM pages for the changes on demand (unless all memory
+ * is configured to be preallocated).
+ *
+ * @returns VBox status.
+ * @param   pVM                 VM Handle.
+ * @param   pDevIns             The device instance owning the ROM.
+ * @param   GCPhys              First physical address in the range.
+ *                              Must be page aligned!
+ * @param   cb                  The size of the range (in bytes).
+ *                              Must be page aligned!
+ * @param   pvBinary            Pointer to the binary data backing the ROM image.
+ * @param   cbBinary            The size of the binary data pvBinary points to.
+ *                              This must be less or equal to @a cb.
+ * @param   fFlags              Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
+ *                              and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
+ * @param   pszDesc             Pointer to description string. This must not be freed.
+ *
+ * @remark  There is no way to remove the rom, automatically on device cleanup or
+ *          manually from the device yet. This isn't difficult in any way, it's
+ *          just not something we expect to be necessary for a while.
+ */
+VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                    const void *pvBinary, uint32_t cbBinary, uint32_t fFlags, const char *pszDesc)
+{
+    Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
+         pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(GCPhys, PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(RT_ALIGN_T(cb, PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
+    AssertReturn(!(fFlags & ~(PGMPHYS_ROM_FLAGS_SHADOWED | PGMPHYS_ROM_FLAGS_PERMANENT_BINARY)), VERR_INVALID_PARAMETER);
+    VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
+
+    const uint32_t cPages = cb >> PAGE_SHIFT;
+
+    /*
+     * Find the ROM location in the ROM list first.
+     */
+    PPGMROMRANGE    pRomPrev = NULL;
+    PPGMROMRANGE    pRom = pVM->pgm.s.pRomRangesR3;
+    while (pRom && GCPhysLast >= pRom->GCPhys)
+    {
+        if (    GCPhys     <= pRom->GCPhysLast
+            &&  GCPhysLast >= pRom->GCPhys)
+            AssertLogRelMsgFailedReturn(("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
+                                         GCPhys, GCPhysLast, pszDesc,
+                                         pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
+                                        VERR_PGM_RAM_CONFLICT);
+        /* next */
+        pRomPrev = pRom;
+        pRom = pRom->pNextR3;
+    }
+
+    /*
+     * Find the RAM location and check for conflicts.
+     *
+     * Conflict detection is a bit different than for RAM
+     * registration since a ROM can be located within a RAM
+     * range. So, what we have to check for is other memory
+     * types (other than RAM that is) and that we don't span
+     * more than one RAM range (layz).
+     */
+    bool            fRamExists = false;
+    PPGMRAMRANGE    pRamPrev = NULL;
+    PPGMRAMRANGE    pRam = pVM->pgm.s.pRamRangesR3;
+    while (pRam && GCPhysLast >= pRam->GCPhys)
+    {
+        if (    GCPhys     <= pRam->GCPhysLast
+            &&  GCPhysLast >= pRam->GCPhys)
+        {
+            /* completely within? */
+            AssertLogRelMsgReturn(   GCPhys     >= pRam->GCPhys
+                                  && GCPhysLast <= pRam->GCPhysLast,
+                                  ("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
+                                   GCPhys, GCPhysLast, pszDesc,
+                                   pRam->GCPhys, pRam->GCPhysLast, pRam->pszDesc),
+                                  VERR_PGM_RAM_CONFLICT);
+            fRamExists = true;
+            break;
+        }
+
+        /* next */
+        pRamPrev = pRam;
+        pRam = pRam->pNextR3;
+    }
+    if (fRamExists)
+    {
+        PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+        uint32_t cPagesLeft = cPages;
+        while (cPagesLeft-- > 0)
+        {
+            AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
+                                  ("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
+                                   pRam->GCPhys + ((RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << PAGE_SHIFT),
+                                   pPage, GCPhys, GCPhysLast, pszDesc), VERR_PGM_RAM_CONFLICT);
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+            pPage++;
+        }
+    }
+
+    /*
+     * Update the base memory reservation if necessary.
+     */
+    uint32_t cExtraBaseCost = fRamExists ? 0 : cPages;
+    if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        cExtraBaseCost += cPages;
+    if (cExtraBaseCost)
+    {
+        int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Allocate memory for the virgin copy of the RAM.
+     */
+    PGMMALLOCATEPAGESREQ pReq;
+    int rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cPages, GMMACCOUNT_BASE);
+    AssertRCReturn(rc, rc);
+
+    for (uint32_t iPage = 0; iPage < cPages; iPage++)
+    {
+        pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << PAGE_SHIFT);
+        pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
+        pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
+    }
+
+    pgmLock(pVM);
+    rc = GMMR3AllocatePagesPerform(pVM, pReq);
+    pgmUnlock(pVM);
+    if (RT_FAILURE(rc))
+    {
+        GMMR3AllocatePagesCleanup(pReq);
+        return rc;
+    }
+
+    /*
+     * Allocate the new ROM range and RAM range (if necessary).
+     */
+    PPGMROMRANGE pRomNew;
+    rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMROMRANGE, aPages[cPages]), 0, MM_TAG_PGM_PHYS, (void **)&pRomNew);
+    if (RT_SUCCESS(rc))
+    {
+        PPGMRAMRANGE pRamNew = NULL;
+        if (!fRamExists)
+            rc = MMHyperAlloc(pVM, RT_OFFSETOF(PGMRAMRANGE, aPages[cPages]), sizeof(PGMPAGE), MM_TAG_PGM_PHYS, (void **)&pRamNew);
+        if (RT_SUCCESS(rc))
+        {
+            pgmLock(pVM);
+
+            /*
+             * Initialize and insert the RAM range (if required).
+             */
+            PPGMROMPAGE pRomPage = &pRomNew->aPages[0];
+            if (!fRamExists)
+            {
+                pRamNew->pSelfR0       = MMHyperCCToR0(pVM, pRamNew);
+                pRamNew->pSelfRC       = MMHyperCCToRC(pVM, pRamNew);
+                pRamNew->GCPhys        = GCPhys;
+                pRamNew->GCPhysLast    = GCPhysLast;
+                pRamNew->cb            = cb;
+                pRamNew->pszDesc       = pszDesc;
+                pRamNew->fFlags        = PGM_RAM_RANGE_FLAGS_AD_HOC_ROM;
+                pRamNew->pvR3          = NULL;
+                pRamNew->paLSPages     = NULL;
+
+                PPGMPAGE pPage = &pRamNew->aPages[0];
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
+                {
+                    PGM_PAGE_INIT(pPage,
+                                  pReq->aPages[iPage].HCPhysGCPhys,
+                                  pReq->aPages[iPage].idPage,
+                                  PGMPAGETYPE_ROM,
+                                  PGM_PAGE_STATE_ALLOCATED);
+
+                    pRomPage->Virgin = *pPage;
+                }
+
+                pVM->pgm.s.cAllPages += cPages;
+                pgmR3PhysLinkRamRange(pVM, pRamNew, pRamPrev);
+            }
+            else
+            {
+                PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pPage++, pRomPage++)
+                {
+                    PGM_PAGE_SET_TYPE(pPage,   PGMPAGETYPE_ROM);
+                    PGM_PAGE_SET_HCPHYS(pPage, pReq->aPages[iPage].HCPhysGCPhys);
+                    PGM_PAGE_SET_STATE(pPage,  PGM_PAGE_STATE_ALLOCATED);
+                    PGM_PAGE_SET_PAGEID(pPage, pReq->aPages[iPage].idPage);
+                    PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
+                    PGM_PAGE_SET_PTE_INDEX(pPage, 0);
+                    PGM_PAGE_SET_TRACKING(pPage, 0);
+
+                    pRomPage->Virgin = *pPage;
+                }
+
+                pRamNew = pRam;
+
+                pVM->pgm.s.cZeroPages -= cPages;
+            }
+            pVM->pgm.s.cPrivatePages += cPages;
+
+            /* Flush physical page map TLB. */
+            PGMPhysInvalidatePageMapTLB(pVM);
+
+            pgmUnlock(pVM);
+
+
+            /*
+             * !HACK ALERT!  REM + (Shadowed) ROM ==> mess.
+             *
+             * If it's shadowed we'll register the handler after the ROM notification
+             * so we get the access handler callbacks that we should. If it isn't
+             * shadowed we'll do it the other way around to make REM use the built-in
+             * ROM behavior and not the handler behavior (which is to route all access
+             * to PGM atm).
+             */
+            if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+            {
+                REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, true /* fShadowed */);
+                rc = PGMR3HandlerPhysicalRegister(pVM,
+                                                  fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
+                                                  ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
+                                                  : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
+                                                  GCPhys, GCPhysLast,
+                                                  pgmR3PhysRomWriteHandler, pRomNew,
+                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
+                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
+            }
+            else
+            {
+                rc = PGMR3HandlerPhysicalRegister(pVM,
+                                                  fFlags & PGMPHYS_ROM_FLAGS_SHADOWED
+                                                  ? PGMPHYSHANDLERTYPE_PHYSICAL_ALL
+                                                  : PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
+                                                  GCPhys, GCPhysLast,
+                                                  pgmR3PhysRomWriteHandler, pRomNew,
+                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToR0(pVM, pRomNew),
+                                                  NULL, "pgmPhysRomWriteHandler", MMHyperCCToRC(pVM, pRomNew), pszDesc);
+                REMR3NotifyPhysRomRegister(pVM, GCPhys, cb, NULL, false /* fShadowed */);
+            }
+            if (RT_SUCCESS(rc))
+            {
+                pgmLock(pVM);
+
+                /*
+                 * Copy the image over to the virgin pages.
+                 * This must be done after linking in the RAM range.
+                 */
+                size_t          cbBinaryLeft = cbBinary;
+                PPGMPAGE        pRamPage     = &pRamNew->aPages[(GCPhys - pRamNew->GCPhys) >> PAGE_SHIFT];
+                for (uint32_t iPage = 0; iPage < cPages; iPage++, pRamPage++)
+                {
+                    void *pvDstPage;
+                    rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << PAGE_SHIFT), &pvDstPage);
+                    if (RT_FAILURE(rc))
+                    {
+                        VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
+                        break;
+                    }
+                    if (cbBinaryLeft >= PAGE_SIZE)
+                    {
+                        memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), PAGE_SIZE);
+                        cbBinaryLeft -= PAGE_SIZE;
+                    }
+                    else
+                    {
+                        ASMMemZeroPage(pvDstPage); /* (shouldn't be necessary, but can't hurt either) */
+                        if (cbBinaryLeft > 0)
+                        {
+                            memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << PAGE_SHIFT), cbBinaryLeft);
+                            cbBinaryLeft = 0;
+                        }
+                    }
+                }
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Initialize the ROM range.
+                     * Note that the Virgin member of the pages has already been initialized above.
+                     */
+                    pRomNew->GCPhys     = GCPhys;
+                    pRomNew->GCPhysLast = GCPhysLast;
+                    pRomNew->cb         = cb;
+                    pRomNew->fFlags     = fFlags;
+                    pRomNew->idSavedState = UINT8_MAX;
+                    pRomNew->cbOriginal = cbBinary;
+#ifdef VBOX_STRICT
+                    pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
+                                        ? pvBinary : RTMemDup(pvBinary, cbBinary);
+#else
+                    pRomNew->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY ? pvBinary : NULL;
+#endif
+                    pRomNew->pszDesc    = pszDesc;
+
+                    for (unsigned iPage = 0; iPage < cPages; iPage++)
+                    {
+                        PPGMROMPAGE pPage = &pRomNew->aPages[iPage];
+                        pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
+                        PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
+                    }
+
+                    /* update the page count stats for the shadow pages. */
+                    if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+                    {
+                        pVM->pgm.s.cZeroPages += cPages;
+                        pVM->pgm.s.cAllPages  += cPages;
+                    }
+
+                    /*
+                     * Insert the ROM range, tell REM and return successfully.
+                     */
+                    pRomNew->pNextR3 = pRom;
+                    pRomNew->pNextR0 = pRom ? MMHyperCCToR0(pVM, pRom) : NIL_RTR0PTR;
+                    pRomNew->pNextRC = pRom ? MMHyperCCToRC(pVM, pRom) : NIL_RTRCPTR;
+
+                    if (pRomPrev)
+                    {
+                        pRomPrev->pNextR3 = pRomNew;
+                        pRomPrev->pNextR0 = MMHyperCCToR0(pVM, pRomNew);
+                        pRomPrev->pNextRC = MMHyperCCToRC(pVM, pRomNew);
+                    }
+                    else
+                    {
+                        pVM->pgm.s.pRomRangesR3 = pRomNew;
+                        pVM->pgm.s.pRomRangesR0 = MMHyperCCToR0(pVM, pRomNew);
+                        pVM->pgm.s.pRomRangesRC = MMHyperCCToRC(pVM, pRomNew);
+                    }
+
+                    PGMPhysInvalidatePageMapTLB(pVM);
+                    GMMR3AllocatePagesCleanup(pReq);
+                    pgmUnlock(pVM);
+                    return VINF_SUCCESS;
+                }
+
+                /* bail out */
+
+                pgmUnlock(pVM);
+                int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
+                AssertRC(rc2);
+                pgmLock(pVM);
+            }
+
+            if (!fRamExists)
+            {
+                pgmR3PhysUnlinkRamRange2(pVM, pRamNew, pRamPrev);
+                MMHyperFree(pVM, pRamNew);
+            }
+        }
+        MMHyperFree(pVM, pRomNew);
+    }
+
+    /** @todo Purge the mapping cache or something... */
+    GMMR3FreeAllocatedPages(pVM, pReq);
+    GMMR3AllocatePagesCleanup(pReq);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * \#PF Handler callback for ROM write accesses.
+ *
+ * @returns VINF_SUCCESS if the handler have carried out the operation.
+ * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
+ * @param   pVM             VM Handle.
+ * @param   GCPhys          The physical address the guest is writing to.
+ * @param   pvPhys          The HC mapping of that address.
+ * @param   pvBuf           What the guest is reading/writing.
+ * @param   cbBuf           How much it's reading/writing.
+ * @param   enmAccessType   The access type.
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) pgmR3PhysRomWriteHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                                                  PGMACCESSTYPE enmAccessType, void *pvUser)
+{
+    PPGMROMRANGE    pRom     = (PPGMROMRANGE)pvUser;
+    const uint32_t  iPage    = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
+    Assert(iPage < (pRom->cb >> PAGE_SHIFT));
+    PPGMROMPAGE     pRomPage = &pRom->aPages[iPage];
+    Log5(("pgmR3PhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
+
+    if (enmAccessType == PGMACCESSTYPE_READ)
+    {
+        switch (pRomPage->enmProt)
+        {
+            /*
+             * Take the default action.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+            case PGMROMPROT_READ_ROM_WRITE_RAM:
+            case PGMROMPROT_READ_RAM_WRITE_RAM:
+                return VINF_PGM_HANDLER_DO_DEFAULT;
+
+            default:
+                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
+                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
+                                      VERR_INTERNAL_ERROR);
+        }
+    }
+    else
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+        switch (pRomPage->enmProt)
+        {
+            /*
+             * Ignore writes.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+                return VINF_SUCCESS;
+
+            /*
+             * Write to the RAM page.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_RAM:
+            case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
+            {
+                /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
+                Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> PAGE_SHIFT) == iPage);
+
+                /*
+                 * Take the lock, do lazy allocation, map the page and copy the data.
+                 *
+                 * Note that we have to bypass the mapping TLB since it works on
+                 * guest physical addresses and entering the shadow page would
+                 * kind of screw things up...
+                 */
+                int rc = pgmLock(pVM);
+                AssertRC(rc);
+
+                PPGMPAGE pShadowPage = &pRomPage->Shadow;
+                if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
+                {
+                    pShadowPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+                    AssertLogRelReturn(pShadowPage, VERR_INTERNAL_ERROR);
+                }
+
+                void *pvDstPage;
+                rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
+                if (RT_SUCCESS(rc))
+                {
+                    memcpy((uint8_t *)pvDstPage + (GCPhys & PAGE_OFFSET_MASK), pvBuf, cbBuf);
+                    pRomPage->LiveSave.fWrittenTo = true;
+                }
+
+                pgmUnlock(pVM);
+                return rc;
+            }
+
+            default:
+                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
+                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
+                                      VERR_INTERNAL_ERROR);
+        }
+    }
+}
+
+
+/**
+ * Called by PGMR3Reset to reset the shadow, switch to the virgin,
+ * and verify that the virgin part is untouched.
+ *
+ * This is done after the normal memory has been cleared.
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @param   pVM         The VM handle.
+ */
+int pgmR3PhysRomReset(PVM pVM)
+{
+    Assert(PGMIsLockOwner(pVM));
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        const uint32_t cPages = pRom->cb >> PAGE_SHIFT;
+
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        {
+            /*
+             * Reset the physical handler.
+             */
+            int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
+            AssertRCReturn(rc, rc);
+
+            /*
+             * What we do with the shadow pages depends on the memory
+             * preallocation option. If not enabled, we'll just throw
+             * out all the dirty pages and replace them by the zero page.
+             */
+            if (!pVM->pgm.s.fRamPreAlloc)
+            {
+                /* Free the dirty pages. */
+                uint32_t            cPendingPages = 0;
+                PGMMFREEPAGESREQ    pReq;
+                rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+                AssertRCReturn(rc, rc);
+
+                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                    if (    !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
+                        &&  !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
+                    {
+                        Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
+                        rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
+                                             pRom->GCPhys + (iPage << PAGE_SHIFT));
+                        AssertLogRelRCReturn(rc, rc);
+                    }
+
+                if (cPendingPages)
+                {
+                    rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+                    AssertLogRelRCReturn(rc, rc);
+                }
+                GMMR3FreePagesCleanup(pReq);
+            }
+            else
+            {
+                /* clear all the shadow pages. */
+                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                {
+                    if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
+                        continue;
+                    Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
+                    void *pvDstPage;
+                    const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
+                    rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
+                    if (RT_FAILURE(rc))
+                        break;
+                    ASMMemZeroPage(pvDstPage);
+                }
+                AssertRCReturn(rc, rc);
+            }
+        }
+
+#ifdef VBOX_STRICT
+        /*
+         * Verify that the virgin page is unchanged if possible.
+         */
+        if (pRom->pvOriginal)
+        {
+            size_t         cbSrcLeft = pRom->cbOriginal;
+            uint8_t const *pbSrcPage = (uint8_t const *)pRom->pvOriginal;
+            for (uint32_t iPage = 0; iPage < cPages && cbSrcLeft > 0; iPage++, pbSrcPage += PAGE_SIZE)
+            {
+                const RTGCPHYS GCPhys = pRom->GCPhys + (iPage << PAGE_SHIFT);
+                void const *pvDstPage;
+                int rc = pgmPhysPageMapReadOnly(pVM, &pRom->aPages[iPage].Virgin, GCPhys, &pvDstPage);
+                if (RT_FAILURE(rc))
+                    break;
+
+                if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, PAGE_SIZE)))
+                    LogRel(("pgmR3PhysRomReset: %RGp rom page changed (%s) - loaded saved state?\n",
+                            GCPhys, pRom->pszDesc));
+                cbSrcLeft -= RT_MIN(cbSrcLeft, PAGE_SIZE);
+            }
+        }
+#endif
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called by PGMR3Term to free resources.
+ *
+ * ASSUMES that the caller owns the PGM lock.
+ *
+ * @param   pVM         The VM handle.
+ */
+void pgmR3PhysRomTerm(PVM pVM)
+{
+#ifdef RT_STRICT
+    /*
+     * Free the heap copy of the original bits.
+     */
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (   pRom->pvOriginal
+            && !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
+        {
+            RTMemFree((void *)pRom->pvOriginal);
+            pRom->pvOriginal = NULL;
+        }
+    }
+#endif
+}
+
+
+/**
+ * Change the shadowing of a range of ROM pages.
+ *
+ * This is intended for implementing chipset specific memory registers
+ * and will not be very strict about the input. It will silently ignore
+ * any pages that are not the part of a shadowed ROM.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   GCPhys      Where to start. Page aligned.
+ * @param   cb          How much to change. Page aligned.
+ * @param   enmProt     The new ROM protection.
+ */
+VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
+{
+    /*
+     * Check input
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+    AssertReturn(!(GCPhys & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    AssertReturn(!(cb & PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
+    AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Process the request.
+     */
+    pgmLock(pVM);
+    int  rc = VINF_SUCCESS;
+    bool fFlushTLB = false;
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (    GCPhys     <= pRom->GCPhysLast
+            &&  GCPhysLast >= pRom->GCPhys
+            &&  (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
+        {
+            /*
+             * Iterate the relevant pages and make necessary the changes.
+             */
+            bool fChanges = false;
+            uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
+                                  ? pRom->cb >> PAGE_SHIFT
+                                  : (GCPhysLast - pRom->GCPhys + 1) >> PAGE_SHIFT;
+            for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> PAGE_SHIFT;
+                 iPage < cPages;
+                 iPage++)
+            {
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
+                {
+                    fChanges = true;
+
+                    /* flush references to the page. */
+                    PPGMPAGE pRamPage = pgmPhysGetPage(&pVM->pgm.s, pRom->GCPhys + (iPage << PAGE_SHIFT));
+                    int rc2 = pgmPoolTrackUpdateGCPhys(pVM, pRom->GCPhys + (iPage << PAGE_SHIFT), pRamPage,
+                                                       true /*fFlushPTEs*/, &fFlushTLB);
+                    if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS || RT_FAILURE(rc2)))
+                        rc = rc2;
+
+                    PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
+                    PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
+
+                    *pOld = *pRamPage;
+                    *pRamPage = *pNew;
+                    /** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
+                }
+                pRomPage->enmProt = enmProt;
+            }
+
+            /*
+             * Reset the access handler if we made changes, no need
+             * to optimize this.
+             */
+            if (fChanges)
+            {
+                int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
+                if (RT_FAILURE(rc2))
+                {
+                    pgmUnlock(pVM);
+                    AssertRC(rc);
+                    return rc2;
+                }
+            }
+
+            /* Advance - cb isn't updated. */
+            GCPhys = pRom->GCPhys + (cPages << PAGE_SHIFT);
+        }
+    }
+    pgmUnlock(pVM);
+    if (fFlushTLB)
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Sets the Address Gate 20 state.
+ *
+ * @param   pVCpu       The VCPU to operate on.
+ * @param   fEnable     True if the gate should be enabled.
+ *                      False if the gate should be disabled.
+ */
+VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
+{
+    LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
+    if (pVCpu->pgm.s.fA20Enabled != fEnable)
+    {
+        pVCpu->pgm.s.fA20Enabled = fEnable;
+        pVCpu->pgm.s.GCPhysA20Mask = ~(RTGCPHYS)(!fEnable << 20);
+        REMR3A20Set(pVCpu->pVMR3, pVCpu, fEnable);
+        /** @todo we're not handling this correctly for VT-x / AMD-V. See #2911 */
+    }
+}
+
+#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
+/**
+ * Tree enumeration callback for dealing with age rollover.
+ * It will perform a simple compression of the current age.
+ */
+static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
+{
+    Assert(PGMIsLockOwner((PVM)pvUser));
+    /* Age compression - ASSUMES iNow == 4. */
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
+    if (pChunk->iAge >= UINT32_C(0xffffff00))
+        pChunk->iAge = 3;
+    else if (pChunk->iAge >= UINT32_C(0xfffff000))
+        pChunk->iAge = 2;
+    else if (pChunk->iAge)
+        pChunk->iAge = 1;
+    else /* iAge = 0 */
+        pChunk->iAge = 4;
+    return 0;
+}
+
+
+/**
+ * Tree enumeration callback that updates the chunks that have
+ * been used since the last
+ */
+static DECLCALLBACK(int) pgmR3PhysChunkAgeingCallback(PAVLU32NODECORE pNode, void *pvUser)
+{
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
+    if (!pChunk->iAge)
+    {
+        PVM pVM = (PVM)pvUser;
+        pChunk->iAge = pVM->pgm.s.ChunkR3Map.iNow;
+    }
+    return 0;
+}
+
+
+/**
+ * Performs ageing of the ring-3 chunk mappings.
+ *
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(void) PGMR3PhysChunkAgeing(PVM pVM)
+{
+    pgmLock(pVM);
+    pVM->pgm.s.ChunkR3Map.AgeingCountdown = RT_MIN(pVM->pgm.s.ChunkR3Map.cMax / 4, 1024);
+    pVM->pgm.s.ChunkR3Map.iNow++;
+    if (pVM->pgm.s.ChunkR3Map.iNow == 0)
+    {
+        pVM->pgm.s.ChunkR3Map.iNow = 4;
+        RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingRolloverCallback, pVM);
+    }
+    else
+        RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkAgeingCallback, pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
+ */
+typedef struct PGMR3PHYSCHUNKUNMAPCB
+{
+    PVM                 pVM;            /**< The VM handle. */
+    PPGMCHUNKR3MAP      pChunk;         /**< The chunk to unmap. */
+    uint32_t            iLastAge;       /**< Highest age found so far. */
+} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
+
+
+/**
+ * Callback used to find the mapping that's been unused for
+ * the longest time.
+ */
+static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
+{
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
+    PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
+
+    if (    pChunk->iAge
+        &&  !pChunk->cRefs
+        &&  pArg->iLastAge < pChunk->iAge)
+    {
+        /*
+         * Check that it's not in any of the TLBs.
+         */
+        PVM pVM = pArg->pVM;
+        for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
+            if (pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk == pChunk)
+            {
+                pChunk = NULL;
+                break;
+            }
+        if (pChunk)
+            for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbHC.aEntries); i++)
+                if (pVM->pgm.s.PhysTlbHC.aEntries[i].pMap == pChunk)
+                {
+                    pChunk = NULL;
+                    break;
+                }
+        if (pChunk)
+        {
+            pArg->pChunk = pChunk;
+            pArg->iLastAge = pChunk->iAge;
+        }
+    }
+    return 0;
+}
+
+
+/**
+ * Finds a good candidate for unmapping when the ring-3 mapping cache is full.
+ *
+ * The candidate will not be part of any TLBs, so no need to flush
+ * anything afterwards.
+ *
+ * @returns Chunk id.
+ * @param   pVM         The VM handle.
+ */
+static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
+{
+    Assert(PGMIsLockOwner(pVM));
+
+    /*
+     * Do tree ageing first?
+     */
+    if (pVM->pgm.s.ChunkR3Map.AgeingCountdown-- == 0)
+    {
+        STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
+        PGMR3PhysChunkAgeing(pVM);
+        STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkAging, a);
+    }
+
+    /*
+     * Enumerate the age tree starting with the left most node.
+     */
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
+    PGMR3PHYSCHUNKUNMAPCB Args;
+    Args.pVM      = pVM;
+    Args.pChunk   = NULL;
+    Args.iLastAge = 0;
+    RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /*fFromLeft*/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
+    Assert(Args.pChunk);
+    if (Args.pChunk)
+    {
+        STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
+        return Args.pChunk->Core.Key;
+    }
+
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkFindCandidate, a);
+    return INT32_MAX;
+}
+
+/**
+ * Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU for the EMT we're being called on. Unused.
+ * @param   pvUser      User pointer. Unused
+ *
+ */
+DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    int rc = VINF_SUCCESS;
+    pgmLock(pVM);
+
+    if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
+    {
+        /* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
+        /* todo: also not really efficient to unmap a chunk that contains PD or PT pages. */
+        pgmR3PoolClearAllRendezvous(pVM, &pVM->aCpus[0], NULL /* no need to flush the REM TLB as we already did that above */);
+
+        /*
+         * Request the ring-0 part to unmap a chunk to make space in the mapping cache.
+         */
+        GMMMAPUNMAPCHUNKREQ Req;
+        Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+        Req.Hdr.cbReq = sizeof(Req);
+        Req.pvR3 = NULL;
+        Req.idChunkMap = NIL_GMM_CHUNKID;
+        Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
+
+        if (Req.idChunkUnmap != INT32_MAX)
+        {
+            STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
+            rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
+            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkUnmap, a);
+            if (RT_SUCCESS(rc))
+            {
+                /* remove the unmapped one. */
+                PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
+                AssertRelease(pUnmappedChunk);
+                pUnmappedChunk->pv = NULL;
+                pUnmappedChunk->Core.Key = UINT32_MAX;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+                MMR3HeapFree(pUnmappedChunk);
+#else
+                MMR3UkHeapFree(pVM, pUnmappedChunk, MM_TAG_PGM_CHUNK_MAPPING);
+#endif
+                pVM->pgm.s.ChunkR3Map.c--;
+                pVM->pgm.s.cUnmappedChunks++;
+
+                /* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses) */
+                /* todo: we should not flush chunks which include cr3 mappings. */
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                {
+                    PPGMCPU pPGM = &pVM->aCpus[idCpu].pgm.s;
+
+                    pPGM->pGst32BitPdR3    = NULL;
+                    pPGM->pGstPaePdptR3    = NULL;
+                    pPGM->pGstAmd64Pml4R3  = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+                    pPGM->pGst32BitPdR0    = NIL_RTR0PTR;
+                    pPGM->pGstPaePdptR0    = NIL_RTR0PTR;
+                    pPGM->pGstAmd64Pml4R0  = NIL_RTR0PTR;
+#endif
+                    for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
+                    {
+                        pPGM->apGstPaePDsR3[i]             = NULL;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+                        pPGM->apGstPaePDsR0[i]             = NIL_RTR0PTR;
+#endif
+                    }
+
+                    /* Flush REM TLBs. */
+                    CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+                }
+
+                /* Flush REM translation blocks. */
+                REMFlushTBs(pVM);
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return rc;
+}
+
+/**
+ * Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+void pgmR3PhysUnmapChunk(PVM pVM)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
+    AssertRC(rc);
+}
+#endif /* PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST */
+
+/**
+ * Maps the given chunk into the ring-3 mapping cache.
+ *
+ * This will call ring-0.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   idChunk     The chunk in question.
+ * @param   ppChunk     Where to store the chunk tracking structure.
+ *
+ * @remarks Called from within the PGM critical section.
+ * @remarks Can be called from any thread!
+ */
+int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
+{
+    int rc;
+
+    Assert(PGMIsLockOwner(pVM));
+    /*
+     * Allocate a new tracking structure first.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
+#else
+    PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3UkHeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk), NULL);
+#endif
+    AssertReturn(pChunk, VERR_NO_MEMORY);
+    pChunk->Core.Key = idChunk;
+
+    /*
+     * Request the ring-0 part to map the chunk in question.
+     */
+    GMMMAPUNMAPCHUNKREQ Req;
+    Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+    Req.Hdr.cbReq = sizeof(Req);
+    Req.pvR3 = NULL;
+    Req.idChunkMap = idChunk;
+    Req.idChunkUnmap = NIL_GMM_CHUNKID;
+
+    /* Must be callable from any thread, so can't use VMMR3CallR0. */
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
+    rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatChunkMap, a);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Update the tree.
+         */
+        /* insert the new one. */
+        AssertPtr(Req.pvR3);
+        pChunk->pv = Req.pvR3;
+        bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
+        AssertRelease(fRc);
+        pVM->pgm.s.ChunkR3Map.c++;
+        pVM->pgm.s.cMappedChunks++;
+
+        /* If we're running out of virtual address space, then we should unmap another chunk. */
+        if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
+        {
+#ifdef PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
+            /* Postpone the unmap operation (which requires a rendezvous operation) as we own the PGM lock here. */
+            rc = VMR3ReqCallNoWaitU(pVM->pUVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
+            AssertRC(rc);
+#else
+            AssertFatalFailed();  /* can't happen */
+#endif
+        }
+    }
+    else
+    {
+        AssertRC(rc);
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        MMR3HeapFree(pChunk);
+#else
+        MMR3UkHeapFree(pVM, pChunk, MM_TAG_PGM_CHUNK_MAPPING);
+#endif
+        pChunk = NULL;
+    }
+
+    *ppChunk = pChunk;
+    return rc;
+}
+
+
+/**
+ * For VMMCALLRING3_PGM_MAP_CHUNK, considered internal.
+ *
+ * @returns see pgmR3PhysChunkMap.
+ * @param   pVM         The VM handle.
+ * @param   idChunk     The chunk to map.
+ */
+VMMR3DECL(int) PGMR3PhysChunkMap(PVM pVM, uint32_t idChunk)
+{
+    PPGMCHUNKR3MAP pChunk;
+    int rc;
+
+    pgmLock(pVM);
+    rc = pgmR3PhysChunkMap(pVM, idChunk, &pChunk);
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Invalidates the TLB for the ring-3 mapping cache.
+ *
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
+{
+    pgmLock(pVM);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
+    {
+        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
+        pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
+    }
+    /* The page map TLB references chunks, so invalidate that one too. */
+    PGMPhysInvalidatePageMapTLB(pVM);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Response to VMMCALLRING3_PGM_ALLOCATE_LARGE_PAGE to allocate a large (2MB) page
+ * for use with a nested paging PDE.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory.
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCPhys      GC physical start address of the 2 MB range
+ */
+VMMR3DECL(int) PGMR3PhysAllocateLargeHandyPage(PVM pVM, RTGCPHYS GCPhys)
+{
+#ifdef PGM_WITH_LARGE_PAGES
+    uint64_t u64TimeStamp1, u64TimeStamp2;
+
+    pgmLock(pVM);
+
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
+    u64TimeStamp1 = RTTimeMilliTS();
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_HANDY_PAGE, 0, NULL);
+    u64TimeStamp2 = RTTimeMilliTS();
+    STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatAllocLargePage, a);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pVM->pgm.s.cLargeHandyPages == 1);
+
+        uint32_t idPage = pVM->pgm.s.aLargeHandyPage[0].idPage;
+        RTHCPHYS HCPhys = pVM->pgm.s.aLargeHandyPage[0].HCPhysGCPhys;
+
+        void *pv;
+
+        /* Map the large page into our address space.
+         *
+         * Note: assuming that within the 2 MB range:
+         * - GCPhys + PAGE_SIZE = HCPhys + PAGE_SIZE (whole point of this exercise)
+         * - user space mapping is continuous as well
+         * - page id (GCPhys) + 1 = page id (GCPhys + PAGE_SIZE)
+         */
+        rc = pgmPhysPageMapByPageID(pVM, idPage, HCPhys, &pv);
+        AssertLogRelMsg(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", idPage, HCPhys, rc));
+
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Clear the pages.
+             */
+            STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
+            for (unsigned i = 0; i < _2M/PAGE_SIZE; i++)
+            {
+                ASMMemZeroPage(pv);
+
+                PPGMPAGE pPage;
+                rc = pgmPhysGetPageEx(&pVM->pgm.s, GCPhys, &pPage);
+                AssertRC(rc);
+
+                Assert(PGM_PAGE_IS_ZERO(pPage));
+                STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRZPageReplaceZero);
+                pVM->pgm.s.cZeroPages--;
+
+                /*
+                 * Do the PGMPAGE modifications.
+                 */
+                pVM->pgm.s.cPrivatePages++;
+                PGM_PAGE_SET_HCPHYS(pPage, HCPhys);
+                PGM_PAGE_SET_PAGEID(pPage, idPage);
+                PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
+                PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_PDE);
+                PGM_PAGE_SET_PTE_INDEX(pPage, 0);
+                PGM_PAGE_SET_TRACKING(pPage, 0);
+
+                /* Somewhat dirty assumption that page ids are increasing. */
+                idPage++;
+
+                HCPhys += PAGE_SIZE;
+                GCPhys += PAGE_SIZE;
+
+                pv = (void *)((uintptr_t)pv + PAGE_SIZE);
+
+                Log3(("PGMR3PhysAllocateLargePage: idPage=%#x HCPhys=%RGp\n", idPage, HCPhys));
+            }
+            STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatClearLargePage, b);
+
+            /* Flush all TLBs. */
+            PGM_INVL_ALL_VCPU_TLBS(pVM);
+            PGMPhysInvalidatePageMapTLB(pVM);
+        }
+        pVM->pgm.s.cLargeHandyPages = 0;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        static uint32_t cTimeOut = 0;
+        uint64_t u64TimeStampDelta = u64TimeStamp2 - u64TimeStamp1;
+
+        if (u64TimeStampDelta > 100)
+        {
+            STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatLargePageOverflow);
+            if (    ++cTimeOut > 10
+                ||  u64TimeStampDelta > 1000 /* more than one second forces an early retirement from allocating large pages. */)
+            {
+                /* If repeated attempts to allocate a large page takes more than 100 ms, then we fall back to normal 4k pages.
+                 * E.g. Vista 64 tries to move memory around, which takes a huge amount of time.
+                 */
+                LogRel(("PGMR3PhysAllocateLargePage: allocating large pages takes too long (last attempt %d ms; nr of timeouts %d); DISABLE\n", u64TimeStampDelta, cTimeOut));
+                PGMSetLargePageUsage(pVM, false);
+            }
+        }
+        else
+        if (cTimeOut > 0)
+            cTimeOut--;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif /* PGM_WITH_LARGE_PAGES */
+}
+
+
+/**
+ * Response to VM_FF_PGM_NEED_HANDY_PAGES and VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES.
+ *
+ * This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
+ * signal and clear the out of memory condition. When contracted, this API is
+ * used to try clear the condition when the user wants to resume.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success. FFs cleared.
+ * @retval  VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
+ *          this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
+ *
+ * @param   pVM         The VM handle.
+ *
+ * @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
+ *          in EM.cpp and shouldn't be propagated outside TRPM, HWACCM, EM and
+ *          pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
+ *          handler.
+ */
+VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
+{
+    pgmLock(pVM);
+
+    /*
+     * Allocate more pages, noting down the index of the first new page.
+     */
+    uint32_t iClear = pVM->pgm.s.cHandyPages;
+    AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_INTERNAL_ERROR);
+    Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
+    int rcAlloc = VINF_SUCCESS;
+    int rcSeed  = VINF_SUCCESS;
+    int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
+    while (rc == VERR_GMM_SEED_ME)
+    {
+        void *pvChunk;
+        rcAlloc = rc = SUPR3PageAlloc(GMM_CHUNK_SIZE >> PAGE_SHIFT, &pvChunk);
+        if (RT_SUCCESS(rc))
+        {
+            rcSeed = rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_SEED_CHUNK, (uintptr_t)pvChunk, NULL);
+            if (RT_FAILURE(rc))
+                SUPR3PageFree(pvChunk, GMM_CHUNK_SIZE >> PAGE_SHIFT);
+        }
+        if (RT_SUCCESS(rc))
+            rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
+    }
+
+    /* todo: we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
+    if (    rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
+        &&  pVM->pgm.s.cHandyPages > 0)
+    {
+        /* Still handy pages left, so don't panic. */
+        rc = VINF_SUCCESS;
+    }
+
+    if (RT_SUCCESS(rc))
+    {
+        AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        Assert(pVM->pgm.s.cHandyPages > 0);
+        VM_FF_CLEAR(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
+        VM_FF_CLEAR(pVM, VM_FF_PGM_NO_MEMORY);
+
+        /*
+         * Clear the pages.
+         */
+        while (iClear < pVM->pgm.s.cHandyPages)
+        {
+            PGMMPAGEDESC pPage = &pVM->pgm.s.aHandyPages[iClear];
+            void *pv;
+            rc = pgmPhysPageMapByPageID(pVM, pPage->idPage, pPage->HCPhysGCPhys, &pv);
+            AssertLogRelMsgBreak(RT_SUCCESS(rc), ("idPage=%#x HCPhysGCPhys=%RHp rc=%Rrc\n", pPage->idPage, pPage->HCPhysGCPhys, rc));
+            ASMMemZeroPage(pv);
+            iClear++;
+            Log3(("PGMR3PhysAllocateHandyPages: idPage=%#x HCPhys=%RGp\n", pPage->idPage, pPage->HCPhysGCPhys));
+        }
+    }
+    else
+    {
+        uint64_t cAllocPages, cMaxPages, cBalloonPages;
+
+        /*
+         * We should never get here unless there is a genuine shortage of
+         * memory (or some internal error). Flag the error so the VM can be
+         * suspended ASAP and the user informed. If we're totally out of
+         * handy pages we will return failure.
+         */
+        /* Report the failure. */
+        LogRel(("PGM: Failed to procure handy pages; rc=%Rrc rcAlloc=%Rrc rcSeed=%Rrc cHandyPages=%#x\n"
+                "     cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
+                rc, rcAlloc, rcSeed,
+                pVM->pgm.s.cHandyPages,
+                pVM->pgm.s.cAllPages,
+                pVM->pgm.s.cPrivatePages,
+                pVM->pgm.s.cSharedPages,
+                pVM->pgm.s.cZeroPages));
+
+        if (GMMR3QueryMemoryStats(pVM, &cAllocPages, &cMaxPages, &cBalloonPages) == VINF_SUCCESS)
+        {
+            LogRel(("GMM: Statistics:\n"
+                    "     Allocated pages: %RX64\n"
+                    "     Maximum   pages: %RX64\n"
+                    "     Ballooned pages: %RX64\n", cAllocPages, cMaxPages, cBalloonPages));
+        }
+
+        if (    rc != VERR_NO_MEMORY
+            &&  rc != VERR_LOCK_FAILED)
+        {
+            for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
+            {
+                LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
+                        i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
+                        pVM->pgm.s.aHandyPages[i].idSharedPage));
+                uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
+                if (idPage != NIL_GMM_PAGEID)
+                {
+                    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3;
+                         pRam;
+                         pRam = pRam->pNextR3)
+                    {
+                        uint32_t const cPages = pRam->cb >> PAGE_SHIFT;
+                        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                            if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
+                                LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
+                                        pRam->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
+                    }
+                }
+            }
+        }
+
+        /* Set the FFs and adjust rc. */
+        VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
+        VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
+        if (    rc == VERR_NO_MEMORY
+            ||  rc == VERR_LOCK_FAILED)
+            rc = VINF_EM_NO_MEMORY;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Frees the specified RAM page and replaces it with the ZERO page.
+ *
+ * This is used by ballooning, remapping MMIO2, RAM reset and state loading.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pReq            Pointer to the request.
+ * @param   pcPendingPages  Where the number of pages waiting to be freed are
+ *                          kept.  This will normally be incremented.
+ * @param   pPage           Pointer to the page structure.
+ * @param   GCPhys          The guest physical address of the page, if applicable.
+ *
+ * @remarks The caller must own the PGM lock.
+ */
+int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    /*
+     * Assert sanity.
+     */
+    Assert(PGMIsLockOwner(pVM));
+    if (RT_UNLIKELY(    PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
+                    &&  PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
+    {
+        AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
+        return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
+    }
+    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
+
+    if (    PGM_PAGE_IS_ZERO(pPage)
+        ||  PGM_PAGE_IS_BALLOONED(pPage))
+        return VINF_SUCCESS;
+
+    const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
+    Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
+    if (RT_UNLIKELY(    idPage == NIL_GMM_PAGEID
+                    ||  idPage > GMM_PAGEID_LAST
+                    ||  PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID))
+    {
+        AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
+        return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
+    }
+
+    /* update page count stats. */
+    if (PGM_PAGE_IS_SHARED(pPage))
+        pVM->pgm.s.cSharedPages--;
+    else
+        pVM->pgm.s.cPrivatePages--;
+    pVM->pgm.s.cZeroPages++;
+
+    /* Deal with write monitored pages. */
+    if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
+    {
+        PGM_PAGE_SET_WRITTEN_TO(pPage);
+        pVM->pgm.s.cWrittenToPages++;
+    }
+
+    /*
+     * pPage = ZERO page.
+     */
+    PGM_PAGE_SET_HCPHYS(pPage, pVM->pgm.s.HCPhysZeroPg);
+    PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ZERO);
+    PGM_PAGE_SET_PAGEID(pPage, NIL_GMM_PAGEID);
+    PGM_PAGE_SET_PDE_TYPE(pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
+    PGM_PAGE_SET_PTE_INDEX(pPage, 0);
+    PGM_PAGE_SET_TRACKING(pPage, 0);
+
+    /* Flush physical page map TLB entry. */
+    PGMPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
+
+    /*
+     * Make sure it's not in the handy page array.
+     */
+    for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
+    {
+        if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
+        {
+            pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
+            break;
+        }
+        if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
+        {
+            pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
+            break;
+        }
+    }
+
+    /*
+     * Push it onto the page array.
+     */
+    uint32_t iPage = *pcPendingPages;
+    Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
+    *pcPendingPages += 1;
+
+    pReq->aPages[iPage].idPage = idPage;
+
+    if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
+        return VINF_SUCCESS;
+
+    /*
+     * Flush the pages.
+     */
+    int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
+    if (RT_SUCCESS(rc))
+    {
+        GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
+        *pcPendingPages = 0;
+    }
+    return rc;
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer, with some
+ * additional checks.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
+ *          access handler of some kind.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
+ *          accesses or is odd in any way.
+ * @retval  VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCPhys      The GC physical address to convert.
+ * @param   fWritable   Whether write access is required.
+ * @param   ppv         Where to store the pointer corresponding to GCPhys on
+ *                      success.
+ */
+VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
+{
+    pgmLock(pVM);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(&pVM->pgm.s, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+            rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
+        else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+            rc = VINF_SUCCESS;
+        else
+        {
+            if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
+                rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+            else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+            {
+                /** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
+                 *        in -norawr0 mode. */
+                if (fWritable)
+                    rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+            else
+            {
+                /* Temporarily disabled physical handler(s), since the recompiler
+                   doesn't get notified when it's reset we'll have to pretend it's
+                   operating normally. */
+                if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
+                    rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+                else
+                    rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            int rc2;
+
+            /* Make sure what we return is writable. */
+            if (fWritable)
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ALLOCATED:
+                        break;
+                    case PGM_PAGE_STATE_BALLOONED:
+                        AssertFailed();
+                        break;
+                    case PGM_PAGE_STATE_ZERO:
+                    case PGM_PAGE_STATE_SHARED:
+                        if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
+                            break;
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                        rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
+                        AssertLogRelRCReturn(rc2, rc2);
+                        break;
+                }
+
+            /* Get a ring-3 mapping of the address. */
+            PPGMPAGER3MAPTLBE pTlbe;
+            rc2 = pgmPhysPageQueryTlbe(&pVM->pgm.s, GCPhys, &pTlbe);
+            AssertLogRelRCReturn(rc2, rc2);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & PAGE_OFFSET_MASK));
+            /** @todo mapping/locking hell; this isn't horribly efficient since
+             *        pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
+
+            Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
+        }
+        else
+            Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
+
+        /* else: handler catching all access, no pointer returned. */
+    }
+    else
+        rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMPhysRWTmpl.h	(revision 35333)
@@ -0,0 +1,57 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager and Monitor, Physical Memory Access Template.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/**
+ * Read physical memory. (one byte/word/dword)
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
+ * want to ignore those.
+ *
+ * @param   pVM             VM Handle.
+ * @param   GCPhys          Physical address start reading from.
+ */
+VMMDECL(PGMPHYS_DATATYPE) PGMPHYSFN_READNAME(PVM pVM, RTGCPHYS GCPhys)
+{
+    Assert(VM_IS_EMT(pVM));
+    PGMPHYS_DATATYPE val;
+    PGMPhysRead(pVM, GCPhys, &val, sizeof(val));
+    return val;
+}
+
+
+/**
+ * Write to physical memory. (one byte/word/dword)
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
+ * want to ignore those.
+ *
+ * @param   pVM             VM Handle.
+ * @param   GCPhys          Physical address to write to.
+ * @param   val             What to write.
+ */
+VMMDECL(void) PGMPHYSFN_WRITENAME(PVM pVM, RTGCPHYS GCPhys, PGMPHYS_DATATYPE val)
+{
+    Assert(VM_IS_EMT(pVM));
+    PGMPhysWrite(pVM, GCPhys, &val, sizeof(val));
+}
+
+#undef PGMPHYSFN_READNAME
+#undef PGMPHYSFN_WRITENAME
+#undef PGMPHYS_DATATYPE
+#undef PGMPHYS_DATASIZE
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMPool.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMPool.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMPool.cpp	(revision 35333)
@@ -0,0 +1,1071 @@
+/* $Id$ */
+/** @file
+ * PGM Shadow Page Pool.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_pgm_pool       PGM Shadow Page Pool
+ *
+ * Motivations:
+ *      -# Relationship between shadow page tables and physical guest pages. This
+ *         should allow us to skip most of the global flushes now following access
+ *         handler changes. The main expense is flushing shadow pages.
+ *      -# Limit the pool size if necessary (default is kind of limitless).
+ *      -# Allocate shadow pages from RC. We use to only do this in SyncCR3.
+ *      -# Required for 64-bit guests.
+ *      -# Combining the PD cache and page pool in order to simplify caching.
+ *
+ *
+ * @section sec_pgm_pool_outline    Design Outline
+ *
+ * The shadow page pool tracks pages used for shadowing paging structures (i.e.
+ * page tables, page directory, page directory pointer table and page map
+ * level-4). Each page in the pool has an unique identifier. This identifier is
+ * used to link a guest physical page to a shadow PT. The identifier is a
+ * non-zero value and has a relativly low max value - say 14 bits. This makes it
+ * possible to fit it into the upper bits of the of the aHCPhys entries in the
+ * ram range.
+ *
+ * By restricting host physical memory to the first 48 bits (which is the
+ * announced physical memory range of the K8L chip (scheduled for 2008)), we
+ * can safely use the upper 16 bits for shadow page ID and reference counting.
+ *
+ * Update: The 48 bit assumption will be lifted with the new physical memory
+ * management (PGMPAGE), so we won't have any trouble when someone stuffs 2TB
+ * into a box in some years.
+ *
+ * Now, it's possible for a page to be aliased, i.e. mapped by more than one PT
+ * or PD. This is solved by creating a list of physical cross reference extents
+ * when ever this happens. Each node in the list (extent) is can contain 3 page
+ * pool indexes. The list it self is chained using indexes into the paPhysExt
+ * array.
+ *
+ *
+ * @section sec_pgm_pool_life       Life Cycle of a Shadow Page
+ *
+ * -# The SyncPT function requests a page from the pool.
+ *    The request includes the kind of page it is (PT/PD, PAE/legacy), the
+ *    address of the page it's shadowing, and more.
+ * -# The pool responds to the request by allocating a new page.
+ *    When the cache is enabled, it will first check if it's in the cache.
+ *    Should the pool be exhausted, one of two things can be done:
+ *      -# Flush the whole pool and current CR3.
+ *      -# Use the cache to find a page which can be flushed (~age).
+ * -# The SyncPT function will sync one or more pages and insert it into the
+ *    shadow PD.
+ * -# The SyncPage function may sync more pages on a later \#PFs.
+ * -# The page is freed / flushed in SyncCR3 (perhaps) and some other cases.
+ *    When caching is enabled, the page isn't flush but remains in the cache.
+ *
+ *
+ * @section sec_pgm_pool_impl       Monitoring
+ *
+ * We always monitor PAGE_SIZE chunks of memory. When we've got multiple shadow
+ * pages for the same PAGE_SIZE of guest memory (PAE and mixed PD/PT) the pages
+ * sharing the monitor get linked using the iMonitoredNext/Prev. The head page
+ * is the pvUser to the access handlers.
+ *
+ *
+ * @section sec_pgm_pool_impl       Implementation
+ *
+ * The pool will take pages from the MM page pool. The tracking data
+ * (attributes, bitmaps and so on) are allocated from the hypervisor heap. The
+ * pool content can be accessed both by using the page id and the physical
+ * address (HC). The former is managed by means of an array, the latter by an
+ * offset based AVL tree.
+ *
+ * Flushing of a pool page means that we iterate the content (we know what kind
+ * it is) and updates the link information in the ram range.
+ *
+ * ...
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_POOL
+#include <VBox/pgm.h>
+#include <VBox/mm.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include "PGMInline.h"
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <VBox/dbg.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
+#ifdef VBOX_WITH_DEBUGGER
+static DECLCALLBACK(int)  pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+#endif
+
+#ifdef VBOX_WITH_DEBUGGER
+/** Command descriptors. */
+static const DBGCCMD    g_aCmds[] =
+{
+    /* pszCmd,  cArgsMin, cArgsMax, paArgDesc,                cArgDescs,    pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
+    { "pgmpoolcheck",  0, 0,        NULL,                     0,            NULL,               0,          pgmR3PoolCmdCheck,  "",                 "Check the pgm pool pages." },
+};
+#endif
+
+/**
+ * Initializes the pool
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+int pgmR3PoolInit(PVM pVM)
+{
+    AssertCompile(NIL_PGMPOOL_IDX == 0);
+    /* pPage->cLocked is an unsigned byte. */
+    AssertCompile(VMM_MAX_CPU_COUNT <= 255);
+
+    /*
+     * Query Pool config.
+     */
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pVM), "/PGM/Pool");
+
+    /* Default pgm pool size equals 1024 pages. */
+    uint16_t cMaxPages = 4*_1M >> PAGE_SHIFT;
+
+#if HC_ARCH_BITS == 64
+    uint64_t cbRam = 0;
+    CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
+
+    /* We should increase the pgm pool size for guests with more than 2 GB of ram */
+    if (cbRam >= UINT64_C(2) * _1G)
+    {
+        /* In the nested paging case we require 2 + 513 * (cbRam/1GB) pages to
+         * store all page table descriptors.
+         */
+        uint64_t u64MaxPages = cbRam  / (_1G / UINT64_C(512));
+        if (u64MaxPages > PGMPOOL_IDX_LAST)
+            cMaxPages = PGMPOOL_IDX_LAST;
+        else
+            cMaxPages = (uint16_t)u64MaxPages;
+    }
+#endif
+
+    /** @cfgm{/PGM/Pool/MaxPages, uint16_t, #pages, 16, 0x3fff, 1024}
+     * The max size of the shadow page pool in pages. The pool will grow dynamically
+     * up to this limit.
+     */
+    int rc = CFGMR3QueryU16Def(pCfg, "MaxPages", &cMaxPages, cMaxPages);
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelMsgReturn(cMaxPages <= PGMPOOL_IDX_LAST && cMaxPages >= RT_ALIGN(PGMPOOL_IDX_FIRST, 16),
+                          ("cMaxPages=%u (%#x)\n", cMaxPages, cMaxPages), VERR_INVALID_PARAMETER);
+    cMaxPages = RT_ALIGN(cMaxPages, 16);
+
+    /** todo:
+     * We need to be much more careful with our allocation strategy here.
+     * For nested paging we don't need pool user info nor extents at all, but we can't check for nested paging here (too early during init to get a confirmation it can be used)
+     * The default for large memory configs is a bit large for shadow paging, so I've restricted the extent maximum to 2k (2k * 16 = 32k of hyper heap)
+     *
+     * Also when large page support is enabled, we typically don't need so much,
+     * although that depends on the availability of 2 MB chunks on the host.
+     */
+
+    /** @cfgm{/PGM/Pool/MaxUsers, uint16_t, #users, MaxUsers, 32K, MaxPages*2}
+     * The max number of shadow page user tracking records. Each shadow page has
+     * zero of other shadow pages (or CR3s) that references it, or uses it if you
+     * like. The structures describing these relationships are allocated from a
+     * fixed sized pool. This configuration variable defines the pool size.
+     */
+    uint16_t cMaxUsers;
+    rc = CFGMR3QueryU16Def(pCfg, "MaxUsers", &cMaxUsers, cMaxPages * 2);
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelMsgReturn(cMaxUsers >= cMaxPages && cMaxPages <= _32K,
+                          ("cMaxUsers=%u (%#x)\n", cMaxUsers, cMaxUsers), VERR_INVALID_PARAMETER);
+
+    /** @cfgm{/PGM/Pool/MaxPhysExts, uint16_t, #extents, 16, MaxPages * 2, MAX(MaxPages*2,0x3fff)}
+     * The max number of extents for tracking aliased guest pages.
+     */
+    uint16_t cMaxPhysExts;
+    rc = CFGMR3QueryU16Def(pCfg, "MaxPhysExts", &cMaxPhysExts, RT_MAX(cMaxPages * 2, 2048 /* 2k max as this eat too much hyper heap */));
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelMsgReturn(cMaxPhysExts >= 16 && cMaxPages <= PGMPOOL_IDX_LAST,
+                          ("cMaxPhysExts=%u (%#x)\n", cMaxPhysExts, cMaxPhysExts), VERR_INVALID_PARAMETER);
+
+    /** @cfgm{/PGM/Pool/ChacheEnabled, bool, true}
+     * Enables or disabling caching of shadow pages. Caching means that we will try
+     * reuse shadow pages instead of recreating them everything SyncCR3, SyncPT or
+     * SyncPage requests one. When reusing a shadow page, we can save time
+     * reconstructing it and it's children.
+     */
+    bool fCacheEnabled;
+    rc = CFGMR3QueryBoolDef(pCfg, "CacheEnabled", &fCacheEnabled, true);
+    AssertLogRelRCReturn(rc, rc);
+
+    LogRel(("pgmR3PoolInit: cMaxPages=%#RX16 cMaxUsers=%#RX16 cMaxPhysExts=%#RX16 fCacheEnable=%RTbool\n",
+             cMaxPages, cMaxUsers, cMaxPhysExts, fCacheEnabled));
+
+    /*
+     * Allocate the data structures.
+     */
+    uint32_t cb = RT_OFFSETOF(PGMPOOL, aPages[cMaxPages]);
+    cb += cMaxUsers * sizeof(PGMPOOLUSER);
+    cb += cMaxPhysExts * sizeof(PGMPOOLPHYSEXT);
+    PPGMPOOL pPool;
+    rc = MMR3HyperAllocOnceNoRel(pVM, cb, 0, MM_TAG_PGM_POOL, (void **)&pPool);
+    if (RT_FAILURE(rc))
+        return rc;
+    pVM->pgm.s.pPoolR3 = pPool;
+    pVM->pgm.s.pPoolR0 = MMHyperR3ToR0(pVM, pPool);
+    pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pPool);
+
+    /*
+     * Initialize it.
+     */
+    pPool->pVMR3     = pVM;
+    pPool->pVMR0     = pVM->pVMR0;
+    pPool->pVMRC     = pVM->pVMRC;
+    pPool->cMaxPages = cMaxPages;
+    pPool->cCurPages = PGMPOOL_IDX_FIRST;
+    pPool->iUserFreeHead = 0;
+    pPool->cMaxUsers = cMaxUsers;
+    PPGMPOOLUSER paUsers = (PPGMPOOLUSER)&pPool->aPages[pPool->cMaxPages];
+    pPool->paUsersR3 = paUsers;
+    pPool->paUsersR0 = MMHyperR3ToR0(pVM, paUsers);
+    pPool->paUsersRC = MMHyperR3ToRC(pVM, paUsers);
+    for (unsigned i = 0; i < cMaxUsers; i++)
+    {
+        paUsers[i].iNext = i + 1;
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iUserTable = 0xfffffffe;
+    }
+    paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
+    pPool->iPhysExtFreeHead = 0;
+    pPool->cMaxPhysExts = cMaxPhysExts;
+    PPGMPOOLPHYSEXT paPhysExts = (PPGMPOOLPHYSEXT)&paUsers[cMaxUsers];
+    pPool->paPhysExtsR3 = paPhysExts;
+    pPool->paPhysExtsR0 = MMHyperR3ToR0(pVM, paPhysExts);
+    pPool->paPhysExtsRC = MMHyperR3ToRC(pVM, paPhysExts);
+    for (unsigned i = 0; i < cMaxPhysExts; i++)
+    {
+        paPhysExts[i].iNext = i + 1;
+        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
+        pPool->aiHash[i] = NIL_PGMPOOL_IDX;
+    pPool->iAgeHead = NIL_PGMPOOL_IDX;
+    pPool->iAgeTail = NIL_PGMPOOL_IDX;
+    pPool->fCacheEnabled = fCacheEnabled;
+    pPool->pfnAccessHandlerR3 = pgmR3PoolAccessHandler;
+    pPool->pszAccessHandler = "Guest Paging Access Handler";
+    pPool->HCPhysTree = 0;
+
+    /* The NIL entry. */
+    Assert(NIL_PGMPOOL_IDX == 0);
+    pPool->aPages[NIL_PGMPOOL_IDX].enmKind = PGMPOOLKIND_INVALID;
+
+    /* The Shadow 32-bit PD. (32 bits guest paging) */
+    pPool->aPages[PGMPOOL_IDX_PD].Core.Key  = NIL_RTHCPHYS;
+    pPool->aPages[PGMPOOL_IDX_PD].GCPhys    = NIL_RTGCPHYS;
+    pPool->aPages[PGMPOOL_IDX_PD].pvPageR3  = 0;
+    pPool->aPages[PGMPOOL_IDX_PD].enmKind   = PGMPOOLKIND_32BIT_PD;
+    pPool->aPages[PGMPOOL_IDX_PD].idx       = PGMPOOL_IDX_PD;
+
+    /* The Shadow PDPT. */
+    pPool->aPages[PGMPOOL_IDX_PDPT].Core.Key  = NIL_RTHCPHYS;
+    pPool->aPages[PGMPOOL_IDX_PDPT].GCPhys    = NIL_RTGCPHYS;
+    pPool->aPages[PGMPOOL_IDX_PDPT].pvPageR3  = 0;
+    pPool->aPages[PGMPOOL_IDX_PDPT].enmKind   = PGMPOOLKIND_PAE_PDPT;
+    pPool->aPages[PGMPOOL_IDX_PDPT].idx       = PGMPOOL_IDX_PDPT;
+
+    /* The Shadow AMD64 CR3. */
+    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].Core.Key  = NIL_RTHCPHYS;
+    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].GCPhys    = NIL_RTGCPHYS;
+    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].pvPageR3  = 0;
+    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].enmKind   = PGMPOOLKIND_64BIT_PML4;
+    pPool->aPages[PGMPOOL_IDX_AMD64_CR3].idx       = PGMPOOL_IDX_AMD64_CR3;
+
+    /* The Nested Paging CR3. */
+    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].Core.Key  = NIL_RTHCPHYS;
+    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].GCPhys    = NIL_RTGCPHYS;
+    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].pvPageR3  = 0;
+    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].enmKind   = PGMPOOLKIND_ROOT_NESTED;
+    pPool->aPages[PGMPOOL_IDX_NESTED_ROOT].idx       = PGMPOOL_IDX_NESTED_ROOT;
+
+    /*
+     * Set common stuff.
+     */
+    for (unsigned iPage = 1; iPage < PGMPOOL_IDX_FIRST; iPage++)
+    {
+        pPool->aPages[iPage].iNext          = NIL_PGMPOOL_IDX;
+        pPool->aPages[iPage].iUserHead      = NIL_PGMPOOL_USER_INDEX;
+        pPool->aPages[iPage].iModifiedNext  = NIL_PGMPOOL_IDX;
+        pPool->aPages[iPage].iModifiedPrev  = NIL_PGMPOOL_IDX;
+        pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
+        pPool->aPages[iPage].iMonitoredNext = NIL_PGMPOOL_IDX;
+        pPool->aPages[iPage].iAgeNext       = NIL_PGMPOOL_IDX;
+        pPool->aPages[iPage].iAgePrev       = NIL_PGMPOOL_IDX;
+        Assert(pPool->aPages[iPage].idx == iPage);
+        Assert(pPool->aPages[iPage].GCPhys == NIL_RTGCPHYS);
+        Assert(!pPool->aPages[iPage].fSeenNonGlobal);
+        Assert(!pPool->aPages[iPage].fMonitored);
+        Assert(!pPool->aPages[iPage].fCached);
+        Assert(!pPool->aPages[iPage].fZeroed);
+        Assert(!pPool->aPages[iPage].fReusedFlushPending);
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Register statistics.
+     */
+    STAM_REG(pVM, &pPool->cCurPages,                    STAMTYPE_U16,       "/PGM/Pool/cCurPages",      STAMUNIT_PAGES,             "Current pool size.");
+    STAM_REG(pVM, &pPool->cMaxPages,                    STAMTYPE_U16,       "/PGM/Pool/cMaxPages",      STAMUNIT_PAGES,             "Max pool size.");
+    STAM_REG(pVM, &pPool->cUsedPages,                   STAMTYPE_U16,       "/PGM/Pool/cUsedPages",     STAMUNIT_PAGES,             "The number of pages currently in use.");
+    STAM_REG(pVM, &pPool->cUsedPagesHigh,               STAMTYPE_U16_RESET, "/PGM/Pool/cUsedPagesHigh", STAMUNIT_PAGES,             "The high watermark for cUsedPages.");
+    STAM_REG(pVM, &pPool->StatAlloc,                  STAMTYPE_PROFILE_ADV, "/PGM/Pool/Alloc",          STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolAlloc.");
+    STAM_REG(pVM, &pPool->StatClearAll,                 STAMTYPE_PROFILE,   "/PGM/Pool/ClearAll",       STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmR3PoolClearAll.");
+    STAM_REG(pVM, &pPool->StatR3Reset,                  STAMTYPE_PROFILE,   "/PGM/Pool/R3Reset",        STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmR3PoolReset.");
+    STAM_REG(pVM, &pPool->StatFlushPage,                STAMTYPE_PROFILE,   "/PGM/Pool/FlushPage",      STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolFlushPage.");
+    STAM_REG(pVM, &pPool->StatFree,                     STAMTYPE_PROFILE,   "/PGM/Pool/Free",           STAMUNIT_TICKS_PER_CALL,    "Profiling of pgmPoolFree.");
+    STAM_REG(pVM, &pPool->StatForceFlushPage,           STAMTYPE_COUNTER,   "/PGM/Pool/FlushForce",     STAMUNIT_OCCURENCES,        "Counting explicit flushes by PGMPoolFlushPage().");
+    STAM_REG(pVM, &pPool->StatForceFlushDirtyPage,      STAMTYPE_COUNTER,   "/PGM/Pool/FlushForceDirty",     STAMUNIT_OCCURENCES,   "Counting explicit flushes of dirty pages by PGMPoolFlushPage().");
+    STAM_REG(pVM, &pPool->StatForceFlushReused,         STAMTYPE_COUNTER,   "/PGM/Pool/FlushReused",    STAMUNIT_OCCURENCES,        "Counting flushes for reused pages.");
+    STAM_REG(pVM, &pPool->StatZeroPage,                 STAMTYPE_PROFILE,   "/PGM/Pool/ZeroPage",       STAMUNIT_TICKS_PER_CALL,    "Profiling time spent zeroing pages. Overlaps with Alloc.");
+    STAM_REG(pVM, &pPool->cMaxUsers,                    STAMTYPE_U16,       "/PGM/Pool/Track/cMaxUsers",            STAMUNIT_COUNT,      "Max user tracking records.");
+    STAM_REG(pVM, &pPool->cPresent,                     STAMTYPE_U32,       "/PGM/Pool/Track/cPresent",             STAMUNIT_COUNT,      "Number of present page table entries.");
+    STAM_REG(pVM, &pPool->StatTrackDeref,               STAMTYPE_PROFILE,   "/PGM/Pool/Track/Deref",                STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackDeref.");
+    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPT,       STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPT",        STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPT.");
+    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTs,      STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPTs",       STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTs.");
+    STAM_REG(pVM, &pPool->StatTrackFlushGCPhysPTsSlow,  STAMTYPE_PROFILE,   "/PGM/Pool/Track/FlushGCPhysPTsSlow",   STAMUNIT_TICKS_PER_CALL, "Profiling of pgmPoolTrackFlushGCPhysPTsSlow.");
+    STAM_REG(pVM, &pPool->StatTrackFlushEntry,          STAMTYPE_COUNTER,   "/PGM/Pool/Track/Entry/Flush",          STAMUNIT_COUNT,          "Nr of flushed entries.");
+    STAM_REG(pVM, &pPool->StatTrackFlushEntryKeep,      STAMTYPE_COUNTER,   "/PGM/Pool/Track/Entry/Update",         STAMUNIT_COUNT,          "Nr of updated entries.");
+    STAM_REG(pVM, &pPool->StatTrackFreeUpOneUser,       STAMTYPE_COUNTER,   "/PGM/Pool/Track/FreeUpOneUser",        STAMUNIT_TICKS_PER_CALL, "The number of times we were out of user tracking records.");
+    STAM_REG(pVM, &pPool->StatTrackDerefGCPhys,         STAMTYPE_PROFILE,   "/PGM/Pool/Track/DrefGCPhys",           STAMUNIT_TICKS_PER_CALL, "Profiling deref activity related tracking GC physical pages.");
+    STAM_REG(pVM, &pPool->StatTrackLinearRamSearches,   STAMTYPE_COUNTER,   "/PGM/Pool/Track/LinearRamSearches",    STAMUNIT_OCCURENCES, "The number of times we had to do linear ram searches.");
+    STAM_REG(pVM, &pPool->StamTrackPhysExtAllocFailures,STAMTYPE_COUNTER,   "/PGM/Pool/Track/PhysExtAllocFailures", STAMUNIT_OCCURENCES, "The number of failing pgmPoolTrackPhysExtAlloc calls.");
+    STAM_REG(pVM, &pPool->StatMonitorRZ,                STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorRZEmulateInstr,    STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFlushPage,       STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFlushReinit,     STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFlushModOverflow,STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFork,            STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
+    STAM_REG(pVM, &pPool->StatMonitorRZHandled,         STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the RC/R0 access we've handled (except REP STOSD).");
+    STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch1,  STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/IntrFailPatch1",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction.");
+    STAM_REG(pVM, &pPool->StatMonitorRZIntrFailPatch2,  STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/IntrFailPatch2",  STAMUNIT_OCCURENCES,     "Times we've failed interpreting a patch code instruction during flushing.");
+    STAM_REG(pVM, &pPool->StatMonitorRZRepPrefix,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
+    STAM_REG(pVM, &pPool->StatMonitorRZRepStosd,        STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/RZ/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPT,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PT",        STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPD,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PD",        STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPDPT,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PDPT",      STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorRZFaultPML4,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/RZ/Fault/PML4",      STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3,                STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3",                 STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorR3EmulateInstr,    STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/EmulateInstr",    STAMUNIT_OCCURENCES,     "Times we've failed interpreting the instruction.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FlushPage,       STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/FlushPage",       STAMUNIT_TICKS_PER_CALL, "Profiling the pgmPoolFlushPage calls made from the R3 access handler.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FlushReinit,     STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/FlushReinit",     STAMUNIT_OCCURENCES,     "Times we've detected a page table reinit.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FlushModOverflow,STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/FlushOverflow",   STAMUNIT_OCCURENCES,     "Counting flushes for pages that are modified too often.");
+    STAM_REG(pVM, &pPool->StatMonitorR3Fork,            STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fork",            STAMUNIT_OCCURENCES,     "Times we've detected fork().");
+    STAM_REG(pVM, &pPool->StatMonitorR3Handled,         STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/Handled",         STAMUNIT_TICKS_PER_CALL, "Profiling the R3 access we've handled (except REP STOSD).");
+    STAM_REG(pVM, &pPool->StatMonitorR3RepPrefix,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/RepPrefix",       STAMUNIT_OCCURENCES,     "The number of times we've seen rep prefixes we can't handle.");
+    STAM_REG(pVM, &pPool->StatMonitorR3RepStosd,        STAMTYPE_PROFILE,   "/PGM/Pool/Monitor/R3/RepStosd",        STAMUNIT_TICKS_PER_CALL, "Profiling the REP STOSD cases we've handled.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPT,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PT",        STAMUNIT_OCCURENCES,     "Nr of handled PT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPD,         STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PD",        STAMUNIT_OCCURENCES,     "Nr of handled PD faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPDPT,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PDPT",      STAMUNIT_OCCURENCES,     "Nr of handled PDPT faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3FaultPML4,       STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Fault/PML4",      STAMUNIT_OCCURENCES,     "Nr of handled PML4 faults.");
+    STAM_REG(pVM, &pPool->StatMonitorR3Async,           STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/R3/Async",           STAMUNIT_OCCURENCES,     "Times we're called in an async thread and need to flush.");
+    STAM_REG(pVM, &pPool->cModifiedPages,               STAMTYPE_U16,       "/PGM/Pool/Monitor/cModifiedPages",     STAMUNIT_PAGES,          "The current cModifiedPages value.");
+    STAM_REG(pVM, &pPool->cModifiedPagesHigh,           STAMTYPE_U16_RESET, "/PGM/Pool/Monitor/cModifiedPagesHigh", STAMUNIT_PAGES,          "The high watermark for cModifiedPages.");
+    STAM_REG(pVM, &pPool->StatResetDirtyPages,          STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/Resets",       STAMUNIT_OCCURENCES,     "Times we've called pgmPoolResetDirtyPages (and there were dirty page).");
+    STAM_REG(pVM, &pPool->StatDirtyPage,                STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/Pages",        STAMUNIT_OCCURENCES,     "Times we've called pgmPoolAddDirtyPage.");
+    STAM_REG(pVM, &pPool->StatDirtyPageDupFlush,        STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/FlushDup",     STAMUNIT_OCCURENCES,     "Times we've had to flush duplicates for dirty page management.");
+    STAM_REG(pVM, &pPool->StatDirtyPageOverFlowFlush,   STAMTYPE_COUNTER,   "/PGM/Pool/Monitor/Dirty/FlushOverflow",STAMUNIT_OCCURENCES,     "Times we've had to flush because of overflow.");
+    STAM_REG(pVM, &pPool->StatCacheHits,                STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Hits",                 STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls satisfied by the cache.");
+    STAM_REG(pVM, &pPool->StatCacheMisses,              STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Misses",               STAMUNIT_OCCURENCES, "The number of pgmPoolAlloc calls not statisfied by the cache.");
+    STAM_REG(pVM, &pPool->StatCacheKindMismatches,      STAMTYPE_COUNTER,   "/PGM/Pool/Cache/KindMismatches",       STAMUNIT_OCCURENCES, "The number of shadow page kind mismatches. (Better be low, preferably 0!)");
+    STAM_REG(pVM, &pPool->StatCacheFreeUpOne,           STAMTYPE_COUNTER,   "/PGM/Pool/Cache/FreeUpOne",            STAMUNIT_OCCURENCES, "The number of times the cache was asked to free up a page.");
+    STAM_REG(pVM, &pPool->StatCacheCacheable,           STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Cacheable",            STAMUNIT_OCCURENCES, "The number of cacheable allocations.");
+    STAM_REG(pVM, &pPool->StatCacheUncacheable,         STAMTYPE_COUNTER,   "/PGM/Pool/Cache/Uncacheable",          STAMUNIT_OCCURENCES, "The number of uncacheable allocations.");
+#endif /* VBOX_WITH_STATISTICS */
+
+#ifdef VBOX_WITH_DEBUGGER
+    /*
+     * Debugger commands.
+     */
+    static bool s_fRegisteredCmds = false;
+    if (!s_fRegisteredCmds)
+    {
+        rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+        if (RT_SUCCESS(rc))
+            s_fRegisteredCmds = true;
+    }
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Relocate the page pool data.
+ *
+ * @param   pVM     The VM handle.
+ */
+void pgmR3PoolRelocate(PVM pVM)
+{
+    pVM->pgm.s.pPoolRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3);
+    pVM->pgm.s.pPoolR3->pVMRC = pVM->pVMRC;
+    pVM->pgm.s.pPoolR3->paUsersRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paUsersR3);
+    pVM->pgm.s.pPoolR3->paPhysExtsRC = MMHyperR3ToRC(pVM, pVM->pgm.s.pPoolR3->paPhysExtsR3);
+    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerRC);
+    AssertReleaseRC(rc);
+    /* init order hack. */
+    if (!pVM->pgm.s.pPoolR3->pfnAccessHandlerR0)
+    {
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "pgmPoolAccessHandler", &pVM->pgm.s.pPoolR3->pfnAccessHandlerR0);
+        AssertReleaseRC(rc);
+    }
+}
+
+
+/**
+ * Grows the shadow page pool.
+ *
+ * I.e. adds more pages to it, assuming that hasn't reached cMaxPages yet.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+VMMR3DECL(int) PGMR3PoolGrow(PVM pVM)
+{
+    PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
+    AssertReturn(pPool->cCurPages < pPool->cMaxPages, VERR_INTERNAL_ERROR);
+
+    pgmLock(pVM);
+
+    /*
+     * How much to grow it by?
+     */
+    uint32_t cPages = pPool->cMaxPages - pPool->cCurPages;
+    cPages = RT_MIN(PGMPOOL_CFG_MAX_GROW, cPages);
+    LogFlow(("PGMR3PoolGrow: Growing the pool by %d (%#x) pages.\n", cPages, cPages));
+
+    for (unsigned i = pPool->cCurPages; cPages-- > 0; i++)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+
+        /* Allocate all pages in low (below 4 GB) memory as 32 bits guests need a page table root in low memory. */
+        pPage->pvPageR3 = MMR3PageAllocLow(pVM);
+        if (!pPage->pvPageR3)
+        {
+            Log(("We're out of memory!! i=%d\n", i));
+            pgmUnlock(pVM);
+            return i ? VINF_SUCCESS : VERR_NO_PAGE_MEMORY;
+        }
+        pPage->Core.Key  = MMPage2Phys(pVM, pPage->pvPageR3);
+        AssertFatal(pPage->Core.Key < _4G);
+        pPage->GCPhys    = NIL_RTGCPHYS;
+        pPage->enmKind   = PGMPOOLKIND_FREE;
+        pPage->idx       = pPage - &pPool->aPages[0];
+        LogFlow(("PGMR3PoolGrow: insert page #%#x - %RHp\n", pPage->idx, pPage->Core.Key));
+        pPage->iNext     = pPool->iFreeHead;
+        pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
+        pPage->iModifiedNext  = NIL_PGMPOOL_IDX;
+        pPage->iModifiedPrev  = NIL_PGMPOOL_IDX;
+        pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+        pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+        pPage->iAgeNext  = NIL_PGMPOOL_IDX;
+        pPage->iAgePrev  = NIL_PGMPOOL_IDX;
+        /* commit it */
+        bool fRc = RTAvloHCPhysInsert(&pPool->HCPhysTree, &pPage->Core); Assert(fRc); NOREF(fRc);
+        pPool->iFreeHead = i;
+        pPool->cCurPages = i + 1;
+    }
+
+    pgmUnlock(pVM);
+    Assert(pPool->cCurPages <= pPool->cMaxPages);
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Worker used by pgmR3PoolAccessHandler when it's invoked by an async thread.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page.
+ */
+static DECLCALLBACK(void) pgmR3PoolFlushReusedPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    /* for the present this should be safe enough I think... */
+    pgmLock(pPool->pVMR3);
+    if (    pPage->fReusedFlushPending
+        &&  pPage->enmKind != PGMPOOLKIND_FREE)
+        pgmPoolFlushPage(pPool, pPage);
+    pgmUnlock(pPool->pVMR3);
+}
+
+
+/**
+ * \#PF Handler callback for PT write accesses.
+ *
+ * The handler can not raise any faults, it's mainly for monitoring write access
+ * to certain pages.
+ *
+ * @returns VINF_SUCCESS if the handler has carried out the operation.
+ * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
+ * @param   pVM             VM Handle.
+ * @param   GCPhys          The physical address the guest is writing to.
+ * @param   pvPhys          The HC mapping of that address.
+ * @param   pvBuf           What the guest is reading/writing.
+ * @param   cbBuf           How much it's reading/writing.
+ * @param   enmAccessType   The access type.
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) pgmR3PoolAccessHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
+{
+    STAM_PROFILE_START(&pVM->pgm.s.pPoolR3->StatMonitorR3, a);
+    PPGMPOOL pPool = pVM->pgm.s.pPoolR3;
+    PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
+    LogFlow(("pgmR3PoolAccessHandler: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
+             GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
+
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    /*
+     * We don't have to be very sophisticated about this since there are relativly few calls here.
+     * However, we must try our best to detect any non-cpu accesses (disk / networking).
+     *
+     * Just to make life more interesting, we'll have to deal with the async threads too.
+     * We cannot flush a page if we're in an async thread because of REM notifications.
+     */
+    pgmLock(pVM);
+    if (PHYS_PAGE_ADDRESS(GCPhys) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
+    {
+        /* Pool page changed while we were waiting for the lock; ignore. */
+        Log(("CPU%d: pgmR3PoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
+        pgmUnlock(pVM);
+        return VINF_PGM_HANDLER_DO_DEFAULT;
+    }
+
+    Assert(pPage->enmKind != PGMPOOLKIND_FREE);
+
+    /* @todo this code doesn't make any sense. remove the if (!pVCpu) block */
+    if (!pVCpu) /** @todo This shouldn't happen any longer, all access handlers will be called on an EMT. All ring-3 handlers, except MMIO, already own the PGM lock. @bugref{3170} */
+    {
+        Log(("pgmR3PoolAccessHandler: async thread, requesting EMT to flush the page: %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
+             pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
+        STAM_COUNTER_INC(&pPool->StatMonitorR3Async);
+        if (!pPage->fReusedFlushPending)
+        {
+            pgmUnlock(pVM);
+            int rc = VMR3ReqCallVoidNoWait(pPool->pVMR3, VMCPUID_ANY, (PFNRT)pgmR3PoolFlushReusedPage, 2, pPool, pPage);
+            AssertRCReturn(rc, rc);
+            pgmLock(pVM);
+            pPage->fReusedFlushPending = true;
+            pPage->cModifications += 0x1000;
+        }
+
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
+        /** @todo r=bird: making unsafe assumption about not crossing entries here! */
+        while (cbBuf > 4)
+        {
+            cbBuf -= 4;
+            pvPhys = (uint8_t *)pvPhys + 4;
+            GCPhys += 4;
+            pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
+        }
+        STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
+    }
+    else if (    (   pPage->cModifications < 96 /* it's cheaper here. */
+                  || pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
+                  )
+             &&  cbBuf <= 4)
+    {
+        /* Clear the shadow entry. */
+        if (!pPage->cModifications++)
+            pgmPoolMonitorModifiedInsert(pPool, pPage);
+        /** @todo r=bird: making unsafe assumption about not crossing entries here! */
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvPhys, 0 /* unknown write size */);
+        STAM_PROFILE_STOP(&pPool->StatMonitorR3, a);
+    }
+    else
+    {
+        pgmPoolMonitorChainFlush(pPool, pPage); /* ASSUME that VERR_PGM_POOL_CLEARED can be ignored here and that FFs will deal with it in due time. */
+        STAM_PROFILE_STOP_EX(&pPool->StatMonitorR3, &pPool->StatMonitorR3FlushPage, a);
+    }
+    pgmUnlock(pVM);
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+/**
+ * Rendezvous callback used by pgmR3PoolClearAll that clears all shadow pages
+ * and all modification counters.
+ *
+ * This is only called on one of the EMTs while the other ones are waiting for
+ * it to complete this function.
+ *
+ * @returns VINF_SUCCESS (VBox strict status code).
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The VMCPU for the EMT we're being called on. Unused.
+ * @param   fpvFlushRemTlb  When not NULL, we'll flush the REM TLB as well.
+ *                          (This is the pvUser, so it has to be void *.)
+ *
+ */
+DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_START(&pPool->StatClearAll, c);
+
+    pgmLock(pVM);
+    Log(("pgmR3PoolClearAllRendezvous: cUsedPages=%d fpvFlushRemTbl=%RTbool\n", pPool->cUsedPages, !!fpvFlushRemTbl));
+
+    /*
+     * Iterate all the pages until we've encountered all that are in use.
+     * This is a simple but not quite optimal solution.
+     */
+    unsigned cModifiedPages = 0; NOREF(cModifiedPages);
+    unsigned cLeft = pPool->cUsedPages;
+    unsigned iPage = pPool->cCurPages;
+    while (--iPage >= PGMPOOL_IDX_FIRST)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
+        if (pPage->GCPhys != NIL_RTGCPHYS)
+        {
+            switch (pPage->enmKind)
+            {
+                /*
+                 * We only care about shadow page tables that reference physical memory
+                 */
+#ifdef PGM_WITH_LARGE_PAGES
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS: /* Large pages reference 2 MB of physical memory, so we must clear them. */
+                    if (pPage->cPresent)
+                    {
+                        PX86PDPAE pShwPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
+                        for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+                        {
+                            if (    pShwPD->a[i].n.u1Present
+                                &&  pShwPD->a[i].b.u1Size)
+                            {
+                                Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
+                                pShwPD->a[i].u = 0;
+                                Assert(pPage->cPresent);
+                                pPage->cPresent--;
+                            }
+                        }
+                        if (pPage->cPresent == 0)
+                            pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                    }
+                    goto default_case;
+
+                case PGMPOOLKIND_PAE_PD_PHYS:   /* Large pages reference 2 MB of physical memory, so we must clear them. */
+                    if (pPage->cPresent)
+                    {
+                        PEPTPD pShwPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
+                        for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+                        {
+                            Assert((pShwPD->a[i].u & UINT64_C(0xfff0000000000f80)) == 0);
+                            if (    pShwPD->a[i].n.u1Present
+                                &&  pShwPD->a[i].b.u1Size)
+                            {
+                                Assert(!(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING));
+                                pShwPD->a[i].u = 0;
+                                Assert(pPage->cPresent);
+                                pPage->cPresent--;
+                            }
+                        }
+                        if (pPage->cPresent == 0)
+                            pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                    }
+                    goto default_case;
+#endif /* PGM_WITH_LARGE_PAGES */
+
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                {
+                    if (pPage->cPresent)
+                    {
+                        void *pvShw = PGMPOOL_PAGE_2_PTR_V2(pPool->CTX_SUFF(pVM), pVCpu, pPage);
+                        STAM_PROFILE_START(&pPool->StatZeroPage, z);
+#if 0
+                        /* Useful check for leaking references; *very* expensive though. */
+                        switch (pPage->enmKind)
+                        {
+                            case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                            case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                            case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                            case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                            case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                            {
+                                bool fFoundFirst = false;
+                                PPGMSHWPTPAE pPT = (PPGMSHWPTPAE)pvShw;
+                                for (unsigned ptIndex = 0; ptIndex < RT_ELEMENTS(pPT->a); ptIndex++)
+                                {
+                                    if (pPT->a[ptIndex].u)
+                                    {
+                                        if (!fFoundFirst)
+                                        {
+                                            AssertFatalMsg(pPage->iFirstPresent <= ptIndex, ("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
+                                            if (pPage->iFirstPresent != ptIndex)
+                                                Log(("ptIndex = %d first present = %d\n", ptIndex, pPage->iFirstPresent));
+                                            fFoundFirst = true;
+                                        }
+                                        if (PGMSHWPTEPAE_IS_P(pPT->a[ptIndex]))
+                                        {
+                                            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pPT->a[ptIndex]), NIL_RTGCPHYS);
+                                            if (pPage->iFirstPresent == ptIndex)
+                                                pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                                        }
+                                    }
+                                }
+                                AssertFatalMsg(pPage->cPresent == 0, ("cPresent = %d pPage = %RGv\n", pPage->cPresent, pPage->GCPhys));
+                                break;
+                            }
+                            default:
+                                break;
+                        }
+#endif
+                        ASMMemZeroPage(pvShw);
+                        STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
+                        pPage->cPresent = 0;
+                        pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+                    }
+                }
+                /* fall thru */
+
+#ifdef PGM_WITH_LARGE_PAGES
+                default_case:
+#endif
+                default:
+                    Assert(!pPage->cModifications || ++cModifiedPages);
+                    Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
+                    Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
+                    pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+                    pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+                    pPage->cModifications = 0;
+                    break;
+
+            }
+            if (!--cLeft)
+                break;
+        }
+    }
+
+    /* swipe the special pages too. */
+    for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
+        if (pPage->GCPhys != NIL_RTGCPHYS)
+        {
+            Assert(!pPage->cModifications || ++cModifiedPages);
+            Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX || pPage->cModifications);
+            Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX || pPage->cModifications);
+            pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+            pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+            pPage->cModifications = 0;
+        }
+    }
+
+#ifndef DEBUG_michael
+    AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
+#endif
+    pPool->iModifiedHead = NIL_PGMPOOL_IDX;
+    pPool->cModifiedPages = 0;
+
+    /*
+     * Clear all the GCPhys links and rebuild the phys ext free list.
+     */
+    for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+            PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
+    }
+
+    pPool->iPhysExtFreeHead = 0;
+    PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+    const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
+    for (unsigned i = 0; i < cMaxPhysExts; i++)
+    {
+        paPhysExts[i].iNext = i + 1;
+        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    /* Reset all dirty pages to reactivate the page monitoring. */
+    /* Note: we must do this *after* clearing all page references and shadow page tables as there might be stale references to
+     *       recently removed MMIO ranges around that might otherwise end up asserting in pgmPoolTracDerefGCPhysHint
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+        PPGMPOOLPAGE pPage;
+        unsigned     idxPage;
+
+        if (pPool->aDirtyPages[i].uIdx == NIL_PGMPOOL_IDX)
+            continue;
+
+        idxPage = pPool->aDirtyPages[i].uIdx;
+        AssertRelease(idxPage != NIL_PGMPOOL_IDX);
+        pPage = &pPool->aPages[idxPage];
+        Assert(pPage->idx == idxPage);
+        Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+        AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, i));
+
+        Log(("Reactivate dirty page %RGp\n", pPage->GCPhys));
+
+        /* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
+        int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & PAGE_BASE_GC_MASK);
+        Assert(rc == VINF_SUCCESS);
+        pPage->fDirty = false;
+
+        pPool->aDirtyPages[i].uIdx = NIL_PGMPOOL_IDX;
+    }
+
+    /* Clear all dirty pages. */
+    pPool->idxFreeDirtyPage = 0;
+    pPool->cDirtyPages      = 0;
+#endif
+
+    /* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        pVM->aCpus[idCpu].pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
+
+    /* Flush job finished. */
+    VM_FF_CLEAR(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
+    pPool->cPresent = 0;
+    pgmUnlock(pVM);
+
+    PGM_INVL_ALL_VCPU_TLBS(pVM);
+
+    if (fpvFlushRemTbl)
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            CPUMSetChangedFlags(&pVM->aCpus[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
+
+    STAM_PROFILE_STOP(&pPool->StatClearAll, c);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Clears the shadow page pool.
+ *
+ * @param   pVM             The VM handle.
+ * @param   fFlushRemTlb    When set, the REM TLB is scheduled for flushing as
+ *                          well.
+ */
+void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb)
+{
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PoolClearAllRendezvous, &fFlushRemTlb);
+    AssertRC(rc);
+}
+
+/**
+ * Protect all pgm pool page table entries to monitor writes
+ *
+ * @param   pVM         The VM handle.
+ *
+ * Remark: assumes the caller will flush all TLBs (!!)
+ */
+void pgmR3PoolWriteProtectPages(PVM pVM)
+{
+    Assert(PGMIsLockOwner(pVM));
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    unsigned cLeft = pPool->cUsedPages;
+    unsigned iPage = pPool->cCurPages;
+    while (--iPage >= PGMPOOL_IDX_FIRST)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
+        if (    pPage->GCPhys != NIL_RTGCPHYS
+            &&  pPage->cPresent)
+        {
+            union
+            {
+                void           *pv;
+                PX86PT          pPT;
+                PPGMSHWPTPAE    pPTPae;
+                PEPTPT          pPTEpt;
+            } uShw;
+            uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+            switch (pPage->enmKind)
+            {
+                /*
+                 * We only care about shadow page tables.
+                 */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPT->a); iShw++)
+                    {
+                        if (uShw.pPT->a[iShw].n.u1Present)
+                            uShw.pPT->a[iShw].n.u1Write = 0;
+                    }
+                    break;
+
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTPae->a); iShw++)
+                    {
+                        if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
+                            PGMSHWPTEPAE_SET_RO(uShw.pPTPae->a[iShw]);
+                    }
+                    break;
+
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                    for (unsigned iShw = 0; iShw < RT_ELEMENTS(uShw.pPTEpt->a); iShw++)
+                    {
+                        if (uShw.pPTEpt->a[iShw].n.u1Present)
+                            uShw.pPTEpt->a[iShw].n.u1Write = 0;
+                    }
+                    break;
+
+                default:
+                    break;
+            }
+            if (!--cLeft)
+                break;
+        }
+    }
+}
+
+#ifdef VBOX_WITH_DEBUGGER
+/**
+ * The '.pgmpoolcheck' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) pgmR3PoolCmdCheck(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires a VM to be selected.\n");
+
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    for (unsigned i = 0; i < pPool->cCurPages; i++)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+        bool fFirstMsg = true;
+
+        /* Todo: cover other paging modes too. */
+        if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+        {
+            PPGMSHWPTPAE pShwPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
+            {
+                PX86PTPAE pGstPT;
+                PGMPAGEMAPLOCK LockPage;
+                int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, pPage->GCPhys, (const void **)&pGstPT, &LockPage);     AssertReleaseRC(rc);
+
+                /* Check if any PTEs are out of sync. */
+                for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
+                {
+                    if (PGMSHWPTEPAE_IS_P(pShwPT->a[j]))
+                    {
+                        RTHCPHYS HCPhys = NIL_RTHCPHYS;
+                        rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[j].u & X86_PTE_PAE_PG_MASK, &HCPhys);
+                        if (   rc != VINF_SUCCESS
+                            || PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[j]) != HCPhys)
+                        {
+                            if (fFirstMsg)
+                            {
+                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
+                                fFirstMsg = false;
+                            }
+                            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch HCPhys: rc=%Rrc idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
+                        }
+                        else if (   PGMSHWPTEPAE_IS_RW(pShwPT->a[j])
+                                 && !pGstPT->a[j].n.u1Write)
+                        {
+                            if (fFirstMsg)
+                            {
+                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
+                                fFirstMsg = false;
+                            }
+                            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch r/w gst/shw: idx=%d guest %RX64 shw=%RX64 vs %RHp\n", j, pGstPT->a[j].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), HCPhys);
+                        }
+                    }
+                }
+                PGMPhysReleasePageMappingLock(pVM, &LockPage);
+            }
+
+            /* Make sure this page table can't be written to from any shadow mapping. */
+            RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
+            int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pPage->GCPhys, &HCPhysPT);
+            AssertMsgRC(rc, ("PGMPhysGCPhys2HCPhys failed with rc=%d for %RGp\n", rc, pPage->GCPhys));
+            if (rc == VINF_SUCCESS)
+            {
+                for (unsigned j = 0; j < pPool->cCurPages; j++)
+                {
+                    PPGMPOOLPAGE pTempPage = &pPool->aPages[j];
+
+                    if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+                    {
+                        PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pTempPage);
+
+                        for (unsigned k = 0; k < RT_ELEMENTS(pShwPT->a); k++)
+                        {
+                            if (    PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[k])
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                                &&  !pPage->fDirty
+# endif
+                                &&  PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[k]) == HCPhysPT)
+                            {
+                                if (fFirstMsg)
+                                {
+                                    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Check pool page %RGp\n", pPage->GCPhys);
+                                    fFirstMsg = false;
+                                }
+                                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Mismatch: r/w: GCPhys=%RGp idx=%d shw %RX64 %RX64\n", pTempPage->GCPhys, k, PGMSHWPTEPAE_GET_LOG(pShwPT->a[k]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[k]));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_DEBUGGER */
Index: /trunk/src/VBox/VMM/VMMR3/PGMSavedState.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMSavedState.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMSavedState.cpp	(revision 35333)
@@ -0,0 +1,3200 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager and Monitor, The Saved State Part.
+ */
+
+/*
+ * Copyright (C) 2006-2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#include <VBox/pgm.h>
+#include <VBox/stam.h>
+#include <VBox/ssm.h>
+#include <VBox/pdmdrv.h>
+#include <VBox/pdmdev.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include "PGMInline.h"
+
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/ftm.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/crc.h>
+#include <iprt/mem.h>
+#include <iprt/sha.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** Saved state data unit version.
+ * @todo remove the guest mappings from the saved state at next version change! */
+#define PGM_SAVED_STATE_VERSION                 12
+/** Saved state before the balloon change. */
+#define PGM_SAVED_STATE_VERSION_PRE_BALLOON     11
+/** Saved state data unit version used during 3.1 development, misses the RAM
+ *  config. */
+#define PGM_SAVED_STATE_VERSION_NO_RAM_CFG      10
+/** Saved state data unit version for 3.0 (pre teleportation). */
+#define PGM_SAVED_STATE_VERSION_3_0_0           9
+/** Saved state data unit version for 2.2.2 and later. */
+#define PGM_SAVED_STATE_VERSION_2_2_2           8
+/** Saved state data unit version for 2.2.0. */
+#define PGM_SAVED_STATE_VERSION_RR_DESC         7
+/** Saved state data unit version. */
+#define PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE   6
+
+
+/** @name Sparse state record types
+ * @{  */
+/** Zero page. No data. */
+#define PGM_STATE_REC_RAM_ZERO          UINT8_C(0x00)
+/** Raw page. */
+#define PGM_STATE_REC_RAM_RAW           UINT8_C(0x01)
+/** Raw MMIO2 page. */
+#define PGM_STATE_REC_MMIO2_RAW         UINT8_C(0x02)
+/** Zero MMIO2 page. */
+#define PGM_STATE_REC_MMIO2_ZERO        UINT8_C(0x03)
+/** Virgin ROM page. Followed by protection (8-bit) and the raw bits. */
+#define PGM_STATE_REC_ROM_VIRGIN        UINT8_C(0x04)
+/** Raw shadowed ROM page. The protection (8-bit) precedes the raw bits. */
+#define PGM_STATE_REC_ROM_SHW_RAW       UINT8_C(0x05)
+/** Zero shadowed ROM page. The protection (8-bit) is the only payload. */
+#define PGM_STATE_REC_ROM_SHW_ZERO      UINT8_C(0x06)
+/** ROM protection (8-bit). */
+#define PGM_STATE_REC_ROM_PROT          UINT8_C(0x07)
+/** The last record type. */
+#define PGM_STATE_REC_LAST              PGM_STATE_REC_ROM_PROT
+/** End marker. */
+#define PGM_STATE_REC_END               UINT8_C(0xff)
+/** Flag indicating that the data is preceded by the page address.
+ *  For RAW pages this is a RTGCPHYS.  For MMIO2 and ROM pages this is a 8-bit
+ *  range ID and a 32-bit page index.
+ */
+#define PGM_STATE_REC_FLAG_ADDR         UINT8_C(0x80)
+/** @} */
+
+/** The CRC-32 for a zero page. */
+#define PGM_STATE_CRC32_ZERO_PAGE       UINT32_C(0xc71c0011)
+/** The CRC-32 for a zero half page. */
+#define PGM_STATE_CRC32_ZERO_HALF_PAGE  UINT32_C(0xf1e8ba9e)
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/** For loading old saved states. (pre-smp) */
+typedef struct
+{
+    /** If set no conflict checks are required.  (boolean) */
+    bool                            fMappingsFixed;
+    /** Size of fixed mapping */
+    uint32_t                        cbMappingFixed;
+    /** Base address (GC) of fixed mapping */
+    RTGCPTR                         GCPtrMappingFixed;
+    /** A20 gate mask.
+     * Our current approach to A20 emulation is to let REM do it and don't bother
+     * anywhere else. The interesting guests will be operating with it enabled anyway.
+     * But should the need arise, we'll subject physical addresses to this mask. */
+    RTGCPHYS                        GCPhysA20Mask;
+    /** A20 gate state - boolean! */
+    bool                            fA20Enabled;
+    /** The guest paging mode. */
+    PGMMODE                         enmGuestMode;
+} PGMOLD;
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/** PGM fields to save/load. */
+
+static const SSMFIELD s_aPGMFields[] =
+{
+    SSMFIELD_ENTRY(         PGM, fMappingsFixed),
+    SSMFIELD_ENTRY_GCPTR(   PGM, GCPtrMappingFixed),
+    SSMFIELD_ENTRY(         PGM, cbMappingFixed),
+    SSMFIELD_ENTRY(         PGM, cBalloonedPages),
+    SSMFIELD_ENTRY_TERM()
+};
+
+static const SSMFIELD s_aPGMFieldsPreBalloon[] =
+{
+    SSMFIELD_ENTRY(         PGM, fMappingsFixed),
+    SSMFIELD_ENTRY_GCPTR(   PGM, GCPtrMappingFixed),
+    SSMFIELD_ENTRY(         PGM, cbMappingFixed),
+    SSMFIELD_ENTRY_TERM()
+};
+
+static const SSMFIELD s_aPGMCpuFields[] =
+{
+    SSMFIELD_ENTRY(         PGMCPU, fA20Enabled),
+    SSMFIELD_ENTRY_GCPHYS(  PGMCPU, GCPhysA20Mask),
+    SSMFIELD_ENTRY(         PGMCPU, enmGuestMode),
+    SSMFIELD_ENTRY_TERM()
+};
+
+static const SSMFIELD s_aPGMFields_Old[] =
+{
+    SSMFIELD_ENTRY(         PGMOLD, fMappingsFixed),
+    SSMFIELD_ENTRY_GCPTR(   PGMOLD, GCPtrMappingFixed),
+    SSMFIELD_ENTRY(         PGMOLD, cbMappingFixed),
+    SSMFIELD_ENTRY(         PGMOLD, fA20Enabled),
+    SSMFIELD_ENTRY_GCPHYS(  PGMOLD, GCPhysA20Mask),
+    SSMFIELD_ENTRY(         PGMOLD, enmGuestMode),
+    SSMFIELD_ENTRY_TERM()
+};
+
+
+/**
+ * Find the ROM tracking structure for the given page.
+ *
+ * @returns Pointer to the ROM page structure. NULL if the caller didn't check
+ *          that it's a ROM page.
+ * @param   pVM         The VM handle.
+ * @param   GCPhys      The address of the ROM page.
+ */
+static PPGMROMPAGE pgmR3GetRomPage(PVM pVM, RTGCPHYS GCPhys) /** @todo change this to take a hint. */
+{
+    for (PPGMROMRANGE pRomRange = pVM->pgm.s.CTX_SUFF(pRomRanges);
+         pRomRange;
+         pRomRange = pRomRange->CTX_SUFF(pNext))
+    {
+        RTGCPHYS off = GCPhys - pRomRange->GCPhys;
+        if (GCPhys - pRomRange->GCPhys < pRomRange->cb)
+            return &pRomRange->aPages[off >> PAGE_SHIFT];
+    }
+    return NULL;
+}
+
+
+/**
+ * Prepares the ROM pages for a live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ */
+static int pgmR3PrepRomPages(PVM pVM)
+{
+    /*
+     * Initialize the live save tracking in the ROM page descriptors.
+     */
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        PPGMRAMRANGE    pRamHint = NULL;;
+        uint32_t const  cPages   = pRom->cb >> PAGE_SHIFT;
+
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            pRom->aPages[iPage].LiveSave.u8Prot           = (uint8_t)PGMROMPROT_INVALID;
+            pRom->aPages[iPage].LiveSave.fWrittenTo       = false;
+            pRom->aPages[iPage].LiveSave.fDirty           = true;
+            pRom->aPages[iPage].LiveSave.fDirtiedRecently = true;
+            if (!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
+            {
+                if (PGMROMPROT_IS_ROM(pRom->aPages[iPage].enmProt))
+                    pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow) && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow);
+                else
+                {
+                    RTGCPHYS GCPhys = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                    PPGMPAGE pPage;
+                    int rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pPage, &pRamHint);
+                    AssertLogRelMsgRC(rc, ("%Rrc GCPhys=%RGp\n", rc, GCPhys));
+                    if (RT_SUCCESS(rc))
+                        pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(pPage) && !PGM_PAGE_IS_BALLOONED(pPage);
+                    else
+                        pRom->aPages[iPage].LiveSave.fWrittenTo = !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow) && !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow);
+                }
+            }
+        }
+
+        pVM->pgm.s.LiveSave.Rom.cDirtyPages += cPages;
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+            pVM->pgm.s.LiveSave.Rom.cDirtyPages += cPages;
+    }
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assigns IDs to the ROM ranges and saves them.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                Saved state handle.
+ */
+static int pgmR3SaveRomRanges(PVM pVM, PSSMHANDLE pSSM)
+{
+    pgmLock(pVM);
+    uint8_t id = 1;
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3, id++)
+    {
+        pRom->idSavedState = id;
+        SSMR3PutU8(pSSM, id);
+        SSMR3PutStrZ(pSSM, "");         /* device name */
+        SSMR3PutU32(pSSM, 0);           /* device instance */
+        SSMR3PutU8(pSSM, 0);            /* region */
+        SSMR3PutStrZ(pSSM, pRom->pszDesc);
+        SSMR3PutGCPhys(pSSM, pRom->GCPhys);
+        int rc = SSMR3PutGCPhys(pSSM, pRom->cb);
+        if (RT_FAILURE(rc))
+            break;
+    }
+    pgmUnlock(pVM);
+    return SSMR3PutU8(pSSM, UINT8_MAX);
+}
+
+
+/**
+ * Loads the ROM range ID assignments.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3LoadRomRanges(PVM pVM, PSSMHANDLE pSSM)
+{
+    Assert(PGMIsLockOwner(pVM));
+
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+        pRom->idSavedState = UINT8_MAX;
+
+    for (;;)
+    {
+        /*
+         * Read the data.
+         */
+        uint8_t id;
+        int rc = SSMR3GetU8(pSSM, &id);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (id == UINT8_MAX)
+        {
+            for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+                AssertLogRelMsg(pRom->idSavedState != UINT8_MAX,
+                                ("The \"%s\" ROM was not found in the saved state. Probably due to some misconfiguration\n",
+                                 pRom->pszDesc));
+            return VINF_SUCCESS;        /* the end */
+        }
+        AssertLogRelReturn(id != 0, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        char szDevName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
+        rc = SSMR3GetStrZ(pSSM, szDevName, sizeof(szDevName));
+        AssertLogRelRCReturn(rc, rc);
+
+        uint32_t    uInstance;
+        SSMR3GetU32(pSSM, &uInstance);
+        uint8_t     iRegion;
+        SSMR3GetU8(pSSM, &iRegion);
+
+        char szDesc[64];
+        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+        AssertLogRelRCReturn(rc, rc);
+
+        RTGCPHYS GCPhys;
+        SSMR3GetGCPhys(pSSM, &GCPhys);
+        RTGCPHYS cb;
+        rc = SSMR3GetGCPhys(pSSM, &cb);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertLogRelMsgReturn(!(GCPhys & PAGE_OFFSET_MASK), ("GCPhys=%RGp %s\n", GCPhys, szDesc), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+        AssertLogRelMsgReturn(!(cb & PAGE_OFFSET_MASK),     ("cb=%RGp %s\n", cb, szDesc),         VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /*
+         * Locate a matching ROM range.
+         */
+        AssertLogRelMsgReturn(   uInstance == 0
+                              && iRegion == 0
+                              && szDevName[0] == '\0',
+                              ("GCPhys=%RGp %s\n", GCPhys, szDesc),
+                              VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+        PPGMROMRANGE pRom;
+        for (pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+        {
+            if (    pRom->idSavedState == UINT8_MAX
+                &&  !strcmp(pRom->pszDesc, szDesc))
+            {
+                pRom->idSavedState = id;
+                break;
+            }
+        }
+        if (!pRom)
+            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("ROM at %RGp by the name '%s' was not found"), GCPhys, szDesc);
+    } /* forever */
+}
+
+
+/**
+ * Scan ROM pages.
+ *
+ * @param   pVM                 The VM handle.
+ */
+static void pgmR3ScanRomPages(PVM pVM)
+{
+    /*
+     * The shadow ROMs.
+     */
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        {
+            uint32_t const cPages = pRom->cb >> PAGE_SHIFT;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                if (pRomPage->LiveSave.fWrittenTo)
+                {
+                    pRomPage->LiveSave.fWrittenTo = false;
+                    if (!pRomPage->LiveSave.fDirty)
+                    {
+                        pRomPage->LiveSave.fDirty = true;
+                        pVM->pgm.s.LiveSave.Rom.cReadyPages--;
+                        pVM->pgm.s.LiveSave.Rom.cDirtyPages++;
+                    }
+                    pRomPage->LiveSave.fDirtiedRecently = true;
+                }
+                else
+                    pRomPage->LiveSave.fDirtiedRecently = false;
+            }
+        }
+    }
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Takes care of the virgin ROM pages in the first pass.
+ *
+ * This is an attempt at simplifying the handling of ROM pages a little bit.
+ * This ASSUMES that no new ROM ranges will be added and that they won't be
+ * relinked in any way.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether we're in a live save or not.
+ */
+static int pgmR3SaveRomVirginPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave)
+{
+    if (FTMIsDeltaLoadSaveActive(pVM))
+        return VINF_SUCCESS;    /* nothing to do as nothing has changed here */
+
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        uint32_t const cPages = pRom->cb >> PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            RTGCPHYS   GCPhys  = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+            PGMROMPROT enmProt = pRom->aPages[iPage].enmProt;
+
+            /* Get the virgin page descriptor. */
+            PPGMPAGE pPage;
+            if (PGMROMPROT_IS_ROM(enmProt))
+                pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+            else
+                pPage = &pRom->aPages[iPage].Virgin;
+
+            /* Get the page bits. (Cannot use pgmPhysGCPhys2CCPtrInternalReadOnly here!) */
+            int rc = VINF_SUCCESS;
+            char abPage[PAGE_SIZE];
+            if (    !PGM_PAGE_IS_ZERO(pPage)
+                &&  !PGM_PAGE_IS_BALLOONED(pPage))
+            {
+                void const *pvPage;
+                rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvPage);
+                if (RT_SUCCESS(rc))
+                    memcpy(abPage, pvPage, PAGE_SIZE);
+            }
+            else
+                ASMMemZeroPage(abPage);
+            pgmUnlock(pVM);
+            AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
+
+            /* Save it. */
+            if (iPage > 0)
+                SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_VIRGIN);
+            else
+            {
+                SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_VIRGIN | PGM_STATE_REC_FLAG_ADDR);
+                SSMR3PutU8(pSSM, pRom->idSavedState);
+                SSMR3PutU32(pSSM, iPage);
+            }
+            SSMR3PutU8(pSSM, (uint8_t)enmProt);
+            rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            /* Update state. */
+            pgmLock(pVM);
+            pRom->aPages[iPage].LiveSave.u8Prot = (uint8_t)enmProt;
+            if (fLiveSave)
+            {
+                pVM->pgm.s.LiveSave.Rom.cDirtyPages--;
+                pVM->pgm.s.LiveSave.Rom.cReadyPages++;
+                pVM->pgm.s.LiveSave.cSavedPages++;
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves dirty pages in the shadowed ROM ranges.
+ *
+ * Used by pgmR3LiveExecPart2 and pgmR3SaveExecMemory.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether it's a live save or not.
+ * @param   fFinalPass          Whether this is the final pass or not.
+ */
+static int pgmR3SaveShadowedRomPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, bool fFinalPass)
+{
+    if (FTMIsDeltaLoadSaveActive(pVM))
+        return VINF_SUCCESS;    /* nothing to do as we deal with those pages separately */
+
+    /*
+     * The Shadowed ROMs.
+     *
+     * ASSUMES that the ROM ranges are fixed.
+     * ASSUMES that all the ROM ranges are mapped.
+     */
+    pgmLock(pVM);
+    for (PPGMROMRANGE pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+    {
+        if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
+        {
+            uint32_t const cPages    = pRom->cb >> PAGE_SHIFT;
+            uint32_t       iPrevPage = cPages;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                if (    !fLiveSave
+                    ||  (   pRomPage->LiveSave.fDirty
+                         && (   (   !pRomPage->LiveSave.fDirtiedRecently
+                                 && !pRomPage->LiveSave.fWrittenTo)
+                             || fFinalPass
+                             )
+                         )
+                    )
+                {
+                    uint8_t     abPage[PAGE_SIZE];
+                    PGMROMPROT  enmProt = pRomPage->enmProt;
+                    RTGCPHYS    GCPhys  = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                    PPGMPAGE    pPage   = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow : pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+                    bool        fZero   = PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_BALLOONED(pPage);
+                    int         rc      = VINF_SUCCESS;
+                    if (!fZero)
+                    {
+                        void const *pvPage;
+                        rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvPage);
+                        if (RT_SUCCESS(rc))
+                            memcpy(abPage, pvPage, PAGE_SIZE);
+                    }
+                    if (fLiveSave && RT_SUCCESS(rc))
+                    {
+                        pRomPage->LiveSave.u8Prot = (uint8_t)enmProt;
+                        pRomPage->LiveSave.fDirty = false;
+                        pVM->pgm.s.LiveSave.Rom.cReadyPages++;
+                        pVM->pgm.s.LiveSave.Rom.cDirtyPages--;
+                        pVM->pgm.s.LiveSave.cSavedPages++;
+                    }
+                    pgmUnlock(pVM);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
+
+                    if (iPage - 1U == iPrevPage && iPage > 0)
+                        SSMR3PutU8(pSSM, (fZero ? PGM_STATE_REC_ROM_SHW_ZERO : PGM_STATE_REC_ROM_SHW_RAW));
+                    else
+                    {
+                        SSMR3PutU8(pSSM, (fZero ? PGM_STATE_REC_ROM_SHW_ZERO : PGM_STATE_REC_ROM_SHW_RAW) | PGM_STATE_REC_FLAG_ADDR);
+                        SSMR3PutU8(pSSM, pRom->idSavedState);
+                        SSMR3PutU32(pSSM, iPage);
+                    }
+                    rc = SSMR3PutU8(pSSM, (uint8_t)enmProt);
+                    if (!fZero)
+                        rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+                    if (RT_FAILURE(rc))
+                        return rc;
+
+                    pgmLock(pVM);
+                    iPrevPage = iPage;
+                }
+                /*
+                 * In the final pass, make sure the protection is in sync.
+                 */
+                else if (   fFinalPass
+                         && pRomPage->LiveSave.u8Prot != pRomPage->enmProt)
+                {
+                    PGMROMPROT enmProt = pRomPage->enmProt;
+                    pRomPage->LiveSave.u8Prot = (uint8_t)enmProt;
+                    pgmUnlock(pVM);
+
+                    if (iPage - 1U == iPrevPage && iPage > 0)
+                        SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_PROT);
+                    else
+                    {
+                        SSMR3PutU8(pSSM, PGM_STATE_REC_ROM_PROT | PGM_STATE_REC_FLAG_ADDR);
+                        SSMR3PutU8(pSSM, pRom->idSavedState);
+                        SSMR3PutU32(pSSM, iPage);
+                    }
+                    int rc = SSMR3PutU8(pSSM, (uint8_t)enmProt);
+                    if (RT_FAILURE(rc))
+                        return rc;
+
+                    pgmLock(pVM);
+                    iPrevPage = iPage;
+                }
+            }
+        }
+    }
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Cleans up ROM pages after a live save.
+ *
+ * @param   pVM                 The VM handle.
+ */
+static void pgmR3DoneRomPages(PVM pVM)
+{
+    NOREF(pVM);
+}
+
+
+/**
+ * Prepares the MMIO2 pages for a live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ */
+static int pgmR3PrepMmio2Pages(PVM pVM)
+{
+    /*
+     * Initialize the live save tracking in the MMIO2 ranges.
+     * ASSUME nothing changes here.
+     */
+    pgmLock(pVM);
+    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
+    {
+        uint32_t const  cPages = pMmio2->RamRange.cb >> PAGE_SHIFT;
+        pgmUnlock(pVM);
+
+        PPGMLIVESAVEMMIO2PAGE paLSPages = (PPGMLIVESAVEMMIO2PAGE)MMR3HeapAllocZ(pVM, MM_TAG_PGM, sizeof(PGMLIVESAVEMMIO2PAGE) * cPages);
+        if (!paLSPages)
+            return VERR_NO_MEMORY;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            /* Initialize it as a dirty zero page. */
+            paLSPages[iPage].fDirty          = true;
+            paLSPages[iPage].cUnchangedScans = 0;
+            paLSPages[iPage].fZero           = true;
+            paLSPages[iPage].u32CrcH1        = PGM_STATE_CRC32_ZERO_HALF_PAGE;
+            paLSPages[iPage].u32CrcH2        = PGM_STATE_CRC32_ZERO_HALF_PAGE;
+        }
+
+        pgmLock(pVM);
+        pMmio2->paLSPages = paLSPages;
+        pVM->pgm.s.LiveSave.Mmio2.cDirtyPages += cPages;
+    }
+    pgmUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assigns IDs to the MMIO2 ranges and saves them.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                Saved state handle.
+ */
+static int pgmR3SaveMmio2Ranges(PVM pVM, PSSMHANDLE pSSM)
+{
+    pgmLock(pVM);
+    uint8_t id = 1;
+    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3, id++)
+    {
+        pMmio2->idSavedState = id;
+        SSMR3PutU8(pSSM, id);
+        SSMR3PutStrZ(pSSM, pMmio2->pDevInsR3->pReg->szName);
+        SSMR3PutU32(pSSM, pMmio2->pDevInsR3->iInstance);
+        SSMR3PutU8(pSSM, pMmio2->iRegion);
+        SSMR3PutStrZ(pSSM, pMmio2->RamRange.pszDesc);
+        int rc = SSMR3PutGCPhys(pSSM, pMmio2->RamRange.cb);
+        if (RT_FAILURE(rc))
+            break;
+    }
+    pgmUnlock(pVM);
+    return SSMR3PutU8(pSSM, UINT8_MAX);
+}
+
+
+/**
+ * Loads the MMIO2 range ID assignments.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3LoadMmio2Ranges(PVM pVM, PSSMHANDLE pSSM)
+{
+    Assert(PGMIsLockOwner(pVM));
+
+    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
+        pMmio2->idSavedState = UINT8_MAX;
+
+    for (;;)
+    {
+        /*
+         * Read the data.
+         */
+        uint8_t id;
+        int rc = SSMR3GetU8(pSSM, &id);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (id == UINT8_MAX)
+        {
+            for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
+                AssertLogRelMsg(pMmio2->idSavedState != UINT8_MAX, ("%s\n", pMmio2->RamRange.pszDesc));
+            return VINF_SUCCESS;        /* the end */
+        }
+        AssertLogRelReturn(id != 0, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        char szDevName[RT_SIZEOFMEMB(PDMDEVREG, szName)];
+        rc = SSMR3GetStrZ(pSSM, szDevName, sizeof(szDevName));
+        AssertLogRelRCReturn(rc, rc);
+
+        uint32_t    uInstance;
+        SSMR3GetU32(pSSM, &uInstance);
+        uint8_t     iRegion;
+        SSMR3GetU8(pSSM, &iRegion);
+
+        char szDesc[64];
+        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+        AssertLogRelRCReturn(rc, rc);
+
+        RTGCPHYS cb;
+        rc = SSMR3GetGCPhys(pSSM, &cb);
+        AssertLogRelMsgReturn(!(cb & PAGE_OFFSET_MASK), ("cb=%RGp %s\n", cb, szDesc), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /*
+         * Locate a matching MMIO2 range.
+         */
+        PPGMMMIO2RANGE pMmio2;
+        for (pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
+        {
+            if (    pMmio2->idSavedState == UINT8_MAX
+                &&  pMmio2->iRegion == iRegion
+                &&  pMmio2->pDevInsR3->iInstance == uInstance
+                &&  !strcmp(pMmio2->pDevInsR3->pReg->szName, szDevName))
+            {
+                pMmio2->idSavedState = id;
+                break;
+            }
+        }
+        if (!pMmio2)
+            return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Failed to locate a MMIO2 range called '%s' owned by %s/%u, region %d"),
+                                    szDesc, szDevName, uInstance, iRegion);
+
+        /*
+         * Validate the configuration, the size of the MMIO2 region should be
+         * the same.
+         */
+        if (cb != pMmio2->RamRange.cb)
+        {
+            LogRel(("PGM: MMIO2 region \"%s\" size mismatch: saved=%RGp config=%RGp\n",
+                    pMmio2->RamRange.pszDesc, cb, pMmio2->RamRange.cb));
+            if (cb > pMmio2->RamRange.cb) /* bad idea? */
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("MMIO2 region \"%s\" size mismatch: saved=%RGp config=%RGp"),
+                                        pMmio2->RamRange.pszDesc, cb, pMmio2->RamRange.cb);
+        }
+    } /* forever */
+}
+
+
+/**
+ * Scans one MMIO2 page.
+ *
+ * @returns True if changed, false if unchanged.
+ *
+ * @param   pVM                 The VM handle
+ * @param   pbPage              The page bits.
+ * @param   pLSPage             The live save tracking structure for the page.
+ *
+ */
+DECLINLINE(bool) pgmR3ScanMmio2Page(PVM pVM, uint8_t const *pbPage, PPGMLIVESAVEMMIO2PAGE pLSPage)
+{
+    /*
+     * Special handling of zero pages.
+     */
+    bool const fZero = pLSPage->fZero;
+    if (fZero)
+    {
+        if (ASMMemIsZeroPage(pbPage))
+        {
+            /* Not modified. */
+            if (pLSPage->fDirty)
+                pLSPage->cUnchangedScans++;
+            return false;
+        }
+
+        pLSPage->fZero    = false;
+        pLSPage->u32CrcH1 = RTCrc32(pbPage, PAGE_SIZE / 2);
+    }
+    else
+    {
+        /*
+         * CRC the first half, if it doesn't match the page is dirty and
+         * we won't check the 2nd half (we'll do that next time).
+         */
+        uint32_t u32CrcH1 = RTCrc32(pbPage, PAGE_SIZE / 2);
+        if (u32CrcH1 == pLSPage->u32CrcH1)
+        {
+            uint32_t u32CrcH2 = RTCrc32(pbPage + PAGE_SIZE / 2, PAGE_SIZE / 2);
+            if (u32CrcH2 == pLSPage->u32CrcH2)
+            {
+                /* Probably not modified. */
+                if (pLSPage->fDirty)
+                    pLSPage->cUnchangedScans++;
+                return false;
+            }
+
+            pLSPage->u32CrcH2 = u32CrcH2;
+        }
+        else
+        {
+            pLSPage->u32CrcH1 = u32CrcH1;
+            if (    u32CrcH1 == PGM_STATE_CRC32_ZERO_HALF_PAGE
+                &&  ASMMemIsZeroPage(pbPage))
+            {
+                pLSPage->u32CrcH2 = PGM_STATE_CRC32_ZERO_HALF_PAGE;
+                pLSPage->fZero    = true;
+            }
+        }
+    }
+
+    /* dirty page path */
+    pLSPage->cUnchangedScans = 0;
+    if (!pLSPage->fDirty)
+    {
+        pLSPage->fDirty = true;
+        pVM->pgm.s.LiveSave.Mmio2.cReadyPages--;
+        pVM->pgm.s.LiveSave.Mmio2.cDirtyPages++;
+        if (fZero)
+            pVM->pgm.s.LiveSave.Mmio2.cZeroPages--;
+    }
+    return true;
+}
+
+
+/**
+ * Scan for MMIO2 page modifications.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   uPass               The pass number.
+ */
+static void pgmR3ScanMmio2Pages(PVM pVM, uint32_t uPass)
+{
+    /*
+     * Since this is a bit expensive we lower the scan rate after a little while.
+     */
+    if (    (    (uPass & 3) != 0
+             &&  uPass > 10)
+        ||  uPass == SSM_PASS_FINAL)
+        return;
+
+    pgmLock(pVM);                       /* paranoia */
+    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
+    {
+        PPGMLIVESAVEMMIO2PAGE paLSPages = pMmio2->paLSPages;
+        uint32_t              cPages    = pMmio2->RamRange.cb >> PAGE_SHIFT;
+        pgmUnlock(pVM);
+
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            uint8_t const *pbPage = (uint8_t const *)pMmio2->pvR3 + iPage * PAGE_SIZE;
+            pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]);
+        }
+
+        pgmLock(pVM);
+    }
+    pgmUnlock(pVM);
+
+}
+
+
+/**
+ * Save quiescent MMIO2 pages.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether it's a live save or not.
+ * @param   uPass               The pass number.
+ */
+static int pgmR3SaveMmio2Pages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, uint32_t uPass)
+{
+    /** @todo implement live saving of MMIO2 pages. (Need some way of telling the
+     *        device that we wish to know about changes.) */
+
+    int rc = VINF_SUCCESS;
+    if (uPass == SSM_PASS_FINAL)
+    {
+        /*
+         * The mop up round.
+         */
+        pgmLock(pVM);
+        for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3;
+             pMmio2 && RT_SUCCESS(rc);
+             pMmio2 = pMmio2->pNextR3)
+        {
+            PPGMLIVESAVEMMIO2PAGE paLSPages = pMmio2->paLSPages;
+            uint8_t const        *pbPage    = (uint8_t const *)pMmio2->RamRange.pvR3;
+            uint32_t              cPages    = pMmio2->RamRange.cb >> PAGE_SHIFT;
+            uint32_t              iPageLast = cPages;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++, pbPage += PAGE_SIZE)
+            {
+                uint8_t u8Type;
+                if (!fLiveSave)
+                    u8Type = ASMMemIsZeroPage(pbPage) ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
+                else
+                {
+                    /* Try figure if it's a clean page, compare the SHA-1 to be really sure. */
+                    if (   !paLSPages[iPage].fDirty
+                        && !pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]))
+                    {
+                        if (paLSPages[iPage].fZero)
+                            continue;
+
+                        uint8_t abSha1Hash[RTSHA1_HASH_SIZE];
+                        RTSha1(pbPage, PAGE_SIZE, abSha1Hash);
+                        if (!memcmp(abSha1Hash, paLSPages[iPage].abSha1Saved, sizeof(abSha1Hash)))
+                            continue;
+                    }
+                    u8Type = paLSPages[iPage].fZero ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
+                    pVM->pgm.s.LiveSave.cSavedPages++;
+                }
+
+                if (iPage != 0 && iPage == iPageLast + 1)
+                    rc = SSMR3PutU8(pSSM, u8Type);
+                else
+                {
+                    SSMR3PutU8(pSSM, u8Type | PGM_STATE_REC_FLAG_ADDR);
+                    SSMR3PutU8(pSSM, pMmio2->idSavedState);
+                    rc = SSMR3PutU32(pSSM, iPage);
+                }
+                if (u8Type == PGM_STATE_REC_MMIO2_RAW)
+                    rc = SSMR3PutMem(pSSM, pbPage, PAGE_SIZE);
+                if (RT_FAILURE(rc))
+                    break;
+                iPageLast = iPage;
+            }
+        }
+        pgmUnlock(pVM);
+    }
+    /*
+     * Reduce the rate after a little while since the current MMIO2 approach is
+     * a bit expensive.
+     * We position it two passes after the scan pass to avoid saving busy pages.
+     */
+    else if (   uPass <= 10
+             || (uPass & 3) == 2)
+    {
+        pgmLock(pVM);
+        for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3;
+             pMmio2 && RT_SUCCESS(rc);
+             pMmio2 = pMmio2->pNextR3)
+        {
+            PPGMLIVESAVEMMIO2PAGE paLSPages = pMmio2->paLSPages;
+            uint8_t const        *pbPage    = (uint8_t const *)pMmio2->RamRange.pvR3;
+            uint32_t              cPages    = pMmio2->RamRange.cb >> PAGE_SHIFT;
+            uint32_t              iPageLast = cPages;
+            pgmUnlock(pVM);
+
+            for (uint32_t iPage = 0; iPage < cPages; iPage++, pbPage += PAGE_SIZE)
+            {
+                /* Skip clean pages and pages which hasn't quiesced. */
+                if (!paLSPages[iPage].fDirty)
+                    continue;
+                if (paLSPages[iPage].cUnchangedScans < 3)
+                    continue;
+                if (pgmR3ScanMmio2Page(pVM, pbPage, &paLSPages[iPage]))
+                    continue;
+
+                /* Save it. */
+                bool const fZero = paLSPages[iPage].fZero;
+                uint8_t abPage[PAGE_SIZE];
+                if (!fZero)
+                {
+                    memcpy(abPage, pbPage, PAGE_SIZE);
+                    RTSha1(abPage, PAGE_SIZE, paLSPages[iPage].abSha1Saved);
+                }
+
+                uint8_t u8Type = paLSPages[iPage].fZero ? PGM_STATE_REC_MMIO2_ZERO : PGM_STATE_REC_MMIO2_RAW;
+                if (iPage != 0 && iPage == iPageLast + 1)
+                    rc = SSMR3PutU8(pSSM, u8Type);
+                else
+                {
+                    SSMR3PutU8(pSSM, u8Type | PGM_STATE_REC_FLAG_ADDR);
+                    SSMR3PutU8(pSSM, pMmio2->idSavedState);
+                    rc = SSMR3PutU32(pSSM, iPage);
+                }
+                if (u8Type == PGM_STATE_REC_MMIO2_RAW)
+                    rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+                if (RT_FAILURE(rc))
+                    break;
+
+                /* Housekeeping. */
+                paLSPages[iPage].fDirty = false;
+                pVM->pgm.s.LiveSave.Mmio2.cDirtyPages--;
+                pVM->pgm.s.LiveSave.Mmio2.cReadyPages++;
+                if (u8Type == PGM_STATE_REC_MMIO2_ZERO)
+                    pVM->pgm.s.LiveSave.Mmio2.cZeroPages++;
+                pVM->pgm.s.LiveSave.cSavedPages++;
+                iPageLast = iPage;
+            }
+
+            pgmLock(pVM);
+        }
+        pgmUnlock(pVM);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Cleans up MMIO2 pages after a live save.
+ *
+ * @param   pVM                 The VM handle.
+ */
+static void pgmR3DoneMmio2Pages(PVM pVM)
+{
+    /*
+     * Free the tracking structures for the MMIO2 pages.
+     * We do the freeing outside the lock in case the VM is running.
+     */
+    pgmLock(pVM);
+    for (PPGMMMIO2RANGE pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
+    {
+        void *pvMmio2ToFree = pMmio2->paLSPages;
+        if (pvMmio2ToFree)
+        {
+            pMmio2->paLSPages = NULL;
+            pgmUnlock(pVM);
+            MMR3HeapFree(pvMmio2ToFree);
+            pgmLock(pVM);
+        }
+    }
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Prepares the RAM pages for a live save.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ */
+static int pgmR3PrepRamPages(PVM pVM)
+{
+
+    /*
+     * Try allocating tracking structures for the ram ranges.
+     *
+     * To avoid lock contention, we leave the lock every time we're allocating
+     * a new array.  This means we'll have to ditch the allocation and start
+     * all over again if the RAM range list changes in-between.
+     *
+     * Note! pgmR3SaveDone will always be called and it is therefore responsible
+     *       for cleaning up.
+     */
+    PPGMRAMRANGE pCur;
+    pgmLock(pVM);
+    do
+    {
+        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (   !pCur->paLSPages
+                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
+            {
+                uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+                uint32_t const  cPages = pCur->cb >> PAGE_SHIFT;
+                pgmUnlock(pVM);
+                PPGMLIVESAVERAMPAGE paLSPages = (PPGMLIVESAVERAMPAGE)MMR3HeapAllocZ(pVM, MM_TAG_PGM, cPages * sizeof(PGMLIVESAVERAMPAGE));
+                if (!paLSPages)
+                    return VERR_NO_MEMORY;
+                pgmLock(pVM);
+                if (pVM->pgm.s.idRamRangesGen != idRamRangesGen)
+                {
+                    pgmUnlock(pVM);
+                    MMR3HeapFree(paLSPages);
+                    pgmLock(pVM);
+                    break;              /* try again */
+                }
+                pCur->paLSPages = paLSPages;
+
+                /*
+                 * Initialize the array.
+                 */
+                uint32_t iPage = cPages;
+                while (iPage-- > 0)
+                {
+                    /** @todo yield critsect! (after moving this away from EMT0) */
+                    PCPGMPAGE pPage = &pCur->aPages[iPage];
+                    paLSPages[iPage].cDirtied               = 0;
+                    paLSPages[iPage].fDirty                 = 1; /* everything is dirty at this time */
+                    paLSPages[iPage].fWriteMonitored        = 0;
+                    paLSPages[iPage].fWriteMonitoredJustNow = 0;
+                    paLSPages[iPage].u2Reserved             = 0;
+                    switch (PGM_PAGE_GET_TYPE(pPage))
+                    {
+                        case PGMPAGETYPE_RAM:
+                            if (    PGM_PAGE_IS_ZERO(pPage)
+                                ||  PGM_PAGE_IS_BALLOONED(pPage))
+                            {
+                                paLSPages[iPage].fZero   = 1;
+                                paLSPages[iPage].fShared = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc  = PGM_STATE_CRC32_ZERO_PAGE;
+#endif
+                            }
+                            else if (PGM_PAGE_IS_SHARED(pPage))
+                            {
+                                paLSPages[iPage].fZero   = 0;
+                                paLSPages[iPage].fShared = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            }
+                            else
+                            {
+                                paLSPages[iPage].fZero   = 0;
+                                paLSPages[iPage].fShared = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            }
+                            paLSPages[iPage].fIgnore     = 0;
+                            pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                            break;
+
+                        case PGMPAGETYPE_ROM_SHADOW:
+                        case PGMPAGETYPE_ROM:
+                        {
+                            paLSPages[iPage].fZero   = 0;
+                            paLSPages[iPage].fShared = 0;
+                            paLSPages[iPage].fDirty  = 0;
+                            paLSPages[iPage].fIgnore = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            pVM->pgm.s.LiveSave.cIgnoredPages++;
+                            break;
+                        }
+
+                        default:
+                            AssertMsgFailed(("%R[pgmpage]", pPage));
+                        case PGMPAGETYPE_MMIO2:
+                        case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
+                            paLSPages[iPage].fZero   = 0;
+                            paLSPages[iPage].fShared = 0;
+                            paLSPages[iPage].fDirty  = 0;
+                            paLSPages[iPage].fIgnore = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            pVM->pgm.s.LiveSave.cIgnoredPages++;
+                            break;
+
+                        case PGMPAGETYPE_MMIO:
+                            paLSPages[iPage].fZero   = 0;
+                            paLSPages[iPage].fShared = 0;
+                            paLSPages[iPage].fDirty  = 0;
+                            paLSPages[iPage].fIgnore = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            paLSPages[iPage].u32Crc  = UINT32_MAX;
+#endif
+                            pVM->pgm.s.LiveSave.cIgnoredPages++;
+                            break;
+                    }
+                }
+            }
+        }
+    } while (pCur);
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves the RAM configuration.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3SaveRamConfig(PVM pVM, PSSMHANDLE pSSM)
+{
+    uint32_t cbRamHole = 0;
+    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHole, MM_RAM_HOLE_SIZE_DEFAULT);
+    AssertRCReturn(rc, rc);
+
+    uint64_t cbRam     = 0;
+    rc = CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRam, 0);
+    AssertRCReturn(rc, rc);
+
+    SSMR3PutU32(pSSM, cbRamHole);
+    return SSMR3PutU64(pSSM, cbRam);
+}
+
+
+/**
+ * Loads and verifies the RAM configuration.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int pgmR3LoadRamConfig(PVM pVM, PSSMHANDLE pSSM)
+{
+    uint32_t cbRamHoleCfg = 0;
+    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "RamHoleSize", &cbRamHoleCfg, MM_RAM_HOLE_SIZE_DEFAULT);
+    AssertRCReturn(rc, rc);
+
+    uint64_t cbRamCfg     = 0;
+    rc = CFGMR3QueryU64Def(CFGMR3GetRoot(pVM), "RamSize", &cbRamCfg, 0);
+    AssertRCReturn(rc, rc);
+
+    uint32_t cbRamHoleSaved;
+    SSMR3GetU32(pSSM, &cbRamHoleSaved);
+
+    uint64_t cbRamSaved;
+    rc = SSMR3GetU64(pSSM, &cbRamSaved);
+    AssertRCReturn(rc, rc);
+
+    if (   cbRamHoleCfg != cbRamHoleSaved
+        || cbRamCfg     != cbRamSaved)
+        return SSMR3SetCfgError(pSSM, RT_SRC_POS, N_("Ram config mismatch: saved=%RX64/%RX32 config=%RX64/%RX32 (RAM/Hole)"),
+                                cbRamSaved, cbRamHoleSaved, cbRamCfg, cbRamHoleCfg);
+    return VINF_SUCCESS;
+}
+
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+
+/**
+ * Calculates the CRC-32 for a RAM page and updates the live save page tracking
+ * info with it.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pCur                The current RAM range.
+ * @param   paLSPages           The current array of live save page tracking
+ *                              structures.
+ * @param   iPage               The page index.
+ */
+static void pgmR3StateCalcCrc32ForRamPage(PVM pVM, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage)
+{
+    RTGCPHYS    GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+    void const *pvPage;
+    int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, &pCur->aPages[iPage], GCPhys, &pvPage);
+    if (RT_SUCCESS(rc))
+        paLSPages[iPage].u32Crc = RTCrc32(pvPage, PAGE_SIZE);
+    else
+        paLSPages[iPage].u32Crc = UINT32_MAX; /* Invalid */
+}
+
+
+/**
+ * Verifies the CRC-32 for a page given it's raw bits.
+ *
+ * @param   pvPage              The page bits.
+ * @param   pCur                The current RAM range.
+ * @param   paLSPages           The current array of live save page tracking
+ *                              structures.
+ * @param   iPage               The page index.
+ */
+static void pgmR3StateVerifyCrc32ForPage(void const *pvPage, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage, const  char *pszWhere)
+{
+    if (paLSPages[iPage].u32Crc != UINT32_MAX)
+    {
+        uint32_t u32Crc = RTCrc32(pvPage, PAGE_SIZE);
+        Assert(   (   !PGM_PAGE_IS_ZERO(&pCur->aPages[iPage])
+                   && !PGM_PAGE_IS_BALLOONED(&pCur->aPages[iPage]))
+               || u32Crc == PGM_STATE_CRC32_ZERO_PAGE);
+        AssertMsg(paLSPages[iPage].u32Crc == u32Crc,
+                  ("%08x != %08x for %RGp %R[pgmpage] %s\n", paLSPages[iPage].u32Crc, u32Crc,
+                   pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT), &pCur->aPages[iPage], pszWhere));
+    }
+}
+
+
+/**
+ * Verifies the CRC-32 for a RAM page.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pCur                The current RAM range.
+ * @param   paLSPages           The current array of live save page tracking
+ *                              structures.
+ * @param   iPage               The page index.
+ */
+static void pgmR3StateVerifyCrc32ForRamPage(PVM pVM, PPGMRAMRANGE pCur, PPGMLIVESAVERAMPAGE paLSPages, uint32_t iPage, const char *pszWhere)
+{
+    if (paLSPages[iPage].u32Crc != UINT32_MAX)
+    {
+        RTGCPHYS    GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+        void const *pvPage;
+        int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, &pCur->aPages[iPage], GCPhys, &pvPage);
+        if (RT_SUCCESS(rc))
+            pgmR3StateVerifyCrc32ForPage(pvPage, pCur, paLSPages, iPage, pszWhere);
+    }
+}
+
+#endif /* PGMLIVESAVERAMPAGE_WITH_CRC32 */
+
+/**
+ * Scan for RAM page modifications and reprotect them.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   fFinalPass          Whether this is the final pass or not.
+ */
+static void pgmR3ScanRamPages(PVM pVM, bool fFinalPass)
+{
+    /*
+     * The RAM.
+     */
+    RTGCPHYS GCPhysCur = 0;
+    PPGMRAMRANGE pCur;
+    pgmLock(pVM);
+    do
+    {
+        uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (    pCur->GCPhysLast > GCPhysCur
+                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
+            {
+                PPGMLIVESAVERAMPAGE paLSPages = pCur->paLSPages;
+                uint32_t         cPages    = pCur->cb >> PAGE_SHIFT;
+                uint32_t         iPage     = GCPhysCur <= pCur->GCPhys ? 0 : (GCPhysCur - pCur->GCPhys) >> PAGE_SHIFT;
+                GCPhysCur = 0;
+                for (; iPage < cPages; iPage++)
+                {
+                    /* Do yield first. */
+                    if (   !fFinalPass
+#ifndef PGMLIVESAVERAMPAGE_WITH_CRC32
+                        && (iPage & 0x7ff) == 0x100
+#endif
+                        && PDMR3CritSectYield(&pVM->pgm.s.CritSect)
+                        && pVM->pgm.s.idRamRangesGen != idRamRangesGen)
+                    {
+                        GCPhysCur = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                        break; /* restart */
+                    }
+
+                    /* Skip already ignored pages. */
+                    if (paLSPages[iPage].fIgnore)
+                        continue;
+
+                    if (RT_LIKELY(PGM_PAGE_GET_TYPE(&pCur->aPages[iPage]) == PGMPAGETYPE_RAM))
+                    {
+                        /*
+                         * A RAM page.
+                         */
+                        switch (PGM_PAGE_GET_STATE(&pCur->aPages[iPage]))
+                        {
+                            case PGM_PAGE_STATE_ALLOCATED:
+                                /** @todo Optimize this: Don't always re-enable write
+                                 * monitoring if the page is known to be very busy. */
+                                if (PGM_PAGE_IS_WRITTEN_TO(&pCur->aPages[iPage]))
+                                {
+                                    Assert(paLSPages[iPage].fWriteMonitored);
+                                    PGM_PAGE_CLEAR_WRITTEN_TO(&pCur->aPages[iPage]);
+                                    Assert(pVM->pgm.s.cWrittenToPages > 0);
+                                    pVM->pgm.s.cWrittenToPages--;
+                                }
+                                else
+                                {
+                                    Assert(!paLSPages[iPage].fWriteMonitored);
+                                    pVM->pgm.s.LiveSave.Ram.cMonitoredPages++;
+                                }
+
+                                if (!paLSPages[iPage].fDirty)
+                                {
+                                    pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                    if (paLSPages[iPage].fZero)
+                                        pVM->pgm.s.LiveSave.Ram.cZeroPages--;
+                                    pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                    if (++paLSPages[iPage].cDirtied > PGMLIVSAVEPAGE_MAX_DIRTIED)
+                                        paLSPages[iPage].cDirtied = PGMLIVSAVEPAGE_MAX_DIRTIED;
+                                }
+
+                                PGM_PAGE_SET_STATE(&pCur->aPages[iPage], PGM_PAGE_STATE_WRITE_MONITORED);
+                                pVM->pgm.s.cMonitoredPages++;
+                                paLSPages[iPage].fWriteMonitored        = 1;
+                                paLSPages[iPage].fWriteMonitoredJustNow = 1;
+                                paLSPages[iPage].fDirty                 = 1;
+                                paLSPages[iPage].fZero                  = 0;
+                                paLSPages[iPage].fShared                = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                paLSPages[iPage].u32Crc                 = UINT32_MAX; /* invalid */
+#endif
+                                break;
+
+                            case PGM_PAGE_STATE_WRITE_MONITORED:
+                                Assert(paLSPages[iPage].fWriteMonitored);
+                                if (PGM_PAGE_GET_WRITE_LOCKS(&pCur->aPages[iPage]) == 0)
+                                {
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    if (paLSPages[iPage].fWriteMonitoredJustNow)
+                                        pgmR3StateCalcCrc32ForRamPage(pVM, pCur, paLSPages, iPage);
+                                    else
+                                        pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "scan");
+#endif
+                                    paLSPages[iPage].fWriteMonitoredJustNow = 0;
+                                }
+                                else
+                                {
+                                    paLSPages[iPage].fWriteMonitoredJustNow = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    paLSPages[iPage].u32Crc                 = UINT32_MAX; /* invalid */
+#endif
+                                    if (!paLSPages[iPage].fDirty)
+                                    {
+                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                        if (++paLSPages[iPage].cDirtied > PGMLIVSAVEPAGE_MAX_DIRTIED)
+                                            paLSPages[iPage].cDirtied = PGMLIVSAVEPAGE_MAX_DIRTIED;
+                                    }
+                                }
+                                break;
+
+                            case PGM_PAGE_STATE_ZERO:
+                                if (!paLSPages[iPage].fZero)
+                                {
+                                    if (!paLSPages[iPage].fDirty)
+                                    {
+                                        paLSPages[iPage].fDirty = 1;
+                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                    }
+                                    paLSPages[iPage].fZero = 1;
+                                    paLSPages[iPage].fShared = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    paLSPages[iPage].u32Crc = PGM_STATE_CRC32_ZERO_PAGE;
+#endif
+                                }
+                                break;
+
+                            case PGM_PAGE_STATE_BALLOONED:
+                                if (!paLSPages[iPage].fZero)
+                                {
+                                    if (!paLSPages[iPage].fDirty)
+                                    {
+                                        paLSPages[iPage].fDirty = 1;
+                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                    }
+                                    paLSPages[iPage].fZero = 1;
+                                    paLSPages[iPage].fShared = 0;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    paLSPages[iPage].u32Crc = PGM_STATE_CRC32_ZERO_PAGE;
+#endif
+                                }
+                                break;
+
+                            case PGM_PAGE_STATE_SHARED:
+                                if (!paLSPages[iPage].fShared)
+                                {
+                                    if (!paLSPages[iPage].fDirty)
+                                    {
+                                        paLSPages[iPage].fDirty = 1;
+                                        pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                                        if (paLSPages[iPage].fZero)
+                                            pVM->pgm.s.LiveSave.Ram.cZeroPages--;
+                                        pVM->pgm.s.LiveSave.Ram.cDirtyPages++;
+                                    }
+                                    paLSPages[iPage].fZero = 0;
+                                    paLSPages[iPage].fShared = 1;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                                    pgmR3StateCalcCrc32ForRamPage(pVM, pCur, paLSPages, iPage);
+#endif
+                                }
+                                break;
+                        }
+                    }
+                    else
+                    {
+                        /*
+                         * All other types => Ignore the page.
+                         */
+                        Assert(!paLSPages[iPage].fIgnore); /* skipped before switch */
+                        paLSPages[iPage].fIgnore = 1;
+                        if (paLSPages[iPage].fWriteMonitored)
+                        {
+                            /** @todo this doesn't hold water when we start monitoring MMIO2 and ROM shadow
+                             *        pages! */
+                            if (RT_UNLIKELY(PGM_PAGE_GET_STATE(&pCur->aPages[iPage]) == PGM_PAGE_STATE_WRITE_MONITORED))
+                            {
+                                AssertMsgFailed(("%R[pgmpage]", &pCur->aPages[iPage])); /* shouldn't happen. */
+                                PGM_PAGE_SET_STATE(&pCur->aPages[iPage], PGM_PAGE_STATE_ALLOCATED);
+                                Assert(pVM->pgm.s.cMonitoredPages > 0);
+                                pVM->pgm.s.cMonitoredPages--;
+                            }
+                            if (PGM_PAGE_IS_WRITTEN_TO(&pCur->aPages[iPage]))
+                            {
+                                PGM_PAGE_CLEAR_WRITTEN_TO(&pCur->aPages[iPage]);
+                                Assert(pVM->pgm.s.cWrittenToPages > 0);
+                                pVM->pgm.s.cWrittenToPages--;
+                            }
+                            pVM->pgm.s.LiveSave.Ram.cMonitoredPages--;
+                        }
+
+                        /** @todo the counting doesn't quite work out here. fix later? */
+                        if (paLSPages[iPage].fDirty)
+                            pVM->pgm.s.LiveSave.Ram.cDirtyPages--;
+                        else
+                        {
+                            pVM->pgm.s.LiveSave.Ram.cReadyPages--;
+                            if (paLSPages[iPage].fZero)
+                                pVM->pgm.s.LiveSave.Ram.cZeroPages--;
+                        }
+                        pVM->pgm.s.LiveSave.cIgnoredPages++;
+                    }
+                } /* for each page in range */
+
+                if (GCPhysCur != 0)
+                    break; /* Yield + ramrange change */
+                GCPhysCur = pCur->GCPhysLast;
+            }
+        } /* for each range */
+    } while (pCur);
+    pgmUnlock(pVM);
+}
+
+
+/**
+ * Save quiescent RAM pages.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ * @param   fLiveSave           Whether it's a live save or not.
+ * @param   uPass               The pass number.
+ */
+static int pgmR3SaveRamPages(PVM pVM, PSSMHANDLE pSSM, bool fLiveSave, uint32_t uPass)
+{
+    /*
+     * The RAM.
+     */
+    RTGCPHYS GCPhysLast = NIL_RTGCPHYS;
+    RTGCPHYS GCPhysCur = 0;
+    PPGMRAMRANGE pCur;
+    bool fFTMDeltaSaveActive = FTMIsDeltaLoadSaveActive(pVM);
+
+    pgmLock(pVM);
+    do
+    {
+        uint32_t const  idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (   pCur->GCPhysLast > GCPhysCur
+                && !PGM_RAM_RANGE_IS_AD_HOC(pCur))
+            {
+                PPGMLIVESAVERAMPAGE paLSPages = pCur->paLSPages;
+                uint32_t         cPages    = pCur->cb >> PAGE_SHIFT;
+                uint32_t         iPage     = GCPhysCur <= pCur->GCPhys ? 0 : (GCPhysCur - pCur->GCPhys) >> PAGE_SHIFT;
+                GCPhysCur = 0;
+                for (; iPage < cPages; iPage++)
+                {
+                    /* Do yield first. */
+                    if (   uPass != SSM_PASS_FINAL
+                        && (iPage & 0x7ff) == 0x100
+                        && PDMR3CritSectYield(&pVM->pgm.s.CritSect)
+                        && pVM->pgm.s.idRamRangesGen != idRamRangesGen)
+                    {
+                        GCPhysCur = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                        break; /* restart */
+                    }
+
+                    PPGMPAGE pCurPage = &pCur->aPages[iPage];
+
+                    /*
+                     * Only save pages that haven't changed since last scan and are dirty.
+                     */
+                    if (    uPass != SSM_PASS_FINAL
+                        &&  paLSPages)
+                    {
+                        if (!paLSPages[iPage].fDirty)
+                            continue;
+                        if (paLSPages[iPage].fWriteMonitoredJustNow)
+                            continue;
+                        if (paLSPages[iPage].fIgnore)
+                            continue;
+                        if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM) /* in case of recent remappings */
+                            continue;
+                        if (    PGM_PAGE_GET_STATE(pCurPage)
+                            !=  (  paLSPages[iPage].fZero
+                                 ? PGM_PAGE_STATE_ZERO
+                                 : paLSPages[iPage].fShared
+                                 ? PGM_PAGE_STATE_SHARED
+                                 : PGM_PAGE_STATE_WRITE_MONITORED))
+                            continue;
+                        if (PGM_PAGE_GET_WRITE_LOCKS(&pCur->aPages[iPage]) > 0)
+                            continue;
+                    }
+                    else
+                    {
+                        if (   paLSPages
+                            && !paLSPages[iPage].fDirty
+                            && !paLSPages[iPage].fIgnore)
+                        {
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM)
+                                pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "save#1");
+#endif
+                            continue;
+                        }
+                        if (PGM_PAGE_GET_TYPE(pCurPage) != PGMPAGETYPE_RAM)
+                            continue;
+                    }
+
+                    /*
+                     * Do the saving outside the PGM critsect since SSM may block on I/O.
+                     */
+                    int         rc;
+                    RTGCPHYS    GCPhys = pCur->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+                    bool        fZero  = PGM_PAGE_IS_ZERO(pCurPage) || PGM_PAGE_IS_BALLOONED(pCurPage);
+                    bool        fSkipped = false;
+
+                    if (!fZero)
+                    {
+                        /*
+                         * Copy the page and then save it outside the lock (since any
+                         * SSM call may block).
+                         */
+                        uint8_t     abPage[PAGE_SIZE];
+                        void const *pvPage;
+                        rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pCurPage, GCPhys, &pvPage);
+                        if (RT_SUCCESS(rc))
+                        {
+                            memcpy(abPage, pvPage, PAGE_SIZE);
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                            if (paLSPages)
+                                pgmR3StateVerifyCrc32ForPage(abPage, pCur, paLSPages, iPage, "save#3");
+#endif
+                        }
+                        pgmUnlock(pVM);
+                        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc GCPhys=%RGp\n", rc, GCPhys), rc);
+
+                        /* Try save some memory when restoring. */
+                        if (!ASMMemIsZeroPage(pvPage))
+                        {
+                            if (fFTMDeltaSaveActive)
+                            {
+                                if (    PGM_PAGE_IS_WRITTEN_TO(pCurPage)
+                                    ||  PGM_PAGE_IS_FT_DIRTY(pCurPage))
+                                {
+                                    if (GCPhys == GCPhysLast + PAGE_SIZE)
+                                        SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW);
+                                    else
+                                    {
+                                        SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW | PGM_STATE_REC_FLAG_ADDR);
+                                        SSMR3PutGCPhys(pSSM, GCPhys);
+                                    }
+                                    rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+                                    PGM_PAGE_CLEAR_WRITTEN_TO(pCurPage);
+                                    PGM_PAGE_CLEAR_FT_DIRTY(pCurPage);
+                                }
+                                /* else nothing changed, so skip it. */
+                                else
+                                    fSkipped = true;
+                            }
+                            else
+                            {
+                                if (GCPhys == GCPhysLast + PAGE_SIZE)
+                                    SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW);
+                                else
+                                {
+                                    SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_RAW | PGM_STATE_REC_FLAG_ADDR);
+                                    SSMR3PutGCPhys(pSSM, GCPhys);
+                                }
+                                rc = SSMR3PutMem(pSSM, abPage, PAGE_SIZE);
+                            }
+                        }
+                        else
+                        {
+                            if (GCPhys == GCPhysLast + PAGE_SIZE)
+                                rc = SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO);
+                            else
+                            {
+                                SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO | PGM_STATE_REC_FLAG_ADDR);
+                                rc = SSMR3PutGCPhys(pSSM, GCPhys);
+                            }
+                        }
+                    }
+                    else
+                    {
+                        /*
+                         * Dirty zero page.
+                         */
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+                        if (paLSPages)
+                            pgmR3StateVerifyCrc32ForRamPage(pVM, pCur, paLSPages, iPage, "save#2");
+#endif
+                        pgmUnlock(pVM);
+
+                        if (GCPhys == GCPhysLast + PAGE_SIZE)
+                            rc = SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO);
+                        else
+                        {
+                            SSMR3PutU8(pSSM, PGM_STATE_REC_RAM_ZERO | PGM_STATE_REC_FLAG_ADDR);
+                            rc = SSMR3PutGCPhys(pSSM, GCPhys);
+                        }
+                    }
+                    if (RT_FAILURE(rc))
+                        return rc;
+
+                    pgmLock(pVM);
+                    if (!fSkipped)
+                        GCPhysLast = GCPhys;
+                    if (paLSPages)
+                    {
+                        paLSPages[iPage].fDirty = 0;
+                        pVM->pgm.s.LiveSave.Ram.cReadyPages++;
+                        if (fZero)
+                            pVM->pgm.s.LiveSave.Ram.cZeroPages++;
+                        pVM->pgm.s.LiveSave.Ram.cDirtyPages--;
+                        pVM->pgm.s.LiveSave.cSavedPages++;
+                    }
+                    if (idRamRangesGen != pVM->pgm.s.idRamRangesGen)
+                    {
+                        GCPhysCur = GCPhys | PAGE_OFFSET_MASK;
+                        break; /* restart */
+                    }
+
+                } /* for each page in range */
+
+                if (GCPhysCur != 0)
+                    break; /* Yield + ramrange change */
+                GCPhysCur = pCur->GCPhysLast;
+            }
+        } /* for each range */
+    } while (pCur);
+
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Cleans up RAM pages after a live save.
+ *
+ * @param   pVM                 The VM handle.
+ */
+static void pgmR3DoneRamPages(PVM pVM)
+{
+    /*
+     * Free the tracking arrays and disable write monitoring.
+     *
+     * Play nice with the PGM lock in case we're called while the VM is still
+     * running.  This means we have to delay the freeing since we wish to use
+     * paLSPages as an indicator of which RAM ranges which we need to scan for
+     * write monitored pages.
+     */
+    void *pvToFree = NULL;
+    PPGMRAMRANGE pCur;
+    uint32_t cMonitoredPages = 0;
+    pgmLock(pVM);
+    do
+    {
+        for (pCur = pVM->pgm.s.pRamRangesR3; pCur; pCur = pCur->pNextR3)
+        {
+            if (pCur->paLSPages)
+            {
+                if (pvToFree)
+                {
+                    uint32_t idRamRangesGen = pVM->pgm.s.idRamRangesGen;
+                    pgmUnlock(pVM);
+                    MMR3HeapFree(pvToFree);
+                    pvToFree = NULL;
+                    pgmLock(pVM);
+                    if (idRamRangesGen != pVM->pgm.s.idRamRangesGen)
+                        break;          /* start over again. */
+                }
+
+                pvToFree = pCur->paLSPages;
+                pCur->paLSPages = NULL;
+
+                uint32_t iPage = pCur->cb >> PAGE_SHIFT;
+                while (iPage--)
+                {
+                    PPGMPAGE pPage = &pCur->aPages[iPage];
+                    PGM_PAGE_CLEAR_WRITTEN_TO(pPage);
+                    if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
+                    {
+                        PGM_PAGE_SET_STATE(pPage, PGM_PAGE_STATE_ALLOCATED);
+                        cMonitoredPages++;
+                    }
+                }
+            }
+        }
+    } while (pCur);
+
+    Assert(pVM->pgm.s.cMonitoredPages >= cMonitoredPages);
+    if (pVM->pgm.s.cMonitoredPages < cMonitoredPages)
+        pVM->pgm.s.cMonitoredPages = 0;
+    else
+        pVM->pgm.s.cMonitoredPages -= cMonitoredPages;
+
+    pgmUnlock(pVM);
+
+    MMR3HeapFree(pvToFree);
+    pvToFree = NULL;
+}
+
+
+/**
+ * Execute a live save pass.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pSSM        The SSM handle.
+ */
+static DECLCALLBACK(int) pgmR3LiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    int rc;
+
+    /*
+     * Save the MMIO2 and ROM range IDs in pass 0.
+     */
+    if (uPass == 0)
+    {
+        rc = pgmR3SaveRamConfig(pVM, pSSM);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = pgmR3SaveRomRanges(pVM, pSSM);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = pgmR3SaveMmio2Ranges(pVM, pSSM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    /*
+     * Reset the page-per-second estimate to avoid inflation by the initial
+     * load of zero pages.  pgmR3LiveVote ASSUMES this is done at pass 7.
+     */
+    else if (uPass == 7)
+    {
+        pVM->pgm.s.LiveSave.cSavedPages  = 0;
+        pVM->pgm.s.LiveSave.uSaveStartNS = RTTimeNanoTS();
+    }
+
+    /*
+     * Do the scanning.
+     */
+    pgmR3ScanRomPages(pVM);
+    pgmR3ScanMmio2Pages(pVM, uPass);
+    pgmR3ScanRamPages(pVM, false /*fFinalPass*/);
+    pgmR3PoolClearAll(pVM, true /*fFlushRemTlb*/); /** @todo this could perhaps be optimized a bit. */
+
+    /*
+     * Save the pages.
+     */
+    if (uPass == 0)
+        rc = pgmR3SaveRomVirginPages(  pVM, pSSM, true /*fLiveSave*/);
+    else
+        rc = VINF_SUCCESS;
+    if (RT_SUCCESS(rc))
+        rc = pgmR3SaveShadowedRomPages(pVM, pSSM, true /*fLiveSave*/, false /*fFinalPass*/);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3SaveMmio2Pages(      pVM, pSSM, true /*fLiveSave*/, uPass);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3SaveRamPages(        pVM, pSSM, true /*fLiveSave*/, uPass);
+    SSMR3PutU8(pSSM, PGM_STATE_REC_END);    /* (Ignore the rc, SSM takes care of it.) */
+
+    return rc;
+}
+
+
+/**
+ * Votes on whether the live save phase is done or not.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pSSM        The SSM handle.
+ * @param   uPass       The data pass.
+ */
+static DECLCALLBACK(int)  pgmR3LiveVote(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    /*
+     * Update and calculate parameters used in the decision making.
+     */
+    const uint32_t cHistoryEntries = RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory);
+
+    /* update history. */
+    pgmLock(pVM);
+    uint32_t const cWrittenToPages = pVM->pgm.s.cWrittenToPages;
+    pgmUnlock(pVM);
+    uint32_t const cDirtyNow = pVM->pgm.s.LiveSave.Rom.cDirtyPages
+                             + pVM->pgm.s.LiveSave.Mmio2.cDirtyPages
+                             + pVM->pgm.s.LiveSave.Ram.cDirtyPages
+                             + cWrittenToPages;
+    uint32_t i = pVM->pgm.s.LiveSave.iDirtyPagesHistory;
+    pVM->pgm.s.LiveSave.acDirtyPagesHistory[i] = cDirtyNow;
+    pVM->pgm.s.LiveSave.iDirtyPagesHistory = (i + 1) % cHistoryEntries;
+
+    /* calc shortterm average (4 passes). */
+    AssertCompile(RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory) > 4);
+    uint64_t cTotal = pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
+    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 1) % cHistoryEntries];
+    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 2) % cHistoryEntries];
+    cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[(i + cHistoryEntries - 3) % cHistoryEntries];
+    uint32_t const cDirtyPagesShort = cTotal / 4;
+    pVM->pgm.s.LiveSave.cDirtyPagesShort = cDirtyPagesShort;
+
+    /* calc longterm average. */
+    cTotal = 0;
+    if (uPass < cHistoryEntries)
+        for (i = 0; i < cHistoryEntries && i <= uPass; i++)
+              cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
+    else
+        for (i = 0; i < cHistoryEntries; i++)
+            cTotal += pVM->pgm.s.LiveSave.acDirtyPagesHistory[i];
+    uint32_t const cDirtyPagesLong = cTotal / cHistoryEntries;
+    pVM->pgm.s.LiveSave.cDirtyPagesLong = cDirtyPagesLong;
+
+    /* estimate the speed */
+    uint64_t cNsElapsed = RTTimeNanoTS() - pVM->pgm.s.LiveSave.uSaveStartNS;
+    uint32_t cPagesPerSecond = (uint32_t)(   pVM->pgm.s.LiveSave.cSavedPages
+                                          / ((long double)cNsElapsed / 1000000000.0) );
+    pVM->pgm.s.LiveSave.cPagesPerSecond = cPagesPerSecond;
+
+    /*
+     * Try make a decision.
+     */
+    if (    cDirtyPagesShort <= cDirtyPagesLong
+        &&  (   cDirtyNow    <= cDirtyPagesShort
+             || cDirtyNow - cDirtyPagesShort < RT_MIN(cDirtyPagesShort / 8, 16)
+            )
+       )
+    {
+        if (uPass > 10)
+        {
+            uint32_t cMsLeftShort   = (uint32_t)(cDirtyPagesShort / (long double)cPagesPerSecond * 1000.0);
+            uint32_t cMsLeftLong    = (uint32_t)(cDirtyPagesLong  / (long double)cPagesPerSecond * 1000.0);
+            uint32_t cMsMaxDowntime = SSMR3HandleMaxDowntime(pSSM);
+            if (cMsMaxDowntime < 32)
+                cMsMaxDowntime = 32;
+            if (   (   cMsLeftLong  <= cMsMaxDowntime
+                    && cMsLeftShort <  cMsMaxDowntime)
+                || cMsLeftShort < cMsMaxDowntime / 2
+               )
+            {
+                Log(("pgmR3LiveVote: VINF_SUCCESS - pass=%d cDirtyPagesShort=%u|%ums cDirtyPagesLong=%u|%ums cMsMaxDowntime=%u\n",
+                     uPass, cDirtyPagesShort, cMsLeftShort, cDirtyPagesLong, cMsLeftLong, cMsMaxDowntime));
+                return VINF_SUCCESS;
+            }
+        }
+        else
+        {
+            if (   (   cDirtyPagesShort <= 128
+                    && cDirtyPagesLong  <= 1024)
+                || cDirtyPagesLong <= 256
+               )
+            {
+                Log(("pgmR3LiveVote: VINF_SUCCESS - pass=%d cDirtyPagesShort=%u cDirtyPagesLong=%u\n", uPass, cDirtyPagesShort, cDirtyPagesLong));
+                return VINF_SUCCESS;
+            }
+        }
+    }
+
+    /*
+     * Come up with a completion percentage.  Currently this is a simple
+     * dirty page (long term) vs. total pages ratio + some pass trickery.
+     */
+    unsigned uPctDirty = (unsigned)(  (long double)cDirtyPagesLong
+                                    / (pVM->pgm.s.cAllPages - pVM->pgm.s.LiveSave.cIgnoredPages - pVM->pgm.s.cZeroPages) );
+    if (uPctDirty <= 100)
+        SSMR3HandleReportLivePercent(pSSM, RT_MIN(100 - uPctDirty, uPass * 2));
+    else
+        AssertMsgFailed(("uPctDirty=%u cDirtyPagesLong=%#x cAllPages=%#x cIgnoredPages=%#x cZeroPages=%#x\n",
+                         uPctDirty, cDirtyPagesLong, pVM->pgm.s.cAllPages, pVM->pgm.s.LiveSave.cIgnoredPages, pVM->pgm.s.cZeroPages));
+
+    return VINF_SSM_VOTE_FOR_ANOTHER_PASS;
+}
+
+
+/**
+ * Prepare for a live save operation.
+ *
+ * This will attempt to allocate and initialize the tracking structures.  It
+ * will also prepare for write monitoring of pages and initialize PGM::LiveSave.
+ * pgmR3SaveDone will do the cleanups.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pSSM        The SSM handle.
+ */
+static DECLCALLBACK(int) pgmR3LivePrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Indicate that we will be using the write monitoring.
+     */
+    pgmLock(pVM);
+    /** @todo find a way of mediating this when more users are added. */
+    if (pVM->pgm.s.fPhysWriteMonitoringEngaged)
+    {
+        pgmUnlock(pVM);
+        AssertLogRelFailedReturn(VERR_INTERNAL_ERROR_2);
+    }
+    pVM->pgm.s.fPhysWriteMonitoringEngaged = true;
+    pgmUnlock(pVM);
+
+    /*
+     * Initialize the statistics.
+     */
+    pVM->pgm.s.LiveSave.Rom.cReadyPages   = 0;
+    pVM->pgm.s.LiveSave.Rom.cDirtyPages   = 0;
+    pVM->pgm.s.LiveSave.Mmio2.cReadyPages = 0;
+    pVM->pgm.s.LiveSave.Mmio2.cDirtyPages = 0;
+    pVM->pgm.s.LiveSave.Ram.cReadyPages   = 0;
+    pVM->pgm.s.LiveSave.Ram.cDirtyPages   = 0;
+    pVM->pgm.s.LiveSave.cIgnoredPages     = 0;
+    pVM->pgm.s.LiveSave.fActive           = true;
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.LiveSave.acDirtyPagesHistory); i++)
+        pVM->pgm.s.LiveSave.acDirtyPagesHistory[i] = UINT32_MAX / 2;
+    pVM->pgm.s.LiveSave.iDirtyPagesHistory = 0;
+    pVM->pgm.s.LiveSave.cSavedPages       = 0;
+    pVM->pgm.s.LiveSave.uSaveStartNS      = RTTimeNanoTS();
+    pVM->pgm.s.LiveSave.cPagesPerSecond   = 8192;
+
+    /*
+     * Per page type.
+     */
+    int rc = pgmR3PrepRomPages(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3PrepMmio2Pages(pVM);
+    if (RT_SUCCESS(rc))
+        rc = pgmR3PrepRamPages(pVM);
+    return rc;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) pgmR3SaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    int         rc;
+    unsigned    i;
+    PPGM        pPGM = &pVM->pgm.s;
+
+    /*
+     * Lock PGM and set the no-more-writes indicator.
+     */
+    pgmLock(pVM);
+    pVM->pgm.s.fNoMorePhysWrites = true;
+
+    /*
+     * Save basic data (required / unaffected by relocation).
+     */
+    bool const fMappingsFixed  = pVM->pgm.s.fMappingsFixed;
+    pVM->pgm.s.fMappingsFixed |= pVM->pgm.s.fMappingsFixedRestored;
+    SSMR3PutStruct(pSSM, pPGM, &s_aPGMFields[0]);
+    pVM->pgm.s.fMappingsFixed  = fMappingsFixed;
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        SSMR3PutStruct(pSSM, &pVM->aCpus[idCpu].pgm.s, &s_aPGMCpuFields[0]);
+
+    /*
+     * The guest mappings.
+     */
+    i = 0;
+    for (PPGMMAPPING pMapping = pPGM->pMappingsR3; pMapping; pMapping = pMapping->pNextR3, i++)
+    {
+        SSMR3PutU32(      pSSM, i);
+        SSMR3PutStrZ(     pSSM, pMapping->pszDesc); /* This is the best unique id we have... */
+        SSMR3PutGCPtr(    pSSM, pMapping->GCPtr);
+        SSMR3PutGCUIntPtr(pSSM, pMapping->cPTs);
+    }
+    rc = SSMR3PutU32(pSSM, ~0); /* terminator. */
+
+    /*
+     * Save the (remainder of the) memory.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        if (pVM->pgm.s.LiveSave.fActive)
+        {
+            pgmR3ScanRomPages(pVM);
+            pgmR3ScanMmio2Pages(pVM, SSM_PASS_FINAL);
+            pgmR3ScanRamPages(pVM, true /*fFinalPass*/);
+
+            rc = pgmR3SaveShadowedRomPages(    pVM, pSSM, true /*fLiveSave*/, true /*fFinalPass*/);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveMmio2Pages(      pVM, pSSM, true /*fLiveSave*/, SSM_PASS_FINAL);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRamPages(        pVM, pSSM, true /*fLiveSave*/, SSM_PASS_FINAL);
+        }
+        else
+        {
+            rc = pgmR3SaveRamConfig(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRomRanges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveMmio2Ranges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRomVirginPages(  pVM, pSSM, false /*fLiveSave*/);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveShadowedRomPages(pVM, pSSM, false /*fLiveSave*/, true /*fFinalPass*/);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveMmio2Pages(      pVM, pSSM, false /*fLiveSave*/, SSM_PASS_FINAL);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3SaveRamPages(        pVM, pSSM, false /*fLiveSave*/, SSM_PASS_FINAL);
+        }
+        SSMR3PutU8(pSSM, PGM_STATE_REC_END);    /* (Ignore the rc, SSM takes of it.) */
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Cleans up after an save state operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) pgmR3SaveDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Do per page type cleanups first.
+     */
+    if (pVM->pgm.s.LiveSave.fActive)
+    {
+        pgmR3DoneRomPages(pVM);
+        pgmR3DoneMmio2Pages(pVM);
+        pgmR3DoneRamPages(pVM);
+    }
+
+    /*
+     * Clear the live save indicator and disengage write monitoring.
+     */
+    pgmLock(pVM);
+    pVM->pgm.s.LiveSave.fActive = false;
+    /** @todo this is blindly assuming that we're the only user of write
+     *        monitoring. Fix this when more users are added. */
+    pVM->pgm.s.fPhysWriteMonitoringEngaged = false;
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Prepare state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) pgmR3LoadPrep(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Call the reset function to make sure all the memory is cleared.
+     */
+    PGMR3Reset(pVM);
+    pVM->pgm.s.LiveSave.fActive = false;
+    NOREF(pSSM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Load an ignored page.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ */
+static int pgmR3LoadPageToDevNullOld(PSSMHANDLE pSSM)
+{
+    uint8_t abPage[PAGE_SIZE];
+    return SSMR3GetMem(pSSM, &abPage[0], sizeof(abPage));
+}
+
+
+/**
+ * Loads a page without any bits in the saved state, i.e. making sure it's
+ * really zero.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   uType           The page type or PGMPAGETYPE_INVALID (old saved
+ *                          state).
+ * @param   pPage           The guest page tracking structure.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The ram range (logging).
+ */
+static int pgmR3LoadPageZeroOld(PVM pVM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    if (    PGM_PAGE_GET_TYPE(pPage) != uType
+        &&  uType != PGMPAGETYPE_INVALID)
+        return VERR_SSM_UNEXPECTED_DATA;
+
+    /* I think this should be sufficient. */
+    if (    !PGM_PAGE_IS_ZERO(pPage)
+        &&  !PGM_PAGE_IS_BALLOONED(pPage))
+        return VERR_SSM_UNEXPECTED_DATA;
+
+    NOREF(pVM);
+    NOREF(GCPhys);
+    NOREF(pRam);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a page from the saved state.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The SSM handle.
+ * @param   uType           The page type or PGMPAGETYEP_INVALID (old saved
+ *                          state).
+ * @param   pPage           The guest page tracking structure.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The ram range (logging).
+ */
+static int pgmR3LoadPageBitsOld(PVM pVM, PSSMHANDLE pSSM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    /*
+     * Match up the type, dealing with MMIO2 aliases (dropped).
+     */
+    AssertLogRelMsgReturn(   PGM_PAGE_GET_TYPE(pPage) == uType
+                          || uType == PGMPAGETYPE_INVALID,
+                          ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc),
+                          VERR_SSM_UNEXPECTED_DATA);
+
+    /*
+     * Load the page.
+     */
+    void *pvPage;
+    int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvPage);
+    if (RT_SUCCESS(rc))
+        rc = SSMR3GetMem(pSSM, pvPage, PAGE_SIZE);
+
+    return rc;
+}
+
+
+/**
+ * Loads a page (counter part to pgmR3SavePage).
+ *
+ * @returns VBox status code, fully bitched errors.
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The SSM handle.
+ * @param   uType           The page type.
+ * @param   pPage           The page.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The RAM range (for error messages).
+ */
+static int pgmR3LoadPageOld(PVM pVM, PSSMHANDLE pSSM, uint8_t uType, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    uint8_t uState;
+    int rc = SSMR3GetU8(pSSM, &uState);
+    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s rc=%Rrc\n", pPage, GCPhys, pRam->pszDesc, rc), rc);
+    if (uState == 0 /* zero */)
+        rc = pgmR3LoadPageZeroOld(pVM, uType, pPage, GCPhys, pRam);
+    else if (uState == 1)
+        rc = pgmR3LoadPageBitsOld(pVM, pSSM, uType, pPage, GCPhys, pRam);
+    else
+        rc = VERR_INTERNAL_ERROR;
+    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] uState=%d uType=%d GCPhys=%RGp %s rc=%Rrc\n",
+                                 pPage, uState, uType, GCPhys, pRam->pszDesc, rc),
+                            rc);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a shadowed ROM page.
+ *
+ * @returns VBox status code, errors are fully bitched.
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The saved state handle.
+ * @param   pPage           The page.
+ * @param   GCPhys          The page address.
+ * @param   pRam            The RAM range (for error messages).
+ */
+static int pgmR3LoadShadowedRomPageOld(PVM pVM, PSSMHANDLE pSSM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPGMRAMRANGE pRam)
+{
+    /*
+     * Load and set the protection first, then load the two pages, the first
+     * one is the active the other is the passive.
+     */
+    PPGMROMPAGE pRomPage = pgmR3GetRomPage(pVM, GCPhys);
+    AssertLogRelMsgReturn(pRomPage, ("GCPhys=%RGp %s\n", GCPhys, pRam->pszDesc), VERR_INTERNAL_ERROR);
+
+    uint8_t uProt;
+    int rc = SSMR3GetU8(pSSM, &uProt);
+    AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] GCPhys=%#x %s\n", pPage, GCPhys, pRam->pszDesc), rc);
+    PGMROMPROT  enmProt = (PGMROMPROT)uProt;
+    AssertLogRelMsgReturn(    enmProt >= PGMROMPROT_INVALID
+                          &&  enmProt <  PGMROMPROT_END,
+                          ("enmProt=%d pPage=%R[pgmpage] GCPhys=%#x %s\n", enmProt, pPage, GCPhys, pRam->pszDesc),
+                          VERR_SSM_UNEXPECTED_DATA);
+
+    if (pRomPage->enmProt != enmProt)
+    {
+        rc = PGMR3PhysRomProtect(pVM, GCPhys, PAGE_SIZE, enmProt);
+        AssertLogRelRCReturn(rc, rc);
+        AssertLogRelReturn(pRomPage->enmProt == enmProt, VERR_INTERNAL_ERROR);
+    }
+
+    PPGMPAGE pPageActive  = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Virgin      : &pRomPage->Shadow;
+    PPGMPAGE pPagePassive = PGMROMPROT_IS_ROM(enmProt) ? &pRomPage->Shadow      : &pRomPage->Virgin;
+    uint8_t  u8ActiveType = PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM        : PGMPAGETYPE_ROM_SHADOW;
+    uint8_t  u8PassiveType= PGMROMPROT_IS_ROM(enmProt) ? PGMPAGETYPE_ROM_SHADOW : PGMPAGETYPE_ROM;
+
+    /** @todo this isn't entirely correct as long as pgmPhysGCPhys2CCPtrInternal is
+     *        used down the line (will the 2nd page will be written to the first
+     *        one because of a false TLB hit since the TLB is using GCPhys and
+     *        doesn't check the HCPhys of the desired page). */
+    rc = pgmR3LoadPageOld(pVM, pSSM, u8ActiveType, pPage, GCPhys, pRam);
+    if (RT_SUCCESS(rc))
+    {
+        *pPageActive = *pPage;
+        rc = pgmR3LoadPageOld(pVM, pSSM, u8PassiveType, pPagePassive, GCPhys, pRam);
+    }
+    return rc;
+}
+
+/**
+ * Ram range flags and bits for older versions of the saved state.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM handle
+ * @param   pSSM        The SSM handle.
+ * @param   uVersion    The saved state version.
+ */
+static int pgmR3LoadMemoryOld(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
+{
+    PPGM pPGM = &pVM->pgm.s;
+
+    /*
+     * Ram range flags and bits.
+     */
+    uint32_t i = 0;
+    for (PPGMRAMRANGE pRam = pPGM->pRamRangesR3; ; pRam = pRam->pNextR3, i++)
+    {
+        /* Check the sequence number / separator. */
+        uint32_t u32Sep;
+        int rc = SSMR3GetU32(pSSM, &u32Sep);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u32Sep == ~0U)
+            break;
+        if (u32Sep != i)
+        {
+            AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+        AssertLogRelReturn(pRam, VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        /* Get the range details. */
+        RTGCPHYS GCPhys;
+        SSMR3GetGCPhys(pSSM, &GCPhys);
+        RTGCPHYS GCPhysLast;
+        SSMR3GetGCPhys(pSSM, &GCPhysLast);
+        RTGCPHYS cb;
+        SSMR3GetGCPhys(pSSM, &cb);
+        uint8_t     fHaveBits;
+        rc = SSMR3GetU8(pSSM, &fHaveBits);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (fHaveBits & ~1)
+        {
+            AssertMsgFailed(("u32Sep=%#x (last)\n", u32Sep));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+        size_t  cchDesc = 0;
+        char    szDesc[256];
+        szDesc[0] = '\0';
+        if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
+        {
+            rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+            if (RT_FAILURE(rc))
+                return rc;
+            /* Since we've modified the description strings in r45878, only compare
+               them if the saved state is more recent. */
+            if (uVersion != PGM_SAVED_STATE_VERSION_RR_DESC)
+                cchDesc = strlen(szDesc);
+        }
+
+        /*
+         * Match it up with the current range.
+         *
+         * Note there is a hack for dealing with the high BIOS mapping
+         * in the old saved state format, this means we might not have
+         * a 1:1 match on success.
+         */
+        if (    (   GCPhys     != pRam->GCPhys
+                 || GCPhysLast != pRam->GCPhysLast
+                 || cb         != pRam->cb
+                 ||  (   cchDesc
+                      && strcmp(szDesc, pRam->pszDesc)) )
+                /* Hack for PDMDevHlpPhysReserve(pDevIns, 0xfff80000, 0x80000, "High ROM Region"); */
+            &&  (   uVersion != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE
+                 || GCPhys     != UINT32_C(0xfff80000)
+                 || GCPhysLast != UINT32_C(0xffffffff)
+                 || pRam->GCPhysLast != GCPhysLast
+                 || pRam->GCPhys     <  GCPhys
+                 || !fHaveBits)
+           )
+        {
+            LogRel(("Ram range: %RGp-%RGp %RGp bytes %s %s\n"
+                    "State    : %RGp-%RGp %RGp bytes %s %s\n",
+                    pRam->GCPhys, pRam->GCPhysLast, pRam->cb, pRam->pvR3 ? "bits" : "nobits", pRam->pszDesc,
+                    GCPhys, GCPhysLast, cb, fHaveBits ? "bits" : "nobits", szDesc));
+            /*
+             * If we're loading a state for debugging purpose, don't make a fuss if
+             * the MMIO and ROM stuff isn't 100% right, just skip the mismatches.
+             */
+            if (    SSMR3HandleGetAfter(pSSM) != SSMAFTER_DEBUG_IT
+                ||  GCPhys < 8 * _1M)
+                return SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                        N_("RAM range mismatch; saved={%RGp-%RGp %RGp bytes %s %s} config={%RGp-%RGp %RGp bytes %s %s}"),
+                                        GCPhys, GCPhysLast, cb, fHaveBits ? "bits" : "nobits", szDesc,
+                                        pRam->GCPhys, pRam->GCPhysLast, pRam->cb, pRam->pvR3 ? "bits" : "nobits", pRam->pszDesc);
+
+            AssertMsgFailed(("debug skipping not implemented, sorry\n"));
+            continue;
+        }
+
+        uint32_t cPages = (GCPhysLast - GCPhys + 1) >> PAGE_SHIFT;
+        if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
+        {
+            /*
+             * Load the pages one by one.
+             */
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
+                PPGMPAGE        pPage      = &pRam->aPages[iPage];
+                uint8_t         uType;
+                rc = SSMR3GetU8(pSSM, &uType);
+                AssertLogRelMsgRCReturn(rc, ("pPage=%R[pgmpage] iPage=%#x GCPhysPage=%#x %s\n", pPage, iPage, GCPhysPage, pRam->pszDesc), rc);
+                if (uType == PGMPAGETYPE_ROM_SHADOW)
+                    rc = pgmR3LoadShadowedRomPageOld(pVM, pSSM, pPage, GCPhysPage, pRam);
+                else
+                    rc = pgmR3LoadPageOld(pVM, pSSM, uType, pPage, GCPhysPage, pRam);
+                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
+            }
+        }
+        else
+        {
+            /*
+             * Old format.
+             */
+
+            /* Of the page flags, pick up MMIO2 and ROM/RESERVED for the !fHaveBits case.
+               The rest is generally irrelevant and wrong since the stuff have to match registrations. */
+            uint32_t fFlags = 0;
+            for (uint32_t iPage = 0; iPage < cPages; iPage++)
+            {
+                uint16_t u16Flags;
+                rc = SSMR3GetU16(pSSM, &u16Flags);
+                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
+                fFlags |= u16Flags;
+            }
+
+            /* Load the bits */
+            if (    !fHaveBits
+                &&  GCPhysLast < UINT32_C(0xe0000000))
+            {
+                /*
+                 * Dynamic chunks.
+                 */
+                const uint32_t cPagesInChunk = (1*1024*1024) >> PAGE_SHIFT;
+                AssertLogRelMsgReturn(cPages % cPagesInChunk == 0,
+                                      ("cPages=%#x cPagesInChunk=%#x\n", cPages, cPagesInChunk, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                for (uint32_t iPage = 0; iPage < cPages; /* incremented by inner loop */ )
+                {
+                    uint8_t fPresent;
+                    rc = SSMR3GetU8(pSSM, &fPresent);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
+                    AssertLogRelMsgReturn(fPresent == (uint8_t)true || fPresent == (uint8_t)false,
+                                          ("fPresent=%#x iPage=%#x GCPhys=%#x %s\n", fPresent, iPage, pRam->GCPhys, pRam->pszDesc),
+                                          VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                    for (uint32_t iChunkPage = 0; iChunkPage < cPagesInChunk; iChunkPage++, iPage++)
+                    {
+                        RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
+                        PPGMPAGE        pPage      = &pRam->aPages[iPage];
+                        if (fPresent)
+                        {
+                            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO)
+                                rc = pgmR3LoadPageToDevNullOld(pSSM);
+                            else
+                                rc = pgmR3LoadPageBitsOld(pVM, pSSM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
+                        }
+                        else
+                            rc = pgmR3LoadPageZeroOld(pVM, PGMPAGETYPE_INVALID, pPage, GCPhysPage, pRam);
+                        AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhysPage=%#x %s\n", rc, iPage, GCPhysPage, pRam->pszDesc), rc);
+                    }
+                }
+            }
+            else if (pRam->pvR3)
+            {
+                /*
+                 * MMIO2.
+                 */
+                AssertLogRelMsgReturn((fFlags & 0x0f) == RT_BIT(3) /*MM_RAM_FLAGS_MMIO2*/,
+                                      ("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                AssertLogRelMsgReturn(pRam->pvR3,
+                                      ("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                rc = SSMR3GetMem(pSSM, pRam->pvR3, pRam->cb);
+                AssertLogRelMsgRCReturn(rc, ("GCPhys=%#x %s\n", pRam->GCPhys, pRam->pszDesc), rc);
+            }
+            else if (GCPhysLast < UINT32_C(0xfff80000))
+            {
+                /*
+                 * PCI MMIO, no pages saved.
+                 */
+            }
+            else
+            {
+                /*
+                 * Load the 0xfff80000..0xffffffff BIOS range.
+                 * It starts with X reserved pages that we have to skip over since
+                 * the RAMRANGE create by the new code won't include those.
+                 */
+                AssertLogRelMsgReturn(   !(fFlags & RT_BIT(3) /*MM_RAM_FLAGS_MMIO2*/)
+                                      && (fFlags  & RT_BIT(0) /*MM_RAM_FLAGS_RESERVED*/),
+                                      ("fFlags=%#x GCPhys=%#x %s\n", fFlags, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                AssertLogRelMsgReturn(GCPhys == UINT32_C(0xfff80000),
+                                      ("GCPhys=%RGp pRamRange{GCPhys=%#x %s}\n", GCPhys, pRam->GCPhys, pRam->pszDesc),
+                                      VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                /* Skip wasted reserved pages before the ROM. */
+                while (GCPhys < pRam->GCPhys)
+                {
+                    rc = pgmR3LoadPageToDevNullOld(pSSM);
+                    GCPhys += PAGE_SIZE;
+                }
+
+                /* Load the bios pages. */
+                cPages = pRam->cb >> PAGE_SHIFT;
+                for (uint32_t iPage = 0; iPage < cPages; iPage++)
+                {
+                    RTGCPHYS const  GCPhysPage = ((RTGCPHYS)iPage << PAGE_SHIFT) + pRam->GCPhys;
+                    PPGMPAGE        pPage      = &pRam->aPages[iPage];
+
+                    AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM,
+                                          ("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, GCPhys),
+                                          VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+                    rc = pgmR3LoadPageBitsOld(pVM, pSSM, PGMPAGETYPE_ROM, pPage, GCPhysPage, pRam);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc iPage=%#x GCPhys=%#x %s\n", rc, iPage, pRam->GCPhys, pRam->pszDesc), rc);
+                }
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for pgmR3Load and pgmR3LoadLocked.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ * @param   uVersion            The saved state version.
+ *
+ * @todo    This needs splitting up if more record types or code twists are
+ *          added...
+ */
+static int pgmR3LoadMemory(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    /*
+     * Process page records until we hit the terminator.
+     */
+    RTGCPHYS        GCPhys   = NIL_RTGCPHYS;
+    PPGMRAMRANGE    pRamHint = NULL;
+    uint8_t         id       = UINT8_MAX;
+    uint32_t        iPage    = UINT32_MAX - 10;
+    PPGMROMRANGE    pRom     = NULL;
+    PPGMMMIO2RANGE  pMmio2   = NULL;
+
+    /*
+     * We batch up pages that should be freed instead of calling GMM for
+     * each and every one of them.  Note that we'll lose the pages in most
+     * failure paths - this should probably be addressed one day.
+     */
+    uint32_t            cPendingPages = 0;
+    PGMMFREEPAGESREQ    pReq;
+    int rc = GMMR3FreePagesPrepare(pVM, &pReq, 128 /* batch size */, GMMACCOUNT_BASE);
+    AssertLogRelRCReturn(rc, rc);
+
+    for (;;)
+    {
+        /*
+         * Get the record type and flags.
+         */
+        uint8_t u8;
+        rc = SSMR3GetU8(pSSM, &u8);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u8 == PGM_STATE_REC_END)
+        {
+            /*
+             * Finish off any pages pending freeing.
+             */
+            if (cPendingPages)
+            {
+                rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
+                AssertLogRelRCReturn(rc, rc);
+            }
+            GMMR3FreePagesCleanup(pReq);
+            return VINF_SUCCESS;
+        }
+        AssertLogRelMsgReturn((u8 & ~PGM_STATE_REC_FLAG_ADDR) <= PGM_STATE_REC_LAST, ("%#x\n", u8), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+        switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+        {
+            /*
+             * RAM page.
+             */
+            case PGM_STATE_REC_RAM_ZERO:
+            case PGM_STATE_REC_RAM_RAW:
+            {
+                /*
+                 * Get the address and resolve it into a page descriptor.
+                 */
+                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
+                    GCPhys += PAGE_SIZE;
+                else
+                {
+                    rc = SSMR3GetGCPhys(pSSM, &GCPhys);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                AssertLogRelMsgReturn(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+                PPGMPAGE pPage;
+                rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pPage, &pRamHint);
+                AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
+
+                /*
+                 * Take action according to the record type.
+                 */
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_RAM_ZERO:
+                    {
+                        if (    PGM_PAGE_IS_ZERO(pPage)
+                            ||  PGM_PAGE_IS_BALLOONED(pPage))
+                            break;
+                        AssertLogRelMsgReturn(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED, ("GCPhys=%RGp %R[pgmpage]\n", GCPhys, pPage), VERR_INTERNAL_ERROR_5);
+
+                        /* If this is a ROM page, we must clear it and not try
+                           free it... */
+                        if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM
+                            || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_ROM_SHADOW)
+                        {
+                            void *pvDstPage;
+                            rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDstPage);
+                            AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp %R[pgmpage] rc=%Rrc\n", GCPhys, pPage, rc), rc);
+                            ASMMemZeroPage(pvDstPage);
+                        }
+                        /* Free it only if it's not part of a previously
+                           allocated large page (no need to clear the page). */
+                        else if (PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE)
+                        {
+                            rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, GCPhys);
+                            AssertRCReturn(rc, rc);
+                        }
+                        break;
+                    }
+
+                    case PGM_STATE_REC_RAM_RAW:
+                    {
+                        void *pvDstPage;
+                        rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDstPage);
+                        AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp %R[pgmpage] rc=%Rrc\n", GCPhys, pPage, rc), rc);
+                        rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
+                        if (RT_FAILURE(rc))
+                            return rc;
+                        break;
+                    }
+
+                    default:
+                        AssertMsgFailedReturn(("%#x\n", u8), VERR_INTERNAL_ERROR);
+                }
+                id = UINT8_MAX;
+                break;
+            }
+
+            /*
+             * MMIO2 page.
+             */
+            case PGM_STATE_REC_MMIO2_RAW:
+            case PGM_STATE_REC_MMIO2_ZERO:
+            {
+                /*
+                 * Get the ID + page number and resolved that into a MMIO2 page.
+                 */
+                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
+                    iPage++;
+                else
+                {
+                    SSMR3GetU8(pSSM, &id);
+                    rc = SSMR3GetU32(pSSM, &iPage);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                if (    !pMmio2
+                    ||  pMmio2->idSavedState != id)
+                {
+                    for (pMmio2 = pVM->pgm.s.pMmio2RangesR3; pMmio2; pMmio2 = pMmio2->pNextR3)
+                        if (pMmio2->idSavedState == id)
+                            break;
+                    AssertLogRelMsgReturn(pMmio2, ("id=%#u iPage=%#x\n", id, iPage), VERR_INTERNAL_ERROR);
+                }
+                AssertLogRelMsgReturn(iPage < (pMmio2->RamRange.cb >> PAGE_SHIFT), ("iPage=%#x cb=%RGp %s\n", iPage, pMmio2->RamRange.cb, pMmio2->RamRange.pszDesc), VERR_INTERNAL_ERROR);
+                void *pvDstPage = (uint8_t *)pMmio2->RamRange.pvR3 + ((size_t)iPage << PAGE_SHIFT);
+
+                /*
+                 * Load the page bits.
+                 */
+                if ((u8 & ~PGM_STATE_REC_FLAG_ADDR) == PGM_STATE_REC_MMIO2_ZERO)
+                    ASMMemZeroPage(pvDstPage);
+                else
+                {
+                    rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                GCPhys = NIL_RTGCPHYS;
+                break;
+            }
+
+            /*
+             * ROM pages.
+             */
+            case PGM_STATE_REC_ROM_VIRGIN:
+            case PGM_STATE_REC_ROM_SHW_RAW:
+            case PGM_STATE_REC_ROM_SHW_ZERO:
+            case PGM_STATE_REC_ROM_PROT:
+            {
+                /*
+                 * Get the ID + page number and resolved that into a ROM page descriptor.
+                 */
+                if (!(u8 & PGM_STATE_REC_FLAG_ADDR))
+                    iPage++;
+                else
+                {
+                    SSMR3GetU8(pSSM, &id);
+                    rc = SSMR3GetU32(pSSM, &iPage);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                if (    !pRom
+                    ||  pRom->idSavedState != id)
+                {
+                    for (pRom = pVM->pgm.s.pRomRangesR3; pRom; pRom = pRom->pNextR3)
+                        if (pRom->idSavedState == id)
+                            break;
+                    AssertLogRelMsgReturn(pRom, ("id=%#u iPage=%#x\n", id, iPage), VERR_INTERNAL_ERROR);
+                }
+                AssertLogRelMsgReturn(iPage < (pRom->cb >> PAGE_SHIFT), ("iPage=%#x cb=%RGp %s\n", iPage, pRom->cb, pRom->pszDesc), VERR_INTERNAL_ERROR);
+                PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
+                GCPhys = pRom->GCPhys + ((RTGCPHYS)iPage << PAGE_SHIFT);
+
+                /*
+                 * Get and set the protection.
+                 */
+                uint8_t u8Prot;
+                rc = SSMR3GetU8(pSSM, &u8Prot);
+                if (RT_FAILURE(rc))
+                    return rc;
+                PGMROMPROT enmProt = (PGMROMPROT)u8Prot;
+                AssertLogRelMsgReturn(enmProt > PGMROMPROT_INVALID && enmProt < PGMROMPROT_END, ("GCPhys=%RGp enmProt=%d\n", GCPhys, enmProt), VERR_INTERNAL_ERROR);
+
+                if (enmProt != pRomPage->enmProt)
+                {
+                    if (RT_UNLIKELY(!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)))
+                        return SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                                N_("Protection change of unshadowed ROM page: GCPhys=%RGp enmProt=%d %s"),
+                                                GCPhys, enmProt, pRom->pszDesc);
+                    rc = PGMR3PhysRomProtect(pVM, GCPhys, PAGE_SIZE, enmProt);
+                    AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp rc=%Rrc\n", GCPhys, rc), rc);
+                    AssertLogRelReturn(pRomPage->enmProt == enmProt, VERR_INTERNAL_ERROR);
+                }
+                if ((u8 & ~PGM_STATE_REC_FLAG_ADDR) == PGM_STATE_REC_ROM_PROT)
+                    break; /* done */
+
+                /*
+                 * Get the right page descriptor.
+                 */
+                PPGMPAGE pRealPage;
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_ROM_VIRGIN:
+                        if (!PGMROMPROT_IS_ROM(enmProt))
+                            pRealPage = &pRomPage->Virgin;
+                        else
+                            pRealPage = NULL;
+                        break;
+
+                    case PGM_STATE_REC_ROM_SHW_RAW:
+                    case PGM_STATE_REC_ROM_SHW_ZERO:
+                        if (RT_UNLIKELY(!(pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)))
+                            return SSMR3SetCfgError(pSSM, RT_SRC_POS,
+                                                    N_("Shadowed / non-shadowed page type mismatch: GCPhys=%RGp enmProt=%d %s"),
+                                                    GCPhys, enmProt, pRom->pszDesc);
+                        if (PGMROMPROT_IS_ROM(enmProt))
+                            pRealPage = &pRomPage->Shadow;
+                        else
+                            pRealPage = NULL;
+                        break;
+
+                    default: AssertLogRelFailedReturn(VERR_INTERNAL_ERROR); /* shut up gcc */
+                }
+                if (!pRealPage)
+                {
+                    rc = pgmPhysGetPageWithHintEx(&pVM->pgm.s, GCPhys, &pRealPage, &pRamHint);
+                    AssertLogRelMsgRCReturn(rc, ("rc=%Rrc %RGp\n", rc, GCPhys), rc);
+                }
+
+                /*
+                 * Make it writable and map it (if necessary).
+                 */
+                void *pvDstPage = NULL;
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_ROM_SHW_ZERO:
+                        if (    PGM_PAGE_IS_ZERO(pRealPage)
+                            ||  PGM_PAGE_IS_BALLOONED(pRealPage))
+                            break;
+                        /** @todo implement zero page replacing. */
+                        /* fall thru */
+                    case PGM_STATE_REC_ROM_VIRGIN:
+                    case PGM_STATE_REC_ROM_SHW_RAW:
+                    {
+                        rc = pgmPhysPageMakeWritableAndMap(pVM, pRealPage, GCPhys, &pvDstPage);
+                        AssertLogRelMsgRCReturn(rc, ("GCPhys=%RGp rc=%Rrc\n", GCPhys, rc), rc);
+                        break;
+                    }
+                }
+
+                /*
+                 * Load the bits.
+                 */
+                switch (u8 & ~PGM_STATE_REC_FLAG_ADDR)
+                {
+                    case PGM_STATE_REC_ROM_SHW_ZERO:
+                        if (pvDstPage)
+                            ASMMemZeroPage(pvDstPage);
+                        break;
+
+                    case PGM_STATE_REC_ROM_VIRGIN:
+                    case PGM_STATE_REC_ROM_SHW_RAW:
+                        rc = SSMR3GetMem(pSSM, pvDstPage, PAGE_SIZE);
+                        if (RT_FAILURE(rc))
+                            return rc;
+                        break;
+                }
+                GCPhys = NIL_RTGCPHYS;
+                break;
+            }
+
+            /*
+             * Unknown type.
+             */
+            default:
+                AssertLogRelMsgFailedReturn(("%#x\n", u8), VERR_INTERNAL_ERROR);
+        }
+    } /* forever */
+}
+
+
+/**
+ * Worker for pgmR3Load.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ * @param   uVersion            The saved state version.
+ */
+static int pgmR3LoadFinalLocked(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion)
+{
+    PPGM        pPGM = &pVM->pgm.s;
+    int         rc;
+    uint32_t    u32Sep;
+
+    /*
+     * Load basic data (required / unaffected by relocation).
+     */
+    if (uVersion >= PGM_SAVED_STATE_VERSION_3_0_0)
+    {
+        if (uVersion > PGM_SAVED_STATE_VERSION_PRE_BALLOON)
+            rc = SSMR3GetStruct(pSSM, pPGM, &s_aPGMFields[0]);
+        else
+            rc = SSMR3GetStruct(pSSM, pPGM, &s_aPGMFieldsPreBalloon[0]);
+
+        AssertLogRelRCReturn(rc, rc);
+
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            rc = SSMR3GetStruct(pSSM, &pVM->aCpus[i].pgm.s, &s_aPGMCpuFields[0]);
+            AssertLogRelRCReturn(rc, rc);
+        }
+    }
+    else if (uVersion >= PGM_SAVED_STATE_VERSION_RR_DESC)
+    {
+        AssertRelease(pVM->cCpus == 1);
+
+        PGMOLD pgmOld;
+        rc = SSMR3GetStruct(pSSM, &pgmOld, &s_aPGMFields_Old[0]);
+        AssertLogRelRCReturn(rc, rc);
+
+        pPGM->fMappingsFixed    = pgmOld.fMappingsFixed;
+        pPGM->GCPtrMappingFixed = pgmOld.GCPtrMappingFixed;
+        pPGM->cbMappingFixed    = pgmOld.cbMappingFixed;
+
+        pVM->aCpus[0].pgm.s.fA20Enabled   = pgmOld.fA20Enabled;
+        pVM->aCpus[0].pgm.s.GCPhysA20Mask = pgmOld.GCPhysA20Mask;
+        pVM->aCpus[0].pgm.s.enmGuestMode  = pgmOld.enmGuestMode;
+    }
+    else
+    {
+        AssertRelease(pVM->cCpus == 1);
+
+        SSMR3GetBool(pSSM,      &pPGM->fMappingsFixed);
+        SSMR3GetGCPtr(pSSM,     &pPGM->GCPtrMappingFixed);
+        SSMR3GetU32(pSSM,       &pPGM->cbMappingFixed);
+
+        uint32_t cbRamSizeIgnored;
+        rc = SSMR3GetU32(pSSM,  &cbRamSizeIgnored);
+        if (RT_FAILURE(rc))
+            return rc;
+        SSMR3GetGCPhys(pSSM,    &pVM->aCpus[0].pgm.s.GCPhysA20Mask);
+
+        uint32_t u32 = 0;
+        SSMR3GetUInt(pSSM,      &u32);
+        pVM->aCpus[0].pgm.s.fA20Enabled = !!u32;
+        SSMR3GetUInt(pSSM,      &pVM->aCpus[0].pgm.s.fSyncFlags);
+        RTUINT uGuestMode;
+        SSMR3GetUInt(pSSM,      &uGuestMode);
+        pVM->aCpus[0].pgm.s.enmGuestMode = (PGMMODE)uGuestMode;
+
+        /* check separator. */
+        SSMR3GetU32(pSSM, &u32Sep);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u32Sep != (uint32_t)~0)
+        {
+            AssertMsgFailed(("u32Sep=%#x (first)\n", u32Sep));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+    }
+
+    /*
+     * The guest mappings - skipped now, see re-fixation in the caller.
+     */
+    uint32_t i = 0;
+    for (;; i++)
+    {
+        rc = SSMR3GetU32(pSSM, &u32Sep);        /* sequence number */
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u32Sep == ~0U)
+            break;
+        AssertMsgReturn(u32Sep == i, ("u32Sep=%#x i=%#x\n", u32Sep, i), VERR_SSM_DATA_UNIT_FORMAT_CHANGED);
+
+        char szDesc[256];
+        rc = SSMR3GetStrZ(pSSM, szDesc, sizeof(szDesc));
+        if (RT_FAILURE(rc))
+            return rc;
+        RTGCPTR GCPtrIgnore;
+        SSMR3GetGCPtr(pSSM, &GCPtrIgnore);      /* GCPtr */
+        rc = SSMR3GetGCPtr(pSSM, &GCPtrIgnore); /* cPTs  */
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Load the RAM contents.
+     */
+    if (uVersion > PGM_SAVED_STATE_VERSION_3_0_0)
+    {
+        if (!pVM->pgm.s.LiveSave.fActive)
+        {
+            if (uVersion > PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
+            {
+                rc = pgmR3LoadRamConfig(pVM, pSSM);
+                if (RT_FAILURE(rc))
+                    return rc;
+            }
+            rc = pgmR3LoadRomRanges(pVM, pSSM);
+            if (RT_FAILURE(rc))
+                return rc;
+            rc = pgmR3LoadMmio2Ranges(pVM, pSSM);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        rc = pgmR3LoadMemory(pVM, pSSM, SSM_PASS_FINAL);
+    }
+    else
+        rc = pgmR3LoadMemoryOld(pVM, pSSM, uVersion);
+
+    /* Refresh balloon accounting. */
+    if (pVM->pgm.s.cBalloonedPages)
+    {
+        rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_INFLATE, pVM->pgm.s.cBalloonedPages);
+        AssertRCReturn(rc, rc);
+    }
+    return rc;
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) pgmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    int     rc;
+    PPGM    pPGM = &pVM->pgm.s;
+
+    /*
+     * Validate version.
+     */
+    if (   (   uPass != SSM_PASS_FINAL
+            && uVersion != PGM_SAVED_STATE_VERSION
+            && uVersion != PGM_SAVED_STATE_VERSION_PRE_BALLOON
+            && uVersion != PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
+        || (   uVersion != PGM_SAVED_STATE_VERSION
+            && uVersion != PGM_SAVED_STATE_VERSION_PRE_BALLOON
+            && uVersion != PGM_SAVED_STATE_VERSION_NO_RAM_CFG
+            && uVersion != PGM_SAVED_STATE_VERSION_3_0_0
+            && uVersion != PGM_SAVED_STATE_VERSION_2_2_2
+            && uVersion != PGM_SAVED_STATE_VERSION_RR_DESC
+            && uVersion != PGM_SAVED_STATE_VERSION_OLD_PHYS_CODE)
+       )
+    {
+        AssertMsgFailed(("pgmR3Load: Invalid version uVersion=%d (current %d)!\n", uVersion, PGM_SAVED_STATE_VERSION));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Do the loading while owning the lock because a bunch of the functions
+     * we're using requires this.
+     */
+    if (uPass != SSM_PASS_FINAL)
+    {
+        pgmLock(pVM);
+        if (uPass != 0)
+            rc = pgmR3LoadMemory(pVM, pSSM, uPass);
+        else
+        {
+            pVM->pgm.s.LiveSave.fActive = true;
+            if (uVersion > PGM_SAVED_STATE_VERSION_NO_RAM_CFG)
+                rc = pgmR3LoadRamConfig(pVM, pSSM);
+            else
+                rc = VINF_SUCCESS;
+            if (RT_SUCCESS(rc))
+                rc = pgmR3LoadRomRanges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3LoadMmio2Ranges(pVM, pSSM);
+            if (RT_SUCCESS(rc))
+                rc = pgmR3LoadMemory(pVM, pSSM, uPass);
+        }
+        pgmUnlock(pVM);
+    }
+    else
+    {
+        pgmLock(pVM);
+        rc = pgmR3LoadFinalLocked(pVM, pSSM, uVersion);
+        pVM->pgm.s.LiveSave.fActive = false;
+        pgmUnlock(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * We require a full resync now.
+             */
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                PVMCPU pVCpu = &pVM->aCpus[i];
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+                pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
+            }
+
+            pgmR3HandlerPhysicalUpdateAll(pVM);
+
+            /*
+             * Change the paging mode and restore PGMCPU::GCPhysCR3.
+             * (The latter requires the CPUM state to be restored already.)
+             */
+            if (CPUMR3IsStateRestorePending(pVM))
+                return SSMR3SetLoadError(pSSM, VERR_WRONG_ORDER, RT_SRC_POS,
+                                         N_("PGM was unexpectedly restored before CPUM"));
+
+            for (VMCPUID i = 0; i < pVM->cCpus; i++)
+            {
+                PVMCPU pVCpu = &pVM->aCpus[i];
+
+                rc = PGMR3ChangeMode(pVM, pVCpu, pVCpu->pgm.s.enmGuestMode);
+                AssertLogRelRCReturn(rc, rc);
+
+                /* Update pVM->pgm.s.GCPhysCR3. */
+                Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS || FTMIsDeltaLoadSaveActive(pVM));
+                RTGCPHYS GCPhysCR3 = CPUMGetGuestCR3(pVCpu);
+                if (    pVCpu->pgm.s.enmGuestMode == PGMMODE_PAE
+                    ||  pVCpu->pgm.s.enmGuestMode == PGMMODE_PAE_NX
+                    ||  pVCpu->pgm.s.enmGuestMode == PGMMODE_AMD64
+                    ||  pVCpu->pgm.s.enmGuestMode == PGMMODE_AMD64_NX)
+                    GCPhysCR3 = (GCPhysCR3 & X86_CR3_PAE_PAGE_MASK);
+                else
+                    GCPhysCR3 = (GCPhysCR3 & X86_CR3_PAGE_MASK);
+                pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+
+                /* Update the PSE, NX flags and validity masks. */
+                pVCpu->pgm.s.fGst32BitPageSizeExtension = CPUMIsGuestPageSizeExtEnabled(pVCpu);
+                PGMNotifyNxeChanged(pVCpu, CPUMIsGuestNXEnabled(pVCpu));
+            }
+
+            /*
+             * Try re-fixate the guest mappings.
+             */
+            pVM->pgm.s.fMappingsFixedRestored = false;
+            if (   pVM->pgm.s.fMappingsFixed
+                && pgmMapAreMappingsEnabled(&pVM->pgm.s))
+            {
+                RTGCPTR     GCPtrFixed    = pVM->pgm.s.GCPtrMappingFixed;
+                uint32_t    cbFixed       = pVM->pgm.s.cbMappingFixed;
+                pVM->pgm.s.fMappingsFixed = false;
+
+                uint32_t    cbRequired;
+                int rc2 = PGMR3MappingsSize(pVM, &cbRequired); AssertRC(rc2);
+                if (   RT_SUCCESS(rc2)
+                    && cbRequired > cbFixed)
+                    rc2 = VERR_OUT_OF_RANGE;
+                if (RT_SUCCESS(rc2))
+                    rc2 = pgmR3MappingsFixInternal(pVM, GCPtrFixed, cbFixed);
+                if (RT_FAILURE(rc2))
+                {
+                    LogRel(("PGM: Unable to re-fixate the guest mappings at %RGv-%RGv: rc=%Rrc (cbRequired=%#x)\n",
+                            GCPtrFixed, GCPtrFixed + cbFixed, rc2, cbRequired));
+                    pVM->pgm.s.fMappingsFixed         = false;
+                    pVM->pgm.s.fMappingsFixedRestored = true;
+                    pVM->pgm.s.GCPtrMappingFixed      = GCPtrFixed;
+                    pVM->pgm.s.cbMappingFixed         = cbFixed;
+                }
+            }
+            else
+            {
+                /* We used to set fixed + disabled while we only use disabled now,
+                   so wipe the state to avoid any confusion. */
+                pVM->pgm.s.fMappingsFixed    = false;
+                pVM->pgm.s.GCPtrMappingFixed = NIL_RTGCPTR;
+                pVM->pgm.s.cbMappingFixed    = 0;
+            }
+
+            /*
+             * If we have floating mappings, do a CR3 sync now to make sure the HMA
+             * doesn't conflict with guest code / data and thereby cause trouble
+             * when restoring other components like PATM.
+             */
+            if (pgmMapAreMappingsFloating(&pVM->pgm.s))
+            {
+                PVMCPU pVCpu = &pVM->aCpus[0];
+                rc = PGMSyncCR3(pVCpu, CPUMGetGuestCR0(pVCpu), CPUMGetGuestCR3(pVCpu),  CPUMGetGuestCR4(pVCpu), true);
+                if (RT_FAILURE(rc))
+                    return SSMR3SetLoadError(pSSM, VERR_WRONG_ORDER, RT_SRC_POS,
+                                             N_("PGMSyncCR3 failed unexpectedly with rc=%Rrc"), rc);
+
+                /* Make sure to re-sync before executing code. */
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+                pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL;
+            }
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Registers the saved state callbacks with SSM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to VM structure.
+ * @param   cbRam   The RAM size.
+ */
+int pgmR3InitSavedState(PVM pVM, uint64_t cbRam)
+{
+    return SSMR3RegisterInternal(pVM, "pgm", 1, PGM_SAVED_STATE_VERSION, (size_t)cbRam + sizeof(PGM),
+                                 pgmR3LivePrep, pgmR3LiveExec, pgmR3LiveVote,
+                                 NULL,          pgmR3SaveExec, pgmR3SaveDone,
+                                 pgmR3LoadPrep, pgmR3Load,     NULL);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/PGMSharedPage.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMSharedPage.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMSharedPage.cpp	(revision 35333)
@@ -0,0 +1,418 @@
+/* $Id$ */
+/** @file
+ * PGM - Page Manager and Monitor, Shared page handling
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_SHARED
+#include <VBox/pgm.h>
+#include <VBox/stam.h>
+#include "PGMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/sup.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+
+#include "PGMInline.h"
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+/** Keep a copy of all registered shared modules for the .pgmcheckduppages debugger command. */
+static PGMMREGISTERSHAREDMODULEREQ g_apSharedModules[512] = {0};
+static unsigned g_cSharedModules = 0;
+#endif
+
+/**
+ * Registers a new shared module for the VM
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle
+ * @param   enmGuestOS          Guest OS type
+ * @param   pszModuleName       Module name
+ * @param   pszVersion          Module version
+ * @param   GCBaseAddr          Module base address
+ * @param   cbModule            Module size
+ * @param   cRegions            Number of shared region descriptors
+ * @param   pRegions            Shared region(s)
+ */
+VMMR3DECL(int) PGMR3SharedModuleRegister(PVM pVM, VBOXOSFAMILY enmGuestOS, char *pszModuleName, char *pszVersion, RTGCPTR GCBaseAddr, uint32_t cbModule,
+                                         unsigned cRegions, VMMDEVSHAREDREGIONDESC *pRegions)
+{
+#ifdef VBOX_WITH_PAGE_SHARING
+    PGMMREGISTERSHAREDMODULEREQ pReq;
+
+    Log(("PGMR3SharedModuleRegister family=%d name=%s version=%s base=%RGv size=%x cRegions=%d\n", enmGuestOS, pszModuleName, pszVersion, GCBaseAddr, cbModule, cRegions));
+
+    /* Sanity check. */
+    AssertReturn(cRegions < VMMDEVSHAREDREGIONDESC_MAX, VERR_INVALID_PARAMETER);
+
+    pReq = (PGMMREGISTERSHAREDMODULEREQ)RTMemAllocZ(RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]));
+    AssertReturn(pReq, VERR_NO_MEMORY);
+
+    pReq->enmGuestOS    = enmGuestOS;
+    pReq->GCBaseAddr    = GCBaseAddr;
+    pReq->cbModule      = cbModule;
+    pReq->cRegions      = cRegions;
+    for (unsigned i = 0; i < cRegions; i++)
+        pReq->aRegions[i] = pRegions[i];
+
+    if (    RTStrCopy(pReq->szName, sizeof(pReq->szName), pszModuleName) != VINF_SUCCESS
+        ||  RTStrCopy(pReq->szVersion, sizeof(pReq->szVersion), pszVersion) != VINF_SUCCESS)
+    {
+        RTMemFree(pReq);
+        return VERR_BUFFER_OVERFLOW;
+    }
+
+    int rc = GMMR3RegisterSharedModule(pVM, pReq);
+# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+    if (rc == VINF_SUCCESS)
+    {
+        PGMMREGISTERSHAREDMODULEREQ *ppSharedModule = NULL;
+
+        if (g_cSharedModules < RT_ELEMENTS(g_apSharedModules))
+        {
+            for (unsigned i = 0; i < RT_ELEMENTS(g_apSharedModules); i++)
+            {
+                if (g_apSharedModules[i] == NULL)
+                {
+                    ppSharedModule = &g_apSharedModules[i];
+                    break;
+                }
+            }
+            Assert(ppSharedModule);
+
+            if (ppSharedModule)
+            {
+                *ppSharedModule = (PGMMREGISTERSHAREDMODULEREQ)RTMemAllocZ(RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]));
+                memcpy(*ppSharedModule, pReq, RT_OFFSETOF(GMMREGISTERSHAREDMODULEREQ, aRegions[cRegions]));
+                g_cSharedModules++;
+            }
+        }
+    }
+# endif
+
+    RTMemFree(pReq);
+    Assert(rc == VINF_SUCCESS || rc == VINF_PGM_SHARED_MODULE_COLLISION || rc == VINF_PGM_SHARED_MODULE_ALREADY_REGISTERED);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+/**
+ * Unregisters a shared module for the VM
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle
+ * @param   pszModuleName       Module name
+ * @param   pszVersion          Module version
+ * @param   GCBaseAddr          Module base address
+ * @param   cbModule            Module size
+ */
+VMMR3DECL(int) PGMR3SharedModuleUnregister(PVM pVM, char *pszModuleName, char *pszVersion, RTGCPTR GCBaseAddr, uint32_t cbModule)
+{
+#ifdef VBOX_WITH_PAGE_SHARING
+    PGMMUNREGISTERSHAREDMODULEREQ pReq;
+
+    Log(("PGMR3SharedModuleUnregister name=%s version=%s base=%RGv size=%x\n", pszModuleName, pszVersion, GCBaseAddr, cbModule));
+
+    pReq = (PGMMUNREGISTERSHAREDMODULEREQ)RTMemAllocZ(sizeof(*pReq));
+    AssertReturn(pReq, VERR_NO_MEMORY);
+
+    pReq->GCBaseAddr    = GCBaseAddr;
+    pReq->cbModule      = cbModule;
+
+    if (    RTStrCopy(pReq->szName, sizeof(pReq->szName), pszModuleName) != VINF_SUCCESS
+        ||  RTStrCopy(pReq->szVersion, sizeof(pReq->szVersion), pszVersion) != VINF_SUCCESS)
+    {
+        RTMemFree(pReq);
+        return VERR_BUFFER_OVERFLOW;
+    }
+    int rc = GMMR3UnregisterSharedModule(pVM, pReq);
+    RTMemFree(pReq);
+
+# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+    for (unsigned i = 0; i < g_cSharedModules; i++)
+    {
+        if (    g_apSharedModules[i]
+            &&  !strcmp(g_apSharedModules[i]->szName, pszModuleName)
+            &&  !strcmp(g_apSharedModules[i]->szVersion, pszVersion))
+        {
+            RTMemFree(g_apSharedModules[i]);
+            g_apSharedModules[i] = NULL;
+            g_cSharedModules--;
+            break;
+        }
+    }
+# endif
+    return rc;
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+#ifdef VBOX_WITH_PAGE_SHARING
+/**
+ * Rendezvous callback that will be called once.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM                 VM handle.
+ * @param   pVCpu               The VMCPU handle for the calling EMT.
+ * @param   pvUser              Not used;
+ */
+static DECLCALLBACK(VBOXSTRICTRC) pgmR3SharedModuleRegRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    VMCPUID idCpu = *(VMCPUID *)pvUser;
+
+    /* Execute on the VCPU that issued the original request to make sure we're in the right cr3 context. */
+    if (pVCpu->idCpu != idCpu)
+    {
+        Assert(pVM->cCpus > 1);
+        return VINF_SUCCESS;
+    }
+
+    /* Flush all pending handy page operations before changing any shared page assignments. */
+    int rc = PGMR3PhysAllocateHandyPages(pVM);
+    AssertRC(rc);
+
+    /* Lock it here as we can't deal with busy locks in this ring-0 path. */
+    pgmLock(pVM);
+    rc = GMMR3CheckSharedModules(pVM);
+    pgmUnlock(pVM);
+    AssertLogRelRC(rc);
+    return rc;
+}
+
+/**
+ * Shared module check helper (called on the way out).
+ *
+ * @param   pVM         The VM handle.
+ * @param   VMCPUID     VCPU id
+ */
+static DECLCALLBACK(void) pgmR3CheckSharedModulesHelper(PVM pVM, VMCPUID idCpu)
+{
+    /* We must stall other VCPUs as we'd otherwise have to send IPI flush commands for every single change we make. */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE, pgmR3SharedModuleRegRendezvous, &idCpu);
+    Assert(rc == VINF_SUCCESS);
+}
+#endif
+
+/**
+ * Check all registered modules for changes.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle
+ */
+VMMR3DECL(int) PGMR3SharedModuleCheckAll(PVM pVM)
+{
+#ifdef VBOX_WITH_PAGE_SHARING
+    /* Queue the actual registration as we are under the IOM lock right now. Perform this operation on the way out. */
+    return VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3CheckSharedModulesHelper, 2, pVM, VMMGetCpuId(pVM));
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+/**
+ * Query the state of a page in a shared module
+ *
+ * @returns VBox status code.
+ * @param   pVM                 VM handle
+ * @param   GCPtrPage           Page address
+ * @param   pfShared            Shared status (out)
+ * @param   puPageFlags         Page flags (out)
+ */
+VMMR3DECL(int) PGMR3SharedModuleGetPageState(PVM pVM, RTGCPTR GCPtrPage, bool *pfShared, uint64_t *puPageFlags)
+{
+#if defined(VBOX_WITH_PAGE_SHARING) && defined(DEBUG)
+    /* Debug only API for the page fusion testcase. */
+    RTGCPHYS GCPhys;
+    uint64_t fFlags;
+
+    pgmLock(pVM);
+
+    int rc = PGMGstGetPage(VMMGetCpu(pVM), GCPtrPage, &fFlags, &GCPhys);
+    switch (rc)
+    {
+    case VINF_SUCCESS:
+    {
+        PPGMPAGE pPage = pgmPhysGetPage(&pVM->pgm.s, GCPhys);
+        if (pPage)
+        {
+            *pfShared    = PGM_PAGE_IS_SHARED(pPage);
+            *puPageFlags = fFlags;
+        }
+        else
+            rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+        break;
+    }
+    case VERR_PAGE_NOT_PRESENT:
+    case VERR_PAGE_TABLE_NOT_PRESENT:
+    case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
+    case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
+        *pfShared = false;
+        *puPageFlags = 0;
+        rc = VINF_SUCCESS;
+        break;
+
+    default:
+        break;
+    }
+
+    pgmUnlock(pVM);
+    return rc;
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+#if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+/**
+ * The '.pgmcheckduppages' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+DECLCALLBACK(int)  pgmR3CmdCheckDuplicatePages(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    unsigned cBallooned = 0;
+    unsigned cShared = 0;
+    unsigned cZero = 0;
+    unsigned cUnique = 0;
+    unsigned cDuplicate = 0;
+    unsigned cAllocZero = 0;
+    unsigned cPages = 0;
+
+    pgmLock(pVM);
+
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.pRamRangesR3; pRam; pRam = pRam->pNextR3)
+    {
+        PPGMPAGE    pPage  = &pRam->aPages[0];
+        RTGCPHYS    GCPhys = pRam->GCPhys;
+        uint32_t    cLeft  = pRam->cb >> PAGE_SHIFT;
+        while (cLeft-- > 0)
+        {
+            if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
+            {
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ZERO:
+                        cZero++;
+                        break;
+
+                    case PGM_PAGE_STATE_BALLOONED:
+                        cBallooned++;
+                        break;
+
+                    case PGM_PAGE_STATE_SHARED:
+                        cShared++;
+                        break;
+
+                    case PGM_PAGE_STATE_ALLOCATED:
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                    {
+                        const void *pvPage;
+                        /* Check if the page was allocated, but completely zero. */
+                        int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvPage);
+                        if (    rc == VINF_SUCCESS
+                            &&  ASMMemIsZeroPage(pvPage))
+                        {
+                            cAllocZero++;
+                        }
+                        else
+                        if (GMMR3IsDuplicatePage(pVM, PGM_PAGE_GET_PAGEID(pPage)))
+                            cDuplicate++;
+                        else
+                            cUnique++;
+
+                        break;
+                    }
+
+                    default:
+                        AssertFailed();
+                        break;
+                }
+            }
+
+            /* next */
+            pPage++;
+            GCPhys += PAGE_SIZE;
+            cPages++;
+            /* Give some feedback for every processed megabyte. */
+            if ((cPages & 0x7f) == 0)
+                pCmdHlp->pfnPrintf(pCmdHlp, NULL, ".");
+        }
+    }
+    pgmUnlock(pVM);
+
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "\nNumber of zero pages      %08x (%d MB)\n", cZero, cZero / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of alloczero pages %08x (%d MB)\n", cAllocZero, cAllocZero / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of ballooned pages %08x (%d MB)\n", cBallooned, cBallooned / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of shared pages    %08x (%d MB)\n", cShared, cShared / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of unique pages    %08x (%d MB)\n", cUnique, cUnique / 256);
+    pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Number of duplicate pages %08x (%d MB)\n", cDuplicate, cDuplicate / 256);
+    return VINF_SUCCESS;
+}
+
+/**
+ * The '.pgmsharedmodules' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+DECLCALLBACK(int)  pgmR3CmdShowSharedModules(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    unsigned i = 0;
+
+    pgmLock(pVM);
+    do
+    {
+        if (g_apSharedModules[i])
+        {
+            pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Shared module %s (%s):\n", g_apSharedModules[i]->szName, g_apSharedModules[i]->szVersion);
+            for (unsigned j = 0; j < g_apSharedModules[i]->cRegions; j++)
+                pCmdHlp->pfnPrintf(pCmdHlp, NULL, "--- Region %d: base %RGv size %x\n", j, g_apSharedModules[i]->aRegions[j].GCRegionAddr, g_apSharedModules[i]->aRegions[j].cbRegion);
+        }
+        i++;
+    }
+    while (i < RT_ELEMENTS(g_apSharedModules));
+    pgmUnlock(pVM);
+
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_STRICT && HC_ARCH_BITS == 64*/
Index: /trunk/src/VBox/VMM/VMMR3/PGMShw.h
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/PGMShw.h	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/PGMShw.h	(revision 35333)
@@ -0,0 +1,269 @@
+/* $Id$ */
+/** @file
+ * VBox - Page Manager / Monitor, Shadow Paging Template.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#undef SHWPT
+#undef PSHWPT
+#undef SHWPTE
+#undef PSHWPTE
+#undef SHWPD
+#undef PSHWPD
+#undef SHWPDE
+#undef PSHWPDE
+#undef SHW_PDE_PG_MASK
+#undef SHW_PD_SHIFT
+#undef SHW_PD_MASK
+#undef SHW_PTE_PG_MASK
+#undef SHW_PT_SHIFT
+#undef SHW_PT_MASK
+#undef SHW_TOTAL_PD_ENTRIES
+#undef SHW_PDPT_SHIFT
+#undef SHW_PDPT_MASK
+#undef SHW_PDPE_PG_MASK
+#undef SHW_POOL_ROOT_IDX
+
+#if PGM_SHW_TYPE == PGM_TYPE_32BIT
+# define SHWPT                  X86PT
+# define PSHWPT                 PX86PT
+# define SHWPTE                 X86PTE
+# define PSHWPTE                PX86PTE
+# define SHWPD                  X86PD
+# define PSHWPD                 PX86PD
+# define SHWPDE                 X86PDE
+# define PSHWPDE                PX86PDE
+# define SHW_PDE_PG_MASK        X86_PDE_PG_MASK
+# define SHW_PD_SHIFT           X86_PD_SHIFT
+# define SHW_PD_MASK            X86_PD_MASK
+# define SHW_TOTAL_PD_ENTRIES   X86_PG_ENTRIES
+# define SHW_PTE_PG_MASK        X86_PTE_PG_MASK
+# define SHW_PT_SHIFT           X86_PT_SHIFT
+# define SHW_PT_MASK            X86_PT_MASK
+# define SHW_POOL_ROOT_IDX      PGMPOOL_IDX_PD
+
+#elif PGM_SHW_TYPE == PGM_TYPE_EPT
+# define SHWPT                  EPTPT
+# define PSHWPT                 PEPTPT
+# define SHWPTE                 EPTPTE
+# define PSHWPTE                PEPTPTE
+# define SHWPD                  EPTPD
+# define PSHWPD                 PEPTPD
+# define SHWPDE                 EPTPDE
+# define PSHWPDE                PEPTPDE
+# define SHW_PDE_PG_MASK        EPT_PDE_PG_MASK
+# define SHW_PD_SHIFT           EPT_PD_SHIFT
+# define SHW_PD_MASK            EPT_PD_MASK
+# define SHW_PTE_PG_MASK        EPT_PTE_PG_MASK
+# define SHW_PT_SHIFT           EPT_PT_SHIFT
+# define SHW_PT_MASK            EPT_PT_MASK
+# define SHW_PDPT_SHIFT         EPT_PDPT_SHIFT
+# define SHW_PDPT_MASK          EPT_PDPT_MASK
+# define SHW_PDPE_PG_MASK       EPT_PDPE_PG_MASK
+# define SHW_TOTAL_PD_ENTRIES   (EPT_PG_AMD64_ENTRIES*EPT_PG_AMD64_PDPE_ENTRIES)
+# define SHW_POOL_ROOT_IDX      PGMPOOL_IDX_NESTED_ROOT      /* do not use! exception is real mode & protected mode without paging. */
+
+#else
+# define SHWPT                  PGMSHWPTPAE
+# define PSHWPT                 PPGMSHWPTPAE
+# define SHWPTE                 PGMSHWPTEPAE
+# define PSHWPTE                PPGMSHWPTEPAE
+# define SHWPD                  X86PDPAE
+# define PSHWPD                 PX86PDPAE
+# define SHWPDE                 X86PDEPAE
+# define PSHWPDE                PX86PDEPAE
+# define SHW_PDE_PG_MASK        X86_PDE_PAE_PG_MASK
+# define SHW_PD_SHIFT           X86_PD_PAE_SHIFT
+# define SHW_PD_MASK            X86_PD_PAE_MASK
+# define SHW_PTE_PG_MASK        X86_PTE_PAE_PG_MASK
+# define SHW_PT_SHIFT           X86_PT_PAE_SHIFT
+# define SHW_PT_MASK            X86_PT_PAE_MASK
+
+# if PGM_SHW_TYPE == PGM_TYPE_AMD64
+#  define SHW_PDPT_SHIFT        X86_PDPT_SHIFT
+#  define SHW_PDPT_MASK         X86_PDPT_MASK_AMD64
+#  define SHW_PDPE_PG_MASK      X86_PDPE_PG_MASK
+#  define SHW_TOTAL_PD_ENTRIES  (X86_PG_AMD64_ENTRIES*X86_PG_AMD64_PDPE_ENTRIES)
+#  define SHW_POOL_ROOT_IDX     PGMPOOL_IDX_AMD64_CR3
+
+# else /* 32 bits PAE mode */
+#  define SHW_PDPT_SHIFT        X86_PDPT_SHIFT
+#  define SHW_PDPT_MASK         X86_PDPT_MASK_PAE
+#  define SHW_PDPE_PG_MASK      X86_PDPE_PG_MASK
+#  define SHW_TOTAL_PD_ENTRIES  (X86_PG_PAE_ENTRIES*X86_PG_PAE_PDPE_ENTRIES)
+#  define SHW_POOL_ROOT_IDX     PGMPOOL_IDX_PDPT
+# endif
+#endif
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+RT_C_DECLS_BEGIN
+/* r3 */
+PGM_SHW_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0);
+PGM_SHW_DECL(int, Enter)(PVMCPU pVCpu, bool fIs64BitsPagingMode);
+PGM_SHW_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta);
+PGM_SHW_DECL(int, Exit)(PVMCPU pVCpu);
+
+/* all */
+PGM_SHW_DECL(int, GetPage)(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys);
+PGM_SHW_DECL(int, ModifyPage)(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags);
+RT_C_DECLS_END
+
+
+/**
+ * Initializes the guest bit of the paging mode data.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   fResolveGCAndR0 Indicate whether or not GC and Ring-0 symbols can be resolved now.
+ *                          This is used early in the init process to avoid trouble with PDM
+ *                          not being initialized yet.
+ */
+PGM_SHW_DECL(int, InitData)(PVM pVM, PPGMMODEDATA pModeData, bool fResolveGCAndR0)
+{
+    Assert(pModeData->uShwType == PGM_SHW_TYPE || pModeData->uShwType == PGM_TYPE_NESTED);
+
+    /* Ring-3 */
+    pModeData->pfnR3ShwRelocate          = PGM_SHW_NAME(Relocate);
+    pModeData->pfnR3ShwExit              = PGM_SHW_NAME(Exit);
+    pModeData->pfnR3ShwGetPage           = PGM_SHW_NAME(GetPage);
+    pModeData->pfnR3ShwModifyPage        = PGM_SHW_NAME(ModifyPage);
+
+    if (fResolveGCAndR0)
+    {
+        int rc;
+
+#if PGM_SHW_TYPE != PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_NESTED && PGM_SHW_TYPE != PGM_TYPE_EPT /* No AMD64 for traditional virtualization, only VT-x and AMD-V. */
+        /* GC */
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_SHW_NAME_RC_STR(GetPage),    &pModeData->pfnRCShwGetPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_RC_STR(GetPage),  rc), rc);
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL,       PGM_SHW_NAME_RC_STR(ModifyPage), &pModeData->pfnRCShwModifyPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_RC_STR(ModifyPage), rc), rc);
+#endif /* Not AMD64 shadow paging. */
+
+        /* Ring-0 */
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_SHW_NAME_R0_STR(GetPage),    &pModeData->pfnR0ShwGetPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_R0_STR(GetPage),  rc), rc);
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL,       PGM_SHW_NAME_R0_STR(ModifyPage), &pModeData->pfnR0ShwModifyPage);
+        AssertMsgRCReturn(rc, ("%s -> rc=%Rrc\n", PGM_SHW_NAME_R0_STR(ModifyPage), rc), rc);
+    }
+    return VINF_SUCCESS;
+}
+
+/**
+ * Enters the shadow mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The VMCPU to operate on.
+ * @param   fIs64BitsPagingMode     New shadow paging mode is for 64 bits? (only relevant for 64 bits guests on a 32 bits AMD-V nested paging host)
+ */
+PGM_SHW_DECL(int, Enter)(PVMCPU pVCpu, bool fIs64BitsPagingMode)
+{
+#if PGM_SHW_TYPE == PGM_TYPE_NESTED || PGM_SHW_TYPE == PGM_TYPE_EPT
+
+# if PGM_SHW_TYPE == PGM_TYPE_NESTED && HC_ARCH_BITS == 32
+    /* Must distinguish between 32 and 64 bits guest paging modes as we'll use a different shadow paging root/mode in both cases. */
+    RTGCPHYS     GCPhysCR3 = (fIs64BitsPagingMode) ? RT_BIT_64(63) : RT_BIT_64(62);
+# else
+    RTGCPHYS     GCPhysCR3 = RT_BIT_64(63);
+# endif
+    PPGMPOOLPAGE pNewShwPageCR3;
+    PVM          pVM       = pVCpu->pVMR3;
+    PPGMPOOL     pPool     = pVM->pgm.s.CTX_SUFF(pPool);
+
+    Assert(HWACCMIsNestedPagingActive(pVM) == pVM->pgm.s.fNestedPaging);
+    Assert(pVM->pgm.s.fNestedPaging);
+    Assert(!pVCpu->pgm.s.pShwPageCR3R3);
+
+    pgmLock(pVM);
+
+    int rc = pgmPoolAllocEx(pVM, GCPhysCR3, PGMPOOLKIND_ROOT_NESTED, PGMPOOLACCESS_DONTCARE, PGMPOOL_IDX_NESTED_ROOT,
+                            GCPhysCR3 >> PAGE_SHIFT, true /*fLockPage*/, &pNewShwPageCR3);
+    AssertFatalRC(rc);
+
+    pVCpu->pgm.s.iShwUser      = PGMPOOL_IDX_NESTED_ROOT;
+    pVCpu->pgm.s.iShwUserTable = GCPhysCR3 >> PAGE_SHIFT;
+    pVCpu->pgm.s.pShwPageCR3R3 = pNewShwPageCR3;
+
+    pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.pShwPageCR3R3);
+    pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.pShwPageCR3R3);
+
+    pgmUnlock(pVM);
+
+    Log(("Enter nested shadow paging mode: root %RHv phys %RHp\n", pVCpu->pgm.s.pShwPageCR3R3, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->Core.Key));
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Relocate any GC pointers related to shadow mode paging.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   offDelta    The relocation offset.
+ */
+PGM_SHW_DECL(int, Relocate)(PVMCPU pVCpu, RTGCPTR offDelta)
+{
+    pVCpu->pgm.s.pShwPageCR3RC += offDelta;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exits the shadow mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The VMCPU to operate on.
+ */
+PGM_SHW_DECL(int, Exit)(PVMCPU pVCpu)
+{
+    PVM pVM = pVCpu->pVMR3;
+
+    if (    (   pVCpu->pgm.s.enmShadowMode == PGMMODE_NESTED
+             || pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT)
+        &&  pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+        Assert(pVCpu->pgm.s.iShwUser == PGMPOOL_IDX_NESTED_ROOT);
+
+        pgmLock(pVM);
+
+        /* Do *not* unlock this page as we have two of them floating around in the 32-bit host & 64-bit guest case.
+         * We currently assert when you try to free one of them; don't bother to really allow this.
+         *
+         * Note that this is two nested paging root pages max. This isn't a leak. They are reused.
+         */
+        /* pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); */
+
+        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), pVCpu->pgm.s.iShwUser, pVCpu->pgm.s.iShwUserTable);
+        pVCpu->pgm.s.pShwPageCR3R3 = 0;
+        pVCpu->pgm.s.pShwPageCR3R0 = 0;
+        pVCpu->pgm.s.pShwPageCR3RC = 0;
+        pVCpu->pgm.s.iShwUser      = 0;
+        pVCpu->pgm.s.iShwUserTable = 0;
+
+        pgmUnlock(pVM);
+
+        Log(("Leave nested shadow paging mode\n"));
+    }
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/SELM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/SELM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/SELM.cpp	(revision 35333)
@@ -0,0 +1,2681 @@
+/* $Id$ */
+/** @file
+ * SELM - The Selector Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_selm   SELM - The Selector Manager
+ *
+ * SELM takes care of GDT, LDT and TSS shadowing in raw-mode, and the injection
+ * of a few hyper selector for the raw-mode context.  In the hardware assisted
+ * virtualization mode its only task is to decode entries in the guest GDT or
+ * LDT once in a while.
+ *
+ * @see grp_selm
+ *
+ *
+ * @section seg_selm_shadowing   Shadowing
+ *
+ * SELMR3UpdateFromCPUM() and SELMR3SyncTSS() does the bulk synchronization
+ * work.  The three structures (GDT, LDT, TSS) are all shadowed wholesale atm.
+ * The idea is to do it in a more on-demand fashion when we get time.  There
+ * also a whole bunch of issues with the current synchronization of all three
+ * tables, see notes and todos in the code.
+ *
+ * When the guest makes changes to the GDT we will try update the shadow copy
+ * without involving SELMR3UpdateFromCPUM(), see selmGCSyncGDTEntry().
+ *
+ * When the guest make LDT changes we'll trigger a full resync of the LDT
+ * (SELMR3UpdateFromCPUM()), which, needless to say, isn't optimal.
+ *
+ * The TSS shadowing is limited to the fields we need to care about, namely SS0
+ * and ESP0.  The Patch Manager makes use of these.  We monitor updates to the
+ * guest TSS and will try keep our SS0 and ESP0 copies up to date this way
+ * rather than go the SELMR3SyncTSS() route.
+ *
+ * When in raw-mode SELM also injects a few extra GDT selectors which are used
+ * by the raw-mode (hyper) context.  These start their life at the high end of
+ * the table and will be relocated when the guest tries to make use of them...
+ * Well, that was that idea at least, only the code isn't quite there yet which
+ * is why we have trouble with guests which actually have a full sized GDT.
+ *
+ * So, the summary of the current GDT, LDT and TSS shadowing is that there is a
+ * lot of relatively simple and enjoyable work to be done, see @bugref{3267}.
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_SELM
+#include <VBox/selm.h>
+#include <VBox/cpum.h>
+#include <VBox/stam.h>
+#include <VBox/mm.h>
+#include <VBox/ssm.h>
+#include <VBox/pgm.h>
+#include <VBox/trpm.h>
+#include <VBox/dbgf.h>
+#include "SELMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/string.h>
+
+
+/**
+ * Enable or disable tracking of Guest's GDT/LDT/TSS.
+ * @{
+ */
+#define SELM_TRACK_GUEST_GDT_CHANGES
+#define SELM_TRACK_GUEST_LDT_CHANGES
+#define SELM_TRACK_GUEST_TSS_CHANGES
+/** @} */
+
+/**
+ * Enable or disable tracking of Shadow GDT/LDT/TSS.
+ * @{
+ */
+#define SELM_TRACK_SHADOW_GDT_CHANGES
+#define SELM_TRACK_SHADOW_LDT_CHANGES
+#define SELM_TRACK_SHADOW_TSS_CHANGES
+/** @} */
+
+
+/** SELM saved state version. */
+#define SELM_SAVED_STATE_VERSION    5
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int)  selmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)  selmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int)  selmR3LoadDone(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)  selmR3GuestGDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
+static DECLCALLBACK(int)  selmR3GuestLDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
+static DECLCALLBACK(int)  selmR3GuestTSSWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
+static DECLCALLBACK(void) selmR3InfoGdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) selmR3InfoGdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) selmR3InfoLdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void) selmR3InfoLdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+//static DECLCALLBACK(void) selmR3InfoTss(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+//static DECLCALLBACK(void) selmR3InfoTssGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+
+/**
+ * Initializes the SELM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) SELMR3Init(PVM pVM)
+{
+    LogFlow(("SELMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     * (The TSS block requires contiguous back.)
+     */
+    AssertCompile(sizeof(pVM->selm.s) <= sizeof(pVM->selm.padding));    AssertRelease(sizeof(pVM->selm.s) <= sizeof(pVM->selm.padding));
+    AssertCompileMemberAlignment(VM, selm.s, 32);                       AssertRelease(!(RT_OFFSETOF(VM, selm.s) & 31));
+#if 0 /* doesn't work */
+    AssertCompile((RT_OFFSETOF(VM, selm.s.Tss)       & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.Tss));
+    AssertCompile((RT_OFFSETOF(VM, selm.s.TssTrap08) & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.TssTrap08));
+#endif
+    AssertRelease((RT_OFFSETOF(VM, selm.s.Tss)       & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.Tss));
+    AssertRelease((RT_OFFSETOF(VM, selm.s.TssTrap08) & PAGE_OFFSET_MASK) <= PAGE_SIZE - sizeof(pVM->selm.s.TssTrap08));
+    AssertRelease(sizeof(pVM->selm.s.Tss.IntRedirBitmap) == 0x20);
+
+    /*
+     * Init the structure.
+     */
+    pVM->selm.s.offVM                                = RT_OFFSETOF(VM, selm);
+    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]         = (SELM_GDT_ELEMENTS - 0x1) << 3;
+    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]         = (SELM_GDT_ELEMENTS - 0x2) << 3;
+    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64]       = (SELM_GDT_ELEMENTS - 0x3) << 3;
+    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]        = (SELM_GDT_ELEMENTS - 0x4) << 3;
+    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = (SELM_GDT_ELEMENTS - 0x5) << 3;
+
+    /*
+     * Allocate GDT table.
+     */
+    int rc = MMR3HyperAllocOnceNoRel(pVM, sizeof(pVM->selm.s.paGdtR3[0]) * SELM_GDT_ELEMENTS,
+                                     PAGE_SIZE, MM_TAG_SELM, (void **)&pVM->selm.s.paGdtR3);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Allocate LDT area.
+     */
+    rc = MMR3HyperAllocOnceNoRel(pVM, _64K + PAGE_SIZE, PAGE_SIZE, MM_TAG_SELM, &pVM->selm.s.pvLdtR3);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Init Guest's and Shadow GDT, LDT, TSS changes control variables.
+     */
+    pVM->selm.s.cbEffGuestGdtLimit = 0;
+    pVM->selm.s.GuestGdtr.pGdt     = RTRCPTR_MAX;
+    pVM->selm.s.GCPtrGuestLdt      = RTRCPTR_MAX;
+    pVM->selm.s.GCPtrGuestTss      = RTRCPTR_MAX;
+
+    pVM->selm.s.paGdtRC            = NIL_RTRCPTR; /* Must be set in SELMR3Relocate because of monitoring. */
+    pVM->selm.s.pvLdtRC            = RTRCPTR_MAX;
+    pVM->selm.s.pvMonShwTssRC      = RTRCPTR_MAX;
+    pVM->selm.s.GCSelTss           = RTSEL_MAX;
+
+    pVM->selm.s.fDisableMonitoring = false;
+    pVM->selm.s.fSyncTSSRing0Stack = false;
+
+    /* The I/O bitmap starts right after the virtual interrupt redirection bitmap. Outside the TSS on purpose; the CPU will not check it
+     * for I/O operations. */
+    pVM->selm.s.Tss.offIoBitmap = sizeof(VBOXTSS);
+    /* bit set to 1 means no redirection */
+    memset(pVM->selm.s.Tss.IntRedirBitmap, 0xff, sizeof(pVM->selm.s.Tss.IntRedirBitmap));
+
+    /*
+     * Register the saved state data unit.
+     */
+    rc = SSMR3RegisterInternal(pVM, "selm", 1, SELM_SAVED_STATE_VERSION, sizeof(SELM),
+                               NULL, NULL, NULL,
+                               NULL, selmR3Save, NULL,
+                               NULL, selmR3Load, selmR3LoadDone);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Statistics.
+     */
+    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestGDTHandled,     STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/GDTInt",  STAMUNIT_OCCURENCES,     "The number of handled writes to the Guest GDT.");
+    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestGDTUnhandled,   STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/GDTEmu",  STAMUNIT_OCCURENCES,     "The number of unhandled writes to the Guest GDT.");
+    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestLDT,            STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/LDT",     STAMUNIT_OCCURENCES,     "The number of writes to the Guest LDT was detected.");
+    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSHandled,     STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSInt",  STAMUNIT_OCCURENCES,     "The number of handled writes to the Guest TSS.");
+    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSRedir,       STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSRedir",STAMUNIT_OCCURENCES,     "The number of handled redir bitmap writes to the Guest TSS.");
+    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSHandledChanged,STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSIntChg", STAMUNIT_OCCURENCES, "The number of handled writes to the Guest TSS where the R0 stack changed.");
+    STAM_REG(pVM, &pVM->selm.s.StatRCWriteGuestTSSUnhandled,   STAMTYPE_COUNTER, "/SELM/GC/Write/Guest/TSSEmu",  STAMUNIT_OCCURENCES,     "The number of unhandled writes to the Guest TSS.");
+    STAM_REG(pVM, &pVM->selm.s.StatTSSSync,                    STAMTYPE_PROFILE, "/PROF/SELM/TSSSync",           STAMUNIT_TICKS_PER_CALL, "Profiling of the SELMR3SyncTSS() body.");
+    STAM_REG(pVM, &pVM->selm.s.StatUpdateFromCPUM,             STAMTYPE_PROFILE, "/PROF/SELM/UpdateFromCPUM",    STAMUNIT_TICKS_PER_CALL, "Profiling of the SELMR3UpdateFromCPUM() body.");
+
+    STAM_REG(pVM, &pVM->selm.s.StatHyperSelsChanged,           STAMTYPE_COUNTER, "/SELM/HyperSels/Changed",      STAMUNIT_OCCURENCES,     "The number of times we had to relocate our hypervisor selectors.");
+    STAM_REG(pVM, &pVM->selm.s.StatScanForHyperSels,           STAMTYPE_COUNTER, "/SELM/HyperSels/Scan",         STAMUNIT_OCCURENCES,     "The number of times we had find free hypervisor selectors.");
+
+    /*
+     * Default action when entering raw mode for the first time
+     */
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+
+    /*
+     * Register info handlers.
+     */
+    DBGFR3InfoRegisterInternal(pVM, "gdt",      "Displays the shadow GDT. No arguments.",   &selmR3InfoGdt);
+    DBGFR3InfoRegisterInternal(pVM, "gdtguest", "Displays the guest GDT. No arguments.",    &selmR3InfoGdtGuest);
+    DBGFR3InfoRegisterInternal(pVM, "ldt",      "Displays the shadow LDT. No arguments.",   &selmR3InfoLdt);
+    DBGFR3InfoRegisterInternal(pVM, "ldtguest", "Displays the guest LDT. No arguments.",    &selmR3InfoLdtGuest);
+    //DBGFR3InfoRegisterInternal(pVM, "tss",      "Displays the shadow TSS. No arguments.",   &selmR3InfoTss);
+    //DBGFR3InfoRegisterInternal(pVM, "tssguest", "Displays the guest TSS. No arguments.",    &selmR3InfoTssGuest);
+
+    return rc;
+}
+
+
+/**
+ * Finalizes HMA page attributes.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+VMMR3DECL(int) SELMR3InitFinalize(PVM pVM)
+{
+    /** @cfgm{/DoubleFault,bool,false}
+     * Enables catching of double faults in the raw-mode context VMM code.  This can
+     * be used when the triple faults or hangs occur and one suspect an unhandled
+     * double fault.  This is not enabled by default because it means making the
+     * hyper selectors writeable for all supervisor code, including the guest's.
+     * The double fault is a task switch and thus requires write access to the GDT
+     * of the TSS (to set it busy), to the old TSS (to store state), and to the Trap
+     * 8 TSS for the back link.
+     */
+    bool f;
+#if defined(DEBUG_bird)
+    int rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "DoubleFault", &f, true);
+#else
+    int rc = CFGMR3QueryBoolDef(CFGMR3GetRoot(pVM), "DoubleFault", &f, false);
+#endif
+    AssertLogRelRCReturn(rc, rc);
+    if (f)
+    {
+        PX86DESC paGdt = pVM->selm.s.paGdtR3;
+        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> 3]), sizeof(paGdt[0]),
+                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
+        AssertRC(rc);
+        rc = PGMMapSetPage(pVM, MMHyperR3ToRC(pVM, &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] >> 3]), sizeof(paGdt[0]),
+                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
+        AssertRC(rc);
+        rc = PGMMapSetPage(pVM, VM_RC_ADDR(pVM, &pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]), sizeof(pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]),
+                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
+        AssertRC(rc);
+        rc = PGMMapSetPage(pVM, VM_RC_ADDR(pVM, &pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08]), sizeof(pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08]),
+                           X86_PTE_RW | X86_PTE_P | X86_PTE_A | X86_PTE_D);
+        AssertRC(rc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Setup the hypervisor GDT selectors in our shadow table
+ *
+ * @param   pVM     The VM handle.
+ */
+static void selmR3SetupHyperGDTSelectors(PVM pVM)
+{
+    PX86DESC paGdt = pVM->selm.s.paGdtR3;
+
+    /*
+     * Set up global code and data descriptors for use in the guest context.
+     * Both are wide open (base 0, limit 4GB)
+     */
+    PX86DESC   pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] >> 3];
+    pDesc->Gen.u16LimitLow      = 0xffff;
+    pDesc->Gen.u4LimitHigh      = 0xf;
+    pDesc->Gen.u16BaseLow       = 0;
+    pDesc->Gen.u8BaseHigh1      = 0;
+    pDesc->Gen.u8BaseHigh2      = 0;
+    pDesc->Gen.u4Type           = X86_SEL_TYPE_ER_ACC;
+    pDesc->Gen.u1DescType       = 1; /* not system, but code/data */
+    pDesc->Gen.u2Dpl            = 0; /* supervisor */
+    pDesc->Gen.u1Present        = 1;
+    pDesc->Gen.u1Available      = 0;
+    pDesc->Gen.u1Long           = 0;
+    pDesc->Gen.u1DefBig         = 1; /* def 32 bit */
+    pDesc->Gen.u1Granularity    = 1; /* 4KB limit */
+
+    /* data */
+    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] >> 3];
+    pDesc->Gen.u16LimitLow      = 0xffff;
+    pDesc->Gen.u4LimitHigh      = 0xf;
+    pDesc->Gen.u16BaseLow       = 0;
+    pDesc->Gen.u8BaseHigh1      = 0;
+    pDesc->Gen.u8BaseHigh2      = 0;
+    pDesc->Gen.u4Type           = X86_SEL_TYPE_RW_ACC;
+    pDesc->Gen.u1DescType       = 1; /* not system, but code/data */
+    pDesc->Gen.u2Dpl            = 0; /* supervisor */
+    pDesc->Gen.u1Present        = 1;
+    pDesc->Gen.u1Available      = 0;
+    pDesc->Gen.u1Long           = 0;
+    pDesc->Gen.u1DefBig         = 1; /* big */
+    pDesc->Gen.u1Granularity    = 1; /* 4KB limit */
+
+    /* 64-bit mode code (& data?) */
+    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] >> 3];
+    pDesc->Gen.u16LimitLow      = 0xffff;
+    pDesc->Gen.u4LimitHigh      = 0xf;
+    pDesc->Gen.u16BaseLow       = 0;
+    pDesc->Gen.u8BaseHigh1      = 0;
+    pDesc->Gen.u8BaseHigh2      = 0;
+    pDesc->Gen.u4Type           = X86_SEL_TYPE_ER_ACC;
+    pDesc->Gen.u1DescType       = 1; /* not system, but code/data */
+    pDesc->Gen.u2Dpl            = 0; /* supervisor */
+    pDesc->Gen.u1Present        = 1;
+    pDesc->Gen.u1Available      = 0;
+    pDesc->Gen.u1Long           = 1; /* The Long (L) attribute bit. */
+    pDesc->Gen.u1DefBig         = 0; /* With L=1 this must be 0. */
+    pDesc->Gen.u1Granularity    = 1; /* 4KB limit */
+
+    /*
+     * TSS descriptor
+     */
+    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] >> 3];
+    RTRCPTR RCPtrTSS = VM_RC_ADDR(pVM, &pVM->selm.s.Tss);
+    pDesc->Gen.u16BaseLow       = RT_LOWORD(RCPtrTSS);
+    pDesc->Gen.u8BaseHigh1      = RT_BYTE3(RCPtrTSS);
+    pDesc->Gen.u8BaseHigh2      = RT_BYTE4(RCPtrTSS);
+    pDesc->Gen.u16LimitLow      = sizeof(VBOXTSS) - 1;
+    pDesc->Gen.u4LimitHigh      = 0;
+    pDesc->Gen.u4Type           = X86_SEL_TYPE_SYS_386_TSS_AVAIL;
+    pDesc->Gen.u1DescType       = 0; /* system */
+    pDesc->Gen.u2Dpl            = 0; /* supervisor */
+    pDesc->Gen.u1Present        = 1;
+    pDesc->Gen.u1Available      = 0;
+    pDesc->Gen.u1Long           = 0;
+    pDesc->Gen.u1DefBig         = 0;
+    pDesc->Gen.u1Granularity    = 0; /* byte limit */
+
+    /*
+     * TSS descriptor for trap 08
+     */
+    pDesc = &paGdt[pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> 3];
+    pDesc->Gen.u16LimitLow      = sizeof(VBOXTSS) - 1;
+    pDesc->Gen.u4LimitHigh      = 0;
+    RCPtrTSS = VM_RC_ADDR(pVM, &pVM->selm.s.TssTrap08);
+    pDesc->Gen.u16BaseLow       = RT_LOWORD(RCPtrTSS);
+    pDesc->Gen.u8BaseHigh1      = RT_BYTE3(RCPtrTSS);
+    pDesc->Gen.u8BaseHigh2      = RT_BYTE4(RCPtrTSS);
+    pDesc->Gen.u4Type           = X86_SEL_TYPE_SYS_386_TSS_AVAIL;
+    pDesc->Gen.u1DescType       = 0; /* system */
+    pDesc->Gen.u2Dpl            = 0; /* supervisor */
+    pDesc->Gen.u1Present        = 1;
+    pDesc->Gen.u1Available      = 0;
+    pDesc->Gen.u1Long           = 0;
+    pDesc->Gen.u1DefBig         = 0;
+    pDesc->Gen.u1Granularity    = 0; /* byte limit */
+}
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The VM.
+ */
+VMMR3DECL(void) SELMR3Relocate(PVM pVM)
+{
+    PX86DESC paGdt = pVM->selm.s.paGdtR3;
+    LogFlow(("SELMR3Relocate\n"));
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        /*
+         * Update GDTR and selector.
+         */
+        CPUMSetHyperGDTR(pVCpu, MMHyperR3ToRC(pVM, paGdt), SELM_GDT_ELEMENTS * sizeof(paGdt[0]) - 1);
+
+        /** @todo selector relocations should be a separate operation? */
+        CPUMSetHyperCS(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]);
+        CPUMSetHyperDS(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]);
+        CPUMSetHyperES(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]);
+        CPUMSetHyperSS(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]);
+        CPUMSetHyperTR(pVCpu, pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]);
+    }
+
+    selmR3SetupHyperGDTSelectors(pVM);
+
+/** @todo SELM must be called when any of the CR3s changes during a cpu mode change. */
+/** @todo PGM knows the proper CR3 values these days, not CPUM. */
+    /*
+     * Update the TSSes.
+     */
+    /* Only applies to raw mode which supports only 1 VCPU */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    /* Current TSS */
+    pVM->selm.s.Tss.cr3     = PGMGetHyperCR3(pVCpu);
+    pVM->selm.s.Tss.ss0     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
+    pVM->selm.s.Tss.esp0    = VMMGetStackRC(pVCpu);
+    pVM->selm.s.Tss.cs      = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS];
+    pVM->selm.s.Tss.ds      = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
+    pVM->selm.s.Tss.es      = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
+    pVM->selm.s.Tss.offIoBitmap = sizeof(VBOXTSS);
+
+    /* trap 08 */
+    pVM->selm.s.TssTrap08.cr3    = PGMGetInterRCCR3(pVM, pVCpu);                   /* this should give use better survival chances. */
+    pVM->selm.s.TssTrap08.ss0    = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
+    pVM->selm.s.TssTrap08.ss     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
+    pVM->selm.s.TssTrap08.esp0   = VMMGetStackRC(pVCpu) - PAGE_SIZE / 2;  /* upper half can be analysed this way. */
+    pVM->selm.s.TssTrap08.esp    = pVM->selm.s.TssTrap08.esp0;
+    pVM->selm.s.TssTrap08.ebp    = pVM->selm.s.TssTrap08.esp0;
+    pVM->selm.s.TssTrap08.cs     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS];
+    pVM->selm.s.TssTrap08.ds     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
+    pVM->selm.s.TssTrap08.es     = pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS];
+    pVM->selm.s.TssTrap08.fs     = 0;
+    pVM->selm.s.TssTrap08.gs     = 0;
+    pVM->selm.s.TssTrap08.selLdt = 0;
+    pVM->selm.s.TssTrap08.eflags = 0x2;    /* all cleared */
+    pVM->selm.s.TssTrap08.ecx    = VM_RC_ADDR(pVM, &pVM->selm.s.Tss);       /* setup ecx to normal Hypervisor TSS address. */
+    pVM->selm.s.TssTrap08.edi    = pVM->selm.s.TssTrap08.ecx;
+    pVM->selm.s.TssTrap08.eax    = pVM->selm.s.TssTrap08.ecx;
+    pVM->selm.s.TssTrap08.edx    = VM_RC_ADDR(pVM, pVM);                    /* setup edx VM address. */
+    pVM->selm.s.TssTrap08.edi    = pVM->selm.s.TssTrap08.edx;
+    pVM->selm.s.TssTrap08.ebx    = pVM->selm.s.TssTrap08.edx;
+    pVM->selm.s.TssTrap08.offIoBitmap = sizeof(VBOXTSS);
+    /* TRPM will be updating the eip */
+
+    if (    !pVM->selm.s.fDisableMonitoring
+        &&  !VMMIsHwVirtExtForced(pVM))
+    {
+        /*
+         * Update shadow GDT/LDT/TSS write access handlers.
+         */
+        int rc;
+#ifdef SELM_TRACK_SHADOW_GDT_CHANGES
+        if (pVM->selm.s.paGdtRC != NIL_RTRCPTR)
+        {
+            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.paGdtRC);
+            AssertRC(rc);
+        }
+        pVM->selm.s.paGdtRC = MMHyperR3ToRC(pVM, paGdt);
+        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->selm.s.paGdtRC,
+                                         pVM->selm.s.paGdtRC + SELM_GDT_ELEMENTS * sizeof(paGdt[0]) - 1,
+                                         0, 0, "selmRCShadowGDTWriteHandler", 0, "Shadow GDT write access handler");
+        AssertRC(rc);
+#endif
+#ifdef SELM_TRACK_SHADOW_TSS_CHANGES
+        if (pVM->selm.s.pvMonShwTssRC != RTRCPTR_MAX)
+        {
+            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvMonShwTssRC);
+            AssertRC(rc);
+        }
+        pVM->selm.s.pvMonShwTssRC = VM_RC_ADDR(pVM, &pVM->selm.s.Tss);
+        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->selm.s.pvMonShwTssRC,
+                                         pVM->selm.s.pvMonShwTssRC + sizeof(pVM->selm.s.Tss) - 1,
+                                         0, 0, "selmRCShadowTSSWriteHandler", 0, "Shadow TSS write access handler");
+        AssertRC(rc);
+#endif
+
+        /*
+         * Update the GC LDT region handler and address.
+         */
+#ifdef SELM_TRACK_SHADOW_LDT_CHANGES
+        if (pVM->selm.s.pvLdtRC != RTRCPTR_MAX)
+        {
+            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvLdtRC);
+            AssertRC(rc);
+        }
+#endif
+        pVM->selm.s.pvLdtRC = MMHyperR3ToRC(pVM, pVM->selm.s.pvLdtR3);
+#ifdef SELM_TRACK_SHADOW_LDT_CHANGES
+        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->selm.s.pvLdtRC,
+                                         pVM->selm.s.pvLdtRC + _64K + PAGE_SIZE - 1,
+                                         0, 0, "selmRCShadowLDTWriteHandler", 0, "Shadow LDT write access handler");
+        AssertRC(rc);
+#endif
+    }
+}
+
+
+/**
+ * Terminates the SELM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) SELMR3Term(PVM pVM)
+{
+    return 0;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the SELM component this means that any GDT/LDT/TSS monitors
+ * needs to be removed.
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) SELMR3Reset(PVM pVM)
+{
+    LogFlow(("SELMR3Reset:\n"));
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Uninstall guest GDT/LDT/TSS write access handlers.
+     */
+    int rc;
+#ifdef SELM_TRACK_GUEST_GDT_CHANGES
+    if (pVM->selm.s.GuestGdtr.pGdt != RTRCPTR_MAX && pVM->selm.s.fGDTRangeRegistered)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GuestGdtr.pGdt);
+        AssertRC(rc);
+        pVM->selm.s.GuestGdtr.pGdt = RTRCPTR_MAX;
+        pVM->selm.s.GuestGdtr.cbGdt = 0;
+    }
+    pVM->selm.s.fGDTRangeRegistered = false;
+#endif
+#ifdef SELM_TRACK_GUEST_LDT_CHANGES
+    if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
+        AssertRC(rc);
+        pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
+    }
+#endif
+#ifdef SELM_TRACK_GUEST_TSS_CHANGES
+    if (pVM->selm.s.GCPtrGuestTss != RTRCPTR_MAX)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestTss);
+        AssertRC(rc);
+        pVM->selm.s.GCPtrGuestTss = RTRCPTR_MAX;
+        pVM->selm.s.GCSelTss      = RTSEL_MAX;
+    }
+#endif
+
+    /*
+     * Re-initialize other members.
+     */
+    pVM->selm.s.cbLdtLimit = 0;
+    pVM->selm.s.offLdtHyper = 0;
+    pVM->selm.s.cbMonitoredGuestTss = 0;
+
+    pVM->selm.s.fSyncTSSRing0Stack = false;
+
+    /*
+     * Default action when entering raw mode for the first time
+     */
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+}
+
+/**
+ * Disable GDT/LDT/TSS monitoring and syncing
+ *
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(void) SELMR3DisableMonitoring(PVM pVM)
+{
+    /*
+     * Uninstall guest GDT/LDT/TSS write access handlers.
+     */
+    int rc;
+#ifdef SELM_TRACK_GUEST_GDT_CHANGES
+    if (pVM->selm.s.GuestGdtr.pGdt != RTRCPTR_MAX && pVM->selm.s.fGDTRangeRegistered)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GuestGdtr.pGdt);
+        AssertRC(rc);
+        pVM->selm.s.GuestGdtr.pGdt = RTRCPTR_MAX;
+        pVM->selm.s.GuestGdtr.cbGdt = 0;
+    }
+    pVM->selm.s.fGDTRangeRegistered = false;
+#endif
+#ifdef SELM_TRACK_GUEST_LDT_CHANGES
+    if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
+        AssertRC(rc);
+        pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
+    }
+#endif
+#ifdef SELM_TRACK_GUEST_TSS_CHANGES
+    if (pVM->selm.s.GCPtrGuestTss != RTRCPTR_MAX)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestTss);
+        AssertRC(rc);
+        pVM->selm.s.GCPtrGuestTss = RTRCPTR_MAX;
+        pVM->selm.s.GCSelTss      = RTSEL_MAX;
+    }
+#endif
+
+    /*
+     * Unregister shadow GDT/LDT/TSS write access handlers.
+     */
+#ifdef SELM_TRACK_SHADOW_GDT_CHANGES
+    if (pVM->selm.s.paGdtRC != NIL_RTRCPTR)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.paGdtRC);
+        AssertRC(rc);
+        pVM->selm.s.paGdtRC = NIL_RTRCPTR;
+    }
+#endif
+#ifdef SELM_TRACK_SHADOW_TSS_CHANGES
+    if (pVM->selm.s.pvMonShwTssRC != RTRCPTR_MAX)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvMonShwTssRC);
+        AssertRC(rc);
+        pVM->selm.s.pvMonShwTssRC = RTRCPTR_MAX;
+    }
+#endif
+#ifdef SELM_TRACK_SHADOW_LDT_CHANGES
+    if (pVM->selm.s.pvLdtRC != RTRCPTR_MAX)
+    {
+        rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.pvLdtRC);
+        AssertRC(rc);
+        pVM->selm.s.pvLdtRC = RTRCPTR_MAX;
+    }
+#endif
+
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+
+    pVM->selm.s.fDisableMonitoring = true;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) selmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("selmR3Save:\n"));
+
+    /*
+     * Save the basic bits - fortunately all the other things can be resynced on load.
+     */
+    PSELM pSelm = &pVM->selm.s;
+
+    SSMR3PutBool(pSSM, pSelm->fDisableMonitoring);
+    SSMR3PutBool(pSSM, pSelm->fSyncTSSRing0Stack);
+    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_CS]);
+    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_DS]);
+    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_CS64]);
+    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_CS64]); /* reserved for DS64. */
+    SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_TSS]);
+    return SSMR3PutSel(pSSM, pSelm->aHyperSel[SELM_HYPER_SEL_TSS_TRAP08]);
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) selmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("selmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (uVersion != SELM_SAVED_STATE_VERSION)
+    {
+        AssertMsgFailed(("selmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Do a reset.
+     */
+    SELMR3Reset(pVM);
+
+    /* Get the monitoring flag. */
+    SSMR3GetBool(pSSM, &pVM->selm.s.fDisableMonitoring);
+
+    /* Get the TSS state flag. */
+    SSMR3GetBool(pSSM, &pVM->selm.s.fSyncTSSRing0Stack);
+
+    /*
+     * Get the selectors.
+     */
+    RTSEL SelCS;
+    SSMR3GetSel(pSSM, &SelCS);
+    RTSEL SelDS;
+    SSMR3GetSel(pSSM, &SelDS);
+    RTSEL SelCS64;
+    SSMR3GetSel(pSSM, &SelCS64);
+    RTSEL SelDS64;
+    SSMR3GetSel(pSSM, &SelDS64);
+    RTSEL SelTSS;
+    SSMR3GetSel(pSSM, &SelTSS);
+    RTSEL SelTSSTrap08;
+    SSMR3GetSel(pSSM, &SelTSSTrap08);
+
+    /* Copy the selectors; they will be checked during relocation. */
+    PSELM pSelm = &pVM->selm.s;
+    pSelm->aHyperSel[SELM_HYPER_SEL_CS]         = SelCS;
+    pSelm->aHyperSel[SELM_HYPER_SEL_DS]         = SelDS;
+    pSelm->aHyperSel[SELM_HYPER_SEL_CS64]       = SelCS64;
+    pSelm->aHyperSel[SELM_HYPER_SEL_TSS]        = SelTSS;
+    pSelm->aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = SelTSSTrap08;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sync the GDT, LDT and TSS after loading the state.
+ *
+ * Just to play save, we set the FFs to force syncing before
+ * executing GC code.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) selmR3LoadDone(PVM pVM, PSSMHANDLE pSSM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    LogFlow(("selmR3LoadDone:\n"));
+
+    /*
+     * Don't do anything if it's a load failure.
+     */
+    int rc = SSMR3HandleGetStatus(pSSM);
+    if (RT_FAILURE(rc))
+        return VINF_SUCCESS;
+
+    /*
+     * Do the syncing if we're in protected mode.
+     */
+    if (PGMGetGuestMode(pVCpu) != PGMMODE_REAL)
+    {
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+        SELMR3UpdateFromCPUM(pVM, pVCpu);
+    }
+
+    /*
+     * Flag everything for resync on next raw mode entry.
+     */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Updates the Guest GDT & LDT virtualization based on current CPU state.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VMCPU to operate on.
+ */
+VMMR3DECL(int) SELMR3UpdateFromCPUM(PVM pVM, PVMCPU pVCpu)
+{
+    int    rc    = VINF_SUCCESS;
+
+    if (pVM->selm.s.fDisableMonitoring)
+    {
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+
+        return VINF_SUCCESS;
+    }
+
+    STAM_PROFILE_START(&pVM->selm.s.StatUpdateFromCPUM, a);
+
+    /*
+     * GDT sync
+     */
+    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_GDT))
+    {
+        /*
+         * Always assume the best
+         */
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_GDT);
+
+        /* If the GDT was changed, then make sure the LDT is checked too */
+        /** @todo only do this if the actual ldtr selector was changed; this is a bit excessive */
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+        /* Same goes for the TSS selector */
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+
+        /*
+         * Get the GDTR and check if there is anything to do (there usually is).
+         */
+        VBOXGDTR    GDTR;
+        CPUMGetGuestGDTR(pVCpu, &GDTR);
+        if (GDTR.cbGdt < sizeof(X86DESC))
+        {
+            Log(("No GDT entries...\n"));
+            STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Read the Guest GDT.
+         * ASSUMES that the entire GDT is in memory.
+         */
+        RTUINT      cbEffLimit = GDTR.cbGdt;
+        PX86DESC    pGDTE = &pVM->selm.s.paGdtR3[1];
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, pGDTE, GDTR.pGdt + sizeof(X86DESC), cbEffLimit + 1 - sizeof(X86DESC));
+        if (RT_FAILURE(rc))
+        {
+            /*
+             * Read it page by page.
+             *
+             * Keep track of the last valid page and delay memsets and
+             * adjust cbEffLimit to reflect the effective size. The latter
+             * is something we do in the belief that the guest will probably
+             * never actually commit the last page, thus allowing us to keep
+             * our selectors in the high end of the GDT.
+             */
+            RTUINT  cbLeft = cbEffLimit + 1 - sizeof(X86DESC);
+            RTGCPTR GCPtrSrc = (RTGCPTR)GDTR.pGdt + sizeof(X86DESC);
+            uint8_t *pu8Dst = (uint8_t *)&pVM->selm.s.paGdtR3[1];
+            uint8_t *pu8DstInvalid = pu8Dst;
+
+            while (cbLeft)
+            {
+                RTUINT cb = PAGE_SIZE - (GCPtrSrc & PAGE_OFFSET_MASK);
+                cb = RT_MIN(cb, cbLeft);
+                rc = PGMPhysSimpleReadGCPtr(pVCpu, pu8Dst, GCPtrSrc, cb);
+                if (RT_SUCCESS(rc))
+                {
+                    if (pu8DstInvalid != pu8Dst)
+                        memset(pu8DstInvalid, 0, pu8Dst - pu8DstInvalid);
+                    GCPtrSrc += cb;
+                    pu8Dst += cb;
+                    pu8DstInvalid = pu8Dst;
+                }
+                else if (   rc == VERR_PAGE_NOT_PRESENT
+                         || rc == VERR_PAGE_TABLE_NOT_PRESENT)
+                {
+                    GCPtrSrc += cb;
+                    pu8Dst += cb;
+                }
+                else
+                {
+                    AssertReleaseMsgFailed(("Couldn't read GDT at %016RX64, rc=%Rrc!\n", GDTR.pGdt, rc));
+                    STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+                    return VERR_NOT_IMPLEMENTED;
+                }
+                cbLeft -= cb;
+            }
+
+            /* any invalid pages at the end? */
+            if (pu8DstInvalid != pu8Dst)
+            {
+                cbEffLimit = pu8DstInvalid - (uint8_t *)pVM->selm.s.paGdtR3 - 1;
+                /* If any GDTEs was invalidated, zero them. */
+                if (cbEffLimit < pVM->selm.s.cbEffGuestGdtLimit)
+                    memset(pu8DstInvalid + cbEffLimit + 1, 0, pVM->selm.s.cbEffGuestGdtLimit - cbEffLimit);
+            }
+
+            /* keep track of the effective limit. */
+            if (cbEffLimit != pVM->selm.s.cbEffGuestGdtLimit)
+            {
+                Log(("SELMR3UpdateFromCPUM: cbEffGuestGdtLimit=%#x -> %#x (actual %#x)\n",
+                     pVM->selm.s.cbEffGuestGdtLimit, cbEffLimit, GDTR.cbGdt));
+                pVM->selm.s.cbEffGuestGdtLimit = cbEffLimit;
+            }
+        }
+
+        /*
+         * Check if the Guest GDT intrudes on our GDT entries.
+         */
+        /** @todo we should try to minimize relocations by making sure our current selectors can be reused. */
+        RTSEL aHyperSel[SELM_HYPER_SEL_MAX];
+        if (cbEffLimit >= SELM_HYPER_DEFAULT_BASE)
+        {
+            PX86DESC    pGDTEStart = pVM->selm.s.paGdtR3;
+            PX86DESC    pGDTECur   = (PX86DESC)((char *)pGDTEStart + GDTR.cbGdt + 1 - sizeof(X86DESC));
+            int         iGDT       = 0;
+
+            Log(("Internal SELM GDT conflict: use non-present entries\n"));
+            STAM_COUNTER_INC(&pVM->selm.s.StatScanForHyperSels);
+            while (pGDTECur > pGDTEStart)
+            {
+                /* We can reuse non-present entries */
+                if (!pGDTECur->Gen.u1Present)
+                {
+                    aHyperSel[iGDT] = ((uintptr_t)pGDTECur - (uintptr_t)pVM->selm.s.paGdtR3) / sizeof(X86DESC);
+                    aHyperSel[iGDT] = aHyperSel[iGDT] << X86_SEL_SHIFT;
+                    Log(("SELM: Found unused GDT %04X\n", aHyperSel[iGDT]));
+                    iGDT++;
+                    if (iGDT >= SELM_HYPER_SEL_MAX)
+                        break;
+                }
+
+                pGDTECur--;
+            }
+            if (iGDT != SELM_HYPER_SEL_MAX)
+            {
+                AssertReleaseMsgFailed(("Internal SELM GDT conflict.\n"));
+                STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+                return VERR_NOT_IMPLEMENTED;
+            }
+        }
+        else
+        {
+            aHyperSel[SELM_HYPER_SEL_CS]         = SELM_HYPER_DEFAULT_SEL_CS;
+            aHyperSel[SELM_HYPER_SEL_DS]         = SELM_HYPER_DEFAULT_SEL_DS;
+            aHyperSel[SELM_HYPER_SEL_CS64]       = SELM_HYPER_DEFAULT_SEL_CS64;
+            aHyperSel[SELM_HYPER_SEL_TSS]        = SELM_HYPER_DEFAULT_SEL_TSS;
+            aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = SELM_HYPER_DEFAULT_SEL_TSS_TRAP08;
+        }
+
+        /*
+         * Work thru the copied GDT entries adjusting them for correct virtualization.
+         */
+        PX86DESC pGDTEEnd = (PX86DESC)((char *)pGDTE + cbEffLimit + 1 - sizeof(X86DESC));
+        while (pGDTE < pGDTEEnd)
+        {
+            if (pGDTE->Gen.u1Present)
+            {
+                /*
+                 * Code and data selectors are generally 1:1, with the
+                 * 'little' adjustment we do for DPL 0 selectors.
+                 */
+                if (pGDTE->Gen.u1DescType)
+                {
+                    /*
+                     * Hack for A-bit against Trap E on read-only GDT.
+                     */
+                    /** @todo Fix this by loading ds and cs before turning off WP. */
+                    pGDTE->Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+
+                    /*
+                     * All DPL 0 code and data segments are squeezed into DPL 1.
+                     *
+                     * We're skipping conforming segments here because those
+                     * cannot give us any trouble.
+                     */
+                    if (    pGDTE->Gen.u2Dpl == 0
+                        &&      (pGDTE->Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+                            !=  (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF) )
+                        pGDTE->Gen.u2Dpl = 1;
+                }
+                else
+                {
+                    /*
+                     * System type selectors are marked not present.
+                     * Recompiler or special handling is required for these.
+                     */
+                    /** @todo what about interrupt gates and rawr0? */
+                    pGDTE->Gen.u1Present = 0;
+                }
+            }
+
+            /* Next GDT entry. */
+            pGDTE++;
+        }
+
+        /*
+         * Check if our hypervisor selectors were changed.
+         */
+        if (    aHyperSel[SELM_HYPER_SEL_CS]         != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]
+            ||  aHyperSel[SELM_HYPER_SEL_DS]         != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]
+            ||  aHyperSel[SELM_HYPER_SEL_CS64]       != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64]
+            ||  aHyperSel[SELM_HYPER_SEL_TSS]        != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]
+            ||  aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] != pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08])
+        {
+            /* Reinitialize our hypervisor GDTs */
+            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS]         = aHyperSel[SELM_HYPER_SEL_CS];
+            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS]         = aHyperSel[SELM_HYPER_SEL_DS];
+            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64]       = aHyperSel[SELM_HYPER_SEL_CS64];
+            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS]        = aHyperSel[SELM_HYPER_SEL_TSS];
+            pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] = aHyperSel[SELM_HYPER_SEL_TSS_TRAP08];
+
+            STAM_COUNTER_INC(&pVM->selm.s.StatHyperSelsChanged);
+
+            /*
+             * Do the relocation callbacks to let everyone update their hyper selector dependencies.
+             * (SELMR3Relocate will call selmR3SetupHyperGDTSelectors() for us.)
+             */
+            VMR3Relocate(pVM, 0);
+        }
+        else if (cbEffLimit >= SELM_HYPER_DEFAULT_BASE)
+            /* We overwrote all entries above, so we have to save them again. */
+            selmR3SetupHyperGDTSelectors(pVM);
+
+        /*
+         * Adjust the cached GDT limit.
+         * Any GDT entries which have been removed must be cleared.
+         */
+        if (pVM->selm.s.GuestGdtr.cbGdt != GDTR.cbGdt)
+        {
+            if (pVM->selm.s.GuestGdtr.cbGdt > GDTR.cbGdt)
+                memset(pGDTE, 0, pVM->selm.s.GuestGdtr.cbGdt - GDTR.cbGdt);
+#ifndef SELM_TRACK_GUEST_GDT_CHANGES
+            pVM->selm.s.GuestGdtr.cbGdt = GDTR.cbGdt;
+#endif
+        }
+
+#ifdef SELM_TRACK_GUEST_GDT_CHANGES
+        /*
+         * Check if Guest's GDTR is changed.
+         */
+        if (    GDTR.pGdt != pVM->selm.s.GuestGdtr.pGdt
+            ||  GDTR.cbGdt != pVM->selm.s.GuestGdtr.cbGdt)
+        {
+            Log(("SELMR3UpdateFromCPUM: Guest's GDT is changed to pGdt=%016RX64 cbGdt=%08X\n", GDTR.pGdt, GDTR.cbGdt));
+
+            /*
+             * [Re]Register write virtual handler for guest's GDT.
+             */
+            if (pVM->selm.s.GuestGdtr.pGdt != RTRCPTR_MAX && pVM->selm.s.fGDTRangeRegistered)
+            {
+                rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GuestGdtr.pGdt);
+                AssertRC(rc);
+            }
+
+            rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, GDTR.pGdt, GDTR.pGdt + GDTR.cbGdt /* already inclusive */,
+                                             0, selmR3GuestGDTWriteHandler, "selmRCGuestGDTWriteHandler", 0, "Guest GDT write access handler");
+            if (RT_FAILURE(rc))
+                return rc;
+
+            /* Update saved Guest GDTR. */
+            pVM->selm.s.GuestGdtr = GDTR;
+            pVM->selm.s.fGDTRangeRegistered = true;
+        }
+#endif
+    }
+
+    /*
+     * TSS sync
+     */
+    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_TSS))
+    {
+        SELMR3SyncTSS(pVM, pVCpu);
+    }
+
+    /*
+     * LDT sync
+     */
+    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_LDT))
+    {
+        /*
+         * Always assume the best
+         */
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_LDT);
+
+        /*
+         * LDT handling is done similarly to the GDT handling with a shadow
+         * array. However, since the LDT is expected to be swappable (at least
+         * some ancient OSes makes it swappable) it must be floating and
+         * synced on a per-page basis.
+         *
+         * Eventually we will change this to be fully on demand. Meaning that
+         * we will only sync pages containing LDT selectors actually used and
+         * let the #PF handler lazily sync pages as they are used.
+         * (This applies to GDT too, when we start making OS/2 fast.)
+         */
+
+        /*
+         * First, determine the current LDT selector.
+         */
+        RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
+        if ((SelLdt & X86_SEL_MASK) == 0)
+        {
+            /* ldtr = 0 - update hyper LDTR and deregister any active handler. */
+            CPUMSetHyperLDTR(pVCpu, 0);
+#ifdef SELM_TRACK_GUEST_LDT_CHANGES
+            if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
+            {
+                rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
+                AssertRC(rc);
+                pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
+            }
+#endif
+            STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Get the LDT selector.
+         */
+        PX86DESC    pDesc = &pVM->selm.s.paGdtR3[SelLdt >> X86_SEL_SHIFT];
+        RTGCPTR     GCPtrLdt = X86DESC_BASE(*pDesc);
+        unsigned    cbLdt = X86DESC_LIMIT(*pDesc);
+        if (pDesc->Gen.u1Granularity)
+            cbLdt = (cbLdt << PAGE_SHIFT) | PAGE_OFFSET_MASK;
+
+        /*
+         * Validate it.
+         */
+        if (    !cbLdt
+            ||  SelLdt >= pVM->selm.s.GuestGdtr.cbGdt
+            ||  pDesc->Gen.u1DescType
+            ||  pDesc->Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+        {
+            AssertMsg(!cbLdt, ("Invalid LDT %04x!\n", SelLdt));
+
+            /* cbLdt > 0:
+             * This is quite impossible, so we do as most people do when faced with
+             * the impossible, we simply ignore it.
+             */
+            CPUMSetHyperLDTR(pVCpu, 0);
+#ifdef SELM_TRACK_GUEST_LDT_CHANGES
+            if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
+            {
+                rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
+                AssertRC(rc);
+                pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
+            }
+#endif
+            STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+            return VINF_SUCCESS;
+        }
+        /** @todo check what intel does about odd limits. */
+        AssertMsg(RT_ALIGN(cbLdt + 1, sizeof(X86DESC)) == cbLdt + 1 && cbLdt <= 0xffff, ("cbLdt=%d\n", cbLdt));
+
+        /*
+         * Use the cached guest ldt address if the descriptor has already been modified (see below)
+         * (this is necessary due to redundant LDT updates; see todo above at GDT sync)
+         */
+        if (MMHyperIsInsideArea(pVM, GCPtrLdt))
+            GCPtrLdt = pVM->selm.s.GCPtrGuestLdt;   /* use the old one */
+
+
+#ifdef SELM_TRACK_GUEST_LDT_CHANGES
+        /** @todo Handle only present LDT segments. */
+    //    if (pDesc->Gen.u1Present)
+        {
+            /*
+             * Check if Guest's LDT address/limit is changed.
+             */
+            if (    GCPtrLdt != pVM->selm.s.GCPtrGuestLdt
+                ||  cbLdt != pVM->selm.s.cbLdtLimit)
+            {
+                Log(("SELMR3UpdateFromCPUM: Guest LDT changed to from %RGv:%04x to %RGv:%04x. (GDTR=%016RX64:%04x)\n",
+                     pVM->selm.s.GCPtrGuestLdt, pVM->selm.s.cbLdtLimit, GCPtrLdt, cbLdt, pVM->selm.s.GuestGdtr.pGdt, pVM->selm.s.GuestGdtr.cbGdt));
+
+                /*
+                 * [Re]Register write virtual handler for guest's GDT.
+                 * In the event of LDT overlapping something, don't install it just assume it's being updated.
+                 */
+                if (pVM->selm.s.GCPtrGuestLdt != RTRCPTR_MAX)
+                {
+                    rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestLdt);
+                    AssertRC(rc);
+                }
+#ifdef DEBUG
+                if (pDesc->Gen.u1Present)
+                    Log(("LDT selector marked not present!!\n"));
+#endif
+                rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, GCPtrLdt, GCPtrLdt + cbLdt /* already inclusive */,
+                                                 0, selmR3GuestLDTWriteHandler, "selmRCGuestLDTWriteHandler", 0, "Guest LDT write access handler");
+                if (rc == VERR_PGM_HANDLER_VIRTUAL_CONFLICT)
+                {
+                    /** @todo investigate the various cases where conflicts happen and try avoid them by enh. the instruction emulation. */
+                    pVM->selm.s.GCPtrGuestLdt = RTRCPTR_MAX;
+                    Log(("WARNING: Guest LDT (%RGv:%04x) conflicted with existing access range!! Assumes LDT is begin updated. (GDTR=%016RX64:%04x)\n",
+                         GCPtrLdt, cbLdt, pVM->selm.s.GuestGdtr.pGdt, pVM->selm.s.GuestGdtr.cbGdt));
+                }
+                else if (RT_SUCCESS(rc))
+                    pVM->selm.s.GCPtrGuestLdt = GCPtrLdt;
+                else
+                {
+                    CPUMSetHyperLDTR(pVCpu, 0);
+                    STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+                    return rc;
+                }
+
+                pVM->selm.s.cbLdtLimit = cbLdt;
+            }
+        }
+#else
+        pVM->selm.s.cbLdtLimit = cbLdt;
+#endif
+
+        /*
+         * Calc Shadow LDT base.
+         */
+        unsigned    off;
+        pVM->selm.s.offLdtHyper = off = (GCPtrLdt & PAGE_OFFSET_MASK);
+        RTGCPTR     GCPtrShadowLDT  = (RTGCPTR)((RTGCUINTPTR)pVM->selm.s.pvLdtRC + off);
+        PX86DESC    pShadowLDT      = (PX86DESC)((uintptr_t)pVM->selm.s.pvLdtR3 + off);
+
+        /*
+         * Enable the LDT selector in the shadow GDT.
+         */
+        pDesc->Gen.u1Present    = 1;
+        pDesc->Gen.u16BaseLow   = RT_LOWORD(GCPtrShadowLDT);
+        pDesc->Gen.u8BaseHigh1  = RT_BYTE3(GCPtrShadowLDT);
+        pDesc->Gen.u8BaseHigh2  = RT_BYTE4(GCPtrShadowLDT);
+        pDesc->Gen.u1Available  = 0;
+        pDesc->Gen.u1Long       = 0;
+        if (cbLdt > 0xffff)
+        {
+            cbLdt = 0xffff;
+            pDesc->Gen.u4LimitHigh  = 0;
+            pDesc->Gen.u16LimitLow  = pDesc->Gen.u1Granularity ? 0xf : 0xffff;
+        }
+
+        /*
+         * Set Hyper LDTR and notify TRPM.
+         */
+        CPUMSetHyperLDTR(pVCpu, SelLdt);
+
+        /*
+         * Loop synchronising the LDT page by page.
+         */
+        /** @todo investigate how intel handle various operations on half present cross page entries. */
+        off = GCPtrLdt & (sizeof(X86DESC) - 1);
+        AssertMsg(!off, ("LDT is not aligned on entry size! GCPtrLdt=%08x\n", GCPtrLdt));
+
+        /* Note: Do not skip the first selector; unlike the GDT, a zero LDT selector is perfectly valid. */
+        unsigned    cbLeft = cbLdt + 1;
+        PX86DESC   pLDTE = pShadowLDT;
+        while (cbLeft)
+        {
+            /*
+             * Read a chunk.
+             */
+            unsigned  cbChunk = PAGE_SIZE - ((RTGCUINTPTR)GCPtrLdt & PAGE_OFFSET_MASK);
+            if (cbChunk > cbLeft)
+                cbChunk = cbLeft;
+            rc = PGMPhysSimpleReadGCPtr(pVCpu, pShadowLDT, GCPtrLdt, cbChunk);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Mark page
+                 */
+                rc = PGMMapSetPage(pVM, GCPtrShadowLDT & PAGE_BASE_GC_MASK, PAGE_SIZE, X86_PTE_P | X86_PTE_A | X86_PTE_D);
+                AssertRC(rc);
+
+                /*
+                 * Loop thru the available LDT entries.
+                 * Figure out where to start and end and the potential cross pageness of
+                 * things adds a little complexity. pLDTE is updated there and not in the
+                 * 'next' part of the loop. The pLDTEEnd is inclusive.
+                 */
+                PX86DESC pLDTEEnd = (PX86DESC)((uintptr_t)pShadowLDT + cbChunk) - 1;
+                if (pLDTE + 1 < pShadowLDT)
+                    pLDTE = (PX86DESC)((uintptr_t)pShadowLDT + off);
+                while (pLDTE <= pLDTEEnd)
+                {
+                    if (pLDTE->Gen.u1Present)
+                    {
+                        /*
+                         * Code and data selectors are generally 1:1, with the
+                         * 'little' adjustment we do for DPL 0 selectors.
+                         */
+                        if (pLDTE->Gen.u1DescType)
+                        {
+                            /*
+                             * Hack for A-bit against Trap E on read-only GDT.
+                             */
+                            /** @todo Fix this by loading ds and cs before turning off WP. */
+                            if (!(pLDTE->Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+                                pLDTE->Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+
+                            /*
+                             * All DPL 0 code and data segments are squeezed into DPL 1.
+                             *
+                             * We're skipping conforming segments here because those
+                             * cannot give us any trouble.
+                             */
+                            if (    pLDTE->Gen.u2Dpl == 0
+                                &&      (pLDTE->Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+                                    !=  (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF) )
+                                pLDTE->Gen.u2Dpl = 1;
+                        }
+                        else
+                        {
+                            /*
+                             * System type selectors are marked not present.
+                             * Recompiler or special handling is required for these.
+                             */
+                            /** @todo what about interrupt gates and rawr0? */
+                            pLDTE->Gen.u1Present = 0;
+                        }
+                    }
+
+                    /* Next LDT entry. */
+                    pLDTE++;
+                }
+            }
+            else
+            {
+                AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("rc=%Rrc\n", rc));
+                rc = PGMMapSetPage(pVM, GCPtrShadowLDT & PAGE_BASE_GC_MASK, PAGE_SIZE, 0);
+                AssertRC(rc);
+            }
+
+            /*
+             * Advance to the next page.
+             */
+            cbLeft          -= cbChunk;
+            GCPtrShadowLDT  += cbChunk;
+            pShadowLDT       = (PX86DESC)((char *)pShadowLDT + cbChunk);
+            GCPtrLdt        += cbChunk;
+        }
+    }
+
+    STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * \#PF Handler callback for virtual access handler ranges.
+ *
+ * Important to realize that a physical page in a range can have aliases, and
+ * for ALL and WRITE handlers these will also trigger.
+ *
+ * @returns VINF_SUCCESS if the handler have carried out the operation.
+ * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
+ * @param   pVM             VM Handle.
+ * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
+ * @param   pvPtr           The HC mapping of that address.
+ * @param   pvBuf           What the guest is reading/writing.
+ * @param   cbBuf           How much it's reading/writing.
+ * @param   enmAccessType   The access type.
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) selmR3GuestGDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
+{
+    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+    Log(("selmR3GuestGDTWriteHandler: write to %RGv size %d\n", GCPtr, cbBuf));
+
+    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_SELM_SYNC_GDT);
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+/**
+ * \#PF Handler callback for virtual access handler ranges.
+ *
+ * Important to realize that a physical page in a range can have aliases, and
+ * for ALL and WRITE handlers these will also trigger.
+ *
+ * @returns VINF_SUCCESS if the handler have carried out the operation.
+ * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
+ * @param   pVM             VM Handle.
+ * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
+ * @param   pvPtr           The HC mapping of that address.
+ * @param   pvBuf           What the guest is reading/writing.
+ * @param   cbBuf           How much it's reading/writing.
+ * @param   enmAccessType   The access type.
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) selmR3GuestLDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
+{
+    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+    Log(("selmR3GuestLDTWriteHandler: write to %RGv size %d\n", GCPtr, cbBuf));
+    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_SELM_SYNC_LDT);
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+/**
+ * \#PF Handler callback for virtual access handler ranges.
+ *
+ * Important to realize that a physical page in a range can have aliases, and
+ * for ALL and WRITE handlers these will also trigger.
+ *
+ * @returns VINF_SUCCESS if the handler have carried out the operation.
+ * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
+ * @param   pVM             VM Handle.
+ * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
+ * @param   pvPtr           The HC mapping of that address.
+ * @param   pvBuf           What the guest is reading/writing.
+ * @param   cbBuf           How much it's reading/writing.
+ * @param   enmAccessType   The access type.
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) selmR3GuestTSSWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
+{
+    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+    Log(("selmR3GuestTSSWriteHandler: write %.*Rhxs to %RGv size %d\n", RT_MIN(8, cbBuf), pvBuf, GCPtr, cbBuf));
+
+    /** @todo This can be optimized by checking for the ESP0 offset and tracking TR
+     *        reloads in REM (setting VM_FF_SELM_SYNC_TSS if TR is reloaded). We
+     *        should probably also deregister the virtual handler if TR.base/size
+     *        changes while we're in REM. */
+
+    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_SELM_SYNC_TSS);
+
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+/**
+ * Synchronize the shadowed fields in the TSS.
+ *
+ * At present we're shadowing the ring-0 stack selector & pointer, and the
+ * interrupt redirection bitmap (if present). We take the lazy approach wrt to
+ * REM and this function is called both if REM made any changes to the TSS or
+ * loaded TR.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VMCPU to operate on.
+ */
+VMMR3DECL(int) SELMR3SyncTSS(PVM pVM, PVMCPU pVCpu)
+{
+    int    rc;
+
+    if (pVM->selm.s.fDisableMonitoring)
+    {
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+        return VINF_SUCCESS;
+    }
+
+    STAM_PROFILE_START(&pVM->selm.s.StatTSSSync, a);
+    Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_TSS));
+
+    /*
+     * Get TR and extract and store the basic info.
+     *
+     * Note! The TSS limit is not checked by the LTR code, so we
+     *       have to be a bit careful with it. We make sure cbTss
+     *       won't be zero if TR is valid and if it's NULL we'll
+     *       make sure cbTss is 0.
+     */
+    CPUMSELREGHID   trHid;
+    RTSEL           SelTss   = CPUMGetGuestTR(pVCpu, &trHid);
+    RTGCPTR         GCPtrTss = trHid.u64Base;
+    uint32_t        cbTss    = trHid.u32Limit;
+    Assert(     (SelTss & X86_SEL_MASK)
+           ||   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
+           ||   (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */));
+    if (SelTss & X86_SEL_MASK)
+    {
+        Assert(!(SelTss & X86_SEL_LDT));
+        Assert(trHid.Attr.n.u1DescType == 0);
+        Assert(   trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY
+               || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+        if (!++cbTss)
+            cbTss = UINT32_MAX;
+    }
+    else
+    {
+        Assert(   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
+               || (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */));
+        cbTss = 0; /* the reset case. */
+    }
+    pVM->selm.s.cbGuestTss     = cbTss;
+    pVM->selm.s.fGuestTss32Bit = trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+                              || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY;
+
+    /*
+     * Figure out the size of what need to monitor.
+     */
+    /* We're not interested in any 16-bit TSSes. */
+    uint32_t cbMonitoredTss = cbTss;
+    if (    trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL
+        &&  trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
+        cbMonitoredTss = 0;
+
+    pVM->selm.s.offGuestIoBitmap = 0;
+    bool fNoRing1Stack = true;
+    if (cbMonitoredTss)
+    {
+        /*
+         * 32-bit TSS. What we're really keen on is the SS0 and ESP0 fields.
+         * If VME is enabled we also want to keep an eye on the interrupt
+         * redirection bitmap.
+         */
+        VBOXTSS Tss;
+        uint32_t cr4 = CPUMGetGuestCR4(pVCpu);
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, &Tss, GCPtrTss, RT_OFFSETOF(VBOXTSS, IntRedirBitmap));
+        if (    !(cr4 & X86_CR4_VME)
+            ||   (  RT_SUCCESS(rc)
+                 && Tss.offIoBitmap < sizeof(VBOXTSS) /* too small */
+                 && Tss.offIoBitmap > cbTss)          /* beyond the end */ /** @todo not sure how the partial case is handled; probably not allowed. */
+           )
+            /* No interrupt redirection bitmap, just ESP0 and SS0. */
+            cbMonitoredTss = RT_UOFFSETOF(VBOXTSS, padding_ss0);
+        else if (RT_SUCCESS(rc))
+        {
+            /*
+             * Everything up to and including the interrupt redirection bitmap. Unfortunately
+             * this can be quite a large chunk. We use to skip it earlier and just hope it
+             * was kind of static...
+             *
+             * Update the virtual interrupt redirection bitmap while we're here.
+             * (It is located in the 32 bytes before TR:offIoBitmap.)
+             */
+            cbMonitoredTss = Tss.offIoBitmap;
+            pVM->selm.s.offGuestIoBitmap = Tss.offIoBitmap;
+
+            uint32_t offRedirBitmap = Tss.offIoBitmap - sizeof(Tss.IntRedirBitmap);
+            rc = PGMPhysSimpleReadGCPtr(pVCpu, &pVM->selm.s.Tss.IntRedirBitmap,
+                                        GCPtrTss + offRedirBitmap, sizeof(Tss.IntRedirBitmap));
+            AssertRC(rc);
+            /** @todo memset the bitmap on failure? */
+            Log2(("Redirection bitmap:\n"));
+            Log2(("%.*Rhxd\n", sizeof(Tss.IntRedirBitmap), &pVM->selm.s.Tss.IntRedirBitmap));
+        }
+        else
+        {
+            cbMonitoredTss = RT_OFFSETOF(VBOXTSS, IntRedirBitmap);
+            pVM->selm.s.offGuestIoBitmap = 0;
+            /** @todo memset the bitmap? */
+        }
+
+        /*
+         * Update the ring 0 stack selector and base address.
+         */
+        if (RT_SUCCESS(rc))
+        {
+#ifdef LOG_ENABLED
+            if (LogIsEnabled())
+            {
+                uint32_t ssr0, espr0;
+                SELMGetRing1Stack(pVM, &ssr0, &espr0);
+                if ((ssr0 & ~1) != Tss.ss0 || espr0 != Tss.esp0)
+                {
+                    RTGCPHYS GCPhys = NIL_RTGCPHYS;
+                    rc = PGMGstGetPage(pVCpu, GCPtrTss, NULL, &GCPhys); AssertRC(rc);
+                    Log(("SELMR3SyncTSS: Updating TSS ring 0 stack to %04X:%08X from %04X:%08X; TSS Phys=%RGp)\n",
+                         Tss.ss0, Tss.esp0, (ssr0 & ~1), espr0,  GCPhys));
+                    AssertMsg(ssr0 != Tss.ss0,
+                              ("ring-1 leak into TSS.SS0! %04X:%08X from %04X:%08X; TSS Phys=%RGp)\n",
+                               Tss.ss0, Tss.esp0, (ssr0 & ~1), espr0,  GCPhys));
+                }
+                Log(("offIoBitmap=%#x\n", Tss.offIoBitmap));
+            }
+#endif /* LOG_ENABLED */
+            AssertMsg(!(Tss.ss0 & 3), ("ring-1 leak into TSS.SS0? %04X:%08X\n", Tss.ss0, Tss.esp0));
+
+            /* Update our TSS structure for the guest's ring 1 stack */
+            selmSetRing1Stack(pVM, Tss.ss0 | 1, Tss.esp0);
+            pVM->selm.s.fSyncTSSRing0Stack = fNoRing1Stack = false;
+        }
+    }
+
+    /*
+     * Flush the ring-1 stack and the direct syscall dispatching if we
+     * cannot obtain SS0:ESP0.
+     */
+    if (fNoRing1Stack)
+    {
+        selmSetRing1Stack(pVM, 0 /* invalid SS */, 0);
+        pVM->selm.s.fSyncTSSRing0Stack = cbMonitoredTss != 0;
+
+        /** @todo handle these dependencies better! */
+        TRPMR3SetGuestTrapHandler(pVM, 0x2E, TRPM_INVALID_HANDLER);
+        TRPMR3SetGuestTrapHandler(pVM, 0x80, TRPM_INVALID_HANDLER);
+    }
+
+    /*
+     * Check for monitor changes and apply them.
+     */
+    if (    GCPtrTss        != pVM->selm.s.GCPtrGuestTss
+        ||  cbMonitoredTss  != pVM->selm.s.cbMonitoredGuestTss)
+    {
+        Log(("SELMR3SyncTSS: Guest's TSS is changed to pTss=%RGv cbMonitoredTss=%08X cbGuestTss=%#08x\n",
+             GCPtrTss, cbMonitoredTss, pVM->selm.s.cbGuestTss));
+
+        /* Release the old range first. */
+        if (pVM->selm.s.GCPtrGuestTss != RTRCPTR_MAX)
+        {
+            rc = PGMHandlerVirtualDeregister(pVM, pVM->selm.s.GCPtrGuestTss);
+            AssertRC(rc);
+        }
+
+        /* Register the write handler if TS != 0. */
+        if (cbMonitoredTss != 0)
+        {
+            rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, GCPtrTss, GCPtrTss + cbMonitoredTss - 1,
+                                             0, selmR3GuestTSSWriteHandler,
+                                             "selmRCGuestTSSWriteHandler", 0, "Guest TSS write access handler");
+            if (RT_FAILURE(rc))
+            {
+                STAM_PROFILE_STOP(&pVM->selm.s.StatUpdateFromCPUM, a);
+                return rc;
+            }
+
+            /* Update saved Guest TSS info. */
+            pVM->selm.s.GCPtrGuestTss       = GCPtrTss;
+            pVM->selm.s.cbMonitoredGuestTss = cbMonitoredTss;
+            pVM->selm.s.GCSelTss            = SelTss;
+        }
+        else
+        {
+            pVM->selm.s.GCPtrGuestTss       = RTRCPTR_MAX;
+            pVM->selm.s.cbMonitoredGuestTss = 0;
+            pVM->selm.s.GCSelTss            = 0;
+        }
+    }
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+
+    STAM_PROFILE_STOP(&pVM->selm.s.StatTSSSync, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Compares the Guest GDT and LDT with the shadow tables.
+ * This is a VBOX_STRICT only function.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM Handle.
+ */
+VMMR3DECL(int) SELMR3DebugCheck(PVM pVM)
+{
+#ifdef VBOX_STRICT
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    /*
+     * Get GDTR and check for conflict.
+     */
+    VBOXGDTR  GDTR;
+    CPUMGetGuestGDTR(pVCpu, &GDTR);
+    if (GDTR.cbGdt == 0)
+        return VINF_SUCCESS;
+
+    if (GDTR.cbGdt >= (unsigned)(pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> X86_SEL_SHIFT))
+        Log(("SELMR3DebugCheck: guest GDT size forced us to look for unused selectors.\n"));
+
+    if (GDTR.cbGdt != pVM->selm.s.GuestGdtr.cbGdt)
+        Log(("SELMR3DebugCheck: limits have changed! new=%d old=%d\n", GDTR.cbGdt, pVM->selm.s.GuestGdtr.cbGdt));
+
+    /*
+     * Loop thru the GDT checking each entry.
+     */
+    RTGCPTR     GCPtrGDTEGuest = GDTR.pGdt;
+    PX86DESC    pGDTE = pVM->selm.s.paGdtR3;
+    PX86DESC    pGDTEEnd = (PX86DESC)((uintptr_t)pGDTE + GDTR.cbGdt);
+    while (pGDTE < pGDTEEnd)
+    {
+        X86DESC    GDTEGuest;
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &GDTEGuest, GCPtrGDTEGuest, sizeof(GDTEGuest));
+        if (RT_SUCCESS(rc))
+        {
+            if (pGDTE->Gen.u1DescType || pGDTE->Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+            {
+                if (   pGDTE->Gen.u16LimitLow != GDTEGuest.Gen.u16LimitLow
+                    || pGDTE->Gen.u4LimitHigh != GDTEGuest.Gen.u4LimitHigh
+                    || pGDTE->Gen.u16BaseLow  != GDTEGuest.Gen.u16BaseLow
+                    || pGDTE->Gen.u8BaseHigh1 != GDTEGuest.Gen.u8BaseHigh1
+                    || pGDTE->Gen.u8BaseHigh2 != GDTEGuest.Gen.u8BaseHigh2
+                    || pGDTE->Gen.u1DefBig    != GDTEGuest.Gen.u1DefBig
+                    || pGDTE->Gen.u1DescType  != GDTEGuest.Gen.u1DescType)
+                {
+                    unsigned iGDT = pGDTE - pVM->selm.s.paGdtR3;
+                    SELMR3DumpDescriptor(*pGDTE, iGDT << 3, "SELMR3DebugCheck: GDT mismatch, shadow");
+                    SELMR3DumpDescriptor(GDTEGuest, iGDT << 3, "SELMR3DebugCheck: GDT mismatch,  guest");
+                }
+            }
+        }
+
+        /* Advance to the next descriptor. */
+        GCPtrGDTEGuest += sizeof(X86DESC);
+        pGDTE++;
+    }
+
+
+    /*
+     * LDT?
+     */
+    RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
+    if ((SelLdt & X86_SEL_MASK) == 0)
+        return VINF_SUCCESS;
+    if (SelLdt > GDTR.cbGdt)
+    {
+        Log(("SELMR3DebugCheck: ldt is out of bound SelLdt=%#x\n", SelLdt));
+        return VERR_INTERNAL_ERROR;
+    }
+    X86DESC    LDTDesc;
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &LDTDesc, GDTR.pGdt + (SelLdt & X86_SEL_MASK), sizeof(LDTDesc));
+    if (RT_FAILURE(rc))
+    {
+        Log(("SELMR3DebugCheck: Failed to read LDT descriptor. rc=%d\n", rc));
+        return rc;
+    }
+    RTGCPTR     GCPtrLDTEGuest = X86DESC_BASE(LDTDesc);
+    unsigned    cbLdt = X86DESC_LIMIT(LDTDesc);
+    if (LDTDesc.Gen.u1Granularity)
+        cbLdt = (cbLdt << PAGE_SHIFT) | PAGE_OFFSET_MASK;
+
+    /*
+     * Validate it.
+     */
+    if (!cbLdt)
+        return VINF_SUCCESS;
+    /** @todo check what intel does about odd limits. */
+    AssertMsg(RT_ALIGN(cbLdt + 1, sizeof(X86DESC)) == cbLdt + 1 && cbLdt <= 0xffff, ("cbLdt=%d\n", cbLdt));
+    if (    LDTDesc.Gen.u1DescType
+        ||  LDTDesc.Gen.u4Type != X86_SEL_TYPE_SYS_LDT
+        ||  SelLdt >= pVM->selm.s.GuestGdtr.cbGdt)
+    {
+        Log(("SELmR3DebugCheck: Invalid LDT %04x!\n", SelLdt));
+        return VERR_INTERNAL_ERROR;
+    }
+
+    /*
+     * Loop thru the LDT checking each entry.
+     */
+    unsigned    off = (GCPtrLDTEGuest & PAGE_OFFSET_MASK);
+    PX86DESC    pLDTE = (PX86DESC)((uintptr_t)pVM->selm.s.pvLdtR3 + off);
+    PX86DESC    pLDTEEnd = (PX86DESC)((uintptr_t)pGDTE + cbLdt);
+    while (pLDTE < pLDTEEnd)
+    {
+        X86DESC    LDTEGuest;
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, &LDTEGuest, GCPtrLDTEGuest, sizeof(LDTEGuest));
+        if (RT_SUCCESS(rc))
+        {
+            if (   pLDTE->Gen.u16LimitLow != LDTEGuest.Gen.u16LimitLow
+                || pLDTE->Gen.u4LimitHigh != LDTEGuest.Gen.u4LimitHigh
+                || pLDTE->Gen.u16BaseLow  != LDTEGuest.Gen.u16BaseLow
+                || pLDTE->Gen.u8BaseHigh1 != LDTEGuest.Gen.u8BaseHigh1
+                || pLDTE->Gen.u8BaseHigh2 != LDTEGuest.Gen.u8BaseHigh2
+                || pLDTE->Gen.u1DefBig    != LDTEGuest.Gen.u1DefBig
+                || pLDTE->Gen.u1DescType  != LDTEGuest.Gen.u1DescType)
+            {
+                unsigned iLDT = pLDTE - (PX86DESC)((uintptr_t)pVM->selm.s.pvLdtR3 + off);
+                SELMR3DumpDescriptor(*pLDTE, iLDT << 3, "SELMR3DebugCheck: LDT mismatch, shadow");
+                SELMR3DumpDescriptor(LDTEGuest, iLDT << 3, "SELMR3DebugCheck: LDT mismatch,  guest");
+            }
+        }
+
+        /* Advance to the next descriptor. */
+        GCPtrLDTEGuest += sizeof(X86DESC);
+        pLDTE++;
+    }
+
+#else  /* !VBOX_STRICT */
+    NOREF(pVM);
+#endif /* !VBOX_STRICT */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates the RawR0 TSS values against the one in the Guest TSS.
+ *
+ * @returns true if it matches.
+ * @returns false and assertions on mismatch..
+ * @param   pVM     VM Handle.
+ */
+VMMR3DECL(bool) SELMR3CheckTSS(PVM pVM)
+{
+#ifdef VBOX_STRICT
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_SELM_SYNC_TSS))
+        return true;
+
+    /*
+     * Get TR and extract the basic info.
+     */
+    CPUMSELREGHID   trHid;
+    RTSEL           SelTss   = CPUMGetGuestTR(pVCpu, &trHid);
+    RTGCPTR         GCPtrTss = trHid.u64Base;
+    uint32_t        cbTss    = trHid.u32Limit;
+    Assert(     (SelTss & X86_SEL_MASK)
+           ||   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
+           ||   (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */));
+    if (SelTss & X86_SEL_MASK)
+    {
+        AssertReturn(!(SelTss & X86_SEL_LDT), false);
+        AssertReturn(trHid.Attr.n.u1DescType == 0, false);
+        AssertReturn(   trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY
+                     || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY,
+                     false);
+        if (!++cbTss)
+            cbTss = UINT32_MAX;
+    }
+    else
+    {
+        AssertReturn(   (cbTss == 0 && GCPtrTss == 0 && trHid.Attr.u == 0 /* TR=0 */)
+                     || (cbTss == 0xffff && GCPtrTss == 0 && trHid.Attr.n.u1Present && trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY /* RESET */),
+                     false);
+        cbTss = 0; /* the reset case. */
+    }
+    AssertMsgReturn(pVM->selm.s.cbGuestTss == cbTss, ("%#x %#x\n", pVM->selm.s.cbGuestTss, cbTss), false);
+    AssertMsgReturn(pVM->selm.s.fGuestTss32Bit == (   trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+                                                   || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY),
+                    ("%RTbool u4Type=%d\n", pVM->selm.s.fGuestTss32Bit, trHid.Attr.n.u4Type),
+                    false);
+    AssertMsgReturn(    pVM->selm.s.GCSelTss == SelTss
+                    ||  (!pVM->selm.s.GCSelTss && !(SelTss & X86_SEL_LDT)),
+                    ("%#x %#x\n", pVM->selm.s.GCSelTss, SelTss),
+                    false);
+    AssertMsgReturn(    pVM->selm.s.GCPtrGuestTss == GCPtrTss
+                    ||  (pVM->selm.s.GCPtrGuestTss == RTRCPTR_MAX && !GCPtrTss),
+                    ("%#RGv %#RGv\n", pVM->selm.s.GCPtrGuestTss, GCPtrTss),
+                    false);
+
+
+    /*
+     * Figure out the size of what need to monitor.
+     */
+    bool fNoRing1Stack = true;
+    /* We're not interested in any 16-bit TSSes. */
+    uint32_t cbMonitoredTss = cbTss;
+    if (    trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL
+        &&  trHid.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
+        cbMonitoredTss = 0;
+    if (cbMonitoredTss)
+    {
+        VBOXTSS Tss;
+        uint32_t cr4 = CPUMGetGuestCR4(pVCpu);
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Tss, GCPtrTss, RT_OFFSETOF(VBOXTSS, IntRedirBitmap));
+        AssertReturn(   rc == VINF_SUCCESS
+                        /* Happens early in XP boot during page table switching. */
+                     || (   (rc == VERR_PAGE_TABLE_NOT_PRESENT || rc == VERR_PAGE_NOT_PRESENT)
+                         && !(CPUMGetGuestEFlags(pVCpu) & X86_EFL_IF)),
+                     false);
+        if (    !(cr4 & X86_CR4_VME)
+            ||   (  RT_SUCCESS(rc)
+                 && Tss.offIoBitmap < sizeof(VBOXTSS) /* too small */
+                 && Tss.offIoBitmap > cbTss)
+           )
+            cbMonitoredTss = RT_UOFFSETOF(VBOXTSS, padding_ss0);
+        else if (RT_SUCCESS(rc))
+        {
+            cbMonitoredTss = Tss.offIoBitmap;
+            AssertMsgReturn(pVM->selm.s.offGuestIoBitmap == Tss.offIoBitmap,
+                            ("#x %#x\n", pVM->selm.s.offGuestIoBitmap, Tss.offIoBitmap),
+                            false);
+
+            /* check the bitmap */
+            uint32_t offRedirBitmap = Tss.offIoBitmap - sizeof(Tss.IntRedirBitmap);
+            rc = PGMPhysSimpleReadGCPtr(pVCpu, &Tss.IntRedirBitmap,
+                                        GCPtrTss + offRedirBitmap, sizeof(Tss.IntRedirBitmap));
+            AssertRCReturn(rc, false);
+            AssertMsgReturn(!memcmp(&Tss.IntRedirBitmap[0], &pVM->selm.s.Tss.IntRedirBitmap[0], sizeof(Tss.IntRedirBitmap)),
+                            ("offIoBitmap=%#x cbTss=%#x\n"
+                             " Guest: %.32Rhxs\n"
+                             "Shadow: %.32Rhxs\n",
+                             Tss.offIoBitmap, cbTss,
+                             &Tss.IntRedirBitmap[0],
+                             &pVM->selm.s.Tss.IntRedirBitmap[0]),
+                            false);
+        }
+        else
+            cbMonitoredTss = RT_OFFSETOF(VBOXTSS, IntRedirBitmap);
+
+        /*
+         * Check SS0 and ESP0.
+         */
+        if (    !pVM->selm.s.fSyncTSSRing0Stack
+            &&  RT_SUCCESS(rc))
+        {
+            if (    Tss.esp0 != pVM->selm.s.Tss.esp1
+                ||  Tss.ss0  != (pVM->selm.s.Tss.ss1 & ~1))
+            {
+                RTGCPHYS GCPhys;
+                rc = PGMGstGetPage(pVCpu, GCPtrTss, NULL, &GCPhys); AssertRC(rc);
+                AssertMsgFailed(("TSS out of sync!! (%04X:%08X vs %04X:%08X (guest)) Tss=%RGv Phys=%RGp\n",
+                                 (pVM->selm.s.Tss.ss1 & ~1), pVM->selm.s.Tss.esp1,
+                                 Tss.ss1, Tss.esp1, GCPtrTss, GCPhys));
+                return false;
+            }
+        }
+        AssertMsgReturn(pVM->selm.s.cbMonitoredGuestTss == cbMonitoredTss, ("%#x %#x\n", pVM->selm.s.cbMonitoredGuestTss, cbMonitoredTss), false);
+    }
+    else
+    {
+        AssertMsgReturn(pVM->selm.s.Tss.ss1 == 0 && pVM->selm.s.Tss.esp1 == 0, ("%04x:%08x\n", pVM->selm.s.Tss.ss1, pVM->selm.s.Tss.esp1), false);
+        AssertReturn(!pVM->selm.s.fSyncTSSRing0Stack, false);
+        AssertMsgReturn(pVM->selm.s.cbMonitoredGuestTss == cbMonitoredTss, ("%#x %#x\n", pVM->selm.s.cbMonitoredGuestTss, cbMonitoredTss), false);
+    }
+
+
+
+    return true;
+
+#else  /* !VBOX_STRICT */
+    NOREF(pVM);
+    return true;
+#endif /* !VBOX_STRICT */
+}
+
+
+/**
+ * Returns flat address and limit of LDT by LDT selector from guest GDTR.
+ *
+ * Fully validate selector.
+ *
+ * @returns VBox status.
+ * @param   pVM       VM Handle.
+ * @param   SelLdt    LDT selector.
+ * @param   ppvLdt    Where to store the flat address of LDT.
+ * @param   pcbLimit  Where to store LDT limit.
+ */
+VMMDECL(int) SELMGetLDTFromSel(PVM pVM, RTSEL SelLdt, PRTGCPTR ppvLdt, unsigned *pcbLimit)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    /* Get guest GDTR. */
+    VBOXGDTR GDTR;
+    CPUMGetGuestGDTR(pVCpu, &GDTR);
+
+    /* Check selector TI and GDT limit. */
+    if (   (SelLdt & X86_SEL_LDT)
+        || SelLdt > GDTR.cbGdt)
+        return VERR_INVALID_SELECTOR;
+
+    /* Read descriptor from GC. */
+    X86DESC Desc;
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, (void *)&Desc, (RTGCPTR)(GDTR.pGdt + (SelLdt & X86_SEL_MASK)), sizeof(Desc));
+    if (RT_FAILURE(rc))
+    {
+        /* fatal */
+        Log(("Can't read LDT descriptor for selector=%04X\n", SelLdt));
+        return VERR_SELECTOR_NOT_PRESENT;
+    }
+
+    /* Check if LDT descriptor is not present. */
+    if (Desc.Gen.u1Present == 0)
+        return VERR_SELECTOR_NOT_PRESENT;
+
+    /* Check LDT descriptor type. */
+    if (    Desc.Gen.u1DescType == 1
+        ||  Desc.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+        return VERR_INVALID_SELECTOR;
+
+    /* LDT descriptor is ok. */
+    if (ppvLdt)
+    {
+        *ppvLdt = (RTGCPTR)X86DESC_BASE(Desc);
+        *pcbLimit = X86DESC_LIMIT(Desc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets information about a 64-bit selector, SELMR3GetSelectorInfo helper.
+ *
+ * See SELMR3GetSelectorInfo for details.
+ *
+ * @returns VBox status code, see SELMR3GetSelectorInfo for details.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   Sel         The selector to get info about.
+ * @param   pSelInfo    Where to store the information.
+ */
+static int selmR3GetSelectorInfo64(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
+{
+    /*
+     * Read it from the guest descriptor table.
+     */
+    X86DESC64   Desc;
+    VBOXGDTR    Gdtr;
+    RTGCPTR     GCPtrDesc;
+    CPUMGetGuestGDTR(pVCpu, &Gdtr);
+    if (!(Sel & X86_SEL_LDT))
+    {
+        /* GDT */
+        if ((unsigned)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
+            return VERR_INVALID_SELECTOR;
+        GCPtrDesc = Gdtr.pGdt + (Sel & X86_SEL_MASK);
+    }
+    else
+    {
+        /*
+         * LDT - must locate the LDT first.
+         */
+        RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
+        if (    (unsigned)(SelLdt & X86_SEL_MASK) < sizeof(X86DESC) /* the first selector is invalid, right? */ /** @todo r=bird: No, I don't think so */
+            ||  (unsigned)(SelLdt & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
+            return VERR_INVALID_SELECTOR;
+        GCPtrDesc = Gdtr.pGdt + (SelLdt & X86_SEL_MASK);
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* validate the LDT descriptor. */
+        if (Desc.Gen.u1Present == 0)
+            return VERR_SELECTOR_NOT_PRESENT;
+        if (    Desc.Gen.u1DescType == 1
+            ||  Desc.Gen.u4Type != AMD64_SEL_TYPE_SYS_LDT)
+            return VERR_INVALID_SELECTOR;
+
+        uint32_t cbLimit = X86DESC_LIMIT(Desc);
+        if (Desc.Gen.u1Granularity)
+            cbLimit = (cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
+        if ((uint32_t)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > cbLimit)
+            return VERR_INVALID_SELECTOR;
+
+        /* calc the descriptor location. */
+        GCPtrDesc = X86DESC64_BASE(Desc);
+        GCPtrDesc += (Sel & X86_SEL_MASK);
+    }
+
+    /* read the descriptor. */
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
+    if (RT_FAILURE(rc))
+    {
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(X86DESC));
+        if (RT_FAILURE(rc))
+            return rc;
+        Desc.au64[1] = 0;
+    }
+
+    /*
+     * Extract the base and limit
+     * (We ignore the present bit here, which is probably a bit silly...)
+     */
+    pSelInfo->Sel     = Sel;
+    pSelInfo->fFlags  = DBGFSELINFO_FLAGS_LONG_MODE;
+    pSelInfo->u.Raw64 = Desc;
+    if (Desc.Gen.u1DescType)
+    {
+        /*
+         * 64-bit code selectors are wide open, it's not possible to detect
+         * 64-bit data or stack selectors without also dragging in assumptions
+         * about current CS (i.e. that's we're executing in 64-bit mode).  So,
+         * the selinfo user needs to deal with this in the context the info is
+         * used unfortunately.
+         */
+        if (    Desc.Gen.u1Long
+            &&  !Desc.Gen.u1DefBig
+            &&  (Desc.Gen.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /* Note! We ignore the segment limit hacks that was added by AMD. */
+            pSelInfo->GCPtrBase = 0;
+            pSelInfo->cbLimit   = ~(RTGCUINTPTR)0;
+        }
+        else
+        {
+            pSelInfo->cbLimit = X86DESC_LIMIT(Desc);
+            if (Desc.Gen.u1Granularity)
+                pSelInfo->cbLimit = (pSelInfo->cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
+            pSelInfo->GCPtrBase = X86DESC_BASE(Desc);
+        }
+        pSelInfo->SelGate = 0;
+    }
+    else if (   Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_LDT
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TSS_AVAIL
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY)
+    {
+        /* Note. LDT descriptors are weird in long mode, we ignore the footnote
+           in the AMD manual here as a simplification. */
+        pSelInfo->GCPtrBase = X86DESC64_BASE(Desc);
+        pSelInfo->cbLimit = X86DESC_LIMIT(Desc);
+        if (Desc.Gen.u1Granularity)
+            pSelInfo->cbLimit = (pSelInfo->cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
+        pSelInfo->SelGate = 0;
+    }
+    else if (   Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_TRAP_GATE
+             || Desc.Gen.u4Type == AMD64_SEL_TYPE_SYS_INT_GATE)
+    {
+        pSelInfo->cbLimit   = X86DESC64_BASE(Desc);
+        pSelInfo->GCPtrBase = Desc.Gate.u16OffsetLow
+                            | ((uint32_t)Desc.Gate.u16OffsetHigh << 16)
+                            | ((uint64_t)Desc.Gate.u32OffsetTop << 32);
+        pSelInfo->SelGate   = Desc.Gate.u16Sel;
+        pSelInfo->fFlags   |= DBGFSELINFO_FLAGS_GATE;
+    }
+    else
+    {
+        pSelInfo->cbLimit   = 0;
+        pSelInfo->GCPtrBase = 0;
+        pSelInfo->SelGate   = 0;
+        pSelInfo->fFlags   |= DBGFSELINFO_FLAGS_INVALID;
+    }
+    if (!Desc.Gen.u1Present)
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_NOT_PRESENT;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for selmR3GetSelectorInfo32 and SELMR3GetShadowSelectorInfo that
+ * interprets a legacy descriptor table entry and fills in the selector info
+ * structure from it.
+ *
+ * @param  pSelInfo     Where to store the selector info. Only the fFlags and
+ *                      Sel members have been initialized.
+ * @param  pDesc        The legacy descriptor to parse.
+ */
+DECLINLINE(void) selmR3SelInfoFromDesc32(PDBGFSELINFO pSelInfo, PCX86DESC pDesc)
+{
+    pSelInfo->u.Raw64.au64[1] = 0;
+    pSelInfo->u.Raw = *pDesc;
+    if (    pDesc->Gen.u1DescType
+        ||  !(pDesc->Gen.u4Type & 4))
+    {
+        pSelInfo->cbLimit = X86DESC_LIMIT(*pDesc);
+        if (pDesc->Gen.u1Granularity)
+            pSelInfo->cbLimit = (pSelInfo->cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
+        pSelInfo->GCPtrBase = X86DESC_BASE(*pDesc);
+        pSelInfo->SelGate = 0;
+    }
+    else if (pDesc->Gen.u4Type != X86_SEL_TYPE_SYS_UNDEFINED4)
+    {
+        pSelInfo->cbLimit = 0;
+        if (pDesc->Gen.u4Type == X86_SEL_TYPE_SYS_TASK_GATE)
+            pSelInfo->GCPtrBase = 0;
+        else
+            pSelInfo->GCPtrBase = pDesc->Gate.u16OffsetLow
+                                | (uint32_t)pDesc->Gate.u16OffsetHigh << 16;
+        pSelInfo->SelGate = pDesc->Gate.u16Sel;
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_GATE;
+    }
+    else
+    {
+        pSelInfo->cbLimit = 0;
+        pSelInfo->GCPtrBase = 0;
+        pSelInfo->SelGate = 0;
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_INVALID;
+    }
+    if (!pDesc->Gen.u1Present)
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_NOT_PRESENT;
+}
+
+
+/**
+ * Gets information about a 64-bit selector, SELMR3GetSelectorInfo helper.
+ *
+ * See SELMR3GetSelectorInfo for details.
+ *
+ * @returns VBox status code, see SELMR3GetSelectorInfo for details.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   Sel         The selector to get info about.
+ * @param   pSelInfo    Where to store the information.
+ */
+static int selmR3GetSelectorInfo32(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
+{
+    /*
+     * Read the descriptor entry
+     */
+    pSelInfo->fFlags = 0;
+    X86DESC Desc;
+    if (    !(Sel & X86_SEL_LDT)
+        && (    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] == (Sel & X86_SEL_MASK)
+            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] == (Sel & X86_SEL_MASK)
+            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] == (Sel & X86_SEL_MASK)
+            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] == (Sel & X86_SEL_MASK)
+            ||  pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] == (Sel & X86_SEL_MASK))
+       )
+    {
+        /*
+         * Hypervisor descriptor.
+         */
+        pSelInfo->fFlags = DBGFSELINFO_FLAGS_HYPER;
+        if (CPUMIsGuestInProtectedMode(pVCpu))
+            pSelInfo->fFlags |= DBGFSELINFO_FLAGS_PROT_MODE;
+        else
+            pSelInfo->fFlags |= DBGFSELINFO_FLAGS_REAL_MODE;
+
+        Desc = pVM->selm.s.paGdtR3[Sel >> X86_SEL_SHIFT];
+    }
+    else if (CPUMIsGuestInProtectedMode(pVCpu))
+    {
+        /*
+         * Read it from the guest descriptor table.
+         */
+        pSelInfo->fFlags = DBGFSELINFO_FLAGS_PROT_MODE;
+
+        VBOXGDTR    Gdtr;
+        RTGCPTR     GCPtrDesc;
+        CPUMGetGuestGDTR(pVCpu, &Gdtr);
+        if (!(Sel & X86_SEL_LDT))
+        {
+            /* GDT */
+            if ((unsigned)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
+                return VERR_INVALID_SELECTOR;
+            GCPtrDesc = Gdtr.pGdt + (Sel & X86_SEL_MASK);
+        }
+        else
+        {
+            /*
+             * LDT - must locate the LDT first...
+             */
+            RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
+            if (    (unsigned)(SelLdt & X86_SEL_MASK) < sizeof(X86DESC) /* the first selector is invalid, right? */ /** @todo r=bird: No, I don't think so */
+                ||  (unsigned)(SelLdt & X86_SEL_MASK) + sizeof(X86DESC) - 1 > (unsigned)Gdtr.cbGdt)
+                return VERR_INVALID_SELECTOR;
+            GCPtrDesc = Gdtr.pGdt + (SelLdt & X86_SEL_MASK);
+            int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
+            if (RT_FAILURE(rc))
+                return rc;
+
+            /* validate the LDT descriptor. */
+            if (Desc.Gen.u1Present == 0)
+                return VERR_SELECTOR_NOT_PRESENT;
+            if (    Desc.Gen.u1DescType == 1
+                ||  Desc.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+                return VERR_INVALID_SELECTOR;
+
+            unsigned cbLimit = X86DESC_LIMIT(Desc);
+            if (Desc.Gen.u1Granularity)
+                cbLimit = (cbLimit << PAGE_SHIFT) | PAGE_OFFSET_MASK;
+            if ((unsigned)(Sel & X86_SEL_MASK) + sizeof(X86DESC) - 1 > cbLimit)
+                return VERR_INVALID_SELECTOR;
+
+            /* calc the descriptor location. */
+            GCPtrDesc = X86DESC_BASE(Desc);
+            GCPtrDesc += (Sel & X86_SEL_MASK);
+        }
+
+        /* read the descriptor. */
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &Desc, GCPtrDesc, sizeof(Desc));
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    else
+    {
+        /*
+         * We're in real mode.
+         */
+        pSelInfo->Sel       = Sel;
+        pSelInfo->GCPtrBase = Sel << 4;
+        pSelInfo->cbLimit   = 0xffff;
+        pSelInfo->fFlags    = DBGFSELINFO_FLAGS_REAL_MODE;
+        pSelInfo->u.Raw64.au64[0] = 0;
+        pSelInfo->u.Raw64.au64[1] = 0;
+        pSelInfo->SelGate   = 0;
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Extract the base and limit or sel:offset for gates.
+     */
+    pSelInfo->Sel = Sel;
+    selmR3SelInfoFromDesc32(pSelInfo, &Desc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets information about a selector.
+ *
+ * Intended for the debugger mostly and will prefer the guest descriptor tables
+ * over the shadow ones.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
+ *          descriptor table.
+ * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
+ *          is not returned if the selector itself isn't present, you have to
+ *          check that for yourself (see DBGFSELINFO::fFlags).
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
+ *          pagetable or page backing the selector table wasn't present.
+ * @returns Other VBox status code on other errors.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       The virtual CPU handle.
+ * @param   Sel         The selector to get info about.
+ * @param   pSelInfo    Where to store the information.
+ */
+VMMR3DECL(int) SELMR3GetSelectorInfo(PVM pVM, PVMCPU pVCpu, RTSEL Sel, PDBGFSELINFO pSelInfo)
+{
+    AssertPtr(pSelInfo);
+    if (CPUMIsGuestInLongMode(pVCpu))
+        return selmR3GetSelectorInfo64(pVM, pVCpu, Sel, pSelInfo);
+    return selmR3GetSelectorInfo32(pVM, pVCpu, Sel, pSelInfo);
+}
+
+
+/**
+ * Gets information about a selector from the shadow tables.
+ *
+ * This is intended to be faster than the SELMR3GetSelectorInfo() method, but
+ * requires that the caller ensures that the shadow tables are up to date.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_INVALID_SELECTOR if the selector isn't fully inside the
+ *          descriptor table.
+ * @retval  VERR_SELECTOR_NOT_PRESENT if the LDT is invalid or not present. This
+ *          is not returned if the selector itself isn't present, you have to
+ *          check that for yourself (see DBGFSELINFO::fFlags).
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT or VERR_PAGE_NOT_PRESENT if the
+ *          pagetable or page backing the selector table wasn't present.
+ * @returns Other VBox status code on other errors.
+ *
+ * @param   pVM         VM handle.
+ * @param   Sel         The selector to get info about.
+ * @param   pSelInfo    Where to store the information.
+ *
+ * @remarks Don't use this when in hardware assisted virtualization mode.
+ */
+VMMR3DECL(int) SELMR3GetShadowSelectorInfo(PVM pVM, RTSEL Sel, PDBGFSELINFO pSelInfo)
+{
+    Assert(pSelInfo);
+
+    /*
+     * Read the descriptor entry
+     */
+    X86DESC Desc;
+    if (!(Sel & X86_SEL_LDT))
+    {
+        /*
+         * Global descriptor.
+         */
+        Desc = pVM->selm.s.paGdtR3[Sel >> X86_SEL_SHIFT];
+        pSelInfo->fFlags =    pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] == (Sel & X86_SEL_MASK)
+                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] == (Sel & X86_SEL_MASK)
+                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] == (Sel & X86_SEL_MASK)
+                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] == (Sel & X86_SEL_MASK)
+                           || pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] == (Sel & X86_SEL_MASK)
+                         ? DBGFSELINFO_FLAGS_HYPER
+                         : 0;
+        /** @todo check that the GDT offset is valid. */
+    }
+    else
+    {
+        /*
+         * Local Descriptor.
+         */
+        PX86DESC paLDT = (PX86DESC)((char *)pVM->selm.s.pvLdtR3 + pVM->selm.s.offLdtHyper);
+        Desc = paLDT[Sel >> X86_SEL_SHIFT];
+        /** @todo check if the LDT page is actually available. */
+        /** @todo check that the LDT offset is valid. */
+        pSelInfo->fFlags = 0;
+    }
+    if (CPUMIsGuestInProtectedMode(VMMGetCpu0(pVM)))
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_PROT_MODE;
+    else
+        pSelInfo->fFlags |= DBGFSELINFO_FLAGS_REAL_MODE;
+
+    /*
+     * Extract the base and limit or sel:offset for gates.
+     */
+    pSelInfo->Sel = Sel;
+    selmR3SelInfoFromDesc32(pSelInfo, &Desc);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Formats a descriptor.
+ *
+ * @param   Desc        Descriptor to format.
+ * @param   Sel         Selector number.
+ * @param   pszOutput   Output buffer.
+ * @param   cchOutput   Size of output buffer.
+ */
+static void selmR3FormatDescriptor(X86DESC Desc, RTSEL Sel, char *pszOutput, size_t cchOutput)
+{
+    /*
+     * Make variable description string.
+     */
+    static struct
+    {
+        unsigned    cch;
+        const char *psz;
+    } const aTypes[32] =
+    {
+#define STRENTRY(str) { sizeof(str) - 1, str }
+        /* system */
+        STRENTRY("Reserved0 "),                  /* 0x00 */
+        STRENTRY("TSS16Avail "),                 /* 0x01 */
+        STRENTRY("LDT "),                        /* 0x02 */
+        STRENTRY("TSS16Busy "),                  /* 0x03 */
+        STRENTRY("Call16 "),                     /* 0x04 */
+        STRENTRY("Task "),                       /* 0x05 */
+        STRENTRY("Int16 "),                      /* 0x06 */
+        STRENTRY("Trap16 "),                     /* 0x07 */
+        STRENTRY("Reserved8 "),                  /* 0x08 */
+        STRENTRY("TSS32Avail "),                 /* 0x09 */
+        STRENTRY("ReservedA "),                  /* 0x0a */
+        STRENTRY("TSS32Busy "),                  /* 0x0b */
+        STRENTRY("Call32 "),                     /* 0x0c */
+        STRENTRY("ReservedD "),                  /* 0x0d */
+        STRENTRY("Int32 "),                      /* 0x0e */
+        STRENTRY("Trap32 "),                     /* 0x0f */
+        /* non system */
+        STRENTRY("DataRO "),                     /* 0x10 */
+        STRENTRY("DataRO Accessed "),            /* 0x11 */
+        STRENTRY("DataRW "),                     /* 0x12 */
+        STRENTRY("DataRW Accessed "),            /* 0x13 */
+        STRENTRY("DataDownRO "),                 /* 0x14 */
+        STRENTRY("DataDownRO Accessed "),        /* 0x15 */
+        STRENTRY("DataDownRW "),                 /* 0x16 */
+        STRENTRY("DataDownRW Accessed "),        /* 0x17 */
+        STRENTRY("CodeEO "),                     /* 0x18 */
+        STRENTRY("CodeEO Accessed "),            /* 0x19 */
+        STRENTRY("CodeER "),                     /* 0x1a */
+        STRENTRY("CodeER Accessed "),            /* 0x1b */
+        STRENTRY("CodeConfEO "),                 /* 0x1c */
+        STRENTRY("CodeConfEO Accessed "),        /* 0x1d */
+        STRENTRY("CodeConfER "),                 /* 0x1e */
+        STRENTRY("CodeConfER Accessed ")         /* 0x1f */
+#undef SYSENTRY
+    };
+#define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
+    char        szMsg[128];
+    char       *psz = &szMsg[0];
+    unsigned    i = Desc.Gen.u1DescType << 4 | Desc.Gen.u4Type;
+    memcpy(psz, aTypes[i].psz, aTypes[i].cch);
+    psz += aTypes[i].cch;
+
+    if (Desc.Gen.u1Present)
+        ADD_STR(psz, "Present ");
+    else
+        ADD_STR(psz, "Not-Present ");
+    if (Desc.Gen.u1Granularity)
+        ADD_STR(psz, "Page ");
+    if (Desc.Gen.u1DefBig)
+        ADD_STR(psz, "32-bit ");
+    else
+        ADD_STR(psz, "16-bit ");
+#undef ADD_STR
+    *psz = '\0';
+
+    /*
+     * Limit and Base and format the output.
+     */
+    uint32_t    u32Limit = X86DESC_LIMIT(Desc);
+    if (Desc.Gen.u1Granularity)
+        u32Limit = u32Limit << PAGE_SHIFT | PAGE_OFFSET_MASK;
+    uint32_t    u32Base =  X86DESC_BASE(Desc);
+
+    RTStrPrintf(pszOutput, cchOutput, "%04x - %08x %08x - base=%08x limit=%08x dpl=%d %s",
+                Sel, Desc.au32[0], Desc.au32[1], u32Base, u32Limit, Desc.Gen.u2Dpl, szMsg);
+}
+
+
+/**
+ * Dumps a descriptor.
+ *
+ * @param   Desc    Descriptor to dump.
+ * @param   Sel     Selector number.
+ * @param   pszMsg  Message to prepend the log entry with.
+ */
+VMMR3DECL(void) SELMR3DumpDescriptor(X86DESC  Desc, RTSEL Sel, const char *pszMsg)
+{
+    char szOutput[128];
+    selmR3FormatDescriptor(Desc, Sel, &szOutput[0], sizeof(szOutput));
+    Log(("%s: %s\n", pszMsg, szOutput));
+    NOREF(szOutput[0]);
+}
+
+
+/**
+ * Display the shadow gdt.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) selmR3InfoGdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    pHlp->pfnPrintf(pHlp, "Shadow GDT (GCAddr=%RRv):\n", MMHyperR3ToRC(pVM, pVM->selm.s.paGdtR3));
+    for (unsigned iGDT = 0; iGDT < SELM_GDT_ELEMENTS; iGDT++)
+    {
+        if (pVM->selm.s.paGdtR3[iGDT].Gen.u1Present)
+        {
+            char szOutput[128];
+            selmR3FormatDescriptor(pVM->selm.s.paGdtR3[iGDT], iGDT << X86_SEL_SHIFT, &szOutput[0], sizeof(szOutput));
+            const char *psz = "";
+            if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS] >> X86_SEL_SHIFT))
+                psz = " HyperCS";
+            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_DS] >> X86_SEL_SHIFT))
+                psz = " HyperDS";
+            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_CS64] >> X86_SEL_SHIFT))
+                psz = " HyperCS64";
+            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS] >> X86_SEL_SHIFT))
+                psz = " HyperTSS";
+            else if (iGDT == ((unsigned)pVM->selm.s.aHyperSel[SELM_HYPER_SEL_TSS_TRAP08] >> X86_SEL_SHIFT))
+                psz = " HyperTSSTrap08";
+            pHlp->pfnPrintf(pHlp, "%s%s\n", szOutput, psz);
+        }
+    }
+}
+
+
+/**
+ * Display the guest gdt.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) selmR3InfoGdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /** @todo SMP support! */
+    PVMCPU      pVCpu = &pVM->aCpus[0];
+
+    VBOXGDTR    GDTR;
+    CPUMGetGuestGDTR(pVCpu, &GDTR);
+    RTGCPTR     GCPtrGDT = GDTR.pGdt;
+    unsigned    cGDTs = ((unsigned)GDTR.cbGdt + 1) / sizeof(X86DESC);
+
+    pHlp->pfnPrintf(pHlp, "Guest GDT (GCAddr=%RGv limit=%x):\n", GCPtrGDT, GDTR.cbGdt);
+    for (unsigned iGDT = 0; iGDT < cGDTs; iGDT++, GCPtrGDT += sizeof(X86DESC))
+    {
+        X86DESC GDTE;
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, &GDTE, GCPtrGDT, sizeof(GDTE));
+        if (RT_SUCCESS(rc))
+        {
+            if (GDTE.Gen.u1Present)
+            {
+                char szOutput[128];
+                selmR3FormatDescriptor(GDTE, iGDT << X86_SEL_SHIFT, &szOutput[0], sizeof(szOutput));
+                pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
+            }
+        }
+        else if (rc == VERR_PAGE_NOT_PRESENT)
+        {
+            if ((GCPtrGDT & PAGE_OFFSET_MASK) + sizeof(X86DESC) - 1 < sizeof(X86DESC))
+                pHlp->pfnPrintf(pHlp, "%04x - page not present (GCAddr=%RGv)\n", iGDT << X86_SEL_SHIFT, GCPtrGDT);
+        }
+        else
+            pHlp->pfnPrintf(pHlp, "%04x - read error rc=%Rrc GCAddr=%RGv\n", iGDT << X86_SEL_SHIFT, rc, GCPtrGDT);
+    }
+}
+
+
+/**
+ * Display the shadow ldt.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) selmR3InfoLdt(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    unsigned    cLDTs = ((unsigned)pVM->selm.s.cbLdtLimit + 1) >> X86_SEL_SHIFT;
+    PX86DESC    paLDT = (PX86DESC)((char *)pVM->selm.s.pvLdtR3 + pVM->selm.s.offLdtHyper);
+    pHlp->pfnPrintf(pHlp, "Shadow LDT (GCAddr=%RRv limit=%#x):\n", pVM->selm.s.pvLdtRC + pVM->selm.s.offLdtHyper, pVM->selm.s.cbLdtLimit);
+    for (unsigned iLDT = 0; iLDT < cLDTs; iLDT++)
+    {
+        if (paLDT[iLDT].Gen.u1Present)
+        {
+            char szOutput[128];
+            selmR3FormatDescriptor(paLDT[iLDT], (iLDT << X86_SEL_SHIFT) | X86_SEL_LDT, &szOutput[0], sizeof(szOutput));
+            pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
+        }
+    }
+}
+
+
+/**
+ * Display the guest ldt.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) selmR3InfoLdtGuest(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    /** @todo SMP support! */
+    PVMCPU      pVCpu = &pVM->aCpus[0];
+
+    RTSEL SelLdt = CPUMGetGuestLDTR(pVCpu);
+    if (!(SelLdt & X86_SEL_MASK))
+    {
+        pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x): Null-Selector\n", SelLdt);
+        return;
+    }
+
+    RTGCPTR     GCPtrLdt;
+    unsigned    cbLdt;
+    int rc = SELMGetLDTFromSel(pVM, SelLdt, &GCPtrLdt, &cbLdt);
+    if (RT_FAILURE(rc))
+    {
+        pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x): rc=%Rrc\n", SelLdt, rc);
+        return;
+    }
+
+    pHlp->pfnPrintf(pHlp, "Guest LDT (Sel=%x GCAddr=%RGv limit=%x):\n", SelLdt, GCPtrLdt, cbLdt);
+    unsigned    cLdts  = (cbLdt + 1) >> X86_SEL_SHIFT;
+    for (unsigned iLdt = 0; iLdt < cLdts; iLdt++, GCPtrLdt += sizeof(X86DESC))
+    {
+        X86DESC LdtE;
+        rc = PGMPhysSimpleReadGCPtr(pVCpu, &LdtE, GCPtrLdt, sizeof(LdtE));
+        if (RT_SUCCESS(rc))
+        {
+            if (LdtE.Gen.u1Present)
+            {
+                char szOutput[128];
+                selmR3FormatDescriptor(LdtE, (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, &szOutput[0], sizeof(szOutput));
+                pHlp->pfnPrintf(pHlp, "%s\n", szOutput);
+            }
+        }
+        else if (rc == VERR_PAGE_NOT_PRESENT)
+        {
+            if ((GCPtrLdt & PAGE_OFFSET_MASK) + sizeof(X86DESC) - 1 < sizeof(X86DESC))
+                pHlp->pfnPrintf(pHlp, "%04x - page not present (GCAddr=%RGv)\n", (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, GCPtrLdt);
+        }
+        else
+            pHlp->pfnPrintf(pHlp, "%04x - read error rc=%Rrc GCAddr=%RGv\n", (iLdt << X86_SEL_SHIFT) | X86_SEL_LDT, rc, GCPtrLdt);
+    }
+}
+
+
+/**
+ * Dumps the hypervisor GDT
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) SELMR3DumpHyperGDT(PVM pVM)
+{
+    DBGFR3Info(pVM, "gdt", NULL, NULL);
+}
+
+
+/**
+ * Dumps the hypervisor LDT
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) SELMR3DumpHyperLDT(PVM pVM)
+{
+    DBGFR3Info(pVM, "ldt", NULL, NULL);
+}
+
+
+/**
+ * Dumps the guest GDT
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) SELMR3DumpGuestGDT(PVM pVM)
+{
+    DBGFR3Info(pVM, "gdtguest", NULL, NULL);
+}
+
+
+/**
+ * Dumps the guest LDT
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) SELMR3DumpGuestLDT(PVM pVM)
+{
+    DBGFR3Info(pVM, "ldtguest", NULL, NULL);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/SSM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/SSM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/SSM.cpp	(revision 35333)
@@ -0,0 +1,9101 @@
+/* $Id$ */
+/** @file
+ * SSM - Saved State Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_ssm        SSM - The Saved State Manager
+ *
+ * The Saved State Manager (SSM) implements facilities for saving and loading a
+ * VM state in a structural manner using callbacks for named data units.
+ *
+ * At init time each of the VMM components, Devices, Drivers and one or two
+ * other things will register data units which they need to save and restore.
+ * Each unit have a unique name (ascii), instance number, and a set of callbacks
+ * associated with it.  The name will be used to identify the unit during
+ * restore.  The callbacks are for the two operations, save and restore.  There
+ * are three callbacks for each of the two - a prepare, a execute and a complete
+ * - giving each component ample opportunity to perform actions both before and
+ * afterwards.
+ *
+ * The SSM provides a number of APIs for encoding and decoding the data: @see
+ * grp_ssm
+ *
+ *
+ *
+ * @section sec_ssm_live_snapshots  Live Snapshots
+ *
+ * The live snapshots feature (LS) is similar to teleportation (TP) and was a
+ * natural first step when implementing TP.  The main differences between LS and
+ * TP are that after a live snapshot we will have a saved state file, disk image
+ * snapshots, and the VM will still be running.
+ *
+ * Compared to normal saved stated and snapshots, the difference is in that the
+ * VM is running while we do most of the saving.  Prior to LS, there was only
+ * one round of callbacks during saving and the VM was paused during it.  With
+ * LS there are 1 or more passes while the VM is still running and a final one
+ * after it has been paused.  The runtime passes are executed on a dedicated
+ * thread running at at the same priority as the EMTs so that the saving doesn't
+ * starve or lose in scheduling questions (note: not implemented yet). The final
+ * pass is done on EMT(0).
+ *
+ * There are a couple of common reasons why LS and TP will fail:
+ *   - Memory configuration changed (PCI memory mappings).
+ *   - Takes too long (TP) / Too much output (LS).
+ *
+ *
+ * The live saving sequence is something like this:
+ *
+ *      -# SSMR3LiveSave is called on EMT0.  It returns a saved state
+ *         handle.
+ *      -# SSMR3LiveDoStep1 is called on a non-EMT.  This will save the major
+ *         parts of the state while the VM may still be running.
+ *      -# The VM is suspended.
+ *      -# SSMR3LiveDoStep2 is called on EMT0 to save the remainder of the state
+ *         in the normal way.
+ *      -# The client does any necessary reconfiguration of harddisks and
+ *         similar.
+ *      -# SSMR3LiveDone is called on EMT0 to close the handle.
+ *      -# The VM is resumed or powered off and destroyed.
+ *
+ *
+ * @section sec_ssm_teleportation   Teleportation
+ *
+ * As mentioned in the previous section, the main differences between this and
+ * live snapshots are in where the saved state is written and what state the
+ * local VM is in afterwards - at least from the VMM point of view.  The
+ * necessary administrative work - establishing the connection to the remote
+ * machine, cloning the VM config on it and doing lowlevel saved state data
+ * transfer - is taken care of by layer above the VMM (i.e.  Main).
+ *
+ * The SSM data format was made streamable for the purpose of teleportation
+ * (v1.2 was the last non-streamable version).
+ *
+ *
+ * @section sec_ssm_format          Saved State Format
+ *
+ * The stream format starts with a header (SSMFILEHDR) that indicates the
+ * version and such things, it is followed by zero or more saved state units
+ * (name + instance + pass), and the stream concludes with a footer
+ * (SSMFILEFTR) that contains unit counts and optionally a checksum for the
+ * entire file.  (In version 1.2 and earlier, the checksum was in the header and
+ * there was no footer.  This meant that the header was updated after the entire
+ * file was written.)
+ *
+ * The saved state units each starts with a variable sized header
+ * (SSMFILEUNITHDRV2) that contains the name, instance and pass.  The data
+ * follows the header and is encoded as records with a 2-8 byte record header
+ * indicating the type, flags and size.  The first byte in the record header
+ * indicates the type and flags:
+ *
+ *   - bits 0..3: Record type:
+ *       - type 0: Invalid.
+ *       - type 1: Terminator with CRC-32 and unit size.
+ *       - type 2: Raw data record.
+ *       - type 3: Raw data compressed by LZF. The data is prefixed by a 8-bit
+ *                 field containing the length of the uncompressed data given in
+ *                 1KB units.
+ *       - type 4: Zero data. The record header is followed by a 8-bit field
+ *                 counting the length of the zero data given in 1KB units.
+ *       - type 5: Named data - length prefixed name followed by the data. This
+ *                 type is not implemented yet as we're missing the API part, so
+ *                 the type assignment is tentative.
+ *       - types 6 thru 15 are current undefined.
+ *   - bit 4: Important (set), can be skipped (clear).
+ *   - bit 5: Undefined flag, must be zero.
+ *   - bit 6: Undefined flag, must be zero.
+ *   - bit 7: "magic" bit, always set.
+ *
+ * Record header byte 2 (optionally thru 7) is the size of the following data
+ * encoded in UTF-8 style.  To make buffering simpler and more efficient during
+ * the save operation, the strict checks enforcing optimal encoding has been
+ * relaxed for the 2 and 3 byte encodings.
+ *
+ * (In version 1.2 and earlier the unit data was compressed and not record
+ * based. The unit header contained the compressed size of the data, i.e. it
+ * needed updating after the data was written.)
+ *
+ *
+ * @section sec_ssm_future          Future Changes
+ *
+ * There are plans to extend SSM to make it easier to be both backwards and
+ * (somewhat) forwards compatible.  One of the new features will be being able
+ * to classify units and data items as unimportant (added to the format in
+ * v2.0).  Another suggested feature is naming data items (also added to the
+ * format in v2.0), perhaps by extending the SSMR3PutStruct API.  Both features
+ * will require API changes, the naming may possibly require both buffering of
+ * the stream as well as some helper managing them.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_SSM
+#include <VBox/ssm.h>
+#include <VBox/dbgf.h>
+#include <VBox/mm.h>
+#include "SSMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/version.h>
+
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/crc.h>
+#include <iprt/file.h>
+#include <iprt/mem.h>
+#include <iprt/param.h>
+#include <iprt/thread.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/uuid.h>
+#include <iprt/zip.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The max length of a unit name. */
+#define SSM_MAX_NAME_SIZE                       48
+
+/** Saved state file magic base string. */
+#define SSMFILEHDR_MAGIC_BASE                   "\177VirtualBox SavedState "
+/** Saved state file magic indicating version 1.x. */
+#define SSMFILEHDR_MAGIC_V1_X                   "\177VirtualBox SavedState V1."
+/** Saved state file v1.1 magic. */
+#define SSMFILEHDR_MAGIC_V1_1                   "\177VirtualBox SavedState V1.1\n"
+/** Saved state file v1.2 magic. */
+#define SSMFILEHDR_MAGIC_V1_2                   "\177VirtualBox SavedState V1.2\n\0\0\0"
+/** Saved state file v2.0 magic. */
+#define SSMFILEHDR_MAGIC_V2_0                   "\177VirtualBox SavedState V2.0\n\0\0\0"
+
+/** @name SSMFILEHDR::fFlags
+ * @{ */
+/** The stream is checksummed up to the footer using CRC-32. */
+#define SSMFILEHDR_FLAGS_STREAM_CRC32           RT_BIT_32(0)
+/** Indicates that the file was produced by a live save. */
+#define SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE       RT_BIT_32(1)
+/** @} */
+
+/** The directory magic. */
+#define SSMFILEDIR_MAGIC                        "\nDir\n\0\0"
+
+/** Saved state file v2.0 magic. */
+#define SSMFILEFTR_MAGIC                        "\nFooter"
+
+/** Data unit magic. */
+#define SSMFILEUNITHDR_MAGIC                    "\nUnit\n\0"
+/** Data end marker magic. */
+#define SSMFILEUNITHDR_END                      "\nTheEnd"
+
+
+/** @name Record Types (data unit)
+ * @{ */
+/** The record type mask. */
+#define SSM_REC_TYPE_MASK                       UINT8_C(0x0f)
+/** Invalid record. */
+#define SSM_REC_TYPE_INVALID                    0
+/** Normal termination record, see SSMRECTERM. */
+#define SSM_REC_TYPE_TERM                       1
+/** Raw data. The data follows the size field without further ado. */
+#define SSM_REC_TYPE_RAW                        2
+/** Raw data compressed by LZF.
+ * The record header is followed by a 8-bit field containing the size of the
+ * uncompressed data in 1KB units.  The compressed data is after it. */
+#define SSM_REC_TYPE_RAW_LZF                    3
+/** Raw zero data.
+ * The record header is followed by a 8-bit field containing the size of the
+ * zero data in 1KB units. */
+#define SSM_REC_TYPE_RAW_ZERO                   4
+/** Named data items.
+ * A length prefix zero terminated string (i.e. max 255) followed by the data.  */
+#define SSM_REC_TYPE_NAMED                      5
+/** Macro for validating the record type.
+ * This can be used with the flags+type byte, no need to mask out the type first. */
+#define SSM_REC_TYPE_IS_VALID(u8Type)           (   ((u8Type) & SSM_REC_TYPE_MASK) >  SSM_REC_TYPE_INVALID \
+                                                 && ((u8Type) & SSM_REC_TYPE_MASK) <= SSM_REC_TYPE_NAMED )
+/** @} */
+
+/** The flag mask. */
+#define SSM_REC_FLAGS_MASK                      UINT8_C(0xf0)
+/** The record is important if this flag is set, if clear it can be omitted. */
+#define SSM_REC_FLAGS_IMPORTANT                 UINT8_C(0x10)
+/** This flag is always set. */
+#define SSM_REC_FLAGS_FIXED                     UINT8_C(0x80)
+/** Macro for validating the flags.
+ * No need to mask the flags out of the flags+type byte before invoking this macro.  */
+#define SSM_REC_FLAGS_ARE_VALID(fFlags)         ( ((fFlags) & UINT8_C(0xe0)) == UINT8_C(0x80) )
+
+/** Macro for validating the type and flags byte in a data record. */
+#define SSM_REC_ARE_TYPE_AND_FLAGS_VALID(u8)    ( SSM_REC_FLAGS_ARE_VALID(u8) && SSM_REC_TYPE_IS_VALID(u8) )
+
+/** @name SSMRECTERM::fFlags
+ * @{ */
+/** There is a CRC-32 value for the stream. */
+#define SSMRECTERM_FLAGS_CRC32                  UINT16_C(0x0001)
+/** @} */
+
+/** Start structure magic. (Isaac Asimov) */
+#define SSMR3STRUCT_BEGIN                       UINT32_C(0x19200102)
+/** End structure magic. (Isaac Asimov) */
+#define SSMR3STRUCT_END                         UINT32_C(0x19920406)
+
+
+/** Number of bytes to log in Log2 and Log4 statements. */
+#define SSM_LOG_BYTES                           16
+
+/** SSMHANDLE::fCancelled value indicating that the operation has been
+ *  cancelled. */
+#define SSMHANDLE_CANCELLED                     UINT32_C(0xdeadbeef)
+/** SSMHANDLE::fCancelled value indicating no cancellation. */
+#define SSMHANDLE_OK                            UINT32_C(0x77777777)
+
+
+/** Macro for checking the u32CRC field of a structure.
+ * The Msg can assume there are  u32ActualCRC and u32CRC in the context. */
+#define SSM_CHECK_CRC32_RET(p, cb, Msg) \
+    do \
+    { \
+        uint32_t u32CRC = (p)->u32CRC; \
+        (p)->u32CRC = 0; \
+        uint32_t u32ActualCRC = RTCrc32((p), (cb)); \
+        (p)->u32CRC = u32CRC; \
+        AssertLogRelMsgReturn(u32ActualCRC == u32CRC, Msg, VERR_SSM_INTEGRITY_CRC); \
+    } while (0)
+
+/** The number of bytes to compress is one block.
+ * Must be a multiple of 1KB.  */
+#define SSM_ZIP_BLOCK_SIZE                      _4K
+AssertCompile(SSM_ZIP_BLOCK_SIZE / _1K * _1K == SSM_ZIP_BLOCK_SIZE);
+
+
+/**
+ * Asserts that the handle is writable and returns with VERR_SSM_INVALID_STATE
+ * if it isn't.
+ */
+#define SSM_ASSERT_WRITEABLE_RET(pSSM) \
+    AssertMsgReturn(   pSSM->enmOp == SSMSTATE_SAVE_EXEC \
+                    || pSSM->enmOp == SSMSTATE_LIVE_EXEC,\
+                    ("Invalid state %d\n", pSSM->enmOp), VERR_SSM_INVALID_STATE);
+
+/**
+ * Asserts that the handle is readable and returns with VERR_SSM_INVALID_STATE
+ * if it isn't.
+ */
+#define SSM_ASSERT_READABLE_RET(pSSM) \
+    AssertMsgReturn(   pSSM->enmOp == SSMSTATE_LOAD_EXEC \
+                    || pSSM->enmOp == SSMSTATE_OPEN_READ,\
+                    ("Invalid state %d\n", pSSM->enmOp), VERR_SSM_INVALID_STATE);
+
+/** Checks for cancellation and returns if pending.
+ * Sets SSMHANDLE::rc to VERR_SSM_CANCELLED (if it still indicates success) and
+ * then returns SSMHANDLE::rc. (Debug logging only.) */
+#define SSM_CHECK_CANCELLED_RET(pSSM) \
+    do \
+    { \
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED)) \
+        { \
+            LogFlow(("%Rfn: Cancelled -> VERR_SSM_CANCELLED\n", __PRETTY_FUNCTION__)); \
+            if (RT_SUCCESS((pSSM)->rc)) \
+                (pSSM)->rc = VERR_SSM_CANCELLED; \
+            return (pSSM)->rc; \
+        } \
+    } while (0)
+
+/**
+ * Asserts that the handle is somewhat valid.  No returns as this is just a
+ * simple safeguard for catching bad API calls. */
+#define SSM_ASSERT_VALID_HANDLE(pSSM) \
+    do \
+    { \
+        AssertPtr(pSSM); \
+        Assert(pSSM->enmOp > SSMSTATE_INVALID && pSSM->enmOp < SSMSTATE_END); \
+    } while (0)
+
+
+/** @def SSM_HOST_IS_MSC_32
+ * Set to 1 if the host is 32-bit MSC, otherwise set to 0.
+ *   */
+#if defined(_MSC_VER) && HC_ARCH_BITS == 32
+# define SSM_HOST_IS_MSC_32     1
+#else
+# define SSM_HOST_IS_MSC_32     0
+#endif
+
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/** SSM state. */
+typedef enum SSMSTATE
+{
+    SSMSTATE_INVALID = 0,
+    SSMSTATE_LIVE_PREP,
+    SSMSTATE_LIVE_STEP1,
+    SSMSTATE_LIVE_EXEC,
+    SSMSTATE_LIVE_VOTE,
+    SSMSTATE_LIVE_STEP2,
+    SSMSTATE_SAVE_PREP,
+    SSMSTATE_SAVE_EXEC,
+    SSMSTATE_SAVE_DONE,
+    SSMSTATE_LOAD_PREP,
+    SSMSTATE_LOAD_EXEC,
+    SSMSTATE_LOAD_DONE,
+    SSMSTATE_OPEN_READ,
+    SSMSTATE_END
+} SSMSTATE;
+
+
+/** Pointer to a SSM stream buffer. */
+typedef struct SSMSTRMBUF *PSSMSTRMBUF;
+/**
+ * A SSM stream buffer.
+ */
+typedef struct SSMSTRMBUF
+{
+    /** The buffer data. */
+    uint8_t                 abData[_64K];
+
+    /** The stream position of this buffer. */
+    uint64_t                offStream;
+    /** The amount of buffered data. */
+    uint32_t                cb;
+    /** End of stream indicator (for read streams only). */
+    bool                    fEndOfStream;
+    /** The nano timestamp set by ssmR3StrmGetFreeBuf. */
+    uint64_t                NanoTS;
+    /** Pointer to the next buffer in the chain. */
+    PSSMSTRMBUF volatile    pNext;
+} SSMSTRMBUF;
+
+/**
+ * SSM stream.
+ *
+ * This is a typical producer / consumer setup with a dedicated I/O thread and
+ * fixed number of buffers for read ahead and write back.
+ */
+typedef struct SSMSTRM
+{
+    /** The stream method table. */
+    PCSSMSTRMOPS            pOps;
+    /** The user argument for the stream methods.
+     * For file based streams, this is the file handle and not a pointer. */
+    void                   *pvUser;
+
+    /** Write (set) or read (clear) stream. */
+    bool                    fWrite;
+    /** Termination indicator. */
+    bool volatile           fTerminating;
+    /** Indicates whether it is necessary to seek before the next buffer is
+     *  read from the stream.  This is used to avoid a seek in ssmR3StrmPeekAt. */
+    bool                    fNeedSeek;
+    /** Stream error status. */
+    int32_t volatile        rc;
+    /** The handle of the I/O thread. This is set to nil when not active. */
+    RTTHREAD                hIoThread;
+    /** Where to seek to. */
+    uint64_t                offNeedSeekTo;
+
+    /** The head of the consumer queue.
+     * For save the consumer is the I/O thread.  For load the I/O thread is the
+     * producer. */
+    PSSMSTRMBUF volatile    pHead;
+    /** Chain of free buffers.
+     * The consumer/producer roles are the inverse of pHead.  */
+    PSSMSTRMBUF volatile    pFree;
+    /** Event that's signalled when pHead is updated. */
+    RTSEMEVENT              hEvtHead;
+    /** Event that's signalled when pFree is updated. */
+    RTSEMEVENT              hEvtFree;
+
+    /** List of pending buffers that has been dequeued from pHead and reversed. */
+    PSSMSTRMBUF             pPending;
+    /** Pointer to the current buffer. */
+    PSSMSTRMBUF             pCur;
+    /** The stream offset of the current buffer. */
+    uint64_t                offCurStream;
+    /** The current buffer offset. */
+    uint32_t                off;
+    /** Whether we're checksumming reads/writes. */
+    bool                    fChecksummed;
+    /** The stream CRC if fChecksummed is set. */
+    uint32_t                u32StreamCRC;
+    /** How far into the buffer u32StreamCRC is up-to-date.
+     * This may lag behind off as it's desirable to checksum as large blocks as
+     * possible.  */
+    uint32_t                offStreamCRC;
+} SSMSTRM;
+/** Pointer to a SSM stream. */
+typedef SSMSTRM *PSSMSTRM;
+
+
+/**
+ * Handle structure.
+ */
+typedef struct SSMHANDLE
+{
+    /** Stream/buffer manager. */
+    SSMSTRM                 Strm;
+
+    /** The VM handle. */
+    PVM                     pVM;
+    /** The current operation. */
+    SSMSTATE                enmOp;
+    /** What to do after save completes. (move the enum) */
+    SSMAFTER                enmAfter;
+    /** Flag indicating that the operation has been cancelled. */
+    uint32_t volatile       fCancelled;
+    /** The current rc of the save operation. */
+    int32_t                 rc;
+    /** Number of compressed bytes left in the current data unit (V1). */
+    uint64_t                cbUnitLeftV1;
+    /** The current uncompressed offset into the data unit. */
+    uint64_t                offUnit;
+    /** Indicates that this is a live save or restore operation. */
+    bool                    fLiveSave;
+
+    /** Pointer to the progress callback function. */
+    PFNVMPROGRESS           pfnProgress;
+    /** User specified argument to the callback function. */
+    void                   *pvUser;
+    /** Next completion percentage. (corresponds to offEstProgress) */
+    unsigned                uPercent;
+    /** The position of the next progress callback in the estimated file. */
+    uint64_t                offEstProgress;
+    /** The estimated total byte count.
+     * (Only valid after the prep.) */
+    uint64_t                cbEstTotal;
+    /** Current position in the estimated file. */
+    uint64_t                offEst;
+    /** End of current unit in the estimated file. */
+    uint64_t                offEstUnitEnd;
+    /** The amount of % we reserve for the 'live' stage */
+    unsigned                uPercentLive;
+    /** The amount of % we reserve for the 'prepare' phase */
+    unsigned                uPercentPrepare;
+    /** The amount of % we reserve for the 'done' stage */
+    unsigned                uPercentDone;
+    /** The lowest value reported via SSMR3HandleReportLivePercent during one
+     * vote run. */
+    unsigned                uReportedLivePercent;
+    /** The filename, NULL if remote stream. */
+    const char             *pszFilename;
+
+    union
+    {
+        /** Write data. */
+        struct
+        {
+            /** Offset into the databuffer. */
+            uint32_t        offDataBuffer;
+            /** Space for the record header.  */
+            uint8_t         abRecHdr[1+7];
+            /** Data buffer. */
+            uint8_t         abDataBuffer[4096];
+            /** The maximum downtime given as milliseconds. */
+            uint32_t        cMsMaxDowntime;
+        } Write;
+
+        /** Read data. */
+        struct
+        {
+            /** V1: The decompressor of the current data unit. */
+            PRTZIPDECOMP    pZipDecompV1;
+            /** The major format version number. */
+            uint32_t        uFmtVerMajor;
+            /** The minor format version number. */
+            uint32_t        uFmtVerMinor;
+
+            /** V2: Unread bytes in the current record. */
+            uint32_t        cbRecLeft;
+            /** V2: Bytes in the data buffer. */
+            uint32_t        cbDataBuffer;
+            /** V2: Current buffer position. */
+            uint32_t        offDataBuffer;
+            /** V2: End of data indicator. */
+            bool            fEndOfData;
+            /** V2: The type and flags byte fo the current record. */
+            uint8_t         u8TypeAndFlags;
+
+            /** @name Context info for SSMR3SetLoadError.
+             * @{  */
+            /** Pointer to the header for the current unit. */
+            PSSMUNIT        pCurUnit;
+            /** The version of the current unit if in the load exec stage. */
+            uint32_t        uCurUnitVer;
+            /** The pass number of the current unit if in the load exec stage. */
+            uint32_t        uCurUnitPass;
+            /** Whether SSMR3SetLoadError[V] has been called.
+             * @note Using ASMAtomicXchgBool because I'm very lazy. */
+            bool volatile   fHaveSetError;
+            /** @} */
+
+            /** RTGCPHYS size in bytes. (Only applicable when loading/reading.) */
+            unsigned        cbGCPhys;
+            /** RTGCPTR size in bytes. (Only applicable when loading/reading.) */
+            unsigned        cbGCPtr;
+            /** Whether cbGCPtr is fixed or settable. */
+            bool            fFixedGCPtrSize;
+
+            /** 32-bit MSC saved this? */
+            bool            fIsHostMsc32;
+            /** "Host OS" dot "architecture", picked up from recent SSM data units. */
+            char            szHostOSAndArch[32];
+
+            /** @name Header info (set by ssmR3ValidateFile)
+             * @{ */
+            /** The size of the file header. */
+            uint32_t        cbFileHdr;
+            /** The major version number. */
+            uint16_t        u16VerMajor;
+            /** The minor version number. */
+            uint16_t        u16VerMinor;
+            /** The build number. */
+            uint32_t        u32VerBuild;
+            /** The SVN revision. */
+            uint32_t        u32SvnRev;
+            /** 32 or 64 depending on the host. */
+            uint8_t         cHostBits;
+            /** Whether the stream is checksummed (SSMFILEHDR_FLAGS_STREAM_CRC32). */
+            bool            fStreamCrc32;
+            /** The CRC of the loaded file. */
+            uint32_t        u32LoadCRC;
+            /** The size of the load file. */
+            uint64_t        cbLoadFile;
+            /** @} */
+
+            /** V2: Data buffer.
+             * @remarks Be extremely careful when changing the size of this buffer! */
+            uint8_t         abDataBuffer[4096];
+
+            /** V2: Decompression buffer for when we cannot use the stream buffer. */
+            uint8_t         abComprBuffer[4096];
+        } Read;
+    } u;
+} SSMHANDLE;
+
+
+/**
+ * Header of the saved state file.
+ *
+ * Added in r5xxxx on 2009-07-2?, VirtualBox v3.0.51.
+ */
+typedef struct SSMFILEHDR
+{
+    /** Magic string which identifies this file as a version of VBox saved state
+     *  file format (SSMFILEHDR_MAGIC_V2_0). */
+    char            szMagic[32];
+    /** The major version number. */
+    uint16_t        u16VerMajor;
+    /** The minor version number. */
+    uint16_t        u16VerMinor;
+    /** The build number. */
+    uint32_t        u32VerBuild;
+    /** The SVN revision. */
+    uint32_t        u32SvnRev;
+    /** 32 or 64 depending on the host. */
+    uint8_t         cHostBits;
+    /** The size of RTGCPHYS. */
+    uint8_t         cbGCPhys;
+    /** The size of RTGCPTR. */
+    uint8_t         cbGCPtr;
+    /** Reserved header space - must be zero. */
+    uint8_t         u8Reserved;
+    /** The number of units that (may) have stored data in the file. */
+    uint32_t        cUnits;
+    /** Flags, see SSMFILEHDR_FLAGS_XXX.  */
+    uint32_t        fFlags;
+    /** The maximum size of decompressed data. */
+    uint32_t        cbMaxDecompr;
+    /** The checksum of this header.
+     * This field is set to zero when calculating the checksum. */
+    uint32_t        u32CRC;
+} SSMFILEHDR;
+AssertCompileSize(SSMFILEHDR, 64);
+AssertCompileMemberOffset(SSMFILEHDR, u32CRC, 60);
+AssertCompileMemberSize(SSMFILEHDR, szMagic, sizeof(SSMFILEHDR_MAGIC_V2_0));
+/** Pointer to a saved state file header. */
+typedef SSMFILEHDR *PSSMFILEHDR;
+/** Pointer to a const saved state file header. */
+typedef SSMFILEHDR const *PCSSMFILEHDR;
+
+
+/**
+ * Header of the saved state file.
+ *
+ * Added in r40980 on 2008-12-15, VirtualBox v2.0.51.
+ *
+ * @remarks This is a superset of SSMFILEHDRV11.
+ */
+typedef struct SSMFILEHDRV12
+{
+    /** Magic string which identifies this file as a version of VBox saved state
+     *  file format (SSMFILEHDR_MAGIC_V1_2). */
+    char            achMagic[32];
+    /** The size of this file. Used to check
+     * whether the save completed and that things are fine otherwise. */
+    uint64_t        cbFile;
+    /** File checksum. The actual calculation skips past the u32CRC field. */
+    uint32_t        u32CRC;
+    /** Padding. */
+    uint32_t        u32Reserved;
+    /** The machine UUID. (Ignored if NIL.) */
+    RTUUID          MachineUuid;
+
+    /** The major version number. */
+    uint16_t        u16VerMajor;
+    /** The minor version number. */
+    uint16_t        u16VerMinor;
+    /** The build number. */
+    uint32_t        u32VerBuild;
+    /** The SVN revision. */
+    uint32_t        u32SvnRev;
+
+    /** 32 or 64 depending on the host. */
+    uint8_t         cHostBits;
+    /** The size of RTGCPHYS. */
+    uint8_t         cbGCPhys;
+    /** The size of RTGCPTR. */
+    uint8_t         cbGCPtr;
+    /** Padding. */
+    uint8_t         au8Reserved;
+} SSMFILEHDRV12;
+AssertCompileSize(SSMFILEHDRV12, 64+16);
+AssertCompileMemberOffset(SSMFILEHDRV12, u32CRC, 40);
+AssertCompileMemberSize(SSMFILEHDRV12, achMagic, sizeof(SSMFILEHDR_MAGIC_V1_2));
+/** Pointer to a saved state file header. */
+typedef SSMFILEHDRV12 *PSSMFILEHDRV12;
+
+
+/**
+ * Header of the saved state file, version 1.1.
+ *
+ * Added in r23677 on 2007-08-17, VirtualBox v1.4.1.
+ */
+typedef struct SSMFILEHDRV11
+{
+    /** Magic string which identifies this file as a version of VBox saved state
+     *  file format (SSMFILEHDR_MAGIC_V1_1). */
+    char            achMagic[32];
+    /** The size of this file. Used to check
+     * whether the save completed and that things are fine otherwise. */
+    uint64_t        cbFile;
+    /** File checksum. The actual calculation skips past the u32CRC field. */
+    uint32_t        u32CRC;
+    /** Padding. */
+    uint32_t        u32Reserved;
+    /** The machine UUID. (Ignored if NIL.) */
+    RTUUID          MachineUuid;
+} SSMFILEHDRV11;
+AssertCompileSize(SSMFILEHDRV11, 64);
+AssertCompileMemberOffset(SSMFILEHDRV11, u32CRC, 40);
+/** Pointer to a saved state file header. */
+typedef SSMFILEHDRV11 *PSSMFILEHDRV11;
+
+
+/**
+ * Data unit header.
+ */
+typedef struct SSMFILEUNITHDRV2
+{
+    /** Magic (SSMFILEUNITHDR_MAGIC or SSMFILEUNITHDR_END). */
+    char            szMagic[8];
+    /** The offset in the saved state stream of the start of this unit.
+     * This is mainly intended for sanity checking. */
+    uint64_t        offStream;
+    /** The CRC-in-progress value this unit starts at. */
+    uint32_t        u32CurStreamCRC;
+    /** The checksum of this structure, including the whole name.
+     * Calculated with this field set to zero.  */
+    uint32_t        u32CRC;
+    /** Data version. */
+    uint32_t        u32Version;
+    /** Instance number. */
+    uint32_t        u32Instance;
+    /** Data pass number. */
+    uint32_t        u32Pass;
+    /** Flags reserved for future extensions. Must be zero. */
+    uint32_t        fFlags;
+    /** Size of the data unit name including the terminator. (bytes) */
+    uint32_t        cbName;
+    /** Data unit name, variable size. */
+    char            szName[SSM_MAX_NAME_SIZE];
+} SSMFILEUNITHDRV2;
+AssertCompileMemberOffset(SSMFILEUNITHDRV2, szName, 44);
+AssertCompileMemberSize(SSMFILEUNITHDRV2, szMagic, sizeof(SSMFILEUNITHDR_MAGIC));
+AssertCompileMemberSize(SSMFILEUNITHDRV2, szMagic, sizeof(SSMFILEUNITHDR_END));
+/** Pointer to SSMFILEUNITHDRV2.  */
+typedef SSMFILEUNITHDRV2 *PSSMFILEUNITHDRV2;
+
+
+/**
+ * Data unit header.
+ *
+ * This is used by v1.0, v1.1 and v1.2 of the format.
+ */
+typedef struct SSMFILEUNITHDRV1
+{
+    /** Magic (SSMFILEUNITHDR_MAGIC or SSMFILEUNITHDR_END). */
+    char            achMagic[8];
+    /** Number of bytes in this data unit including the header. */
+    uint64_t        cbUnit;
+    /** Data version. */
+    uint32_t        u32Version;
+    /** Instance number. */
+    uint32_t        u32Instance;
+    /** Size of the data unit name including the terminator. (bytes) */
+    uint32_t        cchName;
+    /** Data unit name. */
+    char            szName[1];
+} SSMFILEUNITHDRV1;
+/** Pointer to SSMFILEUNITHDR.  */
+typedef SSMFILEUNITHDRV1 *PSSMFILEUNITHDRV1;
+
+
+/**
+ * Termination data record.
+ */
+typedef struct SSMRECTERM
+{
+    uint8_t         u8TypeAndFlags;
+    /** The record size (sizeof(SSMRECTERM) - 2). */
+    uint8_t         cbRec;
+    /** Flags, see SSMRECTERM_FLAGS_CRC32. */
+    uint16_t        fFlags;
+    /** The checksum of the stream up to fFlags (exclusive). */
+    uint32_t        u32StreamCRC;
+    /** The length of this data unit in bytes (including this record). */
+    uint64_t        cbUnit;
+} SSMRECTERM;
+AssertCompileSize(SSMRECTERM, 16);
+AssertCompileMemberAlignment(SSMRECTERM, cbUnit, 8);
+/** Pointer to a termination record. */
+typedef SSMRECTERM *PSSMRECTERM;
+/** Pointer to a const termination record. */
+typedef SSMRECTERM const *PCSSMRECTERM;
+
+
+/**
+ * Directory entry.
+ */
+typedef struct SSMFILEDIRENTRY
+{
+    /** The offset of the data unit. */
+    uint64_t        off;
+    /** The instance number. */
+    uint32_t        u32Instance;
+    /** The CRC-32 of the name excluding the terminator. (lazy bird) */
+    uint32_t        u32NameCRC;
+} SSMFILEDIRENTRY;
+AssertCompileSize(SSMFILEDIRENTRY, 16);
+/** Pointer to a directory entry. */
+typedef SSMFILEDIRENTRY *PSSMFILEDIRENTRY;
+/** Pointer to a const directory entry. */
+typedef SSMFILEDIRENTRY const *PCSSMFILEDIRENTRY;
+
+/**
+ * Directory for the data units from the final pass.
+ *
+ * This is used to speed up SSMR3Seek (it would have to decompress and parse the
+ * whole stream otherwise).
+ */
+typedef struct SSMFILEDIR
+{
+    /** Magic string (SSMFILEDIR_MAGIC). */
+    char            szMagic[8];
+    /** The CRC-32 for the whole directory.
+     * Calculated with this field set to zero. */
+    uint32_t        u32CRC;
+    /** The number of directory entries. */
+    uint32_t        cEntries;
+    /** The directory entries (variable size). */
+    SSMFILEDIRENTRY aEntries[1];
+} SSMFILEDIR;
+AssertCompileSize(SSMFILEDIR, 32);
+/** Pointer to a directory. */
+typedef SSMFILEDIR *PSSMFILEDIR;
+/** Pointer to a const directory. */
+typedef SSMFILEDIR *PSSMFILEDIR;
+
+
+/**
+ * Footer structure
+ */
+typedef struct SSMFILEFTR
+{
+    /** Magic string (SSMFILEFTR_MAGIC). */
+    char            szMagic[8];
+    /** The offset of this record in the stream. */
+    uint64_t        offStream;
+    /** The CRC for the stream.
+     * This is set to zero if SSMFILEHDR_FLAGS_STREAM_CRC32 is clear. */
+    uint32_t        u32StreamCRC;
+    /** Number directory entries. */
+    uint32_t        cDirEntries;
+    /** Reserved footer space - must be zero. */
+    uint32_t        u32Reserved;
+    /** The CRC-32 for this structure.
+     * Calculated with this field set to zero. */
+    uint32_t        u32CRC;
+} SSMFILEFTR;
+AssertCompileSize(SSMFILEFTR, 32);
+/** Pointer to a footer. */
+typedef SSMFILEFTR *PSSMFILEFTR;
+/** Pointer to a const footer. */
+typedef SSMFILEFTR const *PCSSMFILEFTR;
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/** Zeros used by the struct putter.
+ * This must be at least 8 bytes or the code breaks. */
+static uint8_t const    g_abZero[_1K] = {0};
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+#ifndef SSM_STANDALONE
+static int                  ssmR3LazyInit(PVM pVM);
+static DECLCALLBACK(int)    ssmR3SelfLiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass);
+static DECLCALLBACK(int)    ssmR3SelfSaveExec(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)    ssmR3SelfLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int)    ssmR3LiveControlLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static int                  ssmR3Register(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess, const char *pszBefore, PSSMUNIT *ppUnit);
+static int                  ssmR3LiveControlEmit(PSSMHANDLE pSSM, long double lrdPct, uint32_t uPass);
+#endif
+
+static int                  ssmR3StrmWriteBuffers(PSSMSTRM pStrm);
+static int                  ssmR3StrmReadMore(PSSMSTRM pStrm);
+
+static int                  ssmR3DataFlushBuffer(PSSMHANDLE pSSM);
+static int                  ssmR3DataReadRecHdrV2(PSSMHANDLE pSSM);
+
+
+#ifndef SSM_STANDALONE
+
+/**
+ * Cleans up resources allocated by SSM on VM termination.
+ *
+ * @param   pVM                 The VM handle.
+ */
+VMMR3_INT_DECL(void) SSMR3Term(PVM pVM)
+{
+    if (pVM->ssm.s.fInitialized)
+    {
+        pVM->ssm.s.fInitialized = false;
+        RTCritSectDelete(&pVM->ssm.s.CancelCritSect);
+    }
+}
+
+
+/**
+ * Performs lazy initialization of the SSM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM.
+ */
+static int ssmR3LazyInit(PVM pVM)
+{
+    /*
+     * Register a saved state unit which we use to put the VirtualBox version,
+     * revision and similar stuff in.
+     */
+    pVM->ssm.s.fInitialized = true;
+    int rc = SSMR3RegisterInternal(pVM, "SSM", 0 /*uInstance*/, 1 /*uVersion*/, 64 /*cbGuess*/,
+                                   NULL /*pfnLivePrep*/, ssmR3SelfLiveExec, NULL /*pfnLiveVote*/,
+                                   NULL /*pfnSavePrep*/, ssmR3SelfSaveExec, NULL /*pfnSaveDone*/,
+                                   NULL /*pfnSavePrep*/, ssmR3SelfLoadExec, NULL /*pfnSaveDone*/);
+    if (RT_SUCCESS(rc))
+        rc = SSMR3RegisterInternal(pVM, "SSMLiveControl", 0 /*uInstance*/, 1 /*uVersion*/, 1 /*cbGuess*/,
+                                   NULL /*pfnLivePrep*/, NULL /*pfnLiveExec*/,     NULL /*pfnLiveVote*/,
+                                   NULL /*pfnSavePrep*/, NULL /*pfnSaveExec*/,     NULL /*pfnSaveDone*/,
+                                   NULL /*pfnSavePrep*/, ssmR3LiveControlLoadExec, NULL /*pfnSaveDone*/);
+
+    /*
+     * Initialize the cancellation critsect now.
+     */
+    if (RT_SUCCESS(rc))
+        rc = RTCritSectInit(&pVM->ssm.s.CancelCritSect);
+    if (RT_SUCCESS(rc))
+    {
+        STAM_REL_REG_USED(pVM, &pVM->ssm.s.uPass, STAMTYPE_U32, "/SSM/uPass", STAMUNIT_COUNT, "Current pass");
+    }
+
+    pVM->ssm.s.fInitialized = RT_SUCCESS(rc);
+    return rc;
+}
+
+
+/**
+ * Do ssmR3SelfSaveExec in pass 0.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uPass           The data pass number.
+ */
+static DECLCALLBACK(int) ssmR3SelfLiveExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    if (uPass == 0)
+    {
+        int rc = ssmR3SelfSaveExec(pVM, pSSM);
+        if (RT_SUCCESS(rc))
+            rc = VINF_SSM_DONT_CALL_AGAIN;
+        return rc;
+    }
+    AssertFailed();
+    return VERR_INTERNAL_ERROR_3;
+}
+
+
+/**
+ * For saving usful things without having to go thru the tedious process of
+ * adding it to the header.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pSSM            The SSM handle.
+ */
+static DECLCALLBACK(int) ssmR3SelfSaveExec(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * String table containing pairs of variable and value string.
+     * Terminated by two empty strings.
+     */
+    SSMR3PutStrZ(pSSM, "Build Type");
+    SSMR3PutStrZ(pSSM, KBUILD_TYPE);
+    SSMR3PutStrZ(pSSM, "Host OS");
+    SSMR3PutStrZ(pSSM, KBUILD_TARGET "." KBUILD_TARGET_ARCH);
+#ifdef VBOX_OSE
+    SSMR3PutStrZ(pSSM, "OSE");
+    SSMR3PutStrZ(pSSM, "true");
+#endif
+
+    /* terminator */
+    SSMR3PutStrZ(pSSM, "");
+    return SSMR3PutStrZ(pSSM, "");
+}
+
+
+/**
+ * For load the version + revision and stuff.
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uVersion        The version (1).
+ * @param   uPass           The pass.
+ */
+static DECLCALLBACK(int) ssmR3SelfLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    AssertLogRelMsgReturn(uVersion == 1, ("%d", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+
+    /*
+     * The first and last passes contains a {name, value} string table that is
+     * terminated by two emptry strings.  It contains useful informal build
+     * info and can be very handy when something goes wrong after restore.
+     */
+    if (    uPass == 0
+        ||  uPass == SSM_PASS_FINAL)
+    {
+        for (unsigned i = 0; ; i++)
+        {
+            char szVar[128];
+            char szValue[1024];
+            int rc = SSMR3GetStrZ(pSSM, szVar, sizeof(szVar));
+            AssertRCReturn(rc, rc);
+            rc = SSMR3GetStrZ(pSSM, szValue, sizeof(szValue));
+            AssertRCReturn(rc, rc);
+            if (!szVar[0] && !szValue[0])
+                break;
+            if (i == 0)
+                LogRel(("SSM: Saved state info:\n"));
+            LogRel(("SSM:   %s: %s\n", szVar, szValue));
+
+            /*
+             * Detect 32-bit MSC for handling SSMFIELD_ENTRY_PAD_MSC32_AUTO.
+             * Save the Host OS for SSMR3HandleHostOSAndArch
+             */
+            if (!strcmp(szVar, "Host OS"))
+            {
+                bool fIsHostMsc32 = !strcmp(szValue, "win.x86");
+                if (fIsHostMsc32 != pSSM->u.Read.fIsHostMsc32)
+                {
+                    LogRel(("SSM: (fIsHostMsc32 %RTbool => %RTbool)\n", pSSM->u.Read.fIsHostMsc32, fIsHostMsc32));
+                    pSSM->u.Read.fIsHostMsc32 = fIsHostMsc32;
+                }
+
+                size_t cchValue = strlen(szValue);
+                size_t cchCopy = RT_MIN(cchValue, sizeof(pSSM->u.Read.szHostOSAndArch) - 1);
+                Assert(cchValue == cchCopy);
+                memcpy(pSSM->u.Read.szHostOSAndArch, szValue, cchCopy);
+                pSSM->u.Read.szHostOSAndArch[cchCopy] = '\0';
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Load exec callback for the special live save state unit that tracks the
+ * progress of a live save.
+ *
+ * This is saved by ssmR3LiveControlEmit().
+ *
+ * @returns VBox status code.
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   pSSM            The SSM handle.
+ * @param   uVersion        The version (1).
+ * @param   uPass           The pass.
+ */
+static DECLCALLBACK(int) ssmR3LiveControlLoadExec(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    AssertLogRelMsgReturn(uVersion == 1, ("%d", uVersion), VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION);
+
+    uint16_t uPartsPerTenThousand;
+    int rc = SSMR3GetU16(pSSM, &uPartsPerTenThousand);
+    if (RT_SUCCESS(rc))
+    {
+        /* Scale it down to fit in our exec range. */
+        unsigned uPct = (unsigned)(  (long double)uPartsPerTenThousand / 100
+                                   * (100 - pSSM->uPercentPrepare - pSSM->uPercentDone) / 100)
+                      + pSSM->uPercentPrepare;
+        if (uPct != pSSM->uPercent)
+        {
+            AssertMsg(uPct < 100, ("uPct=%d uPartsPerTenThousand=%d uPercentPrepare=%d uPercentDone=%d\n", uPct, uPartsPerTenThousand, pSSM->uPercentPrepare, pSSM->uPercentDone));
+            pSSM->uPercent = uPct;
+            if (pSSM->pfnProgress)
+                pSSM->pfnProgress(pVM, RT_MIN(uPct, 100 - pSSM->uPercentDone), pSSM->pvUser);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Internal registration worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance id.
+ * @param   uVersion        The data unit version.
+ * @param   cbGuess         The guessed data unit size.
+ * @param   pszBefore       Name of data unit to be placed in front of.
+ *                          Optional.
+ * @param   ppUnit          Where to store the inserted unit node.
+ *                          Caller must fill in the missing details.
+ */
+static int ssmR3Register(PVM pVM, const char *pszName, uint32_t uInstance,
+                         uint32_t uVersion, size_t cbGuess, const char *pszBefore, PSSMUNIT *ppUnit)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pszName);
+    AssertReturn(*pszName, VERR_INVALID_PARAMETER);
+    size_t cchName = strlen(pszName);
+    AssertMsgReturn(cchName < SSM_MAX_NAME_SIZE, ("%zu >= %u: %s\n", cchName, SSM_MAX_NAME_SIZE, pszName), VERR_OUT_OF_RANGE);
+
+    AssertReturn(!pszBefore || *pszBefore, VERR_INVALID_PARAMETER);
+    size_t cchBefore = pszBefore ? strlen(pszBefore) : 0;
+    AssertMsgReturn(cchBefore < SSM_MAX_NAME_SIZE, ("%zu >= %u: %s\n", cchBefore, SSM_MAX_NAME_SIZE, pszBefore), VERR_OUT_OF_RANGE);
+
+    /*
+     * Lazy init.
+     */
+    if (!pVM->ssm.s.fInitialized)
+    {
+        int rc = ssmR3LazyInit(pVM);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Walk to the end of the list checking for duplicates as we go.
+     */
+    PSSMUNIT    pUnitBeforePrev = NULL;
+    PSSMUNIT    pUnitBefore     = NULL;
+    PSSMUNIT    pUnitPrev       = NULL;
+    PSSMUNIT    pUnit           = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->u32Instance == uInstance
+            &&  pUnit->cchName == cchName
+            &&  !memcmp(pUnit->szName, pszName, cchName))
+        {
+            AssertMsgFailed(("Duplicate registration %s\n", pszName));
+            return VERR_SSM_UNIT_EXISTS;
+        }
+        if (    pUnit->cchName == cchBefore
+            &&  !pUnitBefore
+            &&  !memcmp(pUnit->szName, pszBefore, cchBefore))
+        {
+            pUnitBeforePrev = pUnitPrev;
+            pUnitBefore     = pUnit;
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    /*
+     * Allocate new node.
+     */
+    pUnit = (PSSMUNIT)MMR3HeapAllocZ(pVM, MM_TAG_SSM, RT_OFFSETOF(SSMUNIT, szName[cchName + 1]));
+    if (!pUnit)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Fill in (some) data. (Stuff is zero'd.)
+     */
+    pUnit->u32Version   = uVersion;
+    pUnit->u32Instance  = uInstance;
+    pUnit->cbGuess      = cbGuess;
+    pUnit->cchName      = cchName;
+    memcpy(pUnit->szName, pszName, cchName);
+
+    /*
+     * Insert
+     */
+    if (pUnitBefore)
+    {
+        pUnit->pNext = pUnitBefore;
+        if (pUnitBeforePrev)
+            pUnitBeforePrev->pNext = pUnit;
+        else
+            pVM->ssm.s.pHead       = pUnit;
+    }
+    else if (pUnitPrev)
+        pUnitPrev->pNext = pUnit;
+    else
+        pVM->ssm.s.pHead = pUnit;
+    pVM->ssm.s.cUnits++;
+
+    *ppUnit = pUnit;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Register a PDM Devices data unit.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pDevIns         Device instance.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ * @param   pszBefore       Name of data unit which we should be put in front
+ *                          of. Optional (NULL).
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ */
+VMMR3DECL(int) SSMR3RegisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess, const char *pszBefore,
+    PFNSSMDEVLIVEPREP pfnLivePrep, PFNSSMDEVLIVEEXEC pfnLiveExec, PFNSSMDEVLIVEVOTE pfnLiveVote,
+    PFNSSMDEVSAVEPREP pfnSavePrep, PFNSSMDEVSAVEEXEC pfnSaveExec, PFNSSMDEVSAVEDONE pfnSaveDone,
+    PFNSSMDEVLOADPREP pfnLoadPrep, PFNSSMDEVLOADEXEC pfnLoadExec, PFNSSMDEVLOADDONE pfnLoadDone)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, pszBefore, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_DEV;
+        pUnit->u.Dev.pfnLivePrep = pfnLivePrep;
+        pUnit->u.Dev.pfnLiveExec = pfnLiveExec;
+        pUnit->u.Dev.pfnLiveVote = pfnLiveVote;
+        pUnit->u.Dev.pfnSavePrep = pfnSavePrep;
+        pUnit->u.Dev.pfnSaveExec = pfnSaveExec;
+        pUnit->u.Dev.pfnSaveDone = pfnSaveDone;
+        pUnit->u.Dev.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.Dev.pfnLoadExec = pfnLoadExec;
+        pUnit->u.Dev.pfnLoadDone = pfnLoadDone;
+        pUnit->u.Dev.pDevIns = pDevIns;
+    }
+    return rc;
+}
+
+
+/**
+ * Register a PDM driver data unit.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pDrvIns         Driver instance.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ */
+VMMR3DECL(int) SSMR3RegisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
+    PFNSSMDRVLIVEPREP pfnLivePrep, PFNSSMDRVLIVEEXEC pfnLiveExec, PFNSSMDRVLIVEVOTE pfnLiveVote,
+    PFNSSMDRVSAVEPREP pfnSavePrep, PFNSSMDRVSAVEEXEC pfnSaveExec, PFNSSMDRVSAVEDONE pfnSaveDone,
+    PFNSSMDRVLOADPREP pfnLoadPrep, PFNSSMDRVLOADEXEC pfnLoadExec, PFNSSMDRVLOADDONE pfnLoadDone)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_DRV;
+        pUnit->u.Drv.pfnSavePrep = pfnSavePrep;
+        pUnit->u.Drv.pfnSaveExec = pfnSaveExec;
+        pUnit->u.Drv.pfnSaveDone = pfnSaveDone;
+        pUnit->u.Drv.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.Drv.pfnLoadExec = pfnLoadExec;
+        pUnit->u.Drv.pfnLoadDone = pfnLoadDone;
+        pUnit->u.Drv.pDrvIns = pDrvIns;
+    }
+    return rc;
+}
+
+
+/**
+ * Register a internal data unit.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ */
+VMMR3DECL(int) SSMR3RegisterInternal(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
+    PFNSSMINTLIVEPREP pfnLivePrep, PFNSSMINTLIVEEXEC pfnLiveExec, PFNSSMINTLIVEVOTE pfnLiveVote,
+    PFNSSMINTSAVEPREP pfnSavePrep, PFNSSMINTSAVEEXEC pfnSaveExec, PFNSSMINTSAVEDONE pfnSaveDone,
+    PFNSSMINTLOADPREP pfnLoadPrep, PFNSSMINTLOADEXEC pfnLoadExec, PFNSSMINTLOADDONE pfnLoadDone)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_INTERNAL;
+        pUnit->u.Internal.pfnLivePrep = pfnLivePrep;
+        pUnit->u.Internal.pfnLiveExec = pfnLiveExec;
+        pUnit->u.Internal.pfnLiveVote = pfnLiveVote;
+        pUnit->u.Internal.pfnSavePrep = pfnSavePrep;
+        pUnit->u.Internal.pfnSaveExec = pfnSaveExec;
+        pUnit->u.Internal.pfnSaveDone = pfnSaveDone;
+        pUnit->u.Internal.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.Internal.pfnLoadExec = pfnLoadExec;
+        pUnit->u.Internal.pfnLoadDone = pfnLoadDone;
+    }
+    return rc;
+}
+
+
+/**
+ * Register an external data unit.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @param   uVersion        Data layout version number.
+ * @param   cbGuess         The approximate amount of data in the unit.
+ *                          Only for progress indicators.
+ *
+ * @param   pfnLivePrep     Prepare live save callback, optional.
+ * @param   pfnLiveExec     Execute live save callback, optional.
+ * @param   pfnLiveVote     Vote live save callback, optional.
+ *
+ * @param   pfnSavePrep     Prepare save callback, optional.
+ * @param   pfnSaveExec     Execute save callback, optional.
+ * @param   pfnSaveDone     Done save callback, optional.
+ *
+ * @param   pfnLoadPrep     Prepare load callback, optional.
+ * @param   pfnLoadExec     Execute load callback, optional.
+ * @param   pfnLoadDone     Done load callback, optional.
+ * @param   pvUser          User argument.
+ */
+VMMR3DECL(int) SSMR3RegisterExternal(PVM pVM, const char *pszName, uint32_t uInstance, uint32_t uVersion, size_t cbGuess,
+    PFNSSMEXTLIVEPREP pfnLivePrep, PFNSSMEXTLIVEEXEC pfnLiveExec, PFNSSMEXTLIVEVOTE pfnLiveVote,
+    PFNSSMEXTSAVEPREP pfnSavePrep, PFNSSMEXTSAVEEXEC pfnSaveExec, PFNSSMEXTSAVEDONE pfnSaveDone,
+    PFNSSMEXTLOADPREP pfnLoadPrep, PFNSSMEXTLOADEXEC pfnLoadExec, PFNSSMEXTLOADDONE pfnLoadDone, void *pvUser)
+{
+    PSSMUNIT pUnit;
+    int rc = ssmR3Register(pVM, pszName, uInstance, uVersion, cbGuess, NULL, &pUnit);
+    if (RT_SUCCESS(rc))
+    {
+        pUnit->enmType = SSMUNITTYPE_EXTERNAL;
+        pUnit->u.External.pfnLivePrep = pfnLivePrep;
+        pUnit->u.External.pfnLiveExec = pfnLiveExec;
+        pUnit->u.External.pfnLiveVote = pfnLiveVote;
+        pUnit->u.External.pfnSavePrep = pfnSavePrep;
+        pUnit->u.External.pfnSaveExec = pfnSaveExec;
+        pUnit->u.External.pfnSaveDone = pfnSaveDone;
+        pUnit->u.External.pfnLoadPrep = pfnLoadPrep;
+        pUnit->u.External.pfnLoadExec = pfnLoadExec;
+        pUnit->u.External.pfnLoadDone = pfnLoadDone;
+        pUnit->u.External.pvUser      = pvUser;
+    }
+    return rc;
+}
+
+
+/**
+ * Deregister one or more PDM Device data units.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pDevIns         Device instance.
+ * @param   pszName         Data unit name.
+ *                          Use NULL to deregister all data units for that device instance.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique.
+ * @remark  Only for dynamic data units and dynamic unloaded modules.
+ */
+VMMR3_INT_DECL(int) SSMR3DeregisterDevice(PVM pVM, PPDMDEVINS pDevIns, const char *pszName, uint32_t uInstance)
+{
+    /*
+     * Validate input.
+     */
+    if (!pDevIns)
+    {
+        AssertMsgFailed(("pDevIns is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Search the list.
+     */
+    size_t      cchName = pszName ? strlen(pszName) : 0;
+    int         rc = pszName ? VERR_SSM_UNIT_NOT_FOUND : VINF_SUCCESS;
+    PSSMUNIT    pUnitPrev = NULL;
+    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->enmType == SSMUNITTYPE_DEV
+            &&  (   !pszName
+                 || (   pUnit->cchName == cchName
+                     && !memcmp(pUnit->szName, pszName, cchName)))
+            &&  pUnit->u32Instance == uInstance
+            )
+        {
+            if (pUnit->u.Dev.pDevIns == pDevIns)
+            {
+                /*
+                 * Unlink it, advance pointer, and free the node.
+                 */
+                PSSMUNIT pFree = pUnit;
+                pUnit = pUnit->pNext;
+                if (pUnitPrev)
+                    pUnitPrev->pNext = pUnit;
+                else
+                    pVM->ssm.s.pHead = pUnit;
+                pVM->ssm.s.cUnits--;
+                Log(("SSM: Removed data unit '%s' (pdm dev).\n", pFree->szName));
+                MMR3HeapFree(pFree);
+
+                if (pszName)
+                    return VINF_SUCCESS;
+                rc = VINF_SUCCESS;
+                continue;
+            }
+            else if (pszName)
+            {
+                AssertMsgFailed(("Caller is not owner! Owner=%p Caller=%p %s\n",
+                                 pUnit->u.Dev.pDevIns, pDevIns, pszName));
+                return VERR_SSM_UNIT_NOT_OWNER;
+            }
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister one ore more PDM Driver data units.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pDrvIns         Driver instance.
+ * @param   pszName         Data unit name.
+ *                          Use NULL to deregister all data units for that driver instance.
+ * @param   uInstance       The instance identifier of the data unit.
+ *                          This must together with the name be unique. Ignored if pszName is NULL.
+ * @remark  Only for dynamic data units and dynamic unloaded modules.
+ */
+VMMR3_INT_DECL(int) SSMR3DeregisterDriver(PVM pVM, PPDMDRVINS pDrvIns, const char *pszName, uint32_t uInstance)
+{
+    /*
+     * Validate input.
+     */
+    if (!pDrvIns)
+    {
+        AssertMsgFailed(("pDrvIns is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Search the list.
+     */
+    size_t      cchName = pszName ? strlen(pszName) : 0;
+    int         rc = pszName ? VERR_SSM_UNIT_NOT_FOUND : VINF_SUCCESS;
+    PSSMUNIT    pUnitPrev = NULL;
+    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->enmType == SSMUNITTYPE_DRV
+            &&  (   !pszName
+                 || (   pUnit->cchName == cchName
+                     && !memcmp(pUnit->szName, pszName, cchName)
+                     && pUnit->u32Instance == uInstance))
+            )
+        {
+            if (pUnit->u.Drv.pDrvIns == pDrvIns)
+            {
+                /*
+                 * Unlink it, advance pointer, and free the node.
+                 */
+                PSSMUNIT pFree = pUnit;
+                pUnit = pUnit->pNext;
+                if (pUnitPrev)
+                    pUnitPrev->pNext = pUnit;
+                else
+                    pVM->ssm.s.pHead = pUnit;
+                pVM->ssm.s.cUnits--;
+                Log(("SSM: Removed data unit '%s' (pdm drv).\n", pFree->szName));
+                MMR3HeapFree(pFree);
+
+                if (pszName)
+                    return VINF_SUCCESS;
+                rc = VINF_SUCCESS;
+                continue;
+            }
+
+            AssertMsgReturn(!pszName,
+                            ("Caller is not owner! Owner=%p Caller=%p %s\n", pUnit->u.Drv.pDrvIns, pDrvIns, pszName),
+                            VERR_SSM_UNIT_NOT_OWNER);
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister a data unit.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   enmType         Unit type
+ * @param   pszName         Data unit name.
+ * @remark  Only for dynamic data units.
+ */
+static int ssmR3DeregisterByNameAndType(PVM pVM, const char *pszName, SSMUNITTYPE enmType)
+{
+    /*
+     * Validate input.
+     */
+    if (!pszName)
+    {
+        AssertMsgFailed(("pszName is NULL!\n"));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * Search the list.
+     */
+    size_t      cchName = strlen(pszName);
+    int         rc = VERR_SSM_UNIT_NOT_FOUND;
+    PSSMUNIT    pUnitPrev = NULL;
+    PSSMUNIT    pUnit = pVM->ssm.s.pHead;
+    while (pUnit)
+    {
+        if (    pUnit->enmType == enmType
+            &&  pUnit->cchName == cchName
+            &&  !memcmp(pUnit->szName, pszName, cchName))
+        {
+            /*
+             * Unlink it, advance pointer, and free the node.
+             */
+            PSSMUNIT pFree = pUnit;
+            pUnit = pUnit->pNext;
+            if (pUnitPrev)
+                pUnitPrev->pNext = pUnit;
+            else
+                pVM->ssm.s.pHead = pUnit;
+            pVM->ssm.s.cUnits--;
+            Log(("SSM: Removed data unit '%s' (type=%d).\n", pFree->szName, enmType));
+            MMR3HeapFree(pFree);
+            return VINF_SUCCESS;
+        }
+
+        /* next */
+        pUnitPrev = pUnit;
+        pUnit = pUnit->pNext;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Deregister an internal data unit.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszName         Data unit name.
+ * @remark  Only for dynamic data units.
+ */
+VMMR3DECL(int) SSMR3DeregisterInternal(PVM pVM, const char *pszName)
+{
+    return ssmR3DeregisterByNameAndType(pVM, pszName, SSMUNITTYPE_INTERNAL);
+}
+
+
+/**
+ * Deregister an external data unit.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszName         Data unit name.
+ * @remark  Only for dynamic data units.
+ */
+VMMR3DECL(int) SSMR3DeregisterExternal(PVM pVM, const char *pszName)
+{
+    return ssmR3DeregisterByNameAndType(pVM, pszName, SSMUNITTYPE_EXTERNAL);
+}
+
+#endif /* !SSM_STANDALONE */
+
+
+/**
+ * Initializes the stream after/before opening the file/whatever.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_MEMORY.
+ * @param   pStrm           The stream handle.
+ * @param   fChecksummed    Whether the stream is to be checksummed while
+ *                          written/read.
+ * @param   cBuffers        The number of buffers.
+ */
+static int ssmR3StrmInitInternal(PSSMSTRM pStrm, bool fChecksummed, uint32_t cBuffers)
+{
+    Assert(cBuffers > 0);
+
+    /*
+     * Init the common data members.
+     */
+    pStrm->fTerminating = false;
+    pStrm->fNeedSeek    = false;
+    pStrm->rc           = VINF_SUCCESS;
+    pStrm->hIoThread    = NIL_RTTHREAD;
+    pStrm->offNeedSeekTo= UINT64_MAX;
+
+    pStrm->pHead        = NULL;
+    pStrm->pFree        = NULL;
+    pStrm->hEvtHead     = NIL_RTSEMEVENT;
+    pStrm->hEvtFree     = NIL_RTSEMEVENT;
+
+    pStrm->pPending     = NULL;
+    pStrm->pCur         = NULL;
+    pStrm->offCurStream = 0;
+    pStrm->off          = 0;
+    pStrm->fChecksummed = fChecksummed;
+    pStrm->u32StreamCRC = fChecksummed ? RTCrc32Start() : 0;
+    pStrm->offStreamCRC = 0;
+
+    /*
+     * Allocate the buffers.  Page align them in case that makes the kernel
+     * and/or cpu happier in some way.
+     */
+    int rc = VINF_SUCCESS;
+    for (uint32_t i = 0; i < cBuffers; i++)
+    {
+        PSSMSTRMBUF pBuf = (PSSMSTRMBUF)RTMemPageAllocZ(sizeof(*pBuf));
+        if (!pBuf)
+        {
+            if (i > 2)
+            {
+                LogRel(("ssmR3StrmAllocBuffer: WARNING: Could only get %d stream buffers.\n", i));
+                break;
+            }
+            LogRel(("ssmR3StrmAllocBuffer: Failed to allocate stream buffers. (i=%d)\n", i));
+            return VERR_NO_MEMORY;
+        }
+
+        /* link it */
+        pBuf->pNext = pStrm->pFree;
+        pStrm->pFree = pBuf;
+    }
+
+    /*
+     * Create the event semaphores.
+     */
+    rc = RTSemEventCreate(&pStrm->hEvtHead);
+    if (RT_FAILURE(rc))
+        return rc;
+    rc = RTSemEventCreate(&pStrm->hEvtFree);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroys a list of buffers.
+ *
+ * @param   pHead            Pointer to the head.
+ */
+static void ssmR3StrmDestroyBufList(PSSMSTRMBUF pHead)
+{
+    while (pHead)
+    {
+        PSSMSTRMBUF pCur = pHead;
+        pHead = pCur->pNext;
+        pCur->pNext = NULL;
+        RTMemPageFree(pCur, sizeof(*pCur));
+    }
+}
+
+
+/**
+ * Cleans up a stream after ssmR3StrmInitInternal has been called (regardless of
+ * it succeeded or not).
+ *
+ * @param   pStrm           The stream handle.
+ */
+static void ssmR3StrmDelete(PSSMSTRM pStrm)
+{
+    RTMemPageFree(pStrm->pCur, sizeof(*pStrm->pCur));
+    pStrm->pCur = NULL;
+    ssmR3StrmDestroyBufList(pStrm->pHead);
+    pStrm->pHead = NULL;
+    ssmR3StrmDestroyBufList(pStrm->pPending);
+    pStrm->pPending = NULL;
+    ssmR3StrmDestroyBufList(pStrm->pFree);
+    pStrm->pFree = NULL;
+
+    RTSemEventDestroy(pStrm->hEvtHead);
+    pStrm->hEvtHead = NIL_RTSEMEVENT;
+
+    RTSemEventDestroy(pStrm->hEvtFree);
+    pStrm->hEvtFree = NIL_RTSEMEVENT;
+}
+
+
+/**
+ * Initializes a stream that uses a method table.
+ *
+ * @returns VBox status code.
+ * @param   pStrm           The stream manager structure.
+ * @param   pStreamOps      The stream method table.
+ * @param   pvUser          The user argument for the stream methods.
+ * @param   fWrite          Whether to open for writing or reading.
+ * @param   fChecksummed    Whether the stream is to be checksummed while
+ *                          written/read.
+ * @param   cBuffers        The number of buffers.
+ */
+static int ssmR3StrmInit(PSSMSTRM pStrm, PCSSMSTRMOPS pStreamOps, void *pvUser, bool fWrite, bool fChecksummed, uint32_t cBuffers)
+{
+    int rc = ssmR3StrmInitInternal(pStrm, fChecksummed, cBuffers);
+    if (RT_SUCCESS(rc))
+    {
+        pStrm->pOps   = pStreamOps;
+        pStrm->pvUser = pvUser;
+        pStrm->fWrite = fWrite;
+        return VINF_SUCCESS;
+    }
+
+    ssmR3StrmDelete(pStrm);
+    pStrm->rc = rc;
+    return rc;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnWrite
+ */
+static DECLCALLBACK(int) ssmR3FileWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
+{
+    Assert(RTFileTell((RTFILE)(uintptr_t)pvUser) == offStream); NOREF(offStream);
+    return RTFileWriteAt((RTFILE)(uintptr_t)pvUser, offStream, pvBuf, cbToWrite, NULL); /** @todo use RTFileWrite */
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnRead
+ */
+static DECLCALLBACK(int) ssmR3FileRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
+{
+    Assert(RTFileTell((RTFILE)(uintptr_t)pvUser) == offStream); NOREF(offStream);
+    return RTFileRead((RTFILE)(uintptr_t)pvUser, pvBuf, cbToRead, pcbRead);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnSeek
+ */
+static DECLCALLBACK(int) ssmR3FileSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
+{
+    return RTFileSeek((RTFILE)(uintptr_t)pvUser, offSeek, uMethod, poffActual);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnTell
+ */
+static DECLCALLBACK(uint64_t) ssmR3FileTell(void *pvUser)
+{
+    return RTFileTell((RTFILE)(uintptr_t)pvUser);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnSize
+ */
+static DECLCALLBACK(int) ssmR3FileSize(void *pvUser, uint64_t *pcb)
+{
+    return RTFileGetSize((RTFILE)(uintptr_t)pvUser, pcb);
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnIsOk
+ */
+static DECLCALLBACK(int) ssmR3FileIsOk(void *pvUser)
+{
+    /*
+     * Check that there is still some space left on the disk.
+     */
+    RTFOFF cbFree;
+    int rc = RTFileQueryFsSizes((RTFILE)(uintptr_t)pvUser, NULL, &cbFree, NULL, NULL);
+#define SSM_MIN_DISK_FREE    ((RTFOFF)( 10 * _1M ))
+    if (RT_SUCCESS(rc))
+    {
+        if (cbFree < SSM_MIN_DISK_FREE)
+        {
+            LogRel(("SSM: Giving up: Low on disk space. (cbFree=%RTfoff, SSM_MIN_DISK_FREE=%RTfoff).\n",
+                    cbFree, SSM_MIN_DISK_FREE));
+            rc = VERR_SSM_LOW_ON_DISK_SPACE;
+        }
+    }
+    else if (rc == VERR_NOT_SUPPORTED)
+        rc = VINF_SUCCESS;
+    else
+        AssertLogRelRC(rc);
+    return rc;
+}
+
+
+/**
+ * @copydoc SSMSTRMOPS::pfnClose
+ */
+static DECLCALLBACK(int) ssmR3FileClose(void *pvUser, bool fCancelled)
+{
+    NOREF(fCancelled);
+    return RTFileClose((RTFILE)(uintptr_t)pvUser);
+}
+
+
+/**
+ * Method table for a file based stream.
+ */
+static SSMSTRMOPS const g_ssmR3FileOps =
+{
+    SSMSTRMOPS_VERSION,
+    ssmR3FileWrite,
+    ssmR3FileRead,
+    ssmR3FileSeek,
+    ssmR3FileTell,
+    ssmR3FileSize,
+    ssmR3FileIsOk,
+    ssmR3FileClose,
+    SSMSTRMOPS_VERSION
+};
+
+
+/**
+ * Opens a file stream.
+ *
+ * @returns VBox status code.
+ * @param   pStrm           The stream manager structure.
+ * @param   pszFilename     The file to open or create.
+ * @param   fWrite          Whether to open for writing or reading.
+ * @param   fChecksummed    Whether the stream is to be checksummed while
+ *                          written/read.
+ * @param   cBuffers        The number of buffers.
+ */
+static int ssmR3StrmOpenFile(PSSMSTRM pStrm, const char *pszFilename, bool fWrite, bool fChecksummed, uint32_t cBuffers)
+{
+    int rc = ssmR3StrmInitInternal(pStrm, fChecksummed, cBuffers);
+    if (RT_SUCCESS(rc))
+    {
+        uint32_t fFlags = fWrite
+                        ? RTFILE_O_READWRITE | RTFILE_O_CREATE_REPLACE | RTFILE_O_DENY_WRITE
+                        : RTFILE_O_READ      | RTFILE_O_OPEN           | RTFILE_O_DENY_WRITE;
+        RTFILE hFile;
+        rc = RTFileOpen(&hFile, pszFilename, fFlags);
+        if (RT_SUCCESS(rc))
+        {
+            pStrm->pOps   = &g_ssmR3FileOps;
+            pStrm->pvUser = (void *)(uintptr_t)hFile;
+            pStrm->fWrite = fWrite;
+            return VINF_SUCCESS;
+        }
+    }
+
+    ssmR3StrmDelete(pStrm);
+    pStrm->rc = rc;
+    return rc;
+}
+
+
+/**
+ * Raise an error condition on the stream.
+ *
+ * @returns true if we raised the error condition, false if the stream already
+ *          had an error condition set.
+ *
+ * @param   pStrm           The stream handle.
+ * @param   rc              The VBox error status code.
+ *
+ * @thread  Any.
+ */
+DECLINLINE(bool) ssmR3StrmSetError(PSSMSTRM pStrm, int rc)
+{
+    Assert(RT_FAILURE_NP(rc));
+    return ASMAtomicCmpXchgS32(&pStrm->rc, rc, VINF_SUCCESS);
+}
+
+
+/**
+ * Puts a buffer into the free list.
+ *
+ * @param   pStrm           The stream handle.
+ * @param   pBuf            The buffer.
+ *
+ * @thread  The consumer.
+ */
+static void ssmR3StrmPutFreeBuf(PSSMSTRM pStrm, PSSMSTRMBUF pBuf)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pCurFreeHead = ASMAtomicUoReadPtrT(&pStrm->pFree, PSSMSTRMBUF);
+        ASMAtomicUoWritePtr(&pBuf->pNext, pCurFreeHead);
+        if (ASMAtomicCmpXchgPtr(&pStrm->pFree, pBuf, pCurFreeHead))
+        {
+            int rc = RTSemEventSignal(pStrm->hEvtFree);
+            AssertRC(rc);
+            return;
+        }
+    }
+}
+
+
+/**
+ * Gets a free buffer, waits for one if necessary.
+ *
+ * @returns Pointer to the buffer on success. NULL if we're terminating.
+ * @param   pStrm           The stream handle.
+ *
+ * @thread  The producer.
+ */
+static PSSMSTRMBUF ssmR3StrmGetFreeBuf(PSSMSTRM pStrm)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pMine = ASMAtomicUoReadPtrT(&pStrm->pFree, PSSMSTRMBUF);
+        if (!pMine)
+        {
+            if (pStrm->fTerminating)
+                return NULL;
+            if (RT_FAILURE(pStrm->rc))
+                return NULL;
+            if (    pStrm->fWrite
+                &&  pStrm->hIoThread == NIL_RTTHREAD)
+            {
+                int rc = ssmR3StrmWriteBuffers(pStrm);
+                if (RT_FAILURE(rc))
+                    return NULL;
+            }
+            int rc = RTSemEventWaitNoResume(pStrm->hEvtFree, 30000);
+            if (    rc == VERR_SEM_DESTROYED
+                ||  pStrm->fTerminating)
+                return NULL;
+            continue;
+        }
+
+        if (ASMAtomicCmpXchgPtr(&pStrm->pFree, pMine->pNext, pMine))
+        {
+            pMine->offStream    = UINT64_MAX;
+            pMine->cb           = 0;
+            pMine->pNext        = NULL;
+            pMine->fEndOfStream = false;
+            pMine->NanoTS       = RTTimeNanoTS();
+            return pMine;
+        }
+    }
+}
+
+
+/**
+ * Puts a buffer onto the queue.
+ *
+ * @param   pBuf        The buffer.
+ *
+ * @thread  The producer.
+ */
+static void ssmR3StrmPutBuf(PSSMSTRM pStrm, PSSMSTRMBUF pBuf)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pCurHead = ASMAtomicUoReadPtrT(&pStrm->pHead, PSSMSTRMBUF);
+        ASMAtomicUoWritePtr(&pBuf->pNext, pCurHead);
+        if (ASMAtomicCmpXchgPtr(&pStrm->pHead, pBuf, pCurHead))
+        {
+            int rc = RTSemEventSignal(pStrm->hEvtHead);
+            AssertRC(rc);
+            return;
+        }
+    }
+}
+
+
+/**
+ * Reverses the list.
+ *
+ * @returns The head of the reversed list.
+ * @param   pHead           The head of the list to reverse.
+ */
+static PSSMSTRMBUF ssmR3StrmReverseList(PSSMSTRMBUF pHead)
+{
+    PSSMSTRMBUF pRevHead = NULL;
+    while (pHead)
+    {
+        PSSMSTRMBUF pCur = pHead;
+        pHead = pCur->pNext;
+        pCur->pNext = pRevHead;
+        pRevHead = pCur;
+    }
+    return pRevHead;
+}
+
+
+/**
+ * Gets one buffer from the queue, will wait for one to become ready if
+ * necessary.
+ *
+ * @returns Pointer to the buffer on success. NULL if we're terminating.
+ * @param   pBuf        The buffer.
+ *
+ * @thread  The consumer.
+ */
+static PSSMSTRMBUF ssmR3StrmGetBuf(PSSMSTRM pStrm)
+{
+    for (;;)
+    {
+        PSSMSTRMBUF pMine = pStrm->pPending;
+        if (pMine)
+        {
+            pStrm->pPending = pMine->pNext;
+            pMine->pNext = NULL;
+            return pMine;
+        }
+
+        pMine = ASMAtomicXchgPtrT(&pStrm->pHead, NULL, PSSMSTRMBUF);
+        if (pMine)
+            pStrm->pPending = ssmR3StrmReverseList(pMine);
+        else
+        {
+            if (pStrm->fTerminating)
+                return NULL;
+            if (RT_FAILURE(pStrm->rc))
+                return NULL;
+            if (    !pStrm->fWrite
+                &&  pStrm->hIoThread == NIL_RTTHREAD)
+            {
+                int rc = ssmR3StrmReadMore(pStrm);
+                if (RT_FAILURE(rc))
+                    return NULL;
+                continue;
+            }
+
+            int rc = RTSemEventWaitNoResume(pStrm->hEvtHead, 30000);
+            if (    rc == VERR_SEM_DESTROYED
+                ||  pStrm->fTerminating)
+                return NULL;
+        }
+    }
+}
+
+
+/**
+ * Flushes the current buffer (both write and read streams).
+ *
+ * @param   pStrm           The stream handle.
+ */
+static void ssmR3StrmFlushCurBuf(PSSMSTRM pStrm)
+{
+    if (pStrm->pCur)
+    {
+        PSSMSTRMBUF pBuf = pStrm->pCur;
+        pStrm->pCur = NULL;
+
+        if (pStrm->fWrite)
+        {
+            uint32_t cb     = pStrm->off;
+            pBuf->cb        = cb;
+            pBuf->offStream = pStrm->offCurStream;
+            if (    pStrm->fChecksummed
+                &&  pStrm->offStreamCRC < cb)
+                pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC,
+                                                     &pBuf->abData[pStrm->offStreamCRC],
+                                                     cb - pStrm->offStreamCRC);
+            pStrm->offCurStream += cb;
+            pStrm->off           = 0;
+            pStrm->offStreamCRC  = 0;
+
+            ssmR3StrmPutBuf(pStrm, pBuf);
+        }
+        else
+        {
+            uint32_t cb = pBuf->cb;
+            if (    pStrm->fChecksummed
+                &&  pStrm->offStreamCRC < cb)
+                pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC,
+                                                     &pBuf->abData[pStrm->offStreamCRC],
+                                                     cb - pStrm->offStreamCRC);
+            pStrm->offCurStream += cb;
+            pStrm->off           = 0;
+            pStrm->offStreamCRC  = 0;
+
+            ssmR3StrmPutFreeBuf(pStrm, pBuf);
+        }
+    }
+}
+
+
+/**
+ * Flush buffered data.
+ *
+ * @returns VBox status code. Returns VINF_EOF if we encounter a buffer with the
+ *          fEndOfStream indicator set.
+ * @param   pStrm           The stream handle.
+ *
+ * @thread  The producer thread.
+ */
+static int ssmR3StrmWriteBuffers(PSSMSTRM pStrm)
+{
+    Assert(pStrm->fWrite);
+
+    /*
+     * Just return if the stream has a pending error condition.
+     */
+    int rc = pStrm->rc;
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Grab the pending list and write it out.
+     */
+    PSSMSTRMBUF pHead = ASMAtomicXchgPtrT(&pStrm->pHead, NULL, PSSMSTRMBUF);
+    if (!pHead)
+        return VINF_SUCCESS;
+    pHead = ssmR3StrmReverseList(pHead);
+
+    while (pHead)
+    {
+        /* pop */
+        PSSMSTRMBUF pCur = pHead;
+        pHead = pCur->pNext;
+
+        /* flush */
+        rc = pStrm->pOps->pfnIsOk(pStrm->pvUser);
+        if (RT_SUCCESS(rc))
+            rc = pStrm->pOps->pfnWrite(pStrm->pvUser, pCur->offStream, &pCur->abData[0], pCur->cb);
+        if (    RT_FAILURE(rc)
+            &&  ssmR3StrmSetError(pStrm, rc))
+            LogRel(("ssmR3StrmWriteBuffers: Write failed with rc=%Rrc at offStream=%#llx\n", rc, pCur->offStream));
+
+        /* free */
+        bool fEndOfStream = pCur->fEndOfStream;
+        ssmR3StrmPutFreeBuf(pStrm, pCur);
+        if (fEndOfStream)
+        {
+            Assert(!pHead);
+            return VINF_EOF;
+        }
+    }
+
+    return pStrm->rc;
+}
+
+
+/**
+ * Closes the stream after first flushing any pending write.
+ *
+ * @returns VBox status code.
+ * @param   pStrm           The stream handle.
+ * @param   fCancelled      Indicates whether the operation was cancelled or
+ *                          not.
+ */
+static int ssmR3StrmClose(PSSMSTRM pStrm, bool fCancelled)
+{
+    /*
+     * Flush, terminate the I/O thread, and close the stream.
+     */
+    if (pStrm->fWrite)
+    {
+        ssmR3StrmFlushCurBuf(pStrm);
+        if (pStrm->hIoThread == NIL_RTTHREAD)
+            ssmR3StrmWriteBuffers(pStrm);
+    }
+
+    if (pStrm->hIoThread != NIL_RTTHREAD)
+        ASMAtomicWriteBool(&pStrm->fTerminating, true);
+
+    int rc;
+    if (pStrm->fWrite)
+    {
+        if (pStrm->hIoThread != NIL_RTTHREAD)
+        {
+            int rc2 = RTSemEventSignal(pStrm->hEvtHead);
+            AssertLogRelRC(rc2);
+            int rc3 = RTThreadWait(pStrm->hIoThread, RT_INDEFINITE_WAIT, NULL);
+            AssertLogRelRC(rc3);
+            pStrm->hIoThread = NIL_RTTHREAD;
+        }
+
+        rc = pStrm->pOps->pfnClose(pStrm->pvUser, fCancelled);
+        if (RT_FAILURE(rc))
+            ssmR3StrmSetError(pStrm, rc);
+    }
+    else
+    {
+        rc = pStrm->pOps->pfnClose(pStrm->pvUser, fCancelled);
+        if (RT_FAILURE(rc))
+            ssmR3StrmSetError(pStrm, rc);
+
+        if (pStrm->hIoThread != NIL_RTTHREAD)
+        {
+            int rc2 = RTSemEventSignal(pStrm->hEvtFree);
+            AssertLogRelRC(rc2);
+            int rc3 = RTThreadWait(pStrm->hIoThread, RT_INDEFINITE_WAIT, NULL);
+            AssertLogRelRC(rc3);
+            pStrm->hIoThread = NIL_RTTHREAD;
+        }
+    }
+
+    pStrm->pOps   = NULL;
+    pStrm->pvUser = NULL;
+
+    rc = pStrm->rc;
+    ssmR3StrmDelete(pStrm);
+
+    return rc;
+}
+
+
+/**
+ * Stream output routine.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   pvBuf       What to write.
+ * @param   cbToWrite   How much to write.
+ *
+ * @thread  The producer in a write stream (never the I/O thread).
+ */
+static int ssmR3StrmWrite(PSSMSTRM pStrm, const void *pvBuf, size_t cbToWrite)
+{
+    AssertReturn(cbToWrite > 0, VINF_SUCCESS);
+    Assert(pStrm->fWrite);
+
+    /*
+     * Squeeze as much as possible into the current buffer.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_LIKELY(pBuf))
+    {
+        uint32_t cbLeft = RT_SIZEOFMEMB(SSMSTRMBUF, abData) - pStrm->off;
+        if (RT_LIKELY(cbLeft >= cbToWrite))
+        {
+            memcpy(&pBuf->abData[pStrm->off], pvBuf, cbToWrite);
+            pStrm->off += (uint32_t)cbToWrite;
+            return VINF_SUCCESS;
+        }
+
+        if (cbLeft > 0)
+        {
+            memcpy(&pBuf->abData[pStrm->off], pvBuf, cbLeft);
+            pStrm->off += cbLeft;
+            cbToWrite  -= cbLeft;
+            pvBuf       = (uint8_t const *)pvBuf + cbLeft;
+        }
+        Assert(pStrm->off == RT_SIZEOFMEMB(SSMSTRMBUF, abData));
+    }
+
+    /*
+     * Need one or more new buffers.
+     */
+    do
+    {
+        /*
+         * Flush the current buffer and replace it with a new one.
+         */
+        ssmR3StrmFlushCurBuf(pStrm);
+        pBuf = ssmR3StrmGetFreeBuf(pStrm);
+        if (!pBuf)
+            break;
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+
+        /*
+         * Copy data to the buffer.
+         */
+        uint32_t cbCopy = RT_SIZEOFMEMB(SSMSTRMBUF, abData);
+        if (cbCopy > cbToWrite)
+            cbCopy = (uint32_t)cbToWrite;
+        memcpy(&pBuf->abData[0], pvBuf, cbCopy);
+        pStrm->off  = cbCopy;
+        cbToWrite  -= cbCopy;
+        pvBuf       = (uint8_t const *)pvBuf + cbCopy;
+    } while (cbToWrite > 0);
+
+    return pStrm->rc;
+}
+
+
+/**
+ * Reserves space in the current buffer so the caller can write directly to the
+ * buffer instead of doing double buffering.
+ *
+ * @returns VBox status code
+ * @param   pStrm       The stream handle.
+ * @param   cb          The amount of buffer space to reserve.
+ * @param   ppb         Where to return the pointer.
+ */
+static int ssmR3StrmReserveWriteBufferSpace(PSSMSTRM pStrm, size_t cb, uint8_t **ppb)
+{
+    Assert(pStrm->fWrite);
+    Assert(RT_SIZEOFMEMB(SSMSTRMBUF, abData) / 4 >= cb);
+
+    /*
+     * Check if there is room in the current buffer, it not flush it.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (pBuf)
+    {
+        uint32_t cbLeft = RT_SIZEOFMEMB(SSMSTRMBUF, abData) - pStrm->off;
+        if (cbLeft >= cb)
+        {
+            *ppb = &pBuf->abData[pStrm->off];
+            return VINF_SUCCESS;
+        }
+
+        ssmR3StrmFlushCurBuf(pStrm);
+    }
+
+    /*
+     * Get a fresh buffer and return a pointer into it.
+     */
+    pBuf = ssmR3StrmGetFreeBuf(pStrm);
+    if (pBuf)
+    {
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+        *ppb = &pBuf->abData[0];
+    }
+    else
+        *ppb = NULL; /* make gcc happy. */
+    return pStrm->rc;
+}
+
+
+/**
+ * Commits buffer space reserved by ssmR3StrmReserveWriteBufferSpace.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   cb          The amount of buffer space to commit.  This can be less
+ *                      that what was reserved initially.
+ */
+static int ssmR3StrmCommitWriteBufferSpace(PSSMSTRM pStrm, size_t cb)
+{
+    Assert(pStrm->pCur);
+    Assert(pStrm->off + cb <= RT_SIZEOFMEMB(SSMSTRMBUF, abData));
+    pStrm->off += (uint32_t)cb;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Marks the end of the stream.
+ *
+ * This will cause the I/O thread to quit waiting for more buffers.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ */
+static int ssmR3StrmSetEnd(PSSMSTRM pStrm)
+{
+    Assert(pStrm->fWrite);
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_UNLIKELY(!pStrm->pCur))
+    {
+        pBuf = ssmR3StrmGetFreeBuf(pStrm);
+        if (!pBuf)
+            return pStrm->rc;
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+    }
+    pBuf->fEndOfStream = true;
+    ssmR3StrmFlushCurBuf(pStrm);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Read more from the stream.
+ *
+ * @returns VBox status code. VERR_EOF gets translated into VINF_EOF.
+ * @param   pStrm       The stream handle.
+ *
+ * @thread  The I/O thread when we got one, otherwise the stream user.
+ */
+static int ssmR3StrmReadMore(PSSMSTRM pStrm)
+{
+    int rc;
+    Log6(("ssmR3StrmReadMore:\n"));
+
+    /*
+     * Undo seek done by ssmR3StrmPeekAt.
+     */
+    if (pStrm->fNeedSeek)
+    {
+        rc = pStrm->pOps->pfnSeek(pStrm->pvUser, pStrm->offNeedSeekTo, RTFILE_SEEK_BEGIN, NULL);
+        if (RT_FAILURE(rc))
+        {
+            if (ssmR3StrmSetError(pStrm, rc))
+                LogRel(("ssmR3StrmReadMore: RTFileSeek(,%#llx,) failed with rc=%Rrc\n", pStrm->offNeedSeekTo, rc));
+            return rc;
+        }
+        pStrm->fNeedSeek     = false;
+        pStrm->offNeedSeekTo = UINT64_MAX;
+    }
+
+    /*
+     * Get a free buffer and try fill it up.
+     */
+    PSSMSTRMBUF pBuf = ssmR3StrmGetFreeBuf(pStrm);
+    if (!pBuf)
+        return pStrm->rc;
+
+    pBuf->offStream = pStrm->pOps->pfnTell(pStrm->pvUser);
+    size_t cbRead   = sizeof(pBuf->abData);
+    rc = pStrm->pOps->pfnRead(pStrm->pvUser, pBuf->offStream, &pBuf->abData[0], cbRead, &cbRead);
+    if (    RT_SUCCESS(rc)
+        &&  cbRead > 0)
+    {
+        pBuf->cb           = (uint32_t)cbRead;
+        pBuf->fEndOfStream = false;
+        Log6(("ssmR3StrmReadMore: %#010llx %#x\n", pBuf->offStream, pBuf->cb));
+        ssmR3StrmPutBuf(pStrm, pBuf);
+    }
+    else if (   (   RT_SUCCESS_NP(rc)
+                 && cbRead == 0)
+             || rc == VERR_EOF)
+    {
+        pBuf->cb           = 0;
+        pBuf->fEndOfStream = true;
+        Log6(("ssmR3StrmReadMore: %#010llx 0 EOF!\n", pBuf->offStream));
+        ssmR3StrmPutBuf(pStrm, pBuf);
+        rc = VINF_EOF;
+    }
+    else
+    {
+        Log6(("ssmR3StrmReadMore: %#010llx rc=%Rrc!\n", pBuf->offStream, rc));
+        if (ssmR3StrmSetError(pStrm, rc))
+            LogRel(("ssmR3StrmReadMore: RTFileRead(,,%#x,) -> %Rrc at offset %#llx\n",
+                    sizeof(pBuf->abData), rc, pBuf->offStream));
+        ssmR3StrmPutFreeBuf(pStrm, pBuf);
+    }
+    return rc;
+}
+
+
+/**
+ * Stream input routine.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   pvBuf       Where to put what we read.
+ * @param   cbToRead    How much to read.
+ */
+static int ssmR3StrmRead(PSSMSTRM pStrm, void *pvBuf, size_t cbToRead)
+{
+    AssertReturn(cbToRead > 0, VINF_SUCCESS);
+    Assert(!pStrm->fWrite);
+
+    /*
+     * Read from the current buffer if we got one.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_LIKELY(pBuf))
+    {
+        Assert(pStrm->off <= pBuf->cb);
+        uint32_t cbLeft = pBuf->cb - pStrm->off;
+        if (cbLeft >= cbToRead)
+        {
+            memcpy(pvBuf, &pBuf->abData[pStrm->off], cbToRead);
+            pStrm->off += (uint32_t)cbToRead;
+            Assert(pStrm->off <= pBuf->cb);
+            return VINF_SUCCESS;
+        }
+        if (cbLeft)
+        {
+            memcpy(pvBuf, &pBuf->abData[pStrm->off], cbLeft);
+            pStrm->off += cbLeft;
+            cbToRead   -= cbLeft;
+            pvBuf       = (uint8_t *)pvBuf + cbLeft;
+        }
+        else if (pBuf->fEndOfStream)
+            return VERR_EOF;
+        Assert(pStrm->off == pBuf->cb);
+    }
+
+    /*
+     * Get more buffers from the stream.
+     */
+    int rc = VINF_SUCCESS;
+    do
+    {
+        /*
+         * Check for EOF first - never flush the EOF buffer.
+         */
+        if (    pBuf
+            &&  pBuf->fEndOfStream)
+            return VERR_EOF;
+
+        /*
+         * Flush the current buffer and get the next one.
+         */
+        ssmR3StrmFlushCurBuf(pStrm);
+        pBuf = ssmR3StrmGetBuf(pStrm);
+        if (!pBuf)
+        {
+            rc = pStrm->rc;
+            break;
+        }
+        pStrm->pCur = pBuf;
+        Assert(pStrm->off == 0);
+        Assert(pStrm->offCurStream == pBuf->offStream);
+        if (!pBuf->cb)
+        {
+            Assert(pBuf->fEndOfStream);
+            return VERR_EOF;
+        }
+
+        /*
+         * Read data from the buffer.
+         */
+        uint32_t cbCopy = pBuf->cb;
+        if (cbCopy > cbToRead)
+            cbCopy = (uint32_t)cbToRead;
+        memcpy(pvBuf, &pBuf->abData[0], cbCopy);
+        pStrm->off  = cbCopy;
+        cbToRead   -= cbCopy;
+        pvBuf       = (uint8_t *)pvBuf + cbCopy;
+        Assert(!pStrm->pCur || pStrm->off <= pStrm->pCur->cb);
+    } while (cbToRead > 0);
+
+    return rc;
+}
+
+
+/**
+ * Reads data from the stream but instead of copying it to some output buffer
+ * the caller gets a pointer to into the current stream buffer.
+ *
+ * The returned pointer becomes invalid after the next stream operation!
+ *
+ * @returns Pointer to the read data residing in the stream buffer.  NULL is
+ *          returned if the request amount of data isn't available in the
+ *          buffer.  The caller must fall back on ssmR3StrmRead when this
+ *          happens.
+ *
+ * @param   pStrm       The stream handle.
+ * @param   cbToRead    The number of bytes to tread.
+ */
+static uint8_t const *ssmR3StrmReadDirect(PSSMSTRM pStrm, size_t cbToRead)
+{
+    AssertReturn(cbToRead > 0, VINF_SUCCESS);
+    Assert(!pStrm->fWrite);
+
+    /*
+     * Too lazy to fetch more data for the odd case that we're
+     * exactly at the boundary between two buffers.
+     */
+    PSSMSTRMBUF pBuf = pStrm->pCur;
+    if (RT_LIKELY(pBuf))
+    {
+        Assert(pStrm->off <= pBuf->cb);
+        uint32_t cbLeft = pBuf->cb - pStrm->off;
+        if (cbLeft >= cbToRead)
+        {
+            uint8_t const *pb = &pBuf->abData[pStrm->off];
+            pStrm->off += (uint32_t)cbToRead;
+            Assert(pStrm->off <= pBuf->cb);
+            return pb;
+        }
+    }
+    return NULL;
+}
+
+
+/**
+ * Check that the stream is OK and flush data that is getting old
+ *
+ * The checking is mainly for testing for cancellation and out of space
+ * conditions.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ */
+static int ssmR3StrmCheckAndFlush(PSSMSTRM pStrm)
+{
+    int rc = pStrm->pOps->pfnIsOk(pStrm->pvUser);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    if (    pStrm->fWrite
+        &&  pStrm->hIoThread != NIL_RTTHREAD
+        &&  !pStrm->pHead /* the worker is probably idle */
+        &&  pStrm->pCur
+        &&  RTTimeNanoTS() - pStrm->pCur->NanoTS > 500*1000*1000 /* 0.5s */
+       )
+        ssmR3StrmFlushCurBuf(pStrm);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Tell current stream position.
+ *
+ * @returns stream position.
+ * @param   pStrm       The stream handle.
+ */
+static uint64_t ssmR3StrmTell(PSSMSTRM pStrm)
+{
+    return pStrm->offCurStream + pStrm->off;
+}
+
+
+/**
+ * Gets the intermediate stream CRC up to the current position.
+ *
+ * @returns CRC.
+ * @param   pStrm       The stream handle.
+ */
+static uint32_t ssmR3StrmCurCRC(PSSMSTRM pStrm)
+{
+    if (!pStrm->fChecksummed)
+        return 0;
+    if (pStrm->offStreamCRC < pStrm->off)
+    {
+        PSSMSTRMBUF pBuf = pStrm->pCur; Assert(pBuf);
+        pStrm->u32StreamCRC = RTCrc32Process(pStrm->u32StreamCRC, &pBuf->abData[pStrm->offStreamCRC], pStrm->off - pStrm->offStreamCRC);
+        pStrm->offStreamCRC = pStrm->off;
+    }
+    else
+        Assert(pStrm->offStreamCRC == pStrm->off);
+    return pStrm->u32StreamCRC;
+}
+
+
+/**
+ * Gets the final stream CRC up to the current position.
+ *
+ * @returns CRC.
+ * @param   pStrm       The stream handle.
+ */
+static uint32_t ssmR3StrmFinalCRC(PSSMSTRM pStrm)
+{
+    if (!pStrm->fChecksummed)
+        return 0;
+    return RTCrc32Finish(ssmR3StrmCurCRC(pStrm));
+}
+
+
+/**
+ * Disables checksumming of the stream.
+ *
+ * @param   pStrm       The stream handle.
+ */
+static void ssmR3StrmDisableChecksumming(PSSMSTRM pStrm)
+{
+    pStrm->fChecksummed = false;
+}
+
+
+/**
+ * Used by SSMR3Seek to position the stream at the new unit.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The strem handle.
+ * @param   off         The seek offset.
+ * @param   uMethod     The seek method.
+ * @param   u32CurCRC   The current CRC at the seek position.
+ */
+static int ssmR3StrmSeek(PSSMSTRM pStrm, int64_t off, uint32_t uMethod, uint32_t u32CurCRC)
+{
+    AssertReturn(!pStrm->fWrite, VERR_NOT_SUPPORTED);
+    AssertReturn(pStrm->hIoThread == NIL_RTTHREAD, VERR_WRONG_ORDER);
+
+    uint64_t offStream;
+    int rc = pStrm->pOps->pfnSeek(pStrm->pvUser, off, uMethod, &offStream);
+    if (RT_SUCCESS(rc))
+    {
+        pStrm->fNeedSeek    = false;
+        pStrm->offNeedSeekTo= UINT64_MAX;
+        pStrm->offCurStream = offStream;
+        pStrm->off          = 0;
+        pStrm->offStreamCRC = 0;
+        if (pStrm->fChecksummed)
+            pStrm->u32StreamCRC = u32CurCRC;
+        if (pStrm->pCur)
+        {
+            ssmR3StrmPutFreeBuf(pStrm, pStrm->pCur);
+            pStrm->pCur = NULL;
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Skip some bytes in the stream.
+ *
+ * This is only used if someone didn't read all of their data in the V1 format,
+ * so don't bother making this very efficient yet.
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle.
+ * @param   offDst      The destination offset.
+ */
+static int ssmR3StrmSkipTo(PSSMSTRM pStrm, uint64_t offDst)
+{
+    /* dead simple - lazy bird! */
+    for (;;)
+    {
+        uint64_t offCur = ssmR3StrmTell(pStrm);
+        AssertReturn(offCur <= offDst, VERR_INTERNAL_ERROR_4);
+        if (offCur == offDst)
+            return VINF_SUCCESS;
+
+        uint8_t abBuf[4096];
+        size_t cbToRead = RT_MIN(sizeof(abBuf), offDst - offCur);
+        int rc = ssmR3StrmRead(pStrm, abBuf, cbToRead);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+}
+
+
+/**
+ * Get the size of the file.
+ *
+ * This does not work for non-file streams!
+ *
+ * @returns The file size, or UINT64_MAX if not a file stream.
+ * @param   pStrm       The stream handle.
+ */
+static uint64_t ssmR3StrmGetSize(PSSMSTRM pStrm)
+{
+    uint64_t cbFile;
+    int rc = pStrm->pOps->pfnSize(pStrm->pvUser, &cbFile);
+    AssertLogRelRCReturn(rc, UINT64_MAX);
+    return cbFile;
+}
+
+
+/***
+ * Tests if the stream is a file stream or not.
+ *
+ * @returns true / false.
+ * @param   pStrm       The stream handle.
+ */
+static bool ssmR3StrmIsFile(PSSMSTRM pStrm)
+{
+    return pStrm->pOps == &g_ssmR3FileOps;
+}
+
+
+/**
+ * Peeks at data in a file stream without buffering anything (or upsetting
+ * the buffering for that matter).
+ *
+ * @returns VBox status code.
+ * @param   pStrm       The stream handle
+ * @param   off         The offset to start peeking at. Use a negative offset to
+ *                      peek at something relative to the end of the file.
+ * @param   pvBuf       Output buffer.
+ * @param   cbToRead    How much to read.
+ * @param   poff        Where to optionally store the position.  Useful when
+ *                      using a negative off.
+ *
+ * @remarks Failures occurring while peeking will not be raised on the stream.
+ */
+static int ssmR3StrmPeekAt(PSSMSTRM pStrm, RTFOFF off, void *pvBuf, size_t cbToRead, uint64_t *poff)
+{
+    AssertReturn(!pStrm->fWrite, VERR_NOT_SUPPORTED);
+    AssertReturn(pStrm->hIoThread == NIL_RTTHREAD, VERR_WRONG_ORDER);
+
+    if (!pStrm->fNeedSeek)
+    {
+        pStrm->fNeedSeek     = true;
+        pStrm->offNeedSeekTo = pStrm->offCurStream + (pStrm->pCur ? pStrm->pCur->cb : 0);
+    }
+    uint64_t offActual;
+    int rc = pStrm->pOps->pfnSeek(pStrm->pvUser, off, off >= 0 ? RTFILE_SEEK_BEGIN : RTFILE_SEEK_END, &offActual);
+    if (RT_SUCCESS(rc))
+    {
+        if (poff)
+            *poff = offActual;
+        rc = pStrm->pOps->pfnRead(pStrm->pvUser, offActual, pvBuf, cbToRead, NULL);
+    }
+
+    return rc;
+}
+
+
+/**
+ * The I/O thread.
+ *
+ * @returns VINF_SUCCESS (ignored).
+ * @param   hSelf       The thread handle.
+ * @param   pvStrm      The stream handle.
+ */
+static DECLCALLBACK(int) ssmR3StrmIoThread(RTTHREAD hSelf, void *pvStrm)
+{
+    PSSMSTRM pStrm = (PSSMSTRM)pvStrm;
+    ASMAtomicWriteHandle(&pStrm->hIoThread, hSelf); /* paranoia */
+
+    Log(("ssmR3StrmIoThread: starts working\n"));
+    if (pStrm->fWrite)
+    {
+        /*
+         * Write until error or terminated.
+         */
+        for (;;)
+        {
+            int rc = ssmR3StrmWriteBuffers(pStrm);
+            if (    RT_FAILURE(rc)
+                ||  rc == VINF_EOF)
+            {
+                Log(("ssmR3StrmIoThread: quitting writing with rc=%Rrc.\n", rc));
+                break;
+            }
+            if (RT_FAILURE(pStrm->rc))
+            {
+                Log(("ssmR3StrmIoThread: quitting writing with stream rc=%Rrc\n", pStrm->rc));
+                break;
+            }
+
+            if (ASMAtomicReadBool(&pStrm->fTerminating))
+            {
+                if (!ASMAtomicReadPtrT(&pStrm->pHead, PSSMSTRMBUF))
+                {
+                    Log(("ssmR3StrmIoThread: quitting writing because of pending termination.\n"));
+                    break;
+                }
+                Log(("ssmR3StrmIoThread: postponing termination because of pending buffers.\n"));
+            }
+            else if (!ASMAtomicReadPtrT(&pStrm->pHead, PSSMSTRMBUF))
+            {
+                rc = RTSemEventWait(pStrm->hEvtHead, RT_INDEFINITE_WAIT);
+                AssertLogRelRC(rc);
+            }
+        }
+
+        if (!ASMAtomicReadBool(&pStrm->fTerminating))
+            RTSemEventSignal(pStrm->hEvtFree);
+    }
+    else
+    {
+        /*
+         * Read until end of file, error or termination.
+         */
+        for (;;)
+        {
+            if (ASMAtomicReadBool(&pStrm->fTerminating))
+            {
+                Log(("ssmR3StrmIoThread: quitting reading because of pending termination.\n"));
+                break;
+            }
+
+            int rc = ssmR3StrmReadMore(pStrm);
+            if (    RT_FAILURE(rc)
+                ||  rc == VINF_EOF)
+            {
+                Log(("ssmR3StrmIoThread: quitting reading with rc=%Rrc\n", rc));
+                break;
+            }
+            if (RT_FAILURE(pStrm->rc))
+            {
+                Log(("ssmR3StrmIoThread: quitting reading with stream rc=%Rrc\n", pStrm->rc));
+                break;
+            }
+        }
+
+        if (!ASMAtomicReadBool(&pStrm->fTerminating))
+            RTSemEventSignal(pStrm->hEvtHead);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Starts the I/O thread for the specified stream.
+ *
+ * @param   pStrm       The stream handle.
+ */
+static void ssmR3StrmStartIoThread(PSSMSTRM pStrm)
+{
+    Assert(pStrm->hIoThread == NIL_RTTHREAD);
+
+    RTTHREAD hThread;
+    int rc = RTThreadCreate(&hThread, ssmR3StrmIoThread, pStrm, 0, RTTHREADTYPE_IO, RTTHREADFLAGS_WAITABLE, "SSM-IO");
+    AssertRCReturnVoid(rc);
+    ASMAtomicWriteHandle(&pStrm->hIoThread, hThread); /* paranoia */
+}
+
+
+/**
+ * Works the progress calculation for non-live saves and restores.
+ *
+ * @param   pSSM        The SSM handle.
+ * @param   cbAdvance   Number of bytes to advance (with in the current unit).
+ */
+static void ssmR3ProgressByByte(PSSMHANDLE pSSM, uint64_t cbAdvance)
+{
+    if (!pSSM->fLiveSave)
+    {
+        /* Can't advance it beyond the estimated end of the unit. */
+        uint64_t cbLeft = pSSM->offEstUnitEnd - pSSM->offEst;
+        if (cbAdvance > cbLeft)
+            cbAdvance = cbLeft;
+        pSSM->offEst += cbAdvance;
+
+        /* uPercentPrepare% prepare, xx% exec, uPercentDone% done+crc. This is not
+           quite right for live save, but the non-live stage there is very short. */
+        while (   pSSM->offEst >= pSSM->offEstProgress
+               && pSSM->uPercent <= 100 - pSSM->uPercentDone)
+        {
+            if (pSSM->pfnProgress)
+                pSSM->pfnProgress(pSSM->pVM, pSSM->uPercent, pSSM->pvUser);
+            pSSM->uPercent++;
+            pSSM->offEstProgress = (pSSM->uPercent - pSSM->uPercentPrepare - pSSM->uPercentLive) * pSSM->cbEstTotal
+                                 / (100 - pSSM->uPercentDone - pSSM->uPercentPrepare - pSSM->uPercentLive);
+        }
+    }
+}
+
+
+/**
+ * Makes the SSM operation cancellable or not (via SSMR3Cancel).
+ *
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The saved state handle. (SSMHANDLE::rc may be set.)
+ * @param   fCancellable    The new state.
+ */
+static void ssmR3SetCancellable(PVM pVM, PSSMHANDLE pSSM, bool fCancellable)
+{
+    RTCritSectEnter(&pVM->ssm.s.CancelCritSect);
+    if (fCancellable)
+    {
+        Assert(!pVM->ssm.s.pSSM);
+        pVM->ssm.s.pSSM = pSSM;
+    }
+    else
+    {
+        if (pVM->ssm.s.pSSM == pSSM)
+            pVM->ssm.s.pSSM = NULL;
+
+        uint32_t fCancelled = ASMAtomicUoReadU32(&pSSM->fCancelled);
+        if (    fCancelled == SSMHANDLE_CANCELLED
+            &&  RT_SUCCESS(pSSM->rc))
+            pSSM->rc = VERR_SSM_CANCELLED;
+    }
+
+    RTCritSectLeave(&pVM->ssm.s.CancelCritSect);
+}
+
+
+/**
+ * Gets the host bit count of the saved state.
+ *
+ * Works for on both save and load handles.
+ *
+ * @returns 32 or 64.
+ * @param   pSSM                The saved state handle.
+ */
+DECLINLINE(uint32_t) ssmR3GetHostBits(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+    {
+        uint32_t cBits = pSSM->u.Read.cHostBits;
+        if (cBits)
+            return cBits;
+    }
+    return HC_ARCH_BITS;
+}
+
+
+/**
+ * Saved state origins on a host using 32-bit MSC?
+ *
+ * Works for on both save and load handles.
+ *
+ * @returns true/false.
+ * @param   pSSM                The saved state handle.
+ */
+DECLINLINE(bool) ssmR3IsHostMsc32(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+        return pSSM->u.Read.fIsHostMsc32;
+    return SSM_HOST_IS_MSC_32;
+}
+
+#ifndef SSM_STANDALONE
+
+/**
+ * Finishes a data unit.
+ * All buffers and compressor instances are flushed and destroyed.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataWriteFinish(PSSMHANDLE pSSM)
+{
+    //Log2(("ssmR3DataWriteFinish: %#010llx start\n", ssmR3StrmTell(&pSSM->Strm)));
+    int rc = ssmR3DataFlushBuffer(pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        pSSM->offUnit = UINT64_MAX;
+        return VINF_SUCCESS;
+    }
+
+    if (RT_SUCCESS(pSSM->rc))
+        pSSM->rc = rc;
+    Log2(("ssmR3DataWriteFinish: failure rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Begins writing the data of a data unit.
+ *
+ * Errors are signalled via pSSM->rc.
+ *
+ * @param   pSSM            The saved state handle.
+ */
+static void ssmR3DataWriteBegin(PSSMHANDLE pSSM)
+{
+    pSSM->offUnit = 0;
+}
+
+
+/**
+ * Writes a record to the current data item in the saved state file.
+ *
+ * @returns VBox status code. Sets pSSM->rc on failure.
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+static int ssmR3DataWriteRaw(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    Log2(("ssmR3DataWriteRaw: %08llx|%08llx: pvBuf=%p cbBuf=%#x %.*Rhxs%s\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pvBuf, cbBuf, RT_MIN(cbBuf, SSM_LOG_BYTES), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
+
+    /*
+     * Check that everything is fine.
+     */
+    if (RT_FAILURE(pSSM->rc))
+        return pSSM->rc;
+
+    /*
+     * Write the data item in 1MB chunks for progress indicator reasons.
+     */
+    while (cbBuf > 0)
+    {
+        size_t cbChunk = RT_MIN(cbBuf, _1M);
+        int rc = ssmR3StrmWrite(&pSSM->Strm, pvBuf, cbChunk);
+        if (RT_FAILURE(rc))
+            return rc;
+        pSSM->offUnit += cbChunk;
+        cbBuf -= cbChunk;
+        pvBuf = (char *)pvBuf + cbChunk;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Writes a record header for the specified amount of data.
+ *
+ * @returns VBox status code. Sets pSSM->rc on failure.
+ * @param   pSSM            The saved state handle
+ * @param   cb              The amount of data.
+ * @param   u8TypeAndFlags  The record type and flags.
+ */
+static int ssmR3DataWriteRecHdr(PSSMHANDLE pSSM, size_t cb, uint8_t u8TypeAndFlags)
+{
+    size_t  cbHdr;
+    uint8_t abHdr[8];
+    abHdr[0] = u8TypeAndFlags;
+    if (cb < 0x80)
+    {
+        cbHdr = 2;
+        abHdr[1] = (uint8_t)cb;
+    }
+    else if (cb < 0x00000800)
+    {
+        cbHdr = 3;
+        abHdr[1] = (uint8_t)(0xc0 | (cb >> 6));
+        abHdr[2] = (uint8_t)(0x80 | (cb & 0x3f));
+    }
+    else if (cb < 0x00010000)
+    {
+        cbHdr = 4;
+        abHdr[1] = (uint8_t)(0xe0 | (cb >> 12));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 6) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | (cb & 0x3f));
+    }
+    else if (cb < 0x00200000)
+    {
+        cbHdr = 5;
+        abHdr[1] = (uint8_t)(0xf0 |  (cb >> 18));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
+        abHdr[4] = (uint8_t)(0x80 |  (cb        & 0x3f));
+    }
+    else if (cb < 0x04000000)
+    {
+        cbHdr = 6;
+        abHdr[1] = (uint8_t)(0xf8 |  (cb >> 24));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 18) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
+        abHdr[4] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
+        abHdr[5] = (uint8_t)(0x80 |  (cb        & 0x3f));
+    }
+    else if (cb <= 0x7fffffff)
+    {
+        cbHdr = 7;
+        abHdr[1] = (uint8_t)(0xfc |  (cb >> 30));
+        abHdr[2] = (uint8_t)(0x80 | ((cb >> 24) & 0x3f));
+        abHdr[3] = (uint8_t)(0x80 | ((cb >> 18) & 0x3f));
+        abHdr[4] = (uint8_t)(0x80 | ((cb >> 12) & 0x3f));
+        abHdr[5] = (uint8_t)(0x80 | ((cb >>  6) & 0x3f));
+        abHdr[6] = (uint8_t)(0x80 | (cb & 0x3f));
+    }
+    else
+        AssertLogRelMsgFailedReturn(("cb=%#x\n", cb), pSSM->rc = VERR_INTERNAL_ERROR);
+
+    Log3(("ssmR3DataWriteRecHdr: %08llx|%08llx/%08x: Type=%02x fImportant=%RTbool cbHdr=%u\n",
+          ssmR3StrmTell(&pSSM->Strm) + cbHdr, pSSM->offUnit + cbHdr, cb, u8TypeAndFlags & SSM_REC_TYPE_MASK, !!(u8TypeAndFlags & SSM_REC_FLAGS_IMPORTANT), cbHdr));
+
+    return ssmR3DataWriteRaw(pSSM, &abHdr[0], cbHdr);
+}
+
+
+/**
+ * Worker that flushes the buffered data.
+ *
+ * @returns VBox status code. Will set pSSM->rc on error.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataFlushBuffer(PSSMHANDLE pSSM)
+{
+    /*
+     * Check how much there current is in the buffer.
+     */
+    uint32_t cb = pSSM->u.Write.offDataBuffer;
+    if (!cb)
+        return pSSM->rc;
+    pSSM->u.Write.offDataBuffer = 0;
+
+    /*
+     * Write a record header and then the data.
+     * (No need for fancy optimizations here any longer since the stream is
+     * fully buffered.)
+     */
+    int rc = ssmR3DataWriteRecHdr(pSSM, cb, SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW);
+    if (RT_SUCCESS(rc))
+        rc = ssmR3DataWriteRaw(pSSM, pSSM->u.Write.abDataBuffer, cb);
+    ssmR3ProgressByByte(pSSM, cb);
+    return rc;
+}
+
+
+/**
+ * ssmR3DataWrite worker that writes big stuff.
+ *
+ * @returns VBox status code
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+static int ssmR3DataWriteBig(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    int rc = ssmR3DataFlushBuffer(pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Split it up into compression blocks.
+         */
+        for (;;)
+        {
+            AssertCompile(SSM_ZIP_BLOCK_SIZE == PAGE_SIZE);
+            if (    cbBuf >= SSM_ZIP_BLOCK_SIZE
+                && (    ((uintptr_t)pvBuf & 0xf)
+                    ||  !ASMMemIsZeroPage(pvBuf))
+               )
+            {
+                /*
+                 * Compress it.
+                 */
+                AssertCompile(1 + 3 + 1 + SSM_ZIP_BLOCK_SIZE < 0x00010000);
+                uint8_t *pb;
+                rc = ssmR3StrmReserveWriteBufferSpace(&pSSM->Strm, 1 + 3 + 1 + SSM_ZIP_BLOCK_SIZE, &pb);
+                if (RT_FAILURE(rc))
+                    break;
+                size_t cbRec = SSM_ZIP_BLOCK_SIZE - (SSM_ZIP_BLOCK_SIZE / 16);
+                rc = RTZipBlockCompress(RTZIPTYPE_LZF, RTZIPLEVEL_FAST, 0 /*fFlags*/,
+                                        pvBuf, SSM_ZIP_BLOCK_SIZE,
+                                        pb + 1 + 3 + 1, cbRec, &cbRec);
+                if (RT_SUCCESS(rc))
+                {
+                    pb[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW_LZF;
+                    pb[4] = SSM_ZIP_BLOCK_SIZE / _1K;
+                    cbRec += 1;
+                }
+                else
+                {
+                    pb[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW;
+                    memcpy(&pb[4], pvBuf, SSM_ZIP_BLOCK_SIZE);
+                    cbRec = SSM_ZIP_BLOCK_SIZE;
+                }
+                pb[1] = (uint8_t)(0xe0 | ( cbRec >> 12));
+                pb[2] = (uint8_t)(0x80 | ((cbRec >>  6) & 0x3f));
+                pb[3] = (uint8_t)(0x80 | ( cbRec        & 0x3f));
+                cbRec += 1 + 3;
+                rc = ssmR3StrmCommitWriteBufferSpace(&pSSM->Strm, cbRec);
+                if (RT_FAILURE(rc))
+                    break;
+
+                pSSM->offUnit += cbRec;
+                ssmR3ProgressByByte(pSSM, SSM_ZIP_BLOCK_SIZE);
+
+                /* advance */
+                if (cbBuf == SSM_ZIP_BLOCK_SIZE)
+                    return VINF_SUCCESS;
+                cbBuf -= SSM_ZIP_BLOCK_SIZE;
+                pvBuf = (uint8_t const*)pvBuf + SSM_ZIP_BLOCK_SIZE;
+            }
+            else if (cbBuf >= SSM_ZIP_BLOCK_SIZE)
+            {
+                /*
+                 * Zero block.
+                 */
+                uint8_t abRec[3];
+                abRec[0] = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW_ZERO;
+                abRec[1] = 1;
+                abRec[2] = SSM_ZIP_BLOCK_SIZE / _1K;
+                Log3(("ssmR3DataWriteBig: %08llx|%08llx/%08x: ZERO\n", ssmR3StrmTell(&pSSM->Strm) + 2, pSSM->offUnit + 2, 1));
+                rc = ssmR3DataWriteRaw(pSSM, &abRec[0], sizeof(abRec));
+                if (RT_FAILURE(rc))
+                    break;
+
+                /* advance */
+                ssmR3ProgressByByte(pSSM, SSM_ZIP_BLOCK_SIZE);
+                if (cbBuf == SSM_ZIP_BLOCK_SIZE)
+                    return VINF_SUCCESS;
+                cbBuf -= SSM_ZIP_BLOCK_SIZE;
+                pvBuf = (uint8_t const*)pvBuf + SSM_ZIP_BLOCK_SIZE;
+            }
+            else
+            {
+                /*
+                 * Less than one block left, store it the simple way.
+                 */
+                rc = ssmR3DataWriteRecHdr(pSSM, cbBuf, SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_RAW);
+                if (RT_SUCCESS(rc))
+                    rc = ssmR3DataWriteRaw(pSSM, pvBuf, cbBuf);
+                ssmR3ProgressByByte(pSSM, cbBuf);
+                break;
+            }
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * ssmR3DataWrite worker that is called when there isn't enough room in the
+ * buffer for the current chunk of data.
+ *
+ * This will first flush the buffer and then add the new bits to it.
+ *
+ * @returns VBox status code
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+static int ssmR3DataWriteFlushAndBuffer(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    int rc = ssmR3DataFlushBuffer(pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        memcpy(&pSSM->u.Write.abDataBuffer[0], pvBuf, cbBuf);
+        pSSM->u.Write.offDataBuffer = (uint32_t)cbBuf;
+    }
+    return rc;
+}
+
+
+/**
+ * Writes data to the current data unit.
+ *
+ * This is an inlined wrapper that optimizes the small writes that so many of
+ * the APIs make.
+ *
+ * @returns VBox status code
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           The bits to write.
+ * @param   cbBuf           The number of bytes to write.
+ */
+DECLINLINE(int) ssmR3DataWrite(PSSMHANDLE pSSM, const void *pvBuf, size_t cbBuf)
+{
+    if (cbBuf > sizeof(pSSM->u.Write.abDataBuffer) / 8)
+        return ssmR3DataWriteBig(pSSM, pvBuf, cbBuf);
+    if (!cbBuf)
+        return VINF_SUCCESS;
+
+    uint32_t off = pSSM->u.Write.offDataBuffer;
+    if (RT_UNLIKELY(cbBuf + off > sizeof(pSSM->u.Write.abDataBuffer)))
+        return ssmR3DataWriteFlushAndBuffer(pSSM, pvBuf, cbBuf);
+
+    memcpy(&pSSM->u.Write.abDataBuffer[off], pvBuf, cbBuf);
+    pSSM->u.Write.offDataBuffer = off + (uint32_t)cbBuf;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Puts a structure.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   paFields        The array of structure fields descriptions.
+ *                          The array must be terminated by a SSMFIELD_ENTRY_TERM().
+ */
+VMMR3DECL(int) SSMR3PutStruct(PSSMHANDLE pSSM, const void *pvStruct, PCSSMFIELD paFields)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+    /* begin marker. */
+    int rc = SSMR3PutU32(pSSM, SSMR3STRUCT_BEGIN);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* put the fields */
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        uint8_t const *pbField = (uint8_t const *)pvStruct + pCur->off;
+        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+        {
+            case SSMFIELDTRANS_NO_TRANSFORMATION:
+                rc = ssmR3DataWrite(pSSM, pbField, pCur->cb);
+                break;
+
+            case SSMFIELDTRANS_GCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPtr(pSSM, *(PRTGCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_GCPHYS:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPhys(pSSM, *(PRTGCPHYS)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutRCPtr(pSSM, *(PRTRCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR_ARRAY:
+            {
+                uint32_t const cEntries = pCur->cb / sizeof(RTRCPTR);
+                AssertMsgReturn(pCur->cb == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = VINF_SUCCESS;
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = SSMR3PutRCPtr(pSSM, ((PRTRCPTR)pbField)[i]);
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("%#x\n", pCur->pfnGetPutOrTransformer), VERR_SSM_FIELD_COMPLEX);
+        }
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /* end marker */
+    return SSMR3PutU32(pSSM, SSMR3STRUCT_END);
+}
+
+
+/**
+ * SSMR3PutStructEx helper that puts a HCPTR that is used as a NULL indicator.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pv              The value to put.
+ * @param   fFlags          SSMSTRUCT_FLAGS_XXX.
+ */
+DECLINLINE(int) ssmR3PutHCPtrNI(PSSMHANDLE pSSM, void *pv, uint32_t fFlags)
+{
+    int rc;
+    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+        rc = ssmR3DataWrite(pSSM, &pv, sizeof(void *));
+    else
+        rc = SSMR3PutBool(pSSM, pv != NULL);
+    return rc;
+}
+
+
+/**
+ * SSMR3PutStructEx helper that puts an arbitrary number of zeros.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   cbToFill        The number of zeros to stuff into the state.
+ */
+static int ssmR3PutZeros(PSSMHANDLE pSSM, uint32_t cbToFill)
+{
+    while (cbToFill > 0)
+    {
+        uint32_t cb = RT_MIN(sizeof(g_abZero), cbToFill);
+        int rc = ssmR3DataWrite(pSSM, g_abZero, cb);
+        if (RT_FAILURE(rc))
+            return rc;
+        cbToFill -= cb;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Puts a structure, extended API.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   cbStruct        The size of the struct (use for validation only).
+ * @param   fFlags          Combination of SSMSTRUCT_FLAGS_XXX defines.
+ * @param   paFields        The array of structure fields descriptions. The
+ *                          array must be terminated by a SSMFIELD_ENTRY_TERM().
+ * @param   pvUser          User argument for any callbacks that paFields might
+ *                          contain.
+ */
+VMMR3DECL(int) SSMR3PutStructEx(PSSMHANDLE pSSM, const void *pvStruct, size_t cbStruct,
+                                uint32_t fFlags, PCSSMFIELD paFields, void *pvUser)
+{
+    int rc;
+
+    /*
+     * Validation.
+     */
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertMsgReturn(!(fFlags & ~SSMSTRUCT_FLAGS_VALID_MASK), ("%#x\n", fFlags), VERR_INVALID_PARAMETER);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+
+    /*
+     * Begin marker.
+     */
+    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
+    {
+        rc = SSMR3PutU32(pSSM, SSMR3STRUCT_BEGIN);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Put the fields
+     */
+    uint32_t    off          = 0;
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        uint32_t const offField =    (!SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer) || pCur->off != UINT32_MAX / 2)
+                                  &&  !SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? pCur->off
+                                : off;
+        uint32_t const cbField  = SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? 0
+                                : SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer)
+                                ? RT_HIWORD(pCur->cb)
+                                : pCur->cb;
+        AssertMsgReturn(   cbField            <= cbStruct
+                        && offField + cbField <= cbStruct
+                        && offField + cbField >= offField,
+                        ("off=%#x cb=%#x cbStruct=%#x (%s)\n", cbField, offField, cbStruct, pCur->pszName),
+                        VERR_SSM_FIELD_OUT_OF_BOUNDS);
+        AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                        || off == offField,
+                        ("off=%#x offField=%#x (%s)\n", off, offField, pCur->pszName),
+                        VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+        rc = VINF_SUCCESS;
+        uint8_t const *pbField = (uint8_t const *)pvStruct + offField;
+        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+        {
+            case SSMFIELDTRANS_NO_TRANSFORMATION:
+                rc = ssmR3DataWrite(pSSM, pbField, cbField);
+                break;
+
+            case SSMFIELDTRANS_GCPHYS:
+                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPhys(pSSM, *(PRTGCPHYS)pbField);
+                break;
+
+            case SSMFIELDTRANS_GCPTR:
+                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutGCPtr(pSSM, *(PRTGCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR:
+                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3PutRCPtr(pSSM, *(PRTRCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR_ARRAY:
+            {
+                uint32_t const cEntries = cbField / sizeof(RTRCPTR);
+                AssertMsgReturn(cbField == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = SSMR3PutRCPtr(pSSM, ((PRTRCPTR)pbField)[i]);
+                break;
+            }
+
+            case SSMFIELDTRANS_HCPTR_NI:
+                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3PutHCPtrNI(pSSM, *(void * const *)pbField, fFlags);
+                break;
+
+            case SSMFIELDTRANS_HCPTR_NI_ARRAY:
+            {
+                uint32_t const cEntries = cbField / sizeof(void *);
+                AssertMsgReturn(cbField == cEntries * sizeof(void *) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = ssmR3PutHCPtrNI(pSSM, ((void * const *)pbField)[i], fFlags);
+                break;
+            }
+
+            case SSMFIELDTRANS_HCPTR_HACK_U32:
+                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                AssertMsgReturn(*(uintptr_t *)pbField <= UINT32_MAX, ("%p (%s)\n", *(uintptr_t *)pbField, pCur->pszName), VERR_SSM_FIELD_INVALID_VALUE);
+                rc = ssmR3DataWrite(pSSM, pbField, sizeof(uint32_t));
+                if ((fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE) && sizeof(void *) != sizeof(uint32_t))
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(uint32_t));
+                break;
+
+
+            case SSMFIELDTRANS_IGNORE:
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3PutZeros(pSSM, cbField);
+                break;
+
+            case SSMFIELDTRANS_IGN_GCPHYS:
+                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPHYS));
+                break;
+
+            case SSMFIELDTRANS_IGN_GCPTR:
+                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPTR));
+                break;
+
+            case SSMFIELDTRANS_IGN_RCPTR:
+                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTRCPTR));
+                break;
+
+            case SSMFIELDTRANS_IGN_HCPTR:
+                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(void *));
+                break;
+
+
+            case SSMFIELDTRANS_OLD:
+                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3PutZeros(pSSM, pCur->cb);
+                break;
+
+            case SSMFIELDTRANS_OLD_GCPHYS:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPHYS));
+                break;
+
+            case SSMFIELDTRANS_OLD_GCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTGCPTR));
+                break;
+
+            case SSMFIELDTRANS_OLD_RCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(RTRCPTR));
+                break;
+
+            case SSMFIELDTRANS_OLD_HCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(void *) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3DataWrite(pSSM, g_abZero, sizeof(void *));
+                break;
+
+            case SSMFIELDTRANS_OLD_PAD_HC:
+                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3PutZeros(pSSM, HC_ARCH_BITS == 64 ? RT_HIWORD(pCur->cb) : RT_LOWORD(pCur->cb));
+                break;
+
+            case SSMFIELDTRANS_OLD_PAD_MSC32:
+                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (SSM_HOST_IS_MSC_32)
+                    rc = ssmR3PutZeros(pSSM, pCur->cb);
+                break;
+
+
+            case SSMFIELDTRANS_PAD_HC:
+            case SSMFIELDTRANS_PAD_HC32:
+            case SSMFIELDTRANS_PAD_HC64:
+            case SSMFIELDTRANS_PAD_HC_AUTO:
+            case SSMFIELDTRANS_PAD_MSC32_AUTO:
+            {
+                uint32_t cb32    = RT_BYTE1(pCur->cb);
+                uint32_t cb64    = RT_BYTE2(pCur->cb);
+                uint32_t cbCtx   =    HC_ARCH_BITS == 64
+                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                       && !SSM_HOST_IS_MSC_32)
+                                 ? cb64 : cb32;
+                uint32_t cbSaved =    ssmR3GetHostBits(pSSM) == 64
+                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                       && !ssmR3IsHostMsc32(pSSM))
+                                 ? cb64 : cb32;
+                AssertMsgReturn(    cbField == cbCtx
+                                &&  (   (   pCur->off == UINT32_MAX / 2
+                                         && (   cbField == 0
+                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_HC_AUTO
+                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                            )
+                                        )
+                                     || (pCur->off != UINT32_MAX / 2 && cbField != 0)
+                                     )
+                                , ("cbField=%#x cb32=%#x cb64=%#x HC_ARCH_BITS=%u cbCtx=%#x cbSaved=%#x off=%#x\n",
+                                   cbField, cb32, cb64, HC_ARCH_BITS, cbCtx, cbSaved, pCur->off),
+                                VERR_SSM_FIELD_INVALID_PADDING_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = ssmR3PutZeros(pSSM, cbSaved);
+                break;
+            }
+
+            default:
+                AssertPtrReturn(pCur->pfnGetPutOrTransformer, VERR_SSM_FIELD_INVALID_CALLBACK);
+                rc = pCur->pfnGetPutOrTransformer(pSSM, pCur, (void *)pvStruct, fFlags, false /*fGetOrPut*/, pvUser);
+                break;
+        }
+        if (RT_FAILURE(rc))
+            return rc;
+
+        off = offField + cbField;
+    }
+    AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                    || off == cbStruct,
+                    ("off=%#x cbStruct=%#x\n", off, cbStruct),
+                    VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+    /*
+     * End marker
+     */
+    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
+    {
+        rc = SSMR3PutU32(pSSM, SSMR3STRUCT_END);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Saves a boolean item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   fBool           Item to save.
+ */
+VMMR3DECL(int) SSMR3PutBool(PSSMHANDLE pSSM, bool fBool)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    uint8_t u8 = fBool; /* enforce 1 byte size */
+    return ssmR3DataWrite(pSSM, &u8, sizeof(u8));
+}
+
+
+/**
+ * Saves a 8-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u8              Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU8(PSSMHANDLE pSSM, uint8_t u8)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u8, sizeof(u8));
+}
+
+
+/**
+ * Saves a 8-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   i8              Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS8(PSSMHANDLE pSSM, int8_t i8)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i8, sizeof(i8));
+}
+
+
+/**
+ * Saves a 16-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u16             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU16(PSSMHANDLE pSSM, uint16_t u16)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u16, sizeof(u16));
+}
+
+
+/**
+ * Saves a 16-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   i16             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS16(PSSMHANDLE pSSM, int16_t i16)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i16, sizeof(i16));
+}
+
+
+/**
+ * Saves a 32-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u32             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU32(PSSMHANDLE pSSM, uint32_t u32)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u32, sizeof(u32));
+}
+
+
+/**
+ * Saves a 32-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   i32             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS32(PSSMHANDLE pSSM, int32_t i32)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i32, sizeof(i32));
+}
+
+
+/**
+ * Saves a 64-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u64             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU64(PSSMHANDLE pSSM, uint64_t u64)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u64, sizeof(u64));
+}
+
+
+/**
+ * Saves a 64-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   i64             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS64(PSSMHANDLE pSSM, int64_t i64)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i64, sizeof(i64));
+}
+
+
+/**
+ * Saves a 128-bit unsigned integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u128            Item to save.
+ */
+VMMR3DECL(int) SSMR3PutU128(PSSMHANDLE pSSM, uint128_t u128)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u128, sizeof(u128));
+}
+
+
+/**
+ * Saves a 128-bit signed integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   i128            Item to save.
+ */
+VMMR3DECL(int) SSMR3PutS128(PSSMHANDLE pSSM, int128_t i128)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i128, sizeof(i128));
+}
+
+
+/**
+ * Saves a VBox unsigned integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u               Item to save.
+ */
+VMMR3DECL(int) SSMR3PutUInt(PSSMHANDLE pSSM, RTUINT u)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u, sizeof(u));
+}
+
+
+/**
+ * Saves a VBox signed integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   i               Item to save.
+ */
+VMMR3DECL(int) SSMR3PutSInt(PSSMHANDLE pSSM, RTINT i)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &i, sizeof(i));
+}
+
+
+/**
+ * Saves a GC natural unsigned integer item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u               Item to save.
+ *
+ * @deprecated Silly type, don't use it.
+ */
+VMMR3DECL(int) SSMR3PutGCUInt(PSSMHANDLE pSSM, RTGCUINT u)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u, sizeof(u));
+}
+
+
+/**
+ * Saves a GC unsigned integer register item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   u               Item to save.
+ */
+VMMR3DECL(int) SSMR3PutGCUIntReg(PSSMHANDLE pSSM, RTGCUINTREG u)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &u, sizeof(u));
+}
+
+
+/**
+ * Saves a 32 bits GC physical address item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPhys          The item to save
+ */
+VMMR3DECL(int) SSMR3PutGCPhys32(PSSMHANDLE pSSM, RTGCPHYS32 GCPhys)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
+}
+
+
+/**
+ * Saves a 64 bits GC physical address item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPhys          The item to save
+ */
+VMMR3DECL(int) SSMR3PutGCPhys64(PSSMHANDLE pSSM, RTGCPHYS64 GCPhys)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
+}
+
+
+/**
+ * Saves a GC physical address item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPhys          The item to save
+ */
+VMMR3DECL(int) SSMR3PutGCPhys(PSSMHANDLE pSSM, RTGCPHYS GCPhys)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPhys, sizeof(GCPhys));
+}
+
+
+/**
+ * Saves a GC virtual address item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPtr           The item to save.
+ */
+VMMR3DECL(int) SSMR3PutGCPtr(PSSMHANDLE pSSM, RTGCPTR GCPtr)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPtr, sizeof(GCPtr));
+}
+
+
+/**
+ * Saves an RC virtual address item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   RCPtr           The item to save.
+ */
+VMMR3DECL(int) SSMR3PutRCPtr(PSSMHANDLE pSSM, RTRCPTR RCPtr)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &RCPtr, sizeof(RCPtr));
+}
+
+
+/**
+ * Saves a GC virtual address (represented as an unsigned integer) item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   GCPtr           The item to save.
+ */
+VMMR3DECL(int) SSMR3PutGCUIntPtr(PSSMHANDLE pSSM, RTGCUINTPTR GCPtr)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &GCPtr, sizeof(GCPtr));
+}
+
+
+/**
+ * Saves a I/O port address item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   IOPort          The item to save.
+ */
+VMMR3DECL(int) SSMR3PutIOPort(PSSMHANDLE pSSM, RTIOPORT IOPort)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &IOPort, sizeof(IOPort));
+}
+
+
+/**
+ * Saves a selector item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   Sel             The item to save.
+ */
+VMMR3DECL(int) SSMR3PutSel(PSSMHANDLE pSSM, RTSEL Sel)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, &Sel, sizeof(Sel));
+}
+
+
+/**
+ * Saves a memory item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pv              Item to save.
+ * @param   cb              Size of the item.
+ */
+VMMR3DECL(int) SSMR3PutMem(PSSMHANDLE pSSM, const void *pv, size_t cb)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataWrite(pSSM, pv, cb);
+}
+
+
+/**
+ * Saves a zero terminated string item to the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   psz             Item to save.
+ */
+VMMR3DECL(int) SSMR3PutStrZ(PSSMHANDLE pSSM, const char *psz)
+{
+    SSM_ASSERT_WRITEABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    size_t cch = strlen(psz);
+    if (cch > _1M)
+    {
+        AssertMsgFailed(("a %zu byte long string, what's this!?!\n", cch));
+        return VERR_TOO_MUCH_DATA;
+    }
+    uint32_t u32 = (uint32_t)cch;
+    int rc = ssmR3DataWrite(pSSM, &u32, sizeof(u32));
+    if (rc)
+        return rc;
+    return ssmR3DataWrite(pSSM, psz, cch);
+}
+
+
+/**
+ * Emits a SSMLiveControl unit with a new progress report.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                    The saved state handle.
+ * @param   lrdPct                  The progress of the the live save.
+ * @param   uPass                   The current pass.
+ */
+static int ssmR3LiveControlEmit(PSSMHANDLE pSSM, long double lrdPct, uint32_t uPass)
+{
+    AssertMsg(lrdPct <= 100.0, ("%u\n", lrdPct * 100));
+
+    /*
+     * Make sure we're in one of the two EXEC states or we may fail.
+     */
+    SSMSTATE enmSavedState = pSSM->enmOp;
+    if (enmSavedState == SSMSTATE_LIVE_VOTE)
+        pSSM->enmOp = SSMSTATE_LIVE_EXEC;
+    else if (enmSavedState == SSMSTATE_SAVE_DONE)
+        pSSM->enmOp = SSMSTATE_SAVE_EXEC;
+
+    /*
+     * Write the unit header.
+     */
+    SSMFILEUNITHDRV2 UnitHdr;
+    memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
+    UnitHdr.offStream       = ssmR3StrmTell(&pSSM->Strm);
+    UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+    UnitHdr.u32CRC          = 0;
+    UnitHdr.u32Version      = 1;
+    UnitHdr.u32Instance     = 0;
+    UnitHdr.u32Pass         = uPass;
+    UnitHdr.fFlags          = 0;
+    UnitHdr.cbName          = sizeof("SSMLiveControl");
+    memcpy(&UnitHdr.szName[0], "SSMLiveControl", UnitHdr.cbName);
+    UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+    Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+         UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+    int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Write the payload.
+         */
+        ssmR3DataWriteBegin(pSSM);
+
+        uint16_t u16PartsPerTenThousand = (uint16_t)(lrdPct * (100 - pSSM->uPercentDone));
+        AssertMsg(u16PartsPerTenThousand <= 10000, ("%u\n", u16PartsPerTenThousand));
+        ssmR3DataWrite(pSSM, &u16PartsPerTenThousand, sizeof(u16PartsPerTenThousand));
+
+        rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Write the termination record and flush the compression stream.
+             */
+            SSMRECTERM TermRec;
+            TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
+            TermRec.cbRec            = sizeof(TermRec) - 2;
+            if (pSSM->Strm.fChecksummed)
+            {
+                TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
+                TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
+            }
+            else
+            {
+                TermRec.fFlags       = 0;
+                TermRec.u32StreamCRC = 0;
+            }
+            TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
+            rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
+            if (RT_SUCCESS(rc))
+                rc = ssmR3DataWriteFinish(pSSM);
+            if (RT_SUCCESS(rc))
+            {
+                pSSM->enmOp = enmSavedState;
+                return rc;
+            }
+        }
+    }
+
+    LogRel(("SSM: Failed to write live control unit. rc=%Rrc\n", rc));
+    if (RT_SUCCESS_NP(pSSM->rc))
+        pSSM->rc = rc;
+    pSSM->enmOp = enmSavedState;
+    return rc;
+}
+
+
+/**
+ * Do the pfnSaveDone run.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoDoneRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Do the done run.
+     */
+    pSSM->enmOp = SSMSTATE_SAVE_DONE;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (    pUnit->u.Common.pfnSaveDone
+            &&  (   pUnit->fCalled
+                 || (!pUnit->u.Common.pfnSavePrep && !pUnit->u.Common.pfnSaveExec)))
+        {
+            int rcOld = pSSM->rc;
+            int rc;
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnSaveDone(pUnit->u.Dev.pDevIns, pSSM);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnSaveDone(pUnit->u.Drv.pDrvIns, pSSM);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnSaveDone(pVM, pSSM);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnSaveDone(pSSM, pUnit->u.External.pvUser);
+                    break;
+                default:
+                    rc = VERR_INTERNAL_ERROR;
+                    break;
+            }
+            if (RT_SUCCESS(rc) && pSSM->rc != rcOld)
+                rc = pSSM->rc;
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("SSM: Done save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                if (RT_SUCCESS_NP(pSSM->rc))
+                    pSSM->rc = rc;
+            }
+        }
+    }
+    return pSSM->rc;
+}
+
+
+/**
+ * Worker for SSMR3LiveDone and SSMR3Save that closes the handle and deletes the
+ * saved state file on failure.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoClose(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    pVM->ssm.s.uPass = 0;
+
+    /*
+     * Make it non-cancellable, close the stream and delete the file on failure.
+     */
+    ssmR3SetCancellable(pVM, pSSM, false);
+    int rc = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    if (RT_SUCCESS(rc))
+        rc = pSSM->rc;
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pSSM->enmOp == SSMSTATE_SAVE_DONE);
+        if (pSSM->pfnProgress)
+            pSSM->pfnProgress(pVM, 100, pSSM->pvUser);
+        LogRel(("SSM: Successfully saved the VM state to '%s'\n",
+                pSSM->pszFilename ? pSSM->pszFilename : "<remote-machine>"));
+    }
+    else
+    {
+        if (pSSM->pszFilename)
+        {
+            int rc2 = RTFileDelete(pSSM->pszFilename);
+            AssertRC(rc2);
+            if (RT_SUCCESS(rc2))
+                LogRel(("SSM: Failed to save the VM state to '%s' (file deleted): %Rrc\n",
+                        pSSM->pszFilename, rc));
+            else
+                LogRel(("SSM: Failed to save the VM state to '%s' (file deletion failed, rc2=%Rrc): %Rrc\n",
+                        pSSM->pszFilename, rc2, rc));
+        }
+        else
+            LogRel(("SSM: Failed to save the VM state.\n"));
+
+        Assert(pSSM->enmOp <= SSMSTATE_SAVE_DONE);
+        if (pSSM->enmOp != SSMSTATE_SAVE_DONE)
+            ssmR3SaveDoDoneRun(pVM, pSSM);
+    }
+
+    /*
+     * Trash the handle before freeing it.
+     */
+    ASMAtomicWriteU32(&pSSM->fCancelled, 0);
+    pSSM->pVM = NULL;
+    pSSM->enmAfter = SSMAFTER_INVALID;
+    pSSM->enmOp = SSMSTATE_INVALID;
+    RTMemFree(pSSM);
+
+    return rc;
+}
+
+
+/**
+ * Closes the SSM handle.
+ *
+ * This must always be called on a handled returned by SSMR3LiveSave.
+ *
+ * @returns VBox status.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
+ *
+ * @thread  EMT(0).
+ */
+VMMR3_INT_DECL(int) SSMR3LiveDone(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3LiveDone: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
+    PVM pVM = pSSM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT0(pVM);
+    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
+                    || pSSM->enmAfter == SSMAFTER_CONTINUE
+                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", pSSM->enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(    pSSM->enmOp >= SSMSTATE_LIVE_PREP
+                    &&  pSSM->enmOp <= SSMSTATE_SAVE_DONE,
+                    ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
+
+    /*
+     * Join paths with SSMR3Save again.
+     */
+    return ssmR3SaveDoClose(pVM, pSSM);
+}
+
+
+/**
+ * Writes the directory.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ * @param   pcEntries           Where to return the number of directory entries.
+ */
+static int ssmR3WriteDirectory(PVM pVM, PSSMHANDLE pSSM, uint32_t *pcEntries)
+{
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Grab some temporary memory for the dictionary.
+     */
+    size_t      cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[pVM->ssm.s.cUnits]);
+    PSSMFILEDIR pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
+    if (!pDir)
+    {
+        LogRel(("ssmR3WriteDirectory: failed to allocate %zu bytes!\n", cbDir));
+        return VERR_NO_TMP_MEMORY;
+    }
+
+    /*
+     * Initialize it.
+     */
+    memcpy(pDir->szMagic, SSMFILEDIR_MAGIC, sizeof(pDir->szMagic));
+    pDir->u32CRC   = 0;
+    pDir->cEntries = 0;
+
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        if (pUnit->offStream != RTFOFF_MIN)
+        {
+            PSSMFILEDIRENTRY pEntry = &pDir->aEntries[pDir->cEntries++];
+            Assert(pDir->cEntries <= pVM->ssm.s.cUnits);
+            Assert(pUnit->offStream >= (RTFOFF)sizeof(SSMFILEHDR));
+            pEntry->off         = pUnit->offStream;
+            pEntry->u32Instance = pUnit->u32Instance;
+            pEntry->u32NameCRC  = RTCrc32(pUnit->szName, pUnit->cchName);
+        }
+
+    /*
+     * Calculate the actual size and CRC-32, then write the directory
+     * out to the stream.
+     */
+    *pcEntries = pDir->cEntries;
+    cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[pDir->cEntries]);
+    pDir->u32CRC = RTCrc32(pDir, cbDir);
+    int rc = ssmR3StrmWrite(&pSSM->Strm, pDir, cbDir);
+    RTMemTmpFree(pDir);
+    return rc;
+}
+
+
+/**
+ * Finalize the saved state stream, i.e. add the end unit, directory
+ * and footer.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoFinalization(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    Assert(RT_SUCCESS(pSSM->rc));
+
+    /*
+     * Write the end unit.
+     */
+    SSMFILEUNITHDRV2 UnitHdr;
+    memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END, sizeof(UnitHdr.szMagic));
+    UnitHdr.offStream       = ssmR3StrmTell(&pSSM->Strm);
+    UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+    UnitHdr.u32CRC          = 0;
+    UnitHdr.u32Version      = 0;
+    UnitHdr.u32Instance     = 0;
+    UnitHdr.u32Pass         = SSM_PASS_FINAL;
+    UnitHdr.fFlags          = 0;
+    UnitHdr.cbName          = 0;
+    UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[0]));
+    Log(("SSM: Unit at %#9llx: END UNIT\n", UnitHdr.offStream));
+    int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[0]));
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed writing the end unit: %Rrc\n", rc));
+        return pSSM->rc = rc;
+    }
+
+    /*
+     * Write the directory for the final units and then the footer.
+     */
+    SSMFILEFTR Footer;
+    rc = ssmR3WriteDirectory(pVM, pSSM, &Footer.cDirEntries);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed writing the directory: %Rrc\n", rc));
+        return pSSM->rc = rc;
+    }
+
+    memcpy(Footer.szMagic, SSMFILEFTR_MAGIC, sizeof(Footer.szMagic));
+    Footer.offStream    = ssmR3StrmTell(&pSSM->Strm);
+    Footer.u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
+    Footer.u32Reserved  = 0;
+    Footer.u32CRC       = 0;
+    Footer.u32CRC       = RTCrc32(&Footer, sizeof(Footer));
+    Log(("SSM: Footer at %#9llx: \n", Footer.offStream));
+    rc = ssmR3StrmWrite(&pSSM->Strm, &Footer, sizeof(Footer));
+    if (RT_SUCCESS(rc))
+        rc = ssmR3StrmSetEnd(&pSSM->Strm);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed writing the footer: %Rrc\n", rc));
+        return pSSM->rc = rc;
+    }
+
+    LogRel(("SSM: Footer at %#llx (%lld), %u directory entries.\n",
+            Footer.offStream, Footer.offStream, Footer.cDirEntries));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Works the progress calculation during the exec part of a live save.
+ *
+ * @param   pSSM        The SSM handle.
+ * @param   iUnit       The current unit number.
+ */
+static void ssmR3ProgressByUnit(PSSMHANDLE pSSM, uint32_t iUnit)
+{
+    if (pSSM->fLiveSave)
+    {
+        unsigned    uPctExec = iUnit * 100 / pSSM->pVM->ssm.s.cUnits;
+        unsigned    cPctExec = 100 - pSSM->uPercentDone - pSSM->uPercentPrepare - pSSM->uPercentLive;
+        long double lrdPct   = (long double)uPctExec * cPctExec / 100 + pSSM->uPercentPrepare + pSSM->uPercentLive;
+        unsigned    uPct     = (unsigned)lrdPct;
+        if (uPct != pSSM->uPercent)
+        {
+            ssmR3LiveControlEmit(pSSM, lrdPct, SSM_PASS_FINAL);
+            pSSM->uPercent =  uPct;
+            pSSM->pfnProgress(pSSM->pVM, uPct, pSSM->pvUser);
+        }
+    }
+}
+
+
+/**
+ * Do the pfnSaveExec run.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoExecRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    AssertRC(pSSM->rc);
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_SAVE_EXEC;
+    unsigned iUnit = 0;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext, iUnit++)
+    {
+        /*
+         * Not all unit have a callback. Skip those which don't and
+         * make sure to keep the progress indicator up to date.
+         */
+        ssmR3ProgressByUnit(pSSM, iUnit);
+        pSSM->offEstUnitEnd += pUnit->cbGuess;
+        if (!pUnit->u.Common.pfnSaveExec)
+        {
+            pUnit->fCalled = true;
+            if (pUnit->cbGuess)
+                ssmR3ProgressByByte(pSSM, pSSM->offEstUnitEnd - pSSM->offEst);
+            continue;
+        }
+        pUnit->offStream = ssmR3StrmTell(&pSSM->Strm);
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!\n"));
+            AssertRC(pSSM->rc);
+            return pSSM->rc = VERR_SSM_CANCELLED;
+        }
+
+        /*
+         * Write data unit header
+         */
+        SSMFILEUNITHDRV2 UnitHdr;
+        memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
+        UnitHdr.offStream       = pUnit->offStream;
+        UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+        UnitHdr.u32CRC          = 0;
+        UnitHdr.u32Version      = pUnit->u32Version;
+        UnitHdr.u32Instance     = pUnit->u32Instance;
+        UnitHdr.u32Pass         = SSM_PASS_FINAL;
+        UnitHdr.fFlags          = 0;
+        UnitHdr.cbName          = (uint32_t)pUnit->cchName + 1;
+        memcpy(&UnitHdr.szName[0], &pUnit->szName[0], UnitHdr.cbName);
+        UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+             UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+        int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed to write unit header. rc=%Rrc\n", rc));
+            return pSSM->rc = rc;
+        }
+
+        /*
+         * Call the execute handler.
+         */
+        ssmR3DataWriteBegin(pSSM);
+        switch (pUnit->enmType)
+        {
+            case SSMUNITTYPE_DEV:
+                rc = pUnit->u.Dev.pfnSaveExec(pUnit->u.Dev.pDevIns, pSSM);
+                break;
+            case SSMUNITTYPE_DRV:
+                rc = pUnit->u.Drv.pfnSaveExec(pUnit->u.Drv.pDrvIns, pSSM);
+                break;
+            case SSMUNITTYPE_INTERNAL:
+                rc = pUnit->u.Internal.pfnSaveExec(pVM, pSSM);
+                break;
+            case SSMUNITTYPE_EXTERNAL:
+                pUnit->u.External.pfnSaveExec(pSSM, pUnit->u.External.pvUser);
+                rc = pSSM->rc;
+                break;
+            default:
+                rc = VERR_INTERNAL_ERROR;
+                break;
+        }
+        pUnit->fCalled = true;
+        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+            pSSM->rc = rc;
+        else
+            rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Execute save failed with rc=%Rrc for data unit '%s'/#%u.\n", rc, pUnit->szName, pUnit->u32Instance));
+            return rc;
+        }
+
+        /*
+         * Write the termination record and flush the compression stream.
+         */
+        SSMRECTERM TermRec;
+        TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
+        TermRec.cbRec            = sizeof(TermRec) - 2;
+        if (pSSM->Strm.fChecksummed)
+        {
+            TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
+            TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
+        }
+        else
+        {
+            TermRec.fFlags       = 0;
+            TermRec.u32StreamCRC = 0;
+        }
+        TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
+        rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
+        if (RT_SUCCESS(rc))
+            rc = ssmR3DataWriteFinish(pSSM);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed terminating unit: %Rrc\n", rc));
+            return pSSM->rc = rc;
+        }
+
+        /*
+         * Advance the progress indicator to the end of the current unit.
+         */
+        ssmR3ProgressByByte(pSSM, pSSM->offEstUnitEnd - pSSM->offEst);
+    } /* for each unit */
+    ssmR3ProgressByUnit(pSSM, pVM->ssm.s.cUnits);
+
+    /* (progress should be pending 99% now) */
+    AssertMsg(   pSSM->uPercent == 101 - pSSM->uPercentDone
+              || pSSM->uPercent == 100 - pSSM->uPercentDone,
+              ("%d\n", pSSM->uPercent));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Do the pfnSavePrep run.
+ *
+ * @returns VBox status code (pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3SaveDoPrepRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+    Assert(RT_SUCCESS(pSSM->rc));
+    pSSM->enmOp = SSMSTATE_SAVE_PREP;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (pUnit->u.Common.pfnSavePrep)
+        {
+            int rc;
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnSavePrep(pUnit->u.Dev.pDevIns, pSSM);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnSavePrep(pUnit->u.Drv.pDrvIns, pSSM);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnSavePrep(pVM, pSSM);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnSavePrep(pSSM, pUnit->u.External.pvUser);
+                    break;
+                default:
+                    rc = VERR_INTERNAL_ERROR;
+                    break;
+            }
+            pUnit->fCalled = true;
+            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                pSSM->rc = rc;
+            else
+                rc = pSSM->rc;
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("SSM: Prepare save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                return rc;
+            }
+        }
+
+        pSSM->cbEstTotal += pUnit->cbGuess;
+    }
+
+    /*
+     * Work the progress indicator if we got one.
+     */
+    if (pSSM->pfnProgress)
+        pSSM->pfnProgress(pVM, pSSM->uPercentPrepare + pSSM->uPercentLive - 1, pSSM->pvUser);
+    pSSM->uPercent = pSSM->uPercentPrepare + pSSM->uPercentLive;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for SSMR3Save and SSMR3LiveSave.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The state handle.
+ *
+ * @thread  EMT(0)
+ */
+static int ssmR3SaveDoCommon(PVM pVM, PSSMHANDLE pSSM)
+{
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Do the work.
+     */
+    int rc = ssmR3SaveDoPrepRun(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        rc = ssmR3SaveDoExecRun(pVM, pSSM);
+        if (RT_SUCCESS(rc))
+            rc = ssmR3SaveDoFinalization(pVM, pSSM);
+    }
+    Assert(pSSM->rc == rc);
+    int rc2 = ssmR3SaveDoDoneRun(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+        rc = rc2;
+
+    return rc;
+}
+
+
+/**
+ * Saves the rest of the state on EMT0.
+ *
+ * @returns VBox status.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
+ *
+ * @thread  Non-EMT thread. Will involve the EMT at the end of the operation.
+ */
+VMMR3_INT_DECL(int) SSMR3LiveDoStep2(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3LiveDoStep2: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
+    PVM pVM = pSSM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_EMT0(pVM);
+    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
+                    || pSSM->enmAfter == SSMAFTER_CONTINUE
+                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", pSSM->enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_LIVE_STEP2, ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
+    AssertRCReturn(pSSM->rc, pSSM->rc);
+
+    /*
+     * Join paths with VMMR3Save.
+     */
+    return ssmR3SaveDoCommon(pVM, pSSM);
+}
+
+
+/**
+ * Writes the file header and clear the per-unit data.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The SSM handle.
+ */
+static int ssmR3WriteHeaderAndClearPerUnitData(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Write the header.
+     */
+    SSMFILEHDR FileHdr;
+    memcpy(&FileHdr.szMagic, SSMFILEHDR_MAGIC_V2_0, sizeof(FileHdr.szMagic));
+    FileHdr.u16VerMajor  = VBOX_VERSION_MAJOR;
+    FileHdr.u16VerMinor  = VBOX_VERSION_MINOR;
+    FileHdr.u32VerBuild  = VBOX_VERSION_BUILD;
+    FileHdr.u32SvnRev    = VMMGetSvnRev();
+    FileHdr.cHostBits    = HC_ARCH_BITS;
+    FileHdr.cbGCPhys     = sizeof(RTGCPHYS);
+    FileHdr.cbGCPtr      = sizeof(RTGCPTR);
+    FileHdr.u8Reserved   = 0;
+    FileHdr.cUnits       = pVM->ssm.s.cUnits;
+    FileHdr.fFlags       = SSMFILEHDR_FLAGS_STREAM_CRC32;
+    if (pSSM->fLiveSave)
+        FileHdr.fFlags  |= SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE;
+    FileHdr.cbMaxDecompr = RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer);
+    FileHdr.u32CRC       = 0;
+    FileHdr.u32CRC       = RTCrc32(&FileHdr, sizeof(FileHdr));
+    int rc = ssmR3StrmWrite(&pSSM->Strm, &FileHdr, sizeof(FileHdr));
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Clear the per unit flags and offsets.
+     */
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        pUnit->fCalled   = false;
+        pUnit->offStream = RTFOFF_MIN;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a new saved state file.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is
+ *                              used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is
+ *                              used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   enmAfter            What to do afterwards.
+ * @param   pfnProgress         The progress callback.
+ * @param   pvProgressUser      The progress callback user argument.
+ * @param   ppSSM               Where to return the pointer to the saved state
+ *                              handle upon successful return.  Free it using
+ *                              RTMemFree after closing the stream.
+ */
+static int ssmR3SaveDoCreateFile(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                 SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, PSSMHANDLE *ppSSM)
+{
+    PSSMHANDLE pSSM = (PSSMHANDLE)RTMemAllocZ(sizeof(*pSSM));
+    if (!pSSM)
+        return VERR_NO_MEMORY;
+
+    pSSM->pVM                       = pVM;
+    pSSM->enmOp                     = SSMSTATE_INVALID;
+    pSSM->enmAfter                  = enmAfter;
+    pSSM->fCancelled                = SSMHANDLE_OK;
+    pSSM->rc                        = VINF_SUCCESS;
+    pSSM->cbUnitLeftV1              = 0;
+    pSSM->offUnit                   = UINT64_MAX;
+    pSSM->fLiveSave                 = false;
+    pSSM->pfnProgress               = pfnProgress;
+    pSSM->pvUser                    = pvProgressUser;
+    pSSM->uPercent                  = 0;
+    pSSM->offEstProgress            = 0;
+    pSSM->cbEstTotal                = 0;
+    pSSM->offEst                    = 0;
+    pSSM->offEstUnitEnd             = 0;
+    pSSM->uPercentLive              = 0;
+    pSSM->uPercentPrepare           = 0;
+    pSSM->uPercentDone              = 0;
+    pSSM->uReportedLivePercent      = 0;
+    pSSM->pszFilename               = pszFilename;
+    pSSM->u.Write.offDataBuffer     = 0;
+    pSSM->u.Write.cMsMaxDowntime    = UINT32_MAX;
+
+    int rc;
+    if (pStreamOps)
+        rc = ssmR3StrmInit(&pSSM->Strm, pStreamOps, pvStreamOpsUser, true /*fWrite*/, true /*fChecksummed*/, 8 /*cBuffers*/);
+    else
+        rc = ssmR3StrmOpenFile(&pSSM->Strm, pszFilename,             true /*fWrite*/, true /*fChecksummed*/, 8 /*cBuffers*/);
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed to create save state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+        RTMemFree(pSSM);
+        return rc;
+    }
+
+    *ppSSM = pSSM;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Start VM save operation.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pszFilename     Name of the file to save the state in. NULL if pStreamOps is used.
+ * @param   pStreamOps      The stream method table. NULL if pszFilename is
+ *                          used.
+ * @param   pvStreamOpsUser The user argument to the stream methods.
+ * @param   enmAfter        What is planned after a successful save operation.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvUser          User argument for the progress callback.
+ *
+ * @thread  EMT
+ */
+VMMR3DECL(int) SSMR3Save(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                         SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvUser)
+{
+    LogFlow(("SSMR3Save: pszFilename=%p:{%s} enmAfter=%d pfnProgress=%p pvUser=%p\n", pszFilename, pszFilename, enmAfter, pfnProgress, pvUser));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(   enmAfter == SSMAFTER_DESTROY
+                    || enmAfter == SSMAFTER_CONTINUE,
+                    ("%d\n", enmAfter),
+                    VERR_INVALID_PARAMETER);
+
+    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
+    if (pStreamOps)
+    {
+        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * Create the saved state file and handle.
+     *
+     * Note that there might be quite some work to do after executing the saving,
+     * so we reserve 20% for the 'Done' period.
+     */
+    PSSMHANDLE pSSM;
+    int rc = ssmR3SaveDoCreateFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser,
+                                   enmAfter, pfnProgress, pvUser, &pSSM);
+    if (RT_FAILURE(rc))
+        return rc;
+    pSSM->uPercentLive    = 0;
+    pSSM->uPercentPrepare = 20;
+    pSSM->uPercentDone    = 2;
+    pSSM->fLiveSave       = false;
+
+    /*
+     * Write the saved state stream header and join paths with
+     * the other save methods for the rest of the job.
+     */
+    Log(("SSM: Starting state save to file '%s'...\n", pszFilename));
+    ssmR3StrmStartIoThread(&pSSM->Strm);
+    rc = ssmR3WriteHeaderAndClearPerUnitData(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        ssmR3SetCancellable(pVM, pSSM, true);
+        ssmR3SaveDoCommon(pVM, pSSM);
+    }
+
+    return ssmR3SaveDoClose(pVM, pSSM);
+}
+
+
+/**
+ * Used by PGM to report the completion percentage of the live stage during the
+ * vote run.
+ *
+ * @param   pSSM                The saved state handle.
+ * @param   uPercent            The completion percentage.
+ */
+VMMR3DECL(void) SSMR3HandleReportLivePercent(PSSMHANDLE pSSM, unsigned uPercent)
+{
+    AssertMsgReturnVoid(pSSM->enmOp == SSMSTATE_LIVE_VOTE, ("%d\n", pSSM->enmOp));
+    AssertReturnVoid(uPercent <= 100);
+    if (uPercent < pSSM->uReportedLivePercent)
+        pSSM->uReportedLivePercent = uPercent;
+}
+
+
+/**
+ * Calls pfnLiveVote for all units.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @retval  VINF_SUCCESS if we can pass on to step 2.
+ * @retval  VINF_SSM_VOTE_FOR_ANOTHER_PASS if we need another pass.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ * @param   uPass               The current pass.
+ */
+static int ssmR3LiveDoVoteRun(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    int rcRet = VINF_SUCCESS;
+    AssertRC(pSSM->rc);
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_LIVE_VOTE;
+
+    unsigned uPrevPrecent = pSSM->uReportedLivePercent;
+    pSSM->uReportedLivePercent = 101;
+
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (    pUnit->u.Common.pfnLiveVote
+            &&  !pUnit->fDoneLive)
+        {
+            int rc;
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnLiveVote(pUnit->u.Dev.pDevIns, pSSM, uPass);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnLiveVote(pUnit->u.Drv.pDrvIns, pSSM, uPass);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnLiveVote(pVM, pSSM, uPass);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnLiveVote(pSSM, pUnit->u.External.pvUser, uPass);
+                    break;
+                default:
+                    rc = VERR_INTERNAL_ERROR;
+                    break;
+            }
+            pUnit->fCalled = true;
+            Assert(pSSM->rc == VINF_SUCCESS);
+            if (rc != VINF_SUCCESS)
+            {
+                if (rc == VINF_SSM_VOTE_FOR_ANOTHER_PASS)
+                {
+                    Log(("ssmR3DoLiveVoteRun: '%s'/#%u -> VINF_SSM_VOTE_FOR_ANOTHER_PASS (pass=%u)\n", pUnit->szName, pUnit->u32Instance, uPass));
+                    rcRet = VINF_SSM_VOTE_FOR_ANOTHER_PASS;
+                }
+                else if (rc == VINF_SSM_VOTE_DONE_DONT_CALL_AGAIN)
+                {
+                    pUnit->fDoneLive = true;
+                    Log(("ssmR3DoLiveVoteRun: '%s'/#%u -> VINF_SSM_VOTE_DONE_DONT_CALL_AGAIN (pass=%u)\n", pUnit->szName, pUnit->u32Instance, uPass));
+                }
+                else
+                {
+                    /*
+                     * rc is usually VERR_SSM_VOTE_FOR_GIVING_UP here, but we allow
+                     * other status codes for better user feed back.  However, no
+                     * other non-error status is allowed.
+                     */
+                    LogRel(("SSM: Error - '%s'/#%u voted %Rrc! (pass=%u)\n", pUnit->szName, pUnit->u32Instance, rc, uPass));
+                    AssertMsgReturn(RT_FAILURE(rc), ("%Rrc; '%s'\n", rc, pUnit->szName), pSSM->rc = VERR_IPE_UNEXPECTED_INFO_STATUS);
+                    return pSSM->rc = rc;
+                }
+            }
+        }
+    }
+    if (rcRet == VINF_SUCCESS)
+    {
+        LogRel(("SSM: Step 1 completed after pass %u.\n", uPass));
+        pSSM->uReportedLivePercent = 100;
+    }
+    else
+    {
+        /*
+         * Work the progress callback.
+         */
+        if (pSSM->uReportedLivePercent > 100)
+            pSSM->uReportedLivePercent = 0;
+        if (   pSSM->uReportedLivePercent != uPrevPrecent
+            && pSSM->pfnProgress
+            && pSSM->uPercentLive)
+        {
+            long double lrdPct = (long double)pSSM->uReportedLivePercent * pSSM->uPercentLive / 100;
+            unsigned    uPct   = (unsigned)lrdPct;
+            if (uPct != pSSM->uPercent)
+            {
+                ssmR3LiveControlEmit(pSSM, lrdPct, uPass);
+                pSSM->uPercent = uPct;
+                pSSM->pfnProgress(pVM, uPct, pSSM->pvUser);
+            }
+        }
+    }
+    return rcRet;
+}
+
+
+/**
+ * Calls pfnLiveExec for all units.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ * @param   uPass               The current pass.
+ */
+static int ssmR3LiveDoExecRun(PVM pVM, PSSMHANDLE pSSM, uint32_t uPass)
+{
+    AssertRC(pSSM->rc);
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_LIVE_EXEC;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        /*
+         * Skip units without a callback (this is most).
+         */
+        if (   !pUnit->u.Common.pfnLiveExec
+            || pUnit->fDoneLive)
+            continue;
+        pUnit->offStream = ssmR3StrmTell(&pSSM->Strm);
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!\n"));
+            AssertRC(pSSM->rc);
+            return pSSM->rc = VERR_SSM_CANCELLED;
+        }
+
+        /*
+         * Write data unit header.
+         */
+        SSMFILEUNITHDRV2 UnitHdr;
+        memcpy(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic));
+        UnitHdr.offStream       = pUnit->offStream;
+        UnitHdr.u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+        UnitHdr.u32CRC          = 0;
+        UnitHdr.u32Version      = pUnit->u32Version;
+        UnitHdr.u32Instance     = pUnit->u32Instance;
+        UnitHdr.u32Pass         = uPass;
+        UnitHdr.fFlags          = 0;
+        UnitHdr.cbName          = (uint32_t)pUnit->cchName + 1;
+        memcpy(&UnitHdr.szName[0], &pUnit->szName[0], UnitHdr.cbName);
+        UnitHdr.u32CRC          = RTCrc32(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+             UnitHdr.offStream, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+        int rc = ssmR3StrmWrite(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]));
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed to write unit header. rc=%Rrc\n", rc));
+            return pSSM->rc = rc;
+        }
+
+        /*
+         * Call the execute handler.
+         */
+        ssmR3DataWriteBegin(pSSM);
+        switch (pUnit->enmType)
+        {
+            case SSMUNITTYPE_DEV:
+                rc = pUnit->u.Dev.pfnLiveExec(pUnit->u.Dev.pDevIns, pSSM, uPass);
+                break;
+            case SSMUNITTYPE_DRV:
+                rc = pUnit->u.Drv.pfnLiveExec(pUnit->u.Drv.pDrvIns, pSSM, uPass);
+                break;
+            case SSMUNITTYPE_INTERNAL:
+                rc = pUnit->u.Internal.pfnLiveExec(pVM, pSSM, uPass);
+                break;
+            case SSMUNITTYPE_EXTERNAL:
+                rc = pUnit->u.External.pfnLiveExec(pSSM, pUnit->u.External.pvUser, uPass);
+                break;
+            default:
+                rc = VERR_INTERNAL_ERROR;
+                break;
+        }
+        pUnit->fCalled = true;
+        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+            pSSM->rc = rc;
+        else
+        {
+            if (rc == VINF_SSM_DONT_CALL_AGAIN)
+                pUnit->fDoneLive = true;
+            rc = ssmR3DataFlushBuffer(pSSM); /* will return SSMHANDLE::rc if it is set */
+        }
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Execute save failed with rc=%Rrc for data unit '%s'/#%u.\n", rc, pUnit->szName, pUnit->u32Instance));
+            if (RT_SUCCESS(pSSM->rc))
+                pSSM->rc = rc;
+            return rc;
+        }
+
+        /*
+         * Write the termination record and flush the compression stream.
+         */
+        SSMRECTERM TermRec;
+        TermRec.u8TypeAndFlags   = SSM_REC_FLAGS_FIXED | SSM_REC_FLAGS_IMPORTANT | SSM_REC_TYPE_TERM;
+        TermRec.cbRec            = sizeof(TermRec) - 2;
+        if (pSSM->Strm.fChecksummed)
+        {
+            TermRec.fFlags       = SSMRECTERM_FLAGS_CRC32;
+            TermRec.u32StreamCRC = RTCrc32Finish(RTCrc32Process(ssmR3StrmCurCRC(&pSSM->Strm), &TermRec, 2));
+        }
+        else
+        {
+            TermRec.fFlags       = 0;
+            TermRec.u32StreamCRC = 0;
+        }
+        TermRec.cbUnit           = pSSM->offUnit + sizeof(TermRec);
+        rc = ssmR3DataWriteRaw(pSSM, &TermRec, sizeof(TermRec));
+        if (RT_SUCCESS(rc))
+            rc = ssmR3DataWriteFinish(pSSM);
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: Failed terminating unit: %Rrc (pass=%u)\n", rc, uPass));
+            return pSSM->rc = rc;
+        }
+    } /* for each unit */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements the live exec+vote loop.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3DoLiveExecVoteLoop(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Calc the max saved state size before we should give up because of insane
+     * amounts of data.
+     */
+#define SSM_MAX_GROWTH_FILE      10000
+#define SSM_MAX_GROWTH_REMOTE   100000
+    uint64_t cbSum = 0;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        cbSum += pUnit->cbGuess;
+    uint64_t cbMax = cbSum * (pSSM->pszFilename ? SSM_MAX_GROWTH_FILE : SSM_MAX_GROWTH_REMOTE);
+    AssertLogRelMsgReturn(cbMax > cbSum, ("cbMax=%#RX64, cbSum=%#RX64\n", cbMax, cbSum), pSSM->rc = VERR_OUT_OF_RANGE);
+    if (cbMax < _1G)
+        cbMax = _1G;
+
+    /*
+     * The pass loop.
+     *
+     * The number of iterations is restricted for two reasons, first
+     * to make sure
+     */
+#define SSM_MAX_PASSES  _1M
+    for (uint32_t uPass = 0; uPass < SSM_MAX_PASSES; uPass++)
+    {
+        pVM->ssm.s.uPass = uPass;
+
+        /*
+         * Save state and vote on whether we need more passes or not.
+         */
+        int rc = ssmR3LiveDoExecRun(pVM, pSSM, uPass);
+        if (RT_FAILURE(rc))
+            return rc;
+        rc = ssmR3LiveDoVoteRun(pVM, pSSM, uPass);
+        if (rc == VINF_SUCCESS)
+        {
+            pSSM->enmOp = SSMSTATE_LIVE_STEP2;
+            return VINF_SUCCESS;
+        }
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Check that we're still within sane data amounts.
+         */
+        uint64_t cbSaved = ssmR3StrmTell(&pSSM->Strm);
+        if (cbSaved > cbMax)
+        {
+            LogRel(("SSM: Giving up: Exceeded max state size. (cbSaved=%#RX64, cbMax=%#RX64)\n", cbSaved, cbMax));
+            return pSSM->rc = VERR_SSM_STATE_GREW_TOO_BIG;
+        }
+
+        /*
+         * Check that the stream is still OK.
+         */
+        rc = ssmR3StrmCheckAndFlush(&pSSM->Strm);
+        if (RT_FAILURE(rc))
+            return pSSM->rc = rc;
+    }
+
+    LogRel(("SSM: Giving up: Too many passes! (%u)\n", SSM_MAX_PASSES));
+    return pSSM->rc = VERR_SSM_TOO_MANY_PASSES;
+}
+
+
+/**
+ * Calls pfnLivePrep for all units.
+ *
+ * @returns VBox status code (no need to check pSSM->rc).
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3DoLivePrepRun(PVM pVM, PSSMHANDLE pSSM)
+{
+    /*
+     * Do the prepare run.
+     */
+    pSSM->rc = VINF_SUCCESS;
+    pSSM->enmOp = SSMSTATE_SAVE_PREP;
+    for (PSSMUNIT pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+    {
+        if (pUnit->u.Common.pfnLivePrep)
+        {
+            int rc;
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnLivePrep(pUnit->u.Dev.pDevIns, pSSM);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnLivePrep(pUnit->u.Drv.pDrvIns, pSSM);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnLivePrep(pVM, pSSM);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnLivePrep(pSSM, pUnit->u.External.pvUser);
+                    break;
+                default:
+                    rc = VERR_INTERNAL_ERROR;
+                    break;
+            }
+            pUnit->fCalled = true;
+            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                pSSM->rc = rc;
+            else
+                rc = pSSM->rc;
+            if (RT_FAILURE(rc))
+            {
+                LogRel(("SSM: Prepare save failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                return rc;
+            }
+        }
+
+        pSSM->cbEstTotal += pUnit->cbGuess;
+    }
+
+    /*
+     * Work the progress indicator if we got one.
+     */
+    if (pSSM->pfnProgress)
+        pSSM->pfnProgress(pVM, 2, pSSM->pvUser);
+    pSSM->uPercent = 2;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Continue a live state saving operation on the worker thread.
+ *
+ * @returns VBox status.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3LiveSave.
+ *
+ * @thread  Non-EMT thread. Will involve the EMT at the end of the operation.
+ */
+VMMR3_INT_DECL(int) SSMR3LiveDoStep1(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3LiveDoStep1: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_POINTER);
+    PVM pVM = pSSM->pVM;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_OTHER_THREAD(pVM);
+    AssertMsgReturn(   pSSM->enmAfter == SSMAFTER_DESTROY
+                    || pSSM->enmAfter == SSMAFTER_CONTINUE
+                    || pSSM->enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", pSSM->enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_LIVE_STEP1, ("%d\n", pSSM->enmOp), VERR_INVALID_STATE);
+    AssertRCReturn(pSSM->rc, pSSM->rc);
+
+    /*
+     * Do the prep run, then the exec+vote cycle.
+     */
+    int rc = ssmR3DoLivePrepRun(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+        rc = ssmR3DoLiveExecVoteLoop(pVM, pSSM);
+    return rc;
+}
+
+
+/**
+ * Start saving the live state.
+ *
+ * Call SSMR3LiveDoStep1, SSMR3LiveDoStep2 and finally SSMR3LiveDone on success.
+ * SSMR3LiveDone should be called even if SSMR3LiveDoStep1 or SSMR3LiveDoStep2
+ * fails.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   cMsMaxDowntime  The maximum downtime given as milliseconds.
+ * @param   pszFilename     Name of the file to save the state in. This string
+ *                          must remain valid until SSMR3LiveDone is called.
+ *                          Must be NULL if pStreamOps is used.
+ * @param   pStreamOps      The stream method table. NULL if pszFilename is
+ *                          used.
+ * @param   pvStreamOpsUser The user argument to the stream methods.
+ * @param   enmAfter        What is planned after a successful save operation.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvProgressUser  User argument for the progress callback.
+ *
+ * @thread  EMT0
+ */
+VMMR3_INT_DECL(int) SSMR3LiveSave(PVM pVM, uint32_t cMsMaxDowntime,
+                                  const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser,
+                                  PSSMHANDLE *ppSSM)
+{
+    LogFlow(("SSMR3LiveSave: cMsMaxDowntime=%u pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p\n",
+             cMsMaxDowntime, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(   enmAfter == SSMAFTER_DESTROY
+                    || enmAfter == SSMAFTER_CONTINUE
+                    || enmAfter == SSMAFTER_TELEPORT,
+                    ("%d\n", enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
+    if (pStreamOps)
+    {
+        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * Create the saved state file and handle.
+     *
+     * Note that there might be quite some work to do after executing the saving,
+     * so we reserve 20% for the 'Done' period.
+     */
+    PSSMHANDLE pSSM;
+    int rc = ssmR3SaveDoCreateFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser,
+                                   enmAfter, pfnProgress, pvProgressUser, &pSSM);
+    if (RT_FAILURE(rc))
+        return rc;
+    pSSM->uPercentLive           = 93;
+    pSSM->uPercentPrepare        = 2;
+    pSSM->uPercentDone           = 2;
+    pSSM->fLiveSave              = true;
+    pSSM->u.Write.cMsMaxDowntime = cMsMaxDowntime;
+
+    /*
+     * Write the saved state stream header and do the prep run for live saving.
+     */
+    Log(("SSM: Starting state save to file '%s'...\n", pszFilename));
+    ssmR3StrmStartIoThread(&pSSM->Strm);
+    rc = ssmR3WriteHeaderAndClearPerUnitData(pVM, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Return and let the requestor thread do the pfnLiveExec/Vote part
+         * via SSMR3SaveFinishLive
+         */
+        pSSM->enmOp = SSMSTATE_LIVE_STEP1;
+        ssmR3SetCancellable(pVM, pSSM, true);
+        *ppSSM = pSSM;
+        return VINF_SUCCESS;
+    }
+    /* bail out. */
+    int rc2 = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    RTMemFree(pSSM);
+    rc2 = RTFileDelete(pszFilename);
+    AssertRC(rc2);
+    return rc;
+}
+
+#endif /* !SSM_STANDALONE */
+
+
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+/* ... Loading and reading starts here ... */
+
+
+/**
+ * Closes the decompressor of a data unit.
+ *
+ * @returns pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataReadFinishV1(PSSMHANDLE pSSM)
+{
+    if (pSSM->u.Read.pZipDecompV1)
+    {
+        int rc = RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
+        AssertRC(rc);
+        pSSM->u.Read.pZipDecompV1 = NULL;
+    }
+    return pSSM->rc;
+}
+
+
+/**
+ * Callback for reading compressed data into the input buffer of the
+ * decompressor, for saved file format version 1.
+ *
+ * @returns VBox status code. Set pSSM->rc on error.
+ * @param   pvSSM       The SSM handle.
+ * @param   pvBuf       Where to store the compressed data.
+ * @param   cbBuf       Size of the buffer.
+ * @param   pcbRead     Number of bytes actually stored in the buffer.
+ */
+static DECLCALLBACK(int) ssmR3ReadInV1(void *pvSSM, void *pvBuf, size_t cbBuf, size_t *pcbRead)
+{
+    PSSMHANDLE  pSSM   = (PSSMHANDLE)pvSSM;
+    size_t      cbRead = cbBuf;
+    if (pSSM->cbUnitLeftV1 < cbBuf)
+        cbRead = (size_t)pSSM->cbUnitLeftV1;
+    if (cbRead)
+    {
+        //Log2(("ssmR3ReadInV1: %#010llx cbBug=%#x cbRead=%#x\n", ssmR3StrmTell(&pSSM->Strm), cbBuf, cbRead));
+        int rc = ssmR3StrmRead(&pSSM->Strm, pvBuf, cbRead);
+        if (RT_SUCCESS(rc))
+        {
+            pSSM->cbUnitLeftV1 -= cbRead;
+            if (pcbRead)
+                *pcbRead = cbRead;
+            ssmR3ProgressByByte(pSSM, cbRead);
+            return VINF_SUCCESS;
+        }
+        return pSSM->rc = rc;
+    }
+
+    if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
+        AssertMsgFailed(("SSM: attempted reading more than the unit!\n"));
+    return pSSM->rc = VERR_SSM_LOADED_TOO_MUCH;
+}
+
+
+/**
+ * Internal read worker for reading data from a version 1 unit.
+ *
+ * @returns VBox status code, pSSM->rc is set on error.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+static int ssmR3DataReadV1(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    /*
+     * Open the decompressor on the first read.
+     */
+    if (!pSSM->u.Read.pZipDecompV1)
+    {
+        pSSM->rc = RTZipDecompCreate(&pSSM->u.Read.pZipDecompV1, pSSM, ssmR3ReadInV1);
+        if (RT_FAILURE(pSSM->rc))
+            return pSSM->rc;
+    }
+
+    /*
+     * Do the requested read.
+     */
+    int rc = pSSM->rc = RTZipDecompress(pSSM->u.Read.pZipDecompV1, pvBuf, cbBuf, NULL);
+    if (RT_SUCCESS(rc))
+    {
+        Log2(("ssmR3DataRead: pvBuf=%p cbBuf=%#x offUnit=%#llx %.*Rhxs%s\n", pvBuf, cbBuf, pSSM->offUnit, RT_MIN(cbBuf, SSM_LOG_BYTES), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
+        pSSM->offUnit += cbBuf;
+        return VINF_SUCCESS;
+    }
+    AssertMsgFailed(("rc=%Rrc cbBuf=%#x\n", rc, cbBuf));
+    return rc;
+}
+
+
+/**
+ * Creates the decompressor for the data unit.
+ *
+ * pSSM->rc will be set on error.
+ *
+ * @param   pSSM            The saved state handle.
+ */
+static void ssmR3DataReadBeginV2(PSSMHANDLE pSSM)
+{
+    Assert(!pSSM->u.Read.cbDataBuffer || pSSM->u.Read.cbDataBuffer == pSSM->u.Read.offDataBuffer);
+    Assert(!pSSM->u.Read.cbRecLeft);
+
+    pSSM->offUnit = 0;
+    pSSM->u.Read.cbRecLeft      = 0;
+    pSSM->u.Read.cbDataBuffer   = 0;
+    pSSM->u.Read.offDataBuffer  = 0;
+    pSSM->u.Read.fEndOfData     = false;
+    pSSM->u.Read.u8TypeAndFlags = 0;
+}
+
+
+/**
+ * Checks for the termination record and closes the decompressor.
+ *
+ * pSSM->rc will be set on error.
+ *
+ * @returns pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataReadFinishV2(PSSMHANDLE pSSM)
+{
+    /*
+     * If we haven't encountered the end of the record, it must be the next one.
+     */
+    int rc = pSSM->rc;
+    if (    !pSSM->u.Read.fEndOfData
+        &&  RT_SUCCESS(rc))
+    {
+        rc = ssmR3DataReadRecHdrV2(pSSM);
+        if (    RT_SUCCESS(rc)
+            &&  !pSSM->u.Read.fEndOfData)
+        {
+            rc = VERR_SSM_LOADED_TOO_LITTLE;
+            AssertFailed();
+        }
+        pSSM->rc = rc;
+    }
+    return rc;
+}
+
+
+/**
+ * Read reader that keep works the progress indicator and unit offset.
+ *
+ * @returns VBox status code. Does NOT set pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to put the bits
+ * @param   cbBuf           How many bytes to read.
+ */
+DECLINLINE(int) ssmR3DataReadV2Raw(PSSMHANDLE pSSM, void *pvBuf, size_t cbToRead)
+{
+    int rc = ssmR3StrmRead(&pSSM->Strm, pvBuf, cbToRead);
+    if (RT_SUCCESS(rc))
+    {
+        pSSM->offUnit += cbToRead;
+        ssmR3ProgressByByte(pSSM, cbToRead);
+        return VINF_SUCCESS;
+    }
+
+    if (rc == VERR_SSM_CANCELLED)
+        return rc;
+
+    if (pSSM->enmAfter != SSMAFTER_DEBUG_IT && rc == VERR_EOF)
+        AssertMsgFailedReturn(("SSM: attempted reading more than the unit! rc=%Rrc\n", rc), VERR_SSM_LOADED_TOO_MUCH);
+    return VERR_SSM_STREAM_ERROR;
+}
+
+
+/**
+ * Reads and checks the LZF "header".
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ * @param   pSSM            The saved state handle..
+ * @param   pcbDecompr      Where to store the size of the decompressed data.
+ */
+DECLINLINE(int) ssmR3DataReadV2RawLzfHdr(PSSMHANDLE pSSM, uint32_t *pcbDecompr)
+{
+    *pcbDecompr = 0; /* shuts up gcc. */
+    AssertLogRelMsgReturn(   pSSM->u.Read.cbRecLeft > 1
+                          && pSSM->u.Read.cbRecLeft <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abComprBuffer) + 2,
+                          ("%#x\n", pSSM->u.Read.cbRecLeft),
+                          VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    uint8_t cKB;
+    int rc = ssmR3DataReadV2Raw(pSSM, &cKB, 1);
+    if (RT_FAILURE(rc))
+        return pSSM->rc = rc;
+    pSSM->u.Read.cbRecLeft -= sizeof(cKB);
+
+    uint32_t cbDecompr = (uint32_t)cKB * _1K;
+    AssertLogRelMsgReturn(   cbDecompr >= pSSM->u.Read.cbRecLeft
+                          && cbDecompr <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer),
+                          ("%#x\n", cbDecompr),
+                          VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    *pcbDecompr = cbDecompr;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reads an LZF block from the stream and decompresses into the specified
+ * buffer.
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ * @param   SSM             The saved state handle.
+ * @param   pvDst           Pointer to the output buffer.
+ * @param   cbDecompr       The size of the decompressed data.
+ */
+static int ssmR3DataReadV2RawLzf(PSSMHANDLE pSSM, void *pvDst, size_t cbDecompr)
+{
+    int         rc;
+    uint32_t    cbCompr    = pSSM->u.Read.cbRecLeft;
+    pSSM->u.Read.cbRecLeft = 0;
+
+    /*
+     * Try use the stream buffer directly to avoid copying things around.
+     */
+    uint8_t const *pb = ssmR3StrmReadDirect(&pSSM->Strm, cbCompr);
+    if (pb)
+    {
+        pSSM->offUnit += cbCompr;
+        ssmR3ProgressByByte(pSSM, cbCompr);
+    }
+    else
+    {
+        rc = ssmR3DataReadV2Raw(pSSM, &pSSM->u.Read.abComprBuffer[0], cbCompr);
+        if (RT_FAILURE(rc))
+            return pSSM->rc = rc;
+        pb = &pSSM->u.Read.abComprBuffer[0];
+    }
+
+    /*
+     * Decompress it.
+     */
+    size_t cbDstActual;
+    rc = RTZipBlockDecompress(RTZIPTYPE_LZF, 0 /*fFlags*/,
+                              pb, cbCompr, NULL /*pcbSrcActual*/,
+                              pvDst, cbDecompr, &cbDstActual);
+    if (RT_SUCCESS(rc))
+    {
+        AssertLogRelMsgReturn(cbDstActual == cbDecompr, ("%#x %#x\n", cbDstActual, cbDecompr), VERR_SSM_INTEGRITY_DECOMPRESSION);
+        return VINF_SUCCESS;
+    }
+
+    AssertLogRelMsgFailed(("cbCompr=%#x cbDecompr=%#x rc=%Rrc\n", cbCompr, cbDecompr, rc));
+    return pSSM->rc = VERR_SSM_INTEGRITY_DECOMPRESSION;
+}
+
+
+/**
+ * Reads and checks the raw zero "header".
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ * @param   pSSM            The saved state handle..
+ * @param   pcbDecompr      Where to store the size of the zero data.
+ */
+DECLINLINE(int) ssmR3DataReadV2RawZeroHdr(PSSMHANDLE pSSM, uint32_t *pcbZero)
+{
+    *pcbZero = 0; /* shuts up gcc. */
+    AssertLogRelMsgReturn(pSSM->u.Read.cbRecLeft == 1, ("%#x\n", pSSM->u.Read.cbRecLeft), VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    uint8_t cKB;
+    int rc = ssmR3DataReadV2Raw(pSSM, &cKB, 1);
+    if (RT_FAILURE(rc))
+        return pSSM->rc = rc;
+    pSSM->u.Read.cbRecLeft = 0;
+
+    uint32_t cbZero = (uint32_t)cKB * _1K;
+    AssertLogRelMsgReturn(cbZero <= RT_SIZEOFMEMB(SSMHANDLE, u.Read.abDataBuffer),
+                          ("%#x\n", cbZero), VERR_SSM_INTEGRITY_DECOMPRESSION);
+
+    *pcbZero = cbZero;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for reading the record header.
+ *
+ * It sets pSSM->u.Read.cbRecLeft, pSSM->u.Read.u8TypeAndFlags and
+ * pSSM->u.Read.fEndOfData.  When a termination record is encounter, it will be
+ * read in full and validated, the fEndOfData indicator is set, and VINF_SUCCESS
+ * is returned.
+ *
+ * @returns VBox status code. Does not set pSSM->rc.
+ * @param   pSSM            The saved state handle.
+ */
+static int ssmR3DataReadRecHdrV2(PSSMHANDLE pSSM)
+{
+    AssertLogRelReturn(!pSSM->u.Read.fEndOfData, VERR_SSM_LOADED_TOO_MUCH);
+
+    /*
+     * Read the two mandatory bytes.
+     */
+    uint8_t  abHdr[8];
+    int rc = ssmR3DataReadV2Raw(pSSM, abHdr, 2);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Validate the first byte and check for the termination records.
+     */
+    pSSM->u.Read.u8TypeAndFlags = abHdr[0];
+    AssertLogRelMsgReturn(SSM_REC_ARE_TYPE_AND_FLAGS_VALID(abHdr[0]), ("%#x %#x\n", abHdr[0], abHdr[1]), VERR_SSM_INTEGRITY_REC_HDR);
+    if ((abHdr[0] & SSM_REC_TYPE_MASK) == SSM_REC_TYPE_TERM)
+    {
+        pSSM->u.Read.cbRecLeft = 0;
+        pSSM->u.Read.fEndOfData = true;
+        AssertLogRelMsgReturn(abHdr[1] == sizeof(SSMRECTERM) - 2, ("%#x\n", abHdr[1]), VERR_SSM_INTEGRITY_REC_TERM);
+        AssertLogRelMsgReturn(abHdr[0] & SSM_REC_FLAGS_IMPORTANT, ("%#x\n", abHdr[0]), VERR_SSM_INTEGRITY_REC_TERM);
+
+        /* get the rest */
+        uint32_t    u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
+        SSMRECTERM  TermRec;
+        rc = ssmR3DataReadV2Raw(pSSM, (uint8_t *)&TermRec + 2, sizeof(SSMRECTERM) - 2);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* validate integrity */
+        AssertLogRelMsgReturn(TermRec.cbUnit == pSSM->offUnit,
+                              ("cbUnit=%#llx offUnit=%#llx\n", TermRec.cbUnit, pSSM->offUnit),
+                              VERR_SSM_INTEGRITY_REC_TERM);
+        AssertLogRelMsgReturn(!(TermRec.fFlags & ~SSMRECTERM_FLAGS_CRC32), ("%#x\n", TermRec.fFlags), VERR_SSM_INTEGRITY_REC_TERM);
+        if (!(TermRec.fFlags & SSMRECTERM_FLAGS_CRC32))
+            AssertLogRelMsgReturn(TermRec.u32StreamCRC == 0, ("%#x\n", TermRec.u32StreamCRC), VERR_SSM_INTEGRITY_REC_TERM);
+        else if (pSSM->Strm.fChecksummed)
+            AssertLogRelMsgReturn(TermRec.u32StreamCRC == u32StreamCRC, ("%#x, %#x\n", TermRec.u32StreamCRC, u32StreamCRC),
+                                  VERR_SSM_INTEGRITY_REC_TERM_CRC);
+
+        Log3(("ssmR3DataReadRecHdrV2: %08llx|%08llx: TERM\n", ssmR3StrmTell(&pSSM->Strm) - sizeof(SSMRECTERM), pSSM->offUnit));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Figure the size. The 2nd byte is encoded in UTF-8 fashion, so this
+     * is can be highly enjoyable.
+     */
+    uint32_t cbHdr = 2;
+    uint32_t cb = abHdr[1];
+    if (!(cb & 0x80))
+        pSSM->u.Read.cbRecLeft = cb;
+    else
+    {
+        /*
+         * Need more data. Figure how much and read it.
+         */
+        if (!(cb & RT_BIT(5)))
+            cb = 2;
+        else if (!(cb & RT_BIT(4)))
+            cb = 3;
+        else if (!(cb & RT_BIT(3)))
+            cb = 4;
+        else if (!(cb & RT_BIT(2)))
+            cb = 5;
+        else if (!(cb & RT_BIT(1)))
+            cb = 6;
+        else
+            AssertLogRelMsgFailedReturn(("Invalid record size byte: %#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+        cbHdr = cb + 1;
+
+        rc = ssmR3DataReadV2Raw(pSSM, &abHdr[2], cb - 1);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Validate what we've read.
+         */
+        switch (cb)
+        {
+            case 6:
+                AssertLogRelMsgReturn((abHdr[6] & 0xc0) == 0x80, ("6/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+            case 5:
+                AssertLogRelMsgReturn((abHdr[5] & 0xc0) == 0x80, ("5/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+            case 4:
+                AssertLogRelMsgReturn((abHdr[4] & 0xc0) == 0x80, ("4/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+            case 3:
+                AssertLogRelMsgReturn((abHdr[3] & 0xc0) == 0x80, ("3/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+            case 2:
+                AssertLogRelMsgReturn((abHdr[2] & 0xc0) == 0x80, ("2/%u: %.*Rhxs\n", cb, cb + 1, &abHdr[0]), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            default:
+                return VERR_INTERNAL_ERROR;
+        }
+
+        /*
+         * Decode it and validate the range.
+         */
+        switch (cb)
+        {
+            case 6:
+                cb =             (abHdr[6] & 0x3f)
+                    | ((uint32_t)(abHdr[5] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[4] & 0x3f) << 12)
+                    | ((uint32_t)(abHdr[3] & 0x3f) << 18)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 24)
+                    | ((uint32_t)(abHdr[1] & 0x01) << 30);
+                AssertLogRelMsgReturn(cb >= 0x04000000 && cb <= 0x7fffffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            case 5:
+                cb =             (abHdr[5] & 0x3f)
+                    | ((uint32_t)(abHdr[4] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[3] & 0x3f) << 12)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 18)
+                    | ((uint32_t)(abHdr[1] & 0x03) << 24);
+                AssertLogRelMsgReturn(cb >= 0x00200000 && cb <= 0x03ffffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            case 4:
+                cb =             (abHdr[4] & 0x3f)
+                    | ((uint32_t)(abHdr[3] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 12)
+                    | ((uint32_t)(abHdr[1] & 0x07) << 18);
+                AssertLogRelMsgReturn(cb >= 0x00010000 && cb <= 0x001fffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+                break;
+            case 3:
+                cb =             (abHdr[3] & 0x3f)
+                    | ((uint32_t)(abHdr[2] & 0x3f) << 6)
+                    | ((uint32_t)(abHdr[1] & 0x0f) << 12);
+#if 0  /* disabled to optimize buffering */
+                AssertLogRelMsgReturn(cb >= 0x00000800 && cb <= 0x0000ffff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+#endif
+                break;
+            case 2:
+                cb =             (abHdr[2] & 0x3f)
+                    | ((uint32_t)(abHdr[1] & 0x1f) << 6);
+#if 0  /* disabled to optimize buffering */
+                AssertLogRelMsgReturn(cb >= 0x00000080 && cb <= 0x000007ff, ("cb=%#x\n", cb), VERR_SSM_INTEGRITY_REC_HDR);
+#endif
+                break;
+            default:
+                return VERR_INTERNAL_ERROR;
+        }
+
+        pSSM->u.Read.cbRecLeft = cb;
+    }
+
+    Log3(("ssmR3DataReadRecHdrV2: %08llx|%08llx/%08x: Type=%02x fImportant=%RTbool cbHdr=%u\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft,
+          pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK,
+          !!(pSSM->u.Read.u8TypeAndFlags & SSM_REC_FLAGS_IMPORTANT),
+          cbHdr
+        )); NOREF(cbHdr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Buffer miss, do an unbuffered read.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+static int ssmR3DataReadUnbufferedV2(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    void   const *pvBufOrg = pvBuf; NOREF(pvBufOrg);
+    size_t const  cbBufOrg = cbBuf; NOREF(cbBufOrg);
+
+    /*
+     * Copy out what we've got in the buffer.
+     */
+    uint32_t off        = pSSM->u.Read.offDataBuffer;
+    int32_t  cbInBuffer = pSSM->u.Read.cbDataBuffer - off;
+    Log4(("ssmR3DataReadUnbufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n", ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, cbInBuffer, cbBufOrg));
+    if (cbInBuffer > 0)
+    {
+        uint32_t const cbToCopy = (uint32_t)cbInBuffer;
+        Assert(cbBuf > cbToCopy);
+        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbToCopy);
+        pvBuf  = (uint8_t *)pvBuf + cbToCopy;
+        cbBuf -= cbToCopy;
+        pSSM->u.Read.cbDataBuffer  = 0;
+        pSSM->u.Read.offDataBuffer = 0;
+    }
+
+    /*
+     * Read data.
+     */
+    do
+    {
+        /*
+         * Read the next record header if no more data.
+         */
+        if (!pSSM->u.Read.cbRecLeft)
+        {
+            int rc = ssmR3DataReadRecHdrV2(pSSM);
+            if (RT_FAILURE(rc))
+                return pSSM->rc = rc;
+        }
+        AssertLogRelMsgReturn(!pSSM->u.Read.fEndOfData, ("cbBuf=%zu", cbBuf), pSSM->rc = VERR_SSM_LOADED_TOO_MUCH);
+
+        /*
+         * Read data from the current record.
+         */
+        uint32_t cbToRead;
+        switch (pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK)
+        {
+            case SSM_REC_TYPE_RAW:
+            {
+                cbToRead = (uint32_t)RT_MIN(cbBuf, pSSM->u.Read.cbRecLeft);
+                int rc = ssmR3DataReadV2Raw(pSSM, pvBuf, cbToRead);
+                if (RT_FAILURE(rc))
+                    return pSSM->rc = rc;
+                pSSM->u.Read.cbRecLeft -= cbToRead;
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_LZF:
+            {
+                int rc = ssmR3DataReadV2RawLzfHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (cbToRead <= cbBuf)
+                {
+                    rc = ssmR3DataReadV2RawLzf(pSSM, pvBuf, cbToRead);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+                else
+                {
+                    /* The output buffer is too small, use the data buffer. */
+                    rc = ssmR3DataReadV2RawLzf(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                    pSSM->u.Read.cbDataBuffer  = cbToRead;
+                    cbToRead = (uint32_t)cbBuf;
+                    pSSM->u.Read.offDataBuffer = cbToRead;
+                    memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                }
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_ZERO:
+            {
+                int rc = ssmR3DataReadV2RawZeroHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                if (cbToRead > cbBuf)
+                {
+                    /* Spill the remainder into the data buffer. */
+                    memset(&pSSM->u.Read.abDataBuffer[0], 0, cbToRead - cbBuf);
+                    pSSM->u.Read.cbDataBuffer  = cbToRead - (uint32_t)cbBuf;
+                    pSSM->u.Read.offDataBuffer = 0;
+                    cbToRead = (uint32_t)cbBuf;
+                }
+                memset(pvBuf, 0, cbToRead);
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("%x\n", pSSM->u.Read.u8TypeAndFlags), VERR_INTERNAL_ERROR_5);
+        }
+
+        cbBuf -= cbToRead;
+        pvBuf = (uint8_t *)pvBuf + cbToRead;
+    } while (cbBuf > 0);
+
+    Log4(("ssmR3DataReadUnBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, 0, cbBufOrg, RT_MIN(SSM_LOG_BYTES, cbBufOrg), pvBufOrg, cbBufOrg > SSM_LOG_BYTES ? "..." : ""));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Buffer miss, do a buffered read.
+ *
+ * @returns VBox status code. Sets pSSM->rc on error.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+static int ssmR3DataReadBufferedV2(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    void   const *pvBufOrg = pvBuf; NOREF(pvBufOrg);
+    size_t const  cbBufOrg = cbBuf; NOREF(cbBufOrg);
+
+    /*
+     * Copy out what we've got in the buffer.
+     */
+    uint32_t off        = pSSM->u.Read.offDataBuffer;
+    int32_t  cbInBuffer = pSSM->u.Read.cbDataBuffer - off;
+    Log4(("ssmR3DataReadBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n", ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, cbInBuffer, cbBufOrg));
+    if (cbInBuffer > 0)
+    {
+        uint32_t const cbToCopy = (uint32_t)cbInBuffer;
+        Assert(cbBuf > cbToCopy);
+        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbToCopy);
+        pvBuf  = (uint8_t *)pvBuf + cbToCopy;
+        cbBuf -= cbToCopy;
+        pSSM->u.Read.cbDataBuffer  = 0;
+        pSSM->u.Read.offDataBuffer = 0;
+    }
+
+    /*
+     * Buffer more data.
+     */
+    do
+    {
+        /*
+         * Read the next record header if no more data.
+         */
+        if (!pSSM->u.Read.cbRecLeft)
+        {
+            int rc = ssmR3DataReadRecHdrV2(pSSM);
+            if (RT_FAILURE(rc))
+                return pSSM->rc = rc;
+        }
+        AssertLogRelMsgReturn(!pSSM->u.Read.fEndOfData, ("cbBuf=%zu", cbBuf), pSSM->rc = VERR_SSM_LOADED_TOO_MUCH);
+
+        /*
+         * Read data from the current record.
+         * LATER: optimize by reading directly into the output buffer for some cases.
+         */
+        uint32_t cbToRead;
+        switch (pSSM->u.Read.u8TypeAndFlags & SSM_REC_TYPE_MASK)
+        {
+            case SSM_REC_TYPE_RAW:
+            {
+                cbToRead = RT_MIN(sizeof(pSSM->u.Read.abDataBuffer), pSSM->u.Read.cbRecLeft);
+                int rc = ssmR3DataReadV2Raw(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                if (RT_FAILURE(rc))
+                    return pSSM->rc = rc;
+                pSSM->u.Read.cbRecLeft   -= cbToRead;
+                pSSM->u.Read.cbDataBuffer = cbToRead;
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_LZF:
+            {
+                int rc = ssmR3DataReadV2RawLzfHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                rc = ssmR3DataReadV2RawLzf(pSSM, &pSSM->u.Read.abDataBuffer[0], cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                pSSM->u.Read.cbDataBuffer = cbToRead;
+                break;
+            }
+
+            case SSM_REC_TYPE_RAW_ZERO:
+            {
+                int rc = ssmR3DataReadV2RawZeroHdr(pSSM, &cbToRead);
+                if (RT_FAILURE(rc))
+                    return rc;
+                memset(&pSSM->u.Read.abDataBuffer[0], 0, cbToRead);
+                pSSM->u.Read.cbDataBuffer = cbToRead;
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("%x\n", pSSM->u.Read.u8TypeAndFlags), VERR_INTERNAL_ERROR_5);
+        }
+        /*pSSM->u.Read.offDataBuffer = 0;*/
+
+        /*
+         * Copy data from the buffer.
+         */
+        uint32_t cbToCopy = (uint32_t)RT_MIN(cbBuf, cbToRead);
+        memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[0], cbToCopy);
+        cbBuf -= cbToCopy;
+        pvBuf = (uint8_t *)pvBuf + cbToCopy;
+        pSSM->u.Read.offDataBuffer = cbToCopy;
+    } while (cbBuf > 0);
+
+    Log4(("ssmR3DataReadBufferedV2: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer,
+          cbBufOrg, RT_MIN(SSM_LOG_BYTES, cbBufOrg), pvBufOrg, cbBufOrg > SSM_LOG_BYTES ? "..." : ""));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined worker that handles format checks and buffered reads.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   pvBuf           Where to store the read data.
+ * @param   cbBuf           Number of bytes to read.
+ */
+DECLINLINE(int) ssmR3DataRead(PSSMHANDLE pSSM, void *pvBuf, size_t cbBuf)
+{
+    /*
+     * Fend off previous errors and V1 data units.
+     */
+    if (RT_FAILURE(pSSM->rc))
+        return pSSM->rc;
+    if (RT_UNLIKELY(pSSM->u.Read.uFmtVerMajor == 1))
+        return ssmR3DataReadV1(pSSM, pvBuf, cbBuf);
+
+    /*
+     * Check if the requested data is buffered.
+     */
+    uint32_t off = pSSM->u.Read.offDataBuffer;
+    if (    off + cbBuf > pSSM->u.Read.cbDataBuffer
+        ||  cbBuf > sizeof(pSSM->u.Read.abDataBuffer))
+    {
+        if (cbBuf <= sizeof(pSSM->u.Read.abDataBuffer) / 8)
+            return ssmR3DataReadBufferedV2(pSSM, pvBuf, cbBuf);
+        return ssmR3DataReadUnbufferedV2(pSSM, pvBuf, cbBuf);
+    }
+
+    memcpy(pvBuf, &pSSM->u.Read.abDataBuffer[off], cbBuf);
+    pSSM->u.Read.offDataBuffer = off + (uint32_t)cbBuf;
+    Log4((cbBuf
+          ? "ssmR3DataRead: %08llx|%08llx/%08x/%08x: cbBuf=%#x %.*Rhxs%s\n"
+          : "ssmR3DataRead: %08llx|%08llx/%08x/%08x: cbBuf=%#x\n",
+          ssmR3StrmTell(&pSSM->Strm), pSSM->offUnit, pSSM->u.Read.cbRecLeft, pSSM->u.Read.cbDataBuffer - pSSM->u.Read.offDataBuffer,
+          cbBuf, RT_MIN(SSM_LOG_BYTES, cbBuf), pvBuf, cbBuf > SSM_LOG_BYTES ? "..." : ""));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets a structure.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   paFields        The array of structure fields descriptions.
+ *                          The array must be terminated by a SSMFIELD_ENTRY_TERM().
+ */
+VMMR3DECL(int) SSMR3GetStruct(PSSMHANDLE pSSM, void *pvStruct, PCSSMFIELD paFields)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+    /* begin marker. */
+    uint32_t u32Magic;
+    int rc = SSMR3GetU32(pSSM, &u32Magic);
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertMsgReturn(u32Magic == SSMR3STRUCT_BEGIN, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
+
+    /* get the fields */
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        uint8_t *pbField = (uint8_t *)pvStruct + pCur->off;
+        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+        {
+            case SSMFIELDTRANS_NO_TRANSFORMATION:
+                rc = ssmR3DataRead(pSSM, pbField, pCur->cb);
+                break;
+
+            case SSMFIELDTRANS_GCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3GetGCPtr(pSSM, (PRTGCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_GCPHYS:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3GetGCPhys(pSSM, (PRTGCPHYS)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR), ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3GetRCPtr(pSSM, (PRTRCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR_ARRAY:
+            {
+                uint32_t const cEntries = pCur->cb / sizeof(RTRCPTR);
+                AssertMsgReturn(pCur->cb == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", pCur->cb, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = VINF_SUCCESS;
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = SSMR3GetRCPtr(pSSM, &((PRTRCPTR)pbField)[i]);
+                break;
+            }
+
+            default:
+                AssertMsgFailedReturn(("%#x\n", pCur->pfnGetPutOrTransformer), VERR_SSM_FIELD_COMPLEX);
+        }
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /* end marker */
+    rc = SSMR3GetU32(pSSM, &u32Magic);
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertMsgReturn(u32Magic == SSMR3STRUCT_END, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
+    return rc;
+}
+
+
+/**
+ * SSMR3GetStructEx helper that gets a HCPTR that is used as a NULL indicator.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The saved state handle.
+ * @param   ppv             Where to return the value (0/1).
+ * @param   fFlags          SSMSTRUCT_FLAGS_XXX.
+ */
+DECLINLINE(int) ssmR3GetHCPtrNI(PSSMHANDLE pSSM, void **ppv, uint32_t fFlags)
+{
+    int rc;
+    if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+    {
+        if (ssmR3GetHostBits(pSSM) == 64)
+        {
+            uint64_t u;
+            rc = ssmR3DataRead(pSSM, &u, sizeof(u));
+            if (RT_SUCCESS(rc))
+                *ppv = (void *)(u ? 1 : 0);
+        }
+        else
+        {
+            uint32_t u;
+            rc = ssmR3DataRead(pSSM, &u, sizeof(u));
+            if (RT_SUCCESS(rc))
+                *ppv = (void *)(u ? 1 : 0);
+        }
+    }
+    else
+    {
+        bool f;
+        rc = SSMR3GetBool(pSSM, &f);
+        if (RT_SUCCESS(rc))
+            *ppv = (void *)(f ? 1 : 0);
+    }
+    return rc;
+}
+
+
+/**
+ * Guts a structure, extended API.
+ *
+ * @returns VBox status code.
+ * @param   pSSM            The saved state handle.
+ * @param   pvStruct        The structure address.
+ * @param   cbStruct        The size of the struct (use for validation only).
+ * @param   fFlags          Combination of SSMSTRUCT_FLAGS_XXX defines.
+ * @param   paFields        The array of structure fields descriptions. The
+ *                          array must be terminated by a SSMFIELD_ENTRY_TERM().
+ * @param   pvUser          User argument for any callbacks that paFields might
+ *                          contain.
+ */
+VMMR3DECL(int) SSMR3GetStructEx(PSSMHANDLE pSSM, void *pvStruct, size_t cbStruct,
+                                uint32_t fFlags, PCSSMFIELD paFields, void *pvUser)
+{
+    int         rc;
+    uint32_t    u32Magic;
+
+    /*
+     * Validation.
+     */
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    AssertMsgReturn(!(fFlags & ~SSMSTRUCT_FLAGS_VALID_MASK), ("%#x\n", fFlags), VERR_INVALID_PARAMETER);
+    AssertPtr(pvStruct);
+    AssertPtr(paFields);
+
+    /*
+     * Begin marker.
+     */
+    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
+    {
+        rc = SSMR3GetU32(pSSM, &u32Magic);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsgReturn(u32Magic == SSMR3STRUCT_BEGIN, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
+    }
+
+    /*
+     * Put the fields
+     */
+    uint32_t off = 0;
+    for (PCSSMFIELD pCur = paFields;
+         pCur->cb != UINT32_MAX && pCur->off != UINT32_MAX;
+         pCur++)
+    {
+        uint32_t const offField =    (!SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer) || pCur->off != UINT32_MAX / 2)
+                                  &&  !SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? pCur->off
+                                : off;
+        uint32_t const cbField  = SSMFIELDTRANS_IS_OLD(pCur->pfnGetPutOrTransformer)
+                                ? 0
+                                : SSMFIELDTRANS_IS_PADDING(pCur->pfnGetPutOrTransformer)
+                                ? RT_HIWORD(pCur->cb)
+                                : pCur->cb;
+        AssertMsgReturn(   cbField            <= cbStruct
+                        && offField + cbField <= cbStruct
+                        && offField + cbField >= offField,
+                        ("off=%#x cb=%#x cbStruct=%#x (%s)\n", cbField, offField, cbStruct, pCur->pszName),
+                        VERR_SSM_FIELD_OUT_OF_BOUNDS);
+        AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                        || off == offField,
+                        ("off=%#x offField=%#x (%s)\n", off, offField, pCur->pszName),
+                        VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+        rc = VINF_SUCCESS;
+        uint8_t       *pbField  = (uint8_t *)pvStruct + offField;
+        switch ((uintptr_t)pCur->pfnGetPutOrTransformer)
+        {
+            case SSMFIELDTRANS_NO_TRANSFORMATION:
+                rc = ssmR3DataRead(pSSM, pbField, cbField);
+                break;
+
+            case SSMFIELDTRANS_GCPHYS:
+                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3GetGCPhys(pSSM, (PRTGCPHYS)pbField);
+                break;
+
+            case SSMFIELDTRANS_GCPTR:
+                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3GetGCPtr(pSSM, (PRTGCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR:
+                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3GetRCPtr(pSSM, (PRTRCPTR)pbField);
+                break;
+
+            case SSMFIELDTRANS_RCPTR_ARRAY:
+            {
+                uint32_t const cEntries = cbField / sizeof(RTRCPTR);
+                AssertMsgReturn(cbField == cEntries * sizeof(RTRCPTR) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = VINF_SUCCESS;
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = SSMR3GetRCPtr(pSSM, &((PRTRCPTR)pbField)[i]);
+                break;
+            }
+
+            case SSMFIELDTRANS_HCPTR_NI:
+                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = ssmR3GetHCPtrNI(pSSM, (void **)pbField, fFlags);
+                break;
+
+            case SSMFIELDTRANS_HCPTR_NI_ARRAY:
+            {
+                uint32_t const cEntries = cbField / sizeof(void *);
+                AssertMsgReturn(cbField == cEntries * sizeof(void *) && cEntries, ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = VINF_SUCCESS;
+                for (uint32_t i = 0; i < cEntries && RT_SUCCESS(rc); i++)
+                    rc = ssmR3GetHCPtrNI(pSSM, &((void **)pbField)[i], fFlags);
+                break;
+            }
+
+            case SSMFIELDTRANS_HCPTR_HACK_U32:
+                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                *(uintptr_t *)pbField = 0;
+                rc = ssmR3DataRead(pSSM, pbField, sizeof(uint32_t));
+                if ((fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE) && ssmR3GetHostBits(pSSM) == 64)
+                {
+                    uint32_t u32;
+                    rc = ssmR3DataRead(pSSM, &u32, sizeof(uint32_t));
+                    AssertMsgReturn(RT_FAILURE(rc) || u32 == 0 || (fFlags & SSMSTRUCT_FLAGS_SAVED_AS_MEM),
+                                    ("high=%#x low=%#x (%s)\n", u32, *(uint32_t *)pbField, pCur->pszName),
+                                    VERR_SSM_FIELD_INVALID_VALUE);
+                }
+                break;
+
+
+            case SSMFIELDTRANS_IGNORE:
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = SSMR3Skip(pSSM, cbField);
+                break;
+
+            case SSMFIELDTRANS_IGN_GCPHYS:
+                AssertMsgReturn(cbField == sizeof(RTGCPHYS), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPhys);
+                break;
+
+            case SSMFIELDTRANS_IGN_GCPTR:
+                AssertMsgReturn(cbField == sizeof(RTGCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPtr);
+                break;
+
+            case SSMFIELDTRANS_IGN_RCPTR:
+                AssertMsgReturn(cbField == sizeof(RTRCPTR), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = SSMR3Skip(pSSM, sizeof(RTRCPTR));
+                break;
+
+            case SSMFIELDTRANS_IGN_HCPTR:
+                AssertMsgReturn(cbField == sizeof(void *), ("%#x (%s)\n", cbField, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) / 8);
+                break;
+
+
+            case SSMFIELDTRANS_OLD:
+                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3Skip(pSSM, pCur->cb);
+                break;
+
+            case SSMFIELDTRANS_OLD_GCPHYS:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPHYS) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPhys);
+                break;
+
+            case SSMFIELDTRANS_OLD_GCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTGCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3Skip(pSSM, pSSM->u.Read.cbGCPtr);
+                break;
+
+            case SSMFIELDTRANS_OLD_RCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(RTRCPTR) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3Skip(pSSM, sizeof(RTRCPTR));
+                break;
+
+            case SSMFIELDTRANS_OLD_HCPTR:
+                AssertMsgReturn(pCur->cb == sizeof(void *) && pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) / 8);
+                break;
+
+            case SSMFIELDTRANS_OLD_PAD_HC:
+                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                rc = SSMR3Skip(pSSM, ssmR3GetHostBits(pSSM) == 64 ? RT_HIWORD(pCur->cb) : RT_LOWORD(pCur->cb));
+                break;
+
+            case SSMFIELDTRANS_OLD_PAD_MSC32:
+                AssertMsgReturn(pCur->off == UINT32_MAX / 2, ("%#x %#x (%s)\n", pCur->cb, pCur->off, pCur->pszName), VERR_SSM_FIELD_INVALID_SIZE);
+                if (ssmR3IsHostMsc32(pSSM))
+                    rc = SSMR3Skip(pSSM, pCur->cb);
+                break;
+
+
+            case SSMFIELDTRANS_PAD_HC:
+            case SSMFIELDTRANS_PAD_HC32:
+            case SSMFIELDTRANS_PAD_HC64:
+            case SSMFIELDTRANS_PAD_HC_AUTO:
+            case SSMFIELDTRANS_PAD_MSC32_AUTO:
+            {
+                uint32_t cb32    = RT_BYTE1(pCur->cb);
+                uint32_t cb64    = RT_BYTE2(pCur->cb);
+                uint32_t cbCtx   =    HC_ARCH_BITS == 64
+                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                       && !SSM_HOST_IS_MSC_32)
+                                 ? cb64 : cb32;
+                uint32_t cbSaved =    ssmR3GetHostBits(pSSM) == 64
+                                   || (   (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                       && !ssmR3IsHostMsc32(pSSM))
+                                 ? cb64 : cb32;
+                AssertMsgReturn(    cbField == cbCtx
+                                &&  (   (   pCur->off == UINT32_MAX / 2
+                                         && (   cbField == 0
+                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_HC_AUTO
+                                             || (uintptr_t)pCur->pfnGetPutOrTransformer == SSMFIELDTRANS_PAD_MSC32_AUTO
+                                            )
+                                        )
+                                     || (pCur->off != UINT32_MAX / 2 && cbField != 0)
+                                     )
+                                , ("cbField=%#x cb32=%#x cb64=%#x HC_ARCH_BITS=%u cbCtx=%#x cbSaved=%#x off=%#x\n",
+                                   cbField, cb32, cb64, HC_ARCH_BITS, cbCtx, cbSaved, pCur->off),
+                                VERR_SSM_FIELD_INVALID_PADDING_SIZE);
+                if (fFlags & SSMSTRUCT_FLAGS_DONT_IGNORE)
+                    rc = SSMR3Skip(pSSM, cbSaved);
+                break;
+            }
+
+            default:
+                AssertPtrReturn(pCur->pfnGetPutOrTransformer, VERR_SSM_FIELD_INVALID_CALLBACK);
+                rc = pCur->pfnGetPutOrTransformer(pSSM, pCur, pvStruct, fFlags, true /*fGetOrPut*/, pvUser);
+                break;
+        }
+        if (RT_FAILURE(rc))
+            return rc;
+
+        off = offField + cbField;
+    }
+    AssertMsgReturn(    !(fFlags & SSMSTRUCT_FLAGS_FULL_STRUCT)
+                    || off == cbStruct,
+                    ("off=%#x cbStruct=%#x\n", off, cbStruct),
+                    VERR_SSM_FIELD_NOT_CONSECUTIVE);
+
+    /*
+     * End marker
+     */
+    if (!(fFlags & SSMSTRUCT_FLAGS_NO_MARKERS))
+    {
+        rc = SSMR3GetU32(pSSM, &u32Magic);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsgReturn(u32Magic == SSMR3STRUCT_END, ("u32Magic=%#RX32\n", u32Magic), VERR_SSM_STRUCTURE_MAGIC);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a boolean item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pfBool          Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetBool(PSSMHANDLE pSSM, bool *pfBool)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    uint8_t u8; /* see SSMR3PutBool */
+    int rc = ssmR3DataRead(pSSM, &u8, sizeof(u8));
+    if (RT_SUCCESS(rc))
+    {
+        Assert(u8 <= 1);
+        *pfBool = !!u8;
+    }
+    return rc;
+}
+
+
+/**
+ * Loads a 8-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu8             Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU8(PSSMHANDLE pSSM, uint8_t *pu8)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu8, sizeof(*pu8));
+}
+
+
+/**
+ * Loads a 8-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pi8             Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS8(PSSMHANDLE pSSM, int8_t *pi8)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi8, sizeof(*pi8));
+}
+
+
+/**
+ * Loads a 16-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu16            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU16(PSSMHANDLE pSSM, uint16_t *pu16)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu16, sizeof(*pu16));
+}
+
+
+/**
+ * Loads a 16-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pi16            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS16(PSSMHANDLE pSSM, int16_t *pi16)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi16, sizeof(*pi16));
+}
+
+
+/**
+ * Loads a 32-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu32            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU32(PSSMHANDLE pSSM, uint32_t *pu32)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu32, sizeof(*pu32));
+}
+
+
+/**
+ * Loads a 32-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pi32            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS32(PSSMHANDLE pSSM, int32_t *pi32)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi32, sizeof(*pi32));
+}
+
+
+/**
+ * Loads a 64-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu64            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU64(PSSMHANDLE pSSM, uint64_t *pu64)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu64, sizeof(*pu64));
+}
+
+
+/**
+ * Loads a 64-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pi64            Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS64(PSSMHANDLE pSSM, int64_t *pi64)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi64, sizeof(*pi64));
+}
+
+
+/**
+ * Loads a 128-bit unsigned integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu128           Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetU128(PSSMHANDLE pSSM, uint128_t *pu128)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu128, sizeof(*pu128));
+}
+
+
+/**
+ * Loads a 128-bit signed integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pi128           Where to store the item.
+ */
+VMMR3DECL(int) SSMR3GetS128(PSSMHANDLE pSSM, int128_t *pi128)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi128, sizeof(*pi128));
+}
+
+
+/**
+ * Loads a VBox unsigned integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu              Where to store the integer.
+ */
+VMMR3DECL(int) SSMR3GetUInt(PSSMHANDLE pSSM, PRTUINT pu)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pu, sizeof(*pu));
+}
+
+
+/**
+ * Loads a VBox signed integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pi              Where to store the integer.
+ */
+VMMR3DECL(int) SSMR3GetSInt(PSSMHANDLE pSSM, PRTINT pi)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pi, sizeof(*pi));
+}
+
+
+/**
+ * Loads a GC natural unsigned integer item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu              Where to store the integer.
+ *
+ * @deprecated Silly type with an incorrect size, don't use it.
+ */
+VMMR3DECL(int) SSMR3GetGCUInt(PSSMHANDLE pSSM, PRTGCUINT pu)
+{
+    AssertCompile(sizeof(RTGCPTR) == sizeof(*pu));
+    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pu);
+}
+
+
+/**
+ * Loads a GC unsigned integer register item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pu              Where to store the integer.
+ */
+VMMR3DECL(int) SSMR3GetGCUIntReg(PSSMHANDLE pSSM, PRTGCUINTREG pu)
+{
+    AssertCompile(sizeof(RTGCPTR) == sizeof(*pu));
+    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pu);
+}
+
+
+/**
+ * Loads a 32 bits GC physical address item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys32(PSSMHANDLE pSSM, PRTGCPHYS32 pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
+}
+
+
+/**
+ * Loads a 64 bits GC physical address item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys64(PSSMHANDLE pSSM, PRTGCPHYS64 pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
+}
+
+
+/**
+ * Loads a GC physical address item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPhys         Where to store the GC physical address.
+ */
+VMMR3DECL(int) SSMR3GetGCPhys(PSSMHANDLE pSSM, PRTGCPHYS pGCPhys)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    /*
+     * Default size?
+     */
+    if (RT_LIKELY(sizeof(*pGCPhys) == pSSM->u.Read.cbGCPhys))
+        return ssmR3DataRead(pSSM, pGCPhys, sizeof(*pGCPhys));
+
+    /*
+     * Fiddly.
+     */
+    Assert(sizeof(*pGCPhys)      == sizeof(uint64_t) || sizeof(*pGCPhys)      == sizeof(uint32_t));
+    Assert(pSSM->u.Read.cbGCPhys == sizeof(uint64_t) || pSSM->u.Read.cbGCPhys == sizeof(uint32_t));
+    if (pSSM->u.Read.cbGCPhys == sizeof(uint64_t))
+    {
+        /* 64-bit saved, 32-bit load: try truncate it. */
+        uint64_t u64;
+        int rc = ssmR3DataRead(pSSM, &u64, sizeof(uint64_t));
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u64 >= _4G)
+            return VERR_SSM_GCPHYS_OVERFLOW;
+        *pGCPhys = (RTGCPHYS)u64;
+        return rc;
+    }
+
+    /* 32-bit saved, 64-bit load: clear the high part. */
+    *pGCPhys = 0;
+    return ssmR3DataRead(pSSM, pGCPhys, sizeof(uint32_t));
+}
+
+
+/**
+ * Loads a GC virtual address item from the current data unit.
+ *
+ * Only applies to in the 1.1 format:
+ *  - SSMR3GetGCPtr
+ *  - SSMR3GetGCUIntPtr
+ *  - SSMR3GetGCUInt
+ *  - SSMR3GetGCUIntReg
+ *
+ * Put functions are not affected.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   cbGCPtr         Size of RTGCPTR
+ *
+ * @remarks This interface only works with saved state version 1.1, if the
+ *          format isn't 1.1 the call will be ignored.
+ */
+VMMR3_INT_DECL(int) SSMR3HandleSetGCPtrSize(PSSMHANDLE pSSM, unsigned cbGCPtr)
+{
+    Assert(cbGCPtr == sizeof(RTGCPTR32) || cbGCPtr == sizeof(RTGCPTR64));
+    if (!pSSM->u.Read.fFixedGCPtrSize)
+    {
+        Log(("SSMR3SetGCPtrSize: %u -> %u bytes\n", pSSM->u.Read.cbGCPtr, cbGCPtr));
+        pSSM->u.Read.cbGCPtr = cbGCPtr;
+        pSSM->u.Read.fFixedGCPtrSize = true;
+    }
+    else if (   pSSM->u.Read.cbGCPtr != cbGCPtr
+             && pSSM->u.Read.uFmtVerMajor == 1
+             && pSSM->u.Read.uFmtVerMinor == 1)
+        AssertMsgFailed(("SSMR3SetGCPtrSize: already fixed at %u bytes; requested %u bytes\n", pSSM->u.Read.cbGCPtr, cbGCPtr));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a GC virtual address item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPtr          Where to store the GC virtual address.
+ */
+VMMR3DECL(int) SSMR3GetGCPtr(PSSMHANDLE pSSM, PRTGCPTR pGCPtr)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    /*
+     * Default size?
+     */
+    if (RT_LIKELY(sizeof(*pGCPtr) == pSSM->u.Read.cbGCPtr))
+        return ssmR3DataRead(pSSM, pGCPtr, sizeof(*pGCPtr));
+
+    /*
+     * Fiddly.
+     */
+    Assert(sizeof(*pGCPtr)      == sizeof(uint64_t) || sizeof(*pGCPtr)      == sizeof(uint32_t));
+    Assert(pSSM->u.Read.cbGCPtr == sizeof(uint64_t) || pSSM->u.Read.cbGCPtr == sizeof(uint32_t));
+    if (pSSM->u.Read.cbGCPtr == sizeof(uint64_t))
+    {
+        /* 64-bit saved, 32-bit load: try truncate it. */
+        uint64_t u64;
+        int rc = ssmR3DataRead(pSSM, &u64, sizeof(uint64_t));
+        if (RT_FAILURE(rc))
+            return rc;
+        if (u64 >= _4G)
+            return VERR_SSM_GCPTR_OVERFLOW;
+        *pGCPtr = (RTGCPTR)u64;
+        return rc;
+    }
+
+    /* 32-bit saved, 64-bit load: clear the high part. */
+    *pGCPtr = 0;
+    return ssmR3DataRead(pSSM, pGCPtr, sizeof(uint32_t));
+}
+
+
+/**
+ * Loads a GC virtual address (represented as unsigned integer) item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pGCPtr          Where to store the GC virtual address.
+ */
+VMMR3DECL(int) SSMR3GetGCUIntPtr(PSSMHANDLE pSSM, PRTGCUINTPTR pGCPtr)
+{
+    AssertCompile(sizeof(RTGCPTR) == sizeof(*pGCPtr));
+    return SSMR3GetGCPtr(pSSM, (PRTGCPTR)pGCPtr);
+}
+
+
+/**
+ * Loads an RC virtual address item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pRCPtr          Where to store the RC virtual address.
+ */
+VMMR3DECL(int) SSMR3GetRCPtr(PSSMHANDLE pSSM, PRTRCPTR pRCPtr)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pRCPtr, sizeof(*pRCPtr));
+}
+
+
+/**
+ * Loads a I/O port address item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pIOPort         Where to store the I/O port address.
+ */
+VMMR3DECL(int) SSMR3GetIOPort(PSSMHANDLE pSSM, PRTIOPORT pIOPort)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pIOPort, sizeof(*pIOPort));
+}
+
+
+/**
+ * Loads a selector item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pSel            Where to store the selector.
+ */
+VMMR3DECL(int) SSMR3GetSel(PSSMHANDLE pSSM, PRTSEL pSel)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pSel, sizeof(*pSel));
+}
+
+
+/**
+ * Loads a memory item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   pv              Where to store the item.
+ * @param   cb              Size of the item.
+ */
+VMMR3DECL(int) SSMR3GetMem(PSSMHANDLE pSSM, void *pv, size_t cb)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    return ssmR3DataRead(pSSM, pv, cb);
+}
+
+
+/**
+ * Loads a string item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   psz             Where to store the item.
+ * @param   cbMax           Max size of the item (including '\\0').
+ */
+VMMR3DECL(int) SSMR3GetStrZ(PSSMHANDLE pSSM, char *psz, size_t cbMax)
+{
+    return SSMR3GetStrZEx(pSSM, psz, cbMax, NULL);
+}
+
+
+/**
+ * Loads a string item from the current data unit.
+ *
+ * @returns VBox status.
+ * @param   pSSM            The saved state handle.
+ * @param   psz             Where to store the item.
+ * @param   cbMax           Max size of the item (including '\\0').
+ * @param   pcbStr          The length of the loaded string excluding the '\\0'. (optional)
+ */
+VMMR3DECL(int) SSMR3GetStrZEx(PSSMHANDLE pSSM, char *psz, size_t cbMax, size_t *pcbStr)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+
+    /* read size prefix. */
+    uint32_t u32;
+    int rc = SSMR3GetU32(pSSM, &u32);
+    if (RT_SUCCESS(rc))
+    {
+        if (pcbStr)
+            *pcbStr = u32;
+        if (u32 < cbMax)
+        {
+            /* terminate and read string content. */
+            psz[u32] = '\0';
+            return ssmR3DataRead(pSSM, psz, u32);
+        }
+        return VERR_TOO_MUCH_DATA;
+    }
+    return rc;
+}
+
+
+/**
+ * Skips a number of bytes in the current data unit.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   cb                  The number of bytes to skip.
+ */
+VMMR3DECL(int) SSMR3Skip(PSSMHANDLE pSSM, size_t cb)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    while (cb > 0)
+    {
+        uint8_t abBuf[8192];
+        size_t  cbCur = RT_MIN(sizeof(abBuf), cb);
+        cb -= cbCur;
+        int rc = ssmR3DataRead(pSSM, abBuf, cbCur);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Skips to the end of the current data unit.
+ *
+ * Since version 2 of the format, the load exec callback have to explicitly call
+ * this API if it wish to be lazy for some reason.  This is because there seldom
+ * is a good reason to not read your entire data unit and it was hiding bugs.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The saved state handle.
+ */
+VMMR3DECL(int) SSMR3SkipToEndOfUnit(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_READABLE_RET(pSSM);
+    SSM_CHECK_CANCELLED_RET(pSSM);
+    if (pSSM->u.Read.uFmtVerMajor >= 2)
+    {
+        /*
+         * Read until we the end of data condition is raised.
+         */
+        pSSM->u.Read.cbDataBuffer  = 0;
+        pSSM->u.Read.offDataBuffer = 0;
+        if (!pSSM->u.Read.fEndOfData)
+        {
+            do
+            {
+                /* read the rest of the current record */
+                while (pSSM->u.Read.cbRecLeft)
+                {
+                    uint8_t  abBuf[8192];
+                    uint32_t cbToRead = RT_MIN(pSSM->u.Read.cbRecLeft, sizeof(abBuf));
+                    int rc = ssmR3DataReadV2Raw(pSSM, abBuf, cbToRead);
+                    if (RT_FAILURE(rc))
+                        return pSSM->rc = rc;
+                    pSSM->u.Read.cbRecLeft -= cbToRead;
+                }
+
+                /* read the next header. */
+                int rc = ssmR3DataReadRecHdrV2(pSSM);
+                if (RT_FAILURE(rc))
+                    return pSSM->rc = rc;
+            } while (!pSSM->u.Read.fEndOfData);
+        }
+    }
+    /* else: Doesn't matter for the version 1 loading. */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Calculate the checksum of a file portion.
+ *
+ * @returns VBox status.
+ * @param   pStrm       The stream handle
+ * @param   off         Where to start checksumming.
+ * @param   cb          How much to checksum.
+ * @param   pu32CRC     Where to store the calculated checksum.
+ */
+static int ssmR3CalcChecksum(PSSMSTRM pStrm, uint64_t off, uint64_t cb, uint32_t *pu32CRC)
+{
+    /*
+     * Allocate a buffer.
+     */
+    const size_t cbBuf = _32K;
+    void *pvBuf = RTMemTmpAlloc(cbBuf);
+    if (!pvBuf)
+        return VERR_NO_TMP_MEMORY;
+
+    /*
+     * Loop reading and calculating CRC32.
+     */
+    int         rc     = VINF_SUCCESS;
+    uint32_t    u32CRC = RTCrc32Start();
+    while (cb > 0)
+    {
+        /* read chunk */
+        size_t cbToRead = cbBuf;
+        if (cb < cbBuf)
+            cbToRead = cb;
+        rc = ssmR3StrmPeekAt(pStrm, off, pvBuf, cbToRead, NULL);
+        if (RT_FAILURE(rc))
+        {
+            AssertMsgFailed(("Failed with rc=%Rrc while calculating crc.\n", rc));
+            RTMemTmpFree(pvBuf);
+            return rc;
+        }
+
+        /* advance */
+        cb  -= cbToRead;
+        off += cbToRead;
+
+        /* calc crc32. */
+        u32CRC = RTCrc32Process(u32CRC, pvBuf, cbToRead);
+    }
+    RTMemTmpFree(pvBuf);
+
+    /* store the calculated crc */
+    u32CRC = RTCrc32Finish(u32CRC);
+    Log(("SSM: u32CRC=0x%08x\n", u32CRC));
+    *pu32CRC = u32CRC;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates a version 2 footer.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pFooter             The footer.
+ * @param   offFooter           The stream offset of the footer.
+ * @param   cDirEntries         The number of directory entries. UINT32_MAX if
+ *                              unknown.
+ * @param   fStreamCrc32        Whether the stream is checksummed using CRC-32.
+ * @param   u32StreamCRC        The stream checksum.
+ */
+static int ssmR3ValidateFooter(PSSMFILEFTR pFooter, uint64_t offFooter, uint32_t cDirEntries, bool fStreamCrc32, uint32_t u32StreamCRC)
+{
+    if (memcmp(pFooter->szMagic, SSMFILEFTR_MAGIC, sizeof(pFooter->szMagic)))
+    {
+        LogRel(("SSM: Bad footer magic: %.*Rhxs\n", sizeof(pFooter->szMagic), &pFooter->szMagic[0]));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    SSM_CHECK_CRC32_RET(pFooter, sizeof(*pFooter), ("Footer CRC mismatch: %08x, correct is %08x\n", u32CRC, u32ActualCRC));
+    if (pFooter->offStream != offFooter)
+    {
+        LogRel(("SSM: SSMFILEFTR::offStream is wrong: %llx, expected %llx\n", pFooter->offStream, offFooter));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    if (pFooter->u32Reserved)
+    {
+        LogRel(("SSM: Reserved footer field isn't zero: %08x\n", pFooter->u32Reserved));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    if (cDirEntries != UINT32_MAX)
+        AssertLogRelMsgReturn(pFooter->cDirEntries == cDirEntries,
+                              ("Footer: cDirEntries=%#x, expected %#x\n", pFooter->cDirEntries, cDirEntries),
+                              VERR_SSM_INTEGRITY_FOOTER);
+    else
+        AssertLogRelMsgReturn(pFooter->cDirEntries < _64K,
+                              ("Footer: cDirEntries=%#x\n", pFooter->cDirEntries),
+                              VERR_SSM_INTEGRITY_FOOTER);
+    if (    !fStreamCrc32
+        &&  pFooter->u32StreamCRC)
+    {
+        LogRel(("SSM: u32StreamCRC field isn't zero, but header says stream checksumming is disabled.\n"));
+        return VERR_SSM_INTEGRITY_FOOTER;
+    }
+    if (    fStreamCrc32
+        &&  pFooter->u32StreamCRC != u32StreamCRC)
+    {
+        LogRel(("SSM: Bad stream CRC: %#x, expected %#x.\n", pFooter->u32StreamCRC, u32StreamCRC));
+        return VERR_SSM_INTEGRITY_CRC;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates the header information stored in the handle.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The handle.
+ * @param   fHaveHostBits   Set if the host bits field is valid.
+ * @param   fHaveVersion    Set if we have a version.
+ */
+static int ssmR3ValidateHeaderInfo(PSSMHANDLE pSSM, bool fHaveHostBits, bool fHaveVersion)
+{
+    Assert(pSSM->u.Read.cbFileHdr < 256 && pSSM->u.Read.cbFileHdr > 32);
+    Assert(pSSM->u.Read.uFmtVerMajor == 1 || pSSM->u.Read.uFmtVerMajor == 2);
+    Assert(pSSM->u.Read.uFmtVerMinor <= 2);
+
+    if (fHaveVersion)
+    {
+        if (    pSSM->u.Read.u16VerMajor == 0
+            ||  pSSM->u.Read.u16VerMajor > 1000
+            ||  pSSM->u.Read.u16VerMinor > 1000
+            ||  pSSM->u.Read.u32VerBuild > _1M
+            ||  pSSM->u.Read.u32SvnRev == 0
+            ||  pSSM->u.Read.u32SvnRev > 10000000 /*100M*/)
+        {
+            LogRel(("SSM: Incorrect version values: %u.%u.%u.r%u\n",
+                    pSSM->u.Read.u16VerMajor, pSSM->u.Read.u16VerMinor, pSSM->u.Read.u32VerBuild, pSSM->u.Read.u32SvnRev));
+            return VERR_SSM_INTEGRITY_VBOX_VERSION;
+        }
+    }
+    else
+        AssertLogRelReturn(   pSSM->u.Read.u16VerMajor == 0
+                           && pSSM->u.Read.u16VerMinor == 0
+                           && pSSM->u.Read.u32VerBuild == 0
+                           && pSSM->u.Read.u32SvnRev   == 0,
+                           VERR_SSM_INTEGRITY_VBOX_VERSION);
+
+    if (fHaveHostBits)
+    {
+        if (    pSSM->u.Read.cHostBits != 32
+            &&  pSSM->u.Read.cHostBits != 64)
+        {
+            LogRel(("SSM: Incorrect cHostBits value: %u\n", pSSM->u.Read.cHostBits));
+            return VERR_SSM_INTEGRITY_HEADER;
+        }
+    }
+    else
+        AssertLogRelReturn(pSSM->u.Read.cHostBits == 0, VERR_SSM_INTEGRITY_HEADER);
+
+    if (    pSSM->u.Read.cbGCPhys != sizeof(uint32_t)
+        &&  pSSM->u.Read.cbGCPhys != sizeof(uint64_t))
+    {
+        LogRel(("SSM: Incorrect cbGCPhys value: %d\n", pSSM->u.Read.cbGCPhys));
+        return VERR_SSM_INTEGRITY_HEADER;
+    }
+    if (    pSSM->u.Read.cbGCPtr != sizeof(uint32_t)
+        &&  pSSM->u.Read.cbGCPtr != sizeof(uint64_t))
+    {
+        LogRel(("SSM: Incorrect cbGCPtr value: %d\n", pSSM->u.Read.cbGCPtr));
+        return VERR_SSM_INTEGRITY_HEADER;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reads the header, detects the format version and performs integrity
+ * validations.
+ *
+ * @returns VBox status.
+ * @param   pSSM                The saved state handle.  A number of field will
+ *                              be updated, mostly header related information.
+ *                              fLiveSave is also set if appropriate.
+ * @param   fChecksumIt         Whether to checksum the file or not.  This will
+ *                              be ignored if it the stream isn't a file.
+ * @param   fChecksumOnRead     Whether to validate the checksum while reading
+ *                              the stream instead of up front. If not possible,
+ *                              verify the checksum up front.
+ * @param   pHdr                Where to store the file header.
+ */
+static int ssmR3HeaderAndValidate(PSSMHANDLE pSSM, bool fChecksumIt, bool fChecksumOnRead)
+{
+    /*
+     * Read and check the header magic.
+     */
+    union
+    {
+        SSMFILEHDR          v2_0;
+        SSMFILEHDRV12       v1_2;
+        SSMFILEHDRV11       v1_1;
+    } uHdr;
+    int rc = ssmR3StrmRead(&pSSM->Strm, &uHdr, sizeof(uHdr.v2_0.szMagic));
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed to read file magic header. rc=%Rrc\n", rc));
+        return rc;
+    }
+    if (memcmp(uHdr.v2_0.szMagic, SSMFILEHDR_MAGIC_BASE, sizeof(SSMFILEHDR_MAGIC_BASE) - 1))
+    {
+        Log(("SSM: Not a saved state file. magic=%.*s\n", sizeof(uHdr.v2_0.szMagic) - 1, uHdr.v2_0.szMagic));
+        return VERR_SSM_INTEGRITY_MAGIC;
+    }
+
+    /*
+     * Find the header size and read the rest.
+     */
+    static const struct
+    {
+        char        szMagic[sizeof(SSMFILEHDR_MAGIC_V2_0)];
+        uint32_t    cbHdr;
+        unsigned    uFmtVerMajor;
+        unsigned    uFmtVerMinor;
+    }   s_aVers[] =
+    {
+        { SSMFILEHDR_MAGIC_V2_0, sizeof(SSMFILEHDR),    2, 0 },
+        { SSMFILEHDR_MAGIC_V1_2, sizeof(SSMFILEHDRV12), 1, 2 },
+        { SSMFILEHDR_MAGIC_V1_1, sizeof(SSMFILEHDRV11), 1, 1 },
+    };
+    int iVer = RT_ELEMENTS(s_aVers);
+    while (iVer-- > 0)
+        if (!memcmp(uHdr.v2_0.szMagic, s_aVers[iVer].szMagic, sizeof(uHdr.v2_0.szMagic)))
+            break;
+    if (iVer < 0)
+    {
+        Log(("SSM: Unknown file format version. magic=%.*s\n", sizeof(uHdr.v2_0.szMagic) - 1, uHdr.v2_0.szMagic));
+        return VERR_SSM_INTEGRITY_VERSION;
+    }
+    pSSM->u.Read.uFmtVerMajor   = s_aVers[iVer].uFmtVerMajor;
+    pSSM->u.Read.uFmtVerMinor   = s_aVers[iVer].uFmtVerMinor;
+    pSSM->u.Read.cbFileHdr      = s_aVers[iVer].cbHdr;
+
+    rc = ssmR3StrmRead(&pSSM->Strm, (uint8_t *)&uHdr + sizeof(uHdr.v2_0.szMagic), pSSM->u.Read.cbFileHdr - sizeof(uHdr.v2_0.szMagic));
+    if (RT_FAILURE(rc))
+    {
+        LogRel(("SSM: Failed to read the file header. rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    /*
+     * Make version specific adjustments.
+     */
+    if (pSSM->u.Read.uFmtVerMajor >= 2)
+    {
+        /*
+         * Version 2.0 and later.
+         */
+        if (pSSM->u.Read.uFmtVerMinor == 0)
+        {
+            /* validate the header. */
+            SSM_CHECK_CRC32_RET(&uHdr.v2_0, sizeof(uHdr.v2_0), ("Header CRC mismatch: %08x, correct is %08x\n", u32CRC, u32ActualCRC));
+            if (uHdr.v2_0.u8Reserved)
+            {
+                LogRel(("SSM: Reserved header field isn't zero: %02x\n", uHdr.v2_0.u8Reserved));
+                return VERR_SSM_INTEGRITY;
+            }
+            if (uHdr.v2_0.fFlags & ~(SSMFILEHDR_FLAGS_STREAM_CRC32 | SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE))
+            {
+                LogRel(("SSM: Unknown header flags: %08x\n", uHdr.v2_0.fFlags));
+                return VERR_SSM_INTEGRITY;
+            }
+            if (    uHdr.v2_0.cbMaxDecompr > sizeof(pSSM->u.Read.abDataBuffer)
+                ||  uHdr.v2_0.cbMaxDecompr < _1K
+                ||  (uHdr.v2_0.cbMaxDecompr & 0xff) != 0)
+            {
+                LogRel(("SSM: The cbMaxDecompr header field is out of range: %#x\n", uHdr.v2_0.cbMaxDecompr));
+                return VERR_SSM_INTEGRITY;
+            }
+
+            /* set the header info. */
+            pSSM->u.Read.cHostBits      = uHdr.v2_0.cHostBits;
+            pSSM->u.Read.u16VerMajor    = uHdr.v2_0.u16VerMajor;
+            pSSM->u.Read.u16VerMinor    = uHdr.v2_0.u16VerMinor;
+            pSSM->u.Read.u32VerBuild    = uHdr.v2_0.u32VerBuild;
+            pSSM->u.Read.u32SvnRev      = uHdr.v2_0.u32SvnRev;
+            pSSM->u.Read.cbGCPhys       = uHdr.v2_0.cbGCPhys;
+            pSSM->u.Read.cbGCPtr        = uHdr.v2_0.cbGCPtr;
+            pSSM->u.Read.fFixedGCPtrSize= true;
+            pSSM->u.Read.fStreamCrc32   = !!(uHdr.v2_0.fFlags & SSMFILEHDR_FLAGS_STREAM_CRC32);
+            pSSM->fLiveSave             = !!(uHdr.v2_0.fFlags & SSMFILEHDR_FLAGS_STREAM_LIVE_SAVE);
+        }
+        else
+            AssertFailedReturn(VERR_INTERNAL_ERROR);
+        if (!pSSM->u.Read.fStreamCrc32)
+            ssmR3StrmDisableChecksumming(&pSSM->Strm);
+
+        /*
+         * Read and validate the footer if it's a file.
+         */
+        if (ssmR3StrmIsFile(&pSSM->Strm))
+        {
+            SSMFILEFTR  Footer;
+            uint64_t    offFooter;
+            rc = ssmR3StrmPeekAt(&pSSM->Strm, -(RTFOFF)sizeof(SSMFILEFTR), &Footer, sizeof(Footer), &offFooter);
+            AssertLogRelRCReturn(rc, rc);
+
+            rc = ssmR3ValidateFooter(&Footer, offFooter, UINT32_MAX, pSSM->u.Read.fStreamCrc32, Footer.u32StreamCRC);
+            if (RT_FAILURE(rc))
+                return rc;
+
+            pSSM->u.Read.cbLoadFile = offFooter + sizeof(Footer);
+            pSSM->u.Read.u32LoadCRC = Footer.u32StreamCRC;
+        }
+        else
+        {
+            pSSM->u.Read.cbLoadFile = UINT64_MAX;
+            pSSM->u.Read.u32LoadCRC = 0;
+        }
+
+        /*
+         * Validate the header info we've set in the handle.
+         */
+        rc = ssmR3ValidateHeaderInfo(pSSM, true /*fHaveHostBits*/, true /*fHaveVersion*/);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Check the checksum if that's called for and possible.
+         */
+        if (    pSSM->u.Read.fStreamCrc32
+            &&  fChecksumIt
+            &&  !fChecksumOnRead
+            &&  ssmR3StrmIsFile(&pSSM->Strm))
+        {
+            uint32_t u32CRC;
+            rc = ssmR3CalcChecksum(&pSSM->Strm, 0, pSSM->u.Read.cbLoadFile - sizeof(SSMFILEFTR), &u32CRC);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (u32CRC != pSSM->u.Read.u32LoadCRC)
+            {
+                LogRel(("SSM: Invalid CRC! Calculated %#010x, in footer %#010x\n", u32CRC, pSSM->u.Read.u32LoadCRC));
+                return VERR_SSM_INTEGRITY_CRC;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Version 1.x of the format.
+         */
+        bool        fHaveHostBits = true;
+        bool        fHaveVersion  = false;
+        RTUUID      MachineUuidFromHdr;
+
+        ssmR3StrmDisableChecksumming(&pSSM->Strm);
+        if (pSSM->u.Read.uFmtVerMinor == 1)
+        {
+            pSSM->u.Read.cHostBits      = 0; /* unknown */
+            pSSM->u.Read.u16VerMajor    = 0;
+            pSSM->u.Read.u16VerMinor    = 0;
+            pSSM->u.Read.u32VerBuild    = 0;
+            pSSM->u.Read.u32SvnRev      = 0;
+            pSSM->u.Read.cbLoadFile     = uHdr.v1_1.cbFile;
+            pSSM->u.Read.u32LoadCRC     = uHdr.v1_1.u32CRC;
+            pSSM->u.Read.cbGCPhys       = sizeof(RTGCPHYS);
+            pSSM->u.Read.cbGCPtr        = sizeof(RTGCPTR);
+            pSSM->u.Read.fFixedGCPtrSize = false; /* settable */
+            pSSM->u.Read.fStreamCrc32   = false;
+
+            MachineUuidFromHdr  = uHdr.v1_1.MachineUuid;
+            fHaveHostBits       = false;
+        }
+        else if (pSSM->u.Read.uFmtVerMinor == 2)
+        {
+            pSSM->u.Read.cHostBits      = uHdr.v1_2.cHostBits;
+            pSSM->u.Read.u16VerMajor    = uHdr.v1_2.u16VerMajor;
+            pSSM->u.Read.u16VerMinor    = uHdr.v1_2.u16VerMinor;
+            pSSM->u.Read.u32VerBuild    = uHdr.v1_2.u32VerBuild;
+            pSSM->u.Read.u32SvnRev      = uHdr.v1_2.u32SvnRev;
+            pSSM->u.Read.cbLoadFile     = uHdr.v1_2.cbFile;
+            pSSM->u.Read.u32LoadCRC     = uHdr.v1_2.u32CRC;
+            pSSM->u.Read.cbGCPhys       = uHdr.v1_2.cbGCPhys;
+            pSSM->u.Read.cbGCPtr        = uHdr.v1_2.cbGCPtr;
+            pSSM->u.Read.fFixedGCPtrSize = true;
+            pSSM->u.Read.fStreamCrc32   = false;
+
+            MachineUuidFromHdr  = uHdr.v1_2.MachineUuid;
+            fHaveVersion        = true;
+        }
+        else
+            AssertFailedReturn(VERR_INTERNAL_ERROR);
+
+        /*
+         * The MachineUuid must be NULL (was never used).
+         */
+        if (!RTUuidIsNull(&MachineUuidFromHdr))
+        {
+            LogRel(("SSM: The UUID of the saved state doesn't match the running VM.\n"));
+            return VERR_SMM_INTEGRITY_MACHINE;
+        }
+
+        /*
+         * Verify the file size.
+         */
+        uint64_t cbFile = ssmR3StrmGetSize(&pSSM->Strm);
+        if (cbFile != pSSM->u.Read.cbLoadFile)
+        {
+            LogRel(("SSM: File size mismatch. hdr.cbFile=%lld actual %lld\n", pSSM->u.Read.cbLoadFile, cbFile));
+            return VERR_SSM_INTEGRITY_SIZE;
+        }
+
+        /*
+         * Validate the header info we've set in the handle.
+         */
+        rc = ssmR3ValidateHeaderInfo(pSSM, fHaveHostBits, fHaveVersion);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Verify the checksum if requested.
+         *
+         * Note! The checksum is not actually generated for the whole file,
+         *       this is of course a bug in the v1.x code that we cannot do
+         *       anything about.
+         */
+        if (    fChecksumIt
+            ||  fChecksumOnRead)
+        {
+            uint32_t u32CRC;
+            rc = ssmR3CalcChecksum(&pSSM->Strm,
+                                   RT_OFFSETOF(SSMFILEHDRV11, u32CRC) + sizeof(uHdr.v1_1.u32CRC),
+                                   cbFile - pSSM->u.Read.cbFileHdr,
+                                   &u32CRC);
+            if (RT_FAILURE(rc))
+                return rc;
+            if (u32CRC != pSSM->u.Read.u32LoadCRC)
+            {
+                LogRel(("SSM: Invalid CRC! Calculated %#010x, in header %#010x\n", u32CRC, pSSM->u.Read.u32LoadCRC));
+                return VERR_SSM_INTEGRITY_CRC;
+            }
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Open a saved state for reading.
+ *
+ * The file will be positioned at the first data unit upon successful return.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pszFilename         The filename. NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream method table. NULL if pszFilename is
+ *                              used.
+ * @param   pvUser              The user argument to the stream methods.
+ * @param   fChecksumIt         Check the checksum for the entire file.
+ * @param   fChecksumOnRead     Whether to validate the checksum while reading
+ *                              the stream instead of up front. If not possible,
+ *                              verify the checksum up front.
+ * @param   pSSM                Pointer to the handle structure. This will be
+ *                              completely initialized on success.
+ * @param   cBuffers            The number of stream buffers.
+ */
+static int ssmR3OpenFile(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvUser,
+                         bool fChecksumIt, bool fChecksumOnRead, uint32_t cBuffers, PSSMHANDLE pSSM)
+{
+    /*
+     * Initialize the handle.
+     */
+    pSSM->pVM                   = pVM;
+    pSSM->enmOp                 = SSMSTATE_INVALID;
+    pSSM->enmAfter              = SSMAFTER_INVALID;
+    pSSM->fCancelled            = SSMHANDLE_OK;
+    pSSM->rc                    = VINF_SUCCESS;
+    pSSM->cbUnitLeftV1          = 0;
+    pSSM->offUnit               = UINT64_MAX;
+    pSSM->fLiveSave             = false;
+    pSSM->pfnProgress           = NULL;
+    pSSM->pvUser                = NULL;
+    pSSM->uPercent              = 0;
+    pSSM->offEstProgress        = 0;
+    pSSM->cbEstTotal            = 0;
+    pSSM->offEst                = 0;
+    pSSM->offEstUnitEnd         = 0;
+    pSSM->uPercentLive          = 0;
+    pSSM->uPercentPrepare       = 5;
+    pSSM->uPercentDone          = 2;
+    pSSM->uReportedLivePercent  = 0;
+    pSSM->pszFilename           = pszFilename;
+
+    pSSM->u.Read.pZipDecompV1   = NULL;
+    pSSM->u.Read.uFmtVerMajor   = UINT32_MAX;
+    pSSM->u.Read.uFmtVerMinor   = UINT32_MAX;
+    pSSM->u.Read.cbFileHdr      = UINT32_MAX;
+    pSSM->u.Read.cbGCPhys       = UINT8_MAX;
+    pSSM->u.Read.cbGCPtr        = UINT8_MAX;
+    pSSM->u.Read.fFixedGCPtrSize= false;
+    pSSM->u.Read.fIsHostMsc32   = SSM_HOST_IS_MSC_32;
+    RT_ZERO(pSSM->u.Read.szHostOSAndArch);
+    pSSM->u.Read.u16VerMajor    = UINT16_MAX;
+    pSSM->u.Read.u16VerMinor    = UINT16_MAX;
+    pSSM->u.Read.u32VerBuild    = UINT32_MAX;
+    pSSM->u.Read.u32SvnRev      = UINT32_MAX;
+    pSSM->u.Read.cHostBits      = UINT8_MAX;
+    pSSM->u.Read.cbLoadFile     = UINT64_MAX;
+
+    pSSM->u.Read.cbRecLeft      = 0;
+    pSSM->u.Read.cbDataBuffer   = 0;
+    pSSM->u.Read.offDataBuffer  = 0;
+    pSSM->u.Read.fEndOfData     = 0;
+    pSSM->u.Read.u8TypeAndFlags = 0;
+
+    pSSM->u.Read.pCurUnit       = NULL;
+    pSSM->u.Read.uCurUnitVer    = UINT32_MAX;
+    pSSM->u.Read.uCurUnitPass   = 0;
+    pSSM->u.Read.fHaveSetError  = false;
+
+    /*
+     * Try open and validate the file.
+     */
+    int rc;
+    if (pStreamOps)
+        rc = ssmR3StrmInit(&pSSM->Strm, pStreamOps, pvUser, false /*fWrite*/, fChecksumOnRead, cBuffers);
+    else
+        rc = ssmR3StrmOpenFile(&pSSM->Strm, pszFilename, false /*fWrite*/, fChecksumOnRead, cBuffers);
+    if (RT_SUCCESS(rc))
+    {
+        rc = ssmR3HeaderAndValidate(pSSM, fChecksumIt, fChecksumOnRead);
+        if (RT_SUCCESS(rc))
+            return rc;
+
+        /* failure path */
+        ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    }
+    else
+        Log(("SSM: Failed to open save state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+    return rc;
+}
+
+
+/**
+ * Verifies the directory.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pDir        The full directory.
+ * @param   cbDir       The size of the directory.
+ * @param   offDir      The directory stream offset.
+ * @param   cDirEntries The directory entry count from the footer.
+ * @param   cbHdr       The header size.
+ * @param   uSvnRev     The SVN revision that saved the state. Bug detection.
+ */
+static int ssmR3ValidateDirectory(PSSMFILEDIR pDir, size_t cbDir, uint64_t offDir, uint32_t cDirEntries,
+                                  uint32_t cbHdr, uint32_t uSvnRev)
+{
+    AssertLogRelReturn(!memcmp(pDir->szMagic, SSMFILEDIR_MAGIC, sizeof(pDir->szMagic)), VERR_SSM_INTEGRITY_DIR_MAGIC);
+    SSM_CHECK_CRC32_RET(pDir, cbDir, ("Bad directory CRC: %08x, actual %08x\n", u32CRC, u32ActualCRC));
+    AssertLogRelMsgReturn(pDir->cEntries == cDirEntries,
+                          ("Bad directory entry count: %#x, expected %#x (from the footer)\n", pDir->cEntries, cDirEntries),
+                           VERR_SSM_INTEGRITY_DIR);
+    AssertLogRelReturn(RT_UOFFSETOF(SSMFILEDIR, aEntries[pDir->cEntries]) == cbDir, VERR_SSM_INTEGRITY_DIR);
+
+    for (uint32_t i = 0; i < pDir->cEntries; i++)
+    {
+        AssertLogRelMsgReturn(  (   pDir->aEntries[i].off >= cbHdr
+                                 && pDir->aEntries[i].off <  offDir)
+                              || (   pDir->aEntries[i].off == 0 /* bug in unreleased code */
+                                  && uSvnRev < 53365),
+                              ("off=%#llx cbHdr=%#x offDir=%#llx\n", pDir->aEntries[i].off, cbHdr, offDir),
+                              VERR_SSM_INTEGRITY_DIR);
+    }
+    return VINF_SUCCESS;
+}
+
+#ifndef SSM_STANDALONE
+
+/**
+ * Find a data unit by name.
+ *
+ * @returns Pointer to the unit.
+ * @returns NULL if not found.
+ *
+ * @param   pVM             VM handle.
+ * @param   pszName         Data unit name.
+ * @param   uInstance       The data unit instance id.
+ */
+static PSSMUNIT ssmR3Find(PVM pVM, const char *pszName, uint32_t uInstance)
+{
+    size_t   cchName = strlen(pszName);
+    PSSMUNIT pUnit = pVM->ssm.s.pHead;
+    while (     pUnit
+           &&   (   pUnit->u32Instance != uInstance
+                 || pUnit->cchName != cchName
+                 || memcmp(pUnit->szName, pszName, cchName)))
+        pUnit = pUnit->pNext;
+    return pUnit;
+}
+
+
+/**
+ * Executes the loading of a V1.X file.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3LoadExecV1(PVM pVM, PSSMHANDLE pSSM)
+{
+    int     rc;
+    char   *pszName = NULL;
+    size_t  cchName = 0;
+    pSSM->enmOp = SSMSTATE_LOAD_EXEC;
+    for (;;)
+    {
+        /*
+         * Save the current file position and read the data unit header.
+         */
+        uint64_t         offUnit = ssmR3StrmTell(&pSSM->Strm);
+        SSMFILEUNITHDRV1 UnitHdr;
+        rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV1, szName));
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Check the magic and see if it's valid and whether it is a end header or not.
+             */
+            if (memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(SSMFILEUNITHDR_MAGIC)))
+            {
+                if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_END, sizeof(SSMFILEUNITHDR_END)))
+                {
+                    Log(("SSM: EndOfFile: offset %#9llx size %9d\n", offUnit, UnitHdr.cbUnit));
+                    /* Complete the progress bar (pending 99% afterwards). */
+                    ssmR3ProgressByByte(pSSM, pSSM->cbEstTotal - pSSM->offEst);
+                    break;
+                }
+                LogRel(("SSM: Invalid unit magic at offset %#llx (%lld), '%.*s'!\n",
+                        offUnit, offUnit, sizeof(UnitHdr.achMagic) - 1, &UnitHdr.achMagic[0]));
+                rc = VERR_SSM_INTEGRITY_UNIT_MAGIC;
+                break;
+            }
+
+            /*
+             * Read the name.
+             * Adjust the name buffer first.
+             */
+            if (cchName < UnitHdr.cchName)
+            {
+                if (pszName)
+                    RTMemTmpFree(pszName);
+                cchName = RT_ALIGN_Z(UnitHdr.cchName, 64);
+                pszName = (char *)RTMemTmpAlloc(cchName);
+            }
+            if (pszName)
+            {
+                rc = ssmR3StrmRead(&pSSM->Strm, pszName, UnitHdr.cchName);
+                if (RT_SUCCESS(rc))
+                {
+                    if (pszName[UnitHdr.cchName - 1])
+                    {
+                        LogRel(("SSM: Unit name '%.*s' was not properly terminated.\n", UnitHdr.cchName, pszName));
+                        rc = VERR_SSM_INTEGRITY_UNIT;
+                        break;
+                    }
+                    Log(("SSM: Data unit: offset %#9llx size %9lld '%s'\n", offUnit, UnitHdr.cbUnit, pszName));
+
+                    /*
+                     * Find the data unit in our internal table.
+                     */
+                    PSSMUNIT pUnit = ssmR3Find(pVM, pszName, UnitHdr.u32Instance);
+                    if (pUnit)
+                    {
+                        /*
+                         * Call the execute handler.
+                         */
+                        pSSM->cbUnitLeftV1 = UnitHdr.cbUnit - RT_OFFSETOF(SSMFILEUNITHDRV1, szName[UnitHdr.cchName]);
+                        pSSM->offUnit = 0;
+                        pSSM->u.Read.uCurUnitVer  = UnitHdr.u32Version;
+                        pSSM->u.Read.uCurUnitPass = SSM_PASS_FINAL;
+                        pSSM->u.Read.pCurUnit     = pUnit;
+                        if (!pUnit->u.Common.pfnLoadExec)
+                        {
+                            LogRel(("SSM: No load exec callback for unit '%s'!\n", pszName));
+                            pSSM->rc = rc = VERR_SSM_NO_LOAD_EXEC;
+                            break;
+                        }
+                        switch (pUnit->enmType)
+                        {
+                            case SSMUNITTYPE_DEV:
+                                rc = pUnit->u.Dev.pfnLoadExec(pUnit->u.Dev.pDevIns, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            case SSMUNITTYPE_DRV:
+                                rc = pUnit->u.Drv.pfnLoadExec(pUnit->u.Drv.pDrvIns, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            case SSMUNITTYPE_INTERNAL:
+                                rc = pUnit->u.Internal.pfnLoadExec(pVM, pSSM, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            case SSMUNITTYPE_EXTERNAL:
+                                rc = pUnit->u.External.pfnLoadExec(pSSM, pUnit->u.External.pvUser, UnitHdr.u32Version, SSM_PASS_FINAL);
+                                break;
+                            default:
+                                rc = VERR_INTERNAL_ERROR;
+                                break;
+                        }
+                        pUnit->fCalled = true;
+                        if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                            pSSM->rc = rc;
+
+                        /*
+                         * Close the reader stream.
+                         */
+                        rc = ssmR3DataReadFinishV1(pSSM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            /*
+                             * Now, we'll check the current position to see if all, or
+                             * more than all, the data was read.
+                             *
+                             * Note! Because of buffering / compression we'll only see the
+                             * really bad ones here.
+                             */
+                            uint64_t off = ssmR3StrmTell(&pSSM->Strm);
+                            int64_t i64Diff = off - (offUnit + UnitHdr.cbUnit);
+                            if (i64Diff < 0)
+                            {
+                                Log(("SSM: Unit '%s' left %lld bytes unread!\n", pszName, -i64Diff));
+                                rc = ssmR3StrmSkipTo(&pSSM->Strm, offUnit + UnitHdr.cbUnit);
+                                ssmR3ProgressByByte(pSSM, offUnit + UnitHdr.cbUnit - pSSM->offEst);
+                            }
+                            else if (i64Diff > 0)
+                            {
+                                LogRel(("SSM: Unit '%s' read %lld bytes too much!\n", pszName, i64Diff));
+                                if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
+                                    rc = VMSetError(pVM, VERR_SSM_LOADED_TOO_MUCH, RT_SRC_POS,
+                                                    N_("Unit '%s' read %lld bytes too much"), pszName, i64Diff);
+                                break;
+                            }
+
+                            pSSM->offUnit = UINT64_MAX;
+                        }
+                        else
+                        {
+                            LogRel(("SSM: Load exec failed for '%s' instance #%u ! (version %u)\n",
+                                    pszName, UnitHdr.u32Instance, UnitHdr.u32Version));
+                            if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
+                            {
+                                if (rc == VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION)
+                                    VMSetError(pVM, rc, RT_SRC_POS, N_("Unsupported version %u of data unit '%s' (instance #%u)"),
+                                               UnitHdr.u32Version, UnitHdr.szName, UnitHdr.u32Instance);
+                                else
+                                    VMSetError(pVM, rc, RT_SRC_POS, N_("Load exec failed for '%s' instance #%u (version %u)"),
+                                               pszName, UnitHdr.u32Instance, UnitHdr.u32Version);
+                            }
+                            break;
+                        }
+
+                        pSSM->u.Read.pCurUnit       = NULL;
+                        pSSM->u.Read.uCurUnitVer    = UINT32_MAX;
+                        pSSM->u.Read.uCurUnitPass   = 0;
+                    }
+                    else
+                    {
+                        /*
+                         * SSM unit wasn't found - ignore this when loading for the debugger.
+                         */
+                        LogRel(("SSM: Found no handler for unit '%s'!\n", pszName));
+                        rc = VERR_SSM_INTEGRITY_UNIT_NOT_FOUND;
+                        if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
+                            break;
+                        rc = ssmR3StrmSkipTo(&pSSM->Strm, offUnit + UnitHdr.cbUnit);
+                    }
+                }
+            }
+            else
+                rc = VERR_NO_TMP_MEMORY;
+        }
+
+        /*
+         * I/O errors ends up here (yea, I know, very nice programming).
+         */
+        if (RT_FAILURE(rc))
+        {
+            LogRel(("SSM: I/O error. rc=%Rrc\n", rc));
+            break;
+        }
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!n"));
+            rc = pSSM->rc;
+            if (RT_SUCCESS(pSSM->rc))
+                pSSM->rc = rc = VERR_SSM_CANCELLED;
+            break;
+        }
+    }
+
+    RTMemTmpFree(pszName);
+    return rc;
+}
+
+
+/**
+ * Reads and verifies the directory and footer.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3LoadDirectoryAndFooter(PSSMHANDLE pSSM)
+{
+    /*
+     * The directory.
+     *
+     * Get the header containing the number of entries first.  Then read the
+     * entries and pass the combined block to the validation function.
+     */
+    uint64_t        off      = ssmR3StrmTell(&pSSM->Strm);
+    size_t const    cbDirHdr = RT_OFFSETOF(SSMFILEDIR, aEntries);
+    SSMFILEDIR      DirHdr;
+    int rc = ssmR3StrmRead(&pSSM->Strm, &DirHdr, cbDirHdr);
+    if (RT_FAILURE(rc))
+        return rc;
+    AssertLogRelMsgReturn(!memcmp(DirHdr.szMagic, SSMFILEDIR_MAGIC, sizeof(DirHdr.szMagic)),
+                          ("Invalid directory magic at %#llx (%lld): %.*Rhxs\n", off, off, sizeof(DirHdr.szMagic), DirHdr.szMagic),
+                          VERR_SSM_INTEGRITY_DIR_MAGIC);
+    AssertLogRelMsgReturn(DirHdr.cEntries < _64K,
+                          ("Too many directory entries at %#llx (%lld): %#x\n", off, off, DirHdr.cEntries),
+                          VERR_SSM_INTEGRITY_DIR);
+
+    size_t      cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[DirHdr.cEntries]);
+    PSSMFILEDIR pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
+    if (!pDir)
+        return VERR_NO_TMP_MEMORY;
+    memcpy(pDir, &DirHdr, cbDirHdr);
+    rc = ssmR3StrmRead(&pSSM->Strm, (uint8_t *)pDir + cbDirHdr, cbDir - cbDirHdr);
+    if (RT_SUCCESS(rc))
+        rc = ssmR3ValidateDirectory(pDir, cbDir, off, DirHdr.cEntries, pSSM->u.Read.cbFileHdr, pSSM->u.Read.u32SvnRev);
+    RTMemTmpFree(pDir);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Read and validate the footer.
+     */
+    off = ssmR3StrmTell(&pSSM->Strm);
+    uint32_t    u32StreamCRC = ssmR3StrmFinalCRC(&pSSM->Strm);
+    SSMFILEFTR  Footer;
+    rc = ssmR3StrmRead(&pSSM->Strm, &Footer, sizeof(Footer));
+    if (RT_FAILURE(rc))
+        return rc;
+    return ssmR3ValidateFooter(&Footer, off, DirHdr.cEntries, pSSM->u.Read.fStreamCrc32, u32StreamCRC);
+}
+
+
+/**
+ * Executes the loading of a V2.X file.
+ *
+ * @returns VBox status code.  May or may not set pSSM->rc, the returned
+ *          status code is ALWAYS the more accurate of the two.
+ * @param   pVM                 The VM handle.
+ * @param   pSSM                The saved state handle.
+ */
+static int ssmR3LoadExecV2(PVM pVM, PSSMHANDLE pSSM)
+{
+    pSSM->enmOp = SSMSTATE_LOAD_EXEC;
+    for (;;)
+    {
+        /*
+         * Read the unit header and check its integrity.
+         */
+        uint64_t            offUnit         = ssmR3StrmTell(&pSSM->Strm);
+        uint32_t            u32CurStreamCRC = ssmR3StrmCurCRC(&pSSM->Strm);
+        SSMFILEUNITHDRV2    UnitHdr;
+        int rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName));
+        if (RT_FAILURE(rc))
+            return rc;
+        if (RT_UNLIKELY(    memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic))
+                        &&  memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END,   sizeof(UnitHdr.szMagic))))
+        {
+            LogRel(("SSM: Unit at %#llx (%lld): Invalid unit magic: %.*Rhxs!\n",
+                    offUnit, offUnit, sizeof(UnitHdr.szMagic) - 1, &UnitHdr.szMagic[0]));
+            pSSM->u.Read.fHaveSetError = true;
+            return VMSetError(pVM, VERR_SSM_INTEGRITY_UNIT_MAGIC, RT_SRC_POS,
+                              N_("Unit at %#llx (%lld): Invalid unit magic"), offUnit, offUnit);
+        }
+        if (UnitHdr.cbName)
+        {
+            AssertLogRelMsgReturn(UnitHdr.cbName <= sizeof(UnitHdr.szName),
+                                  ("Unit at %#llx (%lld): UnitHdr.cbName=%u > %u\n",
+                                   offUnit, offUnit, UnitHdr.cbName, sizeof(UnitHdr.szName)),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            rc = ssmR3StrmRead(&pSSM->Strm, &UnitHdr.szName[0], UnitHdr.cbName);
+            if (RT_FAILURE(rc))
+                return rc;
+            AssertLogRelMsgReturn(!UnitHdr.szName[UnitHdr.cbName - 1],
+                                  ("Unit at %#llx (%lld): Name %.*Rhxs was not properly terminated.\n",
+                                   offUnit, offUnit, UnitHdr.cbName, UnitHdr.szName),
+                                  VERR_SSM_INTEGRITY_UNIT);
+        }
+        SSM_CHECK_CRC32_RET(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]),
+                            ("Unit at %#llx (%lld): CRC mismatch: %08x, correct is %08x\n", offUnit, offUnit, u32CRC, u32ActualCRC));
+        AssertLogRelMsgReturn(UnitHdr.offStream == offUnit,
+                              ("Unit at %#llx (%lld): offStream=%#llx, expected %#llx\n", offUnit, offUnit, UnitHdr.offStream, offUnit),
+                              VERR_SSM_INTEGRITY_UNIT);
+        AssertLogRelMsgReturn(UnitHdr.u32CurStreamCRC == u32CurStreamCRC || !pSSM->Strm.fChecksummed,
+                              ("Unit at %#llx (%lld): Stream CRC mismatch: %08x, correct is %08x\n", offUnit, offUnit, UnitHdr.u32CurStreamCRC, u32CurStreamCRC),
+                              VERR_SSM_INTEGRITY_UNIT);
+        AssertLogRelMsgReturn(!UnitHdr.fFlags, ("Unit at %#llx (%lld): fFlags=%08x\n", offUnit, offUnit, UnitHdr.fFlags),
+                              VERR_SSM_INTEGRITY_UNIT);
+        if (!memcmp(&UnitHdr.szMagic[0], SSMFILEUNITHDR_END,   sizeof(UnitHdr.szMagic)))
+        {
+            AssertLogRelMsgReturn(   UnitHdr.cbName       == 0
+                                  && UnitHdr.u32Instance  == 0
+                                  && UnitHdr.u32Version   == 0
+                                  && UnitHdr.u32Pass      == SSM_PASS_FINAL,
+                                  ("Unit at %#llx (%lld): Malformed END unit\n", offUnit, offUnit),
+                                  VERR_SSM_INTEGRITY_UNIT);
+
+            /*
+             * Complete the progress bar (pending 99% afterwards) and RETURN.
+             */
+            Log(("SSM: Unit at %#9llx: END UNIT\n", offUnit));
+            ssmR3ProgressByByte(pSSM, pSSM->cbEstTotal - pSSM->offEst);
+            return ssmR3LoadDirectoryAndFooter(pSSM);
+        }
+        AssertLogRelMsgReturn(UnitHdr.cbName > 1, ("Unit at %#llx (%lld): No name\n", offUnit, offUnit), VERR_SSM_INTEGRITY);
+
+        Log(("SSM: Unit at %#9llx: '%s', instance %u, pass %#x, version %u\n",
+             offUnit, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass, UnitHdr.u32Version));
+
+        /*
+         * Find the data unit in our internal table.
+         */
+        PSSMUNIT pUnit = ssmR3Find(pVM, UnitHdr.szName, UnitHdr.u32Instance);
+        if (pUnit)
+        {
+            /*
+             * Call the execute handler.
+             */
+            AssertLogRelMsgReturn(pUnit->u.Common.pfnLoadExec,
+                                  ("SSM: No load exec callback for unit '%s'!\n", UnitHdr.szName),
+                                  VERR_SSM_NO_LOAD_EXEC);
+            pSSM->u.Read.uCurUnitVer  = UnitHdr.u32Version;
+            pSSM->u.Read.uCurUnitPass = UnitHdr.u32Pass;
+            pSSM->u.Read.pCurUnit     = pUnit;
+            ssmR3DataReadBeginV2(pSSM);
+            switch (pUnit->enmType)
+            {
+                case SSMUNITTYPE_DEV:
+                    rc = pUnit->u.Dev.pfnLoadExec(pUnit->u.Dev.pDevIns, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                case SSMUNITTYPE_DRV:
+                    rc = pUnit->u.Drv.pfnLoadExec(pUnit->u.Drv.pDrvIns, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                case SSMUNITTYPE_INTERNAL:
+                    rc = pUnit->u.Internal.pfnLoadExec(pVM, pSSM, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                case SSMUNITTYPE_EXTERNAL:
+                    rc = pUnit->u.External.pfnLoadExec(pSSM, pUnit->u.External.pvUser, UnitHdr.u32Version, UnitHdr.u32Pass);
+                    break;
+                default:
+                    rc = VERR_INTERNAL_ERROR;
+                    break;
+            }
+            pUnit->fCalled = true;
+            if (RT_FAILURE(rc) && RT_SUCCESS_NP(pSSM->rc))
+                pSSM->rc = rc;
+            rc = ssmR3DataReadFinishV2(pSSM);
+            if (RT_SUCCESS(rc))
+                pSSM->offUnit = UINT64_MAX;
+            else
+            {
+                LogRel(("SSM: LoadExec failed for '%s' instance #%u (version %u, pass %#x): %Rrc\n",
+                        UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Version, UnitHdr.u32Pass, rc));
+                if (!ASMAtomicXchgBool(&pSSM->u.Read.fHaveSetError, true))
+                {
+                    if (rc == VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION)
+                        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Unsupported version %u of data unit '%s' (instance #%u, pass %#x)"),
+                                        UnitHdr.u32Version, UnitHdr.szName, UnitHdr.u32Instance, UnitHdr.u32Pass);
+                    else
+                        rc = VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to load unit '%s'"), UnitHdr.szName);
+                }
+                return rc;
+            }
+        }
+        else
+        {
+            /*
+             * SSM unit wasn't found - ignore this when loading for the debugger.
+             */
+            LogRel(("SSM: Found no handler for unit '%s' instance #%u!\n", UnitHdr.szName, UnitHdr.u32Instance));
+            if (pSSM->enmAfter != SSMAFTER_DEBUG_IT)
+            {
+                pSSM->u.Read.fHaveSetError = true;
+                return VMSetError(pVM, VERR_SSM_INTEGRITY_UNIT_NOT_FOUND, RT_SRC_POS,
+                                  N_("Found no handler for unit '%s' instance #%u"), UnitHdr.szName, UnitHdr.u32Instance);
+            }
+            SSMR3SkipToEndOfUnit(pSSM);
+            ssmR3DataReadFinishV2(pSSM);
+        }
+
+        /*
+         * Check for cancellation.
+         */
+        if (RT_UNLIKELY(ASMAtomicUoReadU32(&(pSSM)->fCancelled) == SSMHANDLE_CANCELLED))
+        {
+            LogRel(("SSM: Cancelled!\n"));
+            if (RT_SUCCESS(pSSM->rc))
+                pSSM->rc = VERR_SSM_CANCELLED;
+            return pSSM->rc;
+        }
+    }
+    /* won't get here */
+}
+
+
+
+
+/**
+ * Load VM save operation.
+ *
+ * @returns VBox status.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pszFilename     The name of the saved state file. NULL if pStreamOps
+ *                          is used.
+ * @param   pStreamOps      The stream method table. NULL if pszFilename is
+ *                          used.
+ * @param   pvStreamOpsUser The user argument for the stream methods.
+ * @param   enmAfter        What is planned after a successful load operation.
+ *                          Only acceptable values are SSMAFTER_RESUME and SSMAFTER_DEBUG_IT.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvProgressUser  User argument for the progress callback.
+ *
+ * @thread  EMT
+ */
+VMMR3DECL(int) SSMR3Load(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                         SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser)
+{
+    LogFlow(("SSMR3Load: pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p\n",
+             pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(   enmAfter == SSMAFTER_RESUME
+                    || enmAfter == SSMAFTER_TELEPORT
+                    || enmAfter == SSMAFTER_DEBUG_IT,
+                    ("%d\n", enmAfter),
+                    VERR_INVALID_PARAMETER);
+    AssertReturn(!pszFilename != !pStreamOps, VERR_INVALID_PARAMETER);
+    if (pStreamOps)
+    {
+        AssertReturn(pStreamOps->u32Version == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->u32EndVersion == SSMSTRMOPS_VERSION, VERR_INVALID_MAGIC);
+        AssertReturn(pStreamOps->pfnWrite, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnRead, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSeek, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnTell, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnSize, VERR_INVALID_PARAMETER);
+        AssertReturn(pStreamOps->pfnClose, VERR_INVALID_PARAMETER);
+    }
+
+    /*
+     * Create the handle and open the file.
+     */
+    SSMHANDLE Handle;
+    int rc = ssmR3OpenFile(pVM, pszFilename, pStreamOps, pvStreamOpsUser, false /* fChecksumIt */,
+                           true /* fChecksumOnRead */, 8 /*cBuffers*/, &Handle);
+    if (RT_SUCCESS(rc))
+    {
+        ssmR3StrmStartIoThread(&Handle.Strm);
+        ssmR3SetCancellable(pVM, &Handle, true);
+
+        Handle.enmAfter         = enmAfter;
+        Handle.pfnProgress      = pfnProgress;
+        Handle.pvUser           = pvProgressUser;
+        Handle.uPercentLive     = 0;
+        Handle.uPercentPrepare  = 2;
+        Handle.uPercentDone     = 2;
+
+        if (Handle.u.Read.u16VerMajor)
+            LogRel(("SSM: File header: Format %u.%u, VirtualBox Version %u.%u.%u r%u, %u-bit host, cbGCPhys=%u, cbGCPtr=%u\n",
+                    Handle.u.Read.uFmtVerMajor, Handle.u.Read.uFmtVerMinor,
+                    Handle.u.Read.u16VerMajor, Handle.u.Read.u16VerMinor, Handle.u.Read.u32VerBuild, Handle.u.Read.u32SvnRev,
+                    Handle.u.Read.cHostBits, Handle.u.Read.cbGCPhys, Handle.u.Read.cbGCPtr));
+        else
+            LogRel(("SSM: File header: Format %u.%u, %u-bit host, cbGCPhys=%u, cbGCPtr=%u\n" ,
+                    Handle.u.Read.uFmtVerMajor, Handle.u.Read.uFmtVerMinor,
+                    Handle.u.Read.cHostBits, Handle.u.Read.cbGCPhys, Handle.u.Read.cbGCPtr));
+
+        if (pfnProgress)
+            pfnProgress(pVM, Handle.uPercent, pvProgressUser);
+
+        /*
+         * Clear the per unit flags.
+         */
+        PSSMUNIT pUnit;
+        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+            pUnit->fCalled = false;
+
+        /*
+         * Do the prepare run.
+         */
+        Handle.rc = VINF_SUCCESS;
+        Handle.enmOp = SSMSTATE_LOAD_PREP;
+        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        {
+            if (pUnit->u.Common.pfnLoadPrep)
+            {
+                Handle.u.Read.pCurUnit = pUnit;
+                pUnit->fCalled = true;
+                switch (pUnit->enmType)
+                {
+                    case SSMUNITTYPE_DEV:
+                        rc = pUnit->u.Dev.pfnLoadPrep(pUnit->u.Dev.pDevIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_DRV:
+                        rc = pUnit->u.Drv.pfnLoadPrep(pUnit->u.Drv.pDrvIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_INTERNAL:
+                        rc = pUnit->u.Internal.pfnLoadPrep(pVM, &Handle);
+                        break;
+                    case SSMUNITTYPE_EXTERNAL:
+                        rc = pUnit->u.External.pfnLoadPrep(&Handle, pUnit->u.External.pvUser);
+                        break;
+                    default:
+                        rc = VERR_INTERNAL_ERROR;
+                        break;
+                }
+                Handle.u.Read.pCurUnit = NULL;
+                if (RT_FAILURE(rc) && RT_SUCCESS_NP(Handle.rc))
+                    Handle.rc = rc;
+                else
+                    rc = Handle.rc;
+                if (RT_FAILURE(rc))
+                {
+                    LogRel(("SSM: Prepare load failed with rc=%Rrc for data unit '%s.\n", rc, pUnit->szName));
+                    break;
+                }
+            }
+        }
+
+        /* end of prepare % */
+        if (pfnProgress)
+            pfnProgress(pVM, Handle.uPercentPrepare - 1, pvProgressUser);
+        Handle.uPercent      = Handle.uPercentPrepare;
+        Handle.cbEstTotal    = Handle.u.Read.cbLoadFile;
+        Handle.offEstUnitEnd = Handle.u.Read.cbLoadFile;
+
+        /*
+         * Do the execute run.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            if (Handle.u.Read.uFmtVerMajor >= 2)
+                rc = ssmR3LoadExecV2(pVM, &Handle);
+            else
+                rc = ssmR3LoadExecV1(pVM, &Handle);
+            Handle.u.Read.pCurUnit       = NULL;
+            Handle.u.Read.uCurUnitVer    = UINT32_MAX;
+            Handle.u.Read.uCurUnitPass   = 0;
+
+            /* (progress should be pending 99% now) */
+            AssertMsg(   Handle.fLiveSave
+                      || RT_FAILURE(rc)
+                      || Handle.uPercent == 101 - Handle.uPercentDone, ("%d\n", Handle.uPercent));
+        }
+
+        /*
+         * Do the done run.
+         */
+        Handle.rc = rc;
+        Handle.enmOp = SSMSTATE_LOAD_DONE;
+        for (pUnit = pVM->ssm.s.pHead; pUnit; pUnit = pUnit->pNext)
+        {
+            if (    pUnit->u.Common.pfnLoadDone
+                && (   pUnit->fCalled
+                    || (!pUnit->u.Common.pfnLoadPrep && !pUnit->u.Common.pfnLoadExec)))
+            {
+                Handle.u.Read.pCurUnit = pUnit;
+                int const rcOld = Handle.rc;
+                rc = VINF_SUCCESS;
+                switch (pUnit->enmType)
+                {
+                    case SSMUNITTYPE_DEV:
+                        rc = pUnit->u.Dev.pfnLoadDone(pUnit->u.Dev.pDevIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_DRV:
+                        rc = pUnit->u.Drv.pfnLoadDone(pUnit->u.Drv.pDrvIns, &Handle);
+                        break;
+                    case SSMUNITTYPE_INTERNAL:
+                        rc = pUnit->u.Internal.pfnLoadDone(pVM, &Handle);
+                        break;
+                    case SSMUNITTYPE_EXTERNAL:
+                        rc = pUnit->u.External.pfnLoadDone(&Handle, pUnit->u.External.pvUser);
+                        break;
+                    default:
+                        rc = VERR_INTERNAL_ERROR;
+                        break;
+                }
+                Handle.u.Read.pCurUnit = NULL;
+                if (RT_SUCCESS(rc) && Handle.rc != rcOld)
+                    rc = Handle.rc;
+                if (RT_FAILURE(rc))
+                {
+                    LogRel(("SSM: LoadDone failed with rc=%Rrc for data unit '%s' instance #%u.\n",
+                            rc, pUnit->szName, pUnit->u32Instance));
+                    if (!ASMAtomicXchgBool(&Handle.u.Read.fHaveSetError, true))
+                        VMSetError(pVM, rc, RT_SRC_POS, N_("LoadDone failed with rc=%Rrc for data unit '%s' instance #%u."),
+                                   rc, pUnit->szName, pUnit->u32Instance);
+                    if (RT_SUCCESS_NP(Handle.rc))
+                        Handle.rc = rc;
+                }
+            }
+        }
+
+        /* progress */
+        if (pfnProgress)
+            pfnProgress(pVM, 99, pvProgressUser);
+
+        ssmR3SetCancellable(pVM, &Handle, false);
+        ssmR3StrmClose(&Handle.Strm, Handle.rc == VERR_SSM_CANCELLED);
+        rc = Handle.rc;
+    }
+
+    /*
+     * Done
+     */
+    if (RT_SUCCESS(rc))
+    {
+        /* progress */
+        if (pfnProgress)
+            pfnProgress(pVM, 100, pvProgressUser);
+        Log(("SSM: Load of '%s' completed!\n", pszFilename));
+    }
+    return rc;
+}
+
+
+/**
+ * VMSetError wrapper for load errors that inserts the saved state details.
+ *
+ * @returns rc.
+ * @param   pSSM                The saved state handle.
+ * @param   rc                  The status code of the error. Use RT_SRC_POS.
+ * @param   RT_SRC_POS_DECL     The source location.
+ * @param   pszFormat           The message format string.
+ * @param   ...                 Variable argument list.
+ */
+VMMR3DECL(int) SSMR3SetLoadError(PSSMHANDLE pSSM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    rc = SSMR3SetLoadErrorV(pSSM, rc, RT_SRC_POS_ARGS, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * VMSetError wrapper for load errors that inserts the saved state details.
+ *
+ * @returns rc.
+ * @param   pSSM                The saved state handle.
+ * @param   rc                  The status code of the error.
+ * @param   RT_SRC_POS_DECL     The error location, use RT_SRC_POS.
+ * @param   pszFormat           The message format string.
+ * @param   va                  Variable argument list.
+ */
+VMMR3DECL(int) SSMR3SetLoadErrorV(PSSMHANDLE pSSM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list va)
+{
+    /*
+     * Input validations.
+     */
+    SSM_ASSERT_READABLE_RET(pSSM);
+    AssertPtr(pszFormat);
+    Assert(RT_FAILURE_NP(rc));
+
+    /*
+     * Format the incoming error.
+     */
+    char *pszMsg;
+    RTStrAPrintfV(&pszMsg, pszFormat, va);
+    if (!pszMsg)
+    {
+        VMSetError(pSSM->pVM, VERR_NO_MEMORY, RT_SRC_POS,
+                   N_("SSMR3SetLoadErrorV ran out of memory formatting: %s\n"), pszFormat);
+        return rc;
+    }
+
+    /*
+     * Forward to VMSetError with the additional info.
+     */
+    PSSMUNIT    pUnit       = pSSM->u.Read.pCurUnit;
+    const char *pszName     = pUnit ? pUnit->szName      : "unknown";
+    uint32_t    uInstance   = pUnit ? pUnit->u32Instance : 0;
+    if (   pSSM->enmOp == SSMSTATE_LOAD_EXEC
+        && pSSM->u.Read.uCurUnitPass == SSM_PASS_FINAL)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [ver=%u pass=final]"),
+                        pszName, uInstance, pszMsg, pSSM->u.Read.uCurUnitVer);
+    else if (pSSM->enmOp == SSMSTATE_LOAD_EXEC)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [ver=%u pass=#%u]"),
+                        pszName, uInstance, pszMsg, pSSM->u.Read.uCurUnitVer, pSSM->u.Read.uCurUnitPass);
+    else if (pSSM->enmOp == SSMSTATE_LOAD_PREP)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [prep]"),
+                        pszName, uInstance, pszMsg);
+    else if (pSSM->enmOp == SSMSTATE_LOAD_DONE)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [done]"),
+                        pszName, uInstance, pszMsg);
+    else if (pSSM->enmOp == SSMSTATE_OPEN_READ)
+        rc = VMSetError(pSSM->pVM, rc, RT_SRC_POS_ARGS, N_("%s#%u: %s [read]"),
+                        pszName, uInstance, pszMsg);
+    else
+        AssertFailed();
+    pSSM->u.Read.fHaveSetError = true;
+    RTStrFree(pszMsg);
+    return rc;
+}
+
+
+/**
+ * SSMR3SetLoadError wrapper that returns VERR_SSM_LOAD_CONFIG_MISMATCH.
+ *
+ * @returns VERR_SSM_LOAD_CONFIG_MISMATCH.
+ * @param   pSSM                The saved state handle.
+ * @param   RT_SRC_POS_DECL     The error location, use RT_SRC_POS.
+ * @param   pszFormat           The message format string.
+ * @param   va                  Variable argument list.
+ */
+VMMR3DECL(int) SSMR3SetCfgError(PSSMHANDLE pSSM, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = SSMR3SetLoadErrorV(pSSM, VERR_SSM_LOAD_CONFIG_MISMATCH, RT_SRC_POS_ARGS, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+#endif /* !SSM_STANDALONE */
+
+/**
+ * Validates a file as a validate SSM saved state.
+ *
+ * This will only verify the file format, the format and content of individual
+ * data units are not inspected.
+ *
+ * @returns VINF_SUCCESS if valid.
+ * @returns VBox status code on other failures.
+ *
+ * @param   pszFilename     The path to the file to validate.
+ * @param   fChecksumIt     Whether to checksum the file or not.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) SSMR3ValidateFile(const char *pszFilename, bool fChecksumIt)
+{
+    LogFlow(("SSMR3ValidateFile: pszFilename=%p:{%s} fChecksumIt=%RTbool\n", pszFilename, pszFilename, fChecksumIt));
+
+    /*
+     * Try open the file and validate it.
+     */
+    SSMHANDLE Handle;
+    int rc = ssmR3OpenFile(NULL, pszFilename, NULL /*pStreamOps*/, NULL /*pvUser*/, fChecksumIt,
+                           false /*fChecksumOnRead*/, 1 /*cBuffers*/, &Handle);
+    if (RT_SUCCESS(rc))
+        ssmR3StrmClose(&Handle.Strm, false /*fCancelled*/);
+    else
+        Log(("SSM: Failed to open saved state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+    return rc;
+}
+
+
+/**
+ * Opens a saved state file for reading.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pszFilename     The path to the saved state file.
+ * @param   fFlags          Open flags. Reserved, must be 0.
+ * @param   ppSSM           Where to store the SSM handle.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) SSMR3Open(const char *pszFilename, unsigned fFlags, PSSMHANDLE *ppSSM)
+{
+    LogFlow(("SSMR3Open: pszFilename=%p:{%s} fFlags=%#x ppSSM=%p\n", pszFilename, pszFilename, fFlags, ppSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(VALID_PTR(pszFilename), ("%p\n", pszFilename), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(!fFlags, ("%#x\n", fFlags), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(VALID_PTR(ppSSM), ("%p\n", ppSSM), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate a handle.
+     */
+    PSSMHANDLE pSSM = (PSSMHANDLE)RTMemAllocZ(sizeof(*pSSM));
+    AssertReturn(pSSM, VERR_NO_MEMORY);
+
+    /*
+     * Try open the file and validate it.
+     */
+    int rc = ssmR3OpenFile(NULL, pszFilename, NULL /*pStreamOps*/, NULL /*pvUser*/, false /*fChecksumIt*/,
+                           true /*fChecksumOnRead*/, 1 /*cBuffers*/, pSSM);
+    if (RT_SUCCESS(rc))
+    {
+        pSSM->enmAfter = SSMAFTER_OPENED;
+        pSSM->enmOp    = SSMSTATE_OPEN_READ;
+        *ppSSM = pSSM;
+        LogFlow(("SSMR3Open: returns VINF_SUCCESS *ppSSM=%p\n", *ppSSM));
+        return VINF_SUCCESS;
+    }
+
+    Log(("SSMR3Open: Failed to open saved state file '%s', rc=%Rrc.\n",  pszFilename, rc));
+    RTMemFree(pSSM);
+    return rc;
+
+}
+
+
+/**
+ * Closes a saved state file opened by SSMR3Open().
+ *
+ * @returns VBox status code.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3Open().
+ *
+ * @thread  Any, but the caller is responsible for serializing calls per handle.
+ */
+VMMR3DECL(int) SSMR3Close(PSSMHANDLE pSSM)
+{
+    LogFlow(("SSMR3Close: pSSM=%p\n", pSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(VALID_PTR(pSSM), ("%p\n", pSSM), VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmAfter == SSMAFTER_OPENED, ("%d\n", pSSM->enmAfter),VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_OPEN_READ, ("%d\n", pSSM->enmOp), VERR_INVALID_PARAMETER);
+    Assert(pSSM->fCancelled == SSMHANDLE_OK);
+
+    /*
+     * Close the stream and free the handle.
+     */
+    int rc = ssmR3StrmClose(&pSSM->Strm, pSSM->rc == VERR_SSM_CANCELLED);
+    if (pSSM->u.Read.pZipDecompV1)
+    {
+        RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
+        pSSM->u.Read.pZipDecompV1 = NULL;
+    }
+    RTMemFree(pSSM);
+    return rc;
+}
+
+
+/**
+ * Worker for SSMR3Seek that seeks version 1 saved state files.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   pszUnit             The unit to seek to.
+ * @param   iInstance           The particular instance we seek.
+ * @param   piVersion           Where to store the unit version number.
+ */
+static int ssmR3FileSeekV1(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    /*
+     * Walk the data units until we find EOF or a match.
+     */
+    size_t              cbUnitNm = strlen(pszUnit) + 1;
+    AssertLogRelReturn(cbUnitNm <= SSM_MAX_NAME_SIZE, VERR_SSM_UNIT_NOT_FOUND);
+    char                szName[SSM_MAX_NAME_SIZE];
+    SSMFILEUNITHDRV1    UnitHdr;
+    for (RTFOFF off = pSSM->u.Read.cbFileHdr; ; off += UnitHdr.cbUnit)
+    {
+        /*
+         * Read the unit header and verify it.
+         */
+        int rc = ssmR3StrmPeekAt(&pSSM->Strm, off, &UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV1, szName), NULL);
+        AssertRCReturn(rc, rc);
+        if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_MAGIC, sizeof(SSMFILEUNITHDR_MAGIC)))
+        {
+            /*
+             * Does what we've got match, if so read the name.
+             */
+            if (    UnitHdr.u32Instance == iInstance
+                &&  UnitHdr.cchName     == cbUnitNm)
+            {
+                rc = ssmR3StrmPeekAt(&pSSM->Strm, off + RT_OFFSETOF(SSMFILEUNITHDRV1, szName), szName, cbUnitNm, NULL);
+                AssertRCReturn(rc, rc);
+                AssertLogRelMsgReturn(!szName[UnitHdr.cchName - 1],
+                                      (" Unit name '%.*s' was not properly terminated.\n", cbUnitNm, szName),
+                                      VERR_SSM_INTEGRITY_UNIT);
+
+                /*
+                 * Does the name match?
+                 */
+                if (!memcmp(szName, pszUnit, cbUnitNm))
+                {
+                    rc = ssmR3StrmSeek(&pSSM->Strm, off + RT_OFFSETOF(SSMFILEUNITHDRV1, szName) + cbUnitNm, RTFILE_SEEK_BEGIN, 0);
+                    pSSM->cbUnitLeftV1 = UnitHdr.cbUnit - RT_OFFSETOF(SSMFILEUNITHDRV1, szName[cbUnitNm]);
+                    pSSM->offUnit = 0;
+                    if (piVersion)
+                        *piVersion = UnitHdr.u32Version;
+                    return VINF_SUCCESS;
+                }
+            }
+        }
+        else if (!memcmp(&UnitHdr.achMagic[0], SSMFILEUNITHDR_END, sizeof(SSMFILEUNITHDR_END)))
+            return VERR_SSM_UNIT_NOT_FOUND;
+        else
+            AssertLogRelMsgFailedReturn(("Invalid unit magic at offset %RTfoff, '%.*s'!\n",
+                                         off, sizeof(UnitHdr.achMagic) - 1, &UnitHdr.achMagic[0]),
+                                        VERR_SSM_INTEGRITY_UNIT_MAGIC);
+    }
+    /* won't get here. */
+}
+
+
+/**
+ * Worker for ssmR3FileSeekV2 for simplifying memory cleanup.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   pDir                The directory buffer.
+ * @param   cbDir               The size of the directory.
+ * @param   cDirEntries         The number of directory entries.
+ * @param   offDir              The directory offset in the file.
+ * @param   pszUnit             The unit to seek to.
+ * @param   iInstance           The particular instance we seek.
+ * @param   piVersion           Where to store the unit version number.
+ */
+static int ssmR3FileSeekSubV2(PSSMHANDLE pSSM, PSSMFILEDIR pDir, size_t cbDir, uint32_t cDirEntries, uint64_t offDir,
+                              const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    /*
+     * Read it.
+     */
+    int rc = ssmR3StrmPeekAt(&pSSM->Strm, offDir, pDir, cbDir, NULL);
+    AssertLogRelRCReturn(rc, rc);
+    rc = ssmR3ValidateDirectory(pDir, (uint32_t)cbDir, offDir, cDirEntries, pSSM->u.Read.cbFileHdr, pSSM->u.Read.u32SvnRev);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Search the directory.
+     */
+    size_t          cbUnitNm   = strlen(pszUnit) + 1;
+    uint32_t const  u32NameCRC = RTCrc32(pszUnit, cbUnitNm - 1);
+    for (uint32_t i = 0; i < cDirEntries; i++)
+    {
+        if (    pDir->aEntries[i].u32NameCRC  == u32NameCRC
+            &&  pDir->aEntries[i].u32Instance == iInstance
+            &&  pDir->aEntries[i].off != 0 /* bug in unreleased code */
+           )
+        {
+            /*
+             * Read and validate the unit header.
+             */
+            SSMFILEUNITHDRV2    UnitHdr;
+            size_t              cbToRead = sizeof(UnitHdr);
+            if (pDir->aEntries[i].off + cbToRead > offDir)
+            {
+                cbToRead = offDir - pDir->aEntries[i].off;
+                RT_ZERO(UnitHdr);
+            }
+            rc = ssmR3StrmPeekAt(&pSSM->Strm, pDir->aEntries[i].off, &UnitHdr, cbToRead, NULL);
+            AssertLogRelRCReturn(rc, rc);
+
+            AssertLogRelMsgReturn(!memcmp(UnitHdr.szMagic, SSMFILEUNITHDR_MAGIC, sizeof(UnitHdr.szMagic)),
+                                  ("Bad unit header or dictionary offset: i=%u off=%lld\n", i, pDir->aEntries[i].off),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            AssertLogRelMsgReturn(UnitHdr.offStream == pDir->aEntries[i].off,
+                                  ("Bad unit header: i=%d off=%lld offStream=%lld\n", i, pDir->aEntries[i].off, UnitHdr.offStream),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            AssertLogRelMsgReturn(UnitHdr.u32Instance == pDir->aEntries[i].u32Instance,
+                                  ("Bad unit header: i=%d off=%lld u32Instance=%u Dir.u32Instance=%u\n",
+                                   i, pDir->aEntries[i].off, UnitHdr.u32Instance, pDir->aEntries[i].u32Instance),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            uint32_t cbUnitHdr = RT_UOFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]);
+            AssertLogRelMsgReturn(   UnitHdr.cbName > 0
+                                  && UnitHdr.cbName < sizeof(UnitHdr)
+                                  && cbUnitHdr <= cbToRead,
+                                  ("Bad unit header: i=%u off=%lld cbName=%#x cbToRead=%#x\n", i, pDir->aEntries[i].off, UnitHdr.cbName, cbToRead),
+                                  VERR_SSM_INTEGRITY_UNIT);
+            SSM_CHECK_CRC32_RET(&UnitHdr, RT_OFFSETOF(SSMFILEUNITHDRV2, szName[UnitHdr.cbName]),
+                                ("Bad unit header CRC: i=%u off=%lld u32CRC=%#x u32ActualCRC=%#x\n",
+                                 i, pDir->aEntries[i].off, u32CRC, u32ActualCRC));
+
+            /*
+             * Ok, it is valid, get on with the comparing now.
+             */
+            if (    UnitHdr.cbName == cbUnitNm
+                &&  !memcmp(UnitHdr.szName, pszUnit, cbUnitNm))
+            {
+                if (piVersion)
+                    *piVersion = UnitHdr.u32Version;
+                rc = ssmR3StrmSeek(&pSSM->Strm, pDir->aEntries[i].off + cbUnitHdr, RTFILE_SEEK_BEGIN,
+                                   RTCrc32Process(UnitHdr.u32CurStreamCRC, &UnitHdr, cbUnitHdr));
+                AssertLogRelRCReturn(rc, rc);
+                ssmR3DataReadBeginV2(pSSM);
+                return VINF_SUCCESS;
+            }
+        }
+    }
+
+    return VERR_SSM_UNIT_NOT_FOUND;
+}
+
+
+/**
+ * Worker for SSMR3Seek that seeks version 2 saved state files.
+ *
+ * @returns VBox status code.
+ * @param   pSSM                The SSM handle.
+ * @param   pszUnit             The unit to seek to.
+ * @param   iInstance           The particular instance we seek.
+ * @param   piVersion           Where to store the unit version number.
+ */
+static int ssmR3FileSeekV2(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    /*
+     * Read the footer, allocate a temporary buffer for the dictionary and
+     * pass it down to a worker to simplify cleanup.
+     */
+    uint64_t        offFooter;
+    SSMFILEFTR      Footer;
+    int rc = ssmR3StrmPeekAt(&pSSM->Strm, -(RTFOFF)sizeof(Footer), &Footer, sizeof(Footer), &offFooter);
+    AssertLogRelRCReturn(rc, rc);
+    AssertLogRelReturn(!memcmp(Footer.szMagic, SSMFILEFTR_MAGIC, sizeof(Footer.szMagic)), VERR_SSM_INTEGRITY);
+    SSM_CHECK_CRC32_RET(&Footer, sizeof(Footer), ("Bad footer CRC: %08x, actual %08x\n", u32CRC, u32ActualCRC));
+
+    size_t const    cbDir = RT_OFFSETOF(SSMFILEDIR, aEntries[Footer.cDirEntries]);
+    PSSMFILEDIR     pDir  = (PSSMFILEDIR)RTMemTmpAlloc(cbDir);
+    if (RT_UNLIKELY(!pDir))
+        return VERR_NO_TMP_MEMORY;
+    rc = ssmR3FileSeekSubV2(pSSM, pDir, cbDir, Footer.cDirEntries, offFooter - cbDir,
+                            pszUnit, iInstance, piVersion);
+    RTMemTmpFree(pDir);
+
+    return rc;
+}
+
+
+/**
+ * Seeks to a specific data unit.
+ *
+ * After seeking it's possible to use the getters to on
+ * that data unit.
+ *
+ * @returns VBox status code.
+ * @returns VERR_SSM_UNIT_NOT_FOUND if the unit+instance wasn't found.
+ *
+ * @param   pSSM            The SSM handle returned by SSMR3Open().
+ * @param   pszUnit         The name of the data unit.
+ * @param   iInstance       The instance number.
+ * @param   piVersion       Where to store the version number. (Optional)
+ *
+ * @thread  Any, but the caller is responsible for serializing calls per handle.
+ */
+VMMR3DECL(int) SSMR3Seek(PSSMHANDLE pSSM, const char *pszUnit, uint32_t iInstance, uint32_t *piVersion)
+{
+    LogFlow(("SSMR3Seek: pSSM=%p pszUnit=%p:{%s} iInstance=%RU32 piVersion=%p\n",
+             pSSM, pszUnit, pszUnit, iInstance, piVersion));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pSSM, VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmAfter == SSMAFTER_OPENED, ("%d\n", pSSM->enmAfter),VERR_INVALID_PARAMETER);
+    AssertMsgReturn(pSSM->enmOp == SSMSTATE_OPEN_READ, ("%d\n", pSSM->enmOp), VERR_INVALID_PARAMETER);
+    AssertPtrReturn(pszUnit, VERR_INVALID_POINTER);
+    AssertMsgReturn(!piVersion || VALID_PTR(piVersion), ("%p\n", piVersion), VERR_INVALID_POINTER);
+
+    /*
+     * Reset the state.
+     */
+    if (pSSM->u.Read.pZipDecompV1)
+    {
+        RTZipDecompDestroy(pSSM->u.Read.pZipDecompV1);
+        pSSM->u.Read.pZipDecompV1 = NULL;
+    }
+    pSSM->cbUnitLeftV1  = 0;
+    pSSM->offUnit       = UINT64_MAX;
+
+    /*
+     * Call the version specific workers.
+     */
+    if (pSSM->u.Read.uFmtVerMajor >= 2)
+        pSSM->rc = ssmR3FileSeekV2(pSSM, pszUnit, iInstance, piVersion);
+    else
+        pSSM->rc = ssmR3FileSeekV1(pSSM, pszUnit, iInstance, piVersion);
+    return pSSM->rc;
+}
+
+
+
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+/* ... Misc APIs ... */
+
+
+
+/**
+ * Query what the VBox status code of the operation is.
+ *
+ * This can be used for putting and getting a batch of values
+ * without bother checking the result till all the calls have
+ * been made.
+ *
+ * @returns SSMAFTER enum value.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(int) SSMR3HandleGetStatus(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return pSSM->rc;
+}
+
+
+/**
+ * Fail the load operation.
+ *
+ * This is mainly intended for sub item loaders (like timers) which
+ * return code isn't necessarily heeded by the caller but is important
+ * to SSM.
+ *
+ * @returns VBox status code of the handle, or VERR_INVALID_PARAMETER.
+ * @param   pSSM            The saved state handle.
+ * @param   iStatus         Failure status code. This MUST be a VERR_*.
+ */
+VMMR3DECL(int) SSMR3HandleSetStatus(PSSMHANDLE pSSM, int iStatus)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    Assert(pSSM->enmOp != SSMSTATE_LIVE_VOTE);
+    if (RT_FAILURE(iStatus))
+    {
+        int rc = pSSM->rc;
+        if (RT_SUCCESS(rc))
+            pSSM->rc = rc = iStatus;
+        return rc;
+    }
+    AssertMsgFailed(("iStatus=%d %Rrc\n", iStatus, iStatus));
+    return VERR_INVALID_PARAMETER;
+}
+
+
+/**
+ * Get what to do after this operation.
+ *
+ * @returns SSMAFTER enum value.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(SSMAFTER) SSMR3HandleGetAfter(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return pSSM->enmAfter;
+}
+
+
+/**
+ * Checks if it is a live save operation or not.
+ *
+ * @returns True if it is, false if it isn't.
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(bool) SSMR3HandleIsLiveSave(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return pSSM->fLiveSave;
+}
+
+
+/**
+ * Gets the maximum downtime for a live operation.
+ *
+ * @returns The max downtime in milliseconds.  Can be anything from 0 thru
+ *          UINT32_MAX.
+ *
+ * @param   pSSM            The saved state handle.
+ */
+VMMR3DECL(uint32_t) SSMR3HandleMaxDowntime(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    if (pSSM->enmOp <= SSMSTATE_SAVE_DONE)
+        return pSSM->u.Write.cMsMaxDowntime;
+    return UINT32_MAX;
+}
+
+
+/**
+ * Gets the host bit count of a saved state.
+ *
+ * @returns 32 or 64. If pSSM is invalid, 0 is returned.
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.
+ */
+VMMR3DECL(uint32_t) SSMR3HandleHostBits(PSSMHANDLE pSSM)
+{
+    SSM_ASSERT_VALID_HANDLE(pSSM);
+    return ssmR3GetHostBits(pSSM);
+}
+
+
+/**
+ * Get the VirtualBox SVN revision that created the saved state.
+ *
+ * @returns The revision number on success.
+ *          form.  If we don't know, it's 0.
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.  Be VERY careful with this
+ *          function since it will return different values for OSE builds!
+ */
+VMMR3DECL(uint32_t)     SSMR3HandleRevision(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+        return pSSM->u.Read.u32SvnRev;
+#ifdef SSM_STANDALONE
+    return 0;
+#else
+    return VMMGetSvnRev();
+#endif
+}
+
+
+/**
+ * Gets the VirtualBox version that created the saved state.
+ *
+ * @returns VBOX_FULL_VERSION style version number.
+ *          Returns UINT32_MAX if unknown or somehow out of range.
+ *
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.
+ */
+VMMR3DECL(uint32_t)     SSMR3HandleVersion(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+    {
+        if (    !pSSM->u.Read.u16VerMajor
+            &&  !pSSM->u.Read.u16VerMinor
+            &&  !pSSM->u.Read.u32VerBuild)
+            return UINT32_MAX;
+        AssertReturn(pSSM->u.Read.u16VerMajor <=   0xff, UINT32_MAX);
+        AssertReturn(pSSM->u.Read.u16VerMinor <=   0xff, UINT32_MAX);
+        AssertReturn(pSSM->u.Read.u32VerBuild <= 0xffff, UINT32_MAX);
+        return VBOX_FULL_VERSION_MAKE(pSSM->u.Read.u16VerMajor, pSSM->u.Read.u16VerMinor, pSSM->u.Read.u32VerBuild);
+    }
+    return VBOX_FULL_VERSION;
+}
+
+
+/**
+ * Get the host OS and architecture where the saved state was created.
+ *
+ * @returns Pointer to a read only string.  When known, this is on the os.arch
+ *          form.  If we don't know, it's an empty string.
+ * @param   pSSM            The saved state handle.
+ *
+ * @remarks This method should ONLY be used for hacks when loading OLDER saved
+ *          state that have data layout or semantic changes without the
+ *          compulsory version number change.
+ */
+VMMR3DECL(const char *) SSMR3HandleHostOSAndArch(PSSMHANDLE pSSM)
+{
+    if (pSSM->enmOp >= SSMSTATE_LOAD_PREP)
+        return pSSM->u.Read.szHostOSAndArch;
+    return KBUILD_TARGET "." KBUILD_TARGET_ARCH;
+}
+
+
+#ifndef SSM_STANDALONE
+/**
+ * Asynchronously cancels the current SSM operation ASAP.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SSM_NO_PENDING_OPERATION if nothing around that can be
+ *          cancelled.
+ * @retval  VERR_SSM_ALREADY_CANCELLED if the operation as already been
+ *          cancelled.
+ *
+ * @param   pVM                 The VM handle.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) SSMR3Cancel(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    int rc = RTCritSectEnter(&pVM->ssm.s.CancelCritSect);
+    AssertRCReturn(rc, rc);
+
+    PSSMHANDLE pSSM = pVM->ssm.s.pSSM;
+    if (pSSM)
+    {
+        uint32_t u32Old;
+        if (ASMAtomicCmpXchgExU32(&pSSM->fCancelled, SSMHANDLE_CANCELLED, SSMHANDLE_OK, &u32Old))
+        {
+            LogRel(("SSM: Cancelled pending operation\n"));
+            rc = VINF_SUCCESS;
+        }
+        else if (u32Old == SSMHANDLE_CANCELLED)
+            rc = VERR_SSM_ALREADY_CANCELLED;
+        else
+        {
+            AssertLogRelMsgFailed(("fCancelled=%RX32 enmOp=%d\n", u32Old, pSSM->enmOp));
+            rc = VERR_INTERNAL_ERROR_2;
+        }
+    }
+    else
+        rc = VERR_SSM_NO_PENDING_OPERATION;
+
+    RTCritSectLeave(&pVM->ssm.s.CancelCritSect);
+    return rc;
+}
+#endif /* !SSM_STANDALONE */
+
Index: /trunk/src/VBox/VMM/VMMR3/STAM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/STAM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/STAM.cpp	(revision 35333)
@@ -0,0 +1,1993 @@
+/* $Id$ */
+/** @file
+ * STAM - The Statistics Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_stam       STAM - The Statistics Manager
+ *
+ * The purpose for the statistics manager is to present the rest of the system
+ * with a somewhat uniform way of accessing VMM statistics.  STAM sports a
+ * couple of different APIs for accessing them: STAMR3EnumU, STAMR3SnapshotU,
+ * STAMR3DumpU, STAMR3DumpToReleaseLogU and the debugger.  Main is exposing the
+ * XML based one, STAMR3SnapshotU.
+ *
+ * The rest of the VMM together with the devices and drivers registers their
+ * statistics with STAM giving them a name.  The name is hierarchical, the
+ * components separated by slashes ('/') and must start with a slash.
+ *
+ * Each item registered with STAM - also, half incorrectly, called a sample -
+ * has a type, unit, visibility, data pointer and description associated with it
+ * in addition to the name (described above).  The type tells STAM what kind of
+ * structure the pointer is pointing to.  The visibility allows unused
+ * statistics from cluttering the output or showing up in the GUI.  All the bits
+ * together makes STAM able to present the items in a sensible way to the user.
+ * Some types also allows STAM to reset the data, which is very convenient when
+ * digging into specific operations and such.
+ *
+ * PS. The VirtualBox Debugger GUI has a viewer for inspecting the statistics
+ * STAM provides.  You will also find statistics in the release and debug logs.
+ * And as mentioned in the introduction, the debugger console features a couple
+ * of command: .stats and .statsreset.
+ *
+ * @see grp_stam
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_STAM
+#include <VBox/stam.h>
+#include "STAMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+#include <VBox/err.h>
+#include <VBox/dbg.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/alloc.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Argument structure for stamR3PrintOne().
+ */
+typedef struct STAMR3PRINTONEARGS
+{
+    PVM pVM;
+    void *pvArg;
+    DECLCALLBACKMEMBER(void, pfnPrintf)(struct STAMR3PRINTONEARGS *pvArg, const char *pszFormat, ...);
+} STAMR3PRINTONEARGS, *PSTAMR3PRINTONEARGS;
+
+
+/**
+ * Argument structure to stamR3EnumOne().
+ */
+typedef struct STAMR3ENUMONEARGS
+{
+    PVM             pVM;
+    PFNSTAMR3ENUM   pfnEnum;
+    void           *pvUser;
+} STAMR3ENUMONEARGS, *PSTAMR3ENUMONEARGS;
+
+
+/**
+ * The snapshot status structure.
+ * Argument package passed to stamR3SnapshotOne, stamR3SnapshotPrintf and stamR3SnapshotOutput.
+ */
+typedef struct STAMR3SNAPSHOTONE
+{
+    /** Pointer to the buffer start. */
+    char           *pszStart;
+    /** Pointer to the buffer end. */
+    char           *pszEnd;
+    /** Pointer to the current buffer position. */
+    char           *psz;
+    /** The VM handle. */
+    PVM             pVM;
+    /** The number of bytes allocated. */
+    size_t          cbAllocated;
+    /** The status code. */
+    int             rc;
+    /** Whether to include the description strings. */
+    bool            fWithDesc;
+} STAMR3SNAPSHOTONE, *PSTAMR3SNAPSHOTONE;
+
+
+/**
+ * Init record for a ring-0 statistic sample.
+ */
+typedef struct STAMR0SAMPLE
+{
+    /** The GVMMSTATS structure offset of the variable. */
+    unsigned        offVar;
+    /** The type. */
+    STAMTYPE        enmType;
+    /** The unit. */
+    STAMUNIT        enmUnit;
+    /** The name. */
+    const char     *pszName;
+    /** The description. */
+    const char     *pszDesc;
+} STAMR0SAMPLE;
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int                  stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                                            STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc);
+static int                  stamR3ResetOne(PSTAMDESC pDesc, void *pvArg);
+static DECLCALLBACK(void)   stamR3EnumLogPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
+static DECLCALLBACK(void)   stamR3EnumRelLogPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
+static DECLCALLBACK(void)   stamR3EnumPrintf(PSTAMR3PRINTONEARGS pvArg, const char *pszFormat, ...);
+static int                  stamR3SnapshotOne(PSTAMDESC pDesc, void *pvArg);
+static int                  stamR3SnapshotPrintf(PSTAMR3SNAPSHOTONE pThis, const char *pszFormat, ...);
+static int                  stamR3PrintOne(PSTAMDESC pDesc, void *pvArg);
+static int                  stamR3EnumOne(PSTAMDESC pDesc, void *pvArg);
+static bool                 stamR3MultiMatch(const char * const *papszExpressions, unsigned cExpressions, unsigned *piExpression, const char *pszName);
+static char **              stamR3SplitPattern(const char *pszPat, unsigned *pcExpressions, char **ppszCopy);
+static int                  stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0, int (pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg);
+static void                 stamR3Ring0StatsRegisterU(PUVM pUVM);
+static void                 stamR3Ring0StatsUpdateU(PUVM pUVM, const char *pszPat);
+static void                 stamR3Ring0StatsUpdateMultiU(PUVM pUVM, const char * const *papszExpressions, unsigned cExpressions);
+
+#ifdef VBOX_WITH_DEBUGGER
+static DECLCALLBACK(int)    stamR3CmdStats(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+static DECLCALLBACK(void)   stamR3EnumDbgfPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...);
+static DECLCALLBACK(int)    stamR3CmdStatsReset(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+#endif
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#ifdef VBOX_WITH_DEBUGGER
+/** Pattern argument. */
+static const DBGCVARDESC    g_aArgPat[] =
+{
+    /* cTimesMin,   cTimesMax,  enmCategory,            fFlags,                         pszName,        pszDescription */
+    {  0,           1,          DBGCVAR_CAT_STRING,     0,                              "pattern",      "Which samples the command shall be applied to. Use '*' as wildcard. Use ';' to separate expression." }
+};
+
+/** Command descriptors. */
+static const DBGCCMD    g_aCmds[] =
+{
+    /* pszCmd,      cArgsMin, cArgsMax, paArgDesc,          cArgDescs,                  pResultDesc,        fFlags,     pfnHandler          pszSyntax,          ....pszDescription */
+    { "stats",      0,        1,        &g_aArgPat[0],      RT_ELEMENTS(g_aArgPat),        NULL,               0,          stamR3CmdStats,     "[pattern]",        "Display statistics." },
+    { "statsreset", 0,        1,        &g_aArgPat[0],      RT_ELEMENTS(g_aArgPat),        NULL,               0,          stamR3CmdStatsReset,"[pattern]",        "Resets statistics." }
+};
+#endif
+
+
+/**
+ * The GVMM mapping records - sans the host cpus.
+ */
+static const STAMR0SAMPLE g_aGVMMStats[] =
+{
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltCalls", "The number of calls to GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltBlocking),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltBlocking", "The number of times we did go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltTimeouts),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltTimeouts", "The number of times we timed out in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltNotBlocking),  STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltNotBlocking", "The number of times we didn't go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cHaltWakeUps),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/HaltWakeUps", "The number of wake ups done during GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpCalls),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpCalls", "The number of calls to GVMMR0WakeUp." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpNotHalted),  STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpNotHalted", "The number of times the EMT thread wasn't actually halted when GVMMR0WakeUp was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cWakeUpWakeUps),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/WakeUpWakeUps", "The number of wake ups done during GVMMR0WakeUp (not counting the explicit one)." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPokeCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PokeCalls", "The number of calls to GVMMR0Poke." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPokeNotBusy),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PokeNotBusy", "The number of times the EMT thread wasn't actually busy when GVMMR0Poke was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollCalls),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollCalls", "The number of calls to GVMMR0SchedPoll." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollHalts),        STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollHalts", "The number of times the EMT has halted in a GVMMR0SchedPoll call." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedVM.cPollWakeUps),      STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/VM/PollWakeUps", "The number of wake ups done during GVMMR0SchedPoll." },
+
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltCalls", "The number of calls to GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltBlocking),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltBlocking", "The number of times we did go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltTimeouts),    STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltTimeouts", "The number of times we timed out in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltNotBlocking), STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltNotBlocking", "The number of times we didn't go to sleep in GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cHaltWakeUps),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/HaltWakeUps", "The number of wake ups done during GVMMR0SchedHalt." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpCalls),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpCalls", "The number of calls to GVMMR0WakeUp." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpNotHalted), STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpNotHalted", "The number of times the EMT thread wasn't actually halted when GVMMR0WakeUp was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cWakeUpWakeUps),   STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/WakeUpWakeUps", "The number of wake ups done during GVMMR0WakeUp (not counting the explicit one)." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPokeCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PokeCalls", "The number of calls to GVMMR0Poke." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPokeNotBusy),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PokeNotBusy", "The number of times the EMT thread wasn't actually busy when GVMMR0Poke was called." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollCalls),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollCalls", "The number of calls to GVMMR0SchedPoll." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollHalts),       STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollHalts", "The number of times the EMT has halted in a GVMMR0SchedPoll call." },
+    { RT_UOFFSETOF(GVMMSTATS, SchedSum.cPollWakeUps),     STAMTYPE_U64_RESET, STAMUNIT_CALLS, "/GVMM/Sum/PollWakeUps", "The number of wake ups done during GVMMR0SchedPoll." },
+
+    { RT_UOFFSETOF(GVMMSTATS, cVMs),                      STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/VMs", "The number of VMs accessible to the caller." },
+    { RT_UOFFSETOF(GVMMSTATS, cEMTs),                     STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/EMTs", "The number of emulation threads." },
+    { RT_UOFFSETOF(GVMMSTATS, cHostCpus),                 STAMTYPE_U32,       STAMUNIT_CALLS, "/GVMM/HostCPUs", "The number of host CPUs." },
+};
+
+
+/**
+ * Initializes the STAM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) STAMR3InitUVM(PUVM pUVM)
+{
+    LogFlow(("STAMR3Init\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompile(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
+    AssertRelease(sizeof(pUVM->stam.s) <= sizeof(pUVM->stam.padding));
+
+    /*
+     * Setup any fixed pointers and offsets.
+     */
+    int rc = RTSemRWCreate(&pUVM->stam.s.RWSem);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Register the ring-0 statistics (GVMM/GMM).
+     */
+    stamR3Ring0StatsRegisterU(pUVM);
+
+#ifdef VBOX_WITH_DEBUGGER
+    /*
+     * Register debugger commands.
+     */
+    static bool fRegisteredCmds = false;
+    if (!fRegisteredCmds)
+    {
+        rc = DBGCRegisterCommands(&g_aCmds[0], RT_ELEMENTS(g_aCmds));
+        if (RT_SUCCESS(rc))
+            fRegisteredCmds = true;
+    }
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Terminates the STAM.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+VMMR3DECL(void) STAMR3TermUVM(PUVM pUVM)
+{
+    /*
+     * Free used memory and the RWLock.
+     */
+    PSTAMDESC pCur = pUVM->stam.s.pHead;
+    while (pCur)
+    {
+        void *pvFree = pCur;
+        pCur = pCur->pNext;
+        RTMemFree(pvFree);
+    }
+    pUVM->stam.s.pHead = NULL;
+
+    Assert(pUVM->stam.s.RWSem != NIL_RTSEMRW);
+    RTSemRWDestroy(pUVM->stam.s.RWSem);
+    pUVM->stam.s.RWSem = NIL_RTSEMRW;
+}
+
+
+/**
+ * Registers a sample with the statistics manager.
+ *
+ * Statistics are maintained on a per VM basis and is normally registered
+ * during the VM init stage, but there is nothing preventing you from
+ * register them at runtime.
+ *
+ * Use STAMR3Deregister() to deregister statistics at runtime, however do
+ * not bother calling at termination time.
+ *
+ * It is not possible to register the same sample twice.
+ *
+ * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
+ *                      Further nesting is possible.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ */
+VMMR3DECL(int)  STAMR3RegisterU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+    return stamR3RegisterU(pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc);
+}
+
+
+/**
+ * Registers a sample with the statistics manager.
+ *
+ * Statistics are maintained on a per VM basis and is normally registered
+ * during the VM init stage, but there is nothing preventing you from
+ * register them at runtime.
+ *
+ * Use STAMR3Deregister() to deregister statistics at runtime, however do
+ * not bother calling at termination time.
+ *
+ * It is not possible to register the same sample twice.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   pszName     Sample name. The name is on this form "/<component>/<sample>".
+ *                      Further nesting is possible.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ */
+VMMR3DECL(int)  STAMR3Register(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+    return stamR3RegisterU(pVM->pUVM, pvSample, NULL, NULL, enmType, enmVisibility, pszName, enmUnit, pszDesc);
+}
+
+
+/**
+ * Same as STAMR3RegisterU except that the name is specified in a
+ * RTStrPrintf like fashion.
+ *
+ * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterFU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                 const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterVU(pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3Register except that the name is specified in a
+ * RTStrPrintf like fashion.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterF(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3Register except that the name is specified in a
+ * RTStrPrintfV like fashion.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterVU(PUVM pUVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                 const char *pszDesc, const char *pszName, va_list args)
+{
+    AssertReturn(enmType != STAMTYPE_CALLBACK, VERR_INVALID_PARAMETER);
+
+    char *pszFormattedName;
+    RTStrAPrintfV(&pszFormattedName, pszName, args);
+    if (!pszFormattedName)
+        return VERR_NO_MEMORY;
+
+    int rc = STAMR3RegisterU(pUVM, pvSample, enmType, enmVisibility, pszFormattedName, enmUnit, pszDesc);
+    RTStrFree(pszFormattedName);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3Register except that the name is specified in a
+ * RTStrPrintfV like fashion.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ */
+VMMR3DECL(int)  STAMR3RegisterV(PVM pVM, void *pvSample, STAMTYPE enmType, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                const char *pszDesc, const char *pszName, va_list args)
+{
+    return STAMR3RegisterVU(pVM->pUVM, pvSample, enmType, enmVisibility, enmUnit, pszDesc, pszName, args);
+}
+
+
+/**
+ * Similar to STAMR3Register except for the two callbacks, the implied type (STAMTYPE_CALLBACK),
+ * and name given in an RTStrPrintf like fashion.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
+ * @param   pfnPrint    Print the sample.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   ...         Arguments to the format string.
+ * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
+ */
+VMMR3DECL(int)  STAMR3RegisterCallback(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                       PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                                       const char *pszDesc, const char *pszName, ...)
+{
+    va_list args;
+    va_start(args, pszName);
+    int rc = STAMR3RegisterCallbackV(pVM, pvSample, enmVisibility, enmUnit, pfnReset, pfnPrint, pszDesc, pszName, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Same as STAMR3RegisterCallback() except for the ellipsis which is a va_list here.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
+ * @param   pfnPrint    Print the sample.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
+ */
+VMMR3DECL(int)  STAMR3RegisterCallbackV(PVM pVM, void *pvSample, STAMVISIBILITY enmVisibility, STAMUNIT enmUnit,
+                                        PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                                        const char *pszDesc, const char *pszName, va_list args)
+{
+    char *pszFormattedName;
+    RTStrAPrintfV(&pszFormattedName, pszName, args);
+    if (!pszFormattedName)
+        return VERR_NO_MEMORY;
+
+    int rc = stamR3RegisterU(pVM->pUVM, pvSample, pfnReset, pfnPrint, STAMTYPE_CALLBACK, enmVisibility, pszFormattedName, enmUnit, pszDesc);
+    RTStrFree(pszFormattedName);
+    return rc;
+}
+
+
+/**
+ * Divide the strings into sub-strings using '/' as delimiter
+ * and then compare them in strcmp fashion.
+ *
+ * @returns Difference.
+ * @retval  0 if equal.
+ * @retval  < 0 if psz1 is less than psz2.
+ * @retval  > 0 if psz1 greater than psz2.
+ *
+ * @param   psz1        The first string.
+ * @param   psz2        The second string.
+ */
+static int stamR3SlashCompare(const char *psz1, const char *psz2)
+{
+    for (;;)
+    {
+        unsigned int ch1 = *psz1++;
+        unsigned int ch2 = *psz2++;
+        if (ch1 != ch2)
+        {
+            /* slash is end-of-sub-string, so it trumps everything but '\0'. */
+            if (ch1 == '/')
+                return ch2 ? -1 : 1;
+            if (ch2 == '/')
+                return ch1 ? 1 : -1;
+            return ch1 - ch2;
+        }
+
+        /* done? */
+        if (ch1 == '\0')
+            return 0;
+    }
+}
+
+
+/**
+ * Internal worker for the different register calls.
+ *
+ * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample.
+ * @param   pfnReset    Callback for resetting the sample. NULL should be used if the sample can't be reset.
+ * @param   pfnPrint    Print the sample.
+ * @param   enmType     Sample type. This indicates what pvSample is pointing at.
+ * @param   enmVisibility  Visibility type specifying whether unused statistics should be visible or not.
+ * @param   enmUnit     Sample unit.
+ * @param   pszDesc     Sample description.
+ * @param   pszName     The sample name format string.
+ * @param   args        Arguments to the format string.
+ * @remark  There is currently no device or driver variant of this API. Add one if it should become necessary!
+ */
+static int stamR3RegisterU(PUVM pUVM, void *pvSample, PFNSTAMR3CALLBACKRESET pfnReset, PFNSTAMR3CALLBACKPRINT pfnPrint,
+                           STAMTYPE enmType, STAMVISIBILITY enmVisibility, const char *pszName, STAMUNIT enmUnit, const char *pszDesc)
+{
+    STAM_LOCK_WR(pUVM);
+
+    /*
+     * Check if exists.
+     */
+    PSTAMDESC   pPrev = NULL;
+    PSTAMDESC   pCur = pUVM->stam.s.pHead;
+    while (pCur)
+    {
+        int iDiff = strcmp(pCur->pszName, pszName);
+        /* passed it */
+        if (iDiff > 0)
+            break;
+        /* found it. */
+        if (!iDiff)
+        {
+            STAM_UNLOCK_WR(pUVM);
+            AssertMsgFailed(("Duplicate sample name: %s\n", pszName));
+            return VERR_ALREADY_EXISTS;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+
+    /*
+     * Check that the name doesn't screw up sorting order when taking
+     * slashes into account. The QT4 GUI makes some assumptions.
+     * Problematic chars are: !"#$%&'()*+,-.
+     */
+    Assert(pszName[0] == '/');
+    if (pPrev)
+        Assert(stamR3SlashCompare(pPrev->pszName, pszName) < 0);
+    if (pCur)
+        Assert(stamR3SlashCompare(pCur->pszName, pszName) > 0);
+
+#ifdef VBOX_STRICT
+    /*
+     * Check alignment requirements.
+     */
+    switch (enmType)
+    {
+            /* 8 byte / 64-bit */
+        case STAMTYPE_U64:
+        case STAMTYPE_U64_RESET:
+        case STAMTYPE_X64:
+        case STAMTYPE_X64_RESET:
+        case STAMTYPE_COUNTER:
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+            AssertMsg(!((uintptr_t)pvSample & 7), ("%p - %s\n", pvSample, pszName));
+            break;
+
+            /* 4 byte / 32-bit */
+        case STAMTYPE_RATIO_U32:
+        case STAMTYPE_RATIO_U32_RESET:
+        case STAMTYPE_U32:
+        case STAMTYPE_U32_RESET:
+        case STAMTYPE_X32:
+        case STAMTYPE_X32_RESET:
+            AssertMsg(!((uintptr_t)pvSample & 3), ("%p - %s\n", pvSample, pszName));
+            break;
+
+            /* 2 byte / 32-bit */
+        case STAMTYPE_U16:
+        case STAMTYPE_U16_RESET:
+        case STAMTYPE_X16:
+        case STAMTYPE_X16_RESET:
+            AssertMsg(!((uintptr_t)pvSample & 1), ("%p - %s\n", pvSample, pszName));
+            break;
+
+            /* 1 byte / 8-bit / unaligned */
+        case STAMTYPE_U8:
+        case STAMTYPE_U8_RESET:
+        case STAMTYPE_X8:
+        case STAMTYPE_X8_RESET:
+        case STAMTYPE_CALLBACK:
+            break;
+
+        default:
+            AssertMsgFailed(("%d\n", enmType));
+            break;
+    }
+#endif /* VBOX_STRICT */
+
+    /*
+     * Create a new node and insert it at the current location.
+     */
+    int rc;
+    size_t cchName = strlen(pszName) + 1;
+    size_t cchDesc = pszDesc ? strlen(pszDesc) + 1 : 0;
+    PSTAMDESC pNew = (PSTAMDESC)RTMemAlloc(sizeof(*pNew) + cchName + cchDesc);
+    if (pNew)
+    {
+        pNew->pszName       = (char *)memcpy((char *)(pNew + 1), pszName, cchName);
+        pNew->enmType       = enmType;
+        pNew->enmVisibility = enmVisibility;
+        if (enmType != STAMTYPE_CALLBACK)
+            pNew->u.pv      = pvSample;
+        else
+        {
+            pNew->u.Callback.pvSample = pvSample;
+            pNew->u.Callback.pfnReset = pfnReset;
+            pNew->u.Callback.pfnPrint = pfnPrint;
+        }
+        pNew->enmUnit       = enmUnit;
+        pNew->pszDesc       = NULL;
+        if (pszDesc)
+            pNew->pszDesc   = (char *)memcpy((char *)(pNew + 1) + cchName,  pszDesc,  cchDesc);
+
+        pNew->pNext         = pCur;
+        if (pPrev)
+            pPrev->pNext    = pNew;
+        else
+            pUVM->stam.s.pHead = pNew;
+
+        stamR3ResetOne(pNew, pUVM->pVM);
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = VERR_NO_MEMORY;
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+
+/**
+ * Deregisters a sample previously registered by STAR3Register().
+ *
+ * This is intended used for devices which can be unplugged and for
+ * temporary samples.
+ *
+ * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pvSample    Pointer to the sample registered with STAMR3Register().
+ */
+VMMR3DECL(int)  STAMR3DeregisterU(PUVM pUVM, void *pvSample)
+{
+    STAM_LOCK_WR(pUVM);
+
+    /*
+     * Search for it.
+     */
+    int         rc = VERR_INVALID_HANDLE;
+    PSTAMDESC   pPrev = NULL;
+    PSTAMDESC   pCur = pUVM->stam.s.pHead;
+    while (pCur)
+    {
+        if (pCur->u.pv == pvSample)
+        {
+            void *pvFree = pCur;
+            pCur = pCur->pNext;
+            if (pPrev)
+                pPrev->pNext = pCur;
+            else
+                pUVM->stam.s.pHead = pCur;
+
+            RTMemFree(pvFree);
+            rc = VINF_SUCCESS;
+            continue;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+
+/**
+ * Deregisters a sample previously registered by STAR3Register().
+ *
+ * This is intended used for devices which can be unplugged and for
+ * temporary samples.
+ *
+ * @returns VBox status.
+ * @param   pVM         The VM handle.
+ * @param   pvSample    Pointer to the sample registered with STAMR3Register().
+ */
+VMMR3DECL(int)  STAMR3Deregister(PVM pVM, void *pvSample)
+{
+    return STAMR3DeregisterU(pVM->pUVM, pvSample);
+}
+
+
+/**
+ * Resets statistics for the specified VM.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status. (Basically, it cannot fail.)
+ * @param   pVM         The VM handle.
+ * @param   pszPat      The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                      If NULL all samples are reset.
+ * @remarks Don't confuse this with the other 'XYZR3Reset' methods, it's not called at VM reset.
+ */
+VMMR3DECL(int)  STAMR3ResetU(PUVM pUVM, const char *pszPat)
+{
+    int rc = VINF_SUCCESS;
+
+    /* ring-0 */
+    GVMMRESETSTATISTICSSREQ GVMMReq;
+    //GMMRESETSTATISTICSSREQ GMMReq;
+    bool fGVMMMatched = !pszPat || !*pszPat;
+    //bool fGMMMatched = fGVMMMatched;
+    if (fGVMMMatched)
+        memset(&GVMMReq.Stats, 0xff, sizeof(GVMMReq.Stats));
+    else
+    {
+        char *pszCopy;
+        unsigned cExpressions;
+        char **papszExpressions = stamR3SplitPattern(pszPat, &cExpressions, &pszCopy);
+        if (!papszExpressions)
+            return VERR_NO_MEMORY;
+
+        /* GVMM */
+        memset(&GVMMReq.Stats, 0, sizeof(GVMMReq.Stats));
+        for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
+            if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGVMMStats[i].pszName))
+            {
+                *((uint8_t *)&GVMMReq.Stats + g_aGVMMStats[i].offVar) = 0xff;
+                fGVMMMatched = true;
+            }
+        if (!fGVMMMatched)
+        {
+            /** @todo match cpu leaves some rainy day.  */
+        }
+
+        /* GMM */
+//        memset(&GMMReq.Stats, 0, sizeof(GMMReq.Stats));
+//        for (unsigned i = 0; i < RT_ELEMENTS(g_aGMMStats); i++)
+//            if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGMMStats[i].pszName))
+//            {
+//                 *((uint8_t *)&GMMReq.Stats + g_aGMMStats[i].offVar) = 0xff;
+//                 fGMMMatched = true;
+//            }
+
+        RTMemTmpFree(papszExpressions);
+        RTStrFree(pszCopy);
+    }
+
+    STAM_LOCK_WR(pUVM);
+    if (fGVMMMatched)
+    {
+        PVM pVM = pUVM->pVM;
+        GVMMReq.Hdr.cbReq = sizeof(GVMMReq);
+        GVMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+        GVMMReq.pSession = pVM->pSession;
+        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GVMM_RESET_STATISTICS, 0, &GVMMReq.Hdr);
+    }
+
+//    if (fGMMMatched)
+//    {
+//        PVM pVM = pUVM->pVM;
+//        GMMReq.Hdr.cbReq = sizeof(Req);
+//        GMMReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+//        GMMReq.pSession = pVM->pSession;
+//        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, VMMR0_DO_GMM_RESET_STATISTICS, 0, &Req.Hdr);
+//    }
+
+    /* and the reset */
+    stamR3EnumU(pUVM, pszPat, false /* fUpdateRing0 */, stamR3ResetOne, pUVM->pVM);
+
+    STAM_UNLOCK_WR(pUVM);
+    return rc;
+}
+
+/**
+ * Resets statistics for the specified VM.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status. (Basically, it cannot fail.)
+ * @param   pVM         The VM handle.
+ * @param   pszPat      The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                      If NULL all samples are reset.
+ * @remarks Don't confuse this with the other 'XYZR3Reset' methods, it's not called at VM reset.
+ */
+VMMR3DECL(int)  STAMR3Reset(PVM pVM, const char *pszPat)
+{
+    return STAMR3ResetU(pVM->pUVM, pszPat);
+}
+
+
+/**
+ * Resets one statistics sample.
+ * Callback for stamR3EnumU().
+ *
+ * @returns VINF_SUCCESS
+ * @param   pDesc   Pointer to the current descriptor.
+ * @param   pvArg   User argument - The VM handle.
+ */
+static int stamR3ResetOne(PSTAMDESC pDesc, void *pvArg)
+{
+    switch (pDesc->enmType)
+    {
+        case STAMTYPE_COUNTER:
+            ASMAtomicXchgU64(&pDesc->u.pCounter->c, 0);
+            break;
+
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cPeriods, 0);
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicks, 0);
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicksMax, 0);
+            ASMAtomicXchgU64(&pDesc->u.pProfile->cTicksMin, ~0);
+            break;
+
+        case STAMTYPE_RATIO_U32_RESET:
+            ASMAtomicXchgU32(&pDesc->u.pRatioU32->u32A, 0);
+            ASMAtomicXchgU32(&pDesc->u.pRatioU32->u32B, 0);
+            break;
+
+        case STAMTYPE_CALLBACK:
+            if (pDesc->u.Callback.pfnReset)
+                pDesc->u.Callback.pfnReset((PVM)pvArg, pDesc->u.Callback.pvSample);
+            break;
+
+        case STAMTYPE_U8_RESET:
+        case STAMTYPE_X8_RESET:
+            ASMAtomicXchgU8(pDesc->u.pu8, 0);
+            break;
+
+        case STAMTYPE_U16_RESET:
+        case STAMTYPE_X16_RESET:
+            ASMAtomicXchgU16(pDesc->u.pu16, 0);
+            break;
+
+        case STAMTYPE_U32_RESET:
+        case STAMTYPE_X32_RESET:
+            ASMAtomicXchgU32(pDesc->u.pu32, 0);
+            break;
+
+        case STAMTYPE_U64_RESET:
+        case STAMTYPE_X64_RESET:
+            ASMAtomicXchgU64(pDesc->u.pu64, 0);
+            break;
+
+        /* These are custom and will not be touched. */
+        case STAMTYPE_U8:
+        case STAMTYPE_X8:
+        case STAMTYPE_U16:
+        case STAMTYPE_X16:
+        case STAMTYPE_U32:
+        case STAMTYPE_X32:
+        case STAMTYPE_U64:
+        case STAMTYPE_X64:
+        case STAMTYPE_RATIO_U32:
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", pDesc->enmType));
+            break;
+    }
+    NOREF(pvArg);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get a snapshot of the statistics.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status. (Basically, it cannot fail.)
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ * @param   fWithDesc       Whether to include the descriptions.
+ * @param   ppszSnapshot    Where to store the pointer to the snapshot data.
+ *                          The format of the snapshot should be XML, but that will have to be discussed
+ *                          when this function is implemented.
+ *                          The returned pointer must be freed by calling STAMR3SnapshotFree().
+ * @param   pcchSnapshot    Where to store the size of the snapshot data. (Excluding the trailing '\0')
+ */
+VMMR3DECL(int) STAMR3SnapshotU(PUVM pUVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc)
+{
+    STAMR3SNAPSHOTONE State = { NULL, NULL, NULL, pUVM->pVM, 0, VINF_SUCCESS, fWithDesc };
+
+    /*
+     * Write the XML header.
+     */
+    /** @todo Make this proper & valid XML. */
+    stamR3SnapshotPrintf(&State, "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n");
+
+    /*
+     * Write the content.
+     */
+    stamR3SnapshotPrintf(&State, "<Statistics>\n");
+    int rc = stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3SnapshotOne, &State);
+    stamR3SnapshotPrintf(&State, "</Statistics>\n");
+
+    if (RT_SUCCESS(rc))
+        rc = State.rc;
+    else
+    {
+        RTMemFree(State.pszStart);
+        State.pszStart = State.pszEnd = State.psz = NULL;
+        State.cbAllocated = 0;
+    }
+
+    /*
+     * Done.
+     */
+    *ppszSnapshot = State.pszStart;
+    if (pcchSnapshot)
+        *pcchSnapshot = State.psz - State.pszStart;
+    return rc;
+}
+
+
+/**
+ * Get a snapshot of the statistics.
+ * It's possible to select a subset of the samples.
+ *
+ * @returns VBox status. (Basically, it cannot fail.)
+ * @param   pVM             The VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ * @param   fWithDesc       Whether to include the descriptions.
+ * @param   ppszSnapshot    Where to store the pointer to the snapshot data.
+ *                          The format of the snapshot should be XML, but that will have to be discussed
+ *                          when this function is implemented.
+ *                          The returned pointer must be freed by calling STAMR3SnapshotFree().
+ * @param   pcchSnapshot    Where to store the size of the snapshot data.
+ *                          (Excluding the trailing '\\0')
+ */
+VMMR3DECL(int) STAMR3Snapshot(PVM pVM, const char *pszPat, char **ppszSnapshot, size_t *pcchSnapshot, bool fWithDesc)
+{
+    return STAMR3SnapshotU(pVM->pUVM, pszPat, ppszSnapshot, pcchSnapshot, fWithDesc);
+}
+
+
+/**
+ * stamR3EnumU callback employed by STAMR3Snapshot.
+ *
+ * @returns VBox status code, but it's interpreted as 0 == success / !0 == failure by enmR3Enum.
+ * @param   pDesc       The sample.
+ * @param   pvArg       The snapshot status structure.
+ */
+static int stamR3SnapshotOne(PSTAMDESC pDesc, void *pvArg)
+{
+    PSTAMR3SNAPSHOTONE pThis = (PSTAMR3SNAPSHOTONE)pvArg;
+
+    switch (pDesc->enmType)
+    {
+        case STAMTYPE_COUNTER:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pCounter->c == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<Counter c=\"%lld\"", pDesc->u.pCounter->c);
+            break;
+
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pProfile->cPeriods == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<Profile cPeriods=\"%lld\" cTicks=\"%lld\" cTicksMin=\"%lld\" cTicksMax=\"%lld\"",
+                                 pDesc->u.pProfile->cPeriods, pDesc->u.pProfile->cTicks, pDesc->u.pProfile->cTicksMin,
+                                 pDesc->u.pProfile->cTicksMax);
+            break;
+
+        case STAMTYPE_RATIO_U32:
+        case STAMTYPE_RATIO_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && !pDesc->u.pRatioU32->u32A && !pDesc->u.pRatioU32->u32B)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<Ratio32 u32A=\"%lld\" u32B=\"%lld\"",
+                                 pDesc->u.pRatioU32->u32A, pDesc->u.pRatioU32->u32B);
+            break;
+
+        case STAMTYPE_CALLBACK:
+        {
+            char szBuf[512];
+            pDesc->u.Callback.pfnPrint(pThis->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
+            stamR3SnapshotPrintf(pThis, "<Callback val=\"%s\"", szBuf);
+            break;
+        }
+
+        case STAMTYPE_U8:
+        case STAMTYPE_U8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U8 val=\"%u\"", *pDesc->u.pu8);
+            break;
+
+        case STAMTYPE_X8:
+        case STAMTYPE_X8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X8 val=\"%#x\"", *pDesc->u.pu8);
+            break;
+
+        case STAMTYPE_U16:
+        case STAMTYPE_U16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U16 val=\"%u\"", *pDesc->u.pu16);
+            break;
+
+        case STAMTYPE_X16:
+        case STAMTYPE_X16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X16 val=\"%#x\"", *pDesc->u.pu16);
+            break;
+
+        case STAMTYPE_U32:
+        case STAMTYPE_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U32 val=\"%u\"", *pDesc->u.pu32);
+            break;
+
+        case STAMTYPE_X32:
+        case STAMTYPE_X32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X32 val=\"%#x\"", *pDesc->u.pu32);
+            break;
+
+        case STAMTYPE_U64:
+        case STAMTYPE_U64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<U64 val=\"%llu\"", *pDesc->u.pu64);
+            break;
+
+        case STAMTYPE_X64:
+        case STAMTYPE_X64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            stamR3SnapshotPrintf(pThis, "<X64 val=\"%#llx\"", *pDesc->u.pu64);
+            break;
+
+        default:
+            AssertMsgFailed(("%d\n", pDesc->enmType));
+            return 0;
+    }
+
+    stamR3SnapshotPrintf(pThis, " unit=\"%s\"", STAMR3GetUnit(pDesc->enmUnit));
+
+    switch (pDesc->enmVisibility)
+    {
+        default:
+        case STAMVISIBILITY_ALWAYS:
+            break;
+        case STAMVISIBILITY_USED:
+            stamR3SnapshotPrintf(pThis, " vis=\"used\"");
+            break;
+        case STAMVISIBILITY_NOT_GUI:
+            stamR3SnapshotPrintf(pThis, " vis=\"not-gui\"");
+            break;
+    }
+
+    stamR3SnapshotPrintf(pThis, " name=\"%s\"", pDesc->pszName);
+
+    if (pThis->fWithDesc && pDesc->pszDesc)
+    {
+        /*
+         * The description is a bit tricky as it may include chars that
+         * xml requires to be escaped.
+         */
+        const char *pszBadChar = strpbrk(pDesc->pszDesc, "&<>\"'");
+        if (!pszBadChar)
+            return stamR3SnapshotPrintf(pThis, " desc=\"%s\"/>\n", pDesc->pszDesc);
+
+        stamR3SnapshotPrintf(pThis, " desc=\"");
+        const char *pszCur = pDesc->pszDesc;
+        do
+        {
+            stamR3SnapshotPrintf(pThis, "%.*s", pszBadChar - pszCur, pszCur);
+            switch (*pszBadChar)
+            {
+                case '&':   stamR3SnapshotPrintf(pThis, "&amp;");   break;
+                case '<':   stamR3SnapshotPrintf(pThis, "&lt;");    break;
+                case '>':   stamR3SnapshotPrintf(pThis, "&gt;");    break;
+                case '"':   stamR3SnapshotPrintf(pThis, "&quot;");  break;
+                case '\'':  stamR3SnapshotPrintf(pThis, "&apos;");  break;
+                default:    AssertMsgFailed(("%c", *pszBadChar));    break;
+            }
+            pszCur = pszBadChar + 1;
+            pszBadChar = strpbrk(pszCur, "&<>\"'");
+        } while (pszBadChar);
+        return stamR3SnapshotPrintf(pThis, "%s\"/>\n", pszCur);
+    }
+    return stamR3SnapshotPrintf(pThis, "/>\n");
+}
+
+
+/**
+ * Output callback for stamR3SnapshotPrintf.
+ *
+ * @returns number of bytes written.
+ * @param   pvArg       The snapshot status structure.
+ * @param   pach        Pointer to an array of characters (bytes).
+ * @param   cch         The number or chars (bytes) to write from the array.
+ */
+static DECLCALLBACK(size_t) stamR3SnapshotOutput(void *pvArg, const char *pach, size_t cch)
+{
+    PSTAMR3SNAPSHOTONE pThis = (PSTAMR3SNAPSHOTONE)pvArg;
+
+    /*
+     * Make sure we've got space for it.
+     */
+    if (RT_UNLIKELY((uintptr_t)pThis->pszEnd - (uintptr_t)pThis->psz < cch + 1))
+    {
+        if (RT_FAILURE(pThis->rc))
+            return 0;
+
+        size_t cbNewSize = pThis->cbAllocated;
+        if (cbNewSize > cch)
+            cbNewSize *= 2;
+        else
+            cbNewSize += RT_ALIGN(cch + 1, 0x1000);
+        char *pszNew = (char *)RTMemRealloc(pThis->pszStart, cbNewSize);
+        if (!pszNew)
+        {
+            /*
+             * Free up immediately, out-of-memory is bad news and this
+             * isn't an important allocations / API.
+             */
+            pThis->rc = VERR_NO_MEMORY;
+            RTMemFree(pThis->pszStart);
+            pThis->pszStart = pThis->pszEnd = pThis->psz = NULL;
+            pThis->cbAllocated = 0;
+            return 0;
+        }
+
+        pThis->psz = pszNew + (pThis->psz - pThis->pszStart);
+        pThis->pszStart = pszNew;
+        pThis->pszEnd = pszNew + cbNewSize;
+        pThis->cbAllocated = cbNewSize;
+    }
+
+    /*
+     * Copy the chars to the buffer and terminate it.
+     */
+    memcpy(pThis->psz, pach, cch);
+    pThis->psz += cch;
+    *pThis->psz = '\0';
+    return cch;
+}
+
+
+/**
+ * Wrapper around RTStrFormatV for use by the snapshot API.
+ *
+ * @returns VBox status code.
+ * @param   pThis       The snapshot status structure.
+ * @param   pszFormat   The format string.
+ * @param   ...         Optional arguments.
+ */
+static int stamR3SnapshotPrintf(PSTAMR3SNAPSHOTONE pThis, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTStrFormatV(stamR3SnapshotOutput, pThis, NULL, NULL, pszFormat, va);
+    va_end(va);
+    return pThis->rc;
+}
+
+
+/**
+ * Releases a statistics snapshot returned by STAMR3Snapshot().
+ *
+ * @returns VBox status.
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
+ *                          NULL is allowed.
+ */
+VMMR3DECL(int)  STAMR3SnapshotFreeU(PUVM pUVM, char *pszSnapshot)
+{
+    if (!pszSnapshot)
+        RTMemFree(pszSnapshot);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Releases a statistics snapshot returned by STAMR3Snapshot().
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszSnapshot     The snapshot data pointer returned by STAMR3Snapshot().
+ *                          NULL is allowed.
+ */
+VMMR3DECL(int)  STAMR3SnapshotFree(PVM pVM, char *pszSnapshot)
+{
+    return STAMR3SnapshotFreeU(pVM->pUVM, pszSnapshot);
+}
+
+
+/**
+ * Dumps the selected statistics to the log.
+ *
+ * @returns VBox status.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+VMMR3DECL(int)  STAMR3DumpU(PUVM pUVM, const char *pszPat)
+{
+    STAMR3PRINTONEARGS Args;
+    Args.pVM = pUVM->pVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumLogPrintf;
+
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps the selected statistics to the log.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+VMMR3DECL(int)  STAMR3Dump(PVM pVM, const char *pszPat)
+{
+    return STAMR3DumpU(pVM->pUVM, pszPat);
+}
+
+
+/**
+ * Prints to the log.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumLogPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTLogPrintfV(pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * Dumps the selected statistics to the release log.
+ *
+ * @returns VBox status.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+VMMR3DECL(int)  STAMR3DumpToReleaseLogU(PUVM pUVM, const char *pszPat)
+{
+    STAMR3PRINTONEARGS Args;
+    Args.pVM = pUVM->pVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumRelLogPrintf;
+
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Dumps the selected statistics to the release log.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are written to the log.
+ */
+VMMR3DECL(int)  STAMR3DumpToReleaseLog(PVM pVM, const char *pszPat)
+{
+    return STAMR3DumpToReleaseLogU(pVM->pUVM, pszPat);
+}
+
+
+/**
+ * Prints to the release log.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumRelLogPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTLogRelPrintfV(pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * Prints the selected statistics to standard out.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ */
+VMMR3DECL(int)  STAMR3PrintU(PUVM pUVM, const char *pszPat)
+{
+    STAMR3PRINTONEARGS Args;
+    Args.pVM = pUVM->pVM;
+    Args.pvArg = NULL;
+    Args.pfnPrintf = stamR3EnumPrintf;
+
+    stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Prints the selected statistics to standard out.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM handle.
+ * @param   pszPat          The name matching pattern. See somewhere_where_this_is_described_in_detail.
+ *                          If NULL all samples are reset.
+ */
+VMMR3DECL(int)  STAMR3Print(PVM pVM, const char *pszPat)
+{
+    return STAMR3PrintU(pVM->pUVM, pszPat);
+}
+
+
+/**
+ * Prints to stdout.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    RTPrintfV(pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * Prints one sample.
+ * Callback for stamR3EnumU().
+ *
+ * @returns VINF_SUCCESS
+ * @param   pDesc   Pointer to the current descriptor.
+ * @param   pvArg   User argument - STAMR3PRINTONEARGS.
+ */
+static int stamR3PrintOne(PSTAMDESC pDesc, void *pvArg)
+{
+    PSTAMR3PRINTONEARGS pArgs = (PSTAMR3PRINTONEARGS)pvArg;
+
+    switch (pDesc->enmType)
+    {
+        case STAMTYPE_COUNTER:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pCounter->c == 0)
+                return VINF_SUCCESS;
+
+            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s\n", pDesc->pszName, pDesc->u.pCounter->c, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_PROFILE:
+        case STAMTYPE_PROFILE_ADV:
+        {
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && pDesc->u.pProfile->cPeriods == 0)
+                return VINF_SUCCESS;
+
+            uint64_t u64 = pDesc->u.pProfile->cPeriods ? pDesc->u.pProfile->cPeriods : 1;
+            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s (%12llu ticks, %7llu times, max %9llu, min %7lld)\n", pDesc->pszName,
+                             pDesc->u.pProfile->cTicks / u64, STAMR3GetUnit(pDesc->enmUnit),
+                             pDesc->u.pProfile->cTicks, pDesc->u.pProfile->cPeriods, pDesc->u.pProfile->cTicksMax, pDesc->u.pProfile->cTicksMin);
+            break;
+        }
+
+        case STAMTYPE_RATIO_U32:
+        case STAMTYPE_RATIO_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && !pDesc->u.pRatioU32->u32A && !pDesc->u.pRatioU32->u32B)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u:%-8u %s\n", pDesc->pszName,
+                             pDesc->u.pRatioU32->u32A, pDesc->u.pRatioU32->u32B, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_CALLBACK:
+        {
+            char szBuf[512];
+            pDesc->u.Callback.pfnPrint(pArgs->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
+            pArgs->pfnPrintf(pArgs, "%-32s %s %s\n", pDesc->pszName, szBuf, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+        }
+
+        case STAMTYPE_U8:
+        case STAMTYPE_U8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu8, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X8:
+        case STAMTYPE_X8_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu8 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu8, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_U16:
+        case STAMTYPE_U16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu16, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X16:
+        case STAMTYPE_X16_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu16 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu16, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_U32:
+        case STAMTYPE_U32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8u %s\n", pDesc->pszName, *pDesc->u.pu32, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X32:
+        case STAMTYPE_X32_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu32 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8x %s\n", pDesc->pszName, *pDesc->u.pu32, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_U64:
+        case STAMTYPE_U64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8llu %s\n", pDesc->pszName, *pDesc->u.pu64, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        case STAMTYPE_X64:
+        case STAMTYPE_X64_RESET:
+            if (pDesc->enmVisibility == STAMVISIBILITY_USED && *pDesc->u.pu64 == 0)
+                return VINF_SUCCESS;
+            pArgs->pfnPrintf(pArgs, "%-32s %8llx %s\n", pDesc->pszName, *pDesc->u.pu64, STAMR3GetUnit(pDesc->enmUnit));
+            break;
+
+        default:
+            AssertMsgFailed(("enmType=%d\n", pDesc->enmType));
+            break;
+    }
+    NOREF(pvArg);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Enumerate the statistics by the means of a callback function.
+ *
+ * @returns Whatever the callback returns.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPat      The pattern to match samples.
+ * @param   pfnEnum     The callback function.
+ * @param   pvUser      The pvUser argument of the callback function.
+ */
+VMMR3DECL(int) STAMR3EnumU(PUVM pUVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser)
+{
+    STAMR3ENUMONEARGS Args;
+    Args.pVM     = pUVM->pVM;
+    Args.pfnEnum = pfnEnum;
+    Args.pvUser  = pvUser;
+
+    return stamR3EnumU(pUVM, pszPat, true /* fUpdateRing0 */, stamR3EnumOne, &Args);
+}
+
+
+/**
+ * Enumerate the statistics by the means of a callback function.
+ *
+ * @returns Whatever the callback returns.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pszPat      The pattern to match samples.
+ * @param   pfnEnum     The callback function.
+ * @param   pvUser      The pvUser argument of the callback function.
+ */
+VMMR3DECL(int) STAMR3Enum(PVM pVM, const char *pszPat, PFNSTAMR3ENUM pfnEnum, void *pvUser)
+{
+    return STAMR3EnumU(pVM->pUVM, pszPat, pfnEnum, pvUser);
+}
+
+
+/**
+ * Callback function for STARTR3Enum().
+ *
+ * @returns whatever the callback returns.
+ * @param   pDesc       Pointer to the current descriptor.
+ * @param   pvArg       Points to a STAMR3ENUMONEARGS structure.
+ */
+static int stamR3EnumOne(PSTAMDESC pDesc, void *pvArg)
+{
+    PSTAMR3ENUMONEARGS pArgs = (PSTAMR3ENUMONEARGS)pvArg;
+    int rc;
+    if (pDesc->enmType == STAMTYPE_CALLBACK)
+    {
+        /* Give the enumerator something useful. */
+        char szBuf[512];
+        pDesc->u.Callback.pfnPrint(pArgs->pVM, pDesc->u.Callback.pvSample, szBuf, sizeof(szBuf));
+        rc = pArgs->pfnEnum(pDesc->pszName, pDesc->enmType, szBuf, pDesc->enmUnit,
+                            pDesc->enmVisibility, pDesc->pszDesc, pArgs->pvUser);
+    }
+    else
+        rc = pArgs->pfnEnum(pDesc->pszName, pDesc->enmType, pDesc->u.pv, pDesc->enmUnit,
+                            pDesc->enmVisibility, pDesc->pszDesc, pArgs->pvUser);
+    return rc;
+}
+
+
+/**
+ * Match a name against an array of patterns.
+ *
+ * @returns true if it matches, false if it doesn't match.
+ * @param   papszExpressions    The array of pattern expressions.
+ * @param   cExpressions        The number of array entries.
+ * @param   piExpression        Where to read/store the current skip index. Optional.
+ * @param   pszName             The name to match.
+ */
+static bool stamR3MultiMatch(const char * const *papszExpressions, unsigned cExpressions,
+                             unsigned *piExpression, const char *pszName)
+{
+    for (unsigned i = piExpression ? *piExpression : 0; i < cExpressions; i++)
+    {
+        const char *pszPat = papszExpressions[i];
+        if (RTStrSimplePatternMatch(pszPat, pszName))
+        {
+            /* later:
+            if (piExpression && i > *piExpression)
+            {
+                check if we can skip some expressions
+            }*/
+            return true;
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Splits a multi pattern into single ones.
+ *
+ * @returns Pointer to an array of single patterns. Free it with RTMemTmpFree.
+ * @param   pszPat          The pattern to split.
+ * @param   pcExpressions   The number of array elements.
+ * @param   pszCopy         The pattern copy to free using RTStrFree.
+ */
+static char **stamR3SplitPattern(const char *pszPat, unsigned *pcExpressions, char **ppszCopy)
+{
+    Assert(pszPat && *pszPat);
+
+    char *pszCopy = RTStrDup(pszPat);
+    if (!pszCopy)
+        return NULL;
+
+    /* count them & allocate array. */
+    char *psz = pszCopy;
+    unsigned cExpressions = 1;
+    while ((psz = strchr(psz, '|')) != NULL)
+        cExpressions++, psz++;
+
+    char **papszExpressions = (char **)RTMemTmpAllocZ((cExpressions + 1) * sizeof(char *));
+    if (!papszExpressions)
+    {
+        RTStrFree(pszCopy);
+        return NULL;
+    }
+
+    /* split */
+    psz = pszCopy;
+    for (unsigned i = 0;;)
+    {
+        papszExpressions[i] = psz;
+        if (++i >= cExpressions)
+            break;
+        psz = strchr(psz, '|');
+        *psz++ = '\0';
+    }
+
+    /* sort the array, putting '*' last. */
+    /** @todo sort it... */
+
+    *pcExpressions = cExpressions;
+    *ppszCopy = pszCopy;
+    return papszExpressions;
+}
+
+
+/**
+ * Enumerates the nodes selected by a pattern or all nodes if no pattern
+ * is specified.
+ *
+ * The call may lock STAM for writing before calling this function, however do
+ * not lock it for reading as this function may need to write lock STAM.
+ *
+ * @returns The rc from the callback.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   pszPat          Pattern.
+ * @param   fUpdateRing0    Update the ring-0 .
+ * @param   pfnCallback     Callback function which shall be called for matching nodes.
+ *                          If it returns anything but VINF_SUCCESS the enumeration is
+ *                          terminated and the status code returned to the caller.
+ * @param   pvArg           User parameter for the callback.
+ */
+static int stamR3EnumU(PUVM pUVM, const char *pszPat, bool fUpdateRing0,
+                       int (*pfnCallback)(PSTAMDESC pDesc, void *pvArg), void *pvArg)
+{
+    int rc = VINF_SUCCESS;
+
+    /*
+     * All
+     */
+    if (!pszPat || !*pszPat || !strcmp(pszPat, "*"))
+    {
+        if (fUpdateRing0)
+            stamR3Ring0StatsUpdateU(pUVM, "*");
+
+        STAM_LOCK_RD(pUVM);
+        PSTAMDESC pCur = pUVM->stam.s.pHead;
+        while (pCur)
+        {
+            rc = pfnCallback(pCur, pvArg);
+            if (rc)
+                break;
+
+            /* next */
+            pCur = pCur->pNext;
+        }
+        STAM_UNLOCK_RD(pUVM);
+    }
+
+    /*
+     * Single expression pattern.
+     */
+    else if (!strchr(pszPat, '|'))
+    {
+        if (fUpdateRing0)
+            stamR3Ring0StatsUpdateU(pUVM, pszPat);
+
+        STAM_LOCK_RD(pUVM);
+        /** @todo This needs to be optimized since the GUI is using this path for the VM info dialog.
+         * Note that it's doing exact matching. Organizing the samples in a tree would speed up thing
+         * no end (at least for debug and profile builds). */
+        for (PSTAMDESC pCur = pUVM->stam.s.pHead; pCur; pCur = pCur->pNext)
+            if (RTStrSimplePatternMatch(pszPat, pCur->pszName))
+            {
+                rc = pfnCallback(pCur, pvArg);
+                if (rc)
+                    break;
+            }
+        STAM_UNLOCK_RD(pUVM);
+    }
+
+    /*
+     * Multi expression pattern.
+     */
+    else
+    {
+        /*
+         * Split up the pattern first.
+         */
+        char *pszCopy;
+        unsigned cExpressions;
+        char **papszExpressions = stamR3SplitPattern(pszPat, &cExpressions, &pszCopy);
+        if (!papszExpressions)
+            return VERR_NO_MEMORY;
+
+        /*
+         * Perform the enumeration.
+         */
+        if (fUpdateRing0)
+            stamR3Ring0StatsUpdateMultiU(pUVM, papszExpressions, cExpressions);
+
+        STAM_LOCK_RD(pUVM);
+        unsigned iExpression = 0;
+        for (PSTAMDESC pCur = pUVM->stam.s.pHead; pCur; pCur = pCur->pNext)
+            if (stamR3MultiMatch(papszExpressions, cExpressions, &iExpression, pCur->pszName))
+            {
+                rc = pfnCallback(pCur, pvArg);
+                if (rc)
+                    break;
+            }
+        STAM_UNLOCK_RD(pUVM);
+
+        RTMemTmpFree(papszExpressions);
+        RTStrFree(pszCopy);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Registers the ring-0 statistics.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ */
+static void stamR3Ring0StatsRegisterU(PUVM pUVM)
+{
+    /* GVMM */
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
+        stamR3RegisterU(pUVM, (uint8_t *)&pUVM->stam.s.GVMMStats + g_aGVMMStats[i].offVar, NULL, NULL,
+                        g_aGVMMStats[i].enmType, STAMVISIBILITY_ALWAYS, g_aGVMMStats[i].pszName,
+                        g_aGVMMStats[i].enmUnit, g_aGVMMStats[i].pszDesc);
+    pUVM->stam.s.cRegisteredHostCpus = 0;
+}
+
+
+/**
+ * Updates the ring-0 statistics (the copy).
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPat      The pattern.
+ */
+static void stamR3Ring0StatsUpdateU(PUVM pUVM, const char *pszPat)
+{
+    stamR3Ring0StatsUpdateMultiU(pUVM, &pszPat, 1);
+}
+
+
+/**
+ * Updates the ring-0 statistics.
+ *
+ * The ring-0 statistics aren't directly addressable from ring-3 and must be
+ * copied when needed.
+ *
+ * @param   pUVM        Pointer to the user mode VM structure.
+ * @param   pszPat      The pattern (for knowing when to skip).
+ */
+static void stamR3Ring0StatsUpdateMultiU(PUVM pUVM, const char * const *papszExpressions, unsigned cExpressions)
+{
+    PVM pVM = pUVM->pVM;
+    if (!pVM || !pVM->pSession)
+        return;
+
+    /* GVMM */
+    bool fUpdate = false;
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aGVMMStats); i++)
+        if (stamR3MultiMatch(papszExpressions, cExpressions, NULL, g_aGVMMStats[i].pszName))
+        {
+            fUpdate = true;
+            break;
+        }
+    if (!fUpdate)
+    {
+        /** @todo check the cpu leaves - rainy day.   */
+    }
+    if (fUpdate)
+    {
+        GVMMQUERYSTATISTICSSREQ Req;
+        Req.Hdr.cbReq = sizeof(Req);
+        Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
+        Req.pSession = pVM->pSession;
+        int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, NIL_VMCPUID, VMMR0_DO_GVMM_QUERY_STATISTICS, 0, &Req.Hdr);
+        if (RT_SUCCESS(rc))
+        {
+            pUVM->stam.s.GVMMStats = Req.Stats;
+
+            /*
+             * Check if the number of host CPUs has changed (it will the first
+             * time around and normally never again).
+             */
+            if (RT_UNLIKELY(pUVM->stam.s.GVMMStats.cHostCpus > pUVM->stam.s.cRegisteredHostCpus))
+            {
+                STAM_LOCK_WR(pUVM);
+                if (RT_UNLIKELY(pUVM->stam.s.GVMMStats.cHostCpus > pUVM->stam.s.cRegisteredHostCpus))
+                {
+                    uint32_t cCpus = pUVM->stam.s.GVMMStats.cHostCpus;
+                    for (uint32_t iCpu  = pUVM->stam.s.cRegisteredHostCpus; iCpu < cCpus; iCpu++)
+                    {
+                        char   szName[120];
+                        size_t cchBase = RTStrPrintf(szName, sizeof(szName), "/GVMM/HostCpus/%u", iCpu);
+                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].idCpu, NULL, NULL,
+                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE, "Host CPU ID");
+                        strcpy(&szName[cchBase], "/idxCpuSet");
+                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].idxCpuSet, NULL, NULL,
+                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_NONE, "CPU Set index");
+                        strcpy(&szName[cchBase], "/DesiredHz");
+                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].uDesiredHz, NULL, NULL,
+                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_HZ, "The desired frequency");
+                        strcpy(&szName[cchBase], "/CurTimerHz");
+                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].uTimerHz, NULL, NULL,
+                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_HZ, "The current timer frequency");
+                        strcpy(&szName[cchBase], "/PPTChanges");
+                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].cChanges, NULL, NULL,
+                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_OCCURENCES, "RTTimerChangeInterval calls");
+                        strcpy(&szName[cchBase], "/PPTStarts");
+                        stamR3RegisterU(pUVM, &pUVM->stam.s.GVMMStats.aHostCpus[iCpu].cStarts, NULL, NULL,
+                                        STAMTYPE_U32, STAMVISIBILITY_ALWAYS, szName, STAMUNIT_OCCURENCES, "RTTimerStart calls");
+                    }
+                    pUVM->stam.s.cRegisteredHostCpus = cCpus;
+                }
+                STAM_UNLOCK_WR(pUVM);
+            }
+        }
+    }
+}
+
+
+/**
+ * Get the unit string.
+ *
+ * @returns Pointer to read only unit string.
+ * @param   enmUnit     The unit.
+ */
+VMMR3DECL(const char *) STAMR3GetUnit(STAMUNIT enmUnit)
+{
+    switch (enmUnit)
+    {
+        case STAMUNIT_NONE:                 return "";
+        case STAMUNIT_CALLS:                return "calls";
+        case STAMUNIT_COUNT:                return "count";
+        case STAMUNIT_BYTES:                return "bytes";
+        case STAMUNIT_PAGES:                return "pages";
+        case STAMUNIT_ERRORS:               return "errors";
+        case STAMUNIT_OCCURENCES:           return "times";
+        case STAMUNIT_TICKS:                return "ticks";
+        case STAMUNIT_TICKS_PER_CALL:       return "ticks/call";
+        case STAMUNIT_TICKS_PER_OCCURENCE:  return "ticks/time";
+        case STAMUNIT_GOOD_BAD:             return "good:bad";
+        case STAMUNIT_MEGABYTES:            return "megabytes";
+        case STAMUNIT_KILOBYTES:            return "kilobytes";
+        case STAMUNIT_NS:                   return "ns";
+        case STAMUNIT_NS_PER_CALL:          return "ns/call";
+        case STAMUNIT_NS_PER_OCCURENCE:     return "ns/time";
+        case STAMUNIT_PCT:                  return "%";
+        case STAMUNIT_HZ:                   return "Hz";
+
+        default:
+            AssertMsgFailed(("Unknown unit %d\n", enmUnit));
+            return "(?unit?)";
+    }
+}
+
+#ifdef VBOX_WITH_DEBUGGER
+
+/**
+ * The '.stats' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) stamR3CmdStats(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires VM to be selected.\n");
+    PUVM pUVM = pVM->pUVM;
+    if (!pUVM->stam.s.pHead)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no statistics present.\n");
+
+    /*
+     * Do the printing.
+     */
+    STAMR3PRINTONEARGS Args;
+    Args.pVM = pVM;
+    Args.pvArg = pCmdHlp;
+    Args.pfnPrintf = stamR3EnumDbgfPrintf;
+
+    return stamR3EnumU(pUVM, cArgs ? paArgs[0].u.pszString : NULL, true /* fUpdateRing0 */, stamR3PrintOne, &Args);
+}
+
+
+/**
+ * Display one sample in the debugger.
+ *
+ * @param   pArgs       Pointer to the print one argument structure.
+ * @param   pszFormat   Format string.
+ * @param   ...         Format arguments.
+ */
+static DECLCALLBACK(void) stamR3EnumDbgfPrintf(PSTAMR3PRINTONEARGS pArgs, const char *pszFormat, ...)
+{
+    PDBGCCMDHLP pCmdHlp = (PDBGCCMDHLP)pArgs->pvArg;
+
+    va_list va;
+    va_start(va, pszFormat);
+    pCmdHlp->pfnPrintfV(pCmdHlp, NULL, pszFormat, va);
+    va_end(va);
+    NOREF(pArgs);
+}
+
+
+/**
+ * The '.statsreset' command.
+ *
+ * @returns VBox status.
+ * @param   pCmd        Pointer to the command descriptor (as registered).
+ * @param   pCmdHlp     Pointer to command helper functions.
+ * @param   pVM         Pointer to the current VM (if any).
+ * @param   paArgs      Pointer to (readonly) array of arguments.
+ * @param   cArgs       Number of arguments in the array.
+ */
+static DECLCALLBACK(int) stamR3CmdStatsReset(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult)
+{
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "error: The command requires VM to be selected.\n");
+    PUVM pUVM = pVM->pUVM;
+    if (!pUVM->stam.s.pHead)
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "Sorry, no statistics present.\n");
+
+    /*
+     * Execute reset.
+     */
+    int rc = STAMR3ResetU(pUVM, cArgs ? paArgs[0].u.pszString : NULL);
+    if (RT_SUCCESS(rc))
+        return pCmdHlp->pfnPrintf(pCmdHlp, NULL, "info: Statistics reset.\n");
+
+    return pCmdHlp->pfnVBoxError(pCmdHlp, rc, "Resetting statistics.\n");
+}
+
+#endif /* VBOX_WITH_DEBUGGER */
+
Index: /trunk/src/VBox/VMM/VMMR3/TM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/TM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/TM.cpp	(revision 35333)
@@ -0,0 +1,3021 @@
+/* $Id$ */
+/** @file
+ * TM - Time Manager.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_tm        TM - The Time Manager
+ *
+ * The Time Manager abstracts the CPU clocks and manages timers used by the VMM,
+ * device and drivers.
+ *
+ * @see grp_tm
+ *
+ *
+ * @section sec_tm_clocks   Clocks
+ *
+ * There are currently 4 clocks:
+ *   - Virtual (guest).
+ *   - Synchronous virtual (guest).
+ *   - CPU Tick (TSC) (guest).  Only current use is rdtsc emulation. Usually a
+ *     function of the virtual clock.
+ *   - Real (host).  This is only used for display updates atm.
+ *
+ * The most important clocks are the three first ones and of these the second is
+ * the most interesting.
+ *
+ *
+ * The synchronous virtual clock is tied to the virtual clock except that it
+ * will take into account timer delivery lag caused by host scheduling.  It will
+ * normally never advance beyond the head timer, and when lagging too far behind
+ * it will gradually speed up to catch up with the virtual clock.  All devices
+ * implementing time sources accessible to and used by the guest is using this
+ * clock (for timers and other things).  This ensures consistency between the
+ * time sources.
+ *
+ * The virtual clock is implemented as an offset to a monotonic, high
+ * resolution, wall clock.  The current time source is using the RTTimeNanoTS()
+ * machinery based upon the Global Info Pages (GIP), that is, we're using TSC
+ * deltas (usually 10 ms) to fill the gaps between GIP updates.  The result is
+ * a fairly high res clock that works in all contexts and on all hosts.  The
+ * virtual clock is paused when the VM isn't in the running state.
+ *
+ * The CPU tick (TSC) is normally virtualized as a function of the synchronous
+ * virtual clock, where the frequency defaults to the host cpu frequency (as we
+ * measure it). In this mode it is possible to configure the frequency. Another
+ * (non-default) option is to use the raw unmodified host TSC values. And yet
+ * another, to tie it to time spent executing guest code.  All these things are
+ * configurable should non-default behavior be desirable.
+ *
+ * The real clock is a monotonic clock (when available) with relatively low
+ * resolution, though this a bit host specific.  Note that we're currently not
+ * servicing timers using the real clock when the VM is not running, this is
+ * simply because it has not been needed yet therefore not implemented.
+ *
+ *
+ * @subsection subsec_tm_timesync Guest Time Sync / UTC time
+ *
+ * Guest time syncing is primarily taken care of by the VMM device.  The
+ * principle is very simple, the guest additions periodically asks the VMM
+ * device what the current UTC time is and makes adjustments accordingly.
+ *
+ * A complicating factor is that the synchronous virtual clock might be doing
+ * catchups and the guest perception is currently a little bit behind the world
+ * but it will (hopefully) be catching up soon as we're feeding timer interrupts
+ * at a slightly higher rate.  Adjusting the guest clock to the current wall
+ * time in the real world would be a bad idea then because the guest will be
+ * advancing too fast and run ahead of world time (if the catchup works out).
+ * To solve this problem TM provides the VMM device with an UTC time source that
+ * gets adjusted with the current lag, so that when the guest eventually catches
+ * up the lag it will be showing correct real world time.
+ *
+ *
+ * @section sec_tm_timers   Timers
+ *
+ * The timers can use any of the TM clocks described in the previous section.
+ * Each clock has its own scheduling facility, or timer queue if you like.
+ * There are a few factors which makes it a bit complex.  First, there is the
+ * usual R0 vs R3 vs. RC thing.  Then there are multiple threads, and then there
+ * is the timer thread that periodically checks whether any timers has expired
+ * without EMT noticing.  On the API level, all but the create and save APIs
+ * must be multithreaded.  EMT will always run the timers.
+ *
+ * The design is using a doubly linked list of active timers which is ordered
+ * by expire date.  This list is only modified by the EMT thread.  Updates to
+ * the list are batched in a singly linked list, which is then processed by the
+ * EMT thread at the first opportunity (immediately, next time EMT modifies a
+ * timer on that clock, or next timer timeout).  Both lists are offset based and
+ * all the elements are therefore allocated from the hyper heap.
+ *
+ * For figuring out when there is need to schedule and run timers TM will:
+ *    - Poll whenever somebody queries the virtual clock.
+ *    - Poll the virtual clocks from the EM and REM loops.
+ *    - Poll the virtual clocks from trap exit path.
+ *    - Poll the virtual clocks and calculate first timeout from the halt loop.
+ *    - Employ a thread which periodically (100Hz) polls all the timer queues.
+ *
+ *
+ * @image html TMTIMER-Statechart-Diagram.gif
+ *
+ * @section sec_tm_timer    Logging
+ *
+ * Level 2: Logs a most of the timer state transitions and queue servicing.
+ * Level 3: Logs a few oddments.
+ * Level 4: Logs TMCLOCK_VIRTUAL_SYNC catch-up events.
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#include <VBox/tm.h>
+#include <VBox/vmm.h>
+#include <VBox/mm.h>
+#include <VBox/ssm.h>
+#include <VBox/dbgf.h>
+#include <VBox/rem.h>
+#include <VBox/pdmapi.h>
+#include <VBox/iom.h>
+#include "TMInternal.h"
+#include <VBox/vm.h>
+
+#include <VBox/pdmdev.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/assert.h>
+#include <iprt/thread.h>
+#include <iprt/time.h>
+#include <iprt/timer.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/env.h>
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The current saved state version.*/
+#define TM_SAVED_STATE_VERSION  3
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static bool                 tmR3HasFixedTSC(PVM pVM);
+static uint64_t             tmR3CalibrateTSC(PVM pVM);
+static DECLCALLBACK(int)    tmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)    tmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(void)   tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t iTick);
+static void                 tmR3TimerQueueRun(PVM pVM, PTMTIMERQUEUE pQueue);
+static void                 tmR3TimerQueueRunVirtualSync(PVM pVM);
+static DECLCALLBACK(int)    tmR3SetWarpDrive(PVM pVM, uint32_t u32Percent);
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+static DECLCALLBACK(void)   tmR3CpuLoadTimer(PVM pVM, PTMTIMER pTimer, void *pvUser);
+#endif
+static DECLCALLBACK(void)   tmR3TimerInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void)   tmR3TimerInfoActive(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+static DECLCALLBACK(void)   tmR3InfoClocks(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/**
+ * Initializes the TM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMM_INT_DECL(int) TMR3Init(PVM pVM)
+{
+    LogFlow(("TMR3Init:\n"));
+
+    /*
+     * Assert alignment and sizes.
+     */
+    AssertCompileMemberAlignment(VM, tm.s, 32);
+    AssertCompile(sizeof(pVM->tm.s) <= sizeof(pVM->tm.padding));
+    AssertCompileMemberAlignment(TM, TimerCritSect, 8);
+    AssertCompileMemberAlignment(TM, VirtualSyncLock, 8);
+
+    /*
+     * Init the structure.
+     */
+    void *pv;
+    int rc = MMHyperAlloc(pVM, sizeof(pVM->tm.s.paTimerQueuesR3[0]) * TMCLOCK_MAX, 0, MM_TAG_TM, &pv);
+    AssertRCReturn(rc, rc);
+    pVM->tm.s.paTimerQueuesR3 = (PTMTIMERQUEUE)pv;
+    pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pv);
+    pVM->tm.s.paTimerQueuesRC = MMHyperR3ToRC(pVM, pv);
+
+    pVM->tm.s.offVM = RT_OFFSETOF(VM, tm.s);
+    pVM->tm.s.idTimerCpu = pVM->cCpus - 1; /* The last CPU. */
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].enmClock        = TMCLOCK_VIRTUAL;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].u64Expire       = INT64_MAX;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].enmClock   = TMCLOCK_VIRTUAL_SYNC;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].u64Expire  = INT64_MAX;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].enmClock           = TMCLOCK_REAL;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].u64Expire          = INT64_MAX;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].enmClock            = TMCLOCK_TSC;
+    pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].u64Expire           = INT64_MAX;
+
+
+    /*
+     * We directly use the GIP to calculate the virtual time. We map the
+     * the GIP into the guest context so we can do this calculation there
+     * as well and save costly world switches.
+     */
+    pVM->tm.s.pvGIPR3 = (void *)g_pSUPGlobalInfoPage;
+    AssertMsgReturn(pVM->tm.s.pvGIPR3, ("GIP support is now required!\n"), VERR_INTERNAL_ERROR);
+    RTHCPHYS HCPhysGIP;
+    rc = SUPR3GipGetPhys(&HCPhysGIP);
+    AssertMsgRCReturn(rc, ("Failed to get GIP physical address!\n"), rc);
+
+    RTGCPTR GCPtr;
+    rc = MMR3HyperMapHCPhys(pVM, pVM->tm.s.pvGIPR3, NIL_RTR0PTR, HCPhysGIP, PAGE_SIZE, "GIP", &GCPtr);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to map GIP into GC, rc=%Rrc!\n", rc));
+        return rc;
+    }
+    pVM->tm.s.pvGIPRC = GCPtr;
+    LogFlow(("TMR3Init: HCPhysGIP=%RHp at %RRv\n", HCPhysGIP, pVM->tm.s.pvGIPRC));
+    MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+
+    /* Check assumptions made in TMAllVirtual.cpp about the GIP update interval. */
+    if (    g_pSUPGlobalInfoPage->u32Magic == SUPGLOBALINFOPAGE_MAGIC
+        &&  g_pSUPGlobalInfoPage->u32UpdateIntervalNS >= 250000000 /* 0.25s */)
+        return VMSetError(pVM, VERR_INTERNAL_ERROR, RT_SRC_POS,
+                          N_("The GIP update interval is too big. u32UpdateIntervalNS=%RU32 (u32UpdateHz=%RU32)"),
+                          g_pSUPGlobalInfoPage->u32UpdateIntervalNS, g_pSUPGlobalInfoPage->u32UpdateHz);
+    LogRel(("TM: GIP - u32Mode=%d (%s) u32UpdateHz=%u\n", g_pSUPGlobalInfoPage->u32Mode,
+            g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_SYNC_TSC ? "SyncTSC"
+            : g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_ASYNC_TSC ? "AsyncTSC" : "Unknown",
+            g_pSUPGlobalInfoPage->u32UpdateHz));
+
+    /*
+     * Setup the VirtualGetRaw backend.
+     */
+    pVM->tm.s.VirtualGetRawDataR3.pu64Prev = &pVM->tm.s.u64VirtualRawPrev;
+    pVM->tm.s.VirtualGetRawDataR3.pfnBad = tmVirtualNanoTSBad;
+    pVM->tm.s.VirtualGetRawDataR3.pfnRediscover = tmVirtualNanoTSRediscover;
+    if (ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_SSE2)
+    {
+        if (g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_SYNC_TSC)
+            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLFenceSync;
+        else
+            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLFenceAsync;
+    }
+    else
+    {
+        if (g_pSUPGlobalInfoPage->u32Mode == SUPGIPMODE_SYNC_TSC)
+            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLegacySync;
+        else
+            pVM->tm.s.pfnVirtualGetRawR3 = RTTimeNanoTSLegacyAsync;
+    }
+
+    pVM->tm.s.VirtualGetRawDataRC.pu64Prev = MMHyperR3ToRC(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
+    pVM->tm.s.VirtualGetRawDataR0.pu64Prev = MMHyperR3ToR0(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
+    AssertReturn(pVM->tm.s.VirtualGetRawDataR0.pu64Prev, VERR_INTERNAL_ERROR);
+    /* The rest is done in TMR3InitFinalize since it's too early to call PDM. */
+
+    /*
+     * Init the locks.
+     */
+    rc = PDMR3CritSectInit(pVM, &pVM->tm.s.TimerCritSect, RT_SRC_POS, "TM Timer Lock");
+    if (RT_FAILURE(rc))
+        return rc;
+    rc = PDMR3CritSectInit(pVM, &pVM->tm.s.VirtualSyncLock, RT_SRC_POS, "TM VirtualSync Lock");
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Get our CFGM node, create it if necessary.
+     */
+    PCFGMNODE pCfgHandle = CFGMR3GetChild(CFGMR3GetRoot(pVM), "TM");
+    if (!pCfgHandle)
+    {
+        rc = CFGMR3InsertNode(CFGMR3GetRoot(pVM), "TM", &pCfgHandle);
+        AssertRCReturn(rc, rc);
+    }
+
+    /*
+     * Determine the TSC configuration and frequency.
+     */
+    /* mode */
+    /** @cfgm{/TM/TSCVirtualized,bool,true}
+     * Use a virtualize TSC, i.e. trap all TSC access. */
+    rc = CFGMR3QueryBool(pCfgHandle, "TSCVirtualized", &pVM->tm.s.fTSCVirtualized);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.fTSCVirtualized = true; /* trap rdtsc */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying bool value \"UseRealTSC\""));
+
+    /* source */
+    /** @cfgm{/TM/UseRealTSC,bool,false}
+     * Use the real TSC as time source for the TSC instead of the synchronous
+     * virtual clock (false, default). */
+    rc = CFGMR3QueryBool(pCfgHandle, "UseRealTSC", &pVM->tm.s.fTSCUseRealTSC);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.fTSCUseRealTSC = false; /* use virtual time */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying bool value \"UseRealTSC\""));
+    if (!pVM->tm.s.fTSCUseRealTSC)
+        pVM->tm.s.fTSCVirtualized = true;
+
+    /* TSC reliability */
+    /** @cfgm{/TM/MaybeUseOffsettedHostTSC,bool,detect}
+     * Whether the CPU has a fixed TSC rate and may be used in offsetted mode with
+     * VT-x/AMD-V execution.  This is autodetected in a very restrictive way by
+     * default. */
+    rc = CFGMR3QueryBool(pCfgHandle, "MaybeUseOffsettedHostTSC", &pVM->tm.s.fMaybeUseOffsettedHostTSC);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+    {
+        if (!pVM->tm.s.fTSCUseRealTSC)
+            pVM->tm.s.fMaybeUseOffsettedHostTSC = tmR3HasFixedTSC(pVM);
+        else
+            pVM->tm.s.fMaybeUseOffsettedHostTSC = true;
+    }
+
+    /** @cfgm{TM/TSCTicksPerSecond, uint32_t, Current TSC frequency from GIP}
+     * The number of TSC ticks per second (i.e. the TSC frequency). This will
+     * override TSCUseRealTSC, TSCVirtualized and MaybeUseOffsettedHostTSC.
+     */
+    rc = CFGMR3QueryU64(pCfgHandle, "TSCTicksPerSecond", &pVM->tm.s.cTSCTicksPerSecond);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+    {
+        pVM->tm.s.cTSCTicksPerSecond = tmR3CalibrateTSC(pVM);
+        if (    !pVM->tm.s.fTSCUseRealTSC
+            &&  pVM->tm.s.cTSCTicksPerSecond >= _4G)
+        {
+            pVM->tm.s.cTSCTicksPerSecond = _4G - 1; /* (A limitation of our math code) */
+            pVM->tm.s.fMaybeUseOffsettedHostTSC = false;
+        }
+    }
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint64_t value \"TSCTicksPerSecond\""));
+    else if (   pVM->tm.s.cTSCTicksPerSecond < _1M
+             || pVM->tm.s.cTSCTicksPerSecond >= _4G)
+        return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                          N_("Configuration error: \"TSCTicksPerSecond\" = %RI64 is not in the range 1MHz..4GHz-1"),
+                          pVM->tm.s.cTSCTicksPerSecond);
+    else
+    {
+        pVM->tm.s.fTSCUseRealTSC = pVM->tm.s.fMaybeUseOffsettedHostTSC = false;
+        pVM->tm.s.fTSCVirtualized = true;
+    }
+
+    /** @cfgm{TM/TSCTiedToExecution, bool, false}
+     * Whether the TSC should be tied to execution. This will exclude most of the
+     * virtualization overhead, but will by default include the time spent in the
+     * halt state (see TM/TSCNotTiedToHalt). This setting will override all other
+     * TSC settings except for TSCTicksPerSecond and TSCNotTiedToHalt, which should
+     * be used avoided or used with great care. Note that this will only work right
+     * together with VT-x or AMD-V, and with a single virtual CPU. */
+    rc = CFGMR3QueryBoolDef(pCfgHandle, "TSCTiedToExecution", &pVM->tm.s.fTSCTiedToExecution, false);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying bool value \"TSCTiedToExecution\""));
+    if (pVM->tm.s.fTSCTiedToExecution)
+    {
+        /* tied to execution, override all other settings. */
+        pVM->tm.s.fTSCVirtualized = true;
+        pVM->tm.s.fTSCUseRealTSC = true;
+        pVM->tm.s.fMaybeUseOffsettedHostTSC = false;
+    }
+
+    /** @cfgm{TM/TSCNotTiedToHalt, bool, true}
+     * For overriding the default of TM/TSCTiedToExecution, i.e. set this to false
+     * to make the TSC freeze during HLT. */
+    rc = CFGMR3QueryBoolDef(pCfgHandle, "TSCNotTiedToHalt", &pVM->tm.s.fTSCNotTiedToHalt, false);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying bool value \"TSCNotTiedToHalt\""));
+
+    /* setup and report */
+    if (pVM->tm.s.fTSCVirtualized)
+        CPUMR3SetCR4Feature(pVM, X86_CR4_TSD, ~X86_CR4_TSD);
+    else
+        CPUMR3SetCR4Feature(pVM, 0, ~X86_CR4_TSD);
+    LogRel(("TM: cTSCTicksPerSecond=%#RX64 (%'RU64) fTSCVirtualized=%RTbool fTSCUseRealTSC=%RTbool\n"
+            "TM: fMaybeUseOffsettedHostTSC=%RTbool TSCTiedToExecution=%RTbool TSCNotTiedToHalt=%RTbool\n",
+            pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.fTSCVirtualized, pVM->tm.s.fTSCUseRealTSC,
+            pVM->tm.s.fMaybeUseOffsettedHostTSC, pVM->tm.s.fTSCTiedToExecution, pVM->tm.s.fTSCNotTiedToHalt));
+
+    /*
+     * Configure the timer synchronous virtual time.
+     */
+    /** @cfgm{TM/ScheduleSlack, uint32_t, ns, 0, UINT32_MAX, 100000}
+     * Scheduling slack when processing timers. */
+    rc = CFGMR3QueryU32(pCfgHandle, "ScheduleSlack", &pVM->tm.s.u32VirtualSyncScheduleSlack);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u32VirtualSyncScheduleSlack           =   100000; /* 0.100ms (ASSUMES virtual time is nanoseconds) */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 32-bit integer value \"ScheduleSlack\""));
+
+    /** @cfgm{TM/CatchUpStopThreshold, uint64_t, ns, 0, UINT64_MAX, 500000}
+     * When to stop a catch-up, considering it successful. */
+    rc = CFGMR3QueryU64(pCfgHandle, "CatchUpStopThreshold", &pVM->tm.s.u64VirtualSyncCatchUpStopThreshold);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u64VirtualSyncCatchUpStopThreshold    =   500000; /* 0.5ms */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpStopThreshold\""));
+
+    /** @cfgm{TM/CatchUpGiveUpThreshold, uint64_t, ns, 0, UINT64_MAX, 60000000000}
+     * When to give up a catch-up attempt. */
+    rc = CFGMR3QueryU64(pCfgHandle, "CatchUpGiveUpThreshold", &pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold  = UINT64_C(60000000000); /* 60 sec */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpGiveUpThreshold\""));
+
+
+    /** @cfgm{TM/CatchUpPrecentage[0..9], uint32_t, %, 1, 2000, various}
+     * The catch-up percent for a given period.  */
+    /** @cfgm{TM/CatchUpStartThreshold[0..9], uint64_t, ns, 0, UINT64_MAX,
+     * The catch-up period threshold, or if you like, when a period starts.  */
+#define TM_CFG_PERIOD(iPeriod, DefStart, DefPct) \
+    do \
+    { \
+        uint64_t u64; \
+        rc = CFGMR3QueryU64(pCfgHandle, "CatchUpStartThreshold" #iPeriod, &u64); \
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND) \
+            u64 = UINT64_C(DefStart); \
+        else if (RT_FAILURE(rc)) \
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying 64-bit integer value \"CatchUpThreshold" #iPeriod "\"")); \
+        if (    (iPeriod > 0 && u64 <= pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod - 1].u64Start) \
+            ||  u64 >= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold) \
+            return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, N_("Configuration error: Invalid start of period #" #iPeriod ": %'RU64"), u64); \
+        pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u64Start = u64; \
+        rc = CFGMR3QueryU32(pCfgHandle, "CatchUpPrecentage" #iPeriod, &pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u32Percentage); \
+        if (rc == VERR_CFGM_VALUE_NOT_FOUND) \
+            pVM->tm.s.aVirtualSyncCatchUpPeriods[iPeriod].u32Percentage = (DefPct); \
+        else if (RT_FAILURE(rc)) \
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("Configuration error: Failed to querying 32-bit integer value \"CatchUpPrecentage" #iPeriod "\"")); \
+    } while (0)
+    /* This needs more tuning. Not sure if we really need so many period and be so gentle. */
+    TM_CFG_PERIOD(0,     750000,   5); /* 0.75ms at 1.05x */
+    TM_CFG_PERIOD(1,    1500000,  10); /* 1.50ms at 1.10x */
+    TM_CFG_PERIOD(2,    8000000,  25); /*    8ms at 1.25x */
+    TM_CFG_PERIOD(3,   30000000,  50); /*   30ms at 1.50x */
+    TM_CFG_PERIOD(4,   75000000,  75); /*   75ms at 1.75x */
+    TM_CFG_PERIOD(5,  175000000, 100); /*  175ms at 2x */
+    TM_CFG_PERIOD(6,  500000000, 200); /*  500ms at 3x */
+    TM_CFG_PERIOD(7, 3000000000, 300); /*    3s  at 4x */
+    TM_CFG_PERIOD(8,30000000000, 400); /*   30s  at 5x */
+    TM_CFG_PERIOD(9,55000000000, 500); /*   55s  at 6x */
+    AssertCompile(RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods) == 10);
+#undef TM_CFG_PERIOD
+
+    /*
+     * Configure real world time (UTC).
+     */
+    /** @cfgm{TM/UTCOffset, int64_t, ns, INT64_MIN, INT64_MAX, 0}
+     * The UTC offset. This is used to put the guest back or forwards in time.  */
+    rc = CFGMR3QueryS64(pCfgHandle, "UTCOffset", &pVM->tm.s.offUTC);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.offUTC = 0; /* ns */
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying 64-bit integer value \"UTCOffset\""));
+
+    /*
+     * Setup the warp drive.
+     */
+    /** @cfgm{TM/WarpDrivePercentage, uint32_t, %, 0, 20000, 100}
+     * The warp drive percentage, 100% is normal speed.  This is used to speed up
+     * or slow down the virtual clock, which can be useful for fast forwarding
+     * borring periods during tests. */
+    rc = CFGMR3QueryU32(pCfgHandle, "WarpDrivePercentage", &pVM->tm.s.u32VirtualWarpDrivePercentage);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        rc = CFGMR3QueryU32(CFGMR3GetRoot(pVM), "WarpDrivePercentage", &pVM->tm.s.u32VirtualWarpDrivePercentage); /* legacy */
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        pVM->tm.s.u32VirtualWarpDrivePercentage = 100;
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"WarpDrivePercent\""));
+    else if (   pVM->tm.s.u32VirtualWarpDrivePercentage < 2
+             || pVM->tm.s.u32VirtualWarpDrivePercentage > 20000)
+        return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS,
+                          N_("Configuration error: \"WarpDrivePercent\" = %RI32 is not in the range 2..20000"),
+                          pVM->tm.s.u32VirtualWarpDrivePercentage);
+    pVM->tm.s.fVirtualWarpDrive = pVM->tm.s.u32VirtualWarpDrivePercentage != 100;
+    if (pVM->tm.s.fVirtualWarpDrive)
+        LogRel(("TM: u32VirtualWarpDrivePercentage=%RI32\n", pVM->tm.s.u32VirtualWarpDrivePercentage));
+
+    /*
+     * Gather the Host Hz configuration values.
+     */
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzMax", &pVM->tm.s.cHostHzMax, 20000);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzMax\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorTimerCpu", &pVM->tm.s.cPctHostHzFudgeFactorTimerCpu, 111);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorTimerCpu\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorOtherCpu", &pVM->tm.s.cPctHostHzFudgeFactorOtherCpu, 110);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorOtherCpu\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp100", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp100, 300);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp100\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp200", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp200, 250);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp200\""));
+
+    rc = CFGMR3QueryU32Def(pCfgHandle, "HostHzFudgeFactorCatchUp400", &pVM->tm.s.cPctHostHzFudgeFactorCatchUp400, 200);
+    if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to querying uint32_t value \"HostHzFudgeFactorCatchUp400\""));
+
+    /*
+     * Start the timer (guard against REM not yielding).
+     */
+    /** @cfgm{TM/TimerMillies, uint32_t, ms, 1, 1000, 10}
+     * The watchdog timer interval.  */
+    uint32_t u32Millies;
+    rc = CFGMR3QueryU32(pCfgHandle, "TimerMillies", &u32Millies);
+    if (rc == VERR_CFGM_VALUE_NOT_FOUND)
+        u32Millies = 10;
+    else if (RT_FAILURE(rc))
+        return VMSetError(pVM, rc, RT_SRC_POS,
+                          N_("Configuration error: Failed to query uint32_t value \"TimerMillies\""));
+    rc = RTTimerCreate(&pVM->tm.s.pTimer, u32Millies, tmR3TimerCallback, pVM);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgFailed(("Failed to create timer, u32Millies=%d rc=%Rrc.\n", u32Millies, rc));
+        return rc;
+    }
+    Log(("TM: Created timer %p firing every %d milliseconds\n", pVM->tm.s.pTimer, u32Millies));
+    pVM->tm.s.u32TimerMillies = u32Millies;
+
+    /*
+     * Register saved state.
+     */
+    rc = SSMR3RegisterInternal(pVM, "tm", 1, TM_SAVED_STATE_VERSION, sizeof(uint64_t) * 8,
+                               NULL, NULL, NULL,
+                               NULL, tmR3Save, NULL,
+                               NULL, tmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register statistics.
+     */
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR3.c1nsSteps,STAMTYPE_U32, "/TM/R3/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR3.cBadPrev, STAMTYPE_U32, "/TM/R3/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR0.c1nsSteps,STAMTYPE_U32, "/TM/R0/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataR0.cBadPrev, STAMTYPE_U32, "/TM/R0/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataRC.c1nsSteps,STAMTYPE_U32, "/TM/RC/1nsSteps",                     STAMUNIT_OCCURENCES, "Virtual time 1ns steps (due to TSC / GIP variations).");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.VirtualGetRawDataRC.cBadPrev, STAMTYPE_U32, "/TM/RC/cBadPrev",                     STAMUNIT_OCCURENCES, "Times the previous virtual time was considered erratic (shouldn't ever happen).");
+    STAM_REL_REG(     pVM,(void*)&pVM->tm.s.offVirtualSync,               STAMTYPE_U64, "/TM/VirtualSync/CurrentOffset",               STAMUNIT_NS, "The current offset. (subtract GivenUp to get the lag)");
+    STAM_REL_REG_USED(pVM,(void*)&pVM->tm.s.offVirtualSyncGivenUp,        STAMTYPE_U64, "/TM/VirtualSync/GivenUp",                     STAMUNIT_NS, "Nanoseconds of the 'CurrentOffset' that's been given up and won't ever be attempted caught up with.");
+    STAM_REL_REG(     pVM,(void*)&pVM->tm.s.uMaxHzHint,                   STAMTYPE_U32, "/TM/MaxHzHint",                               STAMUNIT_HZ, "Max guest timer frequency hint.");
+
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR3.cExpired,    STAMTYPE_U32, "/TM/R3/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR3.cUpdateRaces,STAMTYPE_U32, "/TM/R3/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR0.cExpired,    STAMTYPE_U32, "/TM/R0/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataR0.cUpdateRaces,STAMTYPE_U32, "/TM/R0/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataRC.cExpired,    STAMTYPE_U32, "/TM/RC/cExpired",                     STAMUNIT_OCCURENCES, "Times the TSC interval expired (overlaps 1ns steps).");
+    STAM_REG_USED(pVM,(void *)&pVM->tm.s.VirtualGetRawDataRC.cUpdateRaces,STAMTYPE_U32, "/TM/RC/cUpdateRaces",                 STAMUNIT_OCCURENCES, "Thread races when updating the previous timestamp.");
+    STAM_REG(pVM, &pVM->tm.s.StatDoQueues,                            STAMTYPE_PROFILE, "/TM/DoQueues",                    STAMUNIT_TICKS_PER_CALL, "Profiling timer TMR3TimerQueuesDo.");
+    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL],      STAMTYPE_PROFILE_ADV, "/TM/DoQueues/Virtual",            STAMUNIT_TICKS_PER_CALL, "Time spent on the virtual clock queue.");
+    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], STAMTYPE_PROFILE_ADV, "/TM/DoQueues/VirtualSync",        STAMUNIT_TICKS_PER_CALL, "Time spent on the virtual sync clock queue.");
+    STAM_REG(pVM, &pVM->tm.s.aStatDoQueues[TMCLOCK_REAL],         STAMTYPE_PROFILE_ADV, "/TM/DoQueues/Real",               STAMUNIT_TICKS_PER_CALL, "Time spent on the real clock queue.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatPoll,                                STAMTYPE_COUNTER, "/TM/Poll",                            STAMUNIT_OCCURENCES, "TMTimerPoll calls.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollAlreadySet,                      STAMTYPE_COUNTER, "/TM/Poll/AlreadySet",                 STAMUNIT_OCCURENCES, "TMTimerPoll calls where the FF was already set.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollELoop,                           STAMTYPE_COUNTER, "/TM/Poll/ELoop",                      STAMUNIT_OCCURENCES, "Times TMTimerPoll has given up getting a consistent virtual sync data set.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollMiss,                            STAMTYPE_COUNTER, "/TM/Poll/Miss",                       STAMUNIT_OCCURENCES, "TMTimerPoll calls where nothing had expired.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollRunning,                         STAMTYPE_COUNTER, "/TM/Poll/Running",                    STAMUNIT_OCCURENCES, "TMTimerPoll calls where the queues were being run.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollSimple,                          STAMTYPE_COUNTER, "/TM/Poll/Simple",                     STAMUNIT_OCCURENCES, "TMTimerPoll calls where we could take the simple path.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollVirtual,                         STAMTYPE_COUNTER, "/TM/Poll/HitsVirtual",                STAMUNIT_OCCURENCES, "The number of times TMTimerPoll found an expired TMCLOCK_VIRTUAL queue.");
+    STAM_REG(pVM, &pVM->tm.s.StatPollVirtualSync,                     STAMTYPE_COUNTER, "/TM/Poll/HitsVirtualSync",            STAMUNIT_OCCURENCES, "The number of times TMTimerPoll found an expired TMCLOCK_VIRTUAL_SYNC queue.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatPostponedR3,                         STAMTYPE_COUNTER, "/TM/PostponedR3",                     STAMUNIT_OCCURENCES, "Postponed due to unschedulable state, in ring-3.");
+    STAM_REG(pVM, &pVM->tm.s.StatPostponedRZ,                         STAMTYPE_COUNTER, "/TM/PostponedRZ",                     STAMUNIT_OCCURENCES, "Postponed due to unschedulable state, in ring-0 / RC.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatScheduleOneR3,                       STAMTYPE_PROFILE, "/TM/ScheduleOneR3",               STAMUNIT_TICKS_PER_CALL, "Profiling the scheduling of one queue during a TMTimer* call in EMT.");
+    STAM_REG(pVM, &pVM->tm.s.StatScheduleOneRZ,                       STAMTYPE_PROFILE, "/TM/ScheduleOneRZ",               STAMUNIT_TICKS_PER_CALL, "Profiling the scheduling of one queue during a TMTimer* call in EMT.");
+    STAM_REG(pVM, &pVM->tm.s.StatScheduleSetFF,                       STAMTYPE_COUNTER, "/TM/ScheduleSetFF",                   STAMUNIT_OCCURENCES, "The number of times the timer FF was set instead of doing scheduling.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSet,                            STAMTYPE_COUNTER, "/TM/TimerSet",                        STAMUNIT_OCCURENCES, "Calls");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetOpt,                         STAMTYPE_COUNTER, "/TM/TimerSet/Opt",                    STAMUNIT_OCCURENCES, "Optimized path taken.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetR3,                          STAMTYPE_PROFILE, "/TM/TimerSet/R3",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-3.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRZ,                          STAMTYPE_PROFILE, "/TM/TimerSet/RZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSet calls made in ring-0 / RC.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStActive,                    STAMTYPE_COUNTER, "/TM/TimerSet/StActive",               STAMUNIT_OCCURENCES, "ACTIVE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStExpDeliver,                STAMTYPE_COUNTER, "/TM/TimerSet/StExpDeliver",           STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStOther,                     STAMTYPE_COUNTER, "/TM/TimerSet/StOther",                STAMUNIT_OCCURENCES, "Other states");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendStop,                  STAMTYPE_COUNTER, "/TM/TimerSet/StPendStop",             STAMUNIT_OCCURENCES, "PENDING_STOP");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendStopSched,             STAMTYPE_COUNTER, "/TM/TimerSet/StPendStopSched",        STAMUNIT_OCCURENCES, "PENDING_STOP_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendSched,                 STAMTYPE_COUNTER, "/TM/TimerSet/StPendSched",            STAMUNIT_OCCURENCES, "PENDING_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStPendResched,               STAMTYPE_COUNTER, "/TM/TimerSet/StPendResched",          STAMUNIT_OCCURENCES, "PENDING_RESCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetStStopped,                   STAMTYPE_COUNTER, "/TM/TimerSet/StStopped",              STAMUNIT_OCCURENCES, "STOPPED");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelative,                    STAMTYPE_COUNTER, "/TM/TimerSetRelative",                STAMUNIT_OCCURENCES, "Calls");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeOpt,                 STAMTYPE_COUNTER, "/TM/TimerSetRelative/Opt",            STAMUNIT_OCCURENCES, "Optimized path taken.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeR3,                  STAMTYPE_PROFILE, "/TM/TimerSetRelative/R3",         STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetRelative calls made in ring-3.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeRZ,                  STAMTYPE_PROFILE, "/TM/TimerSetRelative/RZ",         STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerSetReltaive calls made in ring-0 / RC.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeRacyVirtSync,        STAMTYPE_COUNTER, "/TM/TimerSetRelative/RacyVirtSync",   STAMUNIT_OCCURENCES, "Potentially racy virtual sync timer update.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStActive,            STAMTYPE_COUNTER, "/TM/TimerSetRelative/StActive",       STAMUNIT_OCCURENCES, "ACTIVE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStExpDeliver,        STAMTYPE_COUNTER, "/TM/TimerSetRelative/StExpDeliver",   STAMUNIT_OCCURENCES, "EXPIRED_DELIVER");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStOther,             STAMTYPE_COUNTER, "/TM/TimerSetRelative/StOther",        STAMUNIT_OCCURENCES, "Other states");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendStop,          STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendStop",     STAMUNIT_OCCURENCES, "PENDING_STOP");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendStopSched,     STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendStopSched",STAMUNIT_OCCURENCES, "PENDING_STOP_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendSched,         STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendSched",    STAMUNIT_OCCURENCES, "PENDING_SCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStPendResched,       STAMTYPE_COUNTER, "/TM/TimerSetRelative/StPendResched",  STAMUNIT_OCCURENCES, "PENDING_RESCHEDULE");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerSetRelativeStStopped,           STAMTYPE_COUNTER, "/TM/TimerSetRelative/StStopped",      STAMUNIT_OCCURENCES, "STOPPED");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerStopR3,                         STAMTYPE_PROFILE, "/TM/TimerStopR3",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerStop calls made in ring-3.");
+    STAM_REG(pVM, &pVM->tm.s.StatTimerStopRZ,                         STAMTYPE_PROFILE, "/TM/TimerStopRZ",                 STAMUNIT_TICKS_PER_CALL, "Profiling TMTimerStop calls made in ring-0 / RC.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualGet,                          STAMTYPE_COUNTER, "/TM/VirtualGet",                      STAMUNIT_OCCURENCES, "The number of times TMTimerGet was called when the clock was running.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualGetSetFF,                     STAMTYPE_COUNTER, "/TM/VirtualGetSetFF",                 STAMUNIT_OCCURENCES, "Times we set the FF when calling TMTimerGet.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGet,                      STAMTYPE_COUNTER, "/TM/VirtualSyncGet",                  STAMUNIT_OCCURENCES, "The number of times tmVirtualSyncGetEx was called.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetELoop,                 STAMTYPE_COUNTER, "/TM/VirtualSyncGet/ELoop",            STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx has given up getting a consistent virtual sync data set.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetExpired,               STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Expired",          STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx encountered an expired timer stopping the clock.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetLocked,                STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Locked",           STAMUNIT_OCCURENCES, "Times we successfully acquired the lock in tmVirtualSyncGetEx.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetLockless,              STAMTYPE_COUNTER, "/TM/VirtualSyncGet/Lockless",         STAMUNIT_OCCURENCES, "Times tmVirtualSyncGetEx returned without needing to take the lock.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGetSetFF,                 STAMTYPE_COUNTER, "/TM/VirtualSyncGet/SetFF",            STAMUNIT_OCCURENCES, "Times we set the FF when calling tmVirtualSyncGetEx.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualPause,                        STAMTYPE_COUNTER, "/TM/VirtualPause",                    STAMUNIT_OCCURENCES, "The number of times TMR3TimerPause was called.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualResume,                       STAMTYPE_COUNTER, "/TM/VirtualResume",                   STAMUNIT_OCCURENCES, "The number of times TMR3TimerResume was called.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTimerCallbackSetFF,                  STAMTYPE_COUNTER, "/TM/CallbackSetFF",                   STAMUNIT_OCCURENCES, "The number of times the timer callback set FF.");
+
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE010,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE010",      STAMUNIT_OCCURENCES, "In catch-up mode, 10% or lower.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE025,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE025",      STAMUNIT_OCCURENCES, "In catch-up mode, 25%-11%.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupLE100,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupLE100",      STAMUNIT_OCCURENCES, "In catch-up mode, 100%-26%.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCCatchupOther,                     STAMTYPE_COUNTER, "/TM/TSC/Intercept/CatchupOther",      STAMUNIT_OCCURENCES, "In catch-up mode, > 100%.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCNotFixed,                         STAMTYPE_COUNTER, "/TM/TSC/Intercept/NotFixed",          STAMUNIT_OCCURENCES, "TSC is not fixed, it may run at variable speed.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCNotTicking,                       STAMTYPE_COUNTER, "/TM/TSC/Intercept/NotTicking",        STAMUNIT_OCCURENCES, "TSC is not ticking.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCSyncNotTicking,                   STAMTYPE_COUNTER, "/TM/TSC/Intercept/SyncNotTicking",    STAMUNIT_OCCURENCES, "VirtualSync isn't ticking.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCWarp,                             STAMTYPE_COUNTER, "/TM/TSC/Intercept/Warp",              STAMUNIT_OCCURENCES, "Warpdrive is active.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCSet,                              STAMTYPE_COUNTER, "/TM/TSC/Sets",                        STAMUNIT_OCCURENCES, "Calls to TMCpuTickSet.");
+    STAM_REG(pVM, &pVM->tm.s.StatTSCUnderflow,                        STAMTYPE_COUNTER, "/TM/TSC/Underflow",                   STAMUNIT_OCCURENCES, "TSC underflow; corrected with last seen value .");
+#endif /* VBOX_WITH_STATISTICS */
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.offTSCRawSrc,          STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_TICKS, "TSC offset relative the raw source",           "/TM/TSC/offCPU%u", i);
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsTotal,       STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,               "Resettable: Total CPU run time.",   "/TM/CPU/%02u", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecuting,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code.",    "/TM/CPU/%02u/PrfExecuting", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecLong,    STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - long hauls.",    "/TM/CPU/%02u/PrfExecLong", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecShort,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - short stretches.",    "/TM/CPU/%02u/PrfExecShort", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsExecTiny,    STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent executing guest code - tiny bits.",    "/TM/CPU/%02u/PrfExecTiny", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsHalted,      STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent halted.",                  "/TM/CPU/%02u/PrfHalted", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.StatNsOther,       STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_OCCURENCE, "Resettable: Time spent in the VMM or preempted.", "/TM/CPU/%02u/PrfOther", i);
+# endif
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsTotal,              STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Total CPU run time.",                          "/TM/CPU/%02u/cNsTotal", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsExecuting,          STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent executing guest code.",             "/TM/CPU/%02u/cNsExecuting", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsHalted,             STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent halted.",                           "/TM/CPU/%02u/cNsHalted", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cNsOther,              STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_NS,    "Time spent in the VMM or preempted.",          "/TM/CPU/%02u/cNsOther", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cPeriodsExecuting,     STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Times executed guest code.",                   "/TM/CPU/%02u/cPeriodsExecuting", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.cPeriodsHalted,        STAMTYPE_U64, STAMVISIBILITY_ALWAYS, STAMUNIT_COUNT, "Times halted.",                                "/TM/CPU/%02u/cPeriodsHalted", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.CpuLoad.cPctExecuting, STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent executing guest code recently.",    "/TM/CPU/%02u/pctExecuting", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.CpuLoad.cPctHalted,    STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent halted recently.",                  "/TM/CPU/%02u/pctHalted", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].tm.s.CpuLoad.cPctOther,     STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent in the VMM or preempted recently.", "/TM/CPU/%02u/pctOther", i);
+#endif
+    }
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctExecuting, STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent executing guest code recently.",    "/TM/CPU/pctExecuting");
+    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctHalted,    STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent halted recently.",                  "/TM/CPU/pctHalted");
+    STAMR3RegisterF(pVM, &pVM->tm.s.CpuLoad.cPctOther,     STAMTYPE_U8,  STAMVISIBILITY_ALWAYS, STAMUNIT_PCT,   "Time spent in the VMM or preempted recently.", "/TM/CPU/pctOther");
+#endif
+
+#ifdef VBOX_WITH_STATISTICS
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncCatchup,              STAMTYPE_PROFILE_ADV, "/TM/VirtualSync/CatchUp",    STAMUNIT_TICKS_PER_OCCURENCE, "Counting and measuring the times spent catching up.");
+    STAM_REG(pVM, (void *)&pVM->tm.s.fVirtualSyncCatchUp,                  STAMTYPE_U8, "/TM/VirtualSync/CatchUpActive",             STAMUNIT_NONE, "Catch-Up active indicator.");
+    STAM_REG(pVM, (void *)&pVM->tm.s.u32VirtualSyncCatchUpPercentage,     STAMTYPE_U32, "/TM/VirtualSync/CatchUpPercentage",          STAMUNIT_PCT, "The catch-up percentage. (+100/100 to get clock multiplier)");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncFF,                       STAMTYPE_PROFILE, "/TM/VirtualSync/FF",         STAMUNIT_TICKS_PER_OCCURENCE, "Time spent in TMR3VirtualSyncFF by all but the dedicate timer EMT.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGiveUp,                   STAMTYPE_COUNTER, "/TM/VirtualSync/GiveUp",              STAMUNIT_OCCURENCES, "Times the catch-up was abandoned.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncGiveUpBeforeStarting,     STAMTYPE_COUNTER, "/TM/VirtualSync/GiveUpBeforeStarting",STAMUNIT_OCCURENCES, "Times the catch-up was abandoned before even starting. (Typically debugging++.)");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRun,                      STAMTYPE_COUNTER, "/TM/VirtualSync/Run",                 STAMUNIT_OCCURENCES, "Times the virtual sync timer queue was considered.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunRestart,               STAMTYPE_COUNTER, "/TM/VirtualSync/Run/Restarts",        STAMUNIT_OCCURENCES, "Times the clock was restarted after a run.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunStop,                  STAMTYPE_COUNTER, "/TM/VirtualSync/Run/Stop",            STAMUNIT_OCCURENCES, "Times the clock was stopped when calculating the current time before examining the timers.");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunStoppedAlready,        STAMTYPE_COUNTER, "/TM/VirtualSync/Run/StoppedAlready",  STAMUNIT_OCCURENCES, "Times the clock was already stopped elsewhere (TMVirtualSyncGet).");
+    STAM_REG(pVM, &pVM->tm.s.StatVirtualSyncRunSlack,                 STAMTYPE_PROFILE, "/TM/VirtualSync/Run/Slack",     STAMUNIT_NS_PER_OCCURENCE, "The scheduling slack. (Catch-up handed out when running timers.)");
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods); i++)
+    {
+        STAMR3RegisterF(pVM, &pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage,    STAMTYPE_U32, STAMVISIBILITY_ALWAYS,          STAMUNIT_PCT, "The catch-up percentage.",         "/TM/VirtualSync/Periods/%u", i);
+        STAMR3RegisterF(pVM, &pVM->tm.s.aStatVirtualSyncCatchupAdjust[i],           STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,   STAMUNIT_OCCURENCES, "Times adjusted to this period.",   "/TM/VirtualSync/Periods/%u/Adjust", i);
+        STAMR3RegisterF(pVM, &pVM->tm.s.aStatVirtualSyncCatchupInitial[i],          STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS,   STAMUNIT_OCCURENCES, "Times started in this period.",    "/TM/VirtualSync/Periods/%u/Initial", i);
+        STAMR3RegisterF(pVM, &pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u64Start,         STAMTYPE_U64, STAMVISIBILITY_ALWAYS,           STAMUNIT_NS, "Start of this period (lag).",      "/TM/VirtualSync/Periods/%u/Start", i);
+    }
+#endif /* VBOX_WITH_STATISTICS */
+
+    /*
+     * Register info handlers.
+     */
+    DBGFR3InfoRegisterInternalEx(pVM, "timers",       "Dumps all timers. No arguments.",          tmR3TimerInfo,        DBGFINFO_FLAGS_RUN_ON_EMT);
+    DBGFR3InfoRegisterInternalEx(pVM, "activetimers", "Dumps active all timers. No arguments.",   tmR3TimerInfoActive,  DBGFINFO_FLAGS_RUN_ON_EMT);
+    DBGFR3InfoRegisterInternalEx(pVM, "clocks",       "Display the time of the various clocks.",  tmR3InfoClocks,       DBGFINFO_FLAGS_RUN_ON_EMT);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the host CPU has a fixed TSC frequency.
+ *
+ * @returns true if it has, false if it hasn't.
+ *
+ * @remark  This test doesn't bother with very old CPUs that don't do power
+ *          management or any other stuff that might influence the TSC rate.
+ *          This isn't currently relevant.
+ */
+static bool tmR3HasFixedTSC(PVM pVM)
+{
+    if (ASMHasCpuId())
+    {
+        uint32_t uEAX, uEBX, uECX, uEDX;
+
+        if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_AMD)
+        {
+            /*
+             * AuthenticAMD - Check for APM support and that TscInvariant is set.
+             *
+             * This test isn't correct with respect to fixed/non-fixed TSC and
+             * older models, but this isn't relevant since the result is currently
+             * only used for making a decision on AMD-V models.
+             */
+            ASMCpuId(0x80000000, &uEAX, &uEBX, &uECX, &uEDX);
+            if (uEAX >= 0x80000007)
+            {
+                PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
+
+                ASMCpuId(0x80000007, &uEAX, &uEBX, &uECX, &uEDX);
+                if (   (uEDX & X86_CPUID_AMD_ADVPOWER_EDX_TSCINVAR) /* TscInvariant */
+                    && pGip->u32Mode == SUPGIPMODE_SYNC_TSC /* no fixed tsc if the gip timer is in async mode */)
+                    return true;
+            }
+        }
+        else if (CPUMGetHostCpuVendor(pVM) == CPUMCPUVENDOR_INTEL)
+        {
+            /*
+             * GenuineIntel - Check the model number.
+             *
+             * This test is lacking in the same way and for the same reasons
+             * as the AMD test above.
+             */
+            ASMCpuId(1, &uEAX, &uEBX, &uECX, &uEDX);
+            unsigned uModel  = (uEAX >> 4) & 0x0f;
+            unsigned uFamily = (uEAX >> 8) & 0x0f;
+            if (uFamily == 0x0f)
+                uFamily += (uEAX >> 20) & 0xff;
+            if (uFamily >= 0x06)
+                uModel += ((uEAX >> 16) & 0x0f) << 4;
+            if (    (uFamily == 0x0f /*P4*/     && uModel >= 0x03)
+                ||  (uFamily == 0x06 /*P2/P3*/  && uModel >= 0x0e))
+                return true;
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Calibrate the CPU tick.
+ *
+ * @returns Number of ticks per second.
+ */
+static uint64_t tmR3CalibrateTSC(PVM pVM)
+{
+    /*
+     * Use GIP when available present.
+     */
+    uint64_t    u64Hz;
+    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
+    if (    pGip
+        &&  pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC)
+    {
+        unsigned iCpu = pGip->u32Mode != SUPGIPMODE_ASYNC_TSC ? 0 : ASMGetApicId();
+        if (iCpu >= RT_ELEMENTS(pGip->aCPUs))
+            AssertReleaseMsgFailed(("iCpu=%d - the ApicId is too high. send VBox.log and hardware specs!\n", iCpu));
+        else
+        {
+            if (tmR3HasFixedTSC(pVM))
+                /* Sleep a bit to get a more reliable CpuHz value. */
+                RTThreadSleep(32);
+            else
+            {
+                /* Spin for 40ms to try push up the CPU frequency and get a more reliable CpuHz value. */
+                const uint64_t u64 = RTTimeMilliTS();
+                while ((RTTimeMilliTS() - u64) < 40 /*ms*/)
+                    /* nothing */;
+            }
+
+            pGip = g_pSUPGlobalInfoPage;
+            if (    pGip
+                &&  pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC
+                &&  (u64Hz = pGip->aCPUs[iCpu].u64CpuHz)
+                &&  u64Hz != ~(uint64_t)0)
+                return u64Hz;
+        }
+    }
+
+    /* call this once first to make sure it's initialized. */
+    RTTimeNanoTS();
+
+    /*
+     * Yield the CPU to increase our chances of getting
+     * a correct value.
+     */
+    RTThreadYield();                    /* Try avoid interruptions between TSC and NanoTS samplings. */
+    static const unsigned   s_auSleep[5] = { 50, 30, 30, 40, 40 };
+    uint64_t                au64Samples[5];
+    unsigned                i;
+    for (i = 0; i < RT_ELEMENTS(au64Samples); i++)
+    {
+        RTMSINTERVAL    cMillies;
+        int             cTries   = 5;
+        uint64_t        u64Start = ASMReadTSC();
+        uint64_t        u64End;
+        uint64_t        StartTS  = RTTimeNanoTS();
+        uint64_t        EndTS;
+        do
+        {
+            RTThreadSleep(s_auSleep[i]);
+            u64End = ASMReadTSC();
+            EndTS  = RTTimeNanoTS();
+            cMillies = (RTMSINTERVAL)((EndTS - StartTS + 500000) / 1000000);
+        } while (   cMillies == 0       /* the sleep may be interrupted... */
+                 || (cMillies < 20 && --cTries > 0));
+        uint64_t    u64Diff = u64End - u64Start;
+
+        au64Samples[i] = (u64Diff * 1000) / cMillies;
+        AssertMsg(cTries > 0, ("cMillies=%d i=%d\n", cMillies, i));
+    }
+
+    /*
+     * Discard the highest and lowest results and calculate the average.
+     */
+    unsigned iHigh = 0;
+    unsigned iLow = 0;
+    for (i = 1; i < RT_ELEMENTS(au64Samples); i++)
+    {
+        if (au64Samples[i] < au64Samples[iLow])
+            iLow = i;
+        if (au64Samples[i] > au64Samples[iHigh])
+            iHigh = i;
+    }
+    au64Samples[iLow] = 0;
+    au64Samples[iHigh] = 0;
+
+    u64Hz = au64Samples[0];
+    for (i = 1; i < RT_ELEMENTS(au64Samples); i++)
+        u64Hz += au64Samples[i];
+    u64Hz /= RT_ELEMENTS(au64Samples) - 2;
+
+    return u64Hz;
+}
+
+
+/**
+ * Finalizes the TM initialization.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMM_INT_DECL(int) TMR3InitFinalize(PVM pVM)
+{
+    int rc;
+
+    /*
+     * Resolve symbols.
+     */
+    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSBad",          &pVM->tm.s.VirtualGetRawDataRC.pfnBad);
+    AssertRCReturn(rc, rc);
+    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSRediscover",   &pVM->tm.s.VirtualGetRawDataRC.pfnRediscover);
+    AssertRCReturn(rc, rc);
+    if (pVM->tm.s.pfnVirtualGetRawR3       == RTTimeNanoTSLFenceSync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceSync",  &pVM->tm.s.pfnVirtualGetRawRC);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLFenceAsync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceAsync", &pVM->tm.s.pfnVirtualGetRawRC);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacySync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacySync",  &pVM->tm.s.pfnVirtualGetRawRC);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacyAsync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacyAsync", &pVM->tm.s.pfnVirtualGetRawRC);
+    else
+        AssertFatalFailed();
+    AssertRCReturn(rc, rc);
+
+    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "tmVirtualNanoTSBad",          &pVM->tm.s.VirtualGetRawDataR0.pfnBad);
+    AssertRCReturn(rc, rc);
+    rc = PDMR3LdrGetSymbolR0(pVM, NULL, "tmVirtualNanoTSRediscover",   &pVM->tm.s.VirtualGetRawDataR0.pfnRediscover);
+    AssertRCReturn(rc, rc);
+    if (pVM->tm.s.pfnVirtualGetRawR3       == RTTimeNanoTSLFenceSync)
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLFenceSync",  &pVM->tm.s.pfnVirtualGetRawR0);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLFenceAsync)
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLFenceAsync", &pVM->tm.s.pfnVirtualGetRawR0);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacySync)
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLegacySync",  &pVM->tm.s.pfnVirtualGetRawR0);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacyAsync)
+        rc = PDMR3LdrGetSymbolR0(pVM, NULL, "RTTimeNanoTSLegacyAsync", &pVM->tm.s.pfnVirtualGetRawR0);
+    else
+        AssertFatalFailed();
+    AssertRCReturn(rc, rc);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Create a timer for refreshing the CPU load stats.
+     */
+    PTMTIMER pTimer;
+    rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, tmR3CpuLoadTimer, NULL, "CPU Load Timer", &pTimer);
+    if (RT_SUCCESS(rc))
+        rc = TMTimerSetMillies(pTimer, 1000);
+#endif
+
+    return rc;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * @param   pVM     The VM.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMM_INT_DECL(void) TMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    int rc;
+    LogFlow(("TMR3Relocate\n"));
+
+    pVM->tm.s.pvGIPRC = MMHyperR3ToRC(pVM, pVM->tm.s.pvGIPR3);
+    pVM->tm.s.paTimerQueuesRC = MMHyperR3ToRC(pVM, pVM->tm.s.paTimerQueuesR3);
+    pVM->tm.s.paTimerQueuesR0 = MMHyperR3ToR0(pVM, pVM->tm.s.paTimerQueuesR3);
+
+    pVM->tm.s.VirtualGetRawDataRC.pu64Prev = MMHyperR3ToRC(pVM, (void *)&pVM->tm.s.u64VirtualRawPrev);
+    AssertFatal(pVM->tm.s.VirtualGetRawDataRC.pu64Prev);
+    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSBad",          &pVM->tm.s.VirtualGetRawDataRC.pfnBad);
+    AssertFatalRC(rc);
+    rc = PDMR3LdrGetSymbolRC(pVM, NULL, "tmVirtualNanoTSRediscover",   &pVM->tm.s.VirtualGetRawDataRC.pfnRediscover);
+    AssertFatalRC(rc);
+
+    if (pVM->tm.s.pfnVirtualGetRawR3       == RTTimeNanoTSLFenceSync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceSync",  &pVM->tm.s.pfnVirtualGetRawRC);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLFenceAsync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLFenceAsync", &pVM->tm.s.pfnVirtualGetRawRC);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacySync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacySync",  &pVM->tm.s.pfnVirtualGetRawRC);
+    else if (pVM->tm.s.pfnVirtualGetRawR3  == RTTimeNanoTSLegacyAsync)
+        rc = PDMR3LdrGetSymbolRC(pVM, NULL, "RTTimeNanoTSLegacyAsync", &pVM->tm.s.pfnVirtualGetRawRC);
+    else
+        AssertFatalFailed();
+    AssertFatalRC(rc);
+
+    /*
+     * Iterate the timers updating the pVMRC pointers.
+     */
+    for (PTMTIMER pTimer = pVM->tm.s.pCreated; pTimer; pTimer = pTimer->pBigNext)
+    {
+        pTimer->pVMRC = pVM->pVMRC;
+        pTimer->pVMR0 = pVM->pVMR0;
+    }
+}
+
+
+/**
+ * Terminates the TM.
+ *
+ * Termination means cleaning up and freeing all resources,
+ * the VM it self is at this point powered off or suspended.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMM_INT_DECL(int) TMR3Term(PVM pVM)
+{
+    AssertMsg(pVM->tm.s.offVM, ("bad init order!\n"));
+    if (pVM->tm.s.pTimer)
+    {
+        int rc = RTTimerDestroy(pVM->tm.s.pTimer);
+        AssertRC(rc);
+        pVM->tm.s.pTimer = NULL;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the TM component this means that a rescheduling is preformed,
+ * the FF is cleared and but without running the queues. We'll have to
+ * check if this makes sense or not, but it seems like a good idea now....
+ *
+ * @param   pVM     VM handle.
+ */
+VMM_INT_DECL(void) TMR3Reset(PVM pVM)
+{
+    LogFlow(("TMR3Reset:\n"));
+    VM_ASSERT_EMT(pVM);
+    tmTimerLock(pVM);
+
+    /*
+     * Abort any pending catch up.
+     * This isn't perfect...
+     */
+    if (pVM->tm.s.fVirtualSyncCatchUp)
+    {
+        const uint64_t offVirtualNow = TMVirtualGetNoCheck(pVM);
+        const uint64_t offVirtualSyncNow = TMVirtualSyncGetNoCheck(pVM);
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+            STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+
+            const uint64_t offOld = pVM->tm.s.offVirtualSyncGivenUp;
+            const uint64_t offNew = offVirtualNow - offVirtualSyncNow;
+            Assert(offOld <= offNew);
+            ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
+            ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSync, offNew);
+            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+            LogRel(("TM: Aborting catch-up attempt on reset with a %'RU64 ns lag on reset; new total: %'RU64 ns\n", offNew - offOld, offNew));
+        }
+    }
+
+    /*
+     * Process the queues.
+     */
+    for (int i = 0; i < TMCLOCK_MAX; i++)
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[i]);
+#ifdef VBOX_STRICT
+    tmTimerQueuesSanityChecks(pVM, "TMR3Reset");
+#endif
+
+    PVMCPU pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
+    VMCPU_FF_CLEAR(pVCpuDst, VMCPU_FF_TIMER); /** @todo FIXME: this isn't right. */
+    tmTimerUnlock(pVM);
+}
+
+
+/**
+ * Resolve a builtin RC symbol.
+ * Called by PDM when loading or relocating GC modules.
+ *
+ * @returns VBox status
+ * @param   pVM             VM Handle.
+ * @param   pszSymbol       Symbol to resolve.
+ * @param   pRCPtrValue     Where to store the symbol value.
+ * @remark  This has to     work before TMR3Relocate() is called.
+ */
+VMM_INT_DECL(int) TMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
+{
+    if (!strcmp(pszSymbol, "g_pSUPGlobalInfoPage"))
+        *pRCPtrValue = MMHyperR3ToRC(pVM, &pVM->tm.s.pvGIPRC);
+    //else if (..)
+    else
+        return VERR_SYMBOL_NOT_FOUND;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) tmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("tmR3Save:\n"));
+#ifdef VBOX_STRICT
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+        Assert(!pVCpu->tm.s.fTSCTicking);
+    }
+    Assert(!pVM->tm.s.cVirtualTicking);
+    Assert(!pVM->tm.s.fVirtualSyncTicking);
+#endif
+
+    /*
+     * Save the virtual clocks.
+     */
+    /* the virtual clock. */
+    SSMR3PutU64(pSSM, TMCLOCK_FREQ_VIRTUAL);
+    SSMR3PutU64(pSSM, pVM->tm.s.u64Virtual);
+
+    /* the virtual timer synchronous clock. */
+    SSMR3PutU64(pSSM, pVM->tm.s.u64VirtualSync);
+    SSMR3PutU64(pSSM, pVM->tm.s.offVirtualSync);
+    SSMR3PutU64(pSSM, pVM->tm.s.offVirtualSyncGivenUp);
+    SSMR3PutU64(pSSM, pVM->tm.s.u64VirtualSyncCatchUpPrev);
+    SSMR3PutBool(pSSM, pVM->tm.s.fVirtualSyncCatchUp);
+
+    /* real time clock */
+    SSMR3PutU64(pSSM, TMCLOCK_FREQ_REAL);
+
+    /* the cpu tick clock. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+        SSMR3PutU64(pSSM, TMCpuTickGet(pVCpu));
+    }
+    return SSMR3PutU64(pSSM, pVM->tm.s.cTSCTicksPerSecond);
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) tmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("tmR3Load:\n"));
+
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+#ifdef VBOX_STRICT
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+        Assert(!pVCpu->tm.s.fTSCTicking);
+    }
+    Assert(!pVM->tm.s.cVirtualTicking);
+    Assert(!pVM->tm.s.fVirtualSyncTicking);
+#endif
+
+    /*
+     * Validate version.
+     */
+    if (uVersion != TM_SAVED_STATE_VERSION)
+    {
+        AssertMsgFailed(("tmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Load the virtual clock.
+     */
+    pVM->tm.s.cVirtualTicking = 0;
+    /* the virtual clock. */
+    uint64_t u64Hz;
+    int rc = SSMR3GetU64(pSSM, &u64Hz);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (u64Hz != TMCLOCK_FREQ_VIRTUAL)
+    {
+        AssertMsgFailed(("The virtual clock frequency differs! Saved: %'RU64 Binary: %'RU64\n",
+                         u64Hz, TMCLOCK_FREQ_VIRTUAL));
+        return VERR_SSM_VIRTUAL_CLOCK_HZ;
+    }
+    SSMR3GetU64(pSSM, &pVM->tm.s.u64Virtual);
+    pVM->tm.s.u64VirtualOffset = 0;
+
+    /* the virtual timer synchronous clock. */
+    pVM->tm.s.fVirtualSyncTicking = false;
+    uint64_t u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.u64VirtualSync = u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.offVirtualSync = u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.offVirtualSyncGivenUp = u64;
+    SSMR3GetU64(pSSM, &u64);
+    pVM->tm.s.u64VirtualSyncCatchUpPrev = u64;
+    bool f;
+    SSMR3GetBool(pSSM, &f);
+    pVM->tm.s.fVirtualSyncCatchUp = f;
+
+    /* the real clock  */
+    rc = SSMR3GetU64(pSSM, &u64Hz);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (u64Hz != TMCLOCK_FREQ_REAL)
+    {
+        AssertMsgFailed(("The real clock frequency differs! Saved: %'RU64 Binary: %'RU64\n",
+                         u64Hz, TMCLOCK_FREQ_REAL));
+        return VERR_SSM_VIRTUAL_CLOCK_HZ; /* misleading... */
+    }
+
+    /* the cpu tick clock. */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        pVCpu->tm.s.fTSCTicking = false;
+        SSMR3GetU64(pSSM, &pVCpu->tm.s.u64TSC);
+
+        if (pVM->tm.s.fTSCUseRealTSC)
+            pVCpu->tm.s.offTSCRawSrc = 0; /** @todo TSC restore stuff and HWACC. */
+    }
+
+    rc = SSMR3GetU64(pSSM, &u64Hz);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (!pVM->tm.s.fTSCUseRealTSC)
+        pVM->tm.s.cTSCTicksPerSecond = u64Hz;
+
+    LogRel(("TM: cTSCTicksPerSecond=%#RX64 (%'RU64) fTSCVirtualized=%RTbool fTSCUseRealTSC=%RTbool (state load)\n",
+            pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.cTSCTicksPerSecond, pVM->tm.s.fTSCVirtualized, pVM->tm.s.fTSCUseRealTSC));
+
+    /*
+     * Make sure timers get rescheduled immediately.
+     */
+    PVMCPU pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
+    VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal TMR3TimerCreate worker.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   enmClock    The timer clock.
+ * @param   pszDesc     The timer description.
+ * @param   ppTimer     Where to store the timer pointer on success.
+ */
+static int tmr3TimerCreate(PVM pVM, TMCLOCK enmClock, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Allocate the timer.
+     */
+    PTMTIMERR3 pTimer = NULL;
+    if (pVM->tm.s.pFree && VM_IS_EMT(pVM))
+    {
+        pTimer = pVM->tm.s.pFree;
+        pVM->tm.s.pFree = pTimer->pBigNext;
+        Log3(("TM: Recycling timer %p, new free head %p.\n", pTimer, pTimer->pBigNext));
+    }
+
+    if (!pTimer)
+    {
+        int rc = MMHyperAlloc(pVM, sizeof(*pTimer), 0, MM_TAG_TM, (void **)&pTimer);
+        if (RT_FAILURE(rc))
+            return rc;
+        Log3(("TM: Allocated new timer %p\n", pTimer));
+    }
+
+    /*
+     * Initialize it.
+     */
+    pTimer->u64Expire       = 0;
+    pTimer->enmClock        = enmClock;
+    pTimer->pVMR3           = pVM;
+    pTimer->pVMR0           = pVM->pVMR0;
+    pTimer->pVMRC           = pVM->pVMRC;
+    pTimer->enmState        = TMTIMERSTATE_STOPPED;
+    pTimer->offScheduleNext = 0;
+    pTimer->offNext         = 0;
+    pTimer->offPrev         = 0;
+    pTimer->pvUser          = NULL;
+    pTimer->pCritSect       = NULL;
+    pTimer->pszDesc         = pszDesc;
+
+    /* insert into the list of created timers. */
+    tmTimerLock(pVM);
+    pTimer->pBigPrev        = NULL;
+    pTimer->pBigNext        = pVM->tm.s.pCreated;
+    pVM->tm.s.pCreated      = pTimer;
+    if (pTimer->pBigNext)
+        pTimer->pBigNext->pBigPrev = pTimer;
+#ifdef VBOX_STRICT
+    tmTimerQueuesSanityChecks(pVM, "tmR3TimerCreate");
+#endif
+    tmTimerUnlock(pVM);
+
+    *ppTimer = pTimer;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Creates a device timer.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM to create the timer in.
+ * @param   pDevIns         Device instance.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          The user argument to the callback.
+ * @param   fFlags          Timer creation flags, see grp_tm_timer_flags.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ * @param   ppTimer         Where to store the timer on success.
+ */
+VMM_INT_DECL(int) TMR3TimerCreateDevice(PVM pVM, PPDMDEVINS pDevIns, TMCLOCK enmClock, PFNTMTIMERDEV pfnCallback, void *pvUser, uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    AssertReturn(!(fFlags & ~(TMTIMER_FLAGS_NO_CRIT_SECT)), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate and init stuff.
+     */
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, ppTimer);
+    if (RT_SUCCESS(rc))
+    {
+        (*ppTimer)->enmType         = TMTIMERTYPE_DEV;
+        (*ppTimer)->u.Dev.pfnTimer  = pfnCallback;
+        (*ppTimer)->u.Dev.pDevIns   = pDevIns;
+        (*ppTimer)->pvUser          = pvUser;
+        if (fFlags & TMTIMER_FLAGS_DEFAULT_CRIT_SECT)
+        {
+            if (pDevIns->pCritSectR3)
+                (*ppTimer)->pCritSect = pDevIns->pCritSectR3;
+            else
+                (*ppTimer)->pCritSect = IOMR3GetCritSect(pVM);
+        }
+        Log(("TM: Created device timer %p clock %d callback %p '%s'\n", (*ppTimer), enmClock, pfnCallback, pszDesc));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Creates a driver timer.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM to create the timer in.
+ * @param   pDrvIns         Driver instance.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          The user argument to the callback.
+ * @param   fFlags          Timer creation flags, see grp_tm_timer_flags.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ * @param   ppTimer         Where to store the timer on success.
+ */
+VMM_INT_DECL(int) TMR3TimerCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, TMCLOCK enmClock, PFNTMTIMERDRV pfnCallback, void *pvUser,
+                                        uint32_t fFlags, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    AssertReturn(!(fFlags & ~(TMTIMER_FLAGS_NO_CRIT_SECT)), VERR_INVALID_PARAMETER);
+
+    /*
+     * Allocate and init stuff.
+     */
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, ppTimer);
+    if (RT_SUCCESS(rc))
+    {
+        (*ppTimer)->enmType         = TMTIMERTYPE_DRV;
+        (*ppTimer)->u.Drv.pfnTimer  = pfnCallback;
+        (*ppTimer)->u.Drv.pDrvIns   = pDrvIns;
+        (*ppTimer)->pvUser          = pvUser;
+        Log(("TM: Created device timer %p clock %d callback %p '%s'\n", (*ppTimer), enmClock, pfnCallback, pszDesc));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Creates an internal timer.
+ *
+ * @returns VBox status.
+ * @param   pVM             The VM to create the timer in.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          User argument to be passed to the callback.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ * @param   ppTimer         Where to store the timer on success.
+ */
+VMMR3DECL(int) TMR3TimerCreateInternal(PVM pVM, TMCLOCK enmClock, PFNTMTIMERINT pfnCallback, void *pvUser, const char *pszDesc, PPTMTIMERR3 ppTimer)
+{
+    /*
+     * Allocate and init  stuff.
+     */
+    PTMTIMER pTimer;
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, &pTimer);
+    if (RT_SUCCESS(rc))
+    {
+        pTimer->enmType             = TMTIMERTYPE_INTERNAL;
+        pTimer->u.Internal.pfnTimer = pfnCallback;
+        pTimer->pvUser              = pvUser;
+        *ppTimer = pTimer;
+        Log(("TM: Created internal timer %p clock %d callback %p '%s'\n", pTimer, enmClock, pfnCallback, pszDesc));
+    }
+
+    return rc;
+}
+
+/**
+ * Creates an external timer.
+ *
+ * @returns Timer handle on success.
+ * @returns NULL on failure.
+ * @param   pVM             The VM to create the timer in.
+ * @param   enmClock        The clock to use on this timer.
+ * @param   pfnCallback     Callback function.
+ * @param   pvUser          User argument.
+ * @param   pszDesc         Pointer to description string which must stay around
+ *                          until the timer is fully destroyed (i.e. a bit after TMTimerDestroy()).
+ */
+VMMR3DECL(PTMTIMERR3) TMR3TimerCreateExternal(PVM pVM, TMCLOCK enmClock, PFNTMTIMEREXT pfnCallback, void *pvUser, const char *pszDesc)
+{
+    /*
+     * Allocate and init stuff.
+     */
+    PTMTIMERR3 pTimer;
+    int rc = tmr3TimerCreate(pVM, enmClock, pszDesc, &pTimer);
+    if (RT_SUCCESS(rc))
+    {
+        pTimer->enmType             = TMTIMERTYPE_EXTERNAL;
+        pTimer->u.External.pfnTimer = pfnCallback;
+        pTimer->pvUser              = pvUser;
+        Log(("TM: Created external timer %p clock %d callback %p '%s'\n", pTimer, enmClock, pfnCallback, pszDesc));
+        return pTimer;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Destroy a timer
+ *
+ * @returns VBox status.
+ * @param   pTimer          Timer handle as returned by one of the create functions.
+ */
+VMMR3DECL(int) TMR3TimerDestroy(PTMTIMER pTimer)
+{
+    /*
+     * Be extra careful here.
+     */
+    if (!pTimer)
+        return VINF_SUCCESS;
+    AssertPtr(pTimer);
+    Assert((unsigned)pTimer->enmClock < (unsigned)TMCLOCK_MAX);
+
+    PVM             pVM      = pTimer->CTX_SUFF(pVM);
+    PTMTIMERQUEUE   pQueue   = &pVM->tm.s.CTX_SUFF(paTimerQueues)[pTimer->enmClock];
+    bool            fActive  = false;
+    bool            fPending = false;
+
+    AssertMsg(   !pTimer->pCritSect
+              || VMR3GetState(pVM) != VMSTATE_RUNNING
+              || PDMCritSectIsOwner(pTimer->pCritSect), ("%s\n", pTimer->pszDesc));
+
+    /*
+     * The rest of the game happens behind the lock, just
+     * like create does. All the work is done here.
+     */
+    tmTimerLock(pVM);
+    for (int cRetries = 1000;; cRetries--)
+    {
+        /*
+         * Change to the DESTROY state.
+         */
+        TMTIMERSTATE enmState    = pTimer->enmState;
+        TMTIMERSTATE enmNewState = enmState;
+        Log2(("TMTimerDestroy: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d\n",
+              pTimer, tmTimerState(enmState), R3STRING(pTimer->pszDesc), cRetries));
+        switch (enmState)
+        {
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+                break;
+
+            case TMTIMERSTATE_ACTIVE:
+                fActive     = true;
+                break;
+
+            case TMTIMERSTATE_PENDING_STOP:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                fActive     = true;
+                fPending    = true;
+                break;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                fPending    = true;
+                break;
+
+            /*
+             * This shouldn't happen as the caller should make sure there are no races.
+             */
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->pszDesc));
+                tmTimerUnlock(pVM);
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+                AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->pszDesc),
+                                VERR_TM_UNSTABLE_STATE);
+                tmTimerLock(pVM);
+                continue;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_FREE:
+            case TMTIMERSTATE_DESTROY:
+                tmTimerUnlock(pVM);
+                AssertLogRelMsgFailedReturn(("pTimer=%p %s\n", pTimer, tmTimerState(enmState)), VERR_TM_INVALID_STATE);
+
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, R3STRING(pTimer->pszDesc)));
+                tmTimerUnlock(pVM);
+                return VERR_TM_UNKNOWN_STATE;
+        }
+
+        /*
+         * Try switch to the destroy state.
+         * This should always succeed as the caller should make sure there are no race.
+         */
+        bool fRc;
+        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_DESTROY, enmState, fRc);
+        if (fRc)
+            break;
+        AssertMsgFailed(("%p:.enmState=%s %s\n", pTimer, tmTimerState(enmState), pTimer->pszDesc));
+        tmTimerUnlock(pVM);
+        AssertMsgReturn(cRetries > 0, ("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->pszDesc),
+                        VERR_TM_UNSTABLE_STATE);
+        tmTimerLock(pVM);
+    }
+
+    /*
+     * Unlink from the active list.
+     */
+    if (fActive)
+    {
+        const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
+        const PTMTIMER pNext = TMTIMER_GET_NEXT(pTimer);
+        if (pPrev)
+            TMTIMER_SET_NEXT(pPrev, pNext);
+        else
+        {
+            TMTIMER_SET_HEAD(pQueue, pNext);
+            pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
+        }
+        if (pNext)
+            TMTIMER_SET_PREV(pNext, pPrev);
+        pTimer->offNext = 0;
+        pTimer->offPrev = 0;
+    }
+
+    /*
+     * Unlink from the schedule list by running it.
+     */
+    if (fPending)
+    {
+        Log3(("TMR3TimerDestroy: tmTimerQueueSchedule\n"));
+        STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
+        Assert(pQueue->offSchedule);
+        tmTimerQueueSchedule(pVM, pQueue);
+    }
+
+    /*
+     * Read to move the timer from the created list and onto the free list.
+     */
+    Assert(!pTimer->offNext); Assert(!pTimer->offPrev); Assert(!pTimer->offScheduleNext);
+
+    /* unlink from created list */
+    if (pTimer->pBigPrev)
+        pTimer->pBigPrev->pBigNext = pTimer->pBigNext;
+    else
+        pVM->tm.s.pCreated         = pTimer->pBigNext;
+    if (pTimer->pBigNext)
+        pTimer->pBigNext->pBigPrev = pTimer->pBigPrev;
+    pTimer->pBigNext = 0;
+    pTimer->pBigPrev = 0;
+
+    /* free */
+    Log2(("TM: Inserting %p into the free list ahead of %p!\n", pTimer, pVM->tm.s.pFree));
+    TM_SET_STATE(pTimer, TMTIMERSTATE_FREE);
+    pTimer->pBigNext = pVM->tm.s.pFree;
+    pVM->tm.s.pFree = pTimer;
+
+#ifdef VBOX_STRICT
+    tmTimerQueuesSanityChecks(pVM, "TMR3TimerDestroy");
+#endif
+    tmTimerUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy all timers owned by a device.
+ *
+ * @returns VBox status.
+ * @param   pVM             VM handle.
+ * @param   pDevIns         Device which timers should be destroyed.
+ */
+VMM_INT_DECL(int) TMR3TimerDestroyDevice(PVM pVM, PPDMDEVINS pDevIns)
+{
+    LogFlow(("TMR3TimerDestroyDevice: pDevIns=%p\n", pDevIns));
+    if (!pDevIns)
+        return VERR_INVALID_PARAMETER;
+
+    tmTimerLock(pVM);
+    PTMTIMER    pCur = pVM->tm.s.pCreated;
+    while (pCur)
+    {
+        PTMTIMER pDestroy = pCur;
+        pCur = pDestroy->pBigNext;
+        if (    pDestroy->enmType == TMTIMERTYPE_DEV
+            &&  pDestroy->u.Dev.pDevIns == pDevIns)
+        {
+            int rc = TMR3TimerDestroy(pDestroy);
+            AssertRC(rc);
+        }
+    }
+    tmTimerUnlock(pVM);
+
+    LogFlow(("TMR3TimerDestroyDevice: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Destroy all timers owned by a driver.
+ *
+ * @returns VBox status.
+ * @param   pVM             VM handle.
+ * @param   pDrvIns         Driver which timers should be destroyed.
+ */
+VMM_INT_DECL(int) TMR3TimerDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns)
+{
+    LogFlow(("TMR3TimerDestroyDriver: pDrvIns=%p\n", pDrvIns));
+    if (!pDrvIns)
+        return VERR_INVALID_PARAMETER;
+
+    tmTimerLock(pVM);
+    PTMTIMER    pCur = pVM->tm.s.pCreated;
+    while (pCur)
+    {
+        PTMTIMER pDestroy = pCur;
+        pCur = pDestroy->pBigNext;
+        if (    pDestroy->enmType == TMTIMERTYPE_DRV
+            &&  pDestroy->u.Drv.pDrvIns == pDrvIns)
+        {
+            int rc = TMR3TimerDestroy(pDestroy);
+            AssertRC(rc);
+        }
+    }
+    tmTimerUnlock(pVM);
+
+    LogFlow(("TMR3TimerDestroyDriver: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal function for getting the clock time.
+ *
+ * @returns clock time.
+ * @param   pVM         The VM handle.
+ * @param   enmClock    The clock.
+ */
+DECLINLINE(uint64_t) tmClock(PVM pVM, TMCLOCK enmClock)
+{
+    switch (enmClock)
+    {
+        case TMCLOCK_VIRTUAL:       return TMVirtualGet(pVM);
+        case TMCLOCK_VIRTUAL_SYNC:  return TMVirtualSyncGet(pVM);
+        case TMCLOCK_REAL:          return TMRealGet(pVM);
+        case TMCLOCK_TSC:           return TMCpuTickGet(&pVM->aCpus[0] /* just take VCPU 0 */);
+        default:
+            AssertMsgFailed(("enmClock=%d\n", enmClock));
+            return ~(uint64_t)0;
+    }
+}
+
+
+/**
+ * Checks if the sync queue has one or more expired timers.
+ *
+ * @returns true / false.
+ *
+ * @param   pVM         The VM handle.
+ * @param   enmClock    The queue.
+ */
+DECLINLINE(bool) tmR3HasExpiredTimer(PVM pVM, TMCLOCK enmClock)
+{
+    const uint64_t u64Expire = pVM->tm.s.CTX_SUFF(paTimerQueues)[enmClock].u64Expire;
+    return u64Expire != INT64_MAX && u64Expire <= tmClock(pVM, enmClock);
+}
+
+
+/**
+ * Checks for expired timers in all the queues.
+ *
+ * @returns true / false.
+ * @param   pVM         The VM handle.
+ */
+DECLINLINE(bool) tmR3AnyExpiredTimers(PVM pVM)
+{
+    /*
+     * Combine the time calculation for the first two since we're not on EMT
+     * TMVirtualSyncGet only permits EMT.
+     */
+    uint64_t u64Now = TMVirtualGetNoCheck(pVM);
+    if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL].u64Expire <= u64Now)
+        return true;
+    u64Now = pVM->tm.s.fVirtualSyncTicking
+           ? u64Now - pVM->tm.s.offVirtualSync
+           : pVM->tm.s.u64VirtualSync;
+    if (pVM->tm.s.CTX_SUFF(paTimerQueues)[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64Now)
+        return true;
+
+    /*
+     * The remaining timers.
+     */
+    if (tmR3HasExpiredTimer(pVM, TMCLOCK_REAL))
+        return true;
+    if (tmR3HasExpiredTimer(pVM, TMCLOCK_TSC))
+        return true;
+    return false;
+}
+
+
+/**
+ * Schedule timer callback.
+ *
+ * @param   pTimer      Timer handle.
+ * @param   pvUser      VM handle.
+ * @thread  Timer thread.
+ *
+ * @remark  We cannot do the scheduling and queues running from a timer handler
+ *          since it's not executing in EMT, and even if it was it would be async
+ *          and we wouldn't know the state of the affairs.
+ *          So, we'll just raise the timer FF and force any REM execution to exit.
+ */
+static DECLCALLBACK(void) tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint64_t /*iTick*/)
+{
+    PVM     pVM      = (PVM)pvUser;
+    PVMCPU  pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
+
+    AssertCompile(TMCLOCK_MAX == 4);
+#ifdef DEBUG_Sander /* very annoying, keep it private. */
+    if (VMCPU_FF_ISSET(pVCpuDst, VMCPU_FF_TIMER))
+        Log(("tmR3TimerCallback: timer event still pending!!\n"));
+#endif
+    if (    !VMCPU_FF_ISSET(pVCpuDst, VMCPU_FF_TIMER)
+        &&  (   pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule /** @todo FIXME - reconsider offSchedule as a reason for running the timer queues. */
+            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule
+            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule
+            ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offSchedule
+            ||  tmR3AnyExpiredTimers(pVM)
+            )
+        && !VMCPU_FF_ISSET(pVCpuDst, VMCPU_FF_TIMER)
+        && !pVM->tm.s.fRunningQueues
+       )
+    {
+        Log5(("TM(%u): FF: 0 -> 1\n", __LINE__));
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        REMR3NotifyTimerPending(pVM, pVCpuDst);
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM /** @todo | VMNOTIFYFF_FLAGS_POKE ?*/);
+        STAM_COUNTER_INC(&pVM->tm.s.StatTimerCallbackSetFF);
+    }
+}
+
+
+/**
+ * Schedules and runs any pending timers.
+ *
+ * This is normally called from a forced action handler in EMT.
+ *
+ * @param   pVM             The VM to run the timers for.
+ *
+ * @thread  EMT (actually EMT0, but we fend off the others)
+ */
+VMMR3DECL(void) TMR3TimerQueuesDo(PVM pVM)
+{
+    /*
+     * Only the dedicated timer EMT should do stuff here.
+     * (fRunningQueues is only used as an indicator.)
+     */
+    Assert(pVM->tm.s.idTimerCpu < pVM->cCpus);
+    PVMCPU pVCpuDst = &pVM->aCpus[pVM->tm.s.idTimerCpu];
+    if (VMMGetCpu(pVM) != pVCpuDst)
+    {
+        Assert(pVM->cCpus > 1);
+        return;
+    }
+    STAM_PROFILE_START(&pVM->tm.s.StatDoQueues, a);
+    Log2(("TMR3TimerQueuesDo:\n"));
+    Assert(!pVM->tm.s.fRunningQueues);
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningQueues, true);
+    tmTimerLock(pVM);
+
+    /*
+     * Process the queues.
+     */
+    AssertCompile(TMCLOCK_MAX == 4);
+
+    /* TMCLOCK_VIRTUAL_SYNC (see also TMR3VirtualSyncFF) */
+    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], s1);
+    tmVirtualSyncLock(pVM);
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, true);
+    VMCPU_FF_CLEAR(pVCpuDst, VMCPU_FF_TIMER);   /* Clear the FF once we started working for real. */
+
+    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule)
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC]);
+    tmR3TimerQueueRunVirtualSync(pVM);
+    if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */
+        VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, false);
+    tmVirtualSyncUnlock(pVM);
+    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL_SYNC], s1);
+
+    /* TMCLOCK_VIRTUAL */
+    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2);
+    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule)
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]);
+    tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL]);
+    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_VIRTUAL], s2);
+
+    /* TMCLOCK_TSC */
+    Assert(!pVM->tm.s.paTimerQueuesR3[TMCLOCK_TSC].offActive); /* not used */
+
+    /* TMCLOCK_REAL */
+    STAM_PROFILE_ADV_START(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3);
+    if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL].offSchedule)
+        tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]);
+    tmR3TimerQueueRun(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_REAL]);
+    STAM_PROFILE_ADV_STOP(&pVM->tm.s.aStatDoQueues[TMCLOCK_REAL], s3);
+
+#ifdef VBOX_STRICT
+    /* check that we didn't screw up. */
+    tmTimerQueuesSanityChecks(pVM, "TMR3TimerQueuesDo");
+#endif
+
+    /* done */
+    Log2(("TMR3TimerQueuesDo: returns void\n"));
+    ASMAtomicWriteBool(&pVM->tm.s.fRunningQueues, false);
+    tmTimerUnlock(pVM);
+    STAM_PROFILE_STOP(&pVM->tm.s.StatDoQueues, a);
+}
+
+//RT_C_DECLS_BEGIN
+//int     iomLock(PVM pVM);
+//void    iomUnlock(PVM pVM);
+//RT_C_DECLS_END
+
+
+/**
+ * Schedules and runs any pending times in the specified queue.
+ *
+ * This is normally called from a forced action handler in EMT.
+ *
+ * @param   pVM             The VM to run the timers for.
+ * @param   pQueue          The queue to run.
+ */
+static void tmR3TimerQueueRun(PVM pVM, PTMTIMERQUEUE pQueue)
+{
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Run timers.
+     *
+     * We check the clock once and run all timers which are ACTIVE
+     * and have an expire time less or equal to the time we read.
+     *
+     * N.B. A generic unlink must be applied since other threads
+     *      are allowed to mess with any active timer at any time.
+     *      However, we only allow EMT to handle EXPIRED_PENDING
+     *      timers, thus enabling the timer handler function to
+     *      arm the timer again.
+     */
+    PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue);
+    if (!pNext)
+        return;
+    const uint64_t u64Now = tmClock(pVM, pQueue->enmClock);
+    while (pNext && pNext->u64Expire <= u64Now)
+    {
+        PTMTIMER        pTimer    = pNext;
+        pNext = TMTIMER_GET_NEXT(pTimer);
+        PPDMCRITSECT    pCritSect = pTimer->pCritSect;
+        if (pCritSect)
+            PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
+        Log2(("tmR3TimerQueueRun: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, u64Expire=%llx (now=%llx) .pszDesc=%s}\n",
+              pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, pTimer->u64Expire, u64Now, pTimer->pszDesc));
+        bool fRc;
+        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_GET_UNLINK, TMTIMERSTATE_ACTIVE, fRc);
+        if (fRc)
+        {
+            Assert(!pTimer->offScheduleNext); /* this can trigger falsely */
+
+            /* unlink */
+            const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
+            if (pPrev)
+                TMTIMER_SET_NEXT(pPrev, pNext);
+            else
+            {
+                TMTIMER_SET_HEAD(pQueue, pNext);
+                pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
+            }
+            if (pNext)
+                TMTIMER_SET_PREV(pNext, pPrev);
+            pTimer->offNext = 0;
+            pTimer->offPrev = 0;
+
+            /* fire */
+            TM_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_DELIVER);
+            switch (pTimer->enmType)
+            {
+                case TMTIMERTYPE_DEV:       pTimer->u.Dev.pfnTimer(pTimer->u.Dev.pDevIns, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_DRV:       pTimer->u.Drv.pfnTimer(pTimer->u.Drv.pDrvIns, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_INTERNAL:  pTimer->u.Internal.pfnTimer(pVM, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_EXTERNAL:  pTimer->u.External.pfnTimer(pTimer->pvUser); break;
+                default:
+                    AssertMsgFailed(("Invalid timer type %d (%s)\n", pTimer->enmType, pTimer->pszDesc));
+                    break;
+            }
+
+            /* change the state if it wasn't changed already in the handler. */
+            TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_EXPIRED_DELIVER, fRc);
+            Log2(("tmR3TimerQueueRun: new state %s\n", tmTimerState(pTimer->enmState)));
+        }
+        if (pCritSect)
+            PDMCritSectLeave(pCritSect);
+    } /* run loop */
+}
+
+
+/**
+ * Schedules and runs any pending times in the timer queue for the
+ * synchronous virtual clock.
+ *
+ * This scheduling is a bit different from the other queues as it need
+ * to implement the special requirements of the timer synchronous virtual
+ * clock, thus this 2nd queue run function.
+ *
+ * @param   pVM             The VM to run the timers for.
+ *
+ * @remarks The caller must own both the TM/EMT and the Virtual Sync locks.
+ */
+static void tmR3TimerQueueRunVirtualSync(PVM pVM)
+{
+    PTMTIMERQUEUE const pQueue = &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC];
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Any timers?
+     */
+    PTMTIMER pNext = TMTIMER_GET_HEAD(pQueue);
+    if (RT_UNLIKELY(!pNext))
+    {
+        Assert(pVM->tm.s.fVirtualSyncTicking || !pVM->tm.s.cVirtualTicking);
+        return;
+    }
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRun);
+
+    /*
+     * Calculate the time frame for which we will dispatch timers.
+     *
+     * We use a time frame ranging from the current sync time (which is most likely the
+     * same as the head timer) and some configurable period (100000ns) up towards the
+     * current virtual time. This period might also need to be restricted by the catch-up
+     * rate so frequent calls to this function won't accelerate the time too much, however
+     * this will be implemented at a later point if necessary.
+     *
+     * Without this frame we would 1) having to run timers much more frequently
+     * and 2) lag behind at a steady rate.
+     */
+    const uint64_t  u64VirtualNow  = TMVirtualGetNoCheck(pVM);
+    uint64_t const  offSyncGivenUp = pVM->tm.s.offVirtualSyncGivenUp;
+    uint64_t        u64Now;
+    if (!pVM->tm.s.fVirtualSyncTicking)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunStoppedAlready);
+        u64Now = pVM->tm.s.u64VirtualSync;
+#ifdef DEBUG_bird
+        Assert(u64Now <= pNext->u64Expire);
+#endif
+    }
+    else
+    {
+        /* Calc 'now'. */
+        bool        fStopCatchup   = false;
+        bool        fUpdateStuff   = false;
+        uint64_t    off            = pVM->tm.s.offVirtualSync;
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+            uint64_t u64Delta = u64VirtualNow - pVM->tm.s.u64VirtualSyncCatchUpPrev;
+            if (RT_LIKELY(!(u64Delta >> 32)))
+            {
+                uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, pVM->tm.s.u32VirtualSyncCatchUpPercentage, 100);
+                if (off > u64Sub + offSyncGivenUp)
+                {
+                    off -= u64Sub;
+                    Log4(("TM: %'RU64/-%'8RU64: sub %'RU64 [tmR3TimerQueueRunVirtualSync]\n", u64VirtualNow - off, off - offSyncGivenUp, u64Sub));
+                }
+                else
+                {
+                    STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                    fStopCatchup = true;
+                    off = offSyncGivenUp;
+                }
+                fUpdateStuff = true;
+            }
+        }
+        u64Now = u64VirtualNow - off;
+
+        /* Check if stopped by expired timer. */
+        uint64_t u64Expire = pNext->u64Expire;
+        if (u64Now >= pNext->u64Expire)
+        {
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunStop);
+            u64Now = pNext->u64Expire;
+            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64Now);
+            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+            Log4(("TM: %'RU64/-%'8RU64: exp tmr [tmR3TimerQueueRunVirtualSync]\n", u64Now, u64VirtualNow - u64Now - offSyncGivenUp));
+        }
+        else if (fUpdateStuff)
+        {
+            ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, off);
+            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev, u64VirtualNow);
+            if (fStopCatchup)
+            {
+                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+                Log4(("TM: %'RU64/0: caught up [tmR3TimerQueueRunVirtualSync]\n", u64VirtualNow));
+            }
+        }
+    }
+
+    /* calc end of frame. */
+    uint64_t u64Max = u64Now + pVM->tm.s.u32VirtualSyncScheduleSlack;
+    if (u64Max > u64VirtualNow - offSyncGivenUp)
+        u64Max = u64VirtualNow - offSyncGivenUp;
+
+    /* assert sanity */
+#ifdef DEBUG_bird
+    Assert(u64Now <= u64VirtualNow - offSyncGivenUp);
+    Assert(u64Max <= u64VirtualNow - offSyncGivenUp);
+    Assert(u64Now <= u64Max);
+    Assert(offSyncGivenUp == pVM->tm.s.offVirtualSyncGivenUp);
+#endif
+
+    /*
+     * Process the expired timers moving the clock along as we progress.
+     */
+#ifdef DEBUG_bird
+#ifdef VBOX_STRICT
+    uint64_t u64Prev = u64Now; NOREF(u64Prev);
+#endif
+#endif
+    while (pNext && pNext->u64Expire <= u64Max)
+    {
+        PTMTIMER        pTimer    = pNext;
+        pNext = TMTIMER_GET_NEXT(pTimer);
+        PPDMCRITSECT    pCritSect = pTimer->pCritSect;
+        if (pCritSect)
+            PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
+        Log2(("tmR3TimerQueueRun: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, u64Expire=%llx (now=%llx) .pszDesc=%s}\n",
+              pTimer, tmTimerState(pTimer->enmState), pTimer->enmClock, pTimer->enmType, pTimer->u64Expire, u64Now, pTimer->pszDesc));
+        bool fRc;
+        TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_GET_UNLINK, TMTIMERSTATE_ACTIVE, fRc);
+        if (fRc)
+        {
+            /* unlink */
+            const PTMTIMER pPrev = TMTIMER_GET_PREV(pTimer);
+            if (pPrev)
+                TMTIMER_SET_NEXT(pPrev, pNext);
+            else
+            {
+                TMTIMER_SET_HEAD(pQueue, pNext);
+                pQueue->u64Expire = pNext ? pNext->u64Expire : INT64_MAX;
+            }
+            if (pNext)
+                TMTIMER_SET_PREV(pNext, pPrev);
+            pTimer->offNext = 0;
+            pTimer->offPrev = 0;
+
+            /* advance the clock - don't permit timers to be out of order or armed in the 'past'. */
+#ifdef DEBUG_bird
+#ifdef VBOX_STRICT
+            AssertMsg(pTimer->u64Expire >= u64Prev, ("%'RU64 < %'RU64 %s\n", pTimer->u64Expire, u64Prev, pTimer->pszDesc));
+            u64Prev = pTimer->u64Expire;
+#endif
+#endif
+            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, pTimer->u64Expire);
+            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+
+            /* fire */
+            TM_SET_STATE(pTimer, TMTIMERSTATE_EXPIRED_DELIVER);
+            switch (pTimer->enmType)
+            {
+                case TMTIMERTYPE_DEV:       pTimer->u.Dev.pfnTimer(pTimer->u.Dev.pDevIns, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_DRV:       pTimer->u.Drv.pfnTimer(pTimer->u.Drv.pDrvIns, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_INTERNAL:  pTimer->u.Internal.pfnTimer(pVM, pTimer, pTimer->pvUser); break;
+                case TMTIMERTYPE_EXTERNAL:  pTimer->u.External.pfnTimer(pTimer->pvUser); break;
+                default:
+                    AssertMsgFailed(("Invalid timer type %d (%s)\n", pTimer->enmType, pTimer->pszDesc));
+                    break;
+            }
+
+            /* Change the state if it wasn't changed already in the handler.
+               Reset the Hz hint too since this is the same as TMTimerStop. */
+            TM_TRY_SET_STATE(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_EXPIRED_DELIVER, fRc);
+            if (fRc && pTimer->uHzHint)
+            {
+                if (pTimer->uHzHint >= pVM->tm.s.uMaxHzHint)
+                    ASMAtomicWriteBool(&pVM->tm.s.fHzHintNeedsUpdating, true);
+                pTimer->uHzHint = 0;
+            }
+            Log2(("tmR3TimerQueueRun: new state %s\n", tmTimerState(pTimer->enmState)));
+        }
+        if (pCritSect)
+            PDMCritSectLeave(pCritSect);
+    } /* run loop */
+
+    /*
+     * Restart the clock if it was stopped to serve any timers,
+     * and start/adjust catch-up if necessary.
+     */
+    if (    !pVM->tm.s.fVirtualSyncTicking
+        &&  pVM->tm.s.cVirtualTicking)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncRunRestart);
+
+        /* calc the slack we've handed out. */
+        const uint64_t u64VirtualNow2 = TMVirtualGetNoCheck(pVM);
+        Assert(u64VirtualNow2 >= u64VirtualNow);
+#ifdef DEBUG_bird
+        AssertMsg(pVM->tm.s.u64VirtualSync >= u64Now, ("%'RU64 < %'RU64\n", pVM->tm.s.u64VirtualSync, u64Now));
+#endif
+        const uint64_t offSlack = pVM->tm.s.u64VirtualSync - u64Now;
+        STAM_STATS({
+            if (offSlack)
+            {
+                PSTAMPROFILE p = &pVM->tm.s.StatVirtualSyncRunSlack;
+                p->cPeriods++;
+                p->cTicks += offSlack;
+                if (p->cTicksMax < offSlack) p->cTicksMax = offSlack;
+                if (p->cTicksMin > offSlack) p->cTicksMin = offSlack;
+            }
+        });
+
+        /* Let the time run a little bit while we were busy running timers(?). */
+        uint64_t u64Elapsed;
+#define MAX_ELAPSED 30000U /* ns */
+        if (offSlack > MAX_ELAPSED)
+            u64Elapsed = 0;
+        else
+        {
+            u64Elapsed = u64VirtualNow2 - u64VirtualNow;
+            if (u64Elapsed > MAX_ELAPSED)
+                u64Elapsed = MAX_ELAPSED;
+            u64Elapsed = u64Elapsed > offSlack ? u64Elapsed - offSlack : 0;
+        }
+#undef MAX_ELAPSED
+
+        /* Calc the current offset. */
+        uint64_t offNew = u64VirtualNow2 - pVM->tm.s.u64VirtualSync - u64Elapsed;
+        Assert(!(offNew & RT_BIT_64(63)));
+        uint64_t offLag = offNew - pVM->tm.s.offVirtualSyncGivenUp;
+        Assert(!(offLag & RT_BIT_64(63)));
+
+        /*
+         * Deal with starting, adjusting and stopping catchup.
+         */
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+            if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpStopThreshold)
+            {
+                /* stop */
+                STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+                Log4(("TM: %'RU64/-%'8RU64: caught up [pt]\n", u64VirtualNow2 - offNew, offLag));
+            }
+            else if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold)
+            {
+                /* adjust */
+                unsigned i = 0;
+                while (     i + 1 < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods)
+                       &&   offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[i + 1].u64Start)
+                    i++;
+                if (pVM->tm.s.u32VirtualSyncCatchUpPercentage < pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage)
+                {
+                    STAM_COUNTER_INC(&pVM->tm.s.aStatVirtualSyncCatchupAdjust[i]);
+                    ASMAtomicWriteU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage, pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage);
+                    Log4(("TM: %'RU64/%'8RU64: adj %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
+                }
+                pVM->tm.s.u64VirtualSyncCatchUpPrev = u64VirtualNow2;
+            }
+            else
+            {
+                /* give up */
+                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGiveUp);
+                STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
+                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+                Log4(("TM: %'RU64/%'8RU64: give up %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
+                LogRel(("TM: Giving up catch-up attempt at a %'RU64 ns lag; new total: %'RU64 ns\n", offLag, offNew));
+            }
+        }
+        else if (offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[0].u64Start)
+        {
+            if (offLag <= pVM->tm.s.u64VirtualSyncCatchUpGiveUpThreshold)
+            {
+                /* start */
+                STAM_PROFILE_ADV_START(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                unsigned i = 0;
+                while (     i + 1 < RT_ELEMENTS(pVM->tm.s.aVirtualSyncCatchUpPeriods)
+                       &&   offLag >= pVM->tm.s.aVirtualSyncCatchUpPeriods[i + 1].u64Start)
+                    i++;
+                STAM_COUNTER_INC(&pVM->tm.s.aStatVirtualSyncCatchupInitial[i]);
+                ASMAtomicWriteU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage, pVM->tm.s.aVirtualSyncCatchUpPeriods[i].u32Percentage);
+                ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, true);
+                Log4(("TM: %'RU64/%'8RU64: catch-up %u%%\n", u64VirtualNow2 - offNew, offLag, pVM->tm.s.u32VirtualSyncCatchUpPercentage));
+            }
+            else
+            {
+                /* don't bother */
+                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGiveUpBeforeStarting);
+                ASMAtomicWriteU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp, offNew);
+                Log4(("TM: %'RU64/%'8RU64: give up\n", u64VirtualNow2 - offNew, offLag));
+                LogRel(("TM: Not bothering to attempt catching up a %'RU64 ns lag; new total: %'RU64\n", offLag, offNew));
+            }
+        }
+
+        /*
+         * Update the offset and restart the clock.
+         */
+        Assert(!(offNew & RT_BIT_64(63)));
+        ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, offNew);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, true);
+    }
+}
+
+
+/**
+ * Deals with stopped Virtual Sync clock.
+ *
+ * This is called by the forced action flag handling code in EM when it
+ * encounters the VM_FF_TM_VIRTUAL_SYNC flag. It is called by all VCPUs and they
+ * will block on the VirtualSyncLock until the pending timers has been executed
+ * and the clock restarted.
+ *
+ * @param   pVM             The VM to run the timers for.
+ * @param   pVCpu           The virtual CPU we're running at.
+ *
+ * @thread  EMTs
+ */
+VMM_INT_DECL(void) TMR3VirtualSyncFF(PVM pVM, PVMCPU pVCpu)
+{
+    Log2(("TMR3VirtualSyncFF:\n"));
+
+    /*
+     * The EMT doing the timers is diverted to them.
+     */
+    if (pVCpu->idCpu == pVM->tm.s.idTimerCpu)
+        TMR3TimerQueuesDo(pVM);
+    /*
+     * The other EMTs will block on the virtual sync lock and the first owner
+     * will run the queue and thus restarting the clock.
+     *
+     * Note! This is very suboptimal code wrt to resuming execution when there
+     *       are more than two Virtual CPUs, since they will all have to enter
+     *       the critical section one by one. But it's a very simple solution
+     *       which will have to do the job for now.
+     */
+    else
+    {
+        STAM_PROFILE_START(&pVM->tm.s.StatVirtualSyncFF, a);
+        tmVirtualSyncLock(pVM);
+        if (pVM->tm.s.fVirtualSyncTicking)
+        {
+            STAM_PROFILE_STOP(&pVM->tm.s.StatVirtualSyncFF, a); /* before the unlock! */
+            tmVirtualSyncUnlock(pVM);
+            Log2(("TMR3VirtualSyncFF: ticking\n"));
+        }
+        else
+        {
+            tmVirtualSyncUnlock(pVM);
+
+            /* try run it. */
+            tmTimerLock(pVM);
+            tmVirtualSyncLock(pVM);
+            if (pVM->tm.s.fVirtualSyncTicking)
+                Log2(("TMR3VirtualSyncFF: ticking (2)\n"));
+            else
+            {
+                ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, true);
+                Log2(("TMR3VirtualSyncFF: running queue\n"));
+
+                if (pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule)
+                    tmTimerQueueSchedule(pVM, &pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC]);
+                tmR3TimerQueueRunVirtualSync(pVM);
+                if (pVM->tm.s.fVirtualSyncTicking) /** @todo move into tmR3TimerQueueRunVirtualSync - FIXME */
+                    VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+
+                ASMAtomicWriteBool(&pVM->tm.s.fRunningVirtualSyncQueue, false);
+            }
+            STAM_PROFILE_STOP(&pVM->tm.s.StatVirtualSyncFF, a); /* before the unlock! */
+            tmVirtualSyncUnlock(pVM);
+            tmTimerUnlock(pVM);
+        }
+    }
+}
+
+
+/** @name Saved state values
+ * @{ */
+#define TMTIMERSTATE_SAVED_PENDING_STOP         4
+#define TMTIMERSTATE_SAVED_PENDING_SCHEDULE     7
+/** @}  */
+
+
+/**
+ * Saves the state of a timer to a saved state.
+ *
+ * @returns VBox status.
+ * @param   pTimer          Timer to save.
+ * @param   pSSM            Save State Manager handle.
+ */
+VMMR3DECL(int) TMR3TimerSave(PTMTIMERR3 pTimer, PSSMHANDLE pSSM)
+{
+    LogFlow(("TMR3TimerSave: %p:{enmState=%s, .pszDesc={%s}} pSSM=%p\n", pTimer, tmTimerState(pTimer->enmState), pTimer->pszDesc, pSSM));
+    switch (pTimer->enmState)
+    {
+        case TMTIMERSTATE_STOPPED:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+            return SSMR3PutU8(pSSM, TMTIMERSTATE_SAVED_PENDING_STOP);
+
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+            AssertMsgFailed(("u64Expire is being updated! (%s)\n", pTimer->pszDesc));
+            if (!RTThreadYield())
+                RTThreadSleep(1);
+            /* fall thru */
+        case TMTIMERSTATE_ACTIVE:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+            SSMR3PutU8(pSSM, TMTIMERSTATE_SAVED_PENDING_SCHEDULE);
+            return SSMR3PutU64(pSSM, pTimer->u64Expire);
+
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertMsgFailed(("Invalid timer state %d %s (%s)\n", pTimer->enmState, tmTimerState(pTimer->enmState), pTimer->pszDesc));
+            return SSMR3HandleSetStatus(pSSM, VERR_TM_INVALID_STATE);
+    }
+
+    AssertMsgFailed(("Unknown timer state %d (%s)\n", pTimer->enmState, pTimer->pszDesc));
+    return SSMR3HandleSetStatus(pSSM, VERR_TM_UNKNOWN_STATE);
+}
+
+
+/**
+ * Loads the state of a timer from a saved state.
+ *
+ * @returns VBox status.
+ * @param   pTimer          Timer to restore.
+ * @param   pSSM            Save State Manager handle.
+ */
+VMMR3DECL(int) TMR3TimerLoad(PTMTIMERR3 pTimer, PSSMHANDLE pSSM)
+{
+    Assert(pTimer); Assert(pSSM); VM_ASSERT_EMT(pTimer->pVMR3);
+    LogFlow(("TMR3TimerLoad: %p:{enmState=%s, .pszDesc={%s}} pSSM=%p\n", pTimer, tmTimerState(pTimer->enmState), pTimer->pszDesc, pSSM));
+
+    /*
+     * Load the state and validate it.
+     */
+    uint8_t u8State;
+    int rc = SSMR3GetU8(pSSM, &u8State);
+    if (RT_FAILURE(rc))
+        return rc;
+#if 1 /* Workaround for accidental state shift in r47786 (2009-05-26 19:12:12). */  /** @todo remove this in a few weeks! */
+    if (    u8State == TMTIMERSTATE_SAVED_PENDING_STOP + 1
+        ||  u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE + 1)
+        u8State--;
+#endif
+    if (    u8State != TMTIMERSTATE_SAVED_PENDING_STOP
+        &&  u8State != TMTIMERSTATE_SAVED_PENDING_SCHEDULE)
+    {
+        AssertLogRelMsgFailed(("u8State=%d\n", u8State));
+        return SSMR3HandleSetStatus(pSSM, VERR_TM_LOAD_STATE);
+    }
+
+    /* Enter the critical section to make TMTimerSet/Stop happy. */
+    PPDMCRITSECT pCritSect = pTimer->pCritSect;
+    if (pCritSect)
+        PDMCritSectEnter(pCritSect, VERR_INTERNAL_ERROR);
+
+    if (u8State == TMTIMERSTATE_SAVED_PENDING_SCHEDULE)
+    {
+        /*
+         * Load the expire time.
+         */
+        uint64_t u64Expire;
+        rc = SSMR3GetU64(pSSM, &u64Expire);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /*
+         * Set it.
+         */
+        Log(("u8State=%d u64Expire=%llu\n", u8State, u64Expire));
+        rc = TMTimerSet(pTimer, u64Expire);
+    }
+    else
+    {
+        /*
+         * Stop it.
+         */
+        Log(("u8State=%d\n", u8State));
+        rc = TMTimerStop(pTimer);
+    }
+
+    if (pCritSect)
+        PDMCritSectLeave(pCritSect);
+
+    /*
+     * On failure set SSM status.
+     */
+    if (RT_FAILURE(rc))
+        rc = SSMR3HandleSetStatus(pSSM, rc);
+    return rc;
+}
+
+
+/**
+ * Associates a critical section with a timer.
+ *
+ * The critical section will be entered prior to doing the timer call back, thus
+ * avoiding potential races between the timer thread and other threads trying to
+ * stop or adjust the timer expiration while it's being delivered. The timer
+ * thread will leave the critical section when the timer callback returns.
+ *
+ * In strict builds, ownership of the critical section will be asserted by
+ * TMTimerSet, TMTimerStop, TMTimerGetExpire and TMTimerDestroy (when called at
+ * runtime).
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_INVALID_HANDLE if the timer handle is NULL or invalid
+ *          (asserted).
+ * @retval  VERR_INVALID_PARAMETER if pCritSect is NULL or has an invalid magic
+ *          (asserted).
+ * @retval  VERR_ALREADY_EXISTS if a critical section was already associated
+ *          with the timer (asserted).
+ * @retval  VERR_INVALID_STATE if the timer isn't stopped.
+ *
+ * @param   pTimer          The timer handle.
+ * @param   pCritSect       The critical section. The caller must make sure this
+ *                          is around for the life time of the timer.
+ *
+ * @thread  Any, but the caller is responsible for making sure the timer is not
+ *          active.
+ */
+VMMR3DECL(int) TMR3TimerSetCritSect(PTMTIMERR3 pTimer, PPDMCRITSECT pCritSect)
+{
+    AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
+    AssertPtrReturn(pCritSect, VERR_INVALID_PARAMETER);
+    const char *pszName = PDMR3CritSectName(pCritSect); /* exploited for validation */
+    AssertReturn(pszName, VERR_INVALID_PARAMETER);
+    AssertReturn(!pTimer->pCritSect, VERR_ALREADY_EXISTS);
+    AssertReturn(pTimer->enmState == TMTIMERSTATE_STOPPED, VERR_INVALID_STATE);
+    LogFlow(("pTimer=%p (%s) pCritSect=%p (%s)\n", pTimer, pTimer->pszDesc, pCritSect, pszName));
+
+    pTimer->pCritSect = pCritSect;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get the real world UTC time adjusted for VM lag.
+ *
+ * @returns pTime.
+ * @param   pVM             The VM instance.
+ * @param   pTime           Where to store the time.
+ */
+VMM_INT_DECL(PRTTIMESPEC) TMR3UtcNow(PVM pVM, PRTTIMESPEC pTime)
+{
+    RTTimeNow(pTime);
+    RTTimeSpecSubNano(pTime, ASMAtomicReadU64(&pVM->tm.s.offVirtualSync) - ASMAtomicReadU64((uint64_t volatile *)&pVM->tm.s.offVirtualSyncGivenUp));
+    RTTimeSpecAddNano(pTime, pVM->tm.s.offUTC);
+    return pTime;
+}
+
+
+/**
+ * Pauses all clocks except TMCLOCK_REAL.
+ *
+ * @returns VBox status code, all errors are asserted.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The virtual CPU handle.
+ * @thread  EMT corresponding to the virtual CPU handle.
+ */
+VMMR3DECL(int) TMR3NotifySuspend(PVM pVM, PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * The shared virtual clock (includes virtual sync which is tied to it).
+     */
+    tmTimerLock(pVM);                           /* Paranoia: Exploiting the timer lock here. */
+    int rc = tmVirtualPauseLocked(pVM);
+    tmTimerUnlock(pVM);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Pause the TSC last since it is normally linked to the virtual
+     * sync clock, so the above code may actually stop both clock.
+     */
+    rc = tmCpuTickPause(pVM, pVCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Update cNsTotal.
+     */
+    uint32_t uGen = ASMAtomicIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1);
+    pVCpu->tm.s.cNsTotal = RTTimeNanoTS() - pVCpu->tm.s.u64NsTsStartTotal;
+    pVCpu->tm.s.cNsOther = pVCpu->tm.s.cNsTotal - pVCpu->tm.s.cNsExecuting - pVCpu->tm.s.cNsHalted;
+    ASMAtomicWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resumes all clocks except TMCLOCK_REAL.
+ *
+ * @returns VBox status code, all errors are asserted.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The virtual CPU handle.
+ * @thread  EMT corresponding to the virtual CPU handle.
+ */
+VMMR3DECL(int) TMR3NotifyResume(PVM pVM, PVMCPU pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    int rc;
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Set u64NsTsStartTotal.  There is no need to back this out if either of
+     * the two calls below fail.
+     */
+    pVCpu->tm.s.u64NsTsStartTotal = RTTimeNanoTS() - pVCpu->tm.s.cNsTotal;
+#endif
+
+    /*
+     * Resume the TSC first since it is normally linked to the virtual sync
+     * clock, so it may actually not be resumed until we've executed the code
+     * below.
+     */
+    if (!pVM->tm.s.fTSCTiedToExecution)
+    {
+        rc = tmCpuTickResume(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * The shared virtual clock (includes virtual sync which is tied to it).
+     */
+    tmTimerLock(pVM);                           /* Paranoia: Exploiting the timer lock here. */
+    rc = tmVirtualResumeLocked(pVM);
+    tmTimerUnlock(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Sets the warp drive percent of the virtual time.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   u32Percent  The new percentage. 100 means normal operation.
+ */
+VMMDECL(int) TMR3SetWarpDrive(PVM pVM, uint32_t u32Percent)
+{
+    return VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)tmR3SetWarpDrive, 2, pVM, u32Percent);
+}
+
+
+/**
+ * EMT worker for TMR3SetWarpDrive.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   u32Percent  See TMR3SetWarpDrive().
+ * @internal
+ */
+static DECLCALLBACK(int) tmR3SetWarpDrive(PVM pVM, uint32_t u32Percent)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+
+    /*
+     * Validate it.
+     */
+    AssertMsgReturn(u32Percent >= 2 && u32Percent <= 20000,
+                    ("%RX32 is not between 2 and 20000 (inclusive).\n", u32Percent),
+                    VERR_INVALID_PARAMETER);
+
+/** @todo This isn't a feature specific to virtual time, move the variables to
+ * TM level and make it affect TMR3UTCNow as well! */
+
+    /*
+     * If the time is running we'll have to pause it before we can change
+     * the warp drive settings.
+     */
+    tmTimerLock(pVM);                           /* Paranoia: Exploiting the timer lock here. */
+    bool fPaused = !!pVM->tm.s.cVirtualTicking;
+    if (fPaused) /** @todo this isn't really working, but wtf. */
+        TMR3NotifySuspend(pVM, pVCpu);
+
+    pVM->tm.s.u32VirtualWarpDrivePercentage = u32Percent;
+    pVM->tm.s.fVirtualWarpDrive = u32Percent != 100;
+    LogRel(("TM: u32VirtualWarpDrivePercentage=%RI32 fVirtualWarpDrive=%RTbool\n",
+            pVM->tm.s.u32VirtualWarpDrivePercentage, pVM->tm.s.fVirtualWarpDrive));
+
+    if (fPaused)
+        TMR3NotifyResume(pVM, pVCpu);
+    tmTimerUnlock(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the performance information for one virtual CPU as seen by the VMM.
+ *
+ * The returned times covers the period where the VM is running and will be
+ * reset when restoring a previous VM state (at least for the time being).
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_NOT_IMPLEMENTED if not compiled in.
+ * @retval  VERR_INVALID_STATE if the VM handle is bad.
+ * @retval  VERR_INVALID_PARAMETER if idCpu is out of range.
+ *
+ * @param   pVM             The VM handle.
+ * @param   idCpu           The ID of the virtual CPU which times to get.
+ * @param   pcNsTotal       Where to store the total run time (nano seconds) of
+ *                          the CPU, i.e. the sum of the three other returns.
+ *                          Optional.
+ * @param   pcNsExecuting   Where to store the time (nano seconds) spent
+ *                          executing guest code.  Optional.
+ * @param   pcNsHalted      Where to store the time (nano seconds) spent
+ *                          halted.  Optional
+ * @param   pcNsOther       Where to store the time (nano seconds) spent
+ *                          preempted by the host scheduler, on virtualization
+ *                          overhead and on other tasks.
+ */
+VMMR3DECL(int) TMR3GetCpuLoadTimes(PVM pVM, VMCPUID idCpu, uint64_t *pcNsTotal, uint64_t *pcNsExecuting,
+                                   uint64_t *pcNsHalted, uint64_t *pcNsOther)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_STATE);
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_PARAMETER);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Get a stable result set.
+     * This should be way quicker than an EMT request.
+     */
+    PVMCPU      pVCpu        = &pVM->aCpus[idCpu];
+    uint32_t    uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+    uint64_t    cNsTotal     = pVCpu->tm.s.cNsTotal;
+    uint64_t    cNsExecuting = pVCpu->tm.s.cNsExecuting;
+    uint64_t    cNsHalted    = pVCpu->tm.s.cNsHalted;
+    uint64_t    cNsOther     = pVCpu->tm.s.cNsOther;
+    while (   (uTimesGen & 1) /* update in progress */
+           || uTimesGen != ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen))
+    {
+        RTThreadYield();
+        uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+        cNsTotal     = pVCpu->tm.s.cNsTotal;
+        cNsExecuting = pVCpu->tm.s.cNsExecuting;
+        cNsHalted    = pVCpu->tm.s.cNsHalted;
+        cNsOther     = pVCpu->tm.s.cNsOther;
+    }
+
+    /*
+     * Fill in the return values.
+     */
+    if (pcNsTotal)
+        *pcNsTotal = cNsTotal;
+    if (pcNsExecuting)
+        *pcNsExecuting = cNsExecuting;
+    if (pcNsHalted)
+        *pcNsHalted = cNsHalted;
+    if (pcNsOther)
+        *pcNsOther = cNsOther;
+
+    return VINF_SUCCESS;
+
+#else
+    return VERR_NOT_IMPLEMENTED;
+#endif
+}
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+
+/**
+ * Helper for tmR3CpuLoadTimer.
+ * @returns
+ * @param   pState              The state to update.
+ * @param   cNsTotalDelta       Total time.
+ * @param   cNsExecutingDelta   Time executing.
+ * @param   cNsHaltedDelta      Time halted.
+ */
+DECLINLINE(void) tmR3CpuLoadTimerMakeUpdate(PTMCPULOADSTATE pState,
+                                            uint64_t cNsTotal,
+                                            uint64_t cNsExecuting,
+                                            uint64_t cNsHalted)
+{
+    /* Calc deltas */
+    uint64_t cNsTotalDelta      = cNsTotal     - pState->cNsPrevTotal;
+    pState->cNsPrevTotal        = cNsTotal;
+
+    uint64_t cNsExecutingDelta  = cNsExecuting - pState->cNsPrevExecuting;
+    pState->cNsPrevExecuting    = cNsExecuting;
+
+    uint64_t cNsHaltedDelta     = cNsHalted    - pState->cNsPrevHalted;
+    pState->cNsPrevHalted       = cNsHalted;
+
+    /* Calc pcts. */
+    if (!cNsTotalDelta)
+    {
+        pState->cPctExecuting   = 0;
+        pState->cPctHalted      = 100;
+        pState->cPctOther       = 0;
+    }
+    else if (cNsTotalDelta < UINT64_MAX / 4)
+    {
+        pState->cPctExecuting   = (uint8_t)(cNsExecutingDelta    * 100 / cNsTotalDelta);
+        pState->cPctHalted      = (uint8_t)(cNsHaltedDelta       * 100 / cNsTotalDelta);
+        pState->cPctOther       = (uint8_t)((cNsTotalDelta - cNsExecutingDelta - cNsHaltedDelta) * 100 / cNsTotalDelta);
+    }
+    else
+    {
+        pState->cPctExecuting   = 0;
+        pState->cPctHalted      = 100;
+        pState->cPctOther       = 0;
+    }
+}
+
+
+/**
+ * Timer callback that calculates the CPU load since the last time it was
+ * called.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pTimer              The timer.
+ * @param   pvUser              NULL, unused.
+ */
+static DECLCALLBACK(void) tmR3CpuLoadTimer(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    /*
+     * Re-arm the timer first.
+     */
+    int rc = TMTimerSetMillies(pTimer, 1000);
+    AssertLogRelRC(rc);
+    NOREF(pvUser);
+
+    /*
+     * Update the values for each CPU.
+     */
+    uint64_t cNsTotalAll       = 0;
+    uint64_t cNsExecutingAll   = 0;
+    uint64_t cNsHaltedAll      = 0;
+    for (VMCPUID iCpu = 0; iCpu < pVM->cCpus; iCpu++)
+    {
+        PVMCPU      pVCpu = &pVM->aCpus[iCpu];
+
+        /* Try get a stable data set. */
+        uint32_t    cTries       = 3;
+        uint32_t    uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+        uint64_t    cNsTotal     = pVCpu->tm.s.cNsTotal;
+        uint64_t    cNsExecuting = pVCpu->tm.s.cNsExecuting;
+        uint64_t    cNsHalted    = pVCpu->tm.s.cNsHalted;
+        while (RT_UNLIKELY(   (uTimesGen & 1) /* update in progress */
+                           || uTimesGen != ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen)))
+        {
+            if (!--cTries)
+                break;
+            ASMNopPause();
+            uTimesGen    = ASMAtomicReadU32(&pVCpu->tm.s.uTimesGen);
+            cNsTotal     = pVCpu->tm.s.cNsTotal;
+            cNsExecuting = pVCpu->tm.s.cNsExecuting;
+            cNsHalted    = pVCpu->tm.s.cNsHalted;
+        }
+
+        /* Totals */
+        cNsTotalAll     += cNsTotal;
+        cNsExecutingAll += cNsExecuting;
+        cNsHaltedAll    += cNsHalted;
+
+        /* Calc the PCTs and update the state. */
+        tmR3CpuLoadTimerMakeUpdate(&pVCpu->tm.s.CpuLoad, cNsTotal, cNsExecuting, cNsHalted);
+    }
+
+    /*
+     * Update the value for all the CPUs.
+     */
+    tmR3CpuLoadTimerMakeUpdate(&pVM->tm.s.CpuLoad, cNsTotalAll, cNsExecutingAll, cNsHaltedAll);
+
+    /** @todo Try add 1, 5 and 15 min load stats. */
+
+}
+
+#endif /* !VBOX_WITHOUT_NS_ACCOUNTING */
+
+/**
+ * Gets the 5 char clock name for the info tables.
+ *
+ * @returns The name.
+ * @param   enmClock        The clock.
+ */
+DECLINLINE(const char *) tmR3Get5CharClockName(TMCLOCK enmClock)
+{
+    switch (enmClock)
+    {
+        case TMCLOCK_REAL:          return "Real ";
+        case TMCLOCK_VIRTUAL:       return "Virt ";
+        case TMCLOCK_VIRTUAL_SYNC:  return "VrSy ";
+        case TMCLOCK_TSC:           return "TSC  ";
+        default:                    return "Bad  ";
+    }
+}
+
+
+/**
+ * Display all timers.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) tmR3TimerInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    pHlp->pfnPrintf(pHlp,
+                    "Timers (pVM=%p)\n"
+                    "%.*s %.*s %.*s %.*s Clock %18s %18s %6s %-25s Description\n",
+                    pVM,
+                    sizeof(RTR3PTR) * 2,        "pTimerR3        ",
+                    sizeof(int32_t) * 2,        "offNext         ",
+                    sizeof(int32_t) * 2,        "offPrev         ",
+                    sizeof(int32_t) * 2,        "offSched        ",
+                                                "Time",
+                                                "Expire",
+                                                "HzHint",
+                                                "State");
+    tmTimerLock(pVM);
+    for (PTMTIMERR3 pTimer = pVM->tm.s.pCreated; pTimer; pTimer = pTimer->pBigNext)
+    {
+        pHlp->pfnPrintf(pHlp,
+                        "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n",
+                        pTimer,
+                        pTimer->offNext,
+                        pTimer->offPrev,
+                        pTimer->offScheduleNext,
+                        tmR3Get5CharClockName(pTimer->enmClock),
+                        TMTimerGet(pTimer),
+                        pTimer->u64Expire,
+                        pTimer->uHzHint,
+                        tmTimerState(pTimer->enmState),
+                        pTimer->pszDesc);
+    }
+    tmTimerUnlock(pVM);
+}
+
+
+/**
+ * Display all active timers.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) tmR3TimerInfoActive(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+    pHlp->pfnPrintf(pHlp,
+                    "Active Timers (pVM=%p)\n"
+                    "%.*s %.*s %.*s %.*s Clock %18s %18s %6s %-25s Description\n",
+                    pVM,
+                    sizeof(RTR3PTR) * 2,        "pTimerR3        ",
+                    sizeof(int32_t) * 2,        "offNext         ",
+                    sizeof(int32_t) * 2,        "offPrev         ",
+                    sizeof(int32_t) * 2,        "offSched        ",
+                                                "Time",
+                                                "Expire",
+                                                "HzHint",
+                                                "State");
+    for (unsigned iQueue = 0; iQueue < TMCLOCK_MAX; iQueue++)
+    {
+        tmTimerLock(pVM);
+        for (PTMTIMERR3 pTimer = TMTIMER_GET_HEAD(&pVM->tm.s.paTimerQueuesR3[iQueue]);
+             pTimer;
+             pTimer = TMTIMER_GET_NEXT(pTimer))
+        {
+            pHlp->pfnPrintf(pHlp,
+                            "%p %08RX32 %08RX32 %08RX32 %s %18RU64 %18RU64 %6RU32 %-25s %s\n",
+                            pTimer,
+                            pTimer->offNext,
+                            pTimer->offPrev,
+                            pTimer->offScheduleNext,
+                            tmR3Get5CharClockName(pTimer->enmClock),
+                            TMTimerGet(pTimer),
+                            pTimer->u64Expire,
+                            pTimer->uHzHint,
+                            tmTimerState(pTimer->enmState),
+                            pTimer->pszDesc);
+        }
+        tmTimerUnlock(pVM);
+    }
+}
+
+
+/**
+ * Display all clocks.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pHlp        The info helpers.
+ * @param   pszArgs     Arguments, ignored.
+ */
+static DECLCALLBACK(void) tmR3InfoClocks(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    NOREF(pszArgs);
+
+    /*
+     * Read the times first to avoid more than necessary time variation.
+     */
+    const uint64_t u64Virtual     = TMVirtualGet(pVM);
+    const uint64_t u64VirtualSync = TMVirtualSyncGet(pVM);
+    const uint64_t u64Real        = TMRealGet(pVM);
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU   pVCpu  = &pVM->aCpus[i];
+        uint64_t u64TSC = TMCpuTickGet(pVCpu);
+
+        /*
+         * TSC
+         */
+        pHlp->pfnPrintf(pHlp,
+                        "Cpu Tick: %18RU64 (%#016RX64) %RU64Hz %s%s",
+                        u64TSC, u64TSC, TMCpuTicksPerSecond(pVM),
+                        pVCpu->tm.s.fTSCTicking ? "ticking" : "paused",
+                        pVM->tm.s.fTSCVirtualized ? " - virtualized" : "");
+        if (pVM->tm.s.fTSCUseRealTSC)
+        {
+            pHlp->pfnPrintf(pHlp, " - real tsc");
+            if (pVCpu->tm.s.offTSCRawSrc)
+                pHlp->pfnPrintf(pHlp, "\n          offset %RU64", pVCpu->tm.s.offTSCRawSrc);
+        }
+        else
+            pHlp->pfnPrintf(pHlp, " - virtual clock");
+        pHlp->pfnPrintf(pHlp, "\n");
+    }
+
+    /*
+     * virtual
+     */
+    pHlp->pfnPrintf(pHlp,
+                    " Virtual: %18RU64 (%#016RX64) %RU64Hz %s",
+                    u64Virtual, u64Virtual, TMVirtualGetFreq(pVM),
+                    pVM->tm.s.cVirtualTicking ? "ticking" : "paused");
+    if (pVM->tm.s.fVirtualWarpDrive)
+        pHlp->pfnPrintf(pHlp, " WarpDrive %RU32 %%", pVM->tm.s.u32VirtualWarpDrivePercentage);
+    pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * virtual sync
+     */
+    pHlp->pfnPrintf(pHlp,
+                    "VirtSync: %18RU64 (%#016RX64) %s%s",
+                    u64VirtualSync, u64VirtualSync,
+                    pVM->tm.s.fVirtualSyncTicking ? "ticking" : "paused",
+                    pVM->tm.s.fVirtualSyncCatchUp ? " - catchup" : "");
+    if (pVM->tm.s.offVirtualSync)
+    {
+        pHlp->pfnPrintf(pHlp, "\n          offset %RU64", pVM->tm.s.offVirtualSync);
+        if (pVM->tm.s.u32VirtualSyncCatchUpPercentage)
+            pHlp->pfnPrintf(pHlp, "  catch-up rate %u %%", pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+    }
+    pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * real
+     */
+    pHlp->pfnPrintf(pHlp,
+                    "    Real: %18RU64 (%#016RX64) %RU64Hz\n",
+                    u64Real, u64Real, TMRealGetFreq(pVM));
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/TRPM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/TRPM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/TRPM.cpp	(revision 35333)
@@ -0,0 +1,1536 @@
+/* $Id$ */
+/** @file
+ * TRPM - The Trap Monitor.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_trpm   TRPM - The Trap Monitor
+ *
+ * The Trap Monitor (TRPM) is responsible for all trap and interrupt handling in
+ * the VMM.  It plays a major role in raw-mode execution and a lesser one in the
+ * hardware assisted mode.
+ *
+ * Note first, the following will use trap as a collective term for faults,
+ * aborts and traps.
+ *
+ * @see grp_trpm
+ *
+ *
+ * @section sec_trpm_rc     Raw-Mode Context
+ *
+ * When executing in the raw-mode context, TRPM will be managing the IDT and
+ * processing all traps and interrupts.  It will also monitor the guest IDT
+ * because CSAM wishes to know about changes to it (trap/interrupt/syscall
+ * handler patching) and TRPM needs to keep the \#BP gate in sync (ring-3
+ * considerations).  See TRPMR3SyncIDT and CSAMR3CheckGates.
+ *
+ * External interrupts will be forwarded to the host context by the quickest
+ * possible route where they will be reasserted.  The other events will be
+ * categorized into virtualization traps, genuine guest traps and hypervisor
+ * traps.  The latter group may be recoverable depending on when they happen and
+ * whether there is a handler for it, otherwise it will cause a guru meditation.
+ *
+ * TRPM distinguishes the between the first two (virt and guest traps) and the
+ * latter (hyper) by checking the CPL of the trapping code, if CPL == 0 then
+ * it's a hyper trap otherwise it's a virt/guest trap.  There are three trap
+ * dispatcher tables, one ad-hoc for one time traps registered via
+ * TRPMGCSetTempHandler(), one for hyper traps and one for virt/guest traps.
+ * The latter two live in TRPMGCHandlersA.asm, the former in the VM structure.
+ *
+ * The raw-mode context trap handlers found in TRPMGCHandlers.cpp (for the most
+ * part), will call up the other VMM sub-systems depending on what it things
+ * happens.  The two most busy traps are page faults (\#PF) and general
+ * protection fault/trap (\#GP).
+ *
+ * Before resuming guest code after having taken a virtualization trap or
+ * injected a guest trap, TRPM will check for pending forced action and
+ * every now and again let TM check for timed out timers.  This allows code that
+ * is being executed as part of virtualization traps to signal ring-3 exits,
+ * page table resyncs and similar without necessarily using the status code.  It
+ * also make sure we're more responsive to timers and requests from other
+ * threads (necessarily running on some different core/cpu in most cases).
+ *
+ *
+ * @section sec_trpm_all    All Contexts
+ *
+ * TRPM will also dispatch / inject interrupts and traps to the guest, both when
+ * in raw-mode and when in hardware assisted mode.  See TRPMInject().
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_TRPM
+#include <VBox/trpm.h>
+#include <VBox/cpum.h>
+#include <VBox/selm.h>
+#include <VBox/ssm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pgm.h>
+#include "internal/pgm.h"
+#include <VBox/dbgf.h>
+#include <VBox/mm.h>
+#include <VBox/stam.h>
+#include <VBox/csam.h>
+#include <VBox/patm.h>
+#include "TRPMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/em.h>
+#include <VBox/rem.h>
+#include <VBox/hwaccm.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/alloc.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Trap handler function.
+ * @todo need to specialize this as we go along.
+ */
+typedef enum TRPMHANDLER
+{
+    /** Generic Interrupt handler. */
+    TRPM_HANDLER_INT = 0,
+    /** Generic Trap handler. */
+    TRPM_HANDLER_TRAP,
+    /** Trap 8 (\#DF) handler. */
+    TRPM_HANDLER_TRAP_08,
+    /** Trap 12 (\#MC) handler. */
+    TRPM_HANDLER_TRAP_12,
+    /** Max. */
+    TRPM_HANDLER_MAX
+} TRPMHANDLER, *PTRPMHANDLER;
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/** Preinitialized IDT.
+ * The u16OffsetLow is a value of the TRPMHANDLER enum which TRPMR3Relocate()
+ * will use to pick the right address. The u16SegSel is always VMM CS.
+ */
+static VBOXIDTE_GENERIC     g_aIdt[256] =
+{
+/* special trap handler - still, this is an interrupt gate not a trap gate... */
+#define IDTE_TRAP(enm)  { (unsigned)enm,                    0, 0, VBOX_IDTE_TYPE1, VBOX_IDTE_TYPE2_INT_32, 0, 1, 0 }
+/* generic trap handler. */
+#define IDTE_TRAP_GEN() IDTE_TRAP(TRPM_HANDLER_TRAP)
+/* special interrupt handler. */
+#define IDTE_INT(enm)   { (unsigned)enm,                    0, 0, VBOX_IDTE_TYPE1, VBOX_IDTE_TYPE2_INT_32,  0, 1, 0 }
+/* generic interrupt handler. */
+#define IDTE_INT_GEN()  IDTE_INT(TRPM_HANDLER_INT)
+/* special task gate IDT entry (for critical exceptions like #DF). */
+#define IDTE_TASK(enm)  { (unsigned)enm,                    0, 0, VBOX_IDTE_TYPE1, VBOX_IDTE_TYPE2_TASK,    0, 1, 0 }
+/* draft, fixme later when the handler is written. */
+#define IDTE_RESERVED() { 0,                                0, 0,               0,                      0,  0, 0, 0 }
+
+                                        /* N  - M M -  T  - C - D i */
+                                        /* o  - n o -  y  - o - e p */
+                                        /*    - e n -  p  - d - s t */
+                                        /*    -   i -  e  - e - c . */
+                                        /*    -   c -     -   - r   */
+                                        /* ============================================================= */
+    IDTE_TRAP_GEN(),                    /*  0 - #DE - F   - N - Divide error */
+    IDTE_TRAP_GEN(),                    /*  1 - #DB - F/T - N - Single step, INT 1 instruction */
+#ifdef VBOX_WITH_NMI
+    IDTE_TRAP_GEN(),                    /*  2 -     - I   - N - Non-Maskable Interrupt (NMI) */
+#else
+    IDTE_INT_GEN(),                     /*  2 -     - I   - N - Non-Maskable Interrupt (NMI) */
+#endif
+    IDTE_TRAP_GEN(),                    /*  3 - #BP - T   - N - Breakpoint, INT 3 instruction. */
+    IDTE_TRAP_GEN(),                    /*  4 - #OF - T   - N - Overflow, INTO instruction. */
+    IDTE_TRAP_GEN(),                    /*  5 - #BR - F   - N - BOUND Range Exceeded, BOUND instruction. */
+    IDTE_TRAP_GEN(),                    /*  6 - #UD - F   - N - Undefined(/Invalid) Opcode. */
+    IDTE_TRAP_GEN(),                    /*  7 - #NM - F   - N - Device not available, FP or (F)WAIT instruction. */
+    IDTE_TASK(TRPM_HANDLER_TRAP_08),    /*  8 - #DF - A   - 0 - Double fault. */
+    IDTE_TRAP_GEN(),                    /*  9 -     - F   - N - Coprocessor Segment Overrun (obsolete). */
+    IDTE_TRAP_GEN(),                    /*  a - #TS - F   - Y - Invalid TSS, Taskswitch or TSS access. */
+    IDTE_TRAP_GEN(),                    /*  b - #NP - F   - Y - Segment not present. */
+    IDTE_TRAP_GEN(),                    /*  c - #SS - F   - Y - Stack-Segment fault. */
+    IDTE_TRAP_GEN(),                    /*  d - #GP - F   - Y - General protection fault. */
+    IDTE_TRAP_GEN(),                    /*  e - #PF - F   - Y - Page fault. - interrupt gate!!! */
+    IDTE_RESERVED(),                    /*  f -     -     -   - Intel Reserved. Do not use. */
+    IDTE_TRAP_GEN(),                    /* 10 - #MF - F   - N - x86 FPU Floating-Point Error (Math fault), FP or (F)WAIT instruction. */
+    IDTE_TRAP_GEN(),                    /* 11 - #AC - F   - 0 - Alignment Check. */
+    IDTE_TRAP(TRPM_HANDLER_TRAP_12),    /* 12 - #MC - A   - N - Machine Check. */
+    IDTE_TRAP_GEN(),                    /* 13 - #XF - F   - N - SIMD Floating-Point Exception. */
+    IDTE_RESERVED(),                    /* 14 -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 15 -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 16 -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 17 -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 18 -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 19 -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 1a -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 1b -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 1c -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 1d -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 1e -     -     -   - Intel Reserved. Do not use. */
+    IDTE_RESERVED(),                    /* 1f -     -     -   - Intel Reserved. Do not use. */
+    IDTE_INT_GEN(),                     /* 20 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 21 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 22 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 23 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 24 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 25 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 26 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 27 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 28 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 29 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 2a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 2b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 2c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 2d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 2e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 2f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 30 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 31 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 32 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 33 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 34 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 35 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 36 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 37 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 38 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 39 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 3a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 3b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 3c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 3d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 3e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 3f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 40 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 41 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 42 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 43 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 44 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 45 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 46 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 47 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 48 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 49 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 4a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 4b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 4c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 4d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 4e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 4f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 50 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 51 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 52 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 53 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 54 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 55 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 56 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 57 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 58 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 59 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 5a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 5b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 5c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 5d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 5e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 5f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 60 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 61 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 62 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 63 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 64 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 65 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 66 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 67 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 68 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 69 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 6a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 6b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 6c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 6d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 6e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 6f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 70 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 71 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 72 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 73 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 74 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 75 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 76 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 77 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 78 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 79 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 7a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 7b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 7c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 7d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 7e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 7f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 80 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 81 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 82 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 83 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 84 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 85 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 86 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 87 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 88 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 89 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 8a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 8b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 8c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 8d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 8e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 8f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 90 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 91 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 92 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 93 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 94 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 95 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 96 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 97 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 98 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 99 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 9a -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 9b -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 9c -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 9d -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 9e -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* 9f -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a0 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a1 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a2 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a3 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a4 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a5 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a6 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a7 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a8 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* a9 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* aa -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ab -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ac -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ad -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ae -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* af -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b0 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b1 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b2 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b3 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b4 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b5 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b6 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b7 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b8 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* b9 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ba -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* bb -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* bc -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* bd -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* be -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* bf -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c0 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c1 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c2 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c3 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c4 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c5 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c6 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c7 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c8 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* c9 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ca -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* cb -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* cc -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* cd -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ce -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* cf -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d0 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d1 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d2 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d3 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d4 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d5 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d6 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d7 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d8 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* d9 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* da -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* db -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* dc -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* dd -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* de -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* df -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e0 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e1 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e2 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e3 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e4 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e5 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e6 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e7 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e8 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* e9 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ea -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* eb -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ec -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ed -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ee -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ef -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f0 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f1 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f2 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f3 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f4 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f5 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f6 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f7 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f8 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* f9 -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* fa -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* fb -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* fc -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* fd -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* fe -     - I   -   - User defined Interrupts, external of INT n. */
+    IDTE_INT_GEN(),                     /* ff -     - I   -   - User defined Interrupts, external of INT n. */
+#undef IDTE_TRAP
+#undef IDTE_TRAP_GEN
+#undef IDTE_INT
+#undef IDTE_INT_GEN
+#undef IDTE_TASK
+#undef IDTE_UNUSED
+#undef IDTE_RESERVED
+};
+
+
+/** Enable or disable tracking of Guest's IDT. */
+#define TRPM_TRACK_GUEST_IDT_CHANGES
+
+/** Enable or disable tracking of Shadow IDT. */
+#define TRPM_TRACK_SHADOW_IDT_CHANGES
+
+/** TRPM saved state version. */
+#define TRPM_SAVED_STATE_VERSION        9
+#define TRPM_SAVED_STATE_VERSION_UNI    8   /* SMP support bumped the version */
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static DECLCALLBACK(int) trpmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int) trpmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(int) trpmR3GuestIDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
+
+
+/**
+ * Initializes the Trap Manager
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) TRPMR3Init(PVM pVM)
+{
+    LogFlow(("TRPMR3Init\n"));
+
+    /*
+     * Assert sizes and alignments.
+     */
+    AssertRelease(!(RT_OFFSETOF(VM, trpm.s) & 31));
+    AssertRelease(!(RT_OFFSETOF(VM, trpm.s.aIdt) & 15));
+    AssertRelease(sizeof(pVM->trpm.s) <= sizeof(pVM->trpm.padding));
+    AssertRelease(RT_ELEMENTS(pVM->trpm.s.aGuestTrapHandler) == sizeof(pVM->trpm.s.au32IdtPatched)*8);
+
+    /*
+     * Initialize members.
+     */
+    pVM->trpm.s.offVM              = RT_OFFSETOF(VM, trpm);
+    pVM->trpm.s.offTRPMCPU         = RT_OFFSETOF(VM, aCpus[0].trpm) - RT_OFFSETOF(VM, trpm);
+
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        pVCpu->trpm.s.offVM         = RT_OFFSETOF(VM, aCpus[i].trpm);
+        pVCpu->trpm.s.offVMCpu      = RT_OFFSETOF(VMCPU, trpm);
+        pVCpu->trpm.s.uActiveVector = ~0;
+    }
+
+    pVM->trpm.s.GuestIdtr.pIdt     = RTRCPTR_MAX;
+    pVM->trpm.s.pvMonShwIdtRC            = RTRCPTR_MAX;
+    pVM->trpm.s.fDisableMonitoring = false;
+    pVM->trpm.s.fSafeToDropGuestIDTMonitoring = false;
+
+    /*
+     * Read the configuration (if any).
+     */
+    PCFGMNODE pTRPMNode = CFGMR3GetChild(CFGMR3GetRoot(pVM), "TRPM");
+    if (pTRPMNode)
+    {
+        bool f;
+        int rc = CFGMR3QueryBool(pTRPMNode, "SafeToDropGuestIDTMonitoring", &f);
+        if (RT_SUCCESS(rc))
+            pVM->trpm.s.fSafeToDropGuestIDTMonitoring = f;
+    }
+
+    /* write config summary to log */
+    if (pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
+        LogRel(("TRPM: Dropping Guest IDT Monitoring.\n"));
+
+    /*
+     * Initialize the IDT.
+     * The handler addresses will be set in the TRPMR3Relocate() function.
+     */
+    Assert(sizeof(pVM->trpm.s.aIdt) == sizeof(g_aIdt));
+    memcpy(&pVM->trpm.s.aIdt[0], &g_aIdt[0], sizeof(pVM->trpm.s.aIdt));
+
+    /*
+     * Register the saved state data unit.
+     */
+    int rc = SSMR3RegisterInternal(pVM, "trpm", 1, TRPM_SAVED_STATE_VERSION, sizeof(TRPM),
+                                   NULL, NULL, NULL,
+                                   NULL, trpmR3Save, NULL,
+                                   NULL, trpmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Statistics.
+     */
+    STAM_REG(pVM, &pVM->trpm.s.StatRCWriteGuestIDTFault,    STAMTYPE_COUNTER, "/TRPM/RC/IDTWritesFault",    STAMUNIT_OCCURENCES,     "Guest IDT writes the we returned to R3 to handle.");
+    STAM_REG(pVM, &pVM->trpm.s.StatRCWriteGuestIDTHandled,  STAMTYPE_COUNTER, "/TRPM/RC/IDTWritesHandled",  STAMUNIT_OCCURENCES,     "Guest IDT writes that we handled successfully.");
+    STAM_REG(pVM, &pVM->trpm.s.StatSyncIDT,                 STAMTYPE_PROFILE, "/PROF/TRPM/SyncIDT",         STAMUNIT_TICKS_PER_CALL, "Profiling of TRPMR3SyncIDT().");
+
+    /* traps */
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x00],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/00",          STAMUNIT_TICKS_PER_CALL, "#DE - Divide error.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x01],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/01",          STAMUNIT_TICKS_PER_CALL, "#DB - Debug (single step and more).");
+    //STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x02],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/02",          STAMUNIT_TICKS_PER_CALL, "NMI");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x03],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/03",          STAMUNIT_TICKS_PER_CALL, "#BP - Breakpoint.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x04],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/04",          STAMUNIT_TICKS_PER_CALL, "#OF - Overflow.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x05],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/05",          STAMUNIT_TICKS_PER_CALL, "#BR - Bound range exceeded.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x06],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/06",          STAMUNIT_TICKS_PER_CALL, "#UD - Undefined opcode.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x07],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/07",          STAMUNIT_TICKS_PER_CALL, "#NM - Device not available (FPU).");
+    //STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x08],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/08",          STAMUNIT_TICKS_PER_CALL, "#DF - Double fault.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x09],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/09",          STAMUNIT_TICKS_PER_CALL, "#?? - Coprocessor segment overrun (obsolete).");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0a],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0a",          STAMUNIT_TICKS_PER_CALL, "#TS - Task switch fault.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0b],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0b",          STAMUNIT_TICKS_PER_CALL, "#NP - Segment not present.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0c],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0c",          STAMUNIT_TICKS_PER_CALL, "#SS - Stack segment fault.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0d],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0d",          STAMUNIT_TICKS_PER_CALL, "#GP - General protection fault.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0e],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0e",          STAMUNIT_TICKS_PER_CALL, "#PF - Page fault.");
+    //STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x0f],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/0f",          STAMUNIT_TICKS_PER_CALL, "Reserved.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x10],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/10",          STAMUNIT_TICKS_PER_CALL, "#MF - Math fault..");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x11],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/11",          STAMUNIT_TICKS_PER_CALL, "#AC - Alignment check.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x12],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/12",          STAMUNIT_TICKS_PER_CALL, "#MC - Machine check.");
+    STAM_REG(pVM, &pVM->trpm.s.aStatGCTraps[0x13],      STAMTYPE_PROFILE_ADV, "/TRPM/GC/Traps/13",          STAMUNIT_TICKS_PER_CALL, "#XF - SIMD Floating-Point Exception.");
+
+#ifdef VBOX_WITH_STATISTICS
+    rc = MMHyperAlloc(pVM, sizeof(STAMCOUNTER) * 255, 8, MM_TAG_STAM, (void **)&pVM->trpm.s.paStatForwardedIRQR3);
+    AssertRCReturn(rc, rc);
+    pVM->trpm.s.paStatForwardedIRQRC = MMHyperR3ToRC(pVM, pVM->trpm.s.paStatForwardedIRQR3);
+    pVM->trpm.s.paStatForwardedIRQR0 = MMHyperR3ToR0(pVM, pVM->trpm.s.paStatForwardedIRQR3);
+    for (unsigned i = 0; i < 255; i++)
+        STAMR3RegisterF(pVM, &pVM->trpm.s.paStatForwardedIRQR3[i], STAMTYPE_COUNTER, STAMVISIBILITY_USED, STAMUNIT_OCCURENCES, "Forwarded interrupts.",
+                        i < 0x20 ? "/TRPM/ForwardRaw/TRAP/%02X" : "/TRPM/ForwardRaw/IRQ/%02X", i);
+#endif
+
+    STAM_REG(pVM, &pVM->trpm.s.StatForwardProfR3,       STAMTYPE_PROFILE_ADV, "/TRPM/ForwardRaw/ProfR3",   STAMUNIT_TICKS_PER_CALL, "Profiling TRPMForwardTrap.");
+    STAM_REG(pVM, &pVM->trpm.s.StatForwardProfRZ,       STAMTYPE_PROFILE_ADV, "/TRPM/ForwardRaw/ProfRZ",   STAMUNIT_TICKS_PER_CALL, "Profiling TRPMForwardTrap.");
+    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailNoHandler,    STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailNoHandler", STAMUNIT_OCCURENCES,"Failure to forward interrupt in raw mode.");
+    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailPatchAddr,    STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailPatchAddr", STAMUNIT_OCCURENCES,"Failure to forward interrupt in raw mode.");
+    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailR3,           STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailR3",       STAMUNIT_OCCURENCES, "Failure to forward interrupt in raw mode.");
+    STAM_REG(pVM, &pVM->trpm.s.StatForwardFailRZ,           STAMTYPE_COUNTER, "/TRPM/ForwardRaw/FailRZ",       STAMUNIT_OCCURENCES, "Failure to forward interrupt in raw mode.");
+
+    STAM_REG(pVM, &pVM->trpm.s.StatTrap0dDisasm,            STAMTYPE_PROFILE, "/TRPM/RC/Traps/0d/Disasm",   STAMUNIT_TICKS_PER_CALL, "Profiling disassembly part of trpmGCTrap0dHandler.");
+    STAM_REG(pVM, &pVM->trpm.s.StatTrap0dRdTsc,             STAMTYPE_COUNTER, "/TRPM/RC/Traps/0d/RdTsc",        STAMUNIT_OCCURENCES, "Number of RDTSC #GPs.");
+
+    /*
+     * Default action when entering raw mode for the first time
+     */
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+    return 0;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this component.
+ *
+ * This function will be called at init and whenever the VMM need
+ * to relocate itself inside the GC.
+ *
+ * @param   pVM         The VM handle.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void) TRPMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    /* Only applies to raw mode which supports only 1 VCPU. */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    LogFlow(("TRPMR3Relocate\n"));
+    /*
+     * Get the trap handler addresses.
+     *
+     * If VMMGC.gc is screwed, so are we. We'll assert here since it elsewise
+     * would make init order impossible if we should assert the presence of these
+     * exports in TRPMR3Init().
+     */
+    RTRCPTR aRCPtrs[TRPM_HANDLER_MAX];
+    RT_ZERO(aRCPtrs);
+    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerInterupt", &aRCPtrs[TRPM_HANDLER_INT]);
+    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerInterupt in VMMGC.gc!\n"));
+
+    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerGeneric",  &aRCPtrs[TRPM_HANDLER_TRAP]);
+    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerGeneric in VMMGC.gc!\n"));
+
+    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerTrap08",   &aRCPtrs[TRPM_HANDLER_TRAP_08]);
+    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerTrap08 in VMMGC.gc!\n"));
+
+    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerTrap12",   &aRCPtrs[TRPM_HANDLER_TRAP_12]);
+    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerTrap12 in VMMGC.gc!\n"));
+
+    RTSEL SelCS = CPUMGetHyperCS(pVCpu);
+
+    /*
+     * Iterate the idt and set the addresses.
+     */
+    PVBOXIDTE pIdte = &pVM->trpm.s.aIdt[0];
+    PVBOXIDTE_GENERIC pIdteTemplate = &g_aIdt[0];
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->trpm.s.aIdt); i++, pIdte++, pIdteTemplate++)
+    {
+        if (    pIdte->Gen.u1Present
+            &&  !ASMBitTest(&pVM->trpm.s.au32IdtPatched[0], i)
+           )
+        {
+            Assert(pIdteTemplate->u16OffsetLow < TRPM_HANDLER_MAX);
+            RTGCPTR Offset = aRCPtrs[pIdteTemplate->u16OffsetLow];
+            switch (pIdteTemplate->u16OffsetLow)
+            {
+                /*
+                 * Generic handlers have different entrypoints for each possible
+                 * vector number. These entrypoints makes a sort of an array with
+                 * 8 byte entries where the vector number is the index.
+                 * See TRPMGCHandlersA.asm for details.
+                 */
+                case TRPM_HANDLER_INT:
+                case TRPM_HANDLER_TRAP:
+                    Offset += i * 8;
+                    break;
+                case TRPM_HANDLER_TRAP_12:
+                    break;
+                case TRPM_HANDLER_TRAP_08:
+                    /* Handle #DF Task Gate in special way. */
+                    pIdte->Gen.u16SegSel     = SELMGetTrap8Selector(pVM);
+                    pIdte->Gen.u16OffsetLow  = 0;
+                    pIdte->Gen.u16OffsetHigh = 0;
+                    SELMSetTrap8EIP(pVM, Offset);
+                    continue;
+            }
+            /* (non-task gates only ) */
+            pIdte->Gen.u16OffsetLow  = Offset & 0xffff;
+            pIdte->Gen.u16OffsetHigh = Offset >> 16;
+            pIdte->Gen.u16SegSel     = SelCS;
+        }
+    }
+
+    /*
+     * Update IDTR (limit is including!).
+     */
+    CPUMSetHyperIDTR(pVCpu, VM_RC_ADDR(pVM, &pVM->trpm.s.aIdt[0]), sizeof(pVM->trpm.s.aIdt)-1);
+
+    if (    !pVM->trpm.s.fDisableMonitoring
+        &&  !VMMIsHwVirtExtForced(pVM))
+    {
+#ifdef TRPM_TRACK_SHADOW_IDT_CHANGES
+        if (pVM->trpm.s.pvMonShwIdtRC != RTRCPTR_MAX)
+        {
+            rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.pvMonShwIdtRC);
+            AssertRC(rc);
+        }
+        pVM->trpm.s.pvMonShwIdtRC = VM_RC_ADDR(pVM, &pVM->trpm.s.aIdt[0]);
+        rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_HYPERVISOR, pVM->trpm.s.pvMonShwIdtRC, pVM->trpm.s.pvMonShwIdtRC + sizeof(pVM->trpm.s.aIdt) - 1,
+                                         0, 0, "trpmRCShadowIDTWriteHandler", 0, "Shadow IDT write access handler");
+        AssertRC(rc);
+#endif
+    }
+
+    /* Relocate IDT handlers for forwarding guest traps/interrupts. */
+    for (uint32_t iTrap = 0; iTrap < RT_ELEMENTS(pVM->trpm.s.aGuestTrapHandler); iTrap++)
+    {
+        if (pVM->trpm.s.aGuestTrapHandler[iTrap] != TRPM_INVALID_HANDLER)
+        {
+            Log(("TRPMR3Relocate: iGate=%2X Handler %RRv -> %RRv\n", iTrap, pVM->trpm.s.aGuestTrapHandler[iTrap], pVM->trpm.s.aGuestTrapHandler[iTrap] + offDelta));
+            pVM->trpm.s.aGuestTrapHandler[iTrap] += offDelta;
+        }
+
+        if (ASMBitTest(&pVM->trpm.s.au32IdtPatched[0], iTrap))
+        {
+            PVBOXIDTE   pIdteCur = &pVM->trpm.s.aIdt[iTrap];
+            RTGCPTR     pHandler = VBOXIDTE_OFFSET(*pIdteCur);
+
+            Log(("TRPMR3Relocate: *iGate=%2X Handler %RGv -> %RGv\n", iTrap, pHandler, pHandler + offDelta));
+            pHandler += offDelta;
+
+            pIdteCur->Gen.u16OffsetHigh = pHandler >> 16;
+            pIdteCur->Gen.u16OffsetLow  = pHandler & 0xFFFF;
+        }
+    }
+
+#ifdef VBOX_WITH_STATISTICS
+    pVM->trpm.s.paStatForwardedIRQRC += offDelta;
+    pVM->trpm.s.paStatForwardedIRQR0 = MMHyperR3ToR0(pVM, pVM->trpm.s.paStatForwardedIRQR3);
+#endif
+}
+
+
+/**
+ * Terminates the Trap Manager
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(int) TRPMR3Term(PVM pVM)
+{
+    NOREF(pVM);
+    return 0;
+}
+
+
+/**
+ * Resets a virtual CPU.
+ *
+ * Used by TRPMR3Reset and CPU hot plugging.
+ *
+ * @param   pVCpu               The virtual CPU handle.
+ */
+VMMR3DECL(void) TRPMR3ResetCpu(PVMCPU pVCpu)
+{
+    pVCpu->trpm.s.uActiveVector = ~0;
+}
+
+
+/**
+ * The VM is being reset.
+ *
+ * For the TRPM component this means that any IDT write monitors
+ * needs to be removed, any pending trap cleared, and the IDT reset.
+ *
+ * @param   pVM     VM handle.
+ */
+VMMR3DECL(void) TRPMR3Reset(PVM pVM)
+{
+    /*
+     * Deregister any virtual handlers.
+     */
+#ifdef TRPM_TRACK_GUEST_IDT_CHANGES
+    if (pVM->trpm.s.GuestIdtr.pIdt != RTRCPTR_MAX)
+    {
+        if (!pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
+        {
+            int rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.GuestIdtr.pIdt);
+            AssertRC(rc);
+        }
+        pVM->trpm.s.GuestIdtr.pIdt = RTRCPTR_MAX;
+    }
+    pVM->trpm.s.GuestIdtr.cbIdt = 0;
+#endif
+
+    /*
+     * Reinitialize other members calling the relocator to get things right.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        TRPMR3ResetCpu(&pVM->aCpus[i]);
+    memcpy(&pVM->trpm.s.aIdt[0], &g_aIdt[0], sizeof(pVM->trpm.s.aIdt));
+    memset(pVM->trpm.s.aGuestTrapHandler, 0, sizeof(pVM->trpm.s.aGuestTrapHandler));
+    TRPMR3Relocate(pVM, 0);
+
+    /*
+     * Default action when entering raw mode for the first time
+     */
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) trpmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    PTRPM pTrpm = &pVM->trpm.s;
+    LogFlow(("trpmR3Save:\n"));
+
+    /*
+     * Active and saved traps.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PTRPMCPU pTrpmCpu = &pVM->aCpus[i].trpm.s;
+        SSMR3PutUInt(pSSM,      pTrpmCpu->uActiveVector);
+        SSMR3PutUInt(pSSM,      pTrpmCpu->enmActiveType);
+        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uActiveErrorCode);
+        SSMR3PutGCUIntPtr(pSSM, pTrpmCpu->uActiveCR2);
+        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uSavedVector);
+        SSMR3PutUInt(pSSM,      pTrpmCpu->enmSavedType);
+        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uSavedErrorCode);
+        SSMR3PutGCUIntPtr(pSSM, pTrpmCpu->uSavedCR2);
+        SSMR3PutGCUInt(pSSM,    pTrpmCpu->uPrevVector);
+    }
+    SSMR3PutBool(pSSM,      pTrpm->fDisableMonitoring);
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies 1 VCPU */
+    SSMR3PutUInt(pSSM,      VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT));
+    SSMR3PutMem(pSSM,       &pTrpm->au32IdtPatched[0], sizeof(pTrpm->au32IdtPatched));
+    SSMR3PutU32(pSSM, ~0);              /* separator. */
+
+    /*
+     * Save any trampoline gates.
+     */
+    for (uint32_t iTrap = 0; iTrap < RT_ELEMENTS(pTrpm->aGuestTrapHandler); iTrap++)
+    {
+        if (pTrpm->aGuestTrapHandler[iTrap])
+        {
+            SSMR3PutU32(pSSM, iTrap);
+            SSMR3PutGCPtr(pSSM, pTrpm->aGuestTrapHandler[iTrap]);
+            SSMR3PutMem(pSSM, &pTrpm->aIdt[iTrap], sizeof(pTrpm->aIdt[iTrap]));
+        }
+    }
+
+    return SSMR3PutU32(pSSM, ~0);       /* terminator */
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) trpmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("trpmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (    uVersion != TRPM_SAVED_STATE_VERSION
+        &&  uVersion != TRPM_SAVED_STATE_VERSION_UNI)
+    {
+        AssertMsgFailed(("trpmR3Load: Invalid version uVersion=%d!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    /*
+     * Call the reset function to kick out any handled gates and other potential trouble.
+     */
+    TRPMR3Reset(pVM);
+
+    /*
+     * Active and saved traps.
+     */
+    PTRPM pTrpm = &pVM->trpm.s;
+
+    if (uVersion == TRPM_SAVED_STATE_VERSION)
+    {
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PTRPMCPU pTrpmCpu = &pVM->aCpus[i].trpm.s;
+            SSMR3GetUInt(pSSM,      &pTrpmCpu->uActiveVector);
+            SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmActiveType);
+            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uActiveErrorCode);
+            SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uActiveCR2);
+            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedVector);
+            SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmSavedType);
+            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedErrorCode);
+            SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uSavedCR2);
+            SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uPrevVector);
+        }
+
+        SSMR3GetBool(pSSM,      &pVM->trpm.s.fDisableMonitoring);
+    }
+    else
+    {
+        PTRPMCPU pTrpmCpu = &pVM->aCpus[0].trpm.s;
+        SSMR3GetUInt(pSSM,      &pTrpmCpu->uActiveVector);
+        SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmActiveType);
+        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uActiveErrorCode);
+        SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uActiveCR2);
+        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedVector);
+        SSMR3GetUInt(pSSM,      (uint32_t *)&pTrpmCpu->enmSavedType);
+        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uSavedErrorCode);
+        SSMR3GetGCUIntPtr(pSSM, &pTrpmCpu->uSavedCR2);
+        SSMR3GetGCUInt(pSSM,    &pTrpmCpu->uPrevVector);
+
+        RTGCUINT fDisableMonitoring;
+        SSMR3GetGCUInt(pSSM,    &fDisableMonitoring);
+        pTrpm->fDisableMonitoring = !!fDisableMonitoring;
+    }
+
+    RTUINT fSyncIDT;
+    int rc = SSMR3GetUInt(pSSM, &fSyncIDT);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (fSyncIDT & ~1)
+    {
+        AssertMsgFailed(("fSyncIDT=%#x\n", fSyncIDT));
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    }
+    if (fSyncIDT)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies 1 VCPU */
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+    }
+    /* else: cleared by reset call above. */
+
+    SSMR3GetMem(pSSM, &pTrpm->au32IdtPatched[0], sizeof(pTrpm->au32IdtPatched));
+
+    /* check the separator */
+    uint32_t u32Sep;
+    rc = SSMR3GetU32(pSSM, &u32Sep);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (u32Sep != (uint32_t)~0)
+    {
+        AssertMsgFailed(("u32Sep=%#x (first)\n", u32Sep));
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    }
+
+    /*
+     * Restore any trampoline gates.
+     */
+    for (;;)
+    {
+        /* gate number / terminator */
+        uint32_t iTrap;
+        rc = SSMR3GetU32(pSSM, &iTrap);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (iTrap == (uint32_t)~0)
+            break;
+        if (    iTrap >= RT_ELEMENTS(pTrpm->aIdt)
+            ||  pTrpm->aGuestTrapHandler[iTrap])
+        {
+            AssertMsgFailed(("iTrap=%#x\n", iTrap));
+            return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+        }
+
+        /* restore the IDT entry. */
+        RTGCPTR GCPtrHandler;
+        SSMR3GetGCPtr(pSSM, &GCPtrHandler);
+        VBOXIDTE Idte;
+        rc = SSMR3GetMem(pSSM, &Idte, sizeof(Idte));
+        if (RT_FAILURE(rc))
+            return rc;
+        Assert(GCPtrHandler);
+        pTrpm->aIdt[iTrap] = Idte;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Check if gate handlers were updated
+ * (callback for the VMCPU_FF_TRPM_SYNC_IDT forced action).
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The VMCPU handle.
+ */
+VMMR3DECL(int) TRPMR3SyncIDT(PVM pVM, PVMCPU pVCpu)
+{
+    STAM_PROFILE_START(&pVM->trpm.s.StatSyncIDT, a);
+    const bool  fRawRing0 = EMIsRawRing0Enabled(pVM);
+    int         rc;
+
+    if (pVM->trpm.s.fDisableMonitoring)
+    {
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+        return VINF_SUCCESS;    /* Nothing to do */
+    }
+
+    if (fRawRing0 && CSAMIsEnabled(pVM))
+    {
+        /* Clear all handlers */
+        Log(("TRPMR3SyncIDT: Clear all trap handlers.\n"));
+        /** @todo inefficient, but simple */
+        for (unsigned iGate = 0; iGate < 256; iGate++)
+            trpmClearGuestTrapHandler(pVM, iGate);
+
+        /* Scan them all (only the first time) */
+        CSAMR3CheckGates(pVM, 0, 256);
+    }
+
+    /*
+     * Get the IDTR.
+     */
+    VBOXIDTR IDTR;
+    IDTR.pIdt = CPUMGetGuestIDTR(pVCpu, &IDTR.cbIdt);
+    if (!IDTR.cbIdt)
+    {
+        Log(("No IDT entries...\n"));
+        return DBGFSTOP(pVM);
+    }
+
+#ifdef TRPM_TRACK_GUEST_IDT_CHANGES
+    /*
+     * Check if Guest's IDTR has changed.
+     */
+    if (    IDTR.pIdt  != pVM->trpm.s.GuestIdtr.pIdt
+        ||  IDTR.cbIdt != pVM->trpm.s.GuestIdtr.cbIdt)
+    {
+        Log(("TRPMR3UpdateFromCPUM: Guest's IDT is changed to pIdt=%08X cbIdt=%08X\n", IDTR.pIdt, IDTR.cbIdt));
+        if (!pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
+        {
+            /*
+             * [Re]Register write virtual handler for guest's IDT.
+             */
+            if (pVM->trpm.s.GuestIdtr.pIdt != RTRCPTR_MAX)
+            {
+                rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.GuestIdtr.pIdt);
+                AssertRCReturn(rc, rc);
+            }
+            /* limit is including */
+            rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, IDTR.pIdt, IDTR.pIdt + IDTR.cbIdt /* already inclusive */,
+                                             0, trpmR3GuestIDTWriteHandler, "trpmRCGuestIDTWriteHandler", 0, "Guest IDT write access handler");
+
+            if (rc == VERR_PGM_HANDLER_VIRTUAL_CONFLICT)
+            {
+                /* Could be a conflict with CSAM */
+                CSAMR3RemovePage(pVM, IDTR.pIdt);
+                if (PAGE_ADDRESS(IDTR.pIdt) != PAGE_ADDRESS(IDTR.pIdt + IDTR.cbIdt))
+                    CSAMR3RemovePage(pVM, IDTR.pIdt + IDTR.cbIdt);
+
+                rc = PGMR3HandlerVirtualRegister(pVM, PGMVIRTHANDLERTYPE_WRITE, IDTR.pIdt, IDTR.pIdt + IDTR.cbIdt /* already inclusive */,
+                                                 0, trpmR3GuestIDTWriteHandler, "trpmRCGuestIDTWriteHandler", 0, "Guest IDT write access handler");
+            }
+
+            AssertRCReturn(rc, rc);
+        }
+
+        /* Update saved Guest IDTR. */
+        pVM->trpm.s.GuestIdtr = IDTR;
+    }
+#endif
+
+    /*
+     * Sync the interrupt gate.
+     * Should probably check/sync the others too, but for now we'll handle that in #GP.
+     */
+    X86DESC  Idte3;
+    rc = PGMPhysSimpleReadGCPtr(pVCpu, &Idte3, IDTR.pIdt + sizeof(Idte3) * 3,  sizeof(Idte3));
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgRC(rc, ("Failed to read IDT[3]! rc=%Rrc\n", rc));
+        return DBGFSTOP(pVM);
+    }
+    AssertRCReturn(rc, rc);
+    if (fRawRing0)
+        pVM->trpm.s.aIdt[3].Gen.u2DPL = RT_MAX(Idte3.Gen.u2Dpl, 1);
+    else
+        pVM->trpm.s.aIdt[3].Gen.u2DPL = Idte3.Gen.u2Dpl;
+
+    /*
+     * Clear the FF and we're done.
+     */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+    STAM_PROFILE_STOP(&pVM->trpm.s.StatSyncIDT, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Disable IDT monitoring and syncing
+ *
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3DECL(void) TRPMR3DisableMonitoring(PVM pVM)
+{
+    /*
+     * Deregister any virtual handlers.
+     */
+#ifdef TRPM_TRACK_GUEST_IDT_CHANGES
+    if (pVM->trpm.s.GuestIdtr.pIdt != RTRCPTR_MAX)
+    {
+        if (!pVM->trpm.s.fSafeToDropGuestIDTMonitoring)
+        {
+            int rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.GuestIdtr.pIdt);
+            AssertRC(rc);
+        }
+        pVM->trpm.s.GuestIdtr.pIdt = RTRCPTR_MAX;
+    }
+    pVM->trpm.s.GuestIdtr.cbIdt = 0;
+#endif
+
+#ifdef TRPM_TRACK_SHADOW_IDT_CHANGES
+    if (pVM->trpm.s.pvMonShwIdtRC != RTRCPTR_MAX)
+    {
+        int rc = PGMHandlerVirtualDeregister(pVM, pVM->trpm.s.pvMonShwIdtRC);
+        AssertRC(rc);
+        pVM->trpm.s.pvMonShwIdtRC = RTRCPTR_MAX;
+    }
+#endif
+
+    PVMCPU pVCpu = &pVM->aCpus[0];  /* raw mode implies on VCPU */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+
+    pVM->trpm.s.fDisableMonitoring = true;
+}
+
+
+/**
+ * \#PF Handler callback for virtual access handler ranges.
+ *
+ * Important to realize that a physical page in a range can have aliases, and
+ * for ALL and WRITE handlers these will also trigger.
+ *
+ * @returns VINF_SUCCESS if the handler have carried out the operation.
+ * @returns VINF_PGM_HANDLER_DO_DEFAULT if the caller should carry out the access operation.
+ * @param   pVM             VM Handle.
+ * @param   GCPtr           The virtual address the guest is writing to. (not correct if it's an alias!)
+ * @param   pvPtr           The HC mapping of that address.
+ * @param   pvBuf           What the guest is reading/writing.
+ * @param   cbBuf           How much it's reading/writing.
+ * @param   enmAccessType   The access type.
+ * @param   pvUser          User argument.
+ */
+static DECLCALLBACK(int) trpmR3GuestIDTWriteHandler(PVM pVM, RTGCPTR GCPtr, void *pvPtr, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser)
+{
+    Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+    Log(("trpmR3GuestIDTWriteHandler: write to %RGv size %d\n", GCPtr, cbBuf));
+    VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TRPM_SYNC_IDT);
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+/**
+ * Clear passthrough interrupt gate handler (reset to default handler)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   iTrap       Trap/interrupt gate number.
+ */
+VMMR3DECL(int) trpmR3ClearPassThroughHandler(PVM pVM, unsigned iTrap)
+{
+    /* Only applies to raw mode which supports only 1 VCPU. */
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    /** @todo cleanup trpmR3ClearPassThroughHandler()! */
+    RTRCPTR aGCPtrs[TRPM_HANDLER_MAX];
+    int rc;
+
+    memset(aGCPtrs, 0, sizeof(aGCPtrs));
+
+    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "TRPMGCHandlerInterupt", &aGCPtrs[TRPM_HANDLER_INT]);
+    AssertReleaseMsgRC(rc, ("Couldn't find TRPMGCHandlerInterupt in VMMGC.gc!\n"));
+
+    if (    iTrap < TRPM_HANDLER_INT_BASE
+        ||  iTrap >= RT_ELEMENTS(pVM->trpm.s.aIdt))
+    {
+        AssertMsg(iTrap < TRPM_HANDLER_INT_BASE, ("Illegal gate number %#x!\n", iTrap));
+        return VERR_INVALID_PARAMETER;
+    }
+    memcpy(&pVM->trpm.s.aIdt[iTrap], &g_aIdt[iTrap], sizeof(pVM->trpm.s.aIdt[0]));
+
+    /* Unmark it for relocation purposes. */
+    ASMBitClear(&pVM->trpm.s.au32IdtPatched[0], iTrap);
+
+    RTSEL               SelCS         = CPUMGetHyperCS(pVCpu);
+    PVBOXIDTE           pIdte         = &pVM->trpm.s.aIdt[iTrap];
+    PVBOXIDTE_GENERIC   pIdteTemplate = &g_aIdt[iTrap];
+    if (pIdte->Gen.u1Present)
+    {
+        Assert(pIdteTemplate->u16OffsetLow == TRPM_HANDLER_INT);
+        Assert(sizeof(RTRCPTR) == sizeof(aGCPtrs[0]));
+        RTRCPTR Offset = (RTRCPTR)aGCPtrs[pIdteTemplate->u16OffsetLow];
+
+        /*
+         * Generic handlers have different entrypoints for each possible
+         * vector number. These entrypoints make a sort of an array with
+         * 8 byte entries where the vector number is the index.
+         * See TRPMGCHandlersA.asm for details.
+         */
+        Offset += iTrap * 8;
+
+        if (pIdte->Gen.u5Type2 != VBOX_IDTE_TYPE2_TASK)
+        {
+            pIdte->Gen.u16OffsetLow  = Offset & 0xffff;
+            pIdte->Gen.u16OffsetHigh = Offset >> 16;
+            pIdte->Gen.u16SegSel     = SelCS;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Check if address is a gate handler (interrupt or trap).
+ *
+ * @returns gate nr or ~0 is not found
+ *
+ * @param   pVM         VM handle.
+ * @param   GCPtr       GC address to check.
+ */
+VMMR3DECL(uint32_t) TRPMR3QueryGateByHandler(PVM pVM, RTRCPTR GCPtr)
+{
+    for (uint32_t iTrap = 0; iTrap < RT_ELEMENTS(pVM->trpm.s.aGuestTrapHandler); iTrap++)
+    {
+        if (pVM->trpm.s.aGuestTrapHandler[iTrap] == GCPtr)
+            return iTrap;
+
+        /* redundant */
+        if (ASMBitTest(&pVM->trpm.s.au32IdtPatched[0], iTrap))
+        {
+            PVBOXIDTE   pIdte = &pVM->trpm.s.aIdt[iTrap];
+            RTGCPTR     pHandler = VBOXIDTE_OFFSET(*pIdte);
+
+            if (pHandler == GCPtr)
+                return iTrap;
+        }
+    }
+    return ~0;
+}
+
+
+/**
+ * Get guest trap/interrupt gate handler
+ *
+ * @returns Guest trap handler address or TRPM_INVALID_HANDLER if none installed
+ * @param   pVM         The VM to operate on.
+ * @param   iTrap       Interrupt/trap number.
+ */
+VMMR3DECL(RTRCPTR) TRPMR3GetGuestTrapHandler(PVM pVM, unsigned iTrap)
+{
+    AssertReturn(iTrap < RT_ELEMENTS(pVM->trpm.s.aIdt), TRPM_INVALID_HANDLER);
+
+    return pVM->trpm.s.aGuestTrapHandler[iTrap];
+}
+
+
+/**
+ * Set guest trap/interrupt gate handler
+ * Used for setting up trap gates used for kernel calls.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   iTrap       Interrupt/trap number.
+ * @param   pHandler    GC handler pointer
+ */
+VMMR3DECL(int) TRPMR3SetGuestTrapHandler(PVM pVM, unsigned iTrap, RTRCPTR pHandler)
+{
+    /* Only valid in raw mode which implies 1 VCPU */
+    Assert(PATMIsEnabled(pVM) && pVM->cCpus == 1);
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    /*
+     * Validate.
+     */
+    if (iTrap >= RT_ELEMENTS(pVM->trpm.s.aIdt))
+    {
+        AssertMsg(iTrap < TRPM_HANDLER_INT_BASE, ("Illegal gate number %d!\n", iTrap));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    AssertReturn(pHandler == TRPM_INVALID_HANDLER || PATMIsPatchGCAddr(pVM, pHandler), VERR_INVALID_PARAMETER);
+
+    uint16_t    cbIDT;
+    RTGCPTR     GCPtrIDT = CPUMGetGuestIDTR(pVCpu, &cbIDT);
+    if (iTrap * sizeof(VBOXIDTE) >= cbIDT)
+        return VERR_INVALID_PARAMETER;  /* Silently ignore out of range requests. */
+
+    if (pHandler == TRPM_INVALID_HANDLER)
+    {
+        /* clear trap handler */
+        Log(("TRPMR3SetGuestTrapHandler: clear handler %x\n", iTrap));
+        return trpmClearGuestTrapHandler(pVM, iTrap);
+    }
+
+    /*
+     * Read the guest IDT entry.
+     */
+    VBOXIDTE GuestIdte;
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &GuestIdte, GCPtrIDT + iTrap * sizeof(GuestIdte),  sizeof(GuestIdte));
+    if (RT_FAILURE(rc))
+    {
+        AssertMsgRC(rc, ("Failed to read IDTE! rc=%Rrc\n", rc));
+        return rc;
+    }
+
+    if (EMIsRawRing0Enabled(pVM))
+    {
+        /*
+         * Only replace handlers for which we are 100% certain there won't be
+         * any host interrupts.
+         *
+         * 0x2E is safe on Windows because it's the system service interrupt gate. Not
+         * quite certain if this is safe or not on 64-bit Vista, it probably is.
+         *
+         * 0x80 is safe on Linux because it's the syscall vector and is part of the
+         * 32-bit usermode ABI. 64-bit Linux (usually) supports 32-bit processes
+         * and will therefor never assign hardware interrupts to 0x80.
+         *
+         * Exactly why 0x80 is safe on 32-bit Windows is a bit hazy, but it seems
+         * to work ok... However on 64-bit Vista (SMP?) is doesn't work reliably.
+         * Booting Linux/BSD guest will cause system lockups on most of the computers.
+         * -> Update: It seems gate 0x80 is not safe on 32-bits Windows either. See
+         *            defect #3604.
+         *
+         * PORTME - Check if your host keeps any of these gates free from hw ints.
+         *
+         * Note! SELMR3SyncTSS also has code related to this interrupt handler replacing.
+         */
+        /** @todo handle those dependencies better! */
+        /** @todo Solve this in a proper manner. see defect #1186 */
+#if defined(RT_OS_WINDOWS) && defined(RT_ARCH_X86)
+        if (iTrap == 0x2E)
+#elif defined(RT_OS_LINUX)
+        if (iTrap == 0x80)
+#else
+        if (0)
+#endif
+        {
+            if (     GuestIdte.Gen.u1Present
+                &&  (   GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_TRAP_32
+                     || GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_INT_32)
+                &&   GuestIdte.Gen.u2DPL == 3)
+            {
+                PVBOXIDTE   pIdte = &pVM->trpm.s.aIdt[iTrap];
+
+                GuestIdte.Gen.u5Type2 = VBOX_IDTE_TYPE2_TRAP_32;
+                GuestIdte.Gen.u16OffsetHigh = pHandler >> 16;
+                GuestIdte.Gen.u16OffsetLow  = pHandler & 0xFFFF;
+                GuestIdte.Gen.u16SegSel |= 1;  //ring 1
+                *pIdte = GuestIdte;
+
+                /* Mark it for relocation purposes. */
+                ASMBitSet(&pVM->trpm.s.au32IdtPatched[0], iTrap);
+
+                /* Also store it in our guest trap array. */
+                pVM->trpm.s.aGuestTrapHandler[iTrap] = pHandler;
+
+                Log(("Setting trap handler %x to %08X (direct)\n", iTrap, pHandler));
+                return VINF_SUCCESS;
+            }
+            /* ok, let's try to install a trampoline handler then. */
+        }
+    }
+
+    if (     GuestIdte.Gen.u1Present
+        &&  (   GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_TRAP_32
+             || GuestIdte.Gen.u5Type2 == VBOX_IDTE_TYPE2_INT_32)
+        &&  (GuestIdte.Gen.u2DPL == 3 || GuestIdte.Gen.u2DPL == 0))
+    {
+        /*
+         * Save handler which can be used for a trampoline call inside the GC
+         */
+        Log(("Setting trap handler %x to %08X\n", iTrap, pHandler));
+        pVM->trpm.s.aGuestTrapHandler[iTrap] = pHandler;
+        return VINF_SUCCESS;
+    }
+    return VERR_INVALID_PARAMETER;
+}
+
+
+/**
+ * Check if address is a gate handler (interrupt/trap/task/anything).
+ *
+ * @returns True is gate handler, false if not.
+ *
+ * @param   pVM         VM handle.
+ * @param   GCPtr       GC address to check.
+ */
+VMMR3DECL(bool) TRPMR3IsGateHandler(PVM pVM, RTRCPTR GCPtr)
+{
+    /* Only valid in raw mode which implies 1 VCPU */
+    Assert(PATMIsEnabled(pVM) && pVM->cCpus == 1);
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    /*
+     * Read IDTR and calc last entry.
+     */
+    uint16_t    cbIDT;
+    RTGCPTR     GCPtrIDTE = CPUMGetGuestIDTR(pVCpu, &cbIDT);
+    unsigned    cEntries = (cbIDT + 1) / sizeof(VBOXIDTE);
+    if (!cEntries)
+        return false;
+    RTGCPTR   GCPtrIDTELast = GCPtrIDTE + (cEntries - 1) * sizeof(VBOXIDTE);
+
+    /*
+     * Outer loop: iterate pages.
+     */
+    while (GCPtrIDTE <= GCPtrIDTELast)
+    {
+        /*
+         * Convert this page to a HC address.
+         * (This function checks for not-present pages.)
+         */
+        PCVBOXIDTE      pIDTE;
+        PGMPAGEMAPLOCK  Lock;
+        int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrIDTE, (const void **)&pIDTE, &Lock);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Inner Loop: Iterate the data on this page looking for an entry equal to GCPtr.
+             * N.B. Member of the Flat Earth Society...
+             */
+            while (GCPtrIDTE <= GCPtrIDTELast)
+            {
+                if (pIDTE->Gen.u1Present)
+                {
+                    RTRCPTR GCPtrHandler = VBOXIDTE_OFFSET(*pIDTE);
+                    if (GCPtr == GCPtrHandler)
+                    {
+                        PGMPhysReleasePageMappingLock(pVM, &Lock);
+                        return true;
+                    }
+                }
+
+                /* next entry */
+                if ((GCPtrIDTE & PAGE_OFFSET_MASK) + sizeof(VBOXIDTE) >= PAGE_SIZE)
+                {
+                    AssertMsg(!(GCPtrIDTE & (sizeof(VBOXIDTE) - 1)),
+                              ("IDT is crossing pages and it's not aligned! GCPtrIDTE=%#x cbIDT=%#x\n", GCPtrIDTE, cbIDT));
+                    GCPtrIDTE += sizeof(VBOXIDTE);
+                    break;
+                }
+                GCPtrIDTE += sizeof(VBOXIDTE);
+                pIDTE++;
+            }
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+        }
+        else
+        {
+            /* Skip to the next page (if any). Take care not to wrap around the address space. */
+            if ((GCPtrIDTELast >> PAGE_SHIFT) == (GCPtrIDTE >> PAGE_SHIFT))
+                return false;
+            GCPtrIDTE = RT_ALIGN_T(GCPtrIDTE, PAGE_SIZE, RTGCPTR) + PAGE_SIZE + (GCPtrIDTE & (sizeof(VBOXIDTE) - 1));
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Inject event (such as external irq or trap)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pVCpu       The VMCPU to operate on.
+ * @param   enmEvent    Trpm event type
+ */
+VMMR3DECL(int) TRPMR3InjectEvent(PVM pVM, PVMCPU pVCpu, TRPMEVENT enmEvent)
+{
+    PCPUMCTX pCtx;
+    int      rc;
+
+    pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+    Assert(!PATMIsPatchGCAddr(pVM, pCtx->eip));
+    Assert(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
+
+    /* Currently only useful for external hardware interrupts. */
+    Assert(enmEvent == TRPM_HARDWARE_INT);
+
+    if (REMR3QueryPendingInterrupt(pVM, pVCpu) == REM_NO_PENDING_IRQ)
+    {
+#ifdef TRPM_FORWARD_TRAPS_IN_GC
+
+# ifdef LOG_ENABLED
+        DBGFR3InfoLog(pVM, "cpumguest", "TRPMInject");
+        DBGFR3DisasInstrCurrentLog(pVCpu, "TRPMInject");
+# endif
+
+        uint8_t u8Interrupt;
+        rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+        Log(("TRPMR3InjectEvent: CPU%d u8Interrupt=%d (%#x) rc=%Rrc\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc));
+        if (RT_SUCCESS(rc))
+        {
+            if (HWACCMIsEnabled(pVM))
+            {
+                rc = TRPMAssertTrap(pVCpu, u8Interrupt, enmEvent);
+                AssertRC(rc);
+                STAM_COUNTER_INC(&pVM->trpm.s.paStatForwardedIRQR3[u8Interrupt]);
+                return HWACCMR3IsActive(pVCpu) ? VINF_EM_RESCHEDULE_HWACC : VINF_EM_RESCHEDULE_REM;
+            }
+            /* If the guest gate is not patched, then we will check (again) if we can patch it. */
+            if (pVM->trpm.s.aGuestTrapHandler[u8Interrupt] == TRPM_INVALID_HANDLER)
+            {
+                CSAMR3CheckGates(pVM, u8Interrupt, 1);
+                Log(("TRPMR3InjectEvent: recheck gate %x -> valid=%d\n", u8Interrupt, TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) != TRPM_INVALID_HANDLER));
+            }
+
+            if (pVM->trpm.s.aGuestTrapHandler[u8Interrupt] != TRPM_INVALID_HANDLER)
+            {
+                /* Must check pending forced actions as our IDT or GDT might be out of sync */
+                rc = EMR3CheckRawForcedActions(pVM, pVCpu);
+                if (rc == VINF_SUCCESS)
+                {
+                    /* There's a handler -> let's execute it in raw mode */
+                    rc = TRPMForwardTrap(pVCpu, CPUMCTX2CORE(pCtx), u8Interrupt, 0, TRPM_TRAP_NO_ERRORCODE, enmEvent, -1);
+                    if (rc == VINF_SUCCESS /* Don't use RT_SUCCESS */)
+                    {
+                        Assert(!VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_SELM_SYNC_GDT | VMCPU_FF_SELM_SYNC_LDT | VMCPU_FF_TRPM_SYNC_IDT | VMCPU_FF_SELM_SYNC_TSS));
+
+                        STAM_COUNTER_INC(&pVM->trpm.s.paStatForwardedIRQR3[u8Interrupt]);
+                        return VINF_EM_RESCHEDULE_RAW;
+                    }
+                }
+            }
+            else
+                STAM_COUNTER_INC(&pVM->trpm.s.StatForwardFailNoHandler);
+            REMR3NotifyPendingInterrupt(pVM, pVCpu, u8Interrupt);
+        }
+        else
+        {
+            AssertRC(rc);
+            return HWACCMR3IsActive(pVCpu) ? VINF_EM_RESCHEDULE_HWACC : VINF_EM_RESCHEDULE_REM; /* (Heed the halted state if this is changed!) */
+        }
+#else
+        if (HWACCMR3IsActive(pVM))
+        {
+            uint8_t u8Interrupt;
+            rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+            Log(("TRPMR3InjectEvent: u8Interrupt=%d (%#x) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
+            if (RT_SUCCESS(rc))
+            {
+                rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
+                AssertRC(rc);
+                STAM_COUNTER_INC(&pVM->trpm.s.paStatForwardedIRQR3[u8Interrupt]);
+                return VINF_EM_RESCHEDULE_HWACC;
+            }
+        }
+        else
+            AssertRC(rc);
+#endif
+    }
+    /** @todo check if it's safe to translate the patch address to the original guest address.
+     *        this implies a safe state in translated instructions and should take sti successors into account (instruction fusing)
+     */
+    /* Note: if it's a PATM address, then we'll go back to raw mode regardless of the return code below. */
+
+    /* Fall back to the recompiler */
+    return VINF_EM_RESCHEDULE_REM; /* (Heed the halted state if this is changed!) */
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/VBoxVMMDeps.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VBoxVMMDeps.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VBoxVMMDeps.cpp	(revision 35333)
@@ -0,0 +1,64 @@
+/* $Id$ */
+/** @file
+ * VBoxVMM link dependencies - drag all we want into the link!
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#include <VBox/pgm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmqueue.h>
+#include <VBox/vm.h>
+#include <VBox/em.h>
+#include <VBox/iom.h>
+#include <VBox/dbgf.h>
+#include <VBox/dbg.h>
+#include <VBox/ftm.h>
+
+VMMR3DECL(int) VMMDoTest(PVM pVM);
+
+/** Just a dummy global structure containing a bunch of
+ * function pointers to code which is wanted in the link.
+ */
+PFNRT g_apfnDeps[] =
+{
+    (PFNRT)DBGFR3DisasInstrEx,
+    (PFNRT)DBGFR3LogModifyFlags,
+    (PFNRT)DBGFR3StackWalkEnd,
+    (PFNRT)DBGFR3AsSymbolByAddr,
+    (PFNRT)DBGFR3CpuGetMode,
+    (PFNRT)DBGFR3CoreWrite,
+    (PFNRT)DBGFR3MemScan,
+    (PFNRT)DBGFR3RegQueryU8,
+    (PFNRT)EMInterpretInstruction,
+    (PFNRT)IOMIOPortRead,
+    (PFNRT)PDMQueueInsert,
+    (PFNRT)PDMCritSectEnter,
+    (PFNRT)PGMInvalidatePage,
+    (PFNRT)PGMR3DbgR3Ptr2GCPhys,
+    (PFNRT)VMR3Create,
+    (PFNRT)VMMDoTest,
+    (PFNRT)FTMR3PowerOn,
+#ifdef VBOX_WITH_DEBUGGER
+    (PFNRT)DBGCCreate,
+#endif
+#ifdef VBOX_WITH_PAGE_SHARING
+    (PFNRT)PGMR3SharedModuleRegister,
+#endif
+    NULL
+};
Index: /trunk/src/VBox/VMM/VMMR3/VM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VM.cpp	(revision 35333)
@@ -0,0 +1,4252 @@
+/* $Id$ */
+/** @file
+ * VM - Virtual Machine
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/** @page pg_vm     VM API
+ *
+ * This is the encapsulating bit.  It provides the APIs that Main and VBoxBFE
+ * use to create a VMM instance for running a guest in.  It also provides
+ * facilities for queuing request for execution in EMT (serialization purposes
+ * mostly) and for reporting error back to the VMM user (Main/VBoxBFE).
+ *
+ *
+ * @section sec_vm_design   Design Critique / Things To Do
+ *
+ * In hindsight this component is a big design mistake, all this stuff really
+ * belongs in the VMM component.  It just seemed like a kind of ok idea at a
+ * time when the VMM bit was a kind of vague.  'VM' also happened to be the name
+ * of the per-VM instance structure (see vm.h), so it kind of made sense.
+ * However as it turned out, VMM(.cpp) is almost empty all it provides in ring-3
+ * is some minor functionally and some "routing" services.
+ *
+ * Fixing this is just a matter of some more or less straight forward
+ * refactoring, the question is just when someone will get to it. Moving the EMT
+ * would be a good start.
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include <VBox/cfgm.h>
+#include <VBox/vmm.h>
+#include <VBox/gvmm.h>
+#include <VBox/mm.h>
+#include <VBox/cpum.h>
+#include <VBox/selm.h>
+#include <VBox/trpm.h>
+#include <VBox/dbgf.h>
+#include <VBox/pgm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/em.h>
+#include <VBox/rem.h>
+#include <VBox/tm.h>
+#include <VBox/stam.h>
+#include <VBox/patm.h>
+#include <VBox/csam.h>
+#include <VBox/iom.h>
+#include <VBox/ssm.h>
+#include <VBox/ftm.h>
+#include <VBox/hwaccm.h>
+#include "VMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+
+#include <VBox/sup.h>
+#include <VBox/dbg.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/env.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+#include <iprt/semaphore.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * VM destruction callback registration record.
+ */
+typedef struct VMATDTOR
+{
+    /** Pointer to the next record in the list. */
+    struct VMATDTOR        *pNext;
+    /** Pointer to the callback function. */
+    PFNVMATDTOR             pfnAtDtor;
+    /** The user argument. */
+    void                   *pvUser;
+} VMATDTOR;
+/** Pointer to a VM destruction callback registration record. */
+typedef VMATDTOR *PVMATDTOR;
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/** Pointer to the list of VMs. */
+static PUVM         g_pUVMsHead = NULL;
+
+/** Pointer to the list of at VM destruction callbacks. */
+static PVMATDTOR    g_pVMAtDtorHead = NULL;
+/** Lock the g_pVMAtDtorHead list. */
+#define VM_ATDTOR_LOCK() do { } while (0)
+/** Unlock the g_pVMAtDtorHead list. */
+#define VM_ATDTOR_UNLOCK() do { } while (0)
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int                  vmR3CreateUVM(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods, PUVM *ppUVM);
+static int                  vmR3CreateU(PUVM pUVM, uint32_t cCpus, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM);
+static int                  vmR3InitRing3(PVM pVM, PUVM pUVM);
+static int                  vmR3InitRing0(PVM pVM);
+static int                  vmR3InitGC(PVM pVM);
+static int                  vmR3InitDoCompleted(PVM pVM, VMINITCOMPLETED enmWhat);
+static DECLCALLBACK(size_t) vmR3LogPrefixCallback(PRTLOGGER pLogger, char *pchBuf, size_t cchBuf, void *pvUser);
+static void                 vmR3DestroyUVM(PUVM pUVM, uint32_t cMilliesEMTWait);
+static void                 vmR3AtDtor(PVM pVM);
+static bool                 vmR3ValidateStateTransition(VMSTATE enmStateOld, VMSTATE enmStateNew);
+static void                 vmR3DoAtState(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
+static int                  vmR3TrySetState(PVM pVM, const char *pszWho, unsigned cTransitions, ...);
+static void                 vmR3SetStateLocked(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
+static void                 vmR3SetState(PVM pVM, VMSTATE enmStateNew, VMSTATE enmStateOld);
+static int                  vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...);
+
+
+/**
+ * Do global VMM init.
+ *
+ * @returns VBox status code.
+ */
+VMMR3DECL(int)   VMR3GlobalInit(void)
+{
+    /*
+     * Only once.
+     */
+    static bool volatile s_fDone = false;
+    if (s_fDone)
+        return VINF_SUCCESS;
+
+    /*
+     * We're done.
+     */
+    s_fDone = true;
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Creates a virtual machine by calling the supplied configuration constructor.
+ *
+ * On successful returned the VM is powered, i.e. VMR3PowerOn() should be
+ * called to start the execution.
+ *
+ * @returns 0 on success.
+ * @returns VBox error code on failure.
+ * @param   cCpus               Number of virtual CPUs for the new VM.
+ * @param   pVmm2UserMethods    An optional method table that the VMM can use to
+ *                              make the user perform various action, like for
+ *                              instance state saving.
+ * @param   pfnVMAtError        Pointer to callback function for setting VM
+ *                              errors. This was added as an implicit call to
+ *                              VMR3AtErrorRegister() since there is no way the
+ *                              caller can get to the VM handle early enough to
+ *                              do this on its own.
+ *                              This is called in the context of an EMT.
+ * @param   pvUserVM            The user argument passed to pfnVMAtError.
+ * @param   pfnCFGMConstructor  Pointer to callback function for constructing the VM configuration tree.
+ *                              This is called in the context of an EMT0.
+ * @param   pvUserCFGM          The user argument passed to pfnCFGMConstructor.
+ * @param   ppVM                Where to store the 'handle' of the created VM.
+ */
+VMMR3DECL(int)   VMR3Create(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods,
+                            PFNVMATERROR pfnVMAtError, void *pvUserVM,
+                            PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM,
+                            PVM *ppVM)
+{
+    LogFlow(("VMR3Create: cCpus=%RU32 pVmm2UserMethods=%p pfnVMAtError=%p pvUserVM=%p  pfnCFGMConstructor=%p pvUserCFGM=%p ppVM=%p\n",
+             cCpus, pVmm2UserMethods, pfnVMAtError, pvUserVM, pfnCFGMConstructor, pvUserCFGM, ppVM));
+
+    if (pVmm2UserMethods)
+    {
+        AssertPtrReturn(pVmm2UserMethods, VERR_INVALID_POINTER);
+        AssertReturn(pVmm2UserMethods->u32Magic    == VMM2USERMETHODS_MAGIC,   VERR_INVALID_PARAMETER);
+        AssertReturn(pVmm2UserMethods->u32Version  == VMM2USERMETHODS_VERSION, VERR_INVALID_PARAMETER);
+        AssertPtrReturn(pVmm2UserMethods->pfnSaveState, VERR_INVALID_POINTER);
+        AssertReturn(pVmm2UserMethods->u32EndMagic == VMM2USERMETHODS_MAGIC,   VERR_INVALID_PARAMETER);
+    }
+    AssertPtrNullReturn(pfnVMAtError, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnCFGMConstructor, VERR_INVALID_POINTER);
+    AssertPtrReturn(ppVM, VERR_INVALID_POINTER);
+
+    /*
+     * Because of the current hackiness of the applications
+     * we'll have to initialize global stuff from here.
+     * Later the applications will take care of this in a proper way.
+     */
+    static bool fGlobalInitDone = false;
+    if (!fGlobalInitDone)
+    {
+        int rc = VMR3GlobalInit();
+        if (RT_FAILURE(rc))
+            return rc;
+        fGlobalInitDone = true;
+    }
+
+    /*
+     * Validate input.
+     */
+    AssertLogRelMsgReturn(cCpus > 0 && cCpus <= VMM_MAX_CPU_COUNT, ("%RU32\n", cCpus), VERR_TOO_MANY_CPUS);
+
+    /*
+     * Create the UVM so we can register the at-error callback
+     * and consolidate a bit of cleanup code.
+     */
+    PUVM pUVM = NULL;                   /* shuts up gcc */
+    int rc = vmR3CreateUVM(cCpus, pVmm2UserMethods, &pUVM);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (pfnVMAtError)
+        rc = VMR3AtErrorRegisterU(pUVM, pfnVMAtError, pvUserVM);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Initialize the support library creating the session for this VM.
+         */
+        rc = SUPR3Init(&pUVM->vm.s.pSession);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Call vmR3CreateU in the EMT thread and wait for it to finish.
+             *
+             * Note! VMCPUID_ANY is used here because VMR3ReqQueueU would have trouble
+             *       submitting a request to a specific VCPU without a pVM. So, to make
+             *       sure init is running on EMT(0), vmR3EmulationThreadWithId makes sure
+             *       that only EMT(0) is servicing VMCPUID_ANY requests when pVM is NULL.
+             */
+            PVMREQ pReq;
+            rc = VMR3ReqCallU(pUVM, VMCPUID_ANY, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
+                              (PFNRT)vmR3CreateU, 4, pUVM, cCpus, pfnCFGMConstructor, pvUserCFGM);
+            if (RT_SUCCESS(rc))
+            {
+                rc = pReq->iStatus;
+                VMR3ReqFree(pReq);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Success!
+                     */
+                    *ppVM = pUVM->pVM;
+                    LogFlow(("VMR3Create: returns VINF_SUCCESS *ppVM=%p\n", *ppVM));
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+                AssertMsgFailed(("VMR3ReqCallU failed rc=%Rrc\n", rc));
+
+            /*
+             * An error occurred during VM creation. Set the error message directly
+             * using the initial callback, as the callback list doesn't exist yet.
+             */
+            const char *pszError = NULL;
+            switch (rc)
+            {
+                case VERR_VMX_IN_VMX_ROOT_MODE:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox can't operate in VMX root mode. "
+                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
+#else
+                    pszError = N_("VirtualBox can't operate in VMX root mode. Please close all other virtualization programs.");
+#endif
+                    break;
+
+#ifndef RT_OS_DARWIN
+                case VERR_HWACCM_CONFIG_MISMATCH:
+                    pszError = N_("VT-x/AMD-V is either not available on your host or disabled. "
+                                  "This hardware extension is required by the VM configuration");
+                    break;
+#endif
+
+                case VERR_SVM_IN_USE:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox can't enable the AMD-V extension. "
+                                  "Please disable the KVM kernel extension, recompile your kernel and reboot");
+#else
+                    pszError = N_("VirtualBox can't enable the AMD-V extension. Please close all other virtualization programs.");
+#endif
+                    break;
+
+                case VERR_VERSION_MISMATCH:
+                    pszError = N_("VMMR0 driver version mismatch. Please terminate all VMs, make sure that "
+                                  "VBoxNetDHCP is not running and try again. If you still get this error, "
+                                  "re-install VirtualBox");
+                    break;
+
+#ifdef RT_OS_LINUX
+                case VERR_SUPDRV_COMPONENT_NOT_FOUND:
+                    pszError = N_("One of the kernel modules was not successfully loaded. Make sure "
+                                  "that no kernel modules from an older version of VirtualBox exist. "
+                                  "Then try to recompile and reload the kernel modules by executing "
+                                  "'/etc/init.d/vboxdrv setup' as root");
+                    break;
+#endif
+
+                case VERR_RAW_MODE_INVALID_SMP:
+                    pszError = N_("VT-x/AMD-V is either not available on your host or disabled. "
+                                  "VirtualBox requires this hardware extension to emulate more than one "
+                                  "guest CPU");
+                    break;
+
+                case VERR_SUPDRV_KERNEL_TOO_OLD_FOR_VTX:
+#ifdef RT_OS_LINUX
+                    pszError = N_("Because the host kernel is too old, VirtualBox cannot enable the VT-x "
+                                  "extension. Either upgrade your kernel to Linux 2.6.13 or later or disable "
+                                  "the VT-x extension in the VM settings. Note that without VT-x you have "
+                                  "to reduce the number of guest CPUs to one");
+#else
+                    pszError = N_("Because the host kernel is too old, VirtualBox cannot enable the VT-x "
+                                  "extension. Either upgrade your kernel or disable the VT-x extension in the "
+                                  "VM settings. Note that without VT-x you have to reduce the number of guest "
+                                  "CPUs to one");
+#endif
+                    break;
+
+                case VERR_PDM_DEVICE_NOT_FOUND:
+                    pszError = N_("A virtual device is configured in the VM settings but the device "
+                                  "implementation is missing.\n"
+                                  "A possible reason for this error is a missing extension pack. Note "
+                                  "that as of VirtualBox 4.0, certain features (for example USB 2.0 "
+                                  "support and remote desktop) are only available from an 'extension "
+                                  "pack' which must be downloaded and installed separately");
+                    break;
+
+                default:
+                    pszError = N_("Unknown error creating VM");
+                    break;
+            }
+            vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
+        }
+        else
+        {
+            /*
+             * An error occurred at support library initialization time (before the
+             * VM could be created). Set the error message directly using the
+             * initial callback, as the callback list doesn't exist yet.
+             */
+            const char *pszError;
+            switch (rc)
+            {
+                case VERR_VM_DRIVER_LOAD_ERROR:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox kernel driver not loaded. The vboxdrv kernel module "
+                                  "was either not loaded or /dev/vboxdrv is not set up properly. "
+                                  "Re-setup the kernel module by executing "
+                                  "'/etc/init.d/vboxdrv setup' as root");
+#else
+                    pszError = N_("VirtualBox kernel driver not loaded");
+#endif
+                    break;
+                case VERR_VM_DRIVER_OPEN_ERROR:
+                    pszError = N_("VirtualBox kernel driver cannot be opened");
+                    break;
+                case VERR_VM_DRIVER_NOT_ACCESSIBLE:
+#ifdef VBOX_WITH_HARDENING
+                    /* This should only happen if the executable wasn't hardened - bad code/build. */
+                    pszError = N_("VirtualBox kernel driver not accessible, permission problem. "
+                                  "Re-install VirtualBox. If you are building it yourself, you "
+                                  "should make sure it installed correctly and that the setuid "
+                                  "bit is set on the executables calling VMR3Create.");
+#else
+                    /* This should only happen when mixing builds or with the usual /dev/vboxdrv access issues. */
+# if defined(RT_OS_DARWIN)
+                    pszError = N_("VirtualBox KEXT is not accessible, permission problem. "
+                                  "If you have built VirtualBox yourself, make sure that you do not "
+                                  "have the vboxdrv KEXT from a different build or installation loaded.");
+# elif defined(RT_OS_LINUX)
+                    pszError = N_("VirtualBox kernel driver is not accessible, permission problem. "
+                                  "If you have built VirtualBox yourself, make sure that you do "
+                                  "not have the vboxdrv kernel module from a different build or "
+                                  "installation loaded. Also, make sure the vboxdrv udev rule gives "
+                                  "you the permission you need to access the device.");
+# elif defined(RT_OS_WINDOWS)
+                    pszError = N_("VirtualBox kernel driver is not accessible, permission problem.");
+# else /* solaris, freebsd, ++. */
+                    pszError = N_("VirtualBox kernel module is not accessible, permission problem. "
+                                  "If you have built VirtualBox yourself, make sure that you do "
+                                  "not have the vboxdrv kernel module from a different install loaded.");
+# endif
+#endif
+                    break;
+                case VERR_INVALID_HANDLE: /** @todo track down and fix this error. */
+                case VERR_VM_DRIVER_NOT_INSTALLED:
+#ifdef RT_OS_LINUX
+                    pszError = N_("VirtualBox kernel driver not installed. The vboxdrv kernel module "
+                                  "was either not loaded or /dev/vboxdrv was not created for some "
+                                  "reason. Re-setup the kernel module by executing "
+                                  "'/etc/init.d/vboxdrv setup' as root");
+#else
+                    pszError = N_("VirtualBox kernel driver not installed");
+#endif
+                    break;
+                case VERR_NO_MEMORY:
+                    pszError = N_("VirtualBox support library out of memory");
+                    break;
+                case VERR_VERSION_MISMATCH:
+                case VERR_VM_DRIVER_VERSION_MISMATCH:
+                    pszError = N_("The VirtualBox support driver which is running is from a different "
+                                  "version of VirtualBox.  You can correct this by stopping all "
+                                  "running instances of VirtualBox and reinstalling the software.");
+                    break;
+                default:
+                    pszError = N_("Unknown error initializing kernel driver");
+                    AssertMsgFailed(("Add error message for rc=%d (%Rrc)\n", rc, rc));
+            }
+            vmR3SetErrorU(pUVM, rc, RT_SRC_POS, pszError, rc);
+        }
+    }
+
+    /* cleanup */
+    vmR3DestroyUVM(pUVM, 2000);
+    LogFlow(("VMR3Create: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Creates the UVM.
+ *
+ * This will not initialize the support library even if vmR3DestroyUVM
+ * will terminate that.
+ *
+ * @returns VBox status code.
+ * @param   cCpus               Number of virtual CPUs
+ * @param   pVmm2UserMethods    Pointer to the optional VMM -> User method
+ *                              table.
+ * @param   ppUVM               Where to store the UVM pointer.
+ */
+static int vmR3CreateUVM(uint32_t cCpus, PCVMM2USERMETHODS pVmm2UserMethods, PUVM *ppUVM)
+{
+    uint32_t i;
+
+    /*
+     * Create and initialize the UVM.
+     */
+    PUVM pUVM = (PUVM)RTMemPageAllocZ(RT_OFFSETOF(UVM, aCpus[cCpus]));
+    AssertReturn(pUVM, VERR_NO_MEMORY);
+    pUVM->u32Magic          = UVM_MAGIC;
+    pUVM->cCpus             = cCpus;
+    pUVM->pVmm2UserMethods  = pVmm2UserMethods;
+
+    AssertCompile(sizeof(pUVM->vm.s) <= sizeof(pUVM->vm.padding));
+
+    pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
+    pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
+    pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
+
+    pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_BOOTSTRAP;
+
+    /* Initialize the VMCPU array in the UVM. */
+    for (i = 0; i < cCpus; i++)
+    {
+        pUVM->aCpus[i].pUVM   = pUVM;
+        pUVM->aCpus[i].idCpu  = i;
+    }
+
+    /* Allocate a TLS entry to store the VMINTUSERPERVMCPU pointer. */
+    int rc = RTTlsAllocEx(&pUVM->vm.s.idxTLS, NULL);
+    AssertRC(rc);
+    if (RT_SUCCESS(rc))
+    {
+        /* Allocate a halt method event semaphore for each VCPU. */
+        for (i = 0; i < cCpus; i++)
+            pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
+        for (i = 0; i < cCpus; i++)
+        {
+            rc = RTSemEventCreate(&pUVM->aCpus[i].vm.s.EventSemWait);
+            if (RT_FAILURE(rc))
+                break;
+        }
+        if (RT_SUCCESS(rc))
+        {
+            rc = RTCritSectInit(&pUVM->vm.s.AtStateCritSect);
+            if (RT_SUCCESS(rc))
+            {
+                rc = RTCritSectInit(&pUVM->vm.s.AtErrorCritSect);
+                if (RT_SUCCESS(rc))
+                {
+                    /*
+                     * Init fundamental (sub-)components - STAM, MMR3Heap and PDMLdr.
+                     */
+                    rc = STAMR3InitUVM(pUVM);
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = MMR3InitUVM(pUVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = PDMR3InitUVM(pUVM);
+                            if (RT_SUCCESS(rc))
+                            {
+                                /*
+                                 * Start the emulation threads for all VMCPUs.
+                                 */
+                                for (i = 0; i < cCpus; i++)
+                                {
+                                    rc = RTThreadCreateF(&pUVM->aCpus[i].vm.s.ThreadEMT, vmR3EmulationThread, &pUVM->aCpus[i], _1M,
+                                                         RTTHREADTYPE_EMULATION, RTTHREADFLAGS_WAITABLE,
+                                                         cCpus > 1 ? "EMT-%u" : "EMT", i);
+                                    if (RT_FAILURE(rc))
+                                        break;
+
+                                    pUVM->aCpus[i].vm.s.NativeThreadEMT = RTThreadGetNative(pUVM->aCpus[i].vm.s.ThreadEMT);
+                                }
+
+                                if (RT_SUCCESS(rc))
+                                {
+                                    *ppUVM = pUVM;
+                                    return VINF_SUCCESS;
+                                }
+
+                                /* bail out. */
+                                while (i-- > 0)
+                                {
+                                    /** @todo rainy day: terminate the EMTs. */
+                                }
+                                PDMR3TermUVM(pUVM);
+                            }
+                            MMR3TermUVM(pUVM);
+                        }
+                        STAMR3TermUVM(pUVM);
+                    }
+                    RTCritSectDelete(&pUVM->vm.s.AtErrorCritSect);
+                }
+                RTCritSectDelete(&pUVM->vm.s.AtStateCritSect);
+            }
+        }
+        for (i = 0; i < cCpus; i++)
+        {
+            RTSemEventDestroy(pUVM->aCpus[i].vm.s.EventSemWait);
+            pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
+        }
+        RTTlsFree(pUVM->vm.s.idxTLS);
+    }
+    RTMemPageFree(pUVM, RT_OFFSETOF(UVM, aCpus[pUVM->cCpus]));
+    return rc;
+}
+
+
+/**
+ * Creates and initializes the VM.
+ *
+ * @thread EMT
+ */
+static int vmR3CreateU(PUVM pUVM, uint32_t cCpus, PFNCFGMCONSTRUCTOR pfnCFGMConstructor, void *pvUserCFGM)
+{
+    int rc = VINF_SUCCESS;
+
+    /*
+     * Load the VMMR0.r0 module so that we can call GVMMR0CreateVM.
+     */
+    rc = PDMR3LdrLoadVMMR0U(pUVM);
+    if (RT_FAILURE(rc))
+    {
+        /** @todo we need a cleaner solution for this (VERR_VMX_IN_VMX_ROOT_MODE).
+          * bird: what about moving the message down here? Main picks the first message, right? */
+        if (rc == VERR_VMX_IN_VMX_ROOT_MODE)
+            return rc;  /* proper error message set later on */
+        return vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("Failed to load VMMR0.r0"));
+    }
+
+    /*
+     * Request GVMM to create a new VM for us.
+     */
+    GVMMCREATEVMREQ CreateVMReq;
+    CreateVMReq.Hdr.u32Magic    = SUPVMMR0REQHDR_MAGIC;
+    CreateVMReq.Hdr.cbReq       = sizeof(CreateVMReq);
+    CreateVMReq.pSession        = pUVM->vm.s.pSession;
+    CreateVMReq.pVMR0           = NIL_RTR0PTR;
+    CreateVMReq.pVMR3           = NULL;
+    CreateVMReq.cCpus           = cCpus;
+    rc = SUPR3CallVMMR0Ex(NIL_RTR0PTR, NIL_VMCPUID, VMMR0_DO_GVMM_CREATE_VM, 0, &CreateVMReq.Hdr);
+    if (RT_SUCCESS(rc))
+    {
+        PVM pVM = pUVM->pVM = CreateVMReq.pVMR3;
+        AssertRelease(VALID_PTR(pVM));
+        AssertRelease(pVM->pVMR0 == CreateVMReq.pVMR0);
+        AssertRelease(pVM->pSession == pUVM->vm.s.pSession);
+        AssertRelease(pVM->cCpus == cCpus);
+        AssertRelease(pVM->uCpuExecutionCap == 100);
+        AssertRelease(pVM->offVMCPU == RT_UOFFSETOF(VM, aCpus));
+
+        Log(("VMR3Create: Created pUVM=%p pVM=%p pVMR0=%p hSelf=%#x cCpus=%RU32\n",
+             pUVM, pVM, pVM->pVMR0, pVM->hSelf, pVM->cCpus));
+
+        /*
+         * Initialize the VM structure and our internal data (VMINT).
+         */
+        pVM->pUVM = pUVM;
+
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            pVM->aCpus[i].pUVCpu            = &pUVM->aCpus[i];
+            pVM->aCpus[i].idCpu             = i;
+            pVM->aCpus[i].hNativeThread     = pUVM->aCpus[i].vm.s.NativeThreadEMT;
+            Assert(pVM->aCpus[i].hNativeThread != NIL_RTNATIVETHREAD);
+            /* hNativeThreadR0 is initialized on EMT registration. */
+            pUVM->aCpus[i].pVCpu            = &pVM->aCpus[i];
+            pUVM->aCpus[i].pVM              = pVM;
+        }
+
+
+        /*
+         * Init the configuration.
+         */
+        rc = CFGMR3Init(pVM, pfnCFGMConstructor, pvUserCFGM);
+        if (RT_SUCCESS(rc))
+        {
+            PCFGMNODE pRoot = CFGMR3GetRoot(pVM);
+            rc = CFGMR3QueryBoolDef(pRoot, "HwVirtExtForced", &pVM->fHwVirtExtForced, false);
+            if (RT_SUCCESS(rc) && pVM->fHwVirtExtForced)
+                pVM->fHWACCMEnabled = true;
+
+            /*
+             * If executing in fake suplib mode disable RR3 and RR0 in the config.
+             */
+            const char *psz = RTEnvGet("VBOX_SUPLIB_FAKE");
+            if (psz && !strcmp(psz, "fake"))
+            {
+                CFGMR3RemoveValue(pRoot, "RawR3Enabled");
+                CFGMR3InsertInteger(pRoot, "RawR3Enabled", 0);
+                CFGMR3RemoveValue(pRoot, "RawR0Enabled");
+                CFGMR3InsertInteger(pRoot, "RawR0Enabled", 0);
+            }
+
+            /*
+             * Make sure the CPU count in the config data matches.
+             */
+            if (RT_SUCCESS(rc))
+            {
+                uint32_t cCPUsCfg;
+                rc = CFGMR3QueryU32Def(pRoot, "NumCPUs", &cCPUsCfg, 1);
+                AssertLogRelMsgRC(rc, ("Configuration error: Querying \"NumCPUs\" as integer failed, rc=%Rrc\n", rc));
+                if (RT_SUCCESS(rc) && cCPUsCfg != cCpus)
+                {
+                    AssertLogRelMsgFailed(("Configuration error: \"NumCPUs\"=%RU32 and VMR3CreateVM::cCpus=%RU32 does not match!\n",
+                                           cCPUsCfg, cCpus));
+                    rc = VERR_INVALID_PARAMETER;
+                }
+            }
+            if (RT_SUCCESS(rc))
+            {
+                rc = CFGMR3QueryU32Def(pRoot, "CpuExecutionCap", &pVM->uCpuExecutionCap, 100);
+                AssertLogRelMsgRC(rc, ("Configuration error: Querying \"CpuExecutionCap\" as integer failed, rc=%Rrc\n", rc));
+
+                /*
+                 * Init the ring-3 components and ring-3 per cpu data, finishing it off
+                 * by a relocation round (intermediate context finalization will do this).
+                 */
+                rc = vmR3InitRing3(pVM, pUVM);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = PGMR3FinalizeMappings(pVM);
+                    if (RT_SUCCESS(rc))
+                    {
+
+                        LogFlow(("Ring-3 init succeeded\n"));
+
+                        /*
+                         * Init the Ring-0 components.
+                         */
+                        rc = vmR3InitRing0(pVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            /* Relocate again, because some switcher fixups depends on R0 init results. */
+                            VMR3Relocate(pVM, 0);
+
+#ifdef VBOX_WITH_DEBUGGER
+                            /*
+                             * Init the tcp debugger console if we're building
+                             * with debugger support.
+                             */
+                            void *pvUser = NULL;
+                            rc = DBGCTcpCreate(pVM, &pvUser);
+                            if (    RT_SUCCESS(rc)
+                                ||  rc == VERR_NET_ADDRESS_IN_USE)
+                            {
+                                pUVM->vm.s.pvDBGC = pvUser;
+#endif
+                                /*
+                                 * Init the Guest Context components.
+                                 */
+                                rc = vmR3InitGC(pVM);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    /*
+                                     * Now we can safely set the VM halt method to default.
+                                     */
+                                    rc = vmR3SetHaltMethodU(pUVM, VMHALTMETHOD_DEFAULT);
+                                    if (RT_SUCCESS(rc))
+                                    {
+                                        /*
+                                         * Set the state and link into the global list.
+                                         */
+                                        vmR3SetState(pVM, VMSTATE_CREATED, VMSTATE_CREATING);
+                                        pUVM->pNext = g_pUVMsHead;
+                                        g_pUVMsHead = pUVM;
+
+#ifdef LOG_ENABLED
+                                        RTLogSetCustomPrefixCallback(NULL, vmR3LogPrefixCallback, pUVM);
+#endif
+                                        return VINF_SUCCESS;
+                                    }
+                                }
+#ifdef VBOX_WITH_DEBUGGER
+                                DBGCTcpTerminate(pVM, pUVM->vm.s.pvDBGC);
+                                pUVM->vm.s.pvDBGC = NULL;
+                            }
+#endif
+                            //..
+                        }
+                    }
+                    vmR3Destroy(pVM);
+                }
+            }
+            //..
+
+            /* Clean CFGM. */
+            int rc2 = CFGMR3Term(pVM);
+            AssertRC(rc2);
+        }
+
+        /*
+         * Do automatic cleanups while the VM structure is still alive and all
+         * references to it are still working.
+         */
+        PDMR3CritSectTerm(pVM);
+
+        /*
+         * Drop all references to VM and the VMCPU structures, then
+         * tell GVMM to destroy the VM.
+         */
+        pUVM->pVM = NULL;
+        for (VMCPUID i = 0; i < pUVM->cCpus; i++)
+        {
+            pUVM->aCpus[i].pVM = NULL;
+            pUVM->aCpus[i].pVCpu = NULL;
+        }
+        Assert(pUVM->vm.s.enmHaltMethod == VMHALTMETHOD_BOOTSTRAP);
+
+        if (pUVM->cCpus > 1)
+        {
+            /* Poke the other EMTs since they may have stale pVM and pVCpu references
+               on the stack (see VMR3WaitU for instance) if they've been awakened after
+               VM creation. */
+            for (VMCPUID i = 1; i < pUVM->cCpus; i++)
+                VMR3NotifyCpuFFU(&pUVM->aCpus[i], 0);
+            RTThreadSleep(RT_MIN(100 + 25 *(pUVM->cCpus - 1), 500)); /* very sophisticated */
+        }
+
+        int rc2 = SUPR3CallVMMR0Ex(CreateVMReq.pVMR0, 0 /*idCpu*/, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
+        AssertRC(rc2);
+    }
+    else
+        vmR3SetErrorU(pUVM, rc, RT_SRC_POS, N_("VM creation failed (GVMM)"));
+
+    LogFlow(("vmR3CreateU: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Register the calling EMT with GVM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   idCpu       The Virtual CPU ID.
+ */
+static DECLCALLBACK(int) vmR3RegisterEMT(PVM pVM, VMCPUID idCpu)
+{
+    Assert(VMMGetCpuId(pVM) == idCpu);
+    int rc = SUPR3CallVMMR0Ex(pVM->pVMR0, idCpu, VMMR0_DO_GVMM_REGISTER_VMCPU, 0, NULL);
+    if (RT_FAILURE(rc))
+        LogRel(("idCpu=%u rc=%Rrc\n", idCpu, rc));
+    return rc;
+}
+
+
+/**
+ * Initializes all R3 components of the VM
+ */
+static int vmR3InitRing3(PVM pVM, PUVM pUVM)
+{
+    int rc;
+
+    /*
+     * Register the other EMTs with GVM.
+     */
+    for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
+    {
+        rc = VMR3ReqCallWaitU(pUVM, idCpu, (PFNRT)vmR3RegisterEMT, 2, pVM, idCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Init all R3 components, the order here might be important.
+     */
+    rc = MMR3Init(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        STAM_REG(pVM, &pVM->StatTotalInGC,          STAMTYPE_PROFILE_ADV, "/PROF/VM/InGC",          STAMUNIT_TICKS_PER_CALL,    "Profiling the total time spent in GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherToGC,       STAMTYPE_PROFILE_ADV, "/PROF/VM/SwitchToGC",    STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherToHC,       STAMTYPE_PROFILE_ADV, "/PROF/VM/SwitchToHC",    STAMUNIT_TICKS_PER_CALL,    "Profiling switching to HC.");
+        STAM_REG(pVM, &pVM->StatSwitcherSaveRegs,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/SaveRegs", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherSysEnter,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/SysEnter", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherDebug,      STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Debug",    STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherCR0,        STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/CR0",  STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherCR4,        STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/CR4",  STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherLgdt,       STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Lgdt", STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherLidt,       STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Lidt", STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherLldt,       STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/Lldt", STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherTSS,        STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/TSS",  STAMUNIT_TICKS_PER_CALL,    "Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherJmpCR3,     STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/JmpCR3",   STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
+        STAM_REG(pVM, &pVM->StatSwitcherRstrRegs,   STAMTYPE_PROFILE_ADV, "/VM/Switcher/ToGC/RstrRegs", STAMUNIT_TICKS_PER_CALL,"Profiling switching to GC.");
+
+        for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+        {
+            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltYield,           STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state yielding.",  "/PROF/VM/CPU%d/Halt/Yield", idCpu);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlock,           STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state blocking.",  "/PROF/VM/CPU%d/Halt/Block", idCpu);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockOverslept,  STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time wasted by blocking too long.", "/PROF/VM/CPU%d/Halt/BlockOverslept", idCpu);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockInsomnia,   STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time slept when returning to early.","/PROF/VM/CPU%d/Halt/BlockInsomnia", idCpu);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltBlockOnTime,     STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Time slept on time.",                "/PROF/VM/CPU%d/Halt/BlockOnTime", idCpu);
+            AssertRC(rc);
+            rc = STAMR3RegisterF(pVM, &pUVM->aCpus[idCpu].vm.s.StatHaltTimers,          STAMTYPE_PROFILE, STAMVISIBILITY_ALWAYS, STAMUNIT_NS_PER_CALL, "Profiling halted state timer tasks.", "/PROF/VM/CPU%d/Halt/Timers", idCpu);
+            AssertRC(rc);
+        }
+
+        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocNew,   STAMTYPE_COUNTER,     "/VM/Req/AllocNew",       STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a new packet.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRaces, STAMTYPE_COUNTER,     "/VM/Req/AllocRaces",     STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc causing races.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqAllocRecycled, STAMTYPE_COUNTER,  "/VM/Req/AllocRecycled",  STAMUNIT_OCCURENCES,        "Number of VMR3ReqAlloc returning a recycled packet.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqFree,       STAMTYPE_COUNTER,     "/VM/Req/Free",           STAMUNIT_OCCURENCES,        "Number of VMR3ReqFree calls.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqFreeOverflow, STAMTYPE_COUNTER,   "/VM/Req/FreeOverflow",   STAMUNIT_OCCURENCES,        "Number of times the request was actually freed.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqProcessed,  STAMTYPE_COUNTER,     "/VM/Req/Processed",      STAMUNIT_OCCURENCES,        "Number of processed requests (any queue).");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqMoreThan1,  STAMTYPE_COUNTER,     "/VM/Req/MoreThan1",      STAMUNIT_OCCURENCES,        "Number of times there are more than one request on the queue when processing it.");
+        STAM_REG(pVM, &pUVM->vm.s.StatReqPushBackRaces, STAMTYPE_COUNTER,  "/VM/Req/PushBackRaces",  STAMUNIT_OCCURENCES,        "Number of push back races.");
+
+        rc = CPUMR3Init(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = HWACCMR3Init(pVM);
+            if (RT_SUCCESS(rc))
+            {
+                rc = PGMR3Init(pVM);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = REMR3Init(pVM);
+                    if (RT_SUCCESS(rc))
+                    {
+                        rc = MMR3InitPaging(pVM);
+                        if (RT_SUCCESS(rc))
+                            rc = TMR3Init(pVM);
+                        if (RT_SUCCESS(rc))
+                        {
+                            rc = FTMR3Init(pVM);
+                            if (RT_SUCCESS(rc))
+                            {
+                                rc = VMMR3Init(pVM);
+                                if (RT_SUCCESS(rc))
+                                {
+                                    rc = SELMR3Init(pVM);
+                                    if (RT_SUCCESS(rc))
+                                    {
+                                        rc = TRPMR3Init(pVM);
+                                        if (RT_SUCCESS(rc))
+                                        {
+                                            rc = CSAMR3Init(pVM);
+                                            if (RT_SUCCESS(rc))
+                                            {
+                                                rc = PATMR3Init(pVM);
+                                                if (RT_SUCCESS(rc))
+                                                {
+                                                    rc = IOMR3Init(pVM);
+                                                    if (RT_SUCCESS(rc))
+                                                    {
+                                                        rc = EMR3Init(pVM);
+                                                        if (RT_SUCCESS(rc))
+                                                        {
+                                                            rc = DBGFR3Init(pVM);
+                                                            if (RT_SUCCESS(rc))
+                                                            {
+                                                                rc = PDMR3Init(pVM);
+                                                                if (RT_SUCCESS(rc))
+                                                                {
+                                                                    rc = PGMR3InitDynMap(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = MMR3HyperInitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = PATMR3InitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = PGMR3InitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = SELMR3InitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = TMR3InitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = REMR3InitFinalize(pVM);
+                                                                    if (RT_SUCCESS(rc))
+                                                                        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_RING3);
+                                                                    if (RT_SUCCESS(rc))
+                                                                    {
+                                                                        LogFlow(("vmR3InitRing3: returns %Rrc\n", VINF_SUCCESS));
+                                                                        return VINF_SUCCESS;
+                                                                    }
+                                                                    int rc2 = PDMR3Term(pVM);
+                                                                    AssertRC(rc2);
+                                                                }
+                                                                int rc2 = DBGFR3Term(pVM);
+                                                                AssertRC(rc2);
+                                                            }
+                                                            int rc2 = EMR3Term(pVM);
+                                                            AssertRC(rc2);
+                                                        }
+                                                        int rc2 = IOMR3Term(pVM);
+                                                        AssertRC(rc2);
+                                                    }
+                                                    int rc2 = PATMR3Term(pVM);
+                                                    AssertRC(rc2);
+                                                }
+                                                int rc2 = CSAMR3Term(pVM);
+                                                AssertRC(rc2);
+                                            }
+                                            int rc2 = TRPMR3Term(pVM);
+                                            AssertRC(rc2);
+                                        }
+                                        int rc2 = SELMR3Term(pVM);
+                                        AssertRC(rc2);
+                                    }
+                                    int rc2 = VMMR3Term(pVM);
+                                    AssertRC(rc2);
+                                }
+                                int rc2 = FTMR3Term(pVM);
+                                AssertRC(rc2);
+                            }
+                            int rc2 = TMR3Term(pVM);
+                            AssertRC(rc2);
+                        }
+                        int rc2 = REMR3Term(pVM);
+                        AssertRC(rc2);
+                    }
+                    int rc2 = PGMR3Term(pVM);
+                    AssertRC(rc2);
+                }
+                int rc2 = HWACCMR3Term(pVM);
+                AssertRC(rc2);
+            }
+            //int rc2 = CPUMR3Term(pVM);
+            //AssertRC(rc2);
+        }
+        /* MMR3Term is not called here because it'll kill the heap. */
+    }
+
+    LogFlow(("vmR3InitRing3: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Initializes all R0 components of the VM
+ */
+static int vmR3InitRing0(PVM pVM)
+{
+    LogFlow(("vmR3InitRing0:\n"));
+
+    /*
+     * Check for FAKE suplib mode.
+     */
+    int rc = VINF_SUCCESS;
+    const char *psz = RTEnvGet("VBOX_SUPLIB_FAKE");
+    if (!psz || strcmp(psz, "fake"))
+    {
+        /*
+         * Call the VMMR0 component and let it do the init.
+         */
+        rc = VMMR3InitR0(pVM);
+    }
+    else
+        Log(("vmR3InitRing0: skipping because of VBOX_SUPLIB_FAKE=fake\n"));
+
+    /*
+     * Do notifications and return.
+     */
+    if (RT_SUCCESS(rc))
+        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_RING0);
+
+    /** @todo Move this to the VMINITCOMPLETED_RING0 notification handler. */
+    if (RT_SUCCESS(rc))
+    {
+        rc = HWACCMR3InitFinalizeR0(pVM);
+        CPUMR3SetHWVirtEx(pVM, HWACCMIsEnabled(pVM));
+    }
+
+    LogFlow(("vmR3InitRing0: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Initializes all GC components of the VM
+ */
+static int vmR3InitGC(PVM pVM)
+{
+    LogFlow(("vmR3InitGC:\n"));
+
+    /*
+     * Check for FAKE suplib mode.
+     */
+    int rc = VINF_SUCCESS;
+    const char *psz = RTEnvGet("VBOX_SUPLIB_FAKE");
+    if (!psz || strcmp(psz, "fake"))
+    {
+        /*
+         * Call the VMMR0 component and let it do the init.
+         */
+        rc = VMMR3InitRC(pVM);
+    }
+    else
+        Log(("vmR3InitGC: skipping because of VBOX_SUPLIB_FAKE=fake\n"));
+
+    /*
+     * Do notifications and return.
+     */
+    if (RT_SUCCESS(rc))
+        rc = vmR3InitDoCompleted(pVM, VMINITCOMPLETED_GC);
+    LogFlow(("vmR3InitGC: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Do init completed notifications.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   enmWhat     What's completed.
+ */
+static int vmR3InitDoCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    int rc = VMMR3InitCompleted(pVM, enmWhat);
+    if (RT_SUCCESS(rc))
+        rc = HWACCMR3InitCompleted(pVM, enmWhat);
+    return rc;
+}
+
+
+/**
+ * Logger callback for inserting a custom prefix.
+ *
+ * @returns Number of chars written.
+ * @param   pLogger             The logger.
+ * @param   pchBuf              The output buffer.
+ * @param   cchBuf              The output buffer size.
+ * @param   pvUser              Pointer to the UVM structure.
+ */
+static DECLCALLBACK(size_t) vmR3LogPrefixCallback(PRTLOGGER pLogger, char *pchBuf, size_t cchBuf, void *pvUser)
+{
+    AssertReturn(cchBuf >= 2, 0);
+    PUVM        pUVM   = (PUVM)pvUser;
+    PUVMCPU     pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
+    if (pUVCpu)
+    {
+        static const char s_szHex[17] = "0123456789abcdef";
+        VMCPUID const     idCpu       = pUVCpu->idCpu;
+        pchBuf[1] = s_szHex[ idCpu       & 15];
+        pchBuf[0] = s_szHex[(idCpu >> 4) & 15];
+    }
+    else
+    {
+        pchBuf[0] = 'x';
+        pchBuf[1] = 'y';
+    }
+
+    return 2;
+}
+
+
+/**
+ * Calls the relocation functions for all VMM components so they can update
+ * any GC pointers. When this function is called all the basic VM members
+ * have been updated  and the actual memory relocation have been done
+ * by the PGM/MM.
+ *
+ * This is used both on init and on runtime relocations.
+ *
+ * @param   pVM         VM handle.
+ * @param   offDelta    Relocation delta relative to old location.
+ */
+VMMR3DECL(void)   VMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("VMR3Relocate: offDelta=%RGv\n", offDelta));
+
+    /*
+     * The order here is very important!
+     */
+    PGMR3Relocate(pVM, offDelta);
+    PDMR3LdrRelocateU(pVM->pUVM, offDelta);
+    PGMR3Relocate(pVM, 0);              /* Repeat after PDM relocation. */
+    CPUMR3Relocate(pVM);
+    HWACCMR3Relocate(pVM);
+    SELMR3Relocate(pVM);
+    VMMR3Relocate(pVM, offDelta);
+    SELMR3Relocate(pVM);                /* !hack! fix stack! */
+    TRPMR3Relocate(pVM, offDelta);
+    PATMR3Relocate(pVM);
+    CSAMR3Relocate(pVM, offDelta);
+    IOMR3Relocate(pVM, offDelta);
+    EMR3Relocate(pVM);
+    TMR3Relocate(pVM, offDelta);
+    DBGFR3Relocate(pVM, offDelta);
+    PDMR3Relocate(pVM, offDelta);
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3PowerOn.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_SUCCESS. (This is a strict return
+ *          code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3PowerOn(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3PowerOn: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first thread thru here tries to change the state.  We shouldn't be
+     * called again if this fails.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3PowerOn", 1, VMSTATE_POWERING_ON, VMSTATE_CREATED);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(enmVMState == VMSTATE_POWERING_ON,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_INTERNAL_ERROR_4);
+
+    /*
+     * All EMTs changes their state to started.
+     */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+
+    /*
+     * EMT(0) is last thru here and it will make the notification calls
+     * and advance the state.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        PDMR3PowerOn(pVM);
+        vmR3SetState(pVM, VMSTATE_RUNNING, VMSTATE_POWERING_ON);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Powers on the virtual machine.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The VM to power on.
+ *
+ * @thread      Any thread.
+ * @vmstate     Created
+ * @vmstateto   PoweringOn+Running
+ */
+VMMR3DECL(int) VMR3PowerOn(PVM pVM)
+{
+    LogFlow(("VMR3PowerOn: pVM=%p\n", pVM));
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Gather all the EMTs to reduce the init TSC drift and keep
+     * the state changing APIs a bit uniform.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3PowerOn, NULL);
+    LogFlow(("VMR3PowerOn: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Does the suspend notifications.
+ *
+ * @param  pVM      The VM handle.
+ * @thread  EMT(0)
+ */
+static void vmR3SuspendDoWork(PVM pVM)
+{
+    PDMR3Suspend(pVM);
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Suspend.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_SUSPEND. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3Suspend(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3Suspend: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first EMT switches the state to suspending.  If this fails because
+     * something was racing us in one way or the other, there will be no more
+     * calls and thus the state assertion below is not going to annoy anyone.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 2,
+                                 VMSTATE_SUSPENDING,        VMSTATE_RUNNING,
+                                 VMSTATE_SUSPENDING_EXT_LS, VMSTATE_RUNNING_LS);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(    enmVMState == VMSTATE_SUSPENDING
+                    ||  enmVMState == VMSTATE_SUSPENDING_EXT_LS,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_INTERNAL_ERROR_4);
+
+    /*
+     * EMT(0) does the actually suspending *after* all the other CPUs have
+     * been thru here.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        vmR3SuspendDoWork(pVM);
+
+        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 2,
+                                 VMSTATE_SUSPENDED,        VMSTATE_SUSPENDING,
+                                 VMSTATE_SUSPENDED_EXT_LS, VMSTATE_SUSPENDING_EXT_LS);
+        if (RT_FAILURE(rc))
+            return VERR_INTERNAL_ERROR_3;
+    }
+
+    return VINF_EM_SUSPEND;
+}
+
+
+/**
+ * Suspends a running VM.
+ *
+ * @returns VBox status code. When called on EMT, this will be a strict status
+ *          code that has to be propagated up the call stack.
+ *
+ * @param   pVM     The VM to suspend.
+ *
+ * @thread      Any thread.
+ * @vmstate     Running or RunningLS
+ * @vmstateto   Suspending + Suspended or SuspendingExtLS + SuspendedExtLS
+ */
+VMMR3DECL(int) VMR3Suspend(PVM pVM)
+{
+    LogFlow(("VMR3Suspend: pVM=%p\n", pVM));
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Gather all the EMTs to make sure there are no races before
+     * changing the VM state.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3Suspend, NULL);
+    LogFlow(("VMR3Suspend: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Resume.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_RESUME. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3Resume(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3Resume: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first thread thru here tries to change the state.  We shouldn't be
+     * called again if this fails.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3Resume", 1, VMSTATE_RESUMING, VMSTATE_SUSPENDED);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(enmVMState == VMSTATE_RESUMING,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_INTERNAL_ERROR_4);
+
+#if 0
+    /*
+     * All EMTs changes their state to started.
+     */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+#endif
+
+    /*
+     * EMT(0) is last thru here and it will make the notification calls
+     * and advance the state.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        PDMR3Resume(pVM);
+        vmR3SetState(pVM, VMSTATE_RUNNING, VMSTATE_RESUMING);
+        pVM->vm.s.fTeleportedAndNotFullyResumedYet = false;
+    }
+
+    return VINF_EM_RESUME;
+}
+
+
+/**
+ * Resume VM execution.
+ *
+ * @returns VBox status code. When called on EMT, this will be a strict status
+ *          code that has to be propagated up the call stack.
+ *
+ * @param   pVM         The VM to resume.
+ *
+ * @thread      Any thread.
+ * @vmstate     Suspended
+ * @vmstateto   Running
+ */
+VMMR3DECL(int) VMR3Resume(PVM pVM)
+{
+    LogFlow(("VMR3Resume: pVM=%p\n", pVM));
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Gather all the EMTs to make sure there are no races before
+     * changing the VM state.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3Resume, NULL);
+    LogFlow(("VMR3Resume: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Save and VMR3Teleport that suspends the VM
+ * after the live step has been completed.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_RESUME. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          The pfSuspended argument of vmR3SaveTeleport.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3LiveDoSuspend(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3LiveDoSuspend: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    bool *pfSuspended = (bool *)pvUser;
+
+    /*
+     * The first thread thru here tries to change the state.  We shouldn't be
+     * called again if this fails.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1U)
+    {
+        PUVM     pUVM = pVM->pUVM;
+        int      rc;
+
+        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+        VMSTATE enmVMState = pVM->enmVMState;
+        switch (enmVMState)
+        {
+            case VMSTATE_RUNNING_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDING_LS, VMSTATE_RUNNING_LS);
+                rc = VINF_SUCCESS;
+                break;
+
+            case VMSTATE_SUSPENDED_EXT_LS:
+            case VMSTATE_SUSPENDED_LS:          /* (via reset) */
+                rc = VINF_SUCCESS;
+                break;
+
+            case VMSTATE_DEBUGGING_LS:
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            case VMSTATE_OFF_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF, VMSTATE_OFF_LS);
+                rc = VERR_SSM_LIVE_POWERED_OFF;
+                break;
+
+            case VMSTATE_FATAL_ERROR_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_FATAL_ERROR, VMSTATE_FATAL_ERROR_LS);
+                rc = VERR_SSM_LIVE_FATAL_ERROR;
+                break;
+
+            case VMSTATE_GURU_MEDITATION_LS:
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION, VMSTATE_GURU_MEDITATION_LS);
+                rc = VERR_SSM_LIVE_GURU_MEDITATION;
+                break;
+
+            case VMSTATE_POWERING_OFF_LS:
+            case VMSTATE_SUSPENDING_EXT_LS:
+            case VMSTATE_RESETTING_LS:
+            default:
+                AssertMsgFailed(("%s\n", VMR3GetStateName(enmVMState)));
+                rc = VERR_INTERNAL_ERROR_3;
+                break;
+        }
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+        if (RT_FAILURE(rc))
+        {
+            LogFlow(("vmR3LiveDoSuspend: returns %Rrc (state was %s)\n", rc, VMR3GetStateName(enmVMState)));
+            return rc;
+        }
+    }
+
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(enmVMState == VMSTATE_SUSPENDING_LS,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_INTERNAL_ERROR_4);
+
+    /*
+     * Only EMT(0) have work to do since it's last thru here.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        vmR3SuspendDoWork(pVM);
+        int rc = vmR3TrySetState(pVM, "VMR3Suspend", 1,
+                                 VMSTATE_SUSPENDED_LS, VMSTATE_SUSPENDING_LS);
+        if (RT_FAILURE(rc))
+            return VERR_INTERNAL_ERROR_3;
+
+        *pfSuspended = true;
+    }
+
+    return VINF_EM_SUSPEND;
+}
+
+
+/**
+ * EMT rendezvous worker that VMR3Save and VMR3Teleport uses to clean up a
+ * SSMR3LiveDoStep1 failure.
+ *
+ * Doing this as a rendezvous operation avoids all annoying transition
+ * states.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE, VINF_SUCCESS or some specific VERR_SSM_*
+ *          status code. (This is a strict return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          The pfSuspended argument of vmR3SaveTeleport.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3LiveDoStep1Cleanup(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3LiveDoStep1Cleanup: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    bool *pfSuspended = (bool *)pvUser;
+    NOREF(pVCpu);
+
+    int rc = vmR3TrySetState(pVM, "vmR3LiveDoStep1Cleanup", 8,
+                             VMSTATE_OFF,               VMSTATE_OFF_LS,                     /* 1 */
+                             VMSTATE_FATAL_ERROR,       VMSTATE_FATAL_ERROR_LS,             /* 2 */
+                             VMSTATE_GURU_MEDITATION,   VMSTATE_GURU_MEDITATION_LS,         /* 3 */
+                             VMSTATE_SUSPENDED,         VMSTATE_SUSPENDED_LS,               /* 4 */
+                             VMSTATE_SUSPENDED,         VMSTATE_SAVING,
+                             VMSTATE_SUSPENDED,         VMSTATE_SUSPENDED_EXT_LS,
+                             VMSTATE_RUNNING,           VMSTATE_RUNNING_LS,
+                             VMSTATE_DEBUGGING,         VMSTATE_DEBUGGING_LS);
+    if (rc == 1)
+        rc = VERR_SSM_LIVE_POWERED_OFF;
+    else if (rc == 2)
+        rc = VERR_SSM_LIVE_FATAL_ERROR;
+    else if (rc == 3)
+        rc = VERR_SSM_LIVE_GURU_MEDITATION;
+    else if (rc == 4)
+    {
+        *pfSuspended = true;
+        rc = VINF_SUCCESS;
+    }
+    else if (rc > 0)
+        rc = VINF_SUCCESS;
+    return rc;
+}
+
+
+/**
+ * EMT(0) worker for VMR3Save and VMR3Teleport that completes the live save.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SSM_LIVE_SUSPENDED if VMR3Suspend was called.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pSSM            The handle of saved state operation.
+ *
+ * @thread  EMT(0)
+ */
+static DECLCALLBACK(int) vmR3LiveDoStep2(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("vmR3LiveDoStep2: pVM=%p pSSM=%p\n", pVM, pSSM));
+    VM_ASSERT_EMT0(pVM);
+
+    /*
+     * Advance the state and mark if VMR3Suspend was called.
+     */
+    int rc = VINF_SUCCESS;
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    if (enmVMState == VMSTATE_SUSPENDED_LS)
+        vmR3SetState(pVM, VMSTATE_SAVING, VMSTATE_SUSPENDED_LS);
+    else
+    {
+        if (enmVMState != VMSTATE_SAVING)
+            vmR3SetState(pVM, VMSTATE_SAVING, VMSTATE_SUSPENDED_EXT_LS);
+        rc = VINF_SSM_LIVE_SUSPENDED;
+    }
+
+    /*
+     * Finish up and release the handle. Careful with the status codes.
+     */
+    int rc2 = SSMR3LiveDoStep2(pSSM);
+    if (rc == VINF_SUCCESS || (RT_FAILURE(rc2) && RT_SUCCESS(rc)))
+        rc = rc2;
+
+    rc2 = SSMR3LiveDone(pSSM);
+    if (rc == VINF_SUCCESS || (RT_FAILURE(rc2) && RT_SUCCESS(rc)))
+        rc = rc2;
+
+    /*
+     * Advance to the final state and return.
+     */
+    vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_SAVING);
+    Assert(rc > VINF_EM_LAST || rc < VINF_EM_FIRST);
+    return rc;
+}
+
+
+/**
+ * Worker for vmR3SaveTeleport that validates the state and calls SSMR3Save or
+ * SSMR3LiveSave.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   enmAfter            What to do afterwards.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvProgressUser      User argument for the progress callback.
+ * @param   ppSSM               Where to return the saved state handle in case of a
+ *                              live snapshot scenario.
+ * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
+ *
+ * @thread  EMT
+ */
+static DECLCALLBACK(int) vmR3Save(PVM pVM, uint32_t cMsMaxDowntime, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, PSSMHANDLE *ppSSM,
+                                  bool fSkipStateChanges)
+{
+    int rc = VINF_SUCCESS;
+
+    LogFlow(("vmR3Save: pVM=%p cMsMaxDowntime=%u pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p enmAfter=%d pfnProgress=%p pvProgressUser=%p ppSSM=%p\n",
+             pVM, cMsMaxDowntime, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser, ppSSM));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrNull(pszFilename);
+    AssertPtrNull(pStreamOps);
+    AssertPtr(pVM);
+    Assert(   enmAfter == SSMAFTER_DESTROY
+           || enmAfter == SSMAFTER_CONTINUE
+           || enmAfter == SSMAFTER_TELEPORT);
+    AssertPtr(ppSSM);
+    *ppSSM = NULL;
+
+    /*
+     * Change the state and perform/start the saving.
+     */
+    if (!fSkipStateChanges)
+    {
+        rc = vmR3TrySetState(pVM, "VMR3Save", 2,
+                             VMSTATE_SAVING,     VMSTATE_SUSPENDED,
+                             VMSTATE_RUNNING_LS, VMSTATE_RUNNING);
+    }
+    else
+    {
+        Assert(enmAfter != SSMAFTER_TELEPORT);
+        rc = 1;
+    }
+
+    if (rc == 1 && enmAfter != SSMAFTER_TELEPORT)
+    {
+        rc = SSMR3Save(pVM, pszFilename, pStreamOps, pvStreamOpsUser, enmAfter, pfnProgress, pvProgressUser);
+        if (!fSkipStateChanges)
+            vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_SAVING);
+    }
+    else if (rc == 2 || enmAfter == SSMAFTER_TELEPORT)
+    {
+        Assert(!fSkipStateChanges);
+        if (enmAfter == SSMAFTER_TELEPORT)
+            pVM->vm.s.fTeleportedAndNotFullyResumedYet = true;
+        rc = SSMR3LiveSave(pVM, cMsMaxDowntime, pszFilename, pStreamOps, pvStreamOpsUser,
+                           enmAfter, pfnProgress, pvProgressUser, ppSSM);
+        /* (We're not subject to cancellation just yet.) */
+    }
+    else
+        Assert(RT_FAILURE(rc));
+    return rc;
+}
+
+
+/**
+ * Common worker for VMR3Save and VMR3Teleport.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   enmAfter            What to do afterwards.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvProgressUser      User argument for the progress callback.
+ * @param   pfSuspended         Set if we suspended the VM.
+ * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
+ *
+ * @thread  Non-EMT
+ */
+static int vmR3SaveTeleport(PVM pVM, uint32_t cMsMaxDowntime,
+                            const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                            SSMAFTER enmAfter, PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool *pfSuspended,
+                            bool fSkipStateChanges)
+{
+    /*
+     * Request the operation in EMT(0).
+     */
+    PSSMHANDLE pSSM;
+    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/,
+                              (PFNRT)vmR3Save, 10, pVM, cMsMaxDowntime, pszFilename, pStreamOps, pvStreamOpsUser,
+                              enmAfter, pfnProgress, pvProgressUser, &pSSM, fSkipStateChanges);
+    if (    RT_SUCCESS(rc)
+        &&  pSSM)
+    {
+        Assert(!fSkipStateChanges);
+
+        /*
+         * Live snapshot.
+         *
+         * The state handling here is kind of tricky, doing it on EMT(0) helps
+         * a bit. See the VMSTATE diagram for details.
+         */
+        rc = SSMR3LiveDoStep1(pSSM);
+        if (RT_SUCCESS(rc))
+        {
+            if (VMR3GetState(pVM) != VMSTATE_SAVING)
+                for (;;)
+                {
+                    /* Try suspend the VM. */
+                    rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                            vmR3LiveDoSuspend, pfSuspended);
+                    if (rc != VERR_TRY_AGAIN)
+                        break;
+
+                    /* Wait for the state to change. */
+                    RTThreadSleep(250); /** @todo Live Migration: fix this polling wait by some smart use of multiple release event  semaphores.. */
+                }
+            if (RT_SUCCESS(rc))
+                rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3LiveDoStep2, 2, pVM, pSSM);
+            else
+            {
+                int rc2 = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)SSMR3LiveDone, 1, pSSM);
+                AssertMsg(rc2 == rc, ("%Rrc != %Rrc\n", rc2, rc));
+            }
+        }
+        else
+        {
+            int rc2 = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)SSMR3LiveDone, 1, pSSM);
+            AssertMsg(rc2 == rc, ("%Rrc != %Rrc\n", rc2, rc));
+
+            rc2 = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, vmR3LiveDoStep1Cleanup, pfSuspended);
+            if (RT_FAILURE(rc2) && rc == VERR_SSM_CANCELLED)
+                rc = rc2;
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Save current VM state.
+ *
+ * Can be used for both saving the state and creating snapshots.
+ *
+ * When called for a VM in the Running state, the saved state is created live
+ * and the VM is only suspended when the final part of the saving is preformed.
+ * The VM state will not be restored to Running in this case and it's up to the
+ * caller to call VMR3Resume if this is desirable.  (The rational is that the
+ * caller probably wish to reconfigure the disks before resuming the VM.)
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM which state should be saved.
+ * @param   pszFilename         The name of the save state file.
+ * @param   pStreamOps          The stream methods.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   fContinueAfterwards Whether continue execution afterwards or not.
+ *                              When in doubt, set this to true.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvUser              User argument for the progress callback.
+ * @param   pfSuspended         Set if we suspended the VM.
+ *
+ * @thread      Non-EMT.
+ * @vmstate     Suspended or Running
+ * @vmstateto   Saving+Suspended or
+ *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
+ */
+VMMR3DECL(int) VMR3Save(PVM pVM, const char *pszFilename, bool fContinueAfterwards, PFNVMPROGRESS pfnProgress, void *pvUser, bool *pfSuspended)
+{
+    LogFlow(("VMR3Save: pVM=%p pszFilename=%p:{%s} fContinueAfterwards=%RTbool pfnProgress=%p pvUser=%p pfSuspended=%p\n",
+             pVM, pszFilename, pszFilename, fContinueAfterwards, pfnProgress, pvUser, pfSuspended));
+
+    /*
+     * Validate input.
+     */
+    AssertPtr(pfSuspended);
+    *pfSuspended = false;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_OTHER_THREAD(pVM);
+    AssertReturn(VALID_PTR(pszFilename), VERR_INVALID_POINTER);
+    AssertReturn(*pszFilename, VERR_INVALID_PARAMETER);
+    AssertPtrNullReturn(pfnProgress, VERR_INVALID_POINTER);
+
+    /*
+     * Join paths with VMR3Teleport.
+     */
+    SSMAFTER enmAfter = fContinueAfterwards ? SSMAFTER_CONTINUE : SSMAFTER_DESTROY;
+    int rc = vmR3SaveTeleport(pVM, 250 /*cMsMaxDowntime*/,
+                              pszFilename, NULL /* pStreamOps */, NULL /* pvStreamOpsUser */,
+                              enmAfter, pfnProgress, pvUser, pfSuspended,
+                              false /* fSkipStateChanges */);
+    LogFlow(("VMR3Save: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
+    return rc;
+}
+
+/**
+ * Save current VM state (used by FTM)
+ *
+ * Can be used for both saving the state and creating snapshots.
+ *
+ * When called for a VM in the Running state, the saved state is created live
+ * and the VM is only suspended when the final part of the saving is preformed.
+ * The VM state will not be restored to Running in this case and it's up to the
+ * caller to call VMR3Resume if this is desirable.  (The rational is that the
+ * caller probably wish to reconfigure the disks before resuming the VM.)
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM which state should be saved.
+ * @param   pStreamOps          The stream methods.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   pfSuspended         Set if we suspended the VM.
+ * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
+ *
+ * @thread      Any
+ * @vmstate     Suspended or Running
+ * @vmstateto   Saving+Suspended or
+ *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
+ */
+VMMR3DECL(int) VMR3SaveFT(PVM pVM, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser, bool *pfSuspended,
+                          bool fSkipStateChanges)
+{
+    LogFlow(("VMR3SaveFT: pVM=%p pStreamOps=%p pvSteamOpsUser=%p pfSuspended=%p\n",
+             pVM, pStreamOps, pvStreamOpsUser, pfSuspended));
+
+    /*
+     * Validate input.
+     */
+    AssertPtr(pfSuspended);
+    *pfSuspended = false;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertReturn(pStreamOps, VERR_INVALID_PARAMETER);
+
+    /*
+     * Join paths with VMR3Teleport.
+     */
+    int rc = vmR3SaveTeleport(pVM, 250 /*cMsMaxDowntime*/,
+                              NULL, pStreamOps, pvStreamOpsUser,
+                              SSMAFTER_CONTINUE, NULL, NULL, pfSuspended,
+                              fSkipStateChanges);
+    LogFlow(("VMR3SaveFT: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
+    return rc;
+}
+
+
+/**
+ * Teleport the VM (aka live migration).
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM which state should be saved.
+ * @param   cMsMaxDowntime      The maximum downtime given as milliseconds.
+ * @param   pStreamOps          The stream methods.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvProgressUser      User argument for the progress callback.
+ * @param   pfSuspended         Set if we suspended the VM.
+ *
+ * @thread      Non-EMT.
+ * @vmstate     Suspended or Running
+ * @vmstateto   Saving+Suspended or
+ *              RunningLS+SuspendingLS+SuspendedLS+Saving+Suspended.
+ */
+VMMR3DECL(int) VMR3Teleport(PVM pVM, uint32_t cMsMaxDowntime, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                            PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool *pfSuspended)
+{
+    LogFlow(("VMR3Teleport: pVM=%p cMsMaxDowntime=%u pStreamOps=%p pvStreamOps=%p pfnProgress=%p pvProgressUser=%p\n",
+             pVM, cMsMaxDowntime, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtr(pfSuspended);
+    *pfSuspended = false;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    VM_ASSERT_OTHER_THREAD(pVM);
+    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
+    AssertPtrNullReturn(pfnProgress, VERR_INVALID_POINTER);
+
+    /*
+     * Join paths with VMR3Save.
+     */
+    int rc = vmR3SaveTeleport(pVM, cMsMaxDowntime,
+                              NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser,
+                              SSMAFTER_TELEPORT, pfnProgress, pvProgressUser, pfSuspended,
+                              false /* fSkipStateChanges */);
+    LogFlow(("VMR3Teleport: returns %Rrc (*pfSuspended=%RTbool)\n", rc, *pfSuspended));
+    return rc;
+}
+
+
+
+/**
+ * EMT(0) worker for VMR3LoadFromFile and VMR3LoadFromStream.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pszFilename         The name of the file.  NULL if pStreamOps is used.
+ * @param   pStreamOps          The stream methods.  NULL if pszFilename is used.
+ * @param   pvStreamOpsUser     The user argument to the stream methods.
+ * @param   pfnProgress         Progress callback. Optional.
+ * @param   pvUser              User argument for the progress callback.
+ * @param   fTeleporting        Indicates whether we're teleporting or not.
+ * @param   fSkipStateChanges   Set if we're supposed to skip state changes (FTM delta case)
+ *
+ * @thread  EMT.
+ */
+static DECLCALLBACK(int) vmR3Load(PVM pVM, const char *pszFilename, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  PFNVMPROGRESS pfnProgress, void *pvProgressUser, bool fTeleporting,
+                                  bool fSkipStateChanges)
+{
+    int rc = VINF_SUCCESS;
+
+    LogFlow(("vmR3Load: pVM=%p pszFilename=%p:{%s} pStreamOps=%p pvStreamOpsUser=%p pfnProgress=%p pvProgressUser=%p fTeleporting=%RTbool\n",
+             pVM, pszFilename, pszFilename, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser, fTeleporting));
+
+    /*
+     * Validate input (paranoia).
+     */
+    AssertPtr(pVM);
+    AssertPtrNull(pszFilename);
+    AssertPtrNull(pStreamOps);
+    AssertPtrNull(pfnProgress);
+
+    if (!fSkipStateChanges)
+    {
+        /*
+         * Change the state and perform the load.
+         *
+         * Always perform a relocation round afterwards to make sure hypervisor
+         * selectors and such are correct.
+         */
+        rc = vmR3TrySetState(pVM, "VMR3Load", 2,
+                                 VMSTATE_LOADING, VMSTATE_CREATED,
+                                 VMSTATE_LOADING, VMSTATE_SUSPENDED);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+    pVM->vm.s.fTeleportedAndNotFullyResumedYet = fTeleporting;
+
+    uint32_t cErrorsPriorToSave = VMR3GetErrorCount(pVM);
+    rc = SSMR3Load(pVM, pszFilename, pStreamOps, pvStreamOpsUser, SSMAFTER_RESUME, pfnProgress, pvProgressUser);
+    if (RT_SUCCESS(rc))
+    {
+        VMR3Relocate(pVM, 0 /*offDelta*/);
+        if (!fSkipStateChanges)
+            vmR3SetState(pVM, VMSTATE_SUSPENDED, VMSTATE_LOADING);
+    }
+    else
+    {
+        pVM->vm.s.fTeleportedAndNotFullyResumedYet = false;
+        if (!fSkipStateChanges)
+            vmR3SetState(pVM, VMSTATE_LOAD_FAILURE, VMSTATE_LOADING);
+
+        if (cErrorsPriorToSave == VMR3GetErrorCount(pVM))
+            rc = VMSetError(pVM, rc, RT_SRC_POS,
+                            N_("Unable to restore the virtual machine's saved state from '%s'. "
+                               "It may be damaged or from an older version of VirtualBox.  "
+                               "Please discard the saved state before starting the virtual machine"),
+                            pszFilename);
+    }
+
+    return rc;
+}
+
+
+/**
+ * Loads a VM state into a newly created VM or a one that is suspended.
+ *
+ * To restore a saved state on VM startup, call this function and then resume
+ * the VM instead of powering it on.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pszFilename     The name of the save state file.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvUser          User argument for the progress callback.
+ *
+ * @thread      Any thread.
+ * @vmstate     Created, Suspended
+ * @vmstateto   Loading+Suspended
+ */
+VMMR3DECL(int) VMR3LoadFromFile(PVM pVM, const char *pszFilename, PFNVMPROGRESS pfnProgress, void *pvUser)
+{
+    LogFlow(("VMR3LoadFromFile: pVM=%p pszFilename=%p:{%s} pfnProgress=%p pvUser=%p\n",
+             pVM, pszFilename, pszFilename, pfnProgress, pvUser));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pszFilename, VERR_INVALID_POINTER);
+
+    /*
+     * Forward the request to EMT(0).  No need to setup a rendezvous here
+     * since there is no execution taking place when this call is allowed.
+     */
+    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 8,
+                              pVM, pszFilename, (uintptr_t)NULL /*pStreamOps*/, (uintptr_t)NULL /*pvStreamOpsUser*/, pfnProgress, pvUser,
+                              false /*fTeleporting*/, false /* fSkipStateChanges */);
+    LogFlow(("VMR3LoadFromFile: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMR3LoadFromFile for arbitrary file streams.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pStreamOps      The stream methods.
+ * @param   pvStreamOpsUser The user argument to the stream methods.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvProgressUser  User argument for the progress callback.
+ *
+ * @thread      Any thread.
+ * @vmstate     Created, Suspended
+ * @vmstateto   Loading+Suspended
+ */
+VMMR3DECL(int) VMR3LoadFromStream(PVM pVM, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser,
+                                  PFNVMPROGRESS pfnProgress, void *pvProgressUser)
+{
+    LogFlow(("VMR3LoadFromStream: pVM=%p pStreamOps=%p pvStreamOpsUser=%p pfnProgress=%p pvProgressUser=%p\n",
+             pVM, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
+
+    /*
+     * Forward the request to EMT(0).  No need to setup a rendezvous here
+     * since there is no execution taking place when this call is allowed.
+     */
+    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 8,
+                              pVM, (uintptr_t)NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser, pfnProgress, pvProgressUser,
+                              true /*fTeleporting*/, false /* fSkipStateChanges */);
+    LogFlow(("VMR3LoadFromStream: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VMR3LoadFromFileFT for arbitrary file streams.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The VM handle.
+ * @param   pStreamOps      The stream methods.
+ * @param   pvStreamOpsUser The user argument to the stream methods.
+ * @param   pfnProgress     Progress callback. Optional.
+ * @param   pvProgressUser  User argument for the progress callback.
+ *
+ * @thread      Any thread.
+ * @vmstate     Created, Suspended
+ * @vmstateto   Loading+Suspended
+ */
+VMMR3DECL(int) VMR3LoadFromStreamFT(PVM pVM, PCSSMSTRMOPS pStreamOps, void *pvStreamOpsUser)
+{
+    LogFlow(("VMR3LoadFromStreamFT: pVM=%p pStreamOps=%p pvStreamOpsUser=%p\n",
+             pVM, pStreamOps, pvStreamOpsUser));
+
+    /*
+     * Validate input.
+     */
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertPtrReturn(pStreamOps, VERR_INVALID_POINTER);
+
+    /*
+     * Forward the request to EMT(0).  No need to setup a rendezvous here
+     * since there is no execution taking place when this call is allowed.
+     */
+    int rc = VMR3ReqCallWaitU(pVM->pUVM, 0 /*idDstCpu*/, (PFNRT)vmR3Load, 8,
+                              pVM, (uintptr_t)NULL /*pszFilename*/, pStreamOps, pvStreamOpsUser, NULL, NULL,
+                              true /*fTeleporting*/, true /* fSkipStateChanges */);
+    LogFlow(("VMR3LoadFromStream: returns %Rrc\n", rc));
+    return rc;
+}
+
+/**
+ * EMT rendezvous worker for VMR3PowerOff.
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_OFF. (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3PowerOff(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    LogFlow(("vmR3PowerOff: pVM=%p pVCpu=%p/#%u\n", pVM, pVCpu, pVCpu->idCpu));
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first EMT thru here will change the state to PoweringOff.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3PowerOff", 11,
+                                 VMSTATE_POWERING_OFF,    VMSTATE_RUNNING,           /* 1 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_SUSPENDED,         /* 2 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_DEBUGGING,         /* 3 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_LOAD_FAILURE,      /* 4 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_GURU_MEDITATION,   /* 5 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_FATAL_ERROR,       /* 6 */
+                                 VMSTATE_POWERING_OFF,    VMSTATE_CREATED,           /* 7 */   /** @todo update the diagram! */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_RUNNING_LS,        /* 8 */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_DEBUGGING_LS,      /* 9 */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_GURU_MEDITATION_LS,/* 10 */
+                                 VMSTATE_POWERING_OFF_LS, VMSTATE_FATAL_ERROR_LS);   /* 11 */
+        if (RT_FAILURE(rc))
+            return rc;
+        if (rc >= 7)
+            SSMR3Cancel(pVM);
+    }
+
+    /*
+     * Check the state.
+     */
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertMsgReturn(   enmVMState == VMSTATE_POWERING_OFF
+                    || enmVMState == VMSTATE_POWERING_OFF_LS,
+                    ("%s\n", VMR3GetStateName(enmVMState)),
+                    VERR_VM_INVALID_VM_STATE);
+
+    /*
+     * EMT(0) does the actual power off work here *after* all the other EMTs
+     * have been thru and entered the STOPPED state.
+     */
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STOPPED);
+    if (pVCpu->idCpu == 0)
+    {
+        /*
+         * For debugging purposes, we will log a summary of the guest state at this point.
+         */
+        if (enmVMState != VMSTATE_GURU_MEDITATION)
+        {
+            /** @todo SMP support? */
+            /** @todo make the state dumping at VMR3PowerOff optional. */
+            bool fOldBuffered = RTLogRelSetBuffering(true /*fBuffered*/);
+            RTLogRelPrintf("****************** Guest state at power off ******************\n");
+            DBGFR3Info(pVM, "cpumguest", "verbose", DBGFR3InfoLogRelHlp());
+            RTLogRelPrintf("***\n");
+            DBGFR3Info(pVM, "mode", NULL, DBGFR3InfoLogRelHlp());
+            RTLogRelPrintf("***\n");
+            DBGFR3Info(pVM, "activetimers", NULL, DBGFR3InfoLogRelHlp());
+            RTLogRelPrintf("***\n");
+            DBGFR3Info(pVM, "gdt", NULL, DBGFR3InfoLogRelHlp());
+            /** @todo dump guest call stack. */
+#if 1 // "temporary" while debugging #1589
+            RTLogRelPrintf("***\n");
+            uint32_t esp = CPUMGetGuestESP(pVCpu);
+            if (    CPUMGetGuestSS(pVCpu) == 0
+                &&  esp < _64K)
+            {
+                uint8_t abBuf[PAGE_SIZE];
+                RTLogRelPrintf("***\n"
+                               "ss:sp=0000:%04x ", esp);
+                uint32_t Start = esp & ~(uint32_t)63;
+                int rc = PGMPhysSimpleReadGCPhys(pVM, abBuf, Start, 0x100);
+                if (RT_SUCCESS(rc))
+                    RTLogRelPrintf("0000:%04x TO 0000:%04x:\n"
+                                   "%.*Rhxd\n",
+                                   Start, Start + 0x100 - 1,
+                                   0x100, abBuf);
+                else
+                    RTLogRelPrintf("rc=%Rrc\n", rc);
+
+                /* grub ... */
+                if (esp < 0x2000 && esp > 0x1fc0)
+                {
+                    rc = PGMPhysSimpleReadGCPhys(pVM, abBuf, 0x8000, 0x800);
+                    if (RT_SUCCESS(rc))
+                        RTLogRelPrintf("0000:8000 TO 0000:87ff:\n"
+                                       "%.*Rhxd\n",
+                                       0x800, abBuf);
+                }
+                /* microsoft cdrom hang ... */
+                if (true)
+                {
+                    rc = PGMPhysSimpleReadGCPhys(pVM, abBuf, 0x8000, 0x200);
+                    if (RT_SUCCESS(rc))
+                        RTLogRelPrintf("2000:0000 TO 2000:01ff:\n"
+                                       "%.*Rhxd\n",
+                                       0x200, abBuf);
+                }
+            }
+#endif
+            RTLogRelSetBuffering(fOldBuffered);
+            RTLogRelPrintf("************** End of Guest state at power off ***************\n");
+        }
+
+        /*
+         * Perform the power off notifications and advance the state to
+         * Off or OffLS.
+         */
+        PDMR3PowerOff(pVM);
+
+        PUVM pUVM = pVM->pUVM;
+        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+        enmVMState = pVM->enmVMState;
+        if (enmVMState == VMSTATE_POWERING_OFF_LS)
+            vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF_LS, VMSTATE_POWERING_OFF_LS);
+        else
+            vmR3SetStateLocked(pVM, pUVM, VMSTATE_OFF,    VMSTATE_POWERING_OFF);
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+    }
+    return VINF_EM_OFF;
+}
+
+
+/**
+ * Power off the VM.
+ *
+ * @returns VBox status code. When called on EMT, this will be a strict status
+ *          code that has to be propagated up the call stack.
+ *
+ * @param   pVM     The handle of the VM to be powered off.
+ *
+ * @thread      Any thread.
+ * @vmstate     Suspended, Running, Guru Meditation, Load Failure
+ * @vmstateto   Off or OffLS
+ */
+VMMR3DECL(int)   VMR3PowerOff(PVM pVM)
+{
+    LogFlow(("VMR3PowerOff: pVM=%p\n", pVM));
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Gather all the EMTs to make sure there are no races before
+     * changing the VM state.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3PowerOff, NULL);
+    LogFlow(("VMR3PowerOff: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Destroys the VM.
+ *
+ * The VM must be powered off (or never really powered on) to call this
+ * function. The VM handle is destroyed and can no longer be used up successful
+ * return.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM     The handle of the VM which should be destroyed.
+ *
+ * @thread      Any none emulation thread.
+ * @vmstate     Off, Created
+ * @vmstateto   N/A
+ */
+VMMR3DECL(int) VMR3Destroy(PVM pVM)
+{
+    LogFlow(("VMR3Destroy: pVM=%p\n", pVM));
+
+    /*
+     * Validate input.
+     */
+    if (!pVM)
+        return VERR_INVALID_PARAMETER;
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+    AssertLogRelReturn(!VM_IS_EMT(pVM), VERR_VM_THREAD_IS_EMT);
+
+    /*
+     * Change VM state to destroying and unlink the VM.
+     */
+    int rc = vmR3TrySetState(pVM, "VMR3Destroy", 1, VMSTATE_DESTROYING, VMSTATE_OFF);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /** @todo lock this when we start having multiple machines in a process... */
+    PUVM pUVM = pVM->pUVM; AssertPtr(pUVM);
+    if (g_pUVMsHead == pUVM)
+        g_pUVMsHead = pUVM->pNext;
+    else
+    {
+        PUVM pPrev = g_pUVMsHead;
+        while (pPrev && pPrev->pNext != pUVM)
+            pPrev = pPrev->pNext;
+        AssertMsgReturn(pPrev, ("pUVM=%p / pVM=%p  is INVALID!\n", pUVM, pVM), VERR_INVALID_PARAMETER);
+
+        pPrev->pNext = pUVM->pNext;
+    }
+    pUVM->pNext = NULL;
+
+    /*
+     * Notify registered at destruction listeners.
+     */
+    vmR3AtDtor(pVM);
+
+    /*
+     * Call vmR3Destroy on each of the EMTs ending with EMT(0) doing the bulk
+     * of the cleanup.
+     */
+    /* vmR3Destroy on all EMTs, ending with EMT(0). */
+    rc = VMR3ReqCallWaitU(pUVM, VMCPUID_ALL_REVERSE, (PFNRT)vmR3Destroy, 1, pVM);
+    AssertLogRelRC(rc);
+
+    /* Wait for EMTs and destroy the UVM. */
+    vmR3DestroyUVM(pUVM, 30000);
+
+    LogFlow(("VMR3Destroy: returns VINF_SUCCESS\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal destruction worker.
+ *
+ * This is either called from VMR3Destroy via VMR3ReqCallU or from
+ * vmR3EmulationThreadWithId when EMT(0) terminates after having called
+ * VMR3Destroy().
+ *
+ * When called on EMT(0), it will performed the great bulk of the destruction.
+ * When called on the other EMTs, they will do nothing and the whole purpose is
+ * to return VINF_EM_TERMINATE so they break out of their run loops.
+ *
+ * @returns VINF_EM_TERMINATE.
+ * @param   pVM     The VM handle.
+ */
+DECLCALLBACK(int) vmR3Destroy(PVM pVM)
+{
+    PUVM   pUVM  = pVM->pUVM;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu);
+    LogFlow(("vmR3Destroy: pVM=%p pUVM=%p pVCpu=%p idCpu=%u\n", pVM, pUVM, pVCpu, pVCpu->idCpu));
+
+    /*
+     * Only VCPU 0 does the full cleanup (last).
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        /*
+         * Dump statistics to the log.
+         */
+#if defined(VBOX_WITH_STATISTICS) || defined(LOG_ENABLED)
+        RTLogFlags(NULL, "nodisabled nobuffered");
+#endif
+#ifdef VBOX_WITH_STATISTICS
+        STAMR3Dump(pVM, "*");
+#else
+        LogRel(("************************* Statistics *************************\n"));
+        STAMR3DumpToReleaseLog(pVM, "*");
+        LogRel(("********************* End of statistics **********************\n"));
+#endif
+
+        /*
+         * Destroy the VM components.
+         */
+        int rc = TMR3Term(pVM);
+        AssertRC(rc);
+#ifdef VBOX_WITH_DEBUGGER
+        rc = DBGCTcpTerminate(pVM, pUVM->vm.s.pvDBGC);
+        pUVM->vm.s.pvDBGC = NULL;
+#endif
+        AssertRC(rc);
+        rc = FTMR3Term(pVM);
+        AssertRC(rc);
+        rc = DBGFR3Term(pVM);
+        AssertRC(rc);
+        rc = PDMR3Term(pVM);
+        AssertRC(rc);
+        rc = EMR3Term(pVM);
+        AssertRC(rc);
+        rc = IOMR3Term(pVM);
+        AssertRC(rc);
+        rc = CSAMR3Term(pVM);
+        AssertRC(rc);
+        rc = PATMR3Term(pVM);
+        AssertRC(rc);
+        rc = TRPMR3Term(pVM);
+        AssertRC(rc);
+        rc = SELMR3Term(pVM);
+        AssertRC(rc);
+        rc = REMR3Term(pVM);
+        AssertRC(rc);
+        rc = HWACCMR3Term(pVM);
+        AssertRC(rc);
+        rc = PGMR3Term(pVM);
+        AssertRC(rc);
+        rc = VMMR3Term(pVM); /* Terminates the ring-0 code! */
+        AssertRC(rc);
+        rc = CPUMR3Term(pVM);
+        AssertRC(rc);
+        SSMR3Term(pVM);
+        rc = PDMR3CritSectTerm(pVM);
+        AssertRC(rc);
+        rc = MMR3Term(pVM);
+        AssertRC(rc);
+
+        /*
+         * We're done, tell the other EMTs to quit.
+         */
+        ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+        ASMAtomicWriteU32(&pVM->fGlobalForcedActions, VM_FF_CHECK_VM_STATE); /* Can't hurt... */
+        LogFlow(("vmR3Destroy: returning %Rrc\n", VINF_EM_TERMINATE));
+    }
+    return VINF_EM_TERMINATE;
+}
+
+
+/**
+ * Destroys the UVM portion.
+ *
+ * This is called as the final step in the VM destruction or as the cleanup
+ * in case of a creation failure.
+ *
+ * @param   pVM             VM Handle.
+ * @param   cMilliesEMTWait The number of milliseconds to wait for the emulation
+ *                          threads.
+ */
+static void vmR3DestroyUVM(PUVM pUVM, uint32_t cMilliesEMTWait)
+{
+    /*
+     * Signal termination of each the emulation threads and
+     * wait for them to complete.
+     */
+    /* Signal them. */
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+    if (pUVM->pVM)
+        VM_FF_SET(pUVM->pVM, VM_FF_CHECK_VM_STATE); /* Can't hurt... */
+    for (VMCPUID i = 0; i < pUVM->cCpus; i++)
+    {
+        VMR3NotifyGlobalFFU(pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
+        RTSemEventSignal(pUVM->aCpus[i].vm.s.EventSemWait);
+    }
+
+    /* Wait for them. */
+    uint64_t    NanoTS = RTTimeNanoTS();
+    RTTHREAD    hSelf  = RTThreadSelf();
+    ASMAtomicUoWriteBool(&pUVM->vm.s.fTerminateEMT, true);
+    for (VMCPUID i = 0; i < pUVM->cCpus; i++)
+    {
+        RTTHREAD hThread = pUVM->aCpus[i].vm.s.ThreadEMT;
+        if (    hThread != NIL_RTTHREAD
+            &&  hThread != hSelf)
+        {
+            uint64_t cMilliesElapsed = (RTTimeNanoTS() - NanoTS) / 1000000;
+            int rc2 = RTThreadWait(hThread,
+                                   cMilliesElapsed < cMilliesEMTWait
+                                   ? RT_MAX(cMilliesEMTWait - cMilliesElapsed, 2000)
+                                   : 2000,
+                                   NULL);
+            if (rc2 == VERR_TIMEOUT) /* avoid the assertion when debugging. */
+                rc2 = RTThreadWait(hThread, 1000, NULL);
+            AssertLogRelMsgRC(rc2, ("i=%u rc=%Rrc\n", i, rc2));
+            if (RT_SUCCESS(rc2))
+                pUVM->aCpus[0].vm.s.ThreadEMT = NIL_RTTHREAD;
+        }
+    }
+
+    /* Cleanup the semaphores. */
+    for (VMCPUID i = 0; i < pUVM->cCpus; i++)
+    {
+        RTSemEventDestroy(pUVM->aCpus[i].vm.s.EventSemWait);
+        pUVM->aCpus[i].vm.s.EventSemWait = NIL_RTSEMEVENT;
+    }
+
+    /*
+     * Free the event semaphores associated with the request packets.
+     */
+    unsigned cReqs = 0;
+    for (unsigned i = 0; i < RT_ELEMENTS(pUVM->vm.s.apReqFree); i++)
+    {
+        PVMREQ pReq = pUVM->vm.s.apReqFree[i];
+        pUVM->vm.s.apReqFree[i] = NULL;
+        for (; pReq; pReq = pReq->pNext, cReqs++)
+        {
+            pReq->enmState = VMREQSTATE_INVALID;
+            RTSemEventDestroy(pReq->EventSem);
+        }
+    }
+    Assert(cReqs == pUVM->vm.s.cReqFree); NOREF(cReqs);
+
+    /*
+     * Kill all queued requests. (There really shouldn't be any!)
+     */
+    for (unsigned i = 0; i < 10; i++)
+    {
+        PVMREQ pReqHead = ASMAtomicXchgPtrT(&pUVM->vm.s.pReqs, NULL, PVMREQ);
+        AssertMsg(!pReqHead, ("This isn't supposed to happen! VMR3Destroy caller has to serialize this.\n"));
+        if (!pReqHead)
+            break;
+        for (PVMREQ pReq = pReqHead; pReq; pReq = pReq->pNext)
+        {
+            ASMAtomicUoWriteSize(&pReq->iStatus, VERR_INTERNAL_ERROR);
+            ASMAtomicWriteSize(&pReq->enmState, VMREQSTATE_INVALID);
+            RTSemEventSignal(pReq->EventSem);
+            RTThreadSleep(2);
+            RTSemEventDestroy(pReq->EventSem);
+        }
+        /* give them a chance to respond before we free the request memory. */
+        RTThreadSleep(32);
+    }
+
+    /*
+     * Now all queued VCPU requests (again, there shouldn't be any).
+     */
+    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
+    {
+        PUVMCPU pUVCpu = &pUVM->aCpus[idCpu];
+
+        for (unsigned i = 0; i < 10; i++)
+        {
+            PVMREQ pReqHead = ASMAtomicXchgPtrT(&pUVCpu->vm.s.pReqs, NULL, PVMREQ);
+            AssertMsg(!pReqHead, ("This isn't supposed to happen! VMR3Destroy caller has to serialize this.\n"));
+            if (!pReqHead)
+                break;
+            for (PVMREQ pReq = pReqHead; pReq; pReq = pReq->pNext)
+            {
+                ASMAtomicUoWriteSize(&pReq->iStatus, VERR_INTERNAL_ERROR);
+                ASMAtomicWriteSize(&pReq->enmState, VMREQSTATE_INVALID);
+                RTSemEventSignal(pReq->EventSem);
+                RTThreadSleep(2);
+                RTSemEventDestroy(pReq->EventSem);
+            }
+            /* give them a chance to respond before we free the request memory. */
+            RTThreadSleep(32);
+        }
+    }
+
+    /*
+     * Make sure the VMMR0.r0 module and whatever else is unloaded.
+     */
+    PDMR3TermUVM(pUVM);
+
+    /*
+     * Terminate the support library if initialized.
+     */
+    if (pUVM->vm.s.pSession)
+    {
+        int rc = SUPR3Term(false /*fForced*/);
+        AssertRC(rc);
+        pUVM->vm.s.pSession = NIL_RTR0PTR;
+    }
+
+    /*
+     * Destroy the MM heap and free the UVM structure.
+     */
+    MMR3TermUVM(pUVM);
+    STAMR3TermUVM(pUVM);
+
+#ifdef LOG_ENABLED
+    RTLogSetCustomPrefixCallback(NULL, NULL, NULL);
+#endif
+    RTTlsFree(pUVM->vm.s.idxTLS);
+
+    ASMAtomicUoWriteU32(&pUVM->u32Magic, UINT32_MAX);
+    RTMemPageFree(pUVM, RT_OFFSETOF(UVM, aCpus[pUVM->cCpus]));
+
+    RTLogFlush(NULL);
+}
+
+
+/**
+ * Enumerates the VMs in this process.
+ *
+ * @returns Pointer to the next VM.
+ * @returns NULL when no more VMs.
+ * @param   pVMPrev     The previous VM
+ *                      Use NULL to start the enumeration.
+ */
+VMMR3DECL(PVM) VMR3EnumVMs(PVM pVMPrev)
+{
+    /*
+     * This is quick and dirty. It has issues with VM being
+     * destroyed during the enumeration.
+     */
+    PUVM pNext;
+    if (pVMPrev)
+        pNext = pVMPrev->pUVM->pNext;
+    else
+        pNext = g_pUVMsHead;
+    return pNext ? pNext->pVM : NULL;
+}
+
+
+/**
+ * Registers an at VM destruction callback.
+ *
+ * @returns VBox status code.
+ * @param   pfnAtDtor       Pointer to callback.
+ * @param   pvUser          User argument.
+ */
+VMMR3DECL(int) VMR3AtDtorRegister(PFNVMATDTOR pfnAtDtor, void *pvUser)
+{
+    /*
+     * Check if already registered.
+     */
+    VM_ATDTOR_LOCK();
+    PVMATDTOR   pCur = g_pVMAtDtorHead;
+    while (pCur)
+    {
+        if (pfnAtDtor == pCur->pfnAtDtor)
+        {
+            VM_ATDTOR_UNLOCK();
+            AssertMsgFailed(("Already registered at destruction callback %p!\n", pfnAtDtor));
+            return VERR_INVALID_PARAMETER;
+        }
+
+        /* next */
+        pCur = pCur->pNext;
+    }
+    VM_ATDTOR_UNLOCK();
+
+    /*
+     * Allocate new entry.
+     */
+    PVMATDTOR   pVMAtDtor = (PVMATDTOR)RTMemAlloc(sizeof(*pVMAtDtor));
+    if (!pVMAtDtor)
+        return VERR_NO_MEMORY;
+
+    VM_ATDTOR_LOCK();
+    pVMAtDtor->pfnAtDtor = pfnAtDtor;
+    pVMAtDtor->pvUser    = pvUser;
+    pVMAtDtor->pNext     = g_pVMAtDtorHead;
+    g_pVMAtDtorHead      = pVMAtDtor;
+    VM_ATDTOR_UNLOCK();
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters an at VM destruction callback.
+ *
+ * @returns VBox status code.
+ * @param   pfnAtDtor       Pointer to callback.
+ */
+VMMR3DECL(int) VMR3AtDtorDeregister(PFNVMATDTOR pfnAtDtor)
+{
+    /*
+     * Find it, unlink it and free it.
+     */
+    VM_ATDTOR_LOCK();
+    PVMATDTOR   pPrev = NULL;
+    PVMATDTOR   pCur = g_pVMAtDtorHead;
+    while (pCur)
+    {
+        if (pfnAtDtor == pCur->pfnAtDtor)
+        {
+            if (pPrev)
+                pPrev->pNext = pCur->pNext;
+            else
+                g_pVMAtDtorHead = pCur->pNext;
+            pCur->pNext = NULL;
+            VM_ATDTOR_UNLOCK();
+
+            RTMemFree(pCur);
+            return VINF_SUCCESS;
+        }
+
+        /* next */
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    VM_ATDTOR_UNLOCK();
+
+    return VERR_INVALID_PARAMETER;
+}
+
+
+/**
+ * Walks the list of at VM destructor callbacks.
+ * @param   pVM     The VM which is about to be destroyed.
+ */
+static void vmR3AtDtor(PVM pVM)
+{
+    /*
+     * Find it, unlink it and free it.
+     */
+    VM_ATDTOR_LOCK();
+    for (PVMATDTOR pCur = g_pVMAtDtorHead; pCur; pCur = pCur->pNext)
+        pCur->pfnAtDtor(pVM, pCur->pvUser);
+    VM_ATDTOR_UNLOCK();
+}
+
+
+/**
+ * Worker which checks integrity of some internal structures.
+ * This is yet another attempt to track down that AVL tree crash.
+ */
+static void vmR3CheckIntegrity(PVM pVM)
+{
+#ifdef VBOX_STRICT
+    int rc = PGMR3CheckIntegrity(pVM);
+    AssertReleaseRC(rc);
+#endif
+}
+
+
+/**
+ * EMT rendezvous worker for VMR3Reset.
+ *
+ * This is called by the emulation threads as a response to the reset request
+ * issued by VMR3Reset().
+ *
+ * @returns VERR_VM_INVALID_VM_STATE, VINF_EM_RESET or VINF_EM_SUSPEND. (This
+ *          is a strict return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3Reset(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first EMT will try change the state to resetting.  If this fails,
+     * we won't get called for the other EMTs.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMR3Reset", 3,
+                                 VMSTATE_RESETTING,     VMSTATE_RUNNING,
+                                 VMSTATE_RESETTING,     VMSTATE_SUSPENDED,
+                                 VMSTATE_RESETTING_LS,  VMSTATE_RUNNING_LS);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+
+    /*
+     * Check the state.
+     */
+    VMSTATE enmVMState = VMR3GetState(pVM);
+    AssertLogRelMsgReturn(   enmVMState == VMSTATE_RESETTING
+                          || enmVMState == VMSTATE_RESETTING_LS,
+                          ("%s\n", VMR3GetStateName(enmVMState)),
+                          VERR_INTERNAL_ERROR_4);
+
+    /*
+     * EMT(0) does the full cleanup *after* all the other EMTs has been
+     * thru here and been told to enter the EMSTATE_WAIT_SIPI state.
+     *
+     * Because there are per-cpu reset routines and order may/is important,
+     * the following sequence looks a bit ugly...
+     */
+    if (pVCpu->idCpu == 0)
+        vmR3CheckIntegrity(pVM);
+
+    /* Reset the VCpu state. */
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED);
+
+    /* Clear all pending forced actions. */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_ALL_MASK & ~VMCPU_FF_REQUEST);
+
+    /*
+     * Reset the VM components.
+     */
+    if (pVCpu->idCpu == 0)
+    {
+        PATMR3Reset(pVM);
+        CSAMR3Reset(pVM);
+        PGMR3Reset(pVM);                    /* We clear VM RAM in PGMR3Reset. It's vital PDMR3Reset is executed
+                                             * _afterwards_. E.g. ACPI sets up RAM tables during init/reset. */
+/** @todo PGMR3Reset should be called after PDMR3Reset really, because we'll trash OS <-> hardware
+ * communication structures residing in RAM when done in the other order.  I.e. the device must be
+ * quiesced first, then we clear the memory and plan tables. Probably have to make these things
+ * explicit in some way, some memory setup pass or something.
+ * (Example: DevAHCI may assert if memory is zeroed before it has read the FIS.)
+ *
+ * @bugref{4467}
+ */
+        MMR3Reset(pVM);
+        PDMR3Reset(pVM);
+        SELMR3Reset(pVM);
+        TRPMR3Reset(pVM);
+        REMR3Reset(pVM);
+        IOMR3Reset(pVM);
+        CPUMR3Reset(pVM);
+    }
+    CPUMR3ResetCpu(pVCpu);
+    if (pVCpu->idCpu == 0)
+    {
+        TMR3Reset(pVM);
+        EMR3Reset(pVM);
+        HWACCMR3Reset(pVM);                 /* This must come *after* PATM, CSAM, CPUM, SELM and TRPM. */
+
+#ifdef LOG_ENABLED
+        /*
+         * Debug logging.
+         */
+        RTLogPrintf("\n\nThe VM was reset:\n");
+        DBGFR3Info(pVM, "cpum", "verbose", NULL);
+#endif
+
+        /*
+         * Since EMT(0) is the last to go thru here, it will advance the state.
+         * When a live save is active, we will move on to SuspendingLS but
+         * leave it for VMR3Reset to do the actual suspending due to deadlock risks.
+         */
+        PUVM pUVM = pVM->pUVM;
+        RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+        enmVMState = pVM->enmVMState;
+        if (enmVMState == VMSTATE_RESETTING)
+        {
+            if (pUVM->vm.s.enmPrevVMState == VMSTATE_SUSPENDED)
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDED, VMSTATE_RESETTING);
+            else
+                vmR3SetStateLocked(pVM, pUVM, VMSTATE_RUNNING,   VMSTATE_RESETTING);
+        }
+        else
+            vmR3SetStateLocked(pVM, pUVM, VMSTATE_SUSPENDING_LS, VMSTATE_RESETTING_LS);
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+
+        vmR3CheckIntegrity(pVM);
+
+        /*
+         * Do the suspend bit as well.
+         * It only requires some EMT(0) work at present.
+         */
+        if (enmVMState != VMSTATE_RESETTING)
+        {
+            vmR3SuspendDoWork(pVM);
+            vmR3SetState(pVM, VMSTATE_SUSPENDED_LS, VMSTATE_SUSPENDING_LS);
+        }
+    }
+
+    return enmVMState == VMSTATE_RESETTING
+         ? VINF_EM_RESET
+         : VINF_EM_SUSPEND; /** @todo VINF_EM_SUSPEND has lower priority than VINF_EM_RESET, so fix races. Perhaps add a new code for this combined case. */
+}
+
+
+/**
+ * Reset the current VM.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM to reset.
+ */
+VMMR3DECL(int) VMR3Reset(PVM pVM)
+{
+    LogFlow(("VMR3Reset:\n"));
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Gather all the EMTs to make sure there are no races before
+     * changing the VM state.
+     */
+    int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3Reset, NULL);
+    LogFlow(("VMR3Reset: returns %Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Gets the current VM state.
+ *
+ * @returns The current VM state.
+ * @param   pVM             VM handle.
+ * @thread  Any
+ */
+VMMR3DECL(VMSTATE) VMR3GetState(PVM pVM)
+{
+    return pVM->enmVMState;
+}
+
+
+/**
+ * Gets the state name string for a VM state.
+ *
+ * @returns Pointer to the state name. (readonly)
+ * @param   enmState        The state.
+ */
+VMMR3DECL(const char *) VMR3GetStateName(VMSTATE enmState)
+{
+    switch (enmState)
+    {
+        case VMSTATE_CREATING:          return "CREATING";
+        case VMSTATE_CREATED:           return "CREATED";
+        case VMSTATE_LOADING:           return "LOADING";
+        case VMSTATE_POWERING_ON:       return "POWERING_ON";
+        case VMSTATE_RESUMING:          return "RESUMING";
+        case VMSTATE_RUNNING:           return "RUNNING";
+        case VMSTATE_RUNNING_LS:        return "RUNNING_LS";
+        case VMSTATE_RUNNING_FT:        return "RUNNING_FT";
+        case VMSTATE_RESETTING:         return "RESETTING";
+        case VMSTATE_RESETTING_LS:      return "RESETTING_LS";
+        case VMSTATE_SUSPENDED:         return "SUSPENDED";
+        case VMSTATE_SUSPENDED_LS:      return "SUSPENDED_LS";
+        case VMSTATE_SUSPENDED_EXT_LS:  return "SUSPENDED_EXT_LS";
+        case VMSTATE_SUSPENDING:        return "SUSPENDING";
+        case VMSTATE_SUSPENDING_LS:     return "SUSPENDING_LS";
+        case VMSTATE_SUSPENDING_EXT_LS: return "SUSPENDING_EXT_LS";
+        case VMSTATE_SAVING:            return "SAVING";
+        case VMSTATE_DEBUGGING:         return "DEBUGGING";
+        case VMSTATE_DEBUGGING_LS:      return "DEBUGGING_LS";
+        case VMSTATE_POWERING_OFF:      return "POWERING_OFF";
+        case VMSTATE_POWERING_OFF_LS:   return "POWERING_OFF_LS";
+        case VMSTATE_FATAL_ERROR:       return "FATAL_ERROR";
+        case VMSTATE_FATAL_ERROR_LS:    return "FATAL_ERROR_LS";
+        case VMSTATE_GURU_MEDITATION:   return "GURU_MEDITATION";
+        case VMSTATE_GURU_MEDITATION_LS:return "GURU_MEDITATION_LS";
+        case VMSTATE_LOAD_FAILURE:      return "LOAD_FAILURE";
+        case VMSTATE_OFF:               return "OFF";
+        case VMSTATE_OFF_LS:            return "OFF_LS";
+        case VMSTATE_DESTROYING:        return "DESTROYING";
+        case VMSTATE_TERMINATED:        return "TERMINATED";
+
+        default:
+            AssertMsgFailed(("Unknown state %d\n", enmState));
+            return "Unknown!\n";
+    }
+}
+
+
+/**
+ * Validates the state transition in strict builds.
+ *
+ * @returns true if valid, false if not.
+ *
+ * @param   enmStateOld         The old (current) state.
+ * @param   enmStateNew         The proposed new state.
+ *
+ * @remarks The reference for this is found in doc/vp/VMM.vpp, the VMSTATE
+ *          diagram (under State Machine Diagram).
+ */
+static bool vmR3ValidateStateTransition(VMSTATE enmStateOld, VMSTATE enmStateNew)
+{
+#ifdef VBOX_STRICT
+    switch (enmStateOld)
+    {
+        case VMSTATE_CREATING:
+            AssertMsgReturn(enmStateNew == VMSTATE_CREATED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_CREATED:
+            AssertMsgReturn(   enmStateNew == VMSTATE_LOADING
+                            || enmStateNew == VMSTATE_POWERING_ON
+                            || enmStateNew == VMSTATE_POWERING_OFF
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_LOADING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
+                            || enmStateNew == VMSTATE_LOAD_FAILURE
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_POWERING_ON:
+            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
+                            /*|| enmStateNew == VMSTATE_FATAL_ERROR ?*/
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RESUMING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
+                            /*|| enmStateNew == VMSTATE_FATAL_ERROR ?*/
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RUNNING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
+                            || enmStateNew == VMSTATE_SUSPENDING
+                            || enmStateNew == VMSTATE_RESETTING
+                            || enmStateNew == VMSTATE_RUNNING_LS
+                            || enmStateNew == VMSTATE_RUNNING_FT
+                            || enmStateNew == VMSTATE_DEBUGGING
+                            || enmStateNew == VMSTATE_FATAL_ERROR
+                            || enmStateNew == VMSTATE_GURU_MEDITATION
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RUNNING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF_LS
+                            || enmStateNew == VMSTATE_SUSPENDING_LS
+                            || enmStateNew == VMSTATE_SUSPENDING_EXT_LS
+                            || enmStateNew == VMSTATE_RESETTING_LS
+                            || enmStateNew == VMSTATE_RUNNING
+                            || enmStateNew == VMSTATE_DEBUGGING_LS
+                            || enmStateNew == VMSTATE_FATAL_ERROR_LS
+                            || enmStateNew == VMSTATE_GURU_MEDITATION_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RUNNING_FT:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
+                            || enmStateNew == VMSTATE_FATAL_ERROR
+                            || enmStateNew == VMSTATE_GURU_MEDITATION
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RESETTING:
+            AssertMsgReturn(enmStateNew == VMSTATE_RUNNING, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_RESETTING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDING:
+            AssertMsgReturn(enmStateNew == VMSTATE_SUSPENDED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING
+                            || enmStateNew == VMSTATE_SUSPENDED_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDING_EXT_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDING
+                            || enmStateNew == VMSTATE_SUSPENDED_EXT_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDED:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
+                            || enmStateNew == VMSTATE_SAVING
+                            || enmStateNew == VMSTATE_RESETTING
+                            || enmStateNew == VMSTATE_RESUMING
+                            || enmStateNew == VMSTATE_LOADING
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDED_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
+                            || enmStateNew == VMSTATE_SAVING
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SUSPENDED_EXT_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_SUSPENDED
+                            || enmStateNew == VMSTATE_SAVING
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_SAVING:
+            AssertMsgReturn(enmStateNew == VMSTATE_SUSPENDED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_DEBUGGING:
+            AssertMsgReturn(   enmStateNew == VMSTATE_RUNNING
+                            || enmStateNew == VMSTATE_POWERING_OFF
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_DEBUGGING_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_DEBUGGING
+                            || enmStateNew == VMSTATE_RUNNING_LS
+                            || enmStateNew == VMSTATE_POWERING_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_POWERING_OFF:
+            AssertMsgReturn(enmStateNew == VMSTATE_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_POWERING_OFF_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_POWERING_OFF
+                            || enmStateNew == VMSTATE_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_OFF:
+            AssertMsgReturn(enmStateNew == VMSTATE_DESTROYING, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_OFF_LS:
+            AssertMsgReturn(enmStateNew == VMSTATE_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_FATAL_ERROR:
+            AssertMsgReturn(enmStateNew == VMSTATE_POWERING_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_FATAL_ERROR_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_FATAL_ERROR
+                            || enmStateNew == VMSTATE_POWERING_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_GURU_MEDITATION:
+            AssertMsgReturn(   enmStateNew == VMSTATE_DEBUGGING
+                            || enmStateNew == VMSTATE_POWERING_OFF
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_GURU_MEDITATION_LS:
+            AssertMsgReturn(   enmStateNew == VMSTATE_GURU_MEDITATION
+                            || enmStateNew == VMSTATE_DEBUGGING_LS
+                            || enmStateNew == VMSTATE_POWERING_OFF_LS
+                            , ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_LOAD_FAILURE:
+            AssertMsgReturn(enmStateNew == VMSTATE_POWERING_OFF, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_DESTROYING:
+            AssertMsgReturn(enmStateNew == VMSTATE_TERMINATED, ("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+
+        case VMSTATE_TERMINATED:
+        default:
+            AssertMsgFailedReturn(("%s -> %s\n", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)), false);
+            break;
+    }
+#endif /* VBOX_STRICT */
+    return true;
+}
+
+
+/**
+ * Does the state change callouts.
+ *
+ * The caller owns the AtStateCritSect.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pUVM                The UVM handle.
+ * @param   enmStateNew         The New state.
+ * @param   enmStateOld         The old state.
+ */
+static void vmR3DoAtState(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
+{
+    LogRel(("Changing the VM state from '%s' to '%s'.\n", VMR3GetStateName(enmStateOld),  VMR3GetStateName(enmStateNew)));
+
+    for (PVMATSTATE pCur = pUVM->vm.s.pAtState; pCur; pCur = pCur->pNext)
+    {
+        pCur->pfnAtState(pVM, enmStateNew, enmStateOld, pCur->pvUser);
+        if (    enmStateNew     != VMSTATE_DESTROYING
+            &&  pVM->enmVMState == VMSTATE_DESTROYING)
+            break;
+        AssertMsg(pVM->enmVMState == enmStateNew,
+                  ("You are not allowed to change the state while in the change callback, except "
+                   "from destroying the VM. There are restrictions in the way the state changes "
+                   "are propagated up to the EM execution loop and it makes the program flow very "
+                   "difficult to follow. (%s, expected %s, old %s)\n",
+                   VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateNew),
+                   VMR3GetStateName(enmStateOld)));
+    }
+}
+
+
+/**
+ * Sets the current VM state, with the AtStatCritSect already entered.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pUVM                The UVM handle.
+ * @param   enmStateNew         The new state.
+ * @param   enmStateOld         The old state.
+ */
+static void vmR3SetStateLocked(PVM pVM, PUVM pUVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
+{
+    vmR3ValidateStateTransition(enmStateOld, enmStateNew);
+
+    AssertMsg(pVM->enmVMState == enmStateOld,
+              ("%s != %s\n", VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateOld)));
+    pUVM->vm.s.enmPrevVMState = enmStateOld;
+    pVM->enmVMState           = enmStateNew;
+    VM_FF_CLEAR(pVM, VM_FF_CHECK_VM_STATE);
+
+    vmR3DoAtState(pVM, pUVM, enmStateNew, enmStateOld);
+}
+
+
+/**
+ * Sets the current VM state.
+ *
+ * @param   pVM             VM handle.
+ * @param   enmStateNew     The new state.
+ * @param   enmStateOld     The old state (for asserting only).
+ */
+static void vmR3SetState(PVM pVM, VMSTATE enmStateNew, VMSTATE enmStateOld)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    AssertMsg(pVM->enmVMState == enmStateOld,
+              ("%s != %s\n", VMR3GetStateName(pVM->enmVMState), VMR3GetStateName(enmStateOld)));
+    vmR3SetStateLocked(pVM, pUVM, enmStateNew, pVM->enmVMState);
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+}
+
+
+/**
+ * Tries to perform a state transition.
+ *
+ * @returns The 1-based ordinal of the succeeding transition.
+ *          VERR_VM_INVALID_VM_STATE and Assert+LogRel on failure.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pszWho              Who is trying to change it.
+ * @param   cTransitions        The number of transitions in the ellipsis.
+ * @param   ...                 Transition pairs; new, old.
+ */
+static int vmR3TrySetState(PVM pVM, const char *pszWho, unsigned cTransitions, ...)
+{
+    va_list va;
+    VMSTATE enmStateNew = VMSTATE_CREATED;
+    VMSTATE enmStateOld = VMSTATE_CREATED;
+
+#ifdef VBOX_STRICT
+    /*
+     * Validate the input first.
+     */
+    va_start(va, cTransitions);
+    for (unsigned i = 0; i < cTransitions; i++)
+    {
+        enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        vmR3ValidateStateTransition(enmStateOld, enmStateNew);
+    }
+    va_end(va);
+#endif
+
+    /*
+     * Grab the lock and see if any of the proposed transitions works out.
+     */
+    va_start(va, cTransitions);
+    int     rc          = VERR_VM_INVALID_VM_STATE;
+    PUVM    pUVM        = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    VMSTATE enmStateCur = pVM->enmVMState;
+
+    for (unsigned i = 0; i < cTransitions; i++)
+    {
+        enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+        if (enmStateCur == enmStateOld)
+        {
+            vmR3SetStateLocked(pVM, pUVM, enmStateNew, enmStateOld);
+            rc = i + 1;
+            break;
+        }
+    }
+
+    if (RT_FAILURE(rc))
+    {
+        /*
+         * Complain about it.
+         */
+        if (cTransitions == 1)
+        {
+            LogRel(("%s: %s -> %s failed, because the VM state is actually %s\n",
+                    pszWho, VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew), VMR3GetStateName(enmStateCur)));
+            VMSetError(pVM, VERR_VM_INVALID_VM_STATE, RT_SRC_POS,
+                       N_("%s failed because the VM state is %s instead of %s"),
+                       pszWho, VMR3GetStateName(enmStateCur), VMR3GetStateName(enmStateOld));
+            AssertMsgFailed(("%s: %s -> %s failed, because the VM state is actually %s\n",
+                             pszWho, VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew), VMR3GetStateName(enmStateCur)));
+        }
+        else
+        {
+            va_end(va);
+            va_start(va, cTransitions);
+            LogRel(("%s:\n", pszWho));
+            for (unsigned i = 0; i < cTransitions; i++)
+            {
+                enmStateNew = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+                enmStateOld = (VMSTATE)va_arg(va, /*VMSTATE*/int);
+                LogRel(("%s%s -> %s",
+                        i ? ", " : " ", VMR3GetStateName(enmStateOld), VMR3GetStateName(enmStateNew)));
+            }
+            LogRel((" failed, because the VM state is actually %s\n", VMR3GetStateName(enmStateCur)));
+            VMSetError(pVM, VERR_VM_INVALID_VM_STATE, RT_SRC_POS,
+                       N_("%s failed because the current VM state, %s, was not found in the state transition table"),
+                       pszWho, VMR3GetStateName(enmStateCur), VMR3GetStateName(enmStateOld));
+            AssertMsgFailed(("%s - state=%s, see release log for full details. Check the cTransitions passed us.\n",
+                             pszWho, VMR3GetStateName(enmStateCur)));
+        }
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+    va_end(va);
+    Assert(rc > 0 || rc < 0);
+    return rc;
+}
+
+
+/**
+ * Flag a guru meditation ... a hack.
+ *
+ * @param   pVM             The VM handle
+ *
+ * @todo    Rewrite this part. The guru meditation should be flagged
+ *          immediately by the VMM and not by VMEmt.cpp when it's all over.
+ */
+void vmR3SetGuruMeditation(PVM pVM)
+{
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    VMSTATE enmStateCur = pVM->enmVMState;
+    if (enmStateCur == VMSTATE_RUNNING)
+        vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION, VMSTATE_RUNNING);
+    else if (enmStateCur == VMSTATE_RUNNING_LS)
+    {
+        vmR3SetStateLocked(pVM, pUVM, VMSTATE_GURU_MEDITATION_LS, VMSTATE_RUNNING_LS);
+        SSMR3Cancel(pVM);
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+}
+
+
+/**
+ * Called by vmR3EmulationThreadWithId just before the VM structure is freed.
+ *
+ * @param   pVM             The VM handle.
+ */
+void vmR3SetTerminated(PVM pVM)
+{
+    vmR3SetState(pVM, VMSTATE_TERMINATED, VMSTATE_DESTROYING);
+}
+
+
+/**
+ * Checks if the VM was teleported and hasn't been fully resumed yet.
+ *
+ * This applies to both sides of the teleportation since we may leave a working
+ * clone behind and the user is allowed to resume this...
+ *
+ * @returns true / false.
+ * @param   pVM                 The VM handle.
+ * @thread  Any thread.
+ */
+VMMR3DECL(bool) VMR3TeleportedAndNotFullyResumedYet(PVM pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, false);
+    return pVM->vm.s.fTeleportedAndNotFullyResumedYet;
+}
+
+
+/**
+ * Registers a VM state change callback.
+ *
+ * You are not allowed to call any function which changes the VM state from a
+ * state callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pfnAtState      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMR3AtStateRegister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser)
+{
+    LogFlow(("VMR3AtStateRegister: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtState, VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Allocate a new record.
+     */
+    PUVM pUVM = pVM->pUVM;
+    PVMATSTATE pNew = (PVMATSTATE)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
+    if (!pNew)
+        return VERR_NO_MEMORY;
+
+    /* fill */
+    pNew->pfnAtState = pfnAtState;
+    pNew->pvUser     = pvUser;
+
+    /* insert */
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+    pNew->pNext      = *pUVM->vm.s.ppAtStateNext;
+    *pUVM->vm.s.ppAtStateNext = pNew;
+    pUVM->vm.s.ppAtStateNext = &pNew->pNext;
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a VM state change callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   pfnAtState      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMR3AtStateDeregister(PVM pVM, PFNVMATSTATE pfnAtState, void *pvUser)
+{
+    LogFlow(("VMR3AtStateDeregister: pfnAtState=%p pvUser=%p\n", pfnAtState, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtState, VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtStateCritSect);
+
+    /*
+     * Search the list for the entry.
+     */
+    PVMATSTATE pPrev = NULL;
+    PVMATSTATE pCur = pUVM->vm.s.pAtState;
+    while (     pCur
+           &&   (   pCur->pfnAtState != pfnAtState
+                 || pCur->pvUser != pvUser))
+    {
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    if (!pCur)
+    {
+        AssertMsgFailed(("pfnAtState=%p was not found\n", pfnAtState));
+        RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+        return VERR_FILE_NOT_FOUND;
+    }
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+    {
+        pPrev->pNext = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtStateNext = &pPrev->pNext;
+    }
+    else
+    {
+        pUVM->vm.s.pAtState = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtStateNext = &pUVM->vm.s.pAtState;
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtStateCritSect);
+
+    /*
+     * Free it.
+     */
+    pCur->pfnAtState = NULL;
+    pCur->pNext = NULL;
+    MMR3HeapFree(pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Registers a VM error callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pfnAtError      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int)   VMR3AtErrorRegister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser)
+{
+    return VMR3AtErrorRegisterU(pVM->pUVM, pfnAtError, pvUser);
+}
+
+
+/**
+ * Registers a VM error callback.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            The VM handle.
+ * @param   pfnAtError      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int)   VMR3AtErrorRegisterU(PUVM pUVM, PFNVMATERROR pfnAtError, void *pvUser)
+{
+    LogFlow(("VMR3AtErrorRegister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtError, VERR_INVALID_PARAMETER);
+    UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Allocate a new record.
+     */
+    PVMATERROR pNew = (PVMATERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
+    if (!pNew)
+        return VERR_NO_MEMORY;
+
+    /* fill */
+    pNew->pfnAtError = pfnAtError;
+    pNew->pvUser     = pvUser;
+
+    /* insert */
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    pNew->pNext      = *pUVM->vm.s.ppAtErrorNext;
+    *pUVM->vm.s.ppAtErrorNext = pNew;
+    pUVM->vm.s.ppAtErrorNext = &pNew->pNext;
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a VM error callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The VM handle.
+ * @param   pfnAtError      Pointer to callback.
+ * @param   pvUser          User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMR3AtErrorDeregister(PVM pVM, PFNVMATERROR pfnAtError, void *pvUser)
+{
+    LogFlow(("VMR3AtErrorDeregister: pfnAtError=%p pvUser=%p\n", pfnAtError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtError, VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Search the list for the entry.
+     */
+    PVMATERROR pPrev = NULL;
+    PVMATERROR pCur = pUVM->vm.s.pAtError;
+    while (     pCur
+           &&   (   pCur->pfnAtError != pfnAtError
+                 || pCur->pvUser != pvUser))
+    {
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    if (!pCur)
+    {
+        AssertMsgFailed(("pfnAtError=%p was not found\n", pfnAtError));
+        RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+        return VERR_FILE_NOT_FOUND;
+    }
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+    {
+        pPrev->pNext = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtErrorNext = &pPrev->pNext;
+    }
+    else
+    {
+        pUVM->vm.s.pAtError = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtErrorNext = &pUVM->vm.s.pAtError;
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Free it.
+     */
+    pCur->pfnAtError = NULL;
+    pCur->pNext = NULL;
+    MMR3HeapFree(pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Ellipsis to va_list wrapper for calling pfnAtError.
+ */
+static void vmR3SetErrorWorkerDoCall(PVM pVM, PVMATERROR pCur, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    pCur->pfnAtError(pVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va);
+    va_end(va);
+}
+
+
+/**
+ * This is a worker function for GC and Ring-0 calls to VMSetError and VMSetErrorV.
+ * The message is found in VMINT.
+ *
+ * @param   pVM             The VM handle.
+ * @thread  EMT.
+ */
+VMMR3DECL(void) VMR3SetErrorWorker(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    AssertReleaseMsgFailed(("And we have a winner! You get to implement Ring-0 and GC VMSetErrorV! Contracts!\n"));
+
+    /*
+     * Unpack the error (if we managed to format one).
+     */
+    PVMERROR pErr = pVM->vm.s.pErrorR3;
+    const char *pszFile = NULL;
+    const char *pszFunction = NULL;
+    uint32_t    iLine = 0;
+    const char *pszMessage;
+    int32_t     rc = VERR_MM_HYPER_NO_MEMORY;
+    if (pErr)
+    {
+        AssertCompile(sizeof(const char) == sizeof(uint8_t));
+        if (pErr->offFile)
+            pszFile = (const char *)pErr + pErr->offFile;
+        iLine = pErr->iLine;
+        if (pErr->offFunction)
+            pszFunction = (const char *)pErr + pErr->offFunction;
+        if (pErr->offMessage)
+            pszMessage = (const char *)pErr + pErr->offMessage;
+        else
+            pszMessage = "No message!";
+    }
+    else
+        pszMessage = "No message! (Failed to allocate memory to put the error message in!)";
+
+    /*
+     * Call the at error callbacks.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    ASMAtomicIncU32(&pUVM->vm.s.cRuntimeErrors);
+    for (PVMATERROR pCur = pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
+        vmR3SetErrorWorkerDoCall(pVM, pCur, rc, RT_SRC_POS_ARGS, "%s", pszMessage);
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+}
+
+
+/**
+ * Gets the number of errors raised via VMSetError.
+ *
+ * This can be used avoid double error messages.
+ *
+ * @returns The error count.
+ * @param   pVM             The VM handle.
+ */
+VMMR3DECL(uint32_t) VMR3GetErrorCount(PVM pVM)
+{
+    return pVM->pUVM->vm.s.cErrors;
+}
+
+
+/**
+ * Creation time wrapper for vmR3SetErrorUV.
+ *
+ * @returns rc.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   rc              The VBox status code.
+ * @param   RT_SRC_POS_DECL The source position of this error.
+ * @param   pszFormat       Format string.
+ * @param   ...             The arguments.
+ * @thread  Any thread.
+ */
+static int vmR3SetErrorU(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    vmR3SetErrorUV(pUVM, rc, pszFile, iLine, pszFunction, pszFormat, &va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Worker which calls everyone listening to the VM error messages.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   rc              The VBox status code.
+ * @param   RT_SRC_POS_DECL The source position of this error.
+ * @param   pszFormat       Format string.
+ * @param   pArgs           Pointer to the format arguments.
+ * @thread  EMT
+ */
+DECLCALLBACK(void) vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *pArgs)
+{
+    /*
+     * Log the error.
+     */
+    va_list va3;
+    va_copy(va3, *pArgs);
+    RTLogRelPrintf("VMSetError: %s(%d) %s; rc=%Rrc\n"
+                   "VMSetError: %N\n",
+                   pszFile, iLine, pszFunction, rc,
+                   pszFormat, &va3);
+    va_end(va3);
+
+#ifdef LOG_ENABLED
+    va_copy(va3, *pArgs);
+    RTLogPrintf("VMSetError: %s(%d) %s; rc=%Rrc\n"
+                "%N\n",
+                pszFile, iLine, pszFunction, rc,
+                pszFormat, &va3);
+    va_end(va3);
+#endif
+
+    /*
+     * Make a copy of the message.
+     */
+    if (pUVM->pVM)
+        vmSetErrorCopy(pUVM->pVM, rc, RT_SRC_POS_ARGS, pszFormat, *pArgs);
+
+    /*
+     * Call the at error callbacks.
+     */
+    bool fCalledSomeone = false;
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    ASMAtomicIncU32(&pUVM->vm.s.cErrors);
+    for (PVMATERROR pCur = pUVM->vm.s.pAtError; pCur; pCur = pCur->pNext)
+    {
+        va_list va2;
+        va_copy(va2, *pArgs);
+        pCur->pfnAtError(pUVM->pVM, pCur->pvUser, rc, RT_SRC_POS_ARGS, pszFormat, va2);
+        va_end(va2);
+        fCalledSomeone = true;
+    }
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+}
+
+
+/**
+ * Registers a VM runtime error callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pfnAtRuntimeError   Pointer to callback.
+ * @param   pvUser              User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int)   VMR3AtRuntimeErrorRegister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
+{
+    LogFlow(("VMR3AtRuntimeErrorRegister: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtRuntimeError, VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    /*
+     * Allocate a new record.
+     */
+    PUVM pUVM = pVM->pUVM;
+    PVMATRUNTIMEERROR pNew = (PVMATRUNTIMEERROR)MMR3HeapAllocU(pUVM, MM_TAG_VM, sizeof(*pNew));
+    if (!pNew)
+        return VERR_NO_MEMORY;
+
+    /* fill */
+    pNew->pfnAtRuntimeError = pfnAtRuntimeError;
+    pNew->pvUser            = pvUser;
+
+    /* insert */
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    pNew->pNext             = *pUVM->vm.s.ppAtRuntimeErrorNext;
+    *pUVM->vm.s.ppAtRuntimeErrorNext = pNew;
+    pUVM->vm.s.ppAtRuntimeErrorNext = &pNew->pNext;
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregisters a VM runtime error callback.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   pfnAtRuntimeError   Pointer to callback.
+ * @param   pvUser              User argument.
+ * @thread  Any.
+ */
+VMMR3DECL(int)   VMR3AtRuntimeErrorDeregister(PVM pVM, PFNVMATRUNTIMEERROR pfnAtRuntimeError, void *pvUser)
+{
+    LogFlow(("VMR3AtRuntimeErrorDeregister: pfnAtRuntimeError=%p pvUser=%p\n", pfnAtRuntimeError, pvUser));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnAtRuntimeError, VERR_INVALID_PARAMETER);
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Search the list for the entry.
+     */
+    PVMATRUNTIMEERROR pPrev = NULL;
+    PVMATRUNTIMEERROR pCur = pUVM->vm.s.pAtRuntimeError;
+    while (     pCur
+           &&   (   pCur->pfnAtRuntimeError != pfnAtRuntimeError
+                 || pCur->pvUser != pvUser))
+    {
+        pPrev = pCur;
+        pCur = pCur->pNext;
+    }
+    if (!pCur)
+    {
+        AssertMsgFailed(("pfnAtRuntimeError=%p was not found\n", pfnAtRuntimeError));
+        RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+        return VERR_FILE_NOT_FOUND;
+    }
+
+    /*
+     * Unlink it.
+     */
+    if (pPrev)
+    {
+        pPrev->pNext = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtRuntimeErrorNext = &pPrev->pNext;
+    }
+    else
+    {
+        pUVM->vm.s.pAtRuntimeError = pCur->pNext;
+        if (!pCur->pNext)
+            pUVM->vm.s.ppAtRuntimeErrorNext = &pUVM->vm.s.pAtRuntimeError;
+    }
+
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    /*
+     * Free it.
+     */
+    pCur->pfnAtRuntimeError = NULL;
+    pCur->pNext = NULL;
+    MMR3HeapFree(pCur);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * EMT rendezvous worker that vmR3SetRuntimeErrorCommon uses to safely change
+ * the state to FatalError(LS).
+ *
+ * @returns VERR_VM_INVALID_VM_STATE or VINF_EM_SUSPEND.  (This is a strict
+ *          return code, see FNVMMEMTRENDEZVOUS.)
+ *
+ * @param   pVM             The VM handle.
+ * @param   pVCpu           The VMCPU handle of the EMT.
+ * @param   pvUser          Ignored.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3SetRuntimeErrorChangeState(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    NOREF(pVCpu);
+    Assert(!pvUser); NOREF(pvUser);
+
+    /*
+     * The first EMT thru here changes the state.
+     */
+    if (pVCpu->idCpu == pVM->cCpus - 1)
+    {
+        int rc = vmR3TrySetState(pVM, "VMSetRuntimeError", 2,
+                                 VMSTATE_FATAL_ERROR,    VMSTATE_RUNNING,
+                                 VMSTATE_FATAL_ERROR_LS, VMSTATE_RUNNING_LS);
+        if (RT_FAILURE(rc))
+            return rc;
+        if (rc == 2)
+            SSMR3Cancel(pVM);
+
+        VM_FF_SET(pVM, VM_FF_CHECK_VM_STATE);
+    }
+
+    /* This'll make sure we get out of whereever we are (e.g. REM). */
+    return VINF_EM_SUSPEND;
+}
+
+
+/**
+ * Worker for VMR3SetRuntimeErrorWorker and vmR3SetRuntimeErrorV.
+ *
+ * This does the common parts after the error has been saved / retrieved.
+ *
+ * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
+ *
+ * @param   pVM             The VM handle.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Format string.
+ * @param   pVa             Pointer to the format arguments.
+ */
+static int vmR3SetRuntimeErrorCommon(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa)
+{
+    LogRel(("VM: Raising runtime error '%s' (fFlags=%#x)\n", pszErrorId, fFlags));
+
+    /*
+     * Take actions before the call.
+     */
+    int rc;
+    if (fFlags & VMSETRTERR_FLAGS_FATAL)
+        rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING | VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR,
+                                vmR3SetRuntimeErrorChangeState, NULL);
+    else if (fFlags & VMSETRTERR_FLAGS_SUSPEND)
+        rc = VMR3Suspend(pVM);
+    else
+        rc = VINF_SUCCESS;
+
+    /*
+     * Do the callback round.
+     */
+    PUVM pUVM = pVM->pUVM;
+    RTCritSectEnter(&pUVM->vm.s.AtErrorCritSect);
+    ASMAtomicIncU32(&pUVM->vm.s.cRuntimeErrors);
+    for (PVMATRUNTIMEERROR pCur = pUVM->vm.s.pAtRuntimeError; pCur; pCur = pCur->pNext)
+    {
+        va_list va;
+        va_copy(va, *pVa);
+        pCur->pfnAtRuntimeError(pVM, pCur->pvUser, fFlags, pszErrorId, pszFormat, va);
+        va_end(va);
+    }
+    RTCritSectLeave(&pUVM->vm.s.AtErrorCritSect);
+
+    return rc;
+}
+
+
+/**
+ * Ellipsis to va_list wrapper for calling vmR3SetRuntimeErrorCommon.
+ */
+static int vmR3SetRuntimeErrorCommonF(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = vmR3SetRuntimeErrorCommon(pVM, fFlags, pszErrorId, pszFormat, &va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * This is a worker function for RC and Ring-0 calls to VMSetError and
+ * VMSetErrorV.
+ *
+ * The message is found in VMINT.
+ *
+ * @returns VBox status code, see VMSetRuntimeError.
+ * @param   pVM             The VM handle.
+ * @thread  EMT.
+ */
+VMMR3DECL(int) VMR3SetRuntimeErrorWorker(PVM pVM)
+{
+    VM_ASSERT_EMT(pVM);
+    AssertReleaseMsgFailed(("And we have a winner! You get to implement Ring-0 and GC VMSetRuntimeErrorV! Congrats!\n"));
+
+    /*
+     * Unpack the error (if we managed to format one).
+     */
+    const char     *pszErrorId = "SetRuntimeError";
+    const char     *pszMessage = "No message!";
+    uint32_t        fFlags     = VMSETRTERR_FLAGS_FATAL;
+    PVMRUNTIMEERROR pErr       = pVM->vm.s.pRuntimeErrorR3;
+    if (pErr)
+    {
+        AssertCompile(sizeof(const char) == sizeof(uint8_t));
+        if (pErr->offErrorId)
+            pszErrorId = (const char *)pErr + pErr->offErrorId;
+        if (pErr->offMessage)
+            pszMessage = (const char *)pErr + pErr->offMessage;
+        fFlags = pErr->fFlags;
+    }
+
+    /*
+     * Join cause with vmR3SetRuntimeErrorV.
+     */
+    return vmR3SetRuntimeErrorCommonF(pVM, fFlags, pszErrorId, "%s", pszMessage);
+}
+
+
+/**
+ * Worker for VMSetRuntimeErrorV for doing the job on EMT in ring-3.
+ *
+ * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
+ *
+ * @param   pVM             The VM handle.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszMessage      The error message residing the MM heap.
+ *
+ * @thread  EMT
+ */
+DECLCALLBACK(int) vmR3SetRuntimeError(PVM pVM, uint32_t fFlags, const char *pszErrorId, char *pszMessage)
+{
+#if 0 /** @todo make copy of the error msg. */
+    /*
+     * Make a copy of the message.
+     */
+    va_list va2;
+    va_copy(va2, *pVa);
+    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va2);
+    va_end(va2);
+#endif
+
+    /*
+     * Join paths with VMR3SetRuntimeErrorWorker.
+     */
+    int rc = vmR3SetRuntimeErrorCommonF(pVM, fFlags, pszErrorId, "%s", pszMessage);
+    MMR3HeapFree(pszMessage);
+    return rc;
+}
+
+
+/**
+ * Worker for VMSetRuntimeErrorV for doing the job on EMT in ring-3.
+ *
+ * @returns VBox status code with modifications, see VMSetRuntimeErrorV.
+ *
+ * @param   pVM             The VM handle.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Format string.
+ * @param   pVa             Pointer to the format arguments.
+ *
+ * @thread  EMT
+ */
+DECLCALLBACK(int) vmR3SetRuntimeErrorV(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa)
+{
+    /*
+     * Make a copy of the message.
+     */
+    va_list va2;
+    va_copy(va2, *pVa);
+    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va2);
+    va_end(va2);
+
+    /*
+     * Join paths with VMR3SetRuntimeErrorWorker.
+     */
+    return vmR3SetRuntimeErrorCommon(pVM, fFlags, pszErrorId, pszFormat, pVa);
+}
+
+
+/**
+ * Gets the number of runtime errors raised via VMR3SetRuntimeError.
+ *
+ * This can be used avoid double error messages.
+ *
+ * @returns The runtime error count.
+ * @param   pVM             The VM handle.
+ */
+VMMR3DECL(uint32_t) VMR3GetRuntimeErrorCount(PVM pVM)
+{
+    return pVM->pUVM->vm.s.cRuntimeErrors;
+}
+
+
+/**
+ * Gets the ID virtual of the virtual CPU associated with the calling thread.
+ *
+ * @returns The CPU ID. NIL_VMCPUID if the thread isn't an EMT.
+ *
+ * @param   pVM             The VM handle.
+ */
+VMMR3DECL(RTCPUID) VMR3GetVMCPUId(PVM pVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
+    return pUVCpu
+         ? pUVCpu->idCpu
+         : NIL_VMCPUID;
+}
+
+
+/**
+ * Returns the native handle of the current EMT VMCPU thread.
+ *
+ * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
+ * @param   pVM             The VM handle.
+ * @thread  EMT
+ */
+VMMR3DECL(RTNATIVETHREAD) VMR3GetVMCPUNativeThread(PVM pVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
+
+    if (!pUVCpu)
+        return NIL_RTNATIVETHREAD;
+
+    return pUVCpu->vm.s.NativeThreadEMT;
+}
+
+
+/**
+ * Returns the native handle of the current EMT VMCPU thread.
+ *
+ * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
+ * @param   pVM             The VM handle.
+ * @thread  EMT
+ */
+VMMR3DECL(RTNATIVETHREAD) VMR3GetVMCPUNativeThreadU(PUVM pUVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
+
+    if (!pUVCpu)
+        return NIL_RTNATIVETHREAD;
+
+    return pUVCpu->vm.s.NativeThreadEMT;
+}
+
+
+/**
+ * Returns the handle of the current EMT VMCPU thread.
+ *
+ * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
+ * @param   pVM             The VM handle.
+ * @thread  EMT
+ */
+VMMR3DECL(RTTHREAD) VMR3GetVMCPUThread(PVM pVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pVM->pUVM->vm.s.idxTLS);
+
+    if (!pUVCpu)
+        return NIL_RTTHREAD;
+
+    return pUVCpu->vm.s.ThreadEMT;
+}
+
+
+/**
+ * Returns the handle of the current EMT VMCPU thread.
+ *
+ * @returns Handle if this is an EMT thread; NIL_RTNATIVETHREAD otherwise
+ * @param   pVM             The VM handle.
+ * @thread  EMT
+ */
+VMMR3DECL(RTTHREAD) VMR3GetVMCPUThreadU(PUVM pUVM)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
+
+    if (!pUVCpu)
+        return NIL_RTTHREAD;
+
+    return pUVCpu->vm.s.ThreadEMT;
+}
+
+
+/**
+ * Return the package and core id of a CPU.
+ *
+ * @returns VBOX status code.
+ * @param   pVM              The VM to operate on.
+ * @param   idCpu            Virtual CPU to get the ID from.
+ * @param   pidCpuCore       Where to store the core ID of the virtual CPU.
+ * @param   pidCpuPackage    Where to store the package ID of the virtual CPU.
+ *
+ */
+VMMR3DECL(int) VMR3GetCpuCoreAndPackageIdFromCpuId(PVM pVM, VMCPUID idCpu, uint32_t *pidCpuCore, uint32_t *pidCpuPackage)
+{
+    if (idCpu >= pVM->cCpus)
+        return VERR_INVALID_CPU_ID;
+
+#ifdef VBOX_WITH_MULTI_CORE
+    *pidCpuCore    = idCpu;
+    *pidCpuPackage = 0;
+#else
+    *pidCpuCore    = 0;
+    *pidCpuPackage = idCpu;
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for VMR3HotUnplugCpu.
+ *
+ * @returns VINF_EM_WAIT_SPIP (strict status code).
+ * @param   pVM                 The VM handle.
+ * @param   idCpu               The current CPU.
+ */
+static DECLCALLBACK(int) vmR3HotUnplugCpu(PVM pVM, VMCPUID idCpu)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Reset per CPU resources.
+     *
+     * Actually only needed for VT-x because the CPU seems to be still in some
+     * paged mode and startup fails after a new hot plug event. SVM works fine
+     * even without this.
+     */
+    Log(("vmR3HotUnplugCpu for VCPU %u\n", idCpu));
+    PGMR3ResetUnpluggedCpu(pVM, pVCpu);
+    PDMR3ResetCpu(pVCpu);
+    TRPMR3ResetCpu(pVCpu);
+    CPUMR3ResetCpu(pVCpu);
+    EMR3ResetCpu(pVCpu);
+    HWACCMR3ResetCpu(pVCpu);
+    return VINF_EM_WAIT_SIPI;
+}
+
+
+/**
+ * Hot-unplugs a CPU from the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM to operate on.
+ * @param   idCpu   Virtual CPU to perform the hot unplugging operation on.
+ */
+VMMR3DECL(int) VMR3HotUnplugCpu(PVM pVM, VMCPUID idCpu)
+{
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /** @todo r=bird: Don't destroy the EMT, it'll break VMMR3EmtRendezvous and
+     *        broadcast requests.  Just note down somewhere that the CPU is
+     *        offline and send it to SPIP wait.  Maybe modify VMCPUSTATE and push
+     *        it out of the EM loops when offline. */
+    return VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmR3HotUnplugCpu, 2, pVM, idCpu);
+}
+
+
+/**
+ * Hot-plugs a CPU on the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM to operate on.
+ * @param   idCpu   Virtual CPU to perform the hot plugging operation on.
+ */
+VMMR3DECL(int) VMR3HotPlugCpu(PVM pVM, VMCPUID idCpu)
+{
+    AssertReturn(idCpu < pVM->cCpus, VERR_INVALID_CPU_ID);
+
+    /** @todo r-bird: Just mark it online and make sure it waits on SPIP. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Changes the VMM execution cap.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM to operate on.
+ * @param   ulCpuExecutionCap   New CPU execution cap
+ */
+VMMR3DECL(int) VMR3SetCpuExecutionCap(PVM pVM, unsigned ulCpuExecutionCap)
+{
+    AssertReturn(ulCpuExecutionCap > 0 && ulCpuExecutionCap <= 100, VERR_INVALID_PARAMETER);
+
+    Log(("VMR3SetCpuExecutionCap: new priority = %d\n", ulCpuExecutionCap));
+    /* Note: not called from EMT. */
+    pVM->uCpuExecutionCap = ulCpuExecutionCap;
+    return VINF_SUCCESS;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/VMEmt.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMEmt.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMEmt.cpp	(revision 35333)
@@ -0,0 +1,1278 @@
+/* $Id$ */
+/** @file
+ * VM - Virtual Machine, The Emulation Thread.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include <VBox/tm.h>
+#include <VBox/dbgf.h>
+#include <VBox/em.h>
+#include <VBox/pdmapi.h>
+#include <VBox/rem.h>
+#include <VBox/tm.h>
+#include "VMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+#include <iprt/semaphore.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/time.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+int vmR3EmulationThreadWithId(RTTHREAD ThreadSelf, PUVMCPU pUVCpu, VMCPUID idCpu);
+
+
+/**
+ * The emulation thread main function.
+ *
+ * @returns Thread exit code.
+ * @param   ThreadSelf  The handle to the executing thread.
+ * @param   pvArgs      Pointer to the user mode per-VCpu structure (UVMPCU).
+ */
+DECLCALLBACK(int) vmR3EmulationThread(RTTHREAD ThreadSelf, void *pvArgs)
+{
+    PUVMCPU pUVCpu = (PUVMCPU)pvArgs;
+    return vmR3EmulationThreadWithId(ThreadSelf, pUVCpu, pUVCpu->idCpu);
+}
+
+
+/**
+ * The emulation thread main function, with Virtual CPU ID for debugging.
+ *
+ * @returns Thread exit code.
+ * @param   ThreadSelf  The handle to the executing thread.
+ * @param   pUVCpu      Pointer to the user mode per-VCpu structure.
+ * @param   idCpu       The virtual CPU ID, for backtrace purposes.
+ */
+int vmR3EmulationThreadWithId(RTTHREAD ThreadSelf, PUVMCPU pUVCpu, VMCPUID idCpu)
+{
+    PUVM    pUVM = pUVCpu->pUVM;
+    int     rc;
+
+    AssertReleaseMsg(VALID_PTR(pUVM) && pUVM->u32Magic == UVM_MAGIC,
+                     ("Invalid arguments to the emulation thread!\n"));
+
+    rc = RTTlsSet(pUVM->vm.s.idxTLS, pUVCpu);
+    AssertReleaseMsgRCReturn(rc, ("RTTlsSet %x failed with %Rrc\n", pUVM->vm.s.idxTLS, rc), rc);
+
+    /*
+     * The request loop.
+     */
+    rc = VINF_SUCCESS;
+    Log(("vmR3EmulationThread: Emulation thread starting the days work... Thread=%#x pUVM=%p\n", ThreadSelf, pUVM));
+    VMSTATE enmBefore = VMSTATE_CREATED; /* (only used for logging atm.) */
+    for (;;)
+    {
+        /*
+         * During early init there is no pVM, so make a special path
+         * for that to keep things clearly separate.
+         */
+        if (!pUVM->pVM)
+        {
+            /*
+             * Check for termination first.
+             */
+            if (pUVM->vm.s.fTerminateEMT)
+            {
+                rc = VINF_EM_TERMINATE;
+                break;
+            }
+
+            /*
+             * Only the first VCPU may initialize the VM during early init
+             * and must therefore service all VMCPUID_ANY requests.
+             * See also VMR3Create
+             */
+            if (    pUVM->vm.s.pReqs
+                &&  pUVCpu->idCpu == 0)
+            {
+                /*
+                 * Service execute in any EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, VMCPUID_ANY);
+                Log(("vmR3EmulationThread: Req rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), pUVM->pVM ? VMR3GetStateName(pUVM->pVM->enmVMState) : "CREATING"));
+            }
+            else if (pUVCpu->vm.s.pReqs)
+            {
+                /*
+                 * Service execute in specific EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, pUVCpu->idCpu);
+                Log(("vmR3EmulationThread: Req (cpu=%u) rc=%Rrc, VM state %s -> %s\n", pUVCpu->idCpu, rc, VMR3GetStateName(enmBefore), pUVM->pVM ? VMR3GetStateName(pUVM->pVM->enmVMState) : "CREATING"));
+            }
+            else
+            {
+                /*
+                 * Nothing important is pending, so wait for something.
+                 */
+                rc = VMR3WaitU(pUVCpu);
+                if (RT_FAILURE(rc))
+                {
+                    AssertLogRelMsgFailed(("VMR3WaitU failed with %Rrc\n", rc));
+                    break;
+                }
+            }
+        }
+        else
+        {
+            /*
+             * Pending requests which needs servicing?
+             *
+             * We check for state changes in addition to status codes when
+             * servicing requests. (Look after the ifs.)
+             */
+            PVM pVM = pUVM->pVM;
+            enmBefore = pVM->enmVMState;
+            if (pUVM->vm.s.fTerminateEMT)
+            {
+                rc = VINF_EM_TERMINATE;
+                break;
+            }
+
+            if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
+            {
+                rc = VMMR3EmtRendezvousFF(pVM, &pVM->aCpus[idCpu]);
+                Log(("vmR3EmulationThread: Rendezvous rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (pUVM->vm.s.pReqs)
+            {
+                /*
+                 * Service execute in any EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, VMCPUID_ANY);
+                Log(("vmR3EmulationThread: Req rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (pUVCpu->vm.s.pReqs)
+            {
+                /*
+                 * Service execute in specific EMT request.
+                 */
+                rc = VMR3ReqProcessU(pUVM, pUVCpu->idCpu);
+                Log(("vmR3EmulationThread: Req (cpu=%u) rc=%Rrc, VM state %s -> %s\n", pUVCpu->idCpu, rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (VM_FF_ISSET(pVM, VM_FF_DBGF))
+            {
+                /*
+                 * Service the debugger request.
+                 */
+                rc = DBGFR3VMMForcedAction(pVM);
+                Log(("vmR3EmulationThread: Dbg rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else if (VM_FF_TESTANDCLEAR(pVM, VM_FF_RESET))
+            {
+                /*
+                 * Service a delayed reset request.
+                 */
+                rc = VMR3Reset(pVM);
+                VM_FF_CLEAR(pVM, VM_FF_RESET);
+                Log(("vmR3EmulationThread: Reset rc=%Rrc, VM state %s -> %s\n", rc, VMR3GetStateName(enmBefore), VMR3GetStateName(pVM->enmVMState)));
+            }
+            else
+            {
+                /*
+                 * Nothing important is pending, so wait for something.
+                 */
+                rc = VMR3WaitU(pUVCpu);
+                if (RT_FAILURE(rc))
+                {
+                    AssertLogRelMsgFailed(("VMR3WaitU failed with %Rrc\n", rc));
+                    break;
+                }
+            }
+
+            /*
+             * Check for termination requests, these have extremely high priority.
+             */
+            if (    rc == VINF_EM_TERMINATE
+                ||  pUVM->vm.s.fTerminateEMT)
+                break;
+        }
+
+        /*
+         * Some requests (both VMR3Req* and the DBGF) can potentially resume
+         * or start the VM, in that case we'll get a change in VM status
+         * indicating that we're now running.
+         */
+        if (    RT_SUCCESS(rc)
+            &&  pUVM->pVM)
+        {
+            PVM     pVM   = pUVM->pVM;
+            PVMCPU  pVCpu = &pVM->aCpus[idCpu];
+            if (    pVM->enmVMState == VMSTATE_RUNNING
+                &&  VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(pVCpu)))
+            {
+                rc = EMR3ExecuteVM(pVM, pVCpu);
+                Log(("vmR3EmulationThread: EMR3ExecuteVM() -> rc=%Rrc, enmVMState=%d\n", rc, pVM->enmVMState));
+                if (EMGetState(pVCpu) == EMSTATE_GURU_MEDITATION)
+                    vmR3SetGuruMeditation(pVM);
+            }
+        }
+
+    } /* forever */
+
+
+    /*
+     * Cleanup and exit.
+     */
+    Log(("vmR3EmulationThread: Terminating emulation thread! Thread=%#x pUVM=%p rc=%Rrc enmBefore=%d enmVMState=%d\n",
+         ThreadSelf, pUVM, rc, enmBefore, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_TERMINATED));
+    if (   idCpu == 0
+        && pUVM->pVM)
+    {
+        PVM pVM = pUVM->pVM;
+        vmR3SetTerminated(pVM);
+        pUVM->pVM = NULL;
+
+        /** @todo SMP: This isn't 100% safe. We should wait for the other
+         *        threads to finish before destroy the VM. */
+        int rc2 = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_GVMM_DESTROY_VM, 0, NULL);
+        AssertLogRelRC(rc2);
+    }
+
+    pUVCpu->vm.s.NativeThreadEMT = NIL_RTNATIVETHREAD;
+    Log(("vmR3EmulationThread: EMT is terminated.\n"));
+    return rc;
+}
+
+
+/**
+ * Gets the name of a halt method.
+ *
+ * @returns Pointer to a read only string.
+ * @param   enmMethod   The method.
+ */
+static const char *vmR3GetHaltMethodName(VMHALTMETHOD enmMethod)
+{
+    switch (enmMethod)
+    {
+        case VMHALTMETHOD_BOOTSTRAP:    return "bootstrap";
+        case VMHALTMETHOD_DEFAULT:      return "default";
+        case VMHALTMETHOD_OLD:          return "old";
+        case VMHALTMETHOD_1:            return "method1";
+        //case VMHALTMETHOD_2:            return "method2";
+        case VMHALTMETHOD_GLOBAL_1:     return "global1";
+        default:                        return "unknown";
+    }
+}
+
+
+/**
+ * Signal a fatal wait error.
+ *
+ * @returns Fatal error code to be propagated up the call stack.
+ * @param   pUVCpu              The user mode per CPU structure of the calling
+ *                              EMT.
+ * @param   pszFmt              The error format with a single %Rrc in it.
+ * @param   rcFmt               The status code to format.
+ */
+static int vmR3FatalWaitError(PUVMCPU pUVCpu, const char *pszFmt, int rcFmt)
+{
+    /** @todo This is wrong ... raise a fatal error / guru meditation
+     *        instead. */
+    AssertLogRelMsgFailed((pszFmt, rcFmt));
+    ASMAtomicUoWriteBool(&pUVCpu->pUVM->vm.s.fTerminateEMT, true);
+    if (pUVCpu->pVM)
+        VM_FF_SET(pUVCpu->pVM, VM_FF_CHECK_VM_STATE);
+    return VERR_INTERNAL_ERROR;
+}
+
+
+/**
+ * The old halt loop.
+ */
+static DECLCALLBACK(int) vmR3HaltOldDoHalt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t /* u64Now*/)
+{
+    /*
+     * Halt loop.
+     */
+    PVM    pVM   = pUVCpu->pVM;
+    PVMCPU pVCpu = pUVCpu->pVCpu;
+
+    int rc = VINF_SUCCESS;
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+    //unsigned cLoops = 0;
+    for (;;)
+    {
+        /*
+         * Work the timers and check if we can exit.
+         * The poll call gives us the ticks left to the next event in
+         * addition to perhaps set an FF.
+         */
+        uint64_t const u64StartTimers   = RTTimeNanoTS();
+        TMR3TimerQueuesDo(pVM);
+        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
+        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+            break;
+        uint64_t u64NanoTS;
+        TMTimerPollGIP(pVM, pVCpu, &u64NanoTS);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        if (u64NanoTS < 50000)
+        {
+            //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d spin\n", u64NanoTS, cLoops++);
+            /* spin */;
+        }
+        else
+        {
+            VMMR3YieldStop(pVM);
+            //uint64_t u64Start = RTTimeNanoTS();
+            if (u64NanoTS <  870000) /* this is a bit speculative... works fine on linux. */
+            {
+                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d yield", u64NanoTS, cLoops++);
+                uint64_t const u64StartSchedYield   = RTTimeNanoTS();
+                RTThreadYield(); /* this is the best we can do here */
+                uint64_t const cNsElapsedSchedYield = RTTimeNanoTS() - u64StartSchedYield;
+                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltYield, cNsElapsedSchedYield);
+            }
+            else if (u64NanoTS < 2000000)
+            {
+                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep 1ms", u64NanoTS, cLoops++);
+                uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+                rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1);
+                uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
+                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+            }
+            else
+            {
+                //RTLogPrintf("u64NanoTS=%RI64 cLoops=%d sleep %dms", u64NanoTS, cLoops++, (uint32_t)RT_MIN((u64NanoTS - 500000) / 1000000, 15));
+                uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+                rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, RT_MIN((u64NanoTS - 1000000) / 1000000, 15));
+                uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
+                STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+            }
+            //uint64_t u64Slept = RTTimeNanoTS() - u64Start;
+            //RTLogPrintf(" -> rc=%Rrc in %RU64 ns / %RI64 ns delta\n", rc, u64Slept, u64NanoTS - u64Slept);
+        }
+        if (rc == VERR_TIMEOUT)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Initialize the configuration of halt method 1 & 2.
+ *
+ * @return VBox status code. Failure on invalid CFGM data.
+ * @param   pVM     The VM handle.
+ */
+static int vmR3HaltMethod12ReadConfigU(PUVM pUVM)
+{
+    /*
+     * The defaults.
+     */
+#if 1 /* DEBUGGING STUFF - REMOVE LATER */
+    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
+    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   2*1000000;
+    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg =  75*1000000;
+    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  30*1000000;
+    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =  20*1000000;
+#else
+    pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = 4;
+    pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg =   5*1000000;
+    pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = 200*1000000;
+    pUVM->vm.s.Halt.Method12.u32StartSpinningCfg    =  20*1000000;
+    pUVM->vm.s.Halt.Method12.u32StopSpinningCfg     =   2*1000000;
+#endif
+
+    /*
+     * Query overrides.
+     *
+     * I don't have time to bother with niceties such as invalid value checks
+     * here right now. sorry.
+     */
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pUVM->pVM), "/VMM/HaltedMethod1");
+    if (pCfg)
+    {
+        uint32_t u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "LagBlockIntervalDivisor", &u32)))
+            pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MinBlockInterval", &u32)))
+            pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "MaxBlockInterval", &u32)))
+            pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StartSpinning", &u32)))
+            pUVM->vm.s.Halt.Method12.u32StartSpinningCfg = u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "StopSpinning", &u32)))
+            pUVM->vm.s.Halt.Method12.u32StopSpinningCfg = u32;
+        LogRel(("HaltedMethod1 config: %d/%d/%d/%d/%d\n",
+                pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
+                pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
+                pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg,
+                pUVM->vm.s.Halt.Method12.u32StartSpinningCfg,
+                pUVM->vm.s.Halt.Method12.u32StopSpinningCfg));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Initialize halt method 1.
+ *
+ * @return VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltMethod1Init(PUVM pUVM)
+{
+    return vmR3HaltMethod12ReadConfigU(pUVM);
+}
+
+
+/**
+ * Method 1 - Block whenever possible, and when lagging behind
+ * switch to spinning for 10-30ms with occasional blocking until
+ * the lag has been eliminated.
+ */
+static DECLCALLBACK(int) vmR3HaltMethod1Halt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now)
+{
+    PUVM    pUVM    = pUVCpu->pUVM;
+    PVMCPU  pVCpu   = pUVCpu->pVCpu;
+    PVM     pVM     = pUVCpu->pVM;
+
+    /*
+     * To simplify things, we decide up-front whether we should switch to spinning or
+     * not. This makes some ASSUMPTIONS about the cause of the spinning (PIT/RTC/PCNet)
+     * and that it will generate interrupts or other events that will cause us to exit
+     * the halt loop.
+     */
+    bool fBlockOnce = false;
+    bool fSpinning = false;
+    uint32_t u32CatchUpPct = TMVirtualSyncGetCatchUpPct(pVM);
+    if (u32CatchUpPct /* non-zero if catching up */)
+    {
+        if (pUVCpu->vm.s.Halt.Method12.u64StartSpinTS)
+        {
+            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StopSpinningCfg;
+            if (fSpinning)
+            {
+                uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
+                fBlockOnce = u64Now - pUVCpu->vm.s.Halt.Method12.u64LastBlockTS
+                           > RT_MAX(pUVM->vm.s.Halt.Method12.u32MinBlockIntervalCfg,
+                                    RT_MIN(u64Lag / pUVM->vm.s.Halt.Method12.u32LagBlockIntervalDivisorCfg,
+                                           pUVM->vm.s.Halt.Method12.u32MaxBlockIntervalCfg));
+            }
+            else
+            {
+                //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVCpu->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
+                pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = 0;
+            }
+        }
+        else
+        {
+            fSpinning = TMVirtualSyncGetLag(pVM) >= pUVM->vm.s.Halt.Method12.u32StartSpinningCfg;
+            if (fSpinning)
+                pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = u64Now;
+        }
+    }
+    else if (pUVCpu->vm.s.Halt.Method12.u64StartSpinTS)
+    {
+        //RTLogRelPrintf("Stopped spinning (%u ms)\n", (u64Now - pUVCpu->vm.s.Halt.Method12.u64StartSpinTS) / 1000000);
+        pUVCpu->vm.s.Halt.Method12.u64StartSpinTS = 0;
+    }
+
+    /*
+     * Halt loop.
+     */
+    int rc = VINF_SUCCESS;
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+    unsigned cLoops = 0;
+    for (;; cLoops++)
+    {
+        /*
+         * Work the timers and check if we can exit.
+         */
+        uint64_t const u64StartTimers   = RTTimeNanoTS();
+        TMR3TimerQueuesDo(pVM);
+        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
+        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+            break;
+
+        /*
+         * Estimate time left to the next event.
+         */
+        uint64_t u64NanoTS;
+        TMTimerPollGIP(pVM, pVCpu, &u64NanoTS);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+            break;
+
+        /*
+         * Block if we're not spinning and the interval isn't all that small.
+         */
+        if (    (   !fSpinning
+                 || fBlockOnce)
+#if 1 /* DEBUGGING STUFF - REMOVE LATER */
+            &&  u64NanoTS >= 100000) /* 0.100 ms */
+#else
+            &&  u64NanoTS >= 250000) /* 0.250 ms */
+#endif
+        {
+            const uint64_t Start = pUVCpu->vm.s.Halt.Method12.u64LastBlockTS = RTTimeNanoTS();
+            VMMR3YieldStop(pVM);
+
+            uint32_t cMilliSecs = RT_MIN(u64NanoTS / 1000000, 15);
+            if (cMilliSecs <= pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg)
+                cMilliSecs = 1;
+            else
+                cMilliSecs -= pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg;
+
+            //RTLogRelPrintf("u64NanoTS=%RI64 cLoops=%3d sleep %02dms (%7RU64) ", u64NanoTS, cLoops, cMilliSecs, u64NanoTS);
+            uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+            rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, cMilliSecs);
+            uint64_t const cNsElapsedSchedHalt = RTTimeNanoTS() - u64StartSchedHalt;
+            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+
+            if (rc == VERR_TIMEOUT)
+                rc = VINF_SUCCESS;
+            else if (RT_FAILURE(rc))
+            {
+                rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
+                break;
+            }
+
+            /*
+             * Calc the statistics.
+             * Update averages every 16th time, and flush parts of the history every 64th time.
+             */
+            const uint64_t Elapsed = RTTimeNanoTS() - Start;
+            pUVCpu->vm.s.Halt.Method12.cNSBlocked += Elapsed;
+            if (Elapsed > u64NanoTS)
+                pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong += Elapsed - u64NanoTS;
+            pUVCpu->vm.s.Halt.Method12.cBlocks++;
+            if (!(pUVCpu->vm.s.Halt.Method12.cBlocks & 0xf))
+            {
+                pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg = pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong / pUVCpu->vm.s.Halt.Method12.cBlocks;
+                if (!(pUVCpu->vm.s.Halt.Method12.cBlocks & 0x3f))
+                {
+                    pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLong = pUVCpu->vm.s.Halt.Method12.cNSBlockedTooLongAvg * 0x40;
+                    pUVCpu->vm.s.Halt.Method12.cBlocks = 0x40;
+                }
+            }
+            //RTLogRelPrintf(" -> %7RU64 ns / %7RI64 ns delta%s\n", Elapsed, Elapsed - u64NanoTS, fBlockOnce ? " (block once)" : "");
+
+            /*
+             * Clear the block once flag if we actually blocked.
+             */
+            if (    fBlockOnce
+                &&  Elapsed > 100000 /* 0.1 ms */)
+                fBlockOnce = false;
+        }
+    }
+    //if (fSpinning) RTLogRelPrintf("spun for %RU64 ns %u loops; lag=%RU64 pct=%d\n", RTTimeNanoTS() - u64Now, cLoops, TMVirtualSyncGetLag(pVM), u32CatchUpPct);
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Initialize the global 1 halt method.
+ *
+ * @return VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Init(PUVM pUVM)
+{
+    /*
+     * The defaults.
+     */
+    uint32_t cNsResolution = SUPSemEventMultiGetResolution(pUVM->vm.s.pSession);
+    if (cNsResolution > 5*RT_NS_100US)
+        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = 50000;
+    else if (cNsResolution > RT_NS_100US)
+        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = cNsResolution / 4;
+    else
+        pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = 2000;
+
+    /*
+     * Query overrides.
+     *
+     * I don't have time to bother with niceties such as invalid value checks
+     * here right now. sorry.
+     */
+    PCFGMNODE pCfg = CFGMR3GetChild(CFGMR3GetRoot(pUVM->pVM), "/VMM/HaltedGlobal1");
+    if (pCfg)
+    {
+        uint32_t u32;
+        if (RT_SUCCESS(CFGMR3QueryU32(pCfg, "SpinBlockThreshold", &u32)))
+            pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg = u32;
+    }
+    LogRel(("HaltedGlobal1 config: cNsSpinBlockThresholdCfg=%u\n",
+            pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * The global 1 halt method - Block in GMM (ring-0) and let it
+ * try take care of the global scheduling of EMT threads.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Halt(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now)
+{
+    PUVM    pUVM  = pUVCpu->pUVM;
+    PVMCPU  pVCpu = pUVCpu->pVCpu;
+    PVM     pVM   = pUVCpu->pVM;
+    Assert(VMMGetCpu(pVM) == pVCpu);
+
+    /*
+     * Halt loop.
+     */
+    //uint64_t u64NowLog, u64Start;
+    //u64Start = u64NowLog = RTTimeNanoTS();
+    int rc = VINF_SUCCESS;
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+    unsigned cLoops = 0;
+    for (;; cLoops++)
+    {
+        /*
+         * Work the timers and check if we can exit.
+         */
+        uint64_t const u64StartTimers   = RTTimeNanoTS();
+        TMR3TimerQueuesDo(pVM);
+        uint64_t const cNsElapsedTimers = RTTimeNanoTS() - u64StartTimers;
+        STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltTimers, cNsElapsedTimers);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+            break;
+
+        /*
+         * Estimate time left to the next event.
+         */
+        //u64NowLog = RTTimeNanoTS();
+        uint64_t u64Delta;
+        uint64_t u64GipTime = TMTimerPollGIP(pVM, pVCpu, &u64Delta);
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+            break;
+
+        /*
+         * Block if we're not spinning and the interval isn't all that small.
+         */
+        if (u64Delta >= pUVM->vm.s.Halt.Global1.cNsSpinBlockThresholdCfg)
+        {
+            VMMR3YieldStop(pVM);
+            if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+                ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+                break;
+
+            //RTLogPrintf("loop=%-3d  u64GipTime=%'llu / %'llu   now=%'llu / %'llu\n", cLoops, u64GipTime, u64Delta, u64NowLog, u64GipTime - u64NowLog);
+            uint64_t const u64StartSchedHalt   = RTTimeNanoTS();
+            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_HALT, u64GipTime, NULL);
+            uint64_t const u64EndSchedHalt     = RTTimeNanoTS();
+            uint64_t const cNsElapsedSchedHalt = u64EndSchedHalt - u64StartSchedHalt;
+            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlock, cNsElapsedSchedHalt);
+
+            if (rc == VERR_INTERRUPTED)
+                rc = VINF_SUCCESS;
+            else if (RT_FAILURE(rc))
+            {
+                rc = vmR3FatalWaitError(pUVCpu, "VMMR0_DO_GVMM_SCHED_HALT->%Rrc\n", rc);
+                break;
+            }
+            else
+            {
+                int64_t const cNsOverslept = u64EndSchedHalt - u64GipTime;
+                if (cNsOverslept > 50000)
+                    STAM_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockOverslept, cNsOverslept);
+                else if (cNsOverslept < -50000)
+                    STAM_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockInsomnia,  cNsElapsedSchedHalt);
+                else
+                    STAM_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltBlockOnTime,    cNsElapsedSchedHalt);
+            }
+        }
+        /*
+         * When spinning call upon the GVMM and do some wakups once
+         * in a while, it's not like we're actually busy or anything.
+         */
+        else if (!(cLoops & 0x1fff))
+        {
+            uint64_t const u64StartSchedYield   = RTTimeNanoTS();
+            rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_POLL, false /* don't yield */, NULL);
+            uint64_t const cNsElapsedSchedYield = RTTimeNanoTS() - u64StartSchedYield;
+            STAM_REL_PROFILE_ADD_PERIOD(&pUVCpu->vm.s.StatHaltYield, cNsElapsedSchedYield);
+        }
+    }
+    //RTLogPrintf("*** %u loops %'llu;  lag=%RU64\n", cLoops, u64NowLog - u64Start, TMVirtualSyncGetLag(pVM));
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * The global 1 halt method - VMR3Wait() worker.
+ *
+ * @returns VBox status code.
+ * @param   pUVCpu            Pointer to the user mode VMCPU structure.
+ */
+static DECLCALLBACK(int) vmR3HaltGlobal1Wait(PUVMCPU pUVCpu)
+{
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+
+    PVM    pVM   = pUVCpu->pUVM->pVM;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu->idCpu == pUVCpu->idCpu);
+
+    int rc = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Check Relevant FFs.
+         */
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK))
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, VMMR0_DO_GVMM_SCHED_HALT, RTTimeNanoTS() + 1000000000 /* +1s */, NULL);
+        if (rc == VERR_INTERRUPTED)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "VMMR0_DO_GVMM_SCHED_HALT->%Rrc\n", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * The global 1 halt method - VMR3NotifyFF() worker.
+ *
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+static DECLCALLBACK(void) vmR3HaltGlobal1NotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    if (pUVCpu->vm.s.fWait)
+    {
+        int rc = SUPR3CallVMMR0Ex(pUVCpu->pVM->pVMR0, pUVCpu->idCpu, VMMR0_DO_GVMM_SCHED_WAKE_UP, 0, NULL);
+        AssertRC(rc);
+    }
+    else if (   (   (fFlags & VMNOTIFYFF_FLAGS_POKE)
+                 || !(fFlags & VMNOTIFYFF_FLAGS_DONE_REM))
+             && pUVCpu->pVCpu)
+    {
+        VMCPUSTATE enmState = VMCPU_GET_STATE(pUVCpu->pVCpu);
+        if (enmState == VMCPUSTATE_STARTED_EXEC)
+        {
+            if (fFlags & VMNOTIFYFF_FLAGS_POKE)
+            {
+                int rc = SUPR3CallVMMR0Ex(pUVCpu->pVM->pVMR0, pUVCpu->idCpu, VMMR0_DO_GVMM_SCHED_POKE, 0, NULL);
+                AssertRC(rc);
+            }
+        }
+        else if (enmState == VMCPUSTATE_STARTED_EXEC_REM)
+        {
+            if (!(fFlags & VMNOTIFYFF_FLAGS_DONE_REM))
+                REMR3NotifyFF(pUVCpu->pVM);
+        }
+    }
+}
+
+
+/**
+ * Bootstrap VMR3Wait() worker.
+ *
+ * @returns VBox status code.
+ * @param   pUVMCPU            Pointer to the user mode VMCPU structure.
+ */
+static DECLCALLBACK(int) vmR3BootstrapWait(PUVMCPU pUVCpu)
+{
+    PUVM pUVM = pUVCpu->pUVM;
+
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+
+    int rc = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Check Relevant FFs.
+         */
+        if (pUVM->vm.s.pReqs)   /* global requests pending? */
+            break;
+        if (pUVCpu->vm.s.pReqs) /* local requests pending? */
+            break;
+
+        if (    pUVCpu->pVM
+            &&  (   VM_FF_ISPENDING(pUVCpu->pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+                 || VMCPU_FF_ISPENDING(VMMGetCpu(pUVCpu->pVM), VMCPU_FF_EXTERNAL_SUSPENDED_MASK)
+                )
+            )
+            break;
+        if (pUVM->vm.s.fTerminateEMT)
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1000);
+        if (rc == VERR_TIMEOUT)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc\n", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Bootstrap VMR3NotifyFF() worker.
+ *
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+static DECLCALLBACK(void) vmR3BootstrapNotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    if (pUVCpu->vm.s.fWait)
+    {
+        int rc = RTSemEventSignal(pUVCpu->vm.s.EventSemWait);
+        AssertRC(rc);
+    }
+    NOREF(fFlags);
+}
+
+
+/**
+ * Default VMR3Wait() worker.
+ *
+ * @returns VBox status code.
+ * @param   pUVMCPU            Pointer to the user mode VMCPU structure.
+ */
+static DECLCALLBACK(int) vmR3DefaultWait(PUVMCPU pUVCpu)
+{
+    ASMAtomicWriteBool(&pUVCpu->vm.s.fWait, true);
+
+    PVM    pVM   = pUVCpu->pVM;
+    PVMCPU pVCpu = pUVCpu->pVCpu;
+    int    rc    = VINF_SUCCESS;
+    for (;;)
+    {
+        /*
+         * Check Relevant FFs.
+         */
+        if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+            ||  VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK))
+            break;
+
+        /*
+         * Wait for a while. Someone will wake us up or interrupt the call if
+         * anything needs our attention.
+         */
+        rc = RTSemEventWait(pUVCpu->vm.s.EventSemWait, 1000);
+        if (rc == VERR_TIMEOUT)
+            rc = VINF_SUCCESS;
+        else if (RT_FAILURE(rc))
+        {
+            rc = vmR3FatalWaitError(pUVCpu, "RTSemEventWait->%Rrc", rc);
+            break;
+        }
+    }
+
+    ASMAtomicUoWriteBool(&pUVCpu->vm.s.fWait, false);
+    return rc;
+}
+
+
+/**
+ * Default VMR3NotifyFF() worker.
+ *
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+static DECLCALLBACK(void) vmR3DefaultNotifyCpuFF(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    if (pUVCpu->vm.s.fWait)
+    {
+        int rc = RTSemEventSignal(pUVCpu->vm.s.EventSemWait);
+        AssertRC(rc);
+    }
+    else if (   !(fFlags & VMNOTIFYFF_FLAGS_DONE_REM)
+             && pUVCpu->pVCpu
+             && pUVCpu->pVCpu->enmState == VMCPUSTATE_STARTED_EXEC_REM)
+        REMR3NotifyFF(pUVCpu->pVM);
+}
+
+
+/**
+ * Array with halt method descriptors.
+ * VMINT::iHaltMethod contains an index into this array.
+ */
+static const struct VMHALTMETHODDESC
+{
+    /** The halt method id. */
+    VMHALTMETHOD enmHaltMethod;
+    /** The init function for loading config and initialize variables. */
+    DECLR3CALLBACKMEMBER(int,  pfnInit,(PUVM pUVM));
+    /** The term function. */
+    DECLR3CALLBACKMEMBER(void, pfnTerm,(PUVM pUVM));
+    /** The VMR3WaitHaltedU function. */
+    DECLR3CALLBACKMEMBER(int,  pfnHalt,(PUVMCPU pUVCpu, const uint32_t fMask, uint64_t u64Now));
+    /** The VMR3WaitU function. */
+    DECLR3CALLBACKMEMBER(int,  pfnWait,(PUVMCPU pUVCpu));
+    /** The VMR3NotifyCpuFFU function. */
+    DECLR3CALLBACKMEMBER(void, pfnNotifyCpuFF,(PUVMCPU pUVCpu, uint32_t fFlags));
+    /** The VMR3NotifyGlobalFFU function. */
+    DECLR3CALLBACKMEMBER(void, pfnNotifyGlobalFF,(PUVM pUVM, uint32_t fFlags));
+} g_aHaltMethods[] =
+{
+    { VMHALTMETHOD_BOOTSTRAP, NULL,                NULL,   NULL,                vmR3BootstrapWait,   vmR3BootstrapNotifyCpuFF,   NULL },
+    { VMHALTMETHOD_OLD,       NULL,                NULL,   vmR3HaltOldDoHalt,   vmR3DefaultWait,     vmR3DefaultNotifyCpuFF,     NULL },
+    { VMHALTMETHOD_1,         vmR3HaltMethod1Init, NULL,   vmR3HaltMethod1Halt, vmR3DefaultWait,     vmR3DefaultNotifyCpuFF,     NULL },
+    { VMHALTMETHOD_GLOBAL_1,  vmR3HaltGlobal1Init, NULL,   vmR3HaltGlobal1Halt, vmR3HaltGlobal1Wait, vmR3HaltGlobal1NotifyCpuFF, NULL },
+};
+
+
+/**
+ * Notify the emulation thread (EMT) about pending Forced Action (FF).
+ *
+ * This function is called by thread other than EMT to make
+ * sure EMT wakes up and promptly service an FF request.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+VMMR3DECL(void) VMR3NotifyGlobalFFU(PUVM pUVM, uint32_t fFlags)
+{
+    LogFlow(("VMR3NotifyGlobalFFU:\n"));
+    uint32_t iHaldMethod = pUVM->vm.s.iHaltMethod;
+
+    if (g_aHaltMethods[iHaldMethod].pfnNotifyGlobalFF) /** @todo make mandatory. */
+        g_aHaltMethods[iHaldMethod].pfnNotifyGlobalFF(pUVM, fFlags);
+    else
+        for (VMCPUID iCpu = 0; iCpu < pUVM->cCpus; iCpu++)
+            g_aHaltMethods[iHaldMethod].pfnNotifyCpuFF(&pUVM->aCpus[iCpu], fFlags);
+}
+
+
+/**
+ * Notify the emulation thread (EMT) about pending Forced Action (FF).
+ *
+ * This function is called by thread other than EMT to make
+ * sure EMT wakes up and promptly service an FF request.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   fFlags          Notification flags, VMNOTIFYFF_FLAGS_*.
+ */
+VMMR3DECL(void) VMR3NotifyCpuFFU(PUVMCPU pUVCpu, uint32_t fFlags)
+{
+    PUVM pUVM = pUVCpu->pUVM;
+
+    LogFlow(("VMR3NotifyCpuFFU:\n"));
+    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyCpuFF(pUVCpu, fFlags);
+}
+
+
+/**
+ * Halted VM Wait.
+ * Any external event will unblock the thread.
+ *
+ * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
+ *          case an appropriate status code is returned.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ * @param   fIgnoreInterrupts   If set the VM_FF_INTERRUPT flags is ignored.
+ * @thread  The emulation thread.
+ */
+VMMR3DECL(int) VMR3WaitHalted(PVM pVM, PVMCPU pVCpu, bool fIgnoreInterrupts)
+{
+    LogFlow(("VMR3WaitHalted: fIgnoreInterrupts=%d\n", fIgnoreInterrupts));
+
+    /*
+     * Check Relevant FFs.
+     */
+    const uint32_t fMask = !fIgnoreInterrupts
+        ? VMCPU_FF_EXTERNAL_HALTED_MASK
+        : VMCPU_FF_EXTERNAL_HALTED_MASK & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC);
+    if (    VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_HALTED_MASK)
+        ||  VMCPU_FF_ISPENDING(pVCpu, fMask))
+    {
+        LogFlow(("VMR3WaitHalted: returns VINF_SUCCESS (FF %#x FFCPU %#x)\n", pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * The yielder is suspended while we're halting, while TM might have clock(s) running
+     * only at certain times and need to be notified..
+     */
+    if (pVCpu->idCpu == 0)
+        VMMR3YieldSuspend(pVM);
+    TMNotifyStartOfHalt(pVCpu);
+
+    /*
+     * Record halt averages for the last second.
+     */
+    PUVMCPU pUVCpu = pVCpu->pUVCpu;
+    uint64_t u64Now = RTTimeNanoTS();
+    int64_t off = u64Now - pUVCpu->vm.s.u64HaltsStartTS;
+    if (off > 1000000000)
+    {
+        if (off > _4G || !pUVCpu->vm.s.cHalts)
+        {
+            pUVCpu->vm.s.HaltInterval = 1000000000 /* 1 sec */;
+            pUVCpu->vm.s.HaltFrequency = 1;
+        }
+        else
+        {
+            pUVCpu->vm.s.HaltInterval = (uint32_t)off / pUVCpu->vm.s.cHalts;
+            pUVCpu->vm.s.HaltFrequency = ASMMultU64ByU32DivByU32(pUVCpu->vm.s.cHalts, 1000000000, (uint32_t)off);
+        }
+        pUVCpu->vm.s.u64HaltsStartTS = u64Now;
+        pUVCpu->vm.s.cHalts = 0;
+    }
+    pUVCpu->vm.s.cHalts++;
+
+    /*
+     * Do the halt.
+     */
+    Assert(VMCPU_GET_STATE(pVCpu) == VMCPUSTATE_STARTED);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HALTED);
+    PUVM pUVM = pUVCpu->pUVM;
+    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnHalt(pUVCpu, fMask, u64Now);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+
+    /*
+     * Notify TM and resume the yielder
+     */
+    TMNotifyEndOfHalt(pVCpu);
+    if (pVCpu->idCpu == 0)
+        VMMR3YieldResume(pVM);
+
+    LogFlow(("VMR3WaitHalted: returns %Rrc (FF %#x)\n", rc, pVM->fGlobalForcedActions));
+    return rc;
+}
+
+
+/**
+ * Suspended VM Wait.
+ * Only a handful of forced actions will cause the function to
+ * return to the caller.
+ *
+ * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
+ *          case an appropriate status code is returned.
+ * @param   pUVCpu          Pointer to the user mode VMCPU structure.
+ * @thread  The emulation thread.
+ */
+VMMR3DECL(int) VMR3WaitU(PUVMCPU pUVCpu)
+{
+    LogFlow(("VMR3WaitU:\n"));
+
+    /*
+     * Check Relevant FFs.
+     */
+    PVM    pVM   = pUVCpu->pVM;
+    PVMCPU pVCpu = pUVCpu->pVCpu;
+
+    if (    pVM
+        &&  (   VM_FF_ISPENDING(pVM, VM_FF_EXTERNAL_SUSPENDED_MASK)
+             || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_EXTERNAL_SUSPENDED_MASK)
+            )
+        )
+    {
+        LogFlow(("VMR3Wait: returns VINF_SUCCESS (FF %#x)\n", pVM->fGlobalForcedActions));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Do waiting according to the halt method (so VMR3NotifyFF
+     * doesn't have to special case anything).
+     */
+    PUVM pUVM = pUVCpu->pUVM;
+    int rc = g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnWait(pUVCpu);
+    LogFlow(("VMR3WaitU: returns %Rrc (FF %#x)\n", rc, pVM ? pVM->fGlobalForcedActions : 0));
+    return rc;
+}
+
+
+/**
+ * Interface that PDMR3Suspend, PDMR3PowerOff and PDMR3Reset uses when they wait
+ * for the handling of asynchronous notifications to complete.
+ *
+ * @returns VINF_SUCCESS unless a fatal error occurred. In the latter
+ *          case an appropriate status code is returned.
+ * @param   pUVCpu              Pointer to the user mode VMCPU structure.
+ * @thread  The emulation thread.
+ */
+VMMR3_INT_DECL(int) VMR3AsyncPdmNotificationWaitU(PUVMCPU pUVCpu)
+{
+    LogFlow(("VMR3AsyncPdmNotificationWaitU:\n"));
+    return VMR3WaitU(pUVCpu);
+}
+
+
+/**
+ * Interface that PDM the helper asynchronous notification completed methods
+ * uses for EMT0 when it is waiting inside VMR3AsyncPdmNotificationWaitU().
+ *
+ * @param   pUVM                Pointer to the user mode VM structure.
+ */
+VMMR3_INT_DECL(void) VMR3AsyncPdmNotificationWakeupU(PUVM pUVM)
+{
+    LogFlow(("VMR3AsyncPdmNotificationWakeupU:\n"));
+    VM_FF_SET(pUVM->pVM, VM_FF_REQUEST); /* this will have to do for now. */
+    g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnNotifyCpuFF(&pUVM->aCpus[0], 0 /*fFlags*/);
+}
+
+
+/**
+ * Rendezvous callback that will be called once.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM                 VM handle.
+ * @param   pVCpu               The VMCPU handle for the calling EMT.
+ * @param   pvUser              The new g_aHaltMethods index.
+ */
+static DECLCALLBACK(VBOXSTRICTRC) vmR3SetHaltMethodCallback(PVM pVM, PVMCPU pVCpu, void *pvUser)
+{
+    PUVM        pUVM = pVM->pUVM;
+    uintptr_t   i    = (uintptr_t)pvUser;
+    Assert(i < RT_ELEMENTS(g_aHaltMethods));
+    NOREF(pVCpu);
+
+    /*
+     * Terminate the old one.
+     */
+    if (    pUVM->vm.s.enmHaltMethod != VMHALTMETHOD_INVALID
+        &&  g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm)
+    {
+        g_aHaltMethods[pUVM->vm.s.iHaltMethod].pfnTerm(pUVM);
+        pUVM->vm.s.enmHaltMethod = VMHALTMETHOD_INVALID;
+    }
+
+    /* Assert that the failure fallback is where we expect. */
+    Assert(g_aHaltMethods[0].enmHaltMethod == VMHALTMETHOD_BOOTSTRAP);
+    Assert(!g_aHaltMethods[0].pfnTerm && !g_aHaltMethods[0].pfnInit);
+
+    /*
+     * Init the new one.
+     */
+    int rc = VINF_SUCCESS;
+    memset(&pUVM->vm.s.Halt, 0, sizeof(pUVM->vm.s.Halt));
+    if (g_aHaltMethods[i].pfnInit)
+    {
+        rc = g_aHaltMethods[i].pfnInit(pUVM);
+        if (RT_FAILURE(rc))
+        {
+            /* Fall back on the bootstrap method. This requires no
+               init/term (see assertion above), and will always work. */
+            AssertLogRelRC(rc);
+            i = 0;
+        }
+    }
+
+    /*
+     * Commit it.
+     */
+    pUVM->vm.s.enmHaltMethod = g_aHaltMethods[i].enmHaltMethod;
+    ASMAtomicWriteU32(&pUVM->vm.s.iHaltMethod, i);
+
+    return rc;
+}
+
+
+/**
+ * Changes the halt method.
+ *
+ * @returns VBox status code.
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   enmHaltMethod   The new halt method.
+ * @thread  EMT.
+ */
+int vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod)
+{
+    PVM pVM = pUVM->pVM; Assert(pVM);
+    VM_ASSERT_EMT(pVM);
+    AssertReturn(enmHaltMethod > VMHALTMETHOD_INVALID && enmHaltMethod < VMHALTMETHOD_END, VERR_INVALID_PARAMETER);
+
+    /*
+     * Resolve default (can be overridden in the configuration).
+     */
+    if (enmHaltMethod == VMHALTMETHOD_DEFAULT)
+    {
+        uint32_t u32;
+        int rc = CFGMR3QueryU32(CFGMR3GetChild(CFGMR3GetRoot(pVM), "VM"), "HaltMethod", &u32);
+        if (RT_SUCCESS(rc))
+        {
+            enmHaltMethod = (VMHALTMETHOD)u32;
+            if (enmHaltMethod <= VMHALTMETHOD_INVALID || enmHaltMethod >= VMHALTMETHOD_END)
+                return VMSetError(pVM, VERR_INVALID_PARAMETER, RT_SRC_POS, N_("Invalid VM/HaltMethod value %d"), enmHaltMethod);
+        }
+        else if (rc == VERR_CFGM_VALUE_NOT_FOUND || rc == VERR_CFGM_CHILD_NOT_FOUND)
+            return VMSetError(pVM, rc, RT_SRC_POS, N_("Failed to Query VM/HaltMethod as uint32_t"));
+        else
+            enmHaltMethod = VMHALTMETHOD_GLOBAL_1;
+            //enmHaltMethod = VMHALTMETHOD_1;
+            //enmHaltMethod = VMHALTMETHOD_OLD;
+    }
+    LogRel(("VM: Halt method %s (%d)\n", vmR3GetHaltMethodName(enmHaltMethod), enmHaltMethod));
+
+    /*
+     * Find the descriptor.
+     */
+    unsigned i = 0;
+    while (     i < RT_ELEMENTS(g_aHaltMethods)
+           &&   g_aHaltMethods[i].enmHaltMethod != enmHaltMethod)
+        i++;
+    AssertReturn(i < RT_ELEMENTS(g_aHaltMethods), VERR_INVALID_PARAMETER);
+
+    /*
+     * This needs to be done while the other EMTs are not sleeping or otherwise messing around.
+     */
+    return VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, vmR3SetHaltMethodCallback, (void *)(uintptr_t)i);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/VMM.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMM.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMM.cpp	(revision 35333)
@@ -0,0 +1,2315 @@
+/* $Id$ */
+/** @file
+ * VMM - The Virtual Machine Monitor Core.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+//#define NO_SUPCALLR0VMM
+
+/** @page pg_vmm        VMM - The Virtual Machine Monitor
+ *
+ * The VMM component is two things at the moment, it's a component doing a few
+ * management and routing tasks, and it's the whole virtual machine monitor
+ * thing.  For hysterical reasons, it is not doing all the management that one
+ * would expect, this is instead done by @ref pg_vm.  We'll address this
+ * misdesign eventually.
+ *
+ * @see grp_vmm, grp_vm
+ *
+ *
+ * @section sec_vmmstate        VMM State
+ *
+ * @image html VM_Statechart_Diagram.gif
+ *
+ * To be written.
+ *
+ *
+ * @subsection  subsec_vmm_init     VMM Initialization
+ *
+ * To be written.
+ *
+ *
+ * @subsection  subsec_vmm_term     VMM Termination
+ *
+ * To be written.
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm.h>
+#include <VBox/vmapi.h>
+#include <VBox/pgm.h>
+#include <VBox/cfgm.h>
+#include <VBox/pdmqueue.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmapi.h>
+#include <VBox/cpum.h>
+#include <VBox/mm.h>
+#include <VBox/iom.h>
+#include <VBox/trpm.h>
+#include <VBox/selm.h>
+#include <VBox/em.h>
+#include <VBox/sup.h>
+#include <VBox/dbgf.h>
+#include <VBox/csam.h>
+#include <VBox/patm.h>
+#include <VBox/rem.h>
+#include <VBox/ssm.h>
+#include <VBox/tm.h>
+#include "VMMInternal.h"
+#include "VMMSwitcher.h"
+#include <VBox/vm.h>
+#include <VBox/ftm.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/version.h>
+#include <VBox/x86.h>
+#include <VBox/hwaccm.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/time.h>
+#include <iprt/semaphore.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/stdarg.h>
+#include <iprt/ctype.h>
+
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** The saved state version. */
+#define VMM_SAVED_STATE_VERSION     4
+/** The saved state version used by v3.0 and earlier. (Teleportation) */
+#define VMM_SAVED_STATE_VERSION_3_0 3
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int                  vmmR3InitStacks(PVM pVM);
+static int                  vmmR3InitLoggers(PVM pVM);
+static void                 vmmR3InitRegisterStats(PVM pVM);
+static DECLCALLBACK(int)    vmmR3Save(PVM pVM, PSSMHANDLE pSSM);
+static DECLCALLBACK(int)    vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass);
+static DECLCALLBACK(void)   vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser);
+static int                  vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu);
+static DECLCALLBACK(void)   vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+
+/**
+ * Initializes the VMM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3_INT_DECL(int) VMMR3Init(PVM pVM)
+{
+    LogFlow(("VMMR3Init\n"));
+
+    /*
+     * Assert alignment, sizes and order.
+     */
+    AssertMsg(pVM->vmm.s.offVM == 0, ("Already initialized!\n"));
+    AssertCompile(sizeof(pVM->vmm.s) <= sizeof(pVM->vmm.padding));
+    AssertCompile(sizeof(pVM->aCpus[0].vmm.s) <= sizeof(pVM->aCpus[0].vmm.padding));
+
+    /*
+     * Init basic VM VMM members.
+     */
+    pVM->vmm.s.offVM = RT_OFFSETOF(VM, vmm);
+    pVM->vmm.s.pahEvtRendezvousEnterOrdered     = NULL;
+    pVM->vmm.s.hEvtRendezvousEnterOneByOne      = NIL_RTSEMEVENT;
+    pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce  = NIL_RTSEMEVENTMULTI;
+    pVM->vmm.s.hEvtMulRendezvousDone            = NIL_RTSEMEVENTMULTI;
+    pVM->vmm.s.hEvtRendezvousDoneCaller         = NIL_RTSEMEVENT;
+
+    /** @cfgm{YieldEMTInterval, uint32_t, 1, UINT32_MAX, 23, ms}
+     * The EMT yield interval.  The EMT yielding is a hack we employ to play a
+     * bit nicer with the rest of the system (like for instance the GUI).
+     */
+    int rc = CFGMR3QueryU32Def(CFGMR3GetRoot(pVM), "YieldEMTInterval", &pVM->vmm.s.cYieldEveryMillies,
+                               23 /* Value arrived at after experimenting with the grub boot prompt. */);
+    AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"YieldEMTInterval\", rc=%Rrc\n", rc), rc);
+
+
+    /** @cfgm{VMM/UsePeriodicPreemptionTimers, boolean, true}
+     * Controls whether we employ per-cpu preemption timers to limit the time
+     * spent executing guest code.  This option is not available on all
+     * platforms and we will silently ignore this setting then.  If we are
+     * running in VT-x mode, we will use the VMX-preemption timer instead of
+     * this one when possible.
+     */
+    PCFGMNODE pCfgVMM = CFGMR3GetChild(CFGMR3GetRoot(pVM), "VMM");
+    rc = CFGMR3QueryBoolDef(pCfgVMM, "UsePeriodicPreemptionTimers", &pVM->vmm.s.fUsePeriodicPreemptionTimers, true);
+    AssertMsgRCReturn(rc, ("Configuration error. Failed to query \"VMM/UsePeriodicPreemptionTimers\", rc=%Rrc\n", rc), rc);
+
+    /*
+     * Initialize the VMM sync critical section and semaphores.
+     */
+    rc = RTCritSectInit(&pVM->vmm.s.CritSectSync);
+    AssertRCReturn(rc, rc);
+    pVM->vmm.s.pahEvtRendezvousEnterOrdered = (PRTSEMEVENT)MMR3HeapAlloc(pVM, MM_TAG_VMM, sizeof(RTSEMEVENT) * pVM->cCpus);
+    if (!pVM->vmm.s.pahEvtRendezvousEnterOrdered)
+        return VERR_NO_MEMORY;
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        rc = RTSemEventCreate(&pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
+        AssertRCReturn(rc, rc);
+    }
+    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousEnterOneByOne);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventMultiCreate(&pVM->vmm.s.hEvtMulRendezvousDone);
+    AssertRCReturn(rc, rc);
+    rc = RTSemEventCreate(&pVM->vmm.s.hEvtRendezvousDoneCaller);
+    AssertRCReturn(rc, rc);
+
+    /* GC switchers are enabled by default. Turned off by HWACCM. */
+    pVM->vmm.s.fSwitcherDisabled = false;
+
+    /*
+     * Register the saved state data unit.
+     */
+    rc = SSMR3RegisterInternal(pVM, "vmm", 1, VMM_SAVED_STATE_VERSION, VMM_STACK_SIZE + sizeof(RTGCPTR),
+                               NULL, NULL, NULL,
+                               NULL, vmmR3Save, NULL,
+                               NULL, vmmR3Load, NULL);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Register the Ring-0 VM handle with the session for fast ioctl calls.
+     */
+    rc = SUPR3SetVMForFastIOCtl(pVM->pVMR0);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Init various sub-components.
+     */
+    rc = vmmR3SwitcherInit(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        rc = vmmR3InitStacks(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = vmmR3InitLoggers(pVM);
+
+#ifdef VBOX_WITH_NMI
+            /*
+             * Allocate mapping for the host APIC.
+             */
+            if (RT_SUCCESS(rc))
+            {
+                rc = MMR3HyperReserve(pVM, PAGE_SIZE, "Host APIC", &pVM->vmm.s.GCPtrApicBase);
+                AssertRC(rc);
+            }
+#endif
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Debug info and statistics.
+                 */
+                DBGFR3InfoRegisterInternal(pVM, "ff", "Displays the current Forced actions Flags.", vmmR3InfoFF);
+                vmmR3InitRegisterStats(pVM);
+
+                return VINF_SUCCESS;
+            }
+        }
+        /** @todo: Need failure cleanup. */
+
+        //more todo in here?
+        //if (RT_SUCCESS(rc))
+        //{
+        //}
+        //int rc2 = vmmR3TermCoreCode(pVM);
+        //AssertRC(rc2));
+    }
+
+    return rc;
+}
+
+
+/**
+ * Allocate & setup the VMM RC stack(s) (for EMTs).
+ *
+ * The stacks are also used for long jumps in Ring-0.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ *
+ * @remarks The optional guard page gets it protection setup up during R3 init
+ *          completion because of init order issues.
+ */
+static int vmmR3InitStacks(PVM pVM)
+{
+    int rc = VINF_SUCCESS;
+#ifdef VMM_R0_SWITCH_STACK
+    uint32_t fFlags = MMHYPER_AONR_FLAGS_KERNEL_MAPPING;
+#else
+    uint32_t fFlags = 0;
+#endif
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[idCpu];
+
+#ifdef VBOX_STRICT_VMM_STACK
+        rc = MMR3HyperAllocOnceNoRelEx(pVM, PAGE_SIZE + VMM_STACK_SIZE + PAGE_SIZE,
+#else
+        rc = MMR3HyperAllocOnceNoRelEx(pVM, VMM_STACK_SIZE,
+#endif
+                                       PAGE_SIZE, MM_TAG_VMM, fFlags, (void **)&pVCpu->vmm.s.pbEMTStackR3);
+        if (RT_SUCCESS(rc))
+        {
+#ifdef VBOX_STRICT_VMM_STACK
+            pVCpu->vmm.s.pbEMTStackR3 += PAGE_SIZE;
+#endif
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+            /* MMHyperR3ToR0 returns R3 when not doing hardware assisted virtualization. */
+            if (!VMMIsHwVirtExtForced(pVM))
+                pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = NIL_RTR0PTR;
+            else
+#endif
+                pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
+            pVCpu->vmm.s.pbEMTStackRC       = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
+            pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
+            AssertRelease(pVCpu->vmm.s.pbEMTStackRC);
+
+            CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
+        }
+    }
+
+    return rc;
+}
+
+
+/**
+ * Initialize the loggers.
+ *
+ * @returns VBox status code.
+ * @param   pVM         Pointer to the shared VM structure.
+ */
+static int vmmR3InitLoggers(PVM pVM)
+{
+    int rc;
+
+    /*
+     * Allocate RC & R0 Logger instances (they are finalized in the relocator).
+     */
+#ifdef LOG_ENABLED
+    PRTLOGGER pLogger = RTLogDefaultInstance();
+    if (pLogger)
+    {
+        pVM->vmm.s.cbRCLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pLogger->cGroups]);
+        rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCLoggerR3);
+        if (RT_FAILURE(rc))
+            return rc;
+        pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
+
+# ifdef VBOX_WITH_R0_LOGGING
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            PVMCPU pVCpu = &pVM->aCpus[i];
+
+            rc = MMR3HyperAllocOnceNoRelEx(pVM, RT_OFFSETOF(VMMR0LOGGER, Logger.afGroups[pLogger->cGroups]),
+                                           0, MM_TAG_VMM, MMHYPER_AONR_FLAGS_KERNEL_MAPPING,
+                                           (void **)&pVCpu->vmm.s.pR0LoggerR3);
+            if (RT_FAILURE(rc))
+                return rc;
+            pVCpu->vmm.s.pR0LoggerR3->pVM = pVM->pVMR0;
+            //pVCpu->vmm.s.pR0LoggerR3->fCreated = false;
+            pVCpu->vmm.s.pR0LoggerR3->cbLogger = RT_OFFSETOF(RTLOGGER, afGroups[pLogger->cGroups]);
+            pVCpu->vmm.s.pR0LoggerR0 = MMHyperR3ToR0(pVM, pVCpu->vmm.s.pR0LoggerR3);
+        }
+# endif
+    }
+#endif /* LOG_ENABLED */
+
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+    /*
+     * Allocate RC release logger instances (finalized in the relocator).
+     */
+    PRTLOGGER pRelLogger = RTLogRelDefaultInstance();
+    if (pRelLogger)
+    {
+        pVM->vmm.s.cbRCRelLogger = RT_OFFSETOF(RTLOGGERRC, afGroups[pRelLogger->cGroups]);
+        rc = MMR3HyperAllocOnceNoRel(pVM, pVM->vmm.s.cbRCRelLogger, 0, MM_TAG_VMM, (void **)&pVM->vmm.s.pRCRelLoggerR3);
+        if (RT_FAILURE(rc))
+            return rc;
+        pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
+    }
+#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMMR3Init worker that register the statistics with STAM.
+ *
+ * @param   pVM         The shared VM structure.
+ */
+static void vmmR3InitRegisterStats(PVM pVM)
+{
+    /*
+     * Statistics.
+     */
+    STAM_REG(pVM, &pVM->vmm.s.StatRunRC,                    STAMTYPE_COUNTER, "/VMM/RunRC",                     STAMUNIT_OCCURENCES, "Number of context switches.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetNormal,              STAMTYPE_COUNTER, "/VMM/RZRet/Normal",              STAMUNIT_OCCURENCES, "Number of VINF_SUCCESS returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterrupt,           STAMTYPE_COUNTER, "/VMM/RZRet/Interrupt",           STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptHyper,      STAMTYPE_COUNTER, "/VMM/RZRet/InterruptHyper",      STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_HYPER returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetGuestTrap,           STAMTYPE_COUNTER, "/VMM/RZRet/GuestTrap",           STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_GUEST_TRAP returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitch,          STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitch",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRingSwitchInt,       STAMTYPE_COUNTER, "/VMM/RZRet/RingSwitchInt",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_RING_SWITCH_INT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetStaleSelector,       STAMTYPE_COUNTER, "/VMM/RZRet/StaleSelector",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_STALE_SELECTOR returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIRETTrap,            STAMTYPE_COUNTER, "/VMM/RZRet/IRETTrap",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_IRET_TRAP returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetEmulate,             STAMTYPE_COUNTER, "/VMM/RZRet/Emulate",             STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOBlockEmulate,      STAMTYPE_COUNTER, "/VMM/RZRet/EmulateIOBlock",      STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_EMULATE_IO_BLOCK returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchEmulate,        STAMTYPE_COUNTER, "/VMM/RZRet/PatchEmulate",        STAMUNIT_OCCURENCES, "Number of VINF_PATCH_EMULATE_INSTR returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIORead,              STAMTYPE_COUNTER, "/VMM/RZRet/IORead",              STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_READ returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIOWrite,             STAMTYPE_COUNTER, "/VMM/RZRet/IOWrite",             STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_IOPORT_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIORead,            STAMTYPE_COUNTER, "/VMM/RZRet/MMIORead",            STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOWrite,           STAMTYPE_COUNTER, "/VMM/RZRet/MMIOWrite",           STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOReadWrite,       STAMTYPE_COUNTER, "/VMM/RZRet/MMIOReadWrite",       STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_READ_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchRead,       STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchRead",       STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_READ returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMMIOPatchWrite,      STAMTYPE_COUNTER, "/VMM/RZRet/MMIOPatchWrite",      STAMUNIT_OCCURENCES, "Number of VINF_IOM_HC_MMIO_PATCH_WRITE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetLDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/LDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_GDT_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetGDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/GDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_LDT_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetIDTFault,            STAMTYPE_COUNTER, "/VMM/RZRet/IDTFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_IDT_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetTSSFault,            STAMTYPE_COUNTER, "/VMM/RZRet/TSSFault",            STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_TSS_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPDFault,             STAMTYPE_COUNTER, "/VMM/RZRet/PDFault",             STAMUNIT_OCCURENCES, "Number of VINF_EM_EXECUTE_INSTRUCTION_PD_FAULT returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetCSAMTask,            STAMTYPE_COUNTER, "/VMM/RZRet/CSAMTask",            STAMUNIT_OCCURENCES, "Number of VINF_CSAM_PENDING_ACTION returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetSyncCR3,             STAMTYPE_COUNTER, "/VMM/RZRet/SyncCR",              STAMUNIT_OCCURENCES, "Number of VINF_PGM_SYNC_CR3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetMisc,                STAMTYPE_COUNTER, "/VMM/RZRet/Misc",                STAMUNIT_OCCURENCES, "Number of misc returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchInt3,           STAMTYPE_COUNTER, "/VMM/RZRet/PatchInt3",           STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_INT3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchPF,             STAMTYPE_COUNTER, "/VMM/RZRet/PatchPF",             STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_PF returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchGP,             STAMTYPE_COUNTER, "/VMM/RZRet/PatchGP",             STAMUNIT_OCCURENCES, "Number of VINF_PATM_PATCH_TRAP_GP returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchIretIRQ,        STAMTYPE_COUNTER, "/VMM/RZRet/PatchIret",           STAMUNIT_OCCURENCES, "Number of VINF_PATM_PENDING_IRQ_AFTER_IRET returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetRescheduleREM,       STAMTYPE_COUNTER, "/VMM/RZRet/ScheduleREM",         STAMUNIT_OCCURENCES, "Number of VINF_EM_RESCHEDULE_REM returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3,                STAMTYPE_COUNTER, "/VMM/RZRet/ToR3",                STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Unknown,         STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Unknown",        STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3TMVirt,          STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/TMVirt",         STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3HandyPages,      STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Handy",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3PDMQueues,       STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/PDMQueue",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Rendezvous,      STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Rendezvous",     STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3Timer,           STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/Timer",          STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3DMA,             STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/DMA",            STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetToR3CritSect,        STAMTYPE_COUNTER, "/VMM/RZRet/ToR3/CritSect",       STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TO_R3 returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetTimerPending,        STAMTYPE_COUNTER, "/VMM/RZRet/TimerPending",        STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_TIMER_PENDING returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetInterruptPending,    STAMTYPE_COUNTER, "/VMM/RZRet/InterruptPending",    STAMUNIT_OCCURENCES, "Number of VINF_EM_RAW_INTERRUPT_PENDING returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPATMDuplicateFn,     STAMTYPE_COUNTER, "/VMM/RZRet/PATMDuplicateFn",     STAMUNIT_OCCURENCES, "Number of VINF_PATM_DUPLICATE_FUNCTION returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMChangeMode,       STAMTYPE_COUNTER, "/VMM/RZRet/PGMChangeMode",       STAMUNIT_OCCURENCES, "Number of VINF_PGM_CHANGE_MODE returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPGMFlushPending,     STAMTYPE_COUNTER, "/VMM/RZRet/PGMFlushPending",     STAMUNIT_OCCURENCES, "Number of VINF_PGM_POOL_FLUSH_PENDING returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPendingRequest,      STAMTYPE_COUNTER, "/VMM/RZRet/PendingRequest",      STAMUNIT_OCCURENCES, "Number of VINF_EM_PENDING_REQUEST returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetPatchTPR,            STAMTYPE_COUNTER, "/VMM/RZRet/PatchTPR",            STAMUNIT_OCCURENCES, "Number of VINF_EM_HWACCM_PATCH_TPR_INSTR returns.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZRetCallRing3,           STAMTYPE_COUNTER, "/VMM/RZCallR3/Misc",             STAMUNIT_OCCURENCES, "Number of Other ring-3 calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPDMLock,            STAMTYPE_COUNTER, "/VMM/RZCallR3/PDMLock",          STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PDM_LOCK calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMLock,            STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMLock",          STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_LOCK calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMPoolGrow,        STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMPoolGrow",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_POOL_GROW calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMMapChunk,        STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMMapChunk",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_MAP_CHUNK calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallPGMAllocHandy,      STAMTYPE_COUNTER, "/VMM/RZCallR3/PGMAllocHandy",    STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallRemReplay,          STAMTYPE_COUNTER, "/VMM/RZCallR3/REMReplay",        STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallLogFlush,           STAMTYPE_COUNTER, "/VMM/RZCallR3/VMMLogFlush",      STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VMM_LOGGER_FLUSH calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetError,         STAMTYPE_COUNTER, "/VMM/RZCallR3/VMSetError",       STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_ERROR calls.");
+    STAM_REG(pVM, &pVM->vmm.s.StatRZCallVMSetRuntimeError,  STAMTYPE_COUNTER, "/VMM/RZCallR3/VMRuntimeError",   STAMUNIT_OCCURENCES, "Number of VMMCALLRING3_VM_SET_RUNTIME_ERROR calls.");
+
+#ifdef VBOX_WITH_STATISTICS
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedMax,  STAMTYPE_U32_RESET, STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,      "Max amount of stack used.", "/VMM/Stack/CPU%u/Max", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cbUsedAvg,  STAMTYPE_U32,       STAMVISIBILITY_ALWAYS, STAMUNIT_BYTES,      "Average stack usage.",      "/VMM/Stack/CPU%u/Avg", i);
+        STAMR3RegisterF(pVM, &pVM->aCpus[i].vmm.s.CallRing3JmpBufR0.cUsedTotal, STAMTYPE_U64,       STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, "Number of stack usages.",   "/VMM/Stack/CPU%u/Uses", i);
+    }
+#endif
+}
+
+
+/**
+ * Initializes the R0 VMM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3_INT_DECL(int) VMMR3InitR0(PVM pVM)
+{
+    int    rc;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu && pVCpu->idCpu == 0);
+
+#ifdef LOG_ENABLED
+    /*
+     * Initialize the ring-0 logger if we haven't done so yet.
+     */
+    if (    pVCpu->vmm.s.pR0LoggerR3
+        &&  !pVCpu->vmm.s.pR0LoggerR3->fCreated)
+    {
+        rc = VMMR3UpdateLoggers(pVM);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+#endif
+
+    /*
+     * Call Ring-0 entry with init code.
+     */
+    for (;;)
+    {
+#ifdef NO_SUPCALLR0VMM
+        //rc = VERR_GENERAL_FAILURE;
+        rc = VINF_SUCCESS;
+#else
+        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_VMMR0_INIT, VMMGetSvnRev(), NULL);
+#endif
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        if (    pVCpu->vmm.s.pR0LoggerR3
+            &&  pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
+            RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
+#endif
+        if (rc != VINF_VMM_CALL_HOST)
+            break;
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+            break;
+        /* Resume R0 */
+    }
+
+    if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+    {
+        LogRel(("R0 init failed, rc=%Rra\n", rc));
+        if (RT_SUCCESS(rc))
+            rc = VERR_INTERNAL_ERROR;
+    }
+    return rc;
+}
+
+
+/**
+ * Initializes the RC VMM.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ */
+VMMR3_INT_DECL(int) VMMR3InitRC(PVM pVM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu && pVCpu->idCpu == 0);
+
+    /* In VMX mode, there's no need to init RC. */
+    if (pVM->vmm.s.fSwitcherDisabled)
+        return VINF_SUCCESS;
+
+    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
+
+    /*
+     * Call VMMGCInit():
+     *      -# resolve the address.
+     *      -# setup stackframe and EIP to use the trampoline.
+     *      -# do a generic hypervisor call.
+     */
+    RTRCPTR RCPtrEP;
+    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
+    if (RT_SUCCESS(rc))
+    {
+        CPUMHyperSetCtxCore(pVCpu, NULL);
+        CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
+        uint64_t u64TS = RTTimeProgramStartNanoTS();
+        CPUMPushHyper(pVCpu, (uint32_t)(u64TS >> 32));    /* Param 3: The program startup TS - Hi. */
+        CPUMPushHyper(pVCpu, (uint32_t)u64TS);            /* Param 3: The program startup TS - Lo. */
+        CPUMPushHyper(pVCpu, VMMGetSvnRev());             /* Param 2: Version argument. */
+        CPUMPushHyper(pVCpu, VMMGC_DO_VMMGC_INIT);        /* Param 1: Operation. */
+        CPUMPushHyper(pVCpu, pVM->pVMRC);                 /* Param 0: pVM */
+        CPUMPushHyper(pVCpu, 5 * sizeof(RTRCPTR));        /* trampoline param: stacksize.  */
+        CPUMPushHyper(pVCpu, RCPtrEP);                    /* Call EIP. */
+        CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
+        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
+
+        for (;;)
+        {
+#ifdef NO_SUPCALLR0VMM
+            //rc = VERR_GENERAL_FAILURE;
+            rc = VINF_SUCCESS;
+#else
+            rc = SUPR3CallVMMR0(pVM->pVMR0, 0 /* VCPU 0 */, VMMR0_DO_CALL_HYPERVISOR, NULL);
+#endif
+#ifdef LOG_ENABLED
+            PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
+            if (    pLogger
+                &&  pLogger->offScratch > 0)
+                RTLogFlushRC(NULL, pLogger);
+#endif
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+            PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
+            if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
+                RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
+#endif
+            if (rc != VINF_VMM_CALL_HOST)
+                break;
+            rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+            if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+                break;
+        }
+
+        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+        {
+            VMMR3FatalDump(pVM, pVCpu, rc);
+            if (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST)
+                rc = VERR_INTERNAL_ERROR;
+        }
+        AssertRC(rc);
+    }
+    return rc;
+}
+
+
+/**
+ * Called when an init phase completes.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The VM handle.
+ * @param   enmWhat             Which init phase.
+ */
+VMMR3_INT_DECL(int) VMMR3InitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
+{
+    int rc = VINF_SUCCESS;
+
+    switch (enmWhat)
+    {
+        case VMINITCOMPLETED_RING3:
+        {
+            /*
+             * Set page attributes to r/w for stack pages.
+             */
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            {
+                rc = PGMMapSetPage(pVM, pVM->aCpus[idCpu].vmm.s.pbEMTStackRC, VMM_STACK_SIZE,
+                                   X86_PTE_P | X86_PTE_A | X86_PTE_D | X86_PTE_RW);
+                AssertRCReturn(rc, rc);
+            }
+
+            /*
+             * Create the EMT yield timer.
+             */
+            rc = TMR3TimerCreateInternal(pVM, TMCLOCK_REAL, vmmR3YieldEMT, NULL, "EMT Yielder", &pVM->vmm.s.pYieldTimer);
+            AssertRCReturn(rc, rc);
+
+            rc = TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldEveryMillies);
+            AssertRCReturn(rc, rc);
+
+#ifdef VBOX_WITH_NMI
+            /*
+             * Map the host APIC into GC - This is AMD/Intel + Host OS specific!
+             */
+            rc = PGMMap(pVM, pVM->vmm.s.GCPtrApicBase, 0xfee00000, PAGE_SIZE,
+                        X86_PTE_P | X86_PTE_RW | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A | X86_PTE_D);
+            AssertRCReturn(rc, rc);
+#endif
+
+#ifdef VBOX_STRICT_VMM_STACK
+            /*
+             * Setup the stack guard pages: Two inaccessible pages at each sides of the
+             * stack to catch over/under-flows.
+             */
+            for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+            {
+                uint8_t *pbEMTStackR3 = pVM->aCpus[idCpu].vmm.s.pbEMTStackR3;
+
+                memset(pbEMTStackR3 - PAGE_SIZE, 0xcc, PAGE_SIZE);
+                MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE, PAGE_SIZE, true /*fSet*/);
+
+                memset(pbEMTStackR3 + VMM_STACK_SIZE, 0xcc, PAGE_SIZE);
+                MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, true /*fSet*/);
+            }
+            pVM->vmm.s.fStackGuardsStationed = true;
+#endif
+            break;
+        }
+
+        case VMINITCOMPLETED_RING0:
+        {
+            /*
+             * Disable the periodic preemption timers if we can use the
+             * VMX-preemption timer instead.
+             */
+            if (   pVM->vmm.s.fUsePeriodicPreemptionTimers
+                && HWACCMR3IsVmxPreemptionTimerUsed(pVM))
+                pVM->vmm.s.fUsePeriodicPreemptionTimers = false;
+            LogRel(("VMM: fUsePeriodicPreemptionTimers=%RTbool\n", pVM->vmm.s.fUsePeriodicPreemptionTimers));
+            break;
+        }
+
+        default: /* shuts up gcc */
+            break;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Terminate the VMM bits.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVM         The VM handle.
+ */
+VMMR3_INT_DECL(int) VMMR3Term(PVM pVM)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu && pVCpu->idCpu == 0);
+
+    /*
+     * Call Ring-0 entry with termination code.
+     */
+    int rc;
+    for (;;)
+    {
+#ifdef NO_SUPCALLR0VMM
+        //rc = VERR_GENERAL_FAILURE;
+        rc = VINF_SUCCESS;
+#else
+        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, 0 /*idCpu*/, VMMR0_DO_VMMR0_TERM, 0, NULL);
+#endif
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        if (    pVCpu->vmm.s.pR0LoggerR3
+            &&  pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
+            RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
+#endif
+        if (rc != VINF_VMM_CALL_HOST)
+            break;
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+            break;
+        /* Resume R0 */
+    }
+    if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+    {
+        LogRel(("VMMR3Term: R0 term failed, rc=%Rra. (warning)\n", rc));
+        if (RT_SUCCESS(rc))
+            rc = VERR_INTERNAL_ERROR;
+    }
+
+    RTCritSectDelete(&pVM->vmm.s.CritSectSync);
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        RTSemEventDestroy(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i]);
+        pVM->vmm.s.pahEvtRendezvousEnterOrdered[i] = NIL_RTSEMEVENT;
+    }
+    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
+    pVM->vmm.s.hEvtRendezvousEnterOneByOne = NIL_RTSEMEVENT;
+    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
+    pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce = NIL_RTSEMEVENTMULTI;
+    RTSemEventMultiDestroy(pVM->vmm.s.hEvtMulRendezvousDone);
+    pVM->vmm.s.hEvtMulRendezvousDone = NIL_RTSEMEVENTMULTI;
+    RTSemEventDestroy(pVM->vmm.s.hEvtRendezvousDoneCaller);
+    pVM->vmm.s.hEvtRendezvousDoneCaller = NIL_RTSEMEVENT;
+
+#ifdef VBOX_STRICT_VMM_STACK
+    /*
+     * Make the two stack guard pages present again.
+     */
+    if (pVM->vmm.s.fStackGuardsStationed)
+    {
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            uint8_t *pbEMTStackR3 = pVM->aCpus[i].vmm.s.pbEMTStackR3;
+            MMR3HyperSetGuard(pVM, pbEMTStackR3 - PAGE_SIZE,      PAGE_SIZE, false /*fSet*/);
+            MMR3HyperSetGuard(pVM, pbEMTStackR3 + VMM_STACK_SIZE, PAGE_SIZE, false /*fSet*/);
+        }
+        pVM->vmm.s.fStackGuardsStationed = false;
+    }
+#endif
+    return rc;
+}
+
+
+/**
+ * Applies relocations to data and code managed by this
+ * component. This function will be called at init and
+ * whenever the VMM need to relocate it self inside the GC.
+ *
+ * The VMM will need to apply relocations to the core code.
+ *
+ * @param   pVM         The VM handle.
+ * @param   offDelta    The relocation delta.
+ */
+VMMR3_INT_DECL(void) VMMR3Relocate(PVM pVM, RTGCINTPTR offDelta)
+{
+    LogFlow(("VMMR3Relocate: offDelta=%RGv\n", offDelta));
+
+    /*
+     * Recalc the RC address.
+     */
+#ifdef VBOX_WITH_RAW_MODE
+    pVM->vmm.s.pvCoreCodeRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pvCoreCodeR3);
+#endif
+
+    /*
+     * The stack.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU pVCpu = &pVM->aCpus[i];
+
+        CPUMSetHyperESP(pVCpu, CPUMGetHyperESP(pVCpu) + offDelta);
+
+        pVCpu->vmm.s.pbEMTStackRC       = MMHyperR3ToRC(pVM, pVCpu->vmm.s.pbEMTStackR3);
+        pVCpu->vmm.s.pbEMTStackBottomRC = pVCpu->vmm.s.pbEMTStackRC + VMM_STACK_SIZE;
+    }
+
+    /*
+     * All the switchers.
+     */
+    vmmR3SwitcherRelocate(pVM, offDelta);
+
+    /*
+     * Get other RC entry points.
+     */
+    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuest", &pVM->vmm.s.pfnCPUMRCResumeGuest);
+    AssertReleaseMsgRC(rc, ("CPUMGCResumeGuest not found! rc=%Rra\n", rc));
+
+    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "CPUMGCResumeGuestV86", &pVM->vmm.s.pfnCPUMRCResumeGuestV86);
+    AssertReleaseMsgRC(rc, ("CPUMGCResumeGuestV86 not found! rc=%Rra\n", rc));
+
+    /*
+     * Update the logger.
+     */
+    VMMR3UpdateLoggers(pVM);
+}
+
+
+/**
+ * Updates the settings for the RC and R0 loggers.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+VMMR3_INT_DECL(int) VMMR3UpdateLoggers(PVM pVM)
+{
+    /*
+     * Simply clone the logger instance (for RC).
+     */
+    int rc = VINF_SUCCESS;
+    RTRCPTR RCPtrLoggerFlush = 0;
+
+    if (pVM->vmm.s.pRCLoggerR3
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+        || pVM->vmm.s.pRCRelLoggerR3
+#endif
+       )
+    {
+        rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerFlush", &RCPtrLoggerFlush);
+        AssertReleaseMsgRC(rc, ("vmmGCLoggerFlush not found! rc=%Rra\n", rc));
+    }
+
+    if (pVM->vmm.s.pRCLoggerR3)
+    {
+        RTRCPTR RCPtrLoggerWrapper = 0;
+        rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCLoggerWrapper", &RCPtrLoggerWrapper);
+        AssertReleaseMsgRC(rc, ("vmmGCLoggerWrapper not found! rc=%Rra\n", rc));
+
+        pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
+        rc = RTLogCloneRC(NULL /* default */, pVM->vmm.s.pRCLoggerR3, pVM->vmm.s.cbRCLogger,
+                          RCPtrLoggerWrapper,  RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
+        AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
+    }
+
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+    if (pVM->vmm.s.pRCRelLoggerR3)
+    {
+        RTRCPTR RCPtrLoggerWrapper = 0;
+        rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "vmmGCRelLoggerWrapper", &RCPtrLoggerWrapper);
+        AssertReleaseMsgRC(rc, ("vmmGCRelLoggerWrapper not found! rc=%Rra\n", rc));
+
+        pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
+        rc = RTLogCloneRC(RTLogRelDefaultInstance(), pVM->vmm.s.pRCRelLoggerR3, pVM->vmm.s.cbRCRelLogger,
+                          RCPtrLoggerWrapper,  RCPtrLoggerFlush, RTLOGFLAGS_BUFFERED);
+        AssertReleaseMsgRC(rc, ("RTLogCloneRC failed! rc=%Rra\n", rc));
+    }
+#endif /* VBOX_WITH_RC_RELEASE_LOGGING */
+
+#ifdef LOG_ENABLED
+    /*
+     * For the ring-0 EMT logger, we use a per-thread logger instance
+     * in ring-0. Only initialize it once.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        PVMCPU       pVCpu = &pVM->aCpus[i];
+        PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
+        if (pR0LoggerR3)
+        {
+            if (!pR0LoggerR3->fCreated)
+            {
+                RTR0PTR pfnLoggerWrapper = NIL_RTR0PTR;
+                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerWrapper", &pfnLoggerWrapper);
+                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerWrapper not found! rc=%Rra\n", rc), rc);
+
+                RTR0PTR pfnLoggerFlush = NIL_RTR0PTR;
+                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerFlush", &pfnLoggerFlush);
+                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerFlush not found! rc=%Rra\n", rc), rc);
+
+                rc = RTLogCreateForR0(&pR0LoggerR3->Logger, pR0LoggerR3->cbLogger,
+                                      *(PFNRTLOGGER *)&pfnLoggerWrapper, *(PFNRTLOGFLUSH *)&pfnLoggerFlush,
+                                      RTLOGFLAGS_BUFFERED, RTLOGDEST_DUMMY);
+                AssertReleaseMsgRCReturn(rc, ("RTLogCreateForR0 failed! rc=%Rra\n", rc), rc);
+
+                RTR0PTR pfnLoggerPrefix = NIL_RTR0PTR;
+                rc = PDMR3LdrGetSymbolR0(pVM, VMMR0_MAIN_MODULE_NAME, "vmmR0LoggerPrefix", &pfnLoggerPrefix);
+                AssertReleaseMsgRCReturn(rc, ("vmmR0LoggerPrefix not found! rc=%Rra\n", rc), rc);
+                rc = RTLogSetCustomPrefixCallback(&pR0LoggerR3->Logger, *(PFNRTLOGPREFIX *)&pfnLoggerPrefix, NULL);
+                AssertReleaseMsgRCReturn(rc, ("RTLogSetCustomPrefixCallback failed! rc=%Rra\n", rc), rc);
+
+                pR0LoggerR3->idCpu = i;
+                pR0LoggerR3->fCreated = true;
+                pR0LoggerR3->fFlushingDisabled = false;
+
+            }
+
+            rc = RTLogCopyGroupsAndFlags(&pR0LoggerR3->Logger, NULL /* default */, pVM->vmm.s.pRCLoggerR3->fFlags, RTLOGFLAGS_BUFFERED);
+            AssertRC(rc);
+        }
+    }
+#endif
+    return rc;
+}
+
+
+/**
+ * Gets the pointer to a buffer containing the R0/RC RTAssertMsg1Weak output.
+ *
+ * @returns Pointer to the buffer.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(const char *) VMMR3GetRZAssertMsg1(PVM pVM)
+{
+    if (HWACCMIsEnabled(pVM))
+        return pVM->vmm.s.szRing0AssertMsg1;
+
+    RTRCPTR RCPtr;
+    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg1", &RCPtr);
+    if (RT_SUCCESS(rc))
+        return (const char *)MMHyperRCToR3(pVM, RCPtr);
+
+    return NULL;
+}
+
+
+/**
+ * Gets the pointer to a buffer containing the R0/RC RTAssertMsg2Weak output.
+ *
+ * @returns Pointer to the buffer.
+ * @param   pVM         The VM handle.
+ */
+VMMR3DECL(const char *) VMMR3GetRZAssertMsg2(PVM pVM)
+{
+    if (HWACCMIsEnabled(pVM))
+        return pVM->vmm.s.szRing0AssertMsg2;
+
+    RTRCPTR RCPtr;
+    int rc = PDMR3LdrGetSymbolRC(pVM, NULL, "g_szRTAssertMsg2", &RCPtr);
+    if (RT_SUCCESS(rc))
+        return (const char *)MMHyperRCToR3(pVM, RCPtr);
+
+    return NULL;
+}
+
+
+/**
+ * Execute state save operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ */
+static DECLCALLBACK(int) vmmR3Save(PVM pVM, PSSMHANDLE pSSM)
+{
+    LogFlow(("vmmR3Save:\n"));
+
+    /*
+     * Save the started/stopped state of all CPUs except 0 as it will always
+     * be running. This avoids breaking the saved state version. :-)
+     */
+    for (VMCPUID i = 1; i < pVM->cCpus; i++)
+        SSMR3PutBool(pSSM, VMCPUSTATE_IS_STARTED(VMCPU_GET_STATE(&pVM->aCpus[i])));
+
+    return SSMR3PutU32(pSSM, UINT32_MAX); /* terminator */
+}
+
+
+/**
+ * Execute state load operation.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM Handle.
+ * @param   pSSM            SSM operation handle.
+ * @param   uVersion        Data layout version.
+ * @param   uPass           The data pass.
+ */
+static DECLCALLBACK(int) vmmR3Load(PVM pVM, PSSMHANDLE pSSM, uint32_t uVersion, uint32_t uPass)
+{
+    LogFlow(("vmmR3Load:\n"));
+    Assert(uPass == SSM_PASS_FINAL); NOREF(uPass);
+
+    /*
+     * Validate version.
+     */
+    if (    uVersion != VMM_SAVED_STATE_VERSION
+        &&  uVersion != VMM_SAVED_STATE_VERSION_3_0)
+    {
+        AssertMsgFailed(("vmmR3Load: Invalid version uVersion=%u!\n", uVersion));
+        return VERR_SSM_UNSUPPORTED_DATA_UNIT_VERSION;
+    }
+
+    if (uVersion <= VMM_SAVED_STATE_VERSION_3_0)
+    {
+        /* Ignore the stack bottom, stack pointer and stack bits. */
+        RTRCPTR RCPtrIgnored;
+        SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
+        SSMR3GetRCPtr(pSSM, &RCPtrIgnored);
+#ifdef RT_OS_DARWIN
+        if (   SSMR3HandleVersion(pSSM)  >= VBOX_FULL_VERSION_MAKE(3,0,0)
+            && SSMR3HandleVersion(pSSM)  <  VBOX_FULL_VERSION_MAKE(3,1,0)
+            && SSMR3HandleRevision(pSSM) >= 48858
+            && (   !strcmp(SSMR3HandleHostOSAndArch(pSSM), "darwin.x86")
+                || !strcmp(SSMR3HandleHostOSAndArch(pSSM), "") )
+           )
+            SSMR3Skip(pSSM, 16384);
+        else
+            SSMR3Skip(pSSM, 8192);
+#else
+        SSMR3Skip(pSSM, 8192);
+#endif
+    }
+
+    /*
+     * Restore the VMCPU states. VCPU 0 is always started.
+     */
+    VMCPU_SET_STATE(&pVM->aCpus[0], VMCPUSTATE_STARTED);
+    for (VMCPUID i = 1; i < pVM->cCpus; i++)
+    {
+        bool fStarted;
+        int rc = SSMR3GetBool(pSSM, &fStarted);
+        if (RT_FAILURE(rc))
+            return rc;
+        VMCPU_SET_STATE(&pVM->aCpus[i], fStarted ? VMCPUSTATE_STARTED : VMCPUSTATE_STOPPED);
+    }
+
+    /* terminator */
+    uint32_t u32;
+    int rc = SSMR3GetU32(pSSM, &u32);
+    if (RT_FAILURE(rc))
+        return rc;
+    if (u32 != UINT32_MAX)
+    {
+        AssertMsgFailed(("u32=%#x\n", u32));
+        return VERR_SSM_DATA_UNIT_FORMAT_CHANGED;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resolve a builtin RC symbol.
+ *
+ * Called by PDM when loading or relocating RC modules.
+ *
+ * @returns VBox status
+ * @param   pVM             VM Handle.
+ * @param   pszSymbol       Symbol to resolv
+ * @param   pRCPtrValue     Where to store the symbol value.
+ *
+ * @remark  This has to work before VMMR3Relocate() is called.
+ */
+VMMR3_INT_DECL(int) VMMR3GetImportRC(PVM pVM, const char *pszSymbol, PRTRCPTR pRCPtrValue)
+{
+    if (!strcmp(pszSymbol, "g_Logger"))
+    {
+        if (pVM->vmm.s.pRCLoggerR3)
+            pVM->vmm.s.pRCLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCLoggerR3);
+        *pRCPtrValue = pVM->vmm.s.pRCLoggerRC;
+    }
+    else if (!strcmp(pszSymbol, "g_RelLogger"))
+    {
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+        if (pVM->vmm.s.pRCRelLoggerR3)
+            pVM->vmm.s.pRCRelLoggerRC = MMHyperR3ToRC(pVM, pVM->vmm.s.pRCRelLoggerR3);
+        *pRCPtrValue = pVM->vmm.s.pRCRelLoggerRC;
+#else
+        *pRCPtrValue = NIL_RTRCPTR;
+#endif
+    }
+    else
+        return VERR_SYMBOL_NOT_FOUND;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Suspends the CPU yielder.
+ *
+ * @param   pVM             The VM handle.
+ */
+VMMR3_INT_DECL(void) VMMR3YieldSuspend(PVM pVM)
+{
+    VMCPU_ASSERT_EMT(&pVM->aCpus[0]);
+    if (!pVM->vmm.s.cYieldResumeMillies)
+    {
+        uint64_t u64Now = TMTimerGet(pVM->vmm.s.pYieldTimer);
+        uint64_t u64Expire = TMTimerGetExpire(pVM->vmm.s.pYieldTimer);
+        if (u64Now >= u64Expire || u64Expire == ~(uint64_t)0)
+            pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
+        else
+            pVM->vmm.s.cYieldResumeMillies = TMTimerToMilli(pVM->vmm.s.pYieldTimer, u64Expire - u64Now);
+        TMTimerStop(pVM->vmm.s.pYieldTimer);
+    }
+    pVM->vmm.s.u64LastYield = RTTimeNanoTS();
+}
+
+
+/**
+ * Stops the CPU yielder.
+ *
+ * @param   pVM             The VM handle.
+ */
+VMMR3_INT_DECL(void) VMMR3YieldStop(PVM pVM)
+{
+    if (!pVM->vmm.s.cYieldResumeMillies)
+        TMTimerStop(pVM->vmm.s.pYieldTimer);
+    pVM->vmm.s.cYieldResumeMillies = pVM->vmm.s.cYieldEveryMillies;
+    pVM->vmm.s.u64LastYield = RTTimeNanoTS();
+}
+
+
+/**
+ * Resumes the CPU yielder when it has been a suspended or stopped.
+ *
+ * @param   pVM             The VM handle.
+ */
+VMMR3_INT_DECL(void) VMMR3YieldResume(PVM pVM)
+{
+    if (pVM->vmm.s.cYieldResumeMillies)
+    {
+        TMTimerSetMillies(pVM->vmm.s.pYieldTimer, pVM->vmm.s.cYieldResumeMillies);
+        pVM->vmm.s.cYieldResumeMillies = 0;
+    }
+}
+
+
+/**
+ * Internal timer callback function.
+ *
+ * @param   pVM             The VM.
+ * @param   pTimer          The timer handle.
+ * @param   pvUser          User argument specified upon timer creation.
+ */
+static DECLCALLBACK(void) vmmR3YieldEMT(PVM pVM, PTMTIMER pTimer, void *pvUser)
+{
+    /*
+     * This really needs some careful tuning. While we shouldn't be too greedy since
+     * that'll cause the rest of the system to stop up, we shouldn't be too nice either
+     * because that'll cause us to stop up.
+     *
+     * The current logic is to use the default interval when there is no lag worth
+     * mentioning, but when we start accumulating lag we don't bother yielding at all.
+     *
+     * (This depends on the TMCLOCK_VIRTUAL_SYNC to be scheduled before TMCLOCK_REAL
+     * so the lag is up to date.)
+     */
+    const uint64_t u64Lag = TMVirtualSyncGetLag(pVM);
+    if (    u64Lag     <   50000000 /* 50ms */
+        ||  (   u64Lag < 1000000000 /*  1s */
+             && RTTimeNanoTS() - pVM->vmm.s.u64LastYield < 500000000 /* 500 ms */)
+       )
+    {
+        uint64_t u64Elapsed = RTTimeNanoTS();
+        pVM->vmm.s.u64LastYield = u64Elapsed;
+
+        RTThreadYield();
+
+#ifdef LOG_ENABLED
+        u64Elapsed = RTTimeNanoTS() - u64Elapsed;
+        Log(("vmmR3YieldEMT: %RI64 ns\n", u64Elapsed));
+#endif
+    }
+    TMTimerSetMillies(pTimer, pVM->vmm.s.cYieldEveryMillies);
+}
+
+
+/**
+ * Executes guest code in the raw-mode context.
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       The VMCPU to operate on.
+ */
+VMMR3_INT_DECL(int) VMMR3RawRunGC(PVM pVM, PVMCPU pVCpu)
+{
+    Log2(("VMMR3RawRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
+
+    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
+
+    /*
+     * Set the EIP and ESP.
+     */
+    CPUMSetHyperEIP(pVCpu, CPUMGetGuestEFlags(pVCpu) & X86_EFL_VM
+                    ? pVM->vmm.s.pfnCPUMRCResumeGuestV86
+                    : pVM->vmm.s.pfnCPUMRCResumeGuest);
+    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC);
+
+    /*
+     * We hide log flushes (outer) and hypervisor interrupts (inner).
+     */
+    for (;;)
+    {
+#ifdef VBOX_STRICT
+        if (RT_UNLIKELY(!CPUMGetHyperCR3(pVCpu) || CPUMGetHyperCR3(pVCpu) != PGMGetHyperCR3(pVCpu)))
+            EMR3FatalError(pVCpu, VERR_VMM_HYPER_CR3_MISMATCH);
+        PGMMapCheck(pVM);
+#endif
+        int rc;
+        do
+        {
+#ifdef NO_SUPCALLR0VMM
+            rc = VERR_GENERAL_FAILURE;
+#else
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                rc = pVCpu->vmm.s.iLastGZRc;
+#endif
+        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
+
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
+        if (    pLogger
+            &&  pLogger->offScratch > 0)
+            RTLogFlushRC(NULL, pLogger);
+#endif
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+        PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
+        if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
+            RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
+#endif
+        if (rc != VINF_VMM_CALL_HOST)
+        {
+            Log2(("VMMR3RawRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
+            return rc;
+        }
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+        /* Resume GC */
+    }
+}
+
+
+/**
+ * Executes guest code (Intel VT-x and AMD-V).
+ *
+ * @param   pVM         VM handle.
+ * @param   pVCpu       The VMCPU to operate on.
+ */
+VMMR3_INT_DECL(int) VMMR3HwAccRunGC(PVM pVM, PVMCPU pVCpu)
+{
+    Log2(("VMMR3HwAccRunGC: (cs:eip=%04x:%08x)\n", CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
+
+    for (;;)
+    {
+        int rc;
+        do
+        {
+#ifdef NO_SUPCALLR0VMM
+            rc = VERR_GENERAL_FAILURE;
+#else
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HWACC_RUN, pVCpu->idCpu);
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                rc = pVCpu->vmm.s.iLastGZRc;
+#endif
+        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
+
+#if 0 /* todo triggers too often */
+        Assert(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TO_R3));
+#endif
+
+#ifdef LOG_ENABLED
+        /*
+         * Flush the log
+         */
+        PVMMR0LOGGER pR0LoggerR3 = pVCpu->vmm.s.pR0LoggerR3;
+        if (    pR0LoggerR3
+            &&  pR0LoggerR3->Logger.offScratch > 0)
+            RTLogFlushToLogger(&pR0LoggerR3->Logger, NULL);
+#endif /* !LOG_ENABLED */
+        if (rc != VINF_VMM_CALL_HOST)
+        {
+            Log2(("VMMR3HwAccRunGC: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
+            return rc;
+        }
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+        /* Resume R0 */
+    }
+}
+
+/**
+ * VCPU worker for VMMSendSipi.
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   idCpu       Virtual CPU to perform SIPI on
+ * @param   uVector     SIPI vector
+ */
+DECLCALLBACK(int) vmmR3SendSipi(PVM pVM, VMCPUID idCpu, uint32_t uVector)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /** @todo what are we supposed to do if the processor is already running? */
+    if (EMGetState(pVCpu) != EMSTATE_WAIT_SIPI)
+        return VERR_ACCESS_DENIED;
+
+
+    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+
+    pCtx->cs                        = uVector << 8;
+    pCtx->csHid.u64Base             = uVector << 12;
+    pCtx->csHid.u32Limit            = 0x0000ffff;
+    pCtx->rip                       = 0;
+
+    Log(("vmmR3SendSipi for VCPU %d with vector %x\n", uVector));
+
+# if 1 /* If we keep the EMSTATE_WAIT_SIPI method, then move this to EM.cpp. */
+    EMSetState(pVCpu, EMSTATE_HALTED);
+    return VINF_EM_RESCHEDULE;
+# else /* And if we go the VMCPU::enmState way it can stay here. */
+    VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STOPPED);
+    VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
+    return VINF_SUCCESS;
+# endif
+}
+
+DECLCALLBACK(int) vmmR3SendInitIpi(PVM pVM, VMCPUID idCpu)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log(("vmmR3SendInitIpi for VCPU %d\n", idCpu));
+    CPUMR3ResetCpu(pVCpu);
+    return VINF_EM_WAIT_SIPI;
+}
+
+/**
+ * Sends SIPI to the virtual CPU by setting CS:EIP into vector-dependent state
+ * and unhalting processor
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   idCpu       Virtual CPU to perform SIPI on
+ * @param   uVector     SIPI vector
+ */
+VMMR3_INT_DECL(void) VMMR3SendSipi(PVM pVM, VMCPUID idCpu,  uint32_t uVector)
+{
+    AssertReturnVoid(idCpu < pVM->cCpus);
+
+    int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendSipi, 3, pVM, idCpu, uVector);
+    AssertRC(rc);
+}
+
+/**
+ * Sends init IPI to the virtual CPU.
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   idCpu       Virtual CPU to perform int IPI on
+ */
+VMMR3_INT_DECL(void) VMMR3SendInitIpi(PVM pVM, VMCPUID idCpu)
+{
+    AssertReturnVoid(idCpu < pVM->cCpus);
+
+    int rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SendInitIpi, 2, pVM, idCpu);
+    AssertRC(rc);
+}
+
+/**
+ * Registers the guest memory range that can be used for patching
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pPatchMem   Patch memory range
+ * @param   cbPatchMem  Size of the memory range
+ */
+VMMR3DECL(int) VMMR3RegisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    if (HWACCMIsEnabled(pVM))
+        return HWACMMR3EnablePatching(pVM, pPatchMem, cbPatchMem);
+
+    return VERR_NOT_SUPPORTED;
+}
+
+/**
+ * Deregisters the guest memory range that can be used for patching
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   pPatchMem   Patch memory range
+ * @param   cbPatchMem  Size of the memory range
+ */
+VMMR3DECL(int) VMMR3DeregisterPatchMemory(PVM pVM, RTGCPTR pPatchMem, unsigned cbPatchMem)
+{
+    if (HWACCMIsEnabled(pVM))
+        return HWACMMR3DisablePatching(pVM, pPatchMem, cbPatchMem);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VCPU worker for VMMR3SynchronizeAllVCpus.
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   idCpu       Virtual CPU to perform SIPI on
+ * @param   uVector     SIPI vector
+ */
+DECLCALLBACK(int) vmmR3SyncVCpu(PVM pVM)
+{
+    /* Block until the job in the caller has finished. */
+    RTCritSectEnter(&pVM->vmm.s.CritSectSync);
+    RTCritSectLeave(&pVM->vmm.s.CritSectSync);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Atomically execute a callback handler
+ * Note: This is very expensive; avoid using it frequently!
+ *
+ * @param   pVM         The VM to operate on.
+ * @param   pfnHandler  Callback handler
+ * @param   pvUser      User specified parameter
+ *
+ * @thread  EMT
+ * @todo    Remove this if not used again soon.
+ */
+VMMR3DECL(int) VMMR3AtomicExecuteHandler(PVM pVM, PFNATOMICHANDLER pfnHandler, void *pvUser)
+{
+    int    rc;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
+
+    /* Shortcut for the uniprocessor case. */
+    if (pVM->cCpus == 1)
+        return pfnHandler(pVM, pvUser);
+
+    RTCritSectEnter(&pVM->vmm.s.CritSectSync);
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        if (idCpu != pVCpu->idCpu)
+        {
+            rc = VMR3ReqCallNoWaitU(pVM->pUVM, idCpu, (PFNRT)vmmR3SyncVCpu, 1, pVM);
+            AssertRC(rc);
+        }
+    }
+    /* Wait until all other VCPUs are waiting for us. */
+    while (RTCritSectGetWaiters(&pVM->vmm.s.CritSectSync) != (int32_t)(pVM->cCpus - 1))
+        RTThreadSleep(1);
+
+    rc = pfnHandler(pVM, pvUser);
+    RTCritSectLeave(&pVM->vmm.s.CritSectSync);
+    return rc;
+}
+
+
+/**
+ * Count returns and have the last non-caller EMT wake up the caller.
+ *
+ * @returns VBox strict informational status code for EM scheduling. No failures
+ *          will be returned here, those are for the caller only.
+ *
+ * @param   pVM                 The VM handle.
+ */
+DECL_FORCE_INLINE(int) vmmR3EmtRendezvousNonCallerReturn(PVM pVM)
+{
+    int rcRet = ASMAtomicReadS32(&pVM->vmm.s.i32RendezvousStatus);
+    uint32_t cReturned = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsReturned);
+    if (cReturned == pVM->cCpus - 1U)
+    {
+        int rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousDoneCaller);
+        AssertLogRelRC(rc);
+    }
+
+    AssertLogRelMsgReturn(   rcRet <= VINF_SUCCESS
+                          || (rcRet >= VINF_EM_FIRST && rcRet <= VINF_EM_LAST),
+                          ("%Rrc\n", rcRet),
+                          VERR_IPE_UNEXPECTED_INFO_STATUS);
+    return RT_SUCCESS(rcRet) ? rcRet : VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for VMMR3EmtRendezvous and VMMR3EmtRendezvousFF.
+ *
+ * @returns VBox strict informational status code for EM scheduling. No failures
+ *          will be returned here, those are for the caller only.  When
+ *          fIsCaller is set, VINF_SUCCESS is always returned.
+ *
+ * @param   pVM                 The VM handle.
+ * @param   pVCpu               The VMCPU structure for the calling EMT.
+ * @param   fIsCaller           Whether we're the VMMR3EmtRendezvous caller or
+ *                              not.
+ * @param   fFlags              The flags.
+ * @param   pfnRendezvous       The callback.
+ * @param   pvUser              The user argument for the callback.
+ */
+static int vmmR3EmtRendezvousCommon(PVM pVM, PVMCPU pVCpu, bool fIsCaller,
+                                    uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
+{
+    int rc;
+
+    /*
+     * Enter, the last EMT triggers the next callback phase.
+     */
+    uint32_t cEntered = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsEntered);
+    if (cEntered != pVM->cCpus)
+    {
+        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
+        {
+            /* Wait for our turn. */
+            rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, RT_INDEFINITE_WAIT);
+            AssertLogRelRC(rc);
+        }
+        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
+        {
+            /* Wait for the last EMT to arrive and wake everyone up. */
+            rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce, RT_INDEFINITE_WAIT);
+            AssertLogRelRC(rc);
+        }
+        else if (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                 || (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
+        {
+            /* Wait for our turn. */
+            rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
+            AssertLogRelRC(rc);
+        }
+        else
+        {
+            Assert((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE);
+
+            /*
+             * The execute once is handled specially to optimize the code flow.
+             *
+             * The last EMT to arrive will perform the callback and the other
+             * EMTs will wait on the Done/DoneCaller semaphores (instead of
+             * the EnterOneByOne/AllAtOnce) in the meanwhile. When the callback
+             * returns, that EMT will initiate the normal return sequence.
+             */
+            if (!fIsCaller)
+            {
+                rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
+                AssertLogRelRC(rc);
+
+                return vmmR3EmtRendezvousNonCallerReturn(pVM);
+            }
+            return VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * All EMTs are waiting, clear the FF and take action according to the
+         * execution method.
+         */
+        VM_FF_CLEAR(pVM, VM_FF_EMT_RENDEZVOUS);
+
+        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE)
+        {
+            /* Wake up everyone. */
+            rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);
+            AssertLogRelRC(rc);
+        }
+        else if (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                 || (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
+        {
+            /* Figure out who to wake up and wake it up. If it's ourself, then
+               it's easy otherwise wait for our turn. */
+            VMCPUID iFirst = (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING
+                           ? 0
+                           : pVM->cCpus - 1U;
+            if (pVCpu->idCpu != iFirst)
+            {
+                rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[iFirst]);
+                AssertLogRelRC(rc);
+                rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu], RT_INDEFINITE_WAIT);
+                AssertLogRelRC(rc);
+            }
+        }
+        /* else: execute the handler on the current EMT and wake up one or more threads afterwards. */
+    }
+
+
+    /*
+     * Do the callback and update the status if necessary.
+     */
+    if (    !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
+        ||  RT_SUCCESS(ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus)) )
+    {
+        VBOXSTRICTRC rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            AssertLogRelMsg(   rcStrict <= VINF_SUCCESS
+                            || (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
+                            ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            int32_t i32RendezvousStatus;
+            do
+            {
+                i32RendezvousStatus = ASMAtomicUoReadS32(&pVM->vmm.s.i32RendezvousStatus);
+                if (    rcStrict == i32RendezvousStatus
+                    ||  RT_FAILURE(i32RendezvousStatus)
+                    ||  (   i32RendezvousStatus != VINF_SUCCESS
+                         && rcStrict > i32RendezvousStatus))
+                    break;
+            } while (!ASMAtomicCmpXchgS32(&pVM->vmm.s.i32RendezvousStatus, VBOXSTRICTRC_VAL(rcStrict), i32RendezvousStatus));
+        }
+    }
+
+    /*
+     * Increment the done counter and take action depending on whether we're
+     * the last to finish callback execution.
+     */
+    uint32_t cDone = ASMAtomicIncU32(&pVM->vmm.s.cRendezvousEmtsDone);
+    if (    cDone != pVM->cCpus
+        &&  (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE)
+    {
+        /* Signal the next EMT? */
+        if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ONE_BY_ONE)
+        {
+            rc = RTSemEventSignal(pVM->vmm.s.hEvtRendezvousEnterOneByOne);
+            AssertLogRelRC(rc);
+        }
+        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_ASCENDING)
+        {
+            Assert(cDone == pVCpu->idCpu + 1U);
+            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVCpu->idCpu + 1U]);
+            AssertLogRelRC(rc);
+        }
+        else if ((fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) == VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING)
+        {
+            Assert(pVM->cCpus - cDone == pVCpu->idCpu);
+            rc = RTSemEventSignal(pVM->vmm.s.pahEvtRendezvousEnterOrdered[pVM->cCpus - cDone - 1U]);
+            AssertLogRelRC(rc);
+        }
+
+        /* Wait for the rest to finish (the caller waits on hEvtRendezvousDoneCaller). */
+        if (!fIsCaller)
+        {
+            rc = RTSemEventMultiWait(pVM->vmm.s.hEvtMulRendezvousDone, RT_INDEFINITE_WAIT);
+            AssertLogRelRC(rc);
+        }
+    }
+    else
+    {
+        /* Callback execution is all done, tell the rest to return. */
+        rc = RTSemEventMultiSignal(pVM->vmm.s.hEvtMulRendezvousDone);
+        AssertLogRelRC(rc);
+    }
+
+    if (!fIsCaller)
+        return vmmR3EmtRendezvousNonCallerReturn(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Called in response to VM_FF_EMT_RENDEZVOUS.
+ *
+ * @returns VBox strict status code - EM scheduling.  No errors will be returned
+ *          here, nor will any non-EM scheduling status codes be returned.
+ *
+ * @param   pVM         The VM handle
+ * @param   pVCpu       The handle of the calling EMT.
+ *
+ * @thread  EMT
+ */
+VMMR3_INT_DECL(int) VMMR3EmtRendezvousFF(PVM pVM, PVMCPU pVCpu)
+{
+    return vmmR3EmtRendezvousCommon(pVM, pVCpu, false /* fIsCaller */, pVM->vmm.s.fRendezvousFlags,
+                                    pVM->vmm.s.pfnRendezvous, pVM->vmm.s.pvRendezvousUser);
+}
+
+
+/**
+ * EMT rendezvous.
+ *
+ * Gathers all the EMTs and execute some code on each of them, either in a one
+ * by one fashion or all at once.
+ *
+ * @returns VBox strict status code.  This will be the the first error,
+ *          VINF_SUCCESS, or an EM scheduling status code.
+ *
+ * @param   pVM             The VM handle.
+ * @param   fFlags          Flags indicating execution methods. See
+ *                          grp_VMMR3EmtRendezvous_fFlags.
+ * @param   pfnRendezvous   The callback.
+ * @param   pvUser          User argument for the callback.
+ *
+ * @thread  Any.
+ */
+VMMR3DECL(int) VMMR3EmtRendezvous(PVM pVM, uint32_t fFlags, PFNVMMEMTRENDEZVOUS pfnRendezvous, void *pvUser)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsg(   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_INVALID
+              && (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) <= VMMEMTRENDEZVOUS_FLAGS_TYPE_DESCENDING
+              && !(fFlags & ~VMMEMTRENDEZVOUS_FLAGS_VALID_MASK), ("%#x\n", fFlags));
+    AssertMsg(   !(fFlags & VMMEMTRENDEZVOUS_FLAGS_STOP_ON_ERROR)
+              || (   (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE
+                  && (fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK) != VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE),
+              ("type %u\n", fFlags & VMMEMTRENDEZVOUS_FLAGS_TYPE_MASK));
+
+    VBOXSTRICTRC rcStrict;
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    if (!pVCpu)
+        /*
+         * Forward the request to an EMT thread.
+         */
+        rcStrict = VMR3ReqCallWait(pVM, VMCPUID_ANY,
+                                   (PFNRT)VMMR3EmtRendezvous, 4, pVM, fFlags, pfnRendezvous, pvUser);
+    else if (pVM->cCpus == 1)
+        /*
+         * Shortcut for the single EMT case.
+         */
+        rcStrict = pfnRendezvous(pVM, pVCpu, pvUser);
+    else
+    {
+        /*
+         * Spin lock. If busy, wait for the other EMT to finish while keeping a
+         * lookout of the RENDEZVOUS FF.
+         */
+        int rc;
+        rcStrict = VINF_SUCCESS;
+        if (RT_UNLIKELY(!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0)))
+        {
+            while (!ASMAtomicCmpXchgU32(&pVM->vmm.s.u32RendezvousLock, 0x77778888, 0))
+            {
+                if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
+                {
+                    rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
+                    if (    rc != VINF_SUCCESS
+                        &&  (   rcStrict == VINF_SUCCESS
+                             || rcStrict > rc))
+                        rcStrict = rc;
+                    /** @todo Perhaps deal with termination here?  */
+                }
+                ASMNopPause();
+            }
+        }
+        Assert(!VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS));
+
+        /*
+         * Clear the slate. This is a semaphore ping-pong orgy. :-)
+         */
+        for (VMCPUID i = 0; i < pVM->cCpus; i++)
+        {
+            rc = RTSemEventWait(pVM->vmm.s.pahEvtRendezvousEnterOrdered[i], 0);
+            AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        }
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousEnterOneByOne, 0);         AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousEnterAllAtOnce);  AssertLogRelRC(rc);
+        rc = RTSemEventMultiReset(pVM->vmm.s.hEvtMulRendezvousDone);            AssertLogRelRC(rc);
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, 0);            AssertLogRelMsg(rc == VERR_TIMEOUT || rc == VINF_SUCCESS, ("%Rrc\n", rc));
+        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsEntered, 0);
+        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsDone, 0);
+        ASMAtomicWriteU32(&pVM->vmm.s.cRendezvousEmtsReturned, 0);
+        ASMAtomicWriteS32(&pVM->vmm.s.i32RendezvousStatus, VINF_SUCCESS);
+        ASMAtomicWritePtr((void * volatile *)&pVM->vmm.s.pfnRendezvous, (void *)(uintptr_t)pfnRendezvous);
+        ASMAtomicWritePtr(&pVM->vmm.s.pvRendezvousUser, pvUser);
+        ASMAtomicWriteU32(&pVM->vmm.s.fRendezvousFlags, fFlags);
+
+        /*
+         * Set the FF and poke the other EMTs.
+         */
+        VM_FF_SET(pVM, VM_FF_EMT_RENDEZVOUS);
+        VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_POKE);
+
+        /*
+         * Do the same ourselves.
+         */
+        vmmR3EmtRendezvousCommon(pVM, pVCpu, true /* fIsCaller */, fFlags, pfnRendezvous, pvUser);
+
+        /*
+         * The caller waits for the other EMTs to be done and return before doing
+         * the cleanup. This makes away with wakeup / reset races we would otherwise
+         * risk in the multiple release event semaphore code (hEvtRendezvousDoneCaller).
+         */
+        rc = RTSemEventWait(pVM->vmm.s.hEvtRendezvousDoneCaller, RT_INDEFINITE_WAIT);
+        AssertLogRelRC(rc);
+
+        /*
+         * Get the return code and clean up a little bit.
+         */
+        int rcMy = pVM->vmm.s.i32RendezvousStatus;
+        ASMAtomicWriteNullPtr((void * volatile *)&pVM->vmm.s.pfnRendezvous);
+
+        ASMAtomicWriteU32(&pVM->vmm.s.u32RendezvousLock, 0);
+
+        /*
+         * Merge rcStrict and rcMy.
+         */
+        AssertRC(VBOXSTRICTRC_VAL(rcStrict));
+        if (    rcMy != VINF_SUCCESS
+            &&  (   rcStrict == VINF_SUCCESS
+                 || rcStrict > rcMy))
+            rcStrict = rcMy;
+    }
+
+    AssertLogRelMsgReturn(   rcStrict <= VINF_SUCCESS
+                          || (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST),
+                          ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)),
+                          VERR_IPE_UNEXPECTED_INFO_STATUS);
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * Read from the ring 0 jump buffer stack
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idCpu           The ID of the source CPU context (for the address).
+ * @param   R0Addr          Where to start reading.
+ * @param   pvBuf           Where to store the data we've read.
+ * @param   cbRead          The number of bytes to read.
+ */
+VMMR3_INT_DECL(int) VMMR3ReadR0Stack(PVM pVM, VMCPUID idCpu, RTHCUINTPTR R0Addr, void *pvBuf, size_t cbRead)
+{
+    PVMCPU pVCpu = VMMGetCpuById(pVM, idCpu);
+    AssertReturn(pVCpu, VERR_INVALID_PARAMETER);
+
+#ifdef VMM_R0_SWITCH_STACK
+    RTHCUINTPTR off = R0Addr - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3);
+#else
+    RTHCUINTPTR off = pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack - (pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck - R0Addr);
+#endif
+    if (   off          >  VMM_STACK_SIZE
+        || off + cbRead >= VMM_STACK_SIZE)
+        return VERR_INVALID_POINTER;
+
+    memcpy(pvBuf, &pVCpu->vmm.s.pbEMTStackR3[off], cbRead);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Calls a RC function.
+ *
+ * @param   pVM         The VM handle.
+ * @param   RCPtrEntry  The address of the RC function.
+ * @param   cArgs       The number of arguments in the ....
+ * @param   ...         Arguments to the function.
+ */
+VMMR3DECL(int) VMMR3CallRC(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, ...)
+{
+    va_list args;
+    va_start(args, cArgs);
+    int rc = VMMR3CallRCV(pVM, RCPtrEntry, cArgs, args);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Calls a RC function.
+ *
+ * @param   pVM         The VM handle.
+ * @param   RCPtrEntry  The address of the RC function.
+ * @param   cArgs       The number of arguments in the ....
+ * @param   args        Arguments to the function.
+ */
+VMMR3DECL(int) VMMR3CallRCV(PVM pVM, RTRCPTR RCPtrEntry, unsigned cArgs, va_list args)
+{
+    /* Raw mode implies 1 VCPU. */
+    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    Log2(("VMMR3CallGCV: RCPtrEntry=%RRv cArgs=%d\n", RCPtrEntry, cArgs));
+
+    /*
+     * Setup the call frame using the trampoline.
+     */
+    CPUMHyperSetCtxCore(pVCpu, NULL);
+    memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE); /* Clear the stack. */
+    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC - cArgs * sizeof(RTGCUINTPTR32));
+    PRTGCUINTPTR32 pFrame = (PRTGCUINTPTR32)(pVCpu->vmm.s.pbEMTStackR3 + VMM_STACK_SIZE) - cArgs;
+    int i = cArgs;
+    while (i-- > 0)
+        *pFrame++ = va_arg(args, RTGCUINTPTR32);
+
+    CPUMPushHyper(pVCpu, cArgs * sizeof(RTGCUINTPTR32));                          /* stack frame size */
+    CPUMPushHyper(pVCpu, RCPtrEntry);                                             /* what to call */
+    CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
+
+    /*
+     * We hide log flushes (outer) and hypervisor interrupts (inner).
+     */
+    for (;;)
+    {
+        int rc;
+        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
+        do
+        {
+#ifdef NO_SUPCALLR0VMM
+            rc = VERR_GENERAL_FAILURE;
+#else
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                rc = pVCpu->vmm.s.iLastGZRc;
+#endif
+        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
+
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
+        if (    pLogger
+            &&  pLogger->offScratch > 0)
+            RTLogFlushRC(NULL, pLogger);
+#endif
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+        PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
+        if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
+            RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
+#endif
+        if (rc == VERR_TRPM_PANIC || rc == VERR_TRPM_DONT_PANIC)
+            VMMR3FatalDump(pVM, pVCpu, rc);
+        if (rc != VINF_VMM_CALL_HOST)
+        {
+            Log2(("VMMR3CallGCV: returns %Rrc (cs:eip=%04x:%08x)\n", rc, CPUMGetGuestCS(pVCpu), CPUMGetGuestEIP(pVCpu)));
+            return rc;
+        }
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+}
+
+
+/**
+ * Wrapper for SUPR3CallVMMR0Ex which will deal with VINF_VMM_CALL_HOST returns.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   uOperation  Operation to execute.
+ * @param   u64Arg      Constant argument.
+ * @param   pReqHdr     Pointer to a request header. See SUPR3CallVMMR0Ex for
+ *                      details.
+ */
+VMMR3DECL(int) VMMR3CallR0(PVM pVM, uint32_t uOperation, uint64_t u64Arg, PSUPVMMR0REQHDR pReqHdr)
+{
+    PVMCPU pVCpu = VMMGetCpu(pVM);
+    AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
+
+    /*
+     * Call Ring-0 entry with init code.
+     */
+    int rc;
+    for (;;)
+    {
+#ifdef NO_SUPCALLR0VMM
+        rc = VERR_GENERAL_FAILURE;
+#else
+        rc = SUPR3CallVMMR0Ex(pVM->pVMR0, pVCpu->idCpu, uOperation, u64Arg, pReqHdr);
+#endif
+        /*
+         * Flush the logs.
+         */
+#ifdef LOG_ENABLED
+        if (    pVCpu->vmm.s.pR0LoggerR3
+            &&  pVCpu->vmm.s.pR0LoggerR3->Logger.offScratch > 0)
+            RTLogFlushToLogger(&pVCpu->vmm.s.pR0LoggerR3->Logger, NULL);
+#endif
+        if (rc != VINF_VMM_CALL_HOST)
+            break;
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST))
+            break;
+        /* Resume R0 */
+    }
+
+    AssertLogRelMsgReturn(rc == VINF_SUCCESS || RT_FAILURE(rc),
+                          ("uOperation=%u rc=%Rrc\n", uOperation, rc),
+                          VERR_INTERNAL_ERROR);
+    return rc;
+}
+
+
+/**
+ * Resumes executing hypervisor code when interrupted by a queue flush or a
+ * debug event.
+ *
+ * @returns VBox status code.
+ * @param   pVM         VM handle.
+ * @param   pVCpu       VMCPU handle.
+ */
+VMMR3DECL(int) VMMR3ResumeHyper(PVM pVM, PVMCPU pVCpu)
+{
+    Log(("VMMR3ResumeHyper: eip=%RRv esp=%RRv\n", CPUMGetHyperEIP(pVCpu), CPUMGetHyperESP(pVCpu)));
+    AssertReturn(pVM->cCpus == 1, VERR_RAW_MODE_INVALID_SMP);
+
+    /*
+     * We hide log flushes (outer) and hypervisor interrupts (inner).
+     */
+    for (;;)
+    {
+        int rc;
+        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
+        do
+        {
+#ifdef NO_SUPCALLR0VMM
+            rc = VERR_GENERAL_FAILURE;
+#else
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                rc = pVCpu->vmm.s.iLastGZRc;
+#endif
+        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
+
+        /*
+         * Flush the loggers,
+         */
+#ifdef LOG_ENABLED
+        PRTLOGGERRC pLogger = pVM->vmm.s.pRCLoggerR3;
+        if (    pLogger
+            &&  pLogger->offScratch > 0)
+            RTLogFlushRC(NULL, pLogger);
+#endif
+#ifdef VBOX_WITH_RC_RELEASE_LOGGING
+        PRTLOGGERRC pRelLogger = pVM->vmm.s.pRCRelLoggerR3;
+        if (RT_UNLIKELY(pRelLogger && pRelLogger->offScratch > 0))
+            RTLogFlushRC(RTLogRelDefaultInstance(), pRelLogger);
+#endif
+        if (rc == VERR_TRPM_PANIC || rc == VERR_TRPM_DONT_PANIC)
+            VMMR3FatalDump(pVM, pVCpu, rc);
+        if (rc != VINF_VMM_CALL_HOST)
+        {
+            Log(("VMMR3ResumeHyper: returns %Rrc\n", rc));
+            return rc;
+        }
+        rc = vmmR3ServiceCallRing3Request(pVM, pVCpu);
+        if (RT_FAILURE(rc))
+            return rc;
+    }
+}
+
+
+/**
+ * Service a call to the ring-3 host code.
+ *
+ * @returns VBox status code.
+ * @param   pVM     VM handle.
+ * @param   pVCpu   VMCPU handle
+ * @remark  Careful with critsects.
+ */
+static int vmmR3ServiceCallRing3Request(PVM pVM, PVMCPU pVCpu)
+{
+    /*
+     * We must also check for pending critsect exits or else we can deadlock
+     * when entering other critsects here.
+     */
+    if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PDM_CRITSECT))
+        PDMCritSectFF(pVCpu);
+
+    switch (pVCpu->vmm.s.enmCallRing3Operation)
+    {
+        /*
+         * Acquire the PDM lock.
+         */
+        case VMMCALLRING3_PDM_LOCK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PDMR3LockCall(pVM);
+            break;
+        }
+
+        /*
+         * Grow the PGM pool.
+         */
+        case VMMCALLRING3_PGM_POOL_GROW:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PoolGrow(pVM);
+            break;
+        }
+
+        /*
+         * Maps an page allocation chunk into ring-3 so ring-0 can use it.
+         */
+        case VMMCALLRING3_PGM_MAP_CHUNK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysChunkMap(pVM, pVCpu->vmm.s.u64CallRing3Arg);
+            break;
+        }
+
+        /*
+         * Allocates more handy pages.
+         */
+        case VMMCALLRING3_PGM_ALLOCATE_HANDY_PAGES:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateHandyPages(pVM);
+            break;
+        }
+
+        /*
+         * Allocates a large page.
+         */
+        case VMMCALLRING3_PGM_ALLOCATE_LARGE_HANDY_PAGE:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3PhysAllocateLargeHandyPage(pVM, pVCpu->vmm.s.u64CallRing3Arg);
+            break;
+        }
+
+        /*
+         * Acquire the PGM lock.
+         */
+        case VMMCALLRING3_PGM_LOCK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = PGMR3LockCall(pVM);
+            break;
+        }
+
+        /*
+         * Acquire the MM hypervisor heap lock.
+         */
+        case VMMCALLRING3_MMHYPER_LOCK:
+        {
+            pVCpu->vmm.s.rcCallRing3 = MMR3LockCall(pVM);
+            break;
+        }
+
+        /*
+         * Flush REM handler notifications.
+         */
+        case VMMCALLRING3_REM_REPLAY_HANDLER_NOTIFICATIONS:
+        {
+            REMR3ReplayHandlerNotifications(pVM);
+            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
+            break;
+        }
+
+        /*
+         * This is a noop. We just take this route to avoid unnecessary
+         * tests in the loops.
+         */
+        case VMMCALLRING3_VMM_LOGGER_FLUSH:
+            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
+            LogAlways(("*FLUSH*\n"));
+            break;
+
+        /*
+         * Set the VM error message.
+         */
+        case VMMCALLRING3_VM_SET_ERROR:
+            VMR3SetErrorWorker(pVM);
+            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
+            break;
+
+        /*
+         * Set the VM runtime error message.
+         */
+        case VMMCALLRING3_VM_SET_RUNTIME_ERROR:
+            pVCpu->vmm.s.rcCallRing3 = VMR3SetRuntimeErrorWorker(pVM);
+            break;
+
+        /*
+         * Signal a ring 0 hypervisor assertion.
+         * Cancel the longjmp operation that's in progress.
+         */
+        case VMMCALLRING3_VM_R0_ASSERTION:
+            pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
+            pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call = false;
+#ifdef RT_ARCH_X86
+            pVCpu->vmm.s.CallRing3JmpBufR0.eip = 0;
+#else
+            pVCpu->vmm.s.CallRing3JmpBufR0.rip = 0;
+#endif
+#ifdef VMM_R0_SWITCH_STACK
+            *(uint64_t *)pVCpu->vmm.s.pbEMTStackR3 = 0; /* clear marker  */
+#endif
+            LogRel((pVM->vmm.s.szRing0AssertMsg1));
+            LogRel((pVM->vmm.s.szRing0AssertMsg2));
+            return VERR_VMM_RING0_ASSERTION;
+
+        /*
+         * A forced switch to ring 0 for preemption purposes.
+         */
+        case VMMCALLRING3_VM_R0_PREEMPT:
+            pVCpu->vmm.s.rcCallRing3 = VINF_SUCCESS;
+            break;
+
+        case VMMCALLRING3_FTM_SET_CHECKPOINT:
+            pVCpu->vmm.s.rcCallRing3 = FTMR3SetCheckpoint(pVM, (FTMCHECKPOINTTYPE)pVCpu->vmm.s.u64CallRing3Arg);
+            break;
+
+        default:
+            AssertMsgFailed(("enmCallRing3Operation=%d\n", pVCpu->vmm.s.enmCallRing3Operation));
+            return VERR_INTERNAL_ERROR;
+    }
+
+    pVCpu->vmm.s.enmCallRing3Operation = VMMCALLRING3_INVALID;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Displays the Force action Flags.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pHlp        The output helpers.
+ * @param   pszArgs     The additional arguments (ignored).
+ */
+static DECLCALLBACK(void) vmmR3InfoFF(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs)
+{
+    int         c;
+    uint32_t    f;
+#define PRINT_FLAG(prf,flag) do { \
+        if (f & (prf##flag)) \
+        { \
+            static const char *s_psz = #flag; \
+            if (!(c % 6)) \
+                pHlp->pfnPrintf(pHlp, "%s\n    %s", c ? "," : "", s_psz); \
+            else \
+                pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
+            c++; \
+            f &= ~(prf##flag); \
+        } \
+    } while (0)
+
+#define PRINT_GROUP(prf,grp,sfx) do { \
+        if (f & (prf##grp##sfx)) \
+        { \
+            static const char *s_psz = #grp; \
+            if (!(c % 5)) \
+                pHlp->pfnPrintf(pHlp, "%s    %s", c ? ",\n" : "  Groups:\n", s_psz); \
+            else \
+                pHlp->pfnPrintf(pHlp, ", %s", s_psz); \
+            c++; \
+        } \
+    } while (0)
+
+    /*
+     * The global flags.
+     */
+    const uint32_t fGlobalForcedActions = pVM->fGlobalForcedActions;
+    pHlp->pfnPrintf(pHlp, "Global FFs: %#RX32", fGlobalForcedActions);
+
+    /* show the flag mnemonics  */
+    c = 0;
+    f = fGlobalForcedActions;
+    PRINT_FLAG(VM_FF_,TM_VIRTUAL_SYNC);
+    PRINT_FLAG(VM_FF_,PDM_QUEUES);
+    PRINT_FLAG(VM_FF_,PDM_DMA);
+    PRINT_FLAG(VM_FF_,DBGF);
+    PRINT_FLAG(VM_FF_,REQUEST);
+    PRINT_FLAG(VM_FF_,CHECK_VM_STATE);
+    PRINT_FLAG(VM_FF_,RESET);
+    PRINT_FLAG(VM_FF_,EMT_RENDEZVOUS);
+    PRINT_FLAG(VM_FF_,PGM_NEED_HANDY_PAGES);
+    PRINT_FLAG(VM_FF_,PGM_NO_MEMORY);
+    PRINT_FLAG(VM_FF_,PGM_POOL_FLUSH_PENDING);
+    PRINT_FLAG(VM_FF_,REM_HANDLER_NOTIFY);
+    PRINT_FLAG(VM_FF_,DEBUG_SUSPEND);
+    if (f)
+        pHlp->pfnPrintf(pHlp, "%s\n    Unknown bits: %#RX32\n", c ? "," : "", f);
+    else
+        pHlp->pfnPrintf(pHlp, "\n");
+
+    /* the groups */
+    c = 0;
+    f = fGlobalForcedActions;
+    PRINT_GROUP(VM_FF_,EXTERNAL_SUSPENDED,_MASK);
+    PRINT_GROUP(VM_FF_,EXTERNAL_HALTED,_MASK);
+    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE,_MASK);
+    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
+    PRINT_GROUP(VM_FF_,HIGH_PRIORITY_POST,_MASK);
+    PRINT_GROUP(VM_FF_,NORMAL_PRIORITY_POST,_MASK);
+    PRINT_GROUP(VM_FF_,NORMAL_PRIORITY,_MASK);
+    PRINT_GROUP(VM_FF_,ALL_BUT_RAW,_MASK);
+    if (c)
+        pHlp->pfnPrintf(pHlp, "\n");
+
+    /*
+     * Per CPU flags.
+     */
+    for (VMCPUID i = 0; i < pVM->cCpus; i++)
+    {
+        const uint32_t fLocalForcedActions = pVM->aCpus[i].fLocalForcedActions;
+        pHlp->pfnPrintf(pHlp, "CPU %u FFs: %#RX32", i, fLocalForcedActions);
+
+        /* show the flag mnemonics */
+        c = 0;
+        f = fLocalForcedActions;
+        PRINT_FLAG(VMCPU_FF_,INTERRUPT_APIC);
+        PRINT_FLAG(VMCPU_FF_,INTERRUPT_PIC);
+        PRINT_FLAG(VMCPU_FF_,TIMER);
+        PRINT_FLAG(VMCPU_FF_,PDM_CRITSECT);
+        PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3);
+        PRINT_FLAG(VMCPU_FF_,PGM_SYNC_CR3_NON_GLOBAL);
+        PRINT_FLAG(VMCPU_FF_,TLB_FLUSH);
+        PRINT_FLAG(VMCPU_FF_,TRPM_SYNC_IDT);
+        PRINT_FLAG(VMCPU_FF_,SELM_SYNC_TSS);
+        PRINT_FLAG(VMCPU_FF_,SELM_SYNC_GDT);
+        PRINT_FLAG(VMCPU_FF_,SELM_SYNC_LDT);
+        PRINT_FLAG(VMCPU_FF_,INHIBIT_INTERRUPTS);
+        PRINT_FLAG(VMCPU_FF_,CSAM_SCAN_PAGE);
+        PRINT_FLAG(VMCPU_FF_,CSAM_PENDING_ACTION);
+        PRINT_FLAG(VMCPU_FF_,TO_R3);
+        if (f)
+            pHlp->pfnPrintf(pHlp, "%s\n    Unknown bits: %#RX32\n", c ? "," : "", f);
+        else
+            pHlp->pfnPrintf(pHlp, "\n");
+
+        /* the groups */
+        c = 0;
+        f = fLocalForcedActions;
+        PRINT_GROUP(VMCPU_FF_,EXTERNAL_SUSPENDED,_MASK);
+        PRINT_GROUP(VMCPU_FF_,EXTERNAL_HALTED,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_PRE_RAW,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HIGH_PRIORITY_POST,_MASK);
+        PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY_POST,_MASK);
+        PRINT_GROUP(VMCPU_FF_,NORMAL_PRIORITY,_MASK);
+        PRINT_GROUP(VMCPU_FF_,RESUME_GUEST,_MASK);
+        PRINT_GROUP(VMCPU_FF_,HWACCM_TO_R3,_MASK);
+        PRINT_GROUP(VMCPU_FF_,ALL_BUT_RAW,_MASK);
+        if (c)
+            pHlp->pfnPrintf(pHlp, "\n");
+    }
+
+#undef PRINT_FLAG
+#undef PRINT_GROUP
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/VMMCodingGuidelines.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMMCodingGuidelines.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMMCodingGuidelines.cpp	(revision 35333)
@@ -0,0 +1,93 @@
+/* $Id$ */
+/** @file
+ * VMM - Coding Guidelines.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/** @page pg_vmm_guideline                  VMM Coding Guidelines
+ *
+ * The guidelines extends the VBox coding guidelines (@ref pg_vbox_guideline)
+ * and consists of a compulsory part and an optional part. It is very important
+ * that the rules of the compulsory part is followed. That will prevent obvious
+ * bugs, and it will ease porting the code to 32/64 and 64/32 bits setups.
+ *
+ *
+ *
+ * @section sec_vmm_guideline_compulsory        Compulsory
+ *
+ * It is of vital importance is to distinguish between addresses - both virtual
+ * and physical - applying to Guest Context and Host Context. To assist the
+ * coder in this, a set of types and macros have been created. Another vital
+ * thing is that structures shared between the two contexts ends up with the
+ * same size and member offsets in both places. There are types and macros
+ * for that too.
+ *
+ *
+ * The rules:
+ *
+ *      - When declaring pointers in shared structures use the RCPTRTYPE(),
+ *        R0PTRTYPE() and R3PTRTYPE() macros.
+ *
+ *      - Use RTGCPTR and RTHCPTR when dealing with the other context in
+ *        none shared structures, parameter lists, stack variables and such.
+ *
+ *      - Following the above rules, pointers will in a context other than the
+ *        one a pointer was defined for, appear as unsigned integers.
+ *
+ *      - It is NOT permitted to subject a pointer from the other context to pointer
+ *        types of the current context by direct cast or by definition.
+ *
+ *      - When doing pointer arithmetic cast using uintptr_t, intptr_t or char *.
+ *        Never cast a pointer to anything else for this purpose, that will not
+ *        work everywhere! (1)
+ *
+ *      - Physical addresses are also specific to their context. Use RTGCPHYS
+ *        and RTHCPHYS when dealing when them. Both types are unsigned integers.
+ *
+ *      - Integers in shared structures should be using a RT integer type or
+ *        any of the [u]int[0-9]+_t types. (2)
+ *
+ *      - If code is shared between the contexts, GCTYPE() can be used to declare
+ *        things differently. If GCTYPE() usage is extensive, don't share the code.
+ *
+ *      - The context is part of all public symbols which are specific to a single
+ *        context.
+ *
+ *
+ * (1) Talking about porting between 32-bit and 64-bit architectures and even
+ *     between 64-bit platforms. On 64-bit linux int is 32-bit, long is 64-bit.
+ *     However on 64-bit windows both int and long are 32-bit - there is no
+ *     standard 64 bit type (_int64 is not a standard type, it's an stupid
+ *     extension).
+ *
+ * (2) The VBox integer types are RTINT, RTUINT, RTGCINT, RTGCUINT,
+ *     RTGCINTPTR, RTGCUINTPTR, RTHCINT, RTHCUINT, RTHCINTPTR and
+ *     RTHCUINTPTR.
+ *
+ *
+ *
+ * @section sec_vmm_guideline_optional          Optional
+ *
+ * There are the general VBox guidelines, see @ref sec_vbox_guideline_optional.
+ * In addition to these for the following rules applies to the VMM:
+ *
+ *      - Prefixes GCPtr and HCPtr are preferred over suffixes HC and GC of
+ *        pointers.
+ *
+ *      - Prefixes GCPhys and HCPhys are generally used for physical addresses,
+ *        types RTGCPHYS and RTHCPHYS respectively.
+ *
+ */
+
Index: /trunk/src/VBox/VMM/VMMR3/VMMDocsRawMode.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMMDocsRawMode.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMMDocsRawMode.cpp	(revision 35333)
@@ -0,0 +1,34 @@
+/* $Id$ */
+/** @file
+ * This file contains the documentation of the raw-mode execution.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+
+/** @page pg_raw                Raw-mode Code Execution
+ *
+ * This chapter describes the virtualization technique which we call raw-mode
+ * and how VirtualBox makes use of it and implements it.
+ *
+ * @todo Write raw-mode chapter!
+ *
+ *
+ * @section sec_rawr3           Raw-Ring3
+ *
+ *
+ * @section sec_rawr0           Raw-Ring0
+ *
+ *
+ */
Index: /trunk/src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMMGuruMeditation.cpp	(revision 35333)
@@ -0,0 +1,664 @@
+/* $Id$ */
+/** @file
+ * VMM - The Virtual Machine Monitor, Guru Meditation Code.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/trpm.h>
+#include <VBox/dbgf.h>
+#include "VMMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/mm.h>
+#include <VBox/iom.h>
+#include <VBox/em.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/version.h>
+#include <VBox/hwaccm.h>
+#include <iprt/assert.h>
+#include <iprt/time.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+#include <iprt/stdarg.h>
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+/**
+ * Structure to pass to DBGFR3Info() and for doing all other
+ * output during fatal dump.
+ */
+typedef struct VMMR3FATALDUMPINFOHLP
+{
+    /** The helper core. */
+    DBGFINFOHLP Core;
+    /** The release logger instance. */
+    PRTLOGGER   pRelLogger;
+    /** The saved release logger flags. */
+    uint32_t    fRelLoggerFlags;
+    /** The logger instance. */
+    PRTLOGGER   pLogger;
+    /** The saved logger flags. */
+    uint32_t    fLoggerFlags;
+    /** The saved logger destination flags. */
+    uint32_t    fLoggerDestFlags;
+    /** Whether to output to stderr or not. */
+    bool        fStdErr;
+    /** Whether we're still recording the summary or not. */
+    bool        fRecSummary;
+    /** Buffer for the summary. */
+    char        szSummary[4096-2];
+    /** The current summary offset. */
+    size_t      offSummary;
+} VMMR3FATALDUMPINFOHLP, *PVMMR3FATALDUMPINFOHLP;
+/** Pointer to a VMMR3FATALDUMPINFOHLP structure. */
+typedef const VMMR3FATALDUMPINFOHLP *PCVMMR3FATALDUMPINFOHLP;
+
+
+/**
+ * Print formatted string.
+ *
+ * @param   pHlp        Pointer to this structure.
+ * @param   pszFormat   The format string.
+ * @param   ...         Arguments.
+ */
+static DECLCALLBACK(void) vmmR3FatalDumpInfoHlp_pfnPrintf(PCDBGFINFOHLP pHlp, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    pHlp->pfnPrintfV(pHlp, pszFormat, args);
+    va_end(args);
+}
+
+
+/**
+ * Print formatted string.
+ *
+ * @param   pHlp        Pointer to this structure.
+ * @param   pszFormat   The format string.
+ * @param   args        Argument list.
+ */
+static DECLCALLBACK(void) vmmR3FatalDumpInfoHlp_pfnPrintfV(PCDBGFINFOHLP pHlp, const char *pszFormat, va_list args)
+{
+    PVMMR3FATALDUMPINFOHLP pMyHlp = (PVMMR3FATALDUMPINFOHLP)pHlp;
+
+    if (pMyHlp->pRelLogger)
+    {
+        va_list args2;
+        va_copy(args2, args);
+        RTLogLoggerV(pMyHlp->pRelLogger, pszFormat, args2);
+        va_end(args2);
+    }
+    if (pMyHlp->pLogger)
+    {
+        va_list args2;
+        va_copy(args2, args);
+        RTLogLoggerV(pMyHlp->pLogger, pszFormat, args);
+        va_end(args2);
+    }
+    if (pMyHlp->fStdErr)
+    {
+        va_list args2;
+        va_copy(args2, args);
+        RTStrmPrintfV(g_pStdErr, pszFormat, args);
+        va_end(args2);
+    }
+    if (pMyHlp->fRecSummary)
+    {
+        size_t cchLeft = sizeof(pMyHlp->szSummary) - pMyHlp->offSummary;
+        if (cchLeft > 1)
+        {
+            va_list args2;
+            va_copy(args2, args);
+            size_t cch = RTStrPrintfV(&pMyHlp->szSummary[pMyHlp->offSummary], cchLeft, pszFormat, args);
+            va_end(args2);
+            Assert(cch <= cchLeft);
+            pMyHlp->offSummary += cch;
+        }
+    }
+}
+
+
+/**
+ * Initializes the fatal dump output helper.
+ *
+ * @param   pHlp        The structure to initialize.
+ */
+static void vmmR3FatalDumpInfoHlpInit(PVMMR3FATALDUMPINFOHLP pHlp)
+{
+    memset(pHlp, 0, sizeof(*pHlp));
+
+    pHlp->Core.pfnPrintf = vmmR3FatalDumpInfoHlp_pfnPrintf;
+    pHlp->Core.pfnPrintfV = vmmR3FatalDumpInfoHlp_pfnPrintfV;
+
+    /*
+     * The loggers.
+     */
+    pHlp->pRelLogger = RTLogRelDefaultInstance();
+#ifndef LOG_ENABLED
+    if (!pHlp->pRelLogger)
+#endif
+        pHlp->pLogger = RTLogDefaultInstance();
+
+    if (pHlp->pRelLogger)
+    {
+        pHlp->fRelLoggerFlags = pHlp->pRelLogger->fFlags;
+        pHlp->pRelLogger->fFlags &= ~(RTLOGFLAGS_BUFFERED | RTLOGFLAGS_DISABLED);
+    }
+
+    if (pHlp->pLogger)
+    {
+        pHlp->fLoggerFlags     = pHlp->pLogger->fFlags;
+        pHlp->fLoggerDestFlags = pHlp->pLogger->fDestFlags;
+        pHlp->pLogger->fFlags     &= ~(RTLOGFLAGS_BUFFERED | RTLOGFLAGS_DISABLED);
+#ifndef DEBUG_sandervl
+        pHlp->pLogger->fDestFlags |= RTLOGDEST_DEBUGGER;
+#endif
+    }
+
+    /*
+     * Check if we need write to stderr.
+     */
+    pHlp->fStdErr = (!pHlp->pRelLogger || !(pHlp->pRelLogger->fDestFlags & (RTLOGDEST_STDOUT | RTLOGDEST_STDERR)))
+                 && (!pHlp->pLogger || !(pHlp->pLogger->fDestFlags & (RTLOGDEST_STDOUT | RTLOGDEST_STDERR)));
+#ifdef DEBUG_sandervl
+    pHlp->fStdErr = false; /* takes too long to display here */
+#endif
+
+    /*
+     * Init the summary recording.
+     */
+    pHlp->fRecSummary  = true;
+    pHlp->offSummary   = 0;
+    pHlp->szSummary[0] = '\0';
+}
+
+
+/**
+ * Deletes the fatal dump output helper.
+ *
+ * @param   pHlp        The structure to delete.
+ */
+static void vmmR3FatalDumpInfoHlpDelete(PVMMR3FATALDUMPINFOHLP pHlp)
+{
+    if (pHlp->pRelLogger)
+    {
+        RTLogFlush(pHlp->pRelLogger);
+        pHlp->pRelLogger->fFlags = pHlp->fRelLoggerFlags;
+    }
+
+    if (pHlp->pLogger)
+    {
+        RTLogFlush(pHlp->pLogger);
+        pHlp->pLogger->fFlags     = pHlp->fLoggerFlags;
+        pHlp->pLogger->fDestFlags = pHlp->fLoggerDestFlags;
+    }
+}
+
+
+/**
+ * Dumps the VM state on a fatal error.
+ *
+ * @param   pVM         VM Handle.
+ * @param   pVCpu       VMCPU Handle.
+ * @param   rcErr       VBox status code.
+ */
+VMMR3DECL(void) VMMR3FatalDump(PVM pVM, PVMCPU pVCpu, int rcErr)
+{
+    /*
+     * Create our output helper and sync it with the log settings.
+     * This helper will be used for all the output.
+     */
+    VMMR3FATALDUMPINFOHLP   Hlp;
+    PCDBGFINFOHLP           pHlp = &Hlp.Core;
+    vmmR3FatalDumpInfoHlpInit(&Hlp);
+
+    /* Release owned locks to make sure other VCPUs can continue in case they were waiting for one. */
+    PDMR3CritSectLeaveAll(pVM);
+
+    /*
+     * Header.
+     */
+    pHlp->pfnPrintf(pHlp,
+                    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+                    "!!\n"
+                    "!!                 Guru Meditation %d (%Rrc)\n"
+                    "!!\n",
+                    rcErr, rcErr);
+
+    /*
+     * Continue according to context.
+     */
+    bool fDoneHyper = false;
+    switch (rcErr)
+    {
+        /*
+         * Hypervisor errors.
+         */
+        case VERR_VMM_RING0_ASSERTION:
+        case VINF_EM_DBG_HYPER_ASSERTION:
+        case VERR_VMM_RING3_CALL_DISABLED:
+        {
+            const char *pszMsg1 = VMMR3GetRZAssertMsg1(pVM);
+            while (pszMsg1 && *pszMsg1 == '\n')
+                pszMsg1++;
+            const char *pszMsg2 = VMMR3GetRZAssertMsg2(pVM);
+            while (pszMsg2 && *pszMsg2 == '\n')
+                pszMsg2++;
+            pHlp->pfnPrintf(pHlp,
+                            "%s"
+                            "%s",
+                            pszMsg1,
+                            pszMsg2);
+            if (    !pszMsg2
+                ||  !*pszMsg2
+                ||  strchr(pszMsg2, '\0')[-1] != '\n')
+                pHlp->pfnPrintf(pHlp, "\n");
+            /* fall thru */
+        }
+        case VERR_TRPM_DONT_PANIC:
+        case VERR_TRPM_PANIC:
+        case VINF_EM_RAW_STALE_SELECTOR:
+        case VINF_EM_RAW_IRET_TRAP:
+        case VINF_EM_DBG_HYPER_BREAKPOINT:
+        case VINF_EM_DBG_HYPER_STEPPED:
+        case VERR_VMM_HYPER_CR3_MISMATCH:
+        {
+            /*
+             * Active trap? This is only of partial interest when in hardware
+             * assisted virtualization mode, thus the different messages.
+             */
+            uint32_t        uEIP       = CPUMGetHyperEIP(pVCpu);
+            TRPMEVENT       enmType;
+            uint8_t         u8TrapNo   =       0xce;
+            RTGCUINT        uErrorCode = 0xdeadface;
+            RTGCUINTPTR     uCR2       = 0xdeadface;
+            int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrorCode, &uCR2);
+            if (!HWACCMIsEnabled(pVM))
+            {
+                if (RT_SUCCESS(rc2))
+                    pHlp->pfnPrintf(pHlp,
+                                    "!! TRAP=%02x ERRCD=%RGv CR2=%RGv EIP=%RX32 Type=%d\n",
+                                    u8TrapNo, uErrorCode, uCR2, uEIP, enmType);
+                else
+                    pHlp->pfnPrintf(pHlp,
+                                    "!! EIP=%RX32 NOTRAP\n",
+                                    uEIP);
+            }
+            else if (RT_SUCCESS(rc2))
+                pHlp->pfnPrintf(pHlp,
+                                "!! ACTIVE TRAP=%02x ERRCD=%RGv CR2=%RGv PC=%RGr Type=%d (Guest!)\n",
+                                u8TrapNo, uErrorCode, uCR2, CPUMGetGuestRIP(pVCpu), enmType);
+
+            /*
+             * Dump the relevant hypervisor registers and stack.
+             */
+            if (HWACCMIsEnabled(pVM))
+            {
+                if (   rcErr == VERR_VMM_RING0_ASSERTION /* fInRing3Call has already been cleared here. */
+                    || pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call)
+                {
+                    /* Dump the jmpbuf.  */
+                    pHlp->pfnPrintf(pHlp,
+                                    "!!\n"
+                                    "!! CallRing3JmpBuf:\n"
+                                    "!!\n");
+                    pHlp->pfnPrintf(pHlp,
+                                    "SavedEsp=%RHv SavedEbp=%RHv SpResume=%RHv SpCheck=%RHv\n",
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.SavedEsp,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.SpResume,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck);
+                    pHlp->pfnPrintf(pHlp,
+                                    "pvSavedStack=%RHv cbSavedStack=%#x  fInRing3Call=%RTbool\n",
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.pvSavedStack,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.fInRing3Call);
+                    pHlp->pfnPrintf(pHlp,
+                                    "cbUsedMax=%#x cbUsedAvg=%#x cbUsedTotal=%#llx cUsedTotal=%#llx\n",
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedMax,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedAvg,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.cbUsedTotal,
+                                    pVCpu->vmm.s.CallRing3JmpBufR0.cUsedTotal);
+
+                    /* Dump the resume register frame on the stack. */
+                    PRTHCUINTPTR pBP;
+#ifdef VMM_R0_SWITCH_STACK
+                    pBP = (PRTHCUINTPTR)&pVCpu->vmm.s.pbEMTStackR3[  pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp
+                                                                   - MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3)];
+#else
+                    pBP = (PRTHCUINTPTR)&pVCpu->vmm.s.pbEMTStackR3[  pVCpu->vmm.s.CallRing3JmpBufR0.cbSavedStack
+                                                                   - pVCpu->vmm.s.CallRing3JmpBufR0.SpCheck
+                                                                   + pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp];
+#endif
+#if HC_ARCH_BITS == 32
+                    pHlp->pfnPrintf(pHlp,
+                                    "eax=volatile ebx=%08x ecx=volatile edx=volatile esi=%08x edi=%08x\n"
+                                    "eip=%08x esp=%08x ebp=%08x efl=%08x\n"
+                                    ,
+                                    pBP[-3], pBP[-2], pBP[-1],
+                                    pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 8, pBP[0], pBP[-4]);
+#else
+# ifdef RT_OS_WINDOWS
+                    pHlp->pfnPrintf(pHlp,
+                                    "rax=volatile         rbx=%016RX64 rcx=volatile         rdx=volatile\n"
+                                    "rsi=%016RX64 rdi=%016RX64  r8=volatile          r9=volatile        \n"
+                                    "r10=volatile         r11=volatile         r12=%016RX64 r13=%016RX64\n"
+                                    "r14=%016RX64 r15=%016RX64\n"
+                                    "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 rfl=%08RX64\n"
+                                    ,
+                                    pBP[-7],
+                                    pBP[-6], pBP[-5],
+                                    pBP[-4], pBP[-3],
+                                    pBP[-2], pBP[-1],
+                                    pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 16, pBP[0], pBP[-8]);
+# else
+                    pHlp->pfnPrintf(pHlp,
+                                    "rax=volatile         rbx=%016RX64 rcx=volatile         rdx=volatile\n"
+                                    "rsi=volatile         rdi=volatile          r8=volatile          r9=volatile        \n"
+                                    "r10=volatile         r11=volatile         r12=%016RX64 r13=%016RX64\n"
+                                    "r14=%016RX64 r15=%016RX64\n"
+                                    "rip=%016RX64 rsp=%016RX64 rbp=%016RX64 rflags=%08RX64\n"
+                                    ,
+                                    pBP[-5],
+                                    pBP[-4], pBP[-3],
+                                    pBP[-2], pBP[-1],
+                                    pBP[1], pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp - 16, pBP[0], pBP[-6]);
+# endif
+#endif
+
+                    /* Callstack. */
+                    DBGFADDRESS pc;
+                    pc.fFlags    = DBGFADDRESS_FLAGS_RING0 | DBGFADDRESS_FLAGS_VALID;
+#if HC_ARCH_BITS == 64
+                    pc.FlatPtr   = pc.off = pVCpu->vmm.s.CallRing3JmpBufR0.rip;
+#else
+                    pc.FlatPtr   = pc.off = pVCpu->vmm.s.CallRing3JmpBufR0.eip;
+#endif
+                    pc.Sel       = DBGF_SEL_FLAT;
+
+                    DBGFADDRESS ebp;
+                    ebp.fFlags   = DBGFADDRESS_FLAGS_RING0 | DBGFADDRESS_FLAGS_VALID;
+                    ebp.FlatPtr  = ebp.off = pVCpu->vmm.s.CallRing3JmpBufR0.SavedEbp;
+                    ebp.Sel      = DBGF_SEL_FLAT;
+
+                    DBGFADDRESS esp;
+                    esp.fFlags   = DBGFADDRESS_FLAGS_RING0 | DBGFADDRESS_FLAGS_VALID;
+                    esp.Sel      = DBGF_SEL_FLAT;
+                    esp.FlatPtr  = esp.off = pVCpu->vmm.s.CallRing3JmpBufR0.SavedEsp;
+
+                    PCDBGFSTACKFRAME pFirstFrame;
+                    rc2 = DBGFR3StackWalkBeginEx(pVM, pVCpu->idCpu, DBGFCODETYPE_RING0, &ebp, &esp, &pc,
+                                                 DBGFRETURNTYPE_INVALID, &pFirstFrame);
+                    if (RT_SUCCESS(rc2))
+                    {
+                        pHlp->pfnPrintf(pHlp,
+                                        "!!\n"
+                                        "!! Call Stack:\n"
+                                        "!!\n");
+#if HC_ARCH_BITS == 32
+                        pHlp->pfnPrintf(pHlp, "EBP      Ret EBP  Ret CS:EIP    Arg0     Arg1     Arg2     Arg3     CS:EIP        Symbol [line]\n");
+#else
+                        pHlp->pfnPrintf(pHlp, "RBP              Ret RBP          Ret RIP          RIP              Symbol [line]\n");
+#endif
+                        for (PCDBGFSTACKFRAME pFrame = pFirstFrame;
+                             pFrame;
+                             pFrame = DBGFR3StackWalkNext(pFrame))
+                        {
+#if HC_ARCH_BITS == 32
+                            pHlp->pfnPrintf(pHlp,
+                                            "%RHv %RHv %04RX32:%RHv %RHv %RHv %RHv %RHv",
+                                            (RTHCUINTPTR)pFrame->AddrFrame.off,
+                                            (RTHCUINTPTR)pFrame->AddrReturnFrame.off,
+                                            (RTHCUINTPTR)pFrame->AddrReturnPC.Sel,
+                                            (RTHCUINTPTR)pFrame->AddrReturnPC.off,
+                                            pFrame->Args.au32[0],
+                                            pFrame->Args.au32[1],
+                                            pFrame->Args.au32[2],
+                                            pFrame->Args.au32[3]);
+                            pHlp->pfnPrintf(pHlp, " %RTsel:%08RHv", pFrame->AddrPC.Sel, pFrame->AddrPC.off);
+#else
+                            pHlp->pfnPrintf(pHlp,
+                                            "%RHv %RHv %RHv %RHv",
+                                            (RTHCUINTPTR)pFrame->AddrFrame.off,
+                                            (RTHCUINTPTR)pFrame->AddrReturnFrame.off,
+                                            (RTHCUINTPTR)pFrame->AddrReturnPC.off,
+                                            (RTHCUINTPTR)pFrame->AddrPC.off);
+#endif
+                            if (pFrame->pSymPC)
+                            {
+                                RTGCINTPTR offDisp = pFrame->AddrPC.FlatPtr - pFrame->pSymPC->Value;
+                                if (offDisp > 0)
+                                    pHlp->pfnPrintf(pHlp, " %s+%llx", pFrame->pSymPC->szName, (int64_t)offDisp);
+                                else if (offDisp < 0)
+                                    pHlp->pfnPrintf(pHlp, " %s-%llx", pFrame->pSymPC->szName, -(int64_t)offDisp);
+                                else
+                                    pHlp->pfnPrintf(pHlp, " %s", pFrame->pSymPC->szName);
+                            }
+                            if (pFrame->pLinePC)
+                                pHlp->pfnPrintf(pHlp, " [%s @ 0i%d]", pFrame->pLinePC->szFilename, pFrame->pLinePC->uLineNo);
+                            pHlp->pfnPrintf(pHlp, "\n");
+                        }
+                        DBGFR3StackWalkEnd(pFirstFrame);
+                    }
+
+                    /* raw stack */
+                    Hlp.fRecSummary = false;
+                    pHlp->pfnPrintf(pHlp,
+                                    "!!\n"
+                                    "!! Raw stack (mind the direction). \n"
+                                    "!! pbEMTStackR0=%RHv pbEMTStackBottomR0=%RHv VMM_STACK_SIZE=%#x\n"
+                                    "!! pbEmtStackR3=%p\n"
+                                    "!!\n"
+                                    "%.*Rhxd\n",
+                                    MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3),
+                                    MMHyperCCToR0(pVM, pVCpu->vmm.s.pbEMTStackR3) + VMM_STACK_SIZE,
+                                    VMM_STACK_SIZE,
+                                    pVCpu->vmm.s.pbEMTStackR3,
+                                    VMM_STACK_SIZE, pVCpu->vmm.s.pbEMTStackR3);
+                }
+                else
+                {
+                    pHlp->pfnPrintf(pHlp,
+                                    "!! Skipping ring-0 registers and stack, rcErr=%Rrc\n", rcErr);
+                }
+            }
+            else
+            {
+                /*
+                 * Try figure out where eip is.
+                 */
+                /* core code? */
+                if (uEIP - (RTGCUINTPTR)pVM->vmm.s.pvCoreCodeRC < pVM->vmm.s.cbCoreCode)
+                    pHlp->pfnPrintf(pHlp,
+                                "!! EIP is in CoreCode, offset %#x\n",
+                                uEIP - (RTGCUINTPTR)pVM->vmm.s.pvCoreCodeRC);
+                else
+                {   /* ask PDM */  /** @todo ask DBGFR3Sym later? */
+                    char        szModName[64];
+                    RTRCPTR     RCPtrMod;
+                    char        szNearSym1[260];
+                    RTRCPTR     RCPtrNearSym1;
+                    char        szNearSym2[260];
+                    RTRCPTR     RCPtrNearSym2;
+                    int rc = PDMR3LdrQueryRCModFromPC(pVM, uEIP,
+                                                      &szModName[0],  sizeof(szModName),  &RCPtrMod,
+                                                      &szNearSym1[0], sizeof(szNearSym1), &RCPtrNearSym1,
+                                                      &szNearSym2[0], sizeof(szNearSym2), &RCPtrNearSym2);
+                    if (RT_SUCCESS(rc))
+                        pHlp->pfnPrintf(pHlp,
+                                        "!! EIP in %s (%RRv) at rva %x near symbols:\n"
+                                        "!!    %RRv rva %RRv off %08x  %s\n"
+                                        "!!    %RRv rva %RRv off -%08x %s\n",
+                                        szModName,  RCPtrMod, (unsigned)(uEIP - RCPtrMod),
+                                        RCPtrNearSym1, RCPtrNearSym1 - RCPtrMod, (unsigned)(uEIP - RCPtrNearSym1), szNearSym1,
+                                        RCPtrNearSym2, RCPtrNearSym2 - RCPtrMod, (unsigned)(RCPtrNearSym2 - uEIP), szNearSym2);
+                    else
+                        pHlp->pfnPrintf(pHlp,
+                                        "!! EIP is not in any code known to VMM!\n");
+                }
+
+                /* Disassemble the instruction. */
+                char szInstr[256];
+                rc2 = DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0, DBGF_DISAS_FLAGS_CURRENT_HYPER | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                                         &szInstr[0], sizeof(szInstr), NULL);
+                if (RT_SUCCESS(rc2))
+                    pHlp->pfnPrintf(pHlp,
+                                    "!! %s\n", szInstr);
+
+                /* Dump the hypervisor cpu state. */
+                pHlp->pfnPrintf(pHlp,
+                                "!!\n"
+                                "!!\n"
+                                "!!\n");
+                rc2 = DBGFR3Info(pVM, "cpumhyper", "verbose", pHlp);
+                fDoneHyper = true;
+
+                /* Callstack. */
+                PCDBGFSTACKFRAME pFirstFrame;
+                rc2 = DBGFR3StackWalkBegin(pVM, pVCpu->idCpu, DBGFCODETYPE_HYPER, &pFirstFrame);
+                if (RT_SUCCESS(rc2))
+                {
+                    pHlp->pfnPrintf(pHlp,
+                                    "!!\n"
+                                    "!! Call Stack:\n"
+                                    "!!\n"
+                                    "EBP      Ret EBP  Ret CS:EIP    Arg0     Arg1     Arg2     Arg3     CS:EIP        Symbol [line]\n");
+                    for (PCDBGFSTACKFRAME pFrame = pFirstFrame;
+                         pFrame;
+                         pFrame = DBGFR3StackWalkNext(pFrame))
+                    {
+                        pHlp->pfnPrintf(pHlp,
+                                        "%08RX32 %08RX32 %04RX32:%08RX32 %08RX32 %08RX32 %08RX32 %08RX32",
+                                        (uint32_t)pFrame->AddrFrame.off,
+                                        (uint32_t)pFrame->AddrReturnFrame.off,
+                                        (uint32_t)pFrame->AddrReturnPC.Sel,
+                                        (uint32_t)pFrame->AddrReturnPC.off,
+                                        pFrame->Args.au32[0],
+                                        pFrame->Args.au32[1],
+                                        pFrame->Args.au32[2],
+                                        pFrame->Args.au32[3]);
+                        pHlp->pfnPrintf(pHlp, " %RTsel:%08RGv", pFrame->AddrPC.Sel, pFrame->AddrPC.off);
+                        if (pFrame->pSymPC)
+                        {
+                            RTGCINTPTR offDisp = pFrame->AddrPC.FlatPtr - pFrame->pSymPC->Value;
+                            if (offDisp > 0)
+                                pHlp->pfnPrintf(pHlp, " %s+%llx", pFrame->pSymPC->szName, (int64_t)offDisp);
+                            else if (offDisp < 0)
+                                pHlp->pfnPrintf(pHlp, " %s-%llx", pFrame->pSymPC->szName, -(int64_t)offDisp);
+                            else
+                                pHlp->pfnPrintf(pHlp, " %s", pFrame->pSymPC->szName);
+                        }
+                        if (pFrame->pLinePC)
+                            pHlp->pfnPrintf(pHlp, " [%s @ 0i%d]", pFrame->pLinePC->szFilename, pFrame->pLinePC->uLineNo);
+                        pHlp->pfnPrintf(pHlp, "\n");
+                    }
+                    DBGFR3StackWalkEnd(pFirstFrame);
+                }
+
+                /* raw stack */
+                Hlp.fRecSummary = false;
+                pHlp->pfnPrintf(pHlp,
+                                "!!\n"
+                                "!! Raw stack (mind the direction). pbEMTStackRC=%RRv pbEMTStackBottomRC=%RRv\n"
+                                "!!\n"
+                                "%.*Rhxd\n",
+                                pVCpu->vmm.s.pbEMTStackRC, pVCpu->vmm.s.pbEMTStackBottomRC,
+                                VMM_STACK_SIZE, pVCpu->vmm.s.pbEMTStackR3);
+            } /* !HWACCMIsEnabled */
+            break;
+        }
+
+        default:
+        {
+            break;
+        }
+
+    } /* switch (rcErr) */
+    Hlp.fRecSummary = false;
+
+
+    /*
+     * Generic info dumper loop.
+     */
+    static struct
+    {
+        const char *pszInfo;
+        const char *pszArgs;
+    } const     aInfo[] =
+    {
+        { "mappings",       NULL },
+        { "hma",            NULL },
+        { "cpumguest",      "verbose" },
+        { "cpumguestinstr", "verbose" },
+        { "cpumhyper",      "verbose" },
+        { "cpumhost",       "verbose" },
+        { "mode",           "all" },
+        { "cpuid",          "verbose" },
+        { "handlers",       "phys virt hyper stats" },
+        { "timers",         NULL },
+        { "activetimers",   NULL },
+    };
+    for (unsigned i = 0; i < RT_ELEMENTS(aInfo); i++)
+    {
+        if (fDoneHyper && !strcmp(aInfo[i].pszInfo, "cpumhyper"))
+            continue;
+        pHlp->pfnPrintf(pHlp,
+                        "!!\n"
+                        "!! {%s, %s}\n"
+                        "!!\n",
+                        aInfo[i].pszInfo, aInfo[i].pszArgs);
+        DBGFR3Info(pVM, aInfo[i].pszInfo, aInfo[i].pszArgs, pHlp);
+    }
+
+    /* All other info items */
+    DBGFR3InfoMulti(pVM,
+                    "*",
+                    "mappings|hma|cpum|cpumguest|cpumguestinstr|cpumhyper|cpumhost|mode|cpuid"
+                    "|pgmpd|pgmcr3|timers|activetimers|handlers|help",
+                    "!!\n"
+                    "!! {%s}\n"
+                    "!!\n",
+                    pHlp);
+
+
+    /* done */
+    pHlp->pfnPrintf(pHlp,
+                    "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");
+
+
+    /*
+     * Repeat the summary to stderr so we don't have to scroll half a mile up.
+     */
+    if (Hlp.szSummary[0])
+        RTStrmPrintf(g_pStdErr,
+                     "%s"
+                     "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
+                     Hlp.szSummary);
+
+    /*
+     * Delete the output instance (flushing and restoring of flags).
+     */
+    vmmR3FatalDumpInfoHlpDelete(&Hlp);
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/VMMSwitcher.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMMSwitcher.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMMSwitcher.cpp	(revision 35333)
@@ -0,0 +1,1030 @@
+/* $Id$ */
+/** @file
+ * VMM - The Virtual Machine Monitor, World Switcher(s).
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm.h>
+#include <VBox/pgm.h>
+#include <VBox/selm.h>
+#include <VBox/mm.h>
+#include <VBox/sup.h>
+#include "VMMInternal.h"
+#include "VMMSwitcher.h"
+#include <VBox/vm.h>
+#include <VBox/dis.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/alloc.h>
+#include <iprt/asm.h>
+#include <iprt/asm-amd64-x86.h>
+#include <iprt/string.h>
+#include <iprt/ctype.h>
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+/** Array of switcher definitions.
+ * The type and index shall match!
+ */
+static PVMMSWITCHERDEF s_apSwitchers[VMMSWITCHER_MAX] =
+{
+    NULL, /* invalid entry */
+#ifdef VBOX_WITH_RAW_MODE
+# ifndef RT_ARCH_AMD64
+    &vmmR3Switcher32BitTo32Bit_Def,
+    &vmmR3Switcher32BitToPAE_Def,
+    &vmmR3Switcher32BitToAMD64_Def,
+    &vmmR3SwitcherPAETo32Bit_Def,
+    &vmmR3SwitcherPAEToPAE_Def,
+    &vmmR3SwitcherPAEToAMD64_Def,
+    NULL,   //&vmmR3SwitcherPAETo32Bit_Def,
+#  ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
+    &vmmR3SwitcherAMD64ToPAE_Def,
+#  else
+    NULL,   //&vmmR3SwitcherAMD64ToPAE_Def,
+#  endif
+    NULL    //&vmmR3SwitcherAMD64ToAMD64_Def,
+# else  /* RT_ARCH_AMD64 */
+    NULL,   //&vmmR3Switcher32BitTo32Bit_Def,
+    NULL,   //&vmmR3Switcher32BitToPAE_Def,
+    NULL,   //&vmmR3Switcher32BitToAMD64_Def,
+    NULL,   //&vmmR3SwitcherPAETo32Bit_Def,
+    NULL,   //&vmmR3SwitcherPAEToPAE_Def,
+    NULL,   //&vmmR3SwitcherPAEToAMD64_Def,
+    &vmmR3SwitcherAMD64To32Bit_Def,
+    &vmmR3SwitcherAMD64ToPAE_Def,
+    NULL    //&vmmR3SwitcherAMD64ToAMD64_Def,
+# endif /* RT_ARCH_AMD64 */
+#else  /* !VBOX_WITH_RAW_MODE */
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL
+#endif /* !VBOX_WITH_RAW_MODE */
+};
+
+
+/**
+ * VMMR3Init worker that initiates the switcher code (aka core code).
+ *
+ * This is core per VM code which might need fixups and/or for ease of use are
+ * put on linear contiguous backing.
+ *
+ * @returns VBox status code.
+ * @param   pVM     Pointer to the shared VM structure.
+ */
+int vmmR3SwitcherInit(PVM pVM)
+{
+#ifndef VBOX_WITH_RAW_MODE
+    return VINF_SUCCESS;
+#else
+    /*
+     * Calc the size.
+     */
+    unsigned cbCoreCode = 0;
+    for (unsigned iSwitcher = 0; iSwitcher < RT_ELEMENTS(s_apSwitchers); iSwitcher++)
+    {
+        pVM->vmm.s.aoffSwitchers[iSwitcher] = cbCoreCode;
+        PVMMSWITCHERDEF pSwitcher = s_apSwitchers[iSwitcher];
+        if (pSwitcher)
+        {
+            AssertRelease((unsigned)pSwitcher->enmType == iSwitcher);
+            cbCoreCode += RT_ALIGN_32(pSwitcher->cbCode + 1, 32);
+        }
+    }
+
+    /*
+     * Allocate contiguous pages for switchers and deal with
+     * conflicts in the intermediate mapping of the code.
+     */
+    pVM->vmm.s.cbCoreCode = RT_ALIGN_32(cbCoreCode, PAGE_SIZE);
+    pVM->vmm.s.pvCoreCodeR3 = SUPR3ContAlloc(pVM->vmm.s.cbCoreCode >> PAGE_SHIFT, &pVM->vmm.s.pvCoreCodeR0, &pVM->vmm.s.HCPhysCoreCode);
+    int rc = VERR_NO_MEMORY;
+    if (pVM->vmm.s.pvCoreCodeR3)
+    {
+        rc = PGMR3MapIntermediate(pVM, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.HCPhysCoreCode, cbCoreCode);
+        if (rc == VERR_PGM_INTERMEDIATE_PAGING_CONFLICT)
+        {
+            /* try more allocations - Solaris, Linux.  */
+            const unsigned cTries = 8234;
+            struct VMMInitBadTry
+            {
+                RTR0PTR  pvR0;
+                void    *pvR3;
+                RTHCPHYS HCPhys;
+                RTUINT   cb;
+            } *paBadTries = (struct VMMInitBadTry *)RTMemTmpAlloc(sizeof(*paBadTries) * cTries);
+            AssertReturn(paBadTries, VERR_NO_TMP_MEMORY);
+            unsigned i = 0;
+            do
+            {
+                paBadTries[i].pvR3 = pVM->vmm.s.pvCoreCodeR3;
+                paBadTries[i].pvR0 = pVM->vmm.s.pvCoreCodeR0;
+                paBadTries[i].HCPhys = pVM->vmm.s.HCPhysCoreCode;
+                i++;
+                pVM->vmm.s.pvCoreCodeR0 = NIL_RTR0PTR;
+                pVM->vmm.s.HCPhysCoreCode = NIL_RTHCPHYS;
+                pVM->vmm.s.pvCoreCodeR3 = SUPR3ContAlloc(pVM->vmm.s.cbCoreCode >> PAGE_SHIFT, &pVM->vmm.s.pvCoreCodeR0, &pVM->vmm.s.HCPhysCoreCode);
+                if (!pVM->vmm.s.pvCoreCodeR3)
+                    break;
+                rc = PGMR3MapIntermediate(pVM, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.HCPhysCoreCode, cbCoreCode);
+            } while (   rc == VERR_PGM_INTERMEDIATE_PAGING_CONFLICT
+                     && i < cTries - 1);
+
+            /* cleanup */
+            if (RT_FAILURE(rc))
+            {
+                paBadTries[i].pvR3   = pVM->vmm.s.pvCoreCodeR3;
+                paBadTries[i].pvR0   = pVM->vmm.s.pvCoreCodeR0;
+                paBadTries[i].HCPhys = pVM->vmm.s.HCPhysCoreCode;
+                paBadTries[i].cb     = pVM->vmm.s.cbCoreCode;
+                i++;
+                LogRel(("Failed to allocated and map core code: rc=%Rrc\n", rc));
+            }
+            while (i-- > 0)
+            {
+                LogRel(("Core code alloc attempt #%d: pvR3=%p pvR0=%p HCPhys=%RHp\n",
+                        i, paBadTries[i].pvR3, paBadTries[i].pvR0, paBadTries[i].HCPhys));
+                SUPR3ContFree(paBadTries[i].pvR3, paBadTries[i].cb >> PAGE_SHIFT);
+            }
+            RTMemTmpFree(paBadTries);
+        }
+    }
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * copy the code.
+         */
+        for (unsigned iSwitcher = 0; iSwitcher < RT_ELEMENTS(s_apSwitchers); iSwitcher++)
+        {
+            PVMMSWITCHERDEF pSwitcher = s_apSwitchers[iSwitcher];
+            if (pSwitcher)
+                memcpy((uint8_t *)pVM->vmm.s.pvCoreCodeR3 + pVM->vmm.s.aoffSwitchers[iSwitcher],
+                       pSwitcher->pvCode, pSwitcher->cbCode);
+        }
+
+        /*
+         * Map the code into the GC address space.
+         */
+        RTGCPTR GCPtr;
+        rc = MMR3HyperMapHCPhys(pVM, pVM->vmm.s.pvCoreCodeR3, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.HCPhysCoreCode,
+                                cbCoreCode, "Core Code", &GCPtr);
+        if (RT_SUCCESS(rc))
+        {
+            pVM->vmm.s.pvCoreCodeRC = GCPtr;
+            MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
+            LogRel(("CoreCode: R3=%RHv R0=%RHv RC=%RRv Phys=%RHp cb=%#x\n",
+                    pVM->vmm.s.pvCoreCodeR3, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.pvCoreCodeRC, pVM->vmm.s.HCPhysCoreCode, pVM->vmm.s.cbCoreCode));
+
+            /*
+             * Finally, PGM probably has selected a switcher already but we need
+             * to get the routine addresses, so we'll reselect it.
+             * This may legally fail so, we're ignoring the rc.
+             */
+            VMMR3SelectSwitcher(pVM, pVM->vmm.s.enmSwitcher);
+            return rc;
+        }
+
+        /* shit */
+        AssertMsgFailed(("PGMR3Map(,%RRv, %RHp, %#x, 0) failed with rc=%Rrc\n", pVM->vmm.s.pvCoreCodeRC, pVM->vmm.s.HCPhysCoreCode, cbCoreCode, rc));
+        SUPR3ContFree(pVM->vmm.s.pvCoreCodeR3, pVM->vmm.s.cbCoreCode >> PAGE_SHIFT);
+    }
+    else
+        VMSetError(pVM, rc, RT_SRC_POS,
+                   N_("Failed to allocate %d bytes of contiguous memory for the world switcher code"),
+                   cbCoreCode);
+
+    pVM->vmm.s.pvCoreCodeR3 = NULL;
+    pVM->vmm.s.pvCoreCodeR0 = NIL_RTR0PTR;
+    pVM->vmm.s.pvCoreCodeRC = 0;
+    return rc;
+#endif
+}
+
+/**
+ * Relocate the switchers, called by VMMR#Relocate.
+ *
+ * @param   pVM         Pointer to the shared VM structure.
+ * @param   offDelta    The relocation delta.
+ */
+void vmmR3SwitcherRelocate(PVM pVM, RTGCINTPTR offDelta)
+{
+#ifdef VBOX_WITH_RAW_MODE
+    /*
+     * Relocate all the switchers.
+     */
+    for (unsigned iSwitcher = 0; iSwitcher < RT_ELEMENTS(s_apSwitchers); iSwitcher++)
+    {
+        PVMMSWITCHERDEF pSwitcher = s_apSwitchers[iSwitcher];
+        if (pSwitcher && pSwitcher->pfnRelocate)
+        {
+            unsigned off = pVM->vmm.s.aoffSwitchers[iSwitcher];
+            pSwitcher->pfnRelocate(pVM,
+                                   pSwitcher,
+                                   pVM->vmm.s.pvCoreCodeR0 + off,
+                                   (uint8_t *)pVM->vmm.s.pvCoreCodeR3 + off,
+                                   pVM->vmm.s.pvCoreCodeRC + off,
+                                   pVM->vmm.s.HCPhysCoreCode + off);
+        }
+    }
+
+    /*
+     * Recalc the RC address for the current switcher.
+     */
+    PVMMSWITCHERDEF pSwitcher   = s_apSwitchers[pVM->vmm.s.enmSwitcher];
+    RTRCPTR         RCPtr       = pVM->vmm.s.pvCoreCodeRC + pVM->vmm.s.aoffSwitchers[pVM->vmm.s.enmSwitcher];
+    pVM->vmm.s.pfnGuestToHostRC         = RCPtr + pSwitcher->offGCGuestToHost;
+    pVM->vmm.s.pfnCallTrampolineRC      = RCPtr + pSwitcher->offGCCallTrampoline;
+    pVM->pfnVMMGCGuestToHostAsm         = RCPtr + pSwitcher->offGCGuestToHostAsm;
+    pVM->pfnVMMGCGuestToHostAsmHyperCtx = RCPtr + pSwitcher->offGCGuestToHostAsmHyperCtx;
+    pVM->pfnVMMGCGuestToHostAsmGuestCtx = RCPtr + pSwitcher->offGCGuestToHostAsmGuestCtx;
+
+//    AssertFailed();
+#endif
+}
+
+
+/**
+ * Generic switcher code relocator.
+ *
+ * @param   pVM         The VM handle.
+ * @param   pSwitcher   The switcher definition.
+ * @param   pu8CodeR3   Pointer to the core code block for the switcher, ring-3 mapping.
+ * @param   R0PtrCode   Pointer to the core code block for the switcher, ring-0 mapping.
+ * @param   GCPtrCode   The guest context address corresponding to pu8Code.
+ * @param   u32IDCode   The identity mapped (ID) address corresponding to pu8Code.
+ * @param   SelCS       The hypervisor CS selector.
+ * @param   SelDS       The hypervisor DS selector.
+ * @param   SelTSS      The hypervisor TSS selector.
+ * @param   GCPtrGDT    The GC address of the hypervisor GDT.
+ * @param   SelCS64     The 64-bit mode hypervisor CS selector.
+ */
+static void vmmR3SwitcherGenericRelocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode,
+                                         RTSEL SelCS, RTSEL SelDS, RTSEL SelTSS, RTGCPTR GCPtrGDT, RTSEL SelCS64)
+{
+    union
+    {
+        const uint8_t *pu8;
+        const uint16_t *pu16;
+        const uint32_t *pu32;
+        const uint64_t *pu64;
+        const void     *pv;
+        uintptr_t       u;
+    } u;
+    u.pv = pSwitcher->pvFixups;
+
+    /*
+     * Process fixups.
+     */
+    uint8_t u8;
+    while ((u8 = *u.pu8++) != FIX_THE_END)
+    {
+        /*
+         * Get the source (where to write the fixup).
+         */
+        uint32_t offSrc = *u.pu32++;
+        Assert(offSrc < pSwitcher->cbCode);
+        union
+        {
+            uint8_t    *pu8;
+            uint16_t   *pu16;
+            uint32_t   *pu32;
+            uint64_t   *pu64;
+            uintptr_t   u;
+        } uSrc;
+        uSrc.pu8 = pu8CodeR3 + offSrc;
+
+        /* The fixup target and method depends on the type. */
+        switch (u8)
+        {
+            /*
+             * 32-bit relative, source in HC and target in GC.
+             */
+            case FIX_HC_2_GC_NEAR_REL:
+            {
+                Assert(offSrc - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offSrc - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg - pSwitcher->offGCCode < pSwitcher->cbGCCode);
+                *uSrc.pu32 = (uint32_t)((GCPtrCode + offTrg) - (uSrc.u + 4));
+                break;
+            }
+
+            /*
+             * 32-bit relative, source in HC and target in ID.
+             */
+            case FIX_HC_2_ID_NEAR_REL:
+            {
+                Assert(offSrc - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offSrc - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
+                *uSrc.pu32 = (uint32_t)((u32IDCode + offTrg) - (R0PtrCode + offSrc + 4));
+                break;
+            }
+
+            /*
+             * 32-bit relative, source in GC and target in HC.
+             */
+            case FIX_GC_2_HC_NEAR_REL:
+            {
+                Assert(offSrc - pSwitcher->offGCCode < pSwitcher->cbGCCode);
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
+                *uSrc.pu32 = (uint32_t)((R0PtrCode + offTrg) - (GCPtrCode + offSrc + 4));
+                break;
+            }
+
+            /*
+             * 32-bit relative, source in GC and target in ID.
+             */
+            case FIX_GC_2_ID_NEAR_REL:
+            {
+                AssertMsg(offSrc - pSwitcher->offGCCode < pSwitcher->cbGCCode, ("%x - %x < %x\n", offSrc, pSwitcher->offGCCode, pSwitcher->cbGCCode));
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
+                *uSrc.pu32 = (uint32_t)((u32IDCode + offTrg) - (GCPtrCode + offSrc + 4));
+                break;
+            }
+
+            /*
+             * 32-bit relative, source in ID and target in HC.
+             */
+            case FIX_ID_2_HC_NEAR_REL:
+            {
+                Assert(offSrc - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offSrc - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
+                *uSrc.pu32 = (uint32_t)((R0PtrCode + offTrg) - (u32IDCode + offSrc + 4));
+                break;
+            }
+
+            /*
+             * 32-bit relative, source in ID and target in HC.
+             */
+            case FIX_ID_2_GC_NEAR_REL:
+            {
+                Assert(offSrc - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offSrc - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg - pSwitcher->offGCCode < pSwitcher->cbGCCode);
+                *uSrc.pu32 = (uint32_t)((GCPtrCode + offTrg) - (u32IDCode + offSrc + 4));
+                break;
+            }
+
+            /*
+             * 16:32 far jump, target in GC.
+             */
+            case FIX_GC_FAR32:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg - pSwitcher->offGCCode < pSwitcher->cbGCCode);
+                *uSrc.pu32++ = (uint32_t)(GCPtrCode + offTrg);
+                *uSrc.pu16++ = SelCS;
+                break;
+            }
+
+            /*
+             * Make 32-bit GC pointer given CPUM offset.
+             */
+            case FIX_GC_CPUM_OFF:
+            {
+                uint32_t offCPUM = *u.pu32++;
+                Assert(offCPUM < sizeof(pVM->cpum));
+                *uSrc.pu32 = (uint32_t)(VM_RC_ADDR(pVM, &pVM->cpum) + offCPUM);
+                break;
+            }
+
+            /*
+             * Make 32-bit GC pointer given CPUMCPU offset.
+             */
+            case FIX_GC_CPUMCPU_OFF:
+            {
+                uint32_t offCPUM = *u.pu32++;
+                Assert(offCPUM < sizeof(pVM->aCpus[0].cpum));
+                *uSrc.pu32 = (uint32_t)(VM_RC_ADDR(pVM, &pVM->aCpus[0].cpum) + offCPUM);
+                break;
+            }
+
+            /*
+             * Make 32-bit GC pointer given VM offset.
+             */
+            case FIX_GC_VM_OFF:
+            {
+                uint32_t offVM = *u.pu32++;
+                Assert(offVM < sizeof(VM));
+                *uSrc.pu32 = (uint32_t)(VM_RC_ADDR(pVM, pVM) + offVM);
+                break;
+            }
+
+            /*
+             * Make 32-bit HC pointer given CPUM offset.
+             */
+            case FIX_HC_CPUM_OFF:
+            {
+                uint32_t offCPUM = *u.pu32++;
+                Assert(offCPUM < sizeof(pVM->cpum));
+                *uSrc.pu32 = (uint32_t)pVM->pVMR0 + RT_OFFSETOF(VM, cpum) + offCPUM;
+                break;
+            }
+
+            /*
+             * Make 32-bit R0 pointer given VM offset.
+             */
+            case FIX_HC_VM_OFF:
+            {
+                uint32_t offVM = *u.pu32++;
+                Assert(offVM < sizeof(VM));
+                *uSrc.pu32 = (uint32_t)pVM->pVMR0 + offVM;
+                break;
+            }
+
+            /*
+             * Store the 32-Bit CR3 (32-bit) for the intermediate memory context.
+             */
+            case FIX_INTER_32BIT_CR3:
+            {
+
+                *uSrc.pu32 = PGMGetInter32BitCR3(pVM);
+                break;
+            }
+
+            /*
+             * Store the PAE CR3 (32-bit) for the intermediate memory context.
+             */
+            case FIX_INTER_PAE_CR3:
+            {
+
+                *uSrc.pu32 = PGMGetInterPaeCR3(pVM);
+                break;
+            }
+
+            /*
+             * Store the AMD64 CR3 (32-bit) for the intermediate memory context.
+             */
+            case FIX_INTER_AMD64_CR3:
+            {
+
+                *uSrc.pu32 = PGMGetInterAmd64CR3(pVM);
+                break;
+            }
+
+            /*
+             * Store Hypervisor CS (16-bit).
+             */
+            case FIX_HYPER_CS:
+            {
+                *uSrc.pu16 = SelCS;
+                break;
+            }
+
+            /*
+             * Store Hypervisor DS (16-bit).
+             */
+            case FIX_HYPER_DS:
+            {
+                *uSrc.pu16 = SelDS;
+                break;
+            }
+
+            /*
+             * Store Hypervisor TSS (16-bit).
+             */
+            case FIX_HYPER_TSS:
+            {
+                *uSrc.pu16 = SelTSS;
+                break;
+            }
+
+            /*
+             * Store the 32-bit GC address of the 2nd dword of the TSS descriptor (in the GDT).
+             */
+            case FIX_GC_TSS_GDTE_DW2:
+            {
+                RTGCPTR GCPtr = GCPtrGDT + (SelTSS & ~7) + 4;
+                *uSrc.pu32 = (uint32_t)GCPtr;
+                break;
+            }
+
+            /*
+             * Store the EFER or mask for the 32->64 bit switcher.
+             */
+            case FIX_EFER_OR_MASK:
+            {
+                uint32_t u32OrMask = MSR_K6_EFER_LME | MSR_K6_EFER_SCE;
+                /** note: we don't care if cpuid 0x8000001 isn't supported as that implies long mode isn't either, so this switcher would never be used. */
+                if (!!(ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_NX))
+                    u32OrMask |= MSR_K6_EFER_NXE;
+
+                *uSrc.pu32 = u32OrMask;
+                break;
+            }
+
+            /*
+             * Insert relative jump to specified target it FXSAVE/FXRSTOR isn't supported by the cpu.
+             */
+            case FIX_NO_FXSAVE_JMP:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg < pSwitcher->cbCode);
+                if (!CPUMSupportsFXSR(pVM))
+                {
+                    *uSrc.pu8++ = 0xe9; /* jmp rel32 */
+                    *uSrc.pu32++ = offTrg - (offSrc + 5);
+                }
+                else
+                {
+                    *uSrc.pu8++ = *((uint8_t *)pSwitcher->pvCode + offSrc);
+                    *uSrc.pu32++ = *(uint32_t *)((uint8_t *)pSwitcher->pvCode + offSrc + 1);
+                }
+                break;
+            }
+
+            /*
+             * Insert relative jump to specified target it SYSENTER isn't used by the host.
+             */
+            case FIX_NO_SYSENTER_JMP:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg < pSwitcher->cbCode);
+                if (!CPUMIsHostUsingSysEnter(pVM))
+                {
+                    *uSrc.pu8++ = 0xe9; /* jmp rel32 */
+                    *uSrc.pu32++ = offTrg - (offSrc + 5);
+                }
+                else
+                {
+                    *uSrc.pu8++ = *((uint8_t *)pSwitcher->pvCode + offSrc);
+                    *uSrc.pu32++ = *(uint32_t *)((uint8_t *)pSwitcher->pvCode + offSrc + 1);
+                }
+                break;
+            }
+
+            /*
+             * Insert relative jump to specified target it SYSCALL isn't used by the host.
+             */
+            case FIX_NO_SYSCALL_JMP:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offTrg < pSwitcher->cbCode);
+                if (!CPUMIsHostUsingSysCall(pVM))
+                {
+                    *uSrc.pu8++ = 0xe9; /* jmp rel32 */
+                    *uSrc.pu32++ = offTrg - (offSrc + 5);
+                }
+                else
+                {
+                    *uSrc.pu8++ = *((uint8_t *)pSwitcher->pvCode + offSrc);
+                    *uSrc.pu32++ = *(uint32_t *)((uint8_t *)pSwitcher->pvCode + offSrc + 1);
+                }
+                break;
+            }
+
+            /*
+             * 32-bit HC pointer fixup to (HC) target within the code (32-bit offset).
+             */
+            case FIX_HC_32BIT:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offSrc < pSwitcher->cbCode);
+                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
+                *uSrc.pu32 = R0PtrCode + offTrg;
+                break;
+            }
+
+#if defined(RT_ARCH_AMD64) || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+            /*
+             * 64-bit HC Code Selector (no argument).
+             */
+            case FIX_HC_64BIT_CS:
+            {
+                Assert(offSrc < pSwitcher->cbCode);
+# if defined(RT_OS_DARWIN) && defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+                *uSrc.pu16 = 0x80; /* KERNEL64_CS from i386/seg.h */
+# else
+                AssertFatalMsgFailed(("FIX_HC_64BIT_CS not implemented for this host\n"));
+# endif
+                break;
+            }
+
+            /*
+             * 64-bit HC pointer to the CPUM instance data (no argument).
+             */
+            case FIX_HC_64BIT_CPUM:
+            {
+                Assert(offSrc < pSwitcher->cbCode);
+                *uSrc.pu64 = pVM->pVMR0 + RT_OFFSETOF(VM, cpum);
+                break;
+            }
+#endif
+            /*
+             * 64-bit HC pointer fixup to (HC) target within the code (32-bit offset).
+             */
+            case FIX_HC_64BIT:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offSrc < pSwitcher->cbCode);
+                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
+                *uSrc.pu64 = R0PtrCode + offTrg;
+                break;
+            }
+
+#ifdef RT_ARCH_X86
+            case FIX_GC_64_BIT_CPUM_OFF:
+            {
+                uint32_t offCPUM = *u.pu32++;
+                Assert(offCPUM < sizeof(pVM->cpum));
+                *uSrc.pu64 = (uint32_t)(VM_RC_ADDR(pVM, &pVM->cpum) + offCPUM);
+                break;
+            }
+#endif
+
+            /*
+             * 32-bit ID pointer to (ID) target within the code (32-bit offset).
+             */
+            case FIX_ID_32BIT:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offSrc < pSwitcher->cbCode);
+                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
+                *uSrc.pu32 = u32IDCode + offTrg;
+                break;
+            }
+
+            /*
+             * 64-bit ID pointer to (ID) target within the code (32-bit offset).
+             */
+            case FIX_ID_64BIT:
+            case FIX_HC_64BIT_NOCHECK:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offSrc < pSwitcher->cbCode);
+                Assert(u8 == FIX_HC_64BIT_NOCHECK || offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
+                *uSrc.pu64 = u32IDCode + offTrg;
+                break;
+            }
+
+            /*
+             * Far 16:32 ID pointer to 64-bit mode (ID) target within the code (32-bit offset).
+             */
+            case FIX_ID_FAR32_TO_64BIT_MODE:
+            {
+                uint32_t offTrg = *u.pu32++;
+                Assert(offSrc < pSwitcher->cbCode);
+                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
+                *uSrc.pu32++ = u32IDCode + offTrg;
+                *uSrc.pu16 = SelCS64;
+                AssertRelease(SelCS64);
+                break;
+            }
+
+#ifdef VBOX_WITH_NMI
+            /*
+             * 32-bit address to the APIC base.
+             */
+            case FIX_GC_APIC_BASE_32BIT:
+            {
+                *uSrc.pu32 = pVM->vmm.s.GCPtrApicBase;
+                break;
+            }
+#endif
+
+            default:
+                AssertReleaseMsgFailed(("Unknown fixup %d in switcher %s\n", u8, pSwitcher->pszDesc));
+                break;
+        }
+    }
+
+#ifdef LOG_ENABLED
+    /*
+     * If Log2 is enabled disassemble the switcher code.
+     *
+     * The switcher code have 1-2 HC parts, 1 GC part and 0-2 ID parts.
+     */
+    if (LogIs2Enabled())
+    {
+        RTLogPrintf("*** Disassembly of switcher %d '%s' %#x bytes ***\n"
+                    "   R0PtrCode   = %p\n"
+                    "   pu8CodeR3   = %p\n"
+                    "   GCPtrCode   = %RGv\n"
+                    "   u32IDCode   = %08x\n"
+                    "   pVMRC       = %RRv\n"
+                    "   pCPUMRC     = %RRv\n"
+                    "   pVMR3       = %p\n"
+                    "   pCPUMR3     = %p\n"
+                    "   GCPtrGDT    = %RGv\n"
+                    "   InterCR3s   = %08RHp, %08RHp, %08RHp (32-Bit, PAE, AMD64)\n"
+                    "   HyperCR3s   = %08RHp (32-Bit, PAE & AMD64)\n"
+                    "   SelCS       = %04x\n"
+                    "   SelDS       = %04x\n"
+                    "   SelCS64     = %04x\n"
+                    "   SelTSS      = %04x\n",
+                    pSwitcher->enmType, pSwitcher->pszDesc, pSwitcher->cbCode,
+                    R0PtrCode,
+                    pu8CodeR3,
+                    GCPtrCode,
+                    u32IDCode,
+                    VM_RC_ADDR(pVM, pVM),
+                    VM_RC_ADDR(pVM, &pVM->cpum),
+                    pVM,
+                    &pVM->cpum,
+                    GCPtrGDT,
+                    PGMGetInter32BitCR3(pVM), PGMGetInterPaeCR3(pVM), PGMGetInterAmd64CR3(pVM),
+                    PGMGetHyperCR3(VMMGetCpu(pVM)),
+                    SelCS, SelDS, SelCS64, SelTSS);
+
+        uint32_t offCode = 0;
+        while (offCode < pSwitcher->cbCode)
+        {
+            /*
+             * Figure out where this is.
+             */
+            const char *pszDesc = NULL;
+            RTUINTPTR   uBase;
+            uint32_t    cbCode;
+            if (offCode - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0)
+            {
+                pszDesc = "HCCode0";
+                uBase   = R0PtrCode;
+                offCode = pSwitcher->offHCCode0;
+                cbCode  = pSwitcher->cbHCCode0;
+            }
+            else if (offCode - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1)
+            {
+                pszDesc = "HCCode1";
+                uBase   = R0PtrCode;
+                offCode = pSwitcher->offHCCode1;
+                cbCode  = pSwitcher->cbHCCode1;
+            }
+            else if (offCode - pSwitcher->offGCCode < pSwitcher->cbGCCode)
+            {
+                pszDesc = "GCCode";
+                uBase   = GCPtrCode;
+                offCode = pSwitcher->offGCCode;
+                cbCode  = pSwitcher->cbGCCode;
+            }
+            else if (offCode - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0)
+            {
+                pszDesc = "IDCode0";
+                uBase   = u32IDCode;
+                offCode = pSwitcher->offIDCode0;
+                cbCode  = pSwitcher->cbIDCode0;
+            }
+            else if (offCode - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1)
+            {
+                pszDesc = "IDCode1";
+                uBase   = u32IDCode;
+                offCode = pSwitcher->offIDCode1;
+                cbCode  = pSwitcher->cbIDCode1;
+            }
+            else
+            {
+                RTLogPrintf("  %04x: %02x '%c' (nowhere)\n",
+                            offCode, pu8CodeR3[offCode], RT_C_IS_PRINT(pu8CodeR3[offCode]) ? pu8CodeR3[offCode] : ' ');
+                offCode++;
+                continue;
+            }
+
+            /*
+             * Disassemble it.
+             */
+            RTLogPrintf("  %s: offCode=%#x cbCode=%#x\n", pszDesc, offCode, cbCode);
+            DISCPUSTATE Cpu;
+
+            memset(&Cpu, 0, sizeof(Cpu));
+            Cpu.mode = CPUMODE_32BIT;
+            while (cbCode > 0)
+            {
+                /* try label it */
+                if (pSwitcher->offR0HostToGuest == offCode)
+                    RTLogPrintf(" *R0HostToGuest:\n");
+                if (pSwitcher->offGCGuestToHost == offCode)
+                    RTLogPrintf(" *GCGuestToHost:\n");
+                if (pSwitcher->offGCCallTrampoline == offCode)
+                    RTLogPrintf(" *GCCallTrampoline:\n");
+                if (pSwitcher->offGCGuestToHostAsm == offCode)
+                    RTLogPrintf(" *GCGuestToHostAsm:\n");
+                if (pSwitcher->offGCGuestToHostAsmHyperCtx == offCode)
+                    RTLogPrintf(" *GCGuestToHostAsmHyperCtx:\n");
+                if (pSwitcher->offGCGuestToHostAsmGuestCtx == offCode)
+                    RTLogPrintf(" *GCGuestToHostAsmGuestCtx:\n");
+
+                /* disas */
+                uint32_t cbInstr = 0;
+                char szDisas[256];
+                if (RT_SUCCESS(DISInstr(&Cpu, (uintptr_t)pu8CodeR3 + offCode, uBase - (uintptr_t)pu8CodeR3, &cbInstr, szDisas)))
+                    RTLogPrintf("  %04x: %s", offCode, szDisas); //for whatever reason szDisas includes '\n'.
+                else
+                {
+                    RTLogPrintf("  %04x: %02x '%c'\n",
+                                offCode, pu8CodeR3[offCode], RT_C_IS_PRINT(pu8CodeR3[offCode]) ? pu8CodeR3[offCode] : ' ');
+                    cbInstr = 1;
+                }
+                offCode += cbInstr;
+                cbCode -= RT_MIN(cbInstr, cbCode);
+            }
+        }
+    }
+#endif
+}
+
+
+/**
+ * Relocator for the 32-Bit to 32-Bit world switcher.
+ */
+DECLCALLBACK(void) vmmR3Switcher32BitTo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
+}
+
+
+/**
+ * Relocator for the 32-Bit to PAE world switcher.
+ */
+DECLCALLBACK(void) vmmR3Switcher32BitToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
+}
+
+
+/**
+ * Relocator for the 32-Bit to AMD64 world switcher.
+ */
+DECLCALLBACK(void) vmmR3Switcher32BitToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
+}
+
+
+/**
+ * Relocator for the PAE to 32-Bit world switcher.
+ */
+DECLCALLBACK(void) vmmR3SwitcherPAETo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
+}
+
+
+/**
+ * Relocator for the PAE to PAE world switcher.
+ */
+DECLCALLBACK(void) vmmR3SwitcherPAEToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
+}
+
+/**
+ * Relocator for the PAE to AMD64 world switcher.
+ */
+DECLCALLBACK(void) vmmR3SwitcherPAEToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
+}
+
+
+/**
+ * Relocator for the AMD64 to 32-bit world switcher.
+ */
+DECLCALLBACK(void) vmmR3SwitcherAMD64To32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
+}
+
+
+/**
+ * Relocator for the AMD64 to PAE world switcher.
+ */
+DECLCALLBACK(void) vmmR3SwitcherAMD64ToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
+{
+    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
+                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
+}
+
+
+/**
+ * Selects the switcher to be used for switching to GC.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ * @param   enmSwitcher     The new switcher.
+ * @remark  This function may be called before the VMM is initialized.
+ */
+VMMR3_INT_DECL(int) VMMR3SelectSwitcher(PVM pVM, VMMSWITCHER enmSwitcher)
+{
+    /*
+     * Validate input.
+     */
+    if (    enmSwitcher < VMMSWITCHER_INVALID
+        ||  enmSwitcher >= VMMSWITCHER_MAX)
+    {
+        AssertMsgFailed(("Invalid input enmSwitcher=%d\n", enmSwitcher));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    /* Do nothing if the switcher is disabled. */
+    if (pVM->vmm.s.fSwitcherDisabled)
+        return VINF_SUCCESS;
+
+    /*
+     * Select the new switcher.
+     */
+    PVMMSWITCHERDEF pSwitcher = s_apSwitchers[enmSwitcher];
+    if (pSwitcher)
+    {
+        Log(("VMMR3SelectSwitcher: enmSwitcher %d -> %d %s\n", pVM->vmm.s.enmSwitcher, enmSwitcher, pSwitcher->pszDesc));
+        pVM->vmm.s.enmSwitcher = enmSwitcher;
+
+        RTR0PTR     pbCodeR0 = (RTR0PTR)pVM->vmm.s.pvCoreCodeR0 + pVM->vmm.s.aoffSwitchers[enmSwitcher]; /** @todo fix the pvCoreCodeR0 type */
+        pVM->vmm.s.pfnHostToGuestR0 = pbCodeR0 + pSwitcher->offR0HostToGuest;
+
+        RTGCPTR     GCPtr = pVM->vmm.s.pvCoreCodeRC + pVM->vmm.s.aoffSwitchers[enmSwitcher];
+        pVM->vmm.s.pfnGuestToHostRC         = GCPtr + pSwitcher->offGCGuestToHost;
+        pVM->vmm.s.pfnCallTrampolineRC      = GCPtr + pSwitcher->offGCCallTrampoline;
+        pVM->pfnVMMGCGuestToHostAsm         = GCPtr + pSwitcher->offGCGuestToHostAsm;
+        pVM->pfnVMMGCGuestToHostAsmHyperCtx = GCPtr + pSwitcher->offGCGuestToHostAsmHyperCtx;
+        pVM->pfnVMMGCGuestToHostAsmGuestCtx = GCPtr + pSwitcher->offGCGuestToHostAsmGuestCtx;
+        return VINF_SUCCESS;
+    }
+
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Disable the switcher logic permanently.
+ *
+ * @returns VBox status code.
+ * @param   pVM             VM handle.
+ */
+VMMR3_INT_DECL(int) VMMR3DisableSwitcher(PVM pVM)
+{
+/** @todo r=bird: I would suggest that we create a dummy switcher which just does something like:
+ * @code
+ *       mov eax, VERR_INTERNAL_ERROR
+ *       ret
+ * @endcode
+ * And then check for fSwitcherDisabled in VMMR3SelectSwitcher() in order to prevent it from being removed.
+ */
+    pVM->vmm.s.fSwitcherDisabled = true;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the switcher to be used for switching to GC.
+ *
+ * @returns host to guest ring 0 switcher entrypoint
+ * @param   pVM             VM handle.
+ * @param   enmSwitcher     The new switcher.
+ */
+VMMR3_INT_DECL(RTR0PTR) VMMR3GetHostToGuestSwitcher(PVM pVM, VMMSWITCHER enmSwitcher)
+{
+    /*
+     * Validate input.
+     */
+    if (    enmSwitcher < VMMSWITCHER_INVALID
+        ||  enmSwitcher >= VMMSWITCHER_MAX)
+    {
+        AssertMsgFailed(("Invalid input enmSwitcher=%d\n", enmSwitcher));
+        return NIL_RTR0PTR;
+    }
+
+    /*
+     * Select the new switcher.
+     */
+    PVMMSWITCHERDEF pSwitcher = s_apSwitchers[enmSwitcher];
+    if (pSwitcher)
+    {
+        RTR0PTR     pbCodeR0 = (RTR0PTR)pVM->vmm.s.pvCoreCodeR0 + pVM->vmm.s.aoffSwitchers[enmSwitcher]; /** @todo fix the pvCoreCodeR0 type */
+        return pbCodeR0 + pSwitcher->offR0HostToGuest;
+    }
+    return NIL_RTR0PTR;
+}
Index: /trunk/src/VBox/VMM/VMMR3/VMMTests.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMMTests.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMMTests.cpp	(revision 35333)
@@ -0,0 +1,592 @@
+/* $Id$ */
+/** @file
+ * VMM - The Virtual Machine Monitor Core, Tests.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+//#define NO_SUPCALLR0VMM
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <iprt/asm-amd64-x86.h> /* for SUPGetCpuHzFromGIP */
+#include <VBox/vmm.h>
+#include <VBox/pdmapi.h>
+#include <VBox/cpum.h>
+#include <VBox/dbg.h>
+#include <VBox/mm.h>
+#include <VBox/trpm.h>
+#include <VBox/selm.h>
+#include "VMMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/x86.h>
+#include <VBox/hwaccm.h>
+
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/time.h>
+#include <iprt/stream.h>
+#include <iprt/string.h>
+
+
+/**
+ * Performs a testcase.
+ *
+ * @returns return value from the test.
+ * @param   pVM         The VM handle.
+ * @param   enmTestcase The testcase operation to perform.
+ * @param   uVariation  The testcase variation id.
+ */
+static int vmmR3DoGCTest(PVM pVM, VMMGCOPERATION enmTestcase, unsigned uVariation)
+{
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    RTRCPTR RCPtrEP;
+    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    CPUMHyperSetCtxCore(pVCpu, NULL);
+    memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE);
+    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
+    CPUMPushHyper(pVCpu, uVariation);
+    CPUMPushHyper(pVCpu, enmTestcase);
+    CPUMPushHyper(pVCpu, pVM->pVMRC);
+    CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
+    CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
+    CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
+    Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
+    rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
+    if (RT_LIKELY(rc == VINF_SUCCESS))
+        rc = pVCpu->vmm.s.iLastGZRc;
+    return rc;
+}
+
+
+/**
+ * Performs a trap test.
+ *
+ * @returns Return value from the trap test.
+ * @param   pVM         The VM handle.
+ * @param   u8Trap      The trap number to test.
+ * @param   uVariation  The testcase variation.
+ * @param   rcExpect    The expected result.
+ * @param   u32Eax      The expected eax value.
+ * @param   pszFaultEIP The fault address. Pass NULL if this isn't available or doesn't apply.
+ * @param   pszDesc     The test description.
+ */
+static int vmmR3DoTrapTest(PVM pVM, uint8_t u8Trap, unsigned uVariation, int rcExpect, uint32_t u32Eax, const char *pszFaultEIP, const char *pszDesc)
+{
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+    RTPrintf("VMM: testing 0%x / %d - %s\n", u8Trap, uVariation, pszDesc);
+
+    RTRCPTR RCPtrEP;
+    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    CPUMHyperSetCtxCore(pVCpu, NULL);
+    memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE);
+    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
+    CPUMPushHyper(pVCpu, uVariation);
+    CPUMPushHyper(pVCpu, u8Trap + VMMGC_DO_TESTCASE_TRAP_FIRST);
+    CPUMPushHyper(pVCpu, pVM->pVMRC);
+    CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
+    CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
+    CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
+    Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
+    rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
+    if (RT_LIKELY(rc == VINF_SUCCESS))
+        rc = pVCpu->vmm.s.iLastGZRc;
+    bool fDump = false;
+    if (rc != rcExpect)
+    {
+        RTPrintf("VMM: FAILURE - rc=%Rrc expected %Rrc\n", rc, rcExpect);
+        if (rc != VERR_NOT_IMPLEMENTED)
+            fDump = true;
+    }
+    else if (    rcExpect != VINF_SUCCESS
+             &&  u8Trap != 8 /* double fault doesn't dare set TrapNo. */
+             &&  u8Trap != 3 /* guest only, we're not in guest. */
+             &&  u8Trap != 1 /* guest only, we're not in guest. */
+             &&  u8Trap != TRPMGetTrapNo(pVCpu))
+    {
+        RTPrintf("VMM: FAILURE - Trap %#x expected %#x\n", TRPMGetTrapNo(pVCpu), u8Trap);
+        fDump = true;
+    }
+    else if (pszFaultEIP)
+    {
+        RTRCPTR RCPtrFault;
+        int rc2 = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, pszFaultEIP, &RCPtrFault);
+        if (RT_FAILURE(rc2))
+            RTPrintf("VMM: FAILURE - Failed to resolve symbol '%s', %Rrc!\n", pszFaultEIP, rc);
+        else if (RCPtrFault != CPUMGetHyperEIP(pVCpu))
+        {
+            RTPrintf("VMM: FAILURE - EIP=%08RX32 expected %RRv (%s)\n", CPUMGetHyperEIP(pVCpu), RCPtrFault, pszFaultEIP);
+            fDump = true;
+        }
+    }
+    else if (rcExpect != VINF_SUCCESS)
+    {
+        if (CPUMGetHyperSS(pVCpu) == SELMGetHyperDS(pVM))
+            RTPrintf("VMM: FAILURE - ss=%x expected %x\n", CPUMGetHyperSS(pVCpu), SELMGetHyperDS(pVM));
+        if (CPUMGetHyperES(pVCpu) == SELMGetHyperDS(pVM))
+            RTPrintf("VMM: FAILURE - es=%x expected %x\n", CPUMGetHyperES(pVCpu), SELMGetHyperDS(pVM));
+        if (CPUMGetHyperDS(pVCpu) == SELMGetHyperDS(pVM))
+            RTPrintf("VMM: FAILURE - ds=%x expected %x\n", CPUMGetHyperDS(pVCpu), SELMGetHyperDS(pVM));
+        if (CPUMGetHyperFS(pVCpu) == SELMGetHyperDS(pVM))
+            RTPrintf("VMM: FAILURE - fs=%x expected %x\n", CPUMGetHyperFS(pVCpu), SELMGetHyperDS(pVM));
+        if (CPUMGetHyperGS(pVCpu) == SELMGetHyperDS(pVM))
+            RTPrintf("VMM: FAILURE - gs=%x expected %x\n", CPUMGetHyperGS(pVCpu), SELMGetHyperDS(pVM));
+        if (CPUMGetHyperEDI(pVCpu) == 0x01234567)
+            RTPrintf("VMM: FAILURE - edi=%x expected %x\n", CPUMGetHyperEDI(pVCpu), 0x01234567);
+        if (CPUMGetHyperESI(pVCpu) == 0x42000042)
+            RTPrintf("VMM: FAILURE - esi=%x expected %x\n", CPUMGetHyperESI(pVCpu), 0x42000042);
+        if (CPUMGetHyperEBP(pVCpu) == 0xffeeddcc)
+            RTPrintf("VMM: FAILURE - ebp=%x expected %x\n", CPUMGetHyperEBP(pVCpu), 0xffeeddcc);
+        if (CPUMGetHyperEBX(pVCpu) == 0x89abcdef)
+            RTPrintf("VMM: FAILURE - ebx=%x expected %x\n", CPUMGetHyperEBX(pVCpu), 0x89abcdef);
+        if (CPUMGetHyperECX(pVCpu) == 0xffffaaaa)
+            RTPrintf("VMM: FAILURE - ecx=%x expected %x\n", CPUMGetHyperECX(pVCpu), 0xffffaaaa);
+        if (CPUMGetHyperEDX(pVCpu) == 0x77778888)
+            RTPrintf("VMM: FAILURE - edx=%x expected %x\n", CPUMGetHyperEDX(pVCpu), 0x77778888);
+        if (CPUMGetHyperEAX(pVCpu) == u32Eax)
+            RTPrintf("VMM: FAILURE - eax=%x expected %x\n", CPUMGetHyperEAX(pVCpu), u32Eax);
+    }
+    if (fDump)
+        VMMR3FatalDump(pVM, pVCpu, rc);
+    return rc;
+}
+
+
+/* execute the switch. */
+VMMR3DECL(int) VMMDoTest(PVM pVM)
+{
+#if 1
+    PVMCPU pVCpu = &pVM->aCpus[0];
+
+#ifdef NO_SUPCALLR0VMM
+    RTPrintf("NO_SUPCALLR0VMM\n");
+    return VINF_SUCCESS;
+#endif
+
+    /*
+     * Setup stack for calling VMMGCEntry().
+     */
+    RTRCPTR RCPtrEP;
+    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
+    if (RT_SUCCESS(rc))
+    {
+        RTPrintf("VMM: VMMGCEntry=%RRv\n", RCPtrEP);
+
+        /*
+         * Test various crashes which we must be able to recover from.
+         */
+        vmmR3DoTrapTest(pVM, 0x3, 0, VINF_EM_DBG_HYPER_ASSERTION,  0xf0f0f0f0, "vmmGCTestTrap3_FaultEIP", "int3");
+        vmmR3DoTrapTest(pVM, 0x3, 1, VINF_EM_DBG_HYPER_ASSERTION,  0xf0f0f0f0, "vmmGCTestTrap3_FaultEIP", "int3 WP");
+
+#if defined(DEBUG_bird) /* guess most people would like to skip these since they write to com1. */
+        vmmR3DoTrapTest(pVM, 0x8, 0, VERR_TRPM_PANIC,       0x00000000, "vmmGCTestTrap8_FaultEIP", "#DF [#PG]");
+        SELMR3Relocate(pVM); /* this resets the busy flag of the Trap 08 TSS */
+        bool f;
+        rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "DoubleFault", &f);
+#if !defined(DEBUG_bird)
+        if (RT_SUCCESS(rc) && f)
+#endif
+        {
+            /* see triple fault warnings in SELM and VMMGC.cpp. */
+            vmmR3DoTrapTest(pVM, 0x8, 1, VERR_TRPM_PANIC,       0x00000000, "vmmGCTestTrap8_FaultEIP", "#DF [#PG] WP");
+            SELMR3Relocate(pVM); /* this resets the busy flag of the Trap 08 TSS */
+        }
+#endif
+
+        vmmR3DoTrapTest(pVM, 0xd, 0, VERR_TRPM_DONT_PANIC,  0xf0f0f0f0, "vmmGCTestTrap0d_FaultEIP", "ltr #GP");
+        ///@todo find a better \#GP case, on intel ltr will \#PF (busy update?) and not \#GP.
+        //vmmR3DoTrapTest(pVM, 0xd, 1, VERR_TRPM_DONT_PANIC,  0xf0f0f0f0, "vmmGCTestTrap0d_FaultEIP", "ltr #GP WP");
+
+        vmmR3DoTrapTest(pVM, 0xe, 0, VERR_TRPM_DONT_PANIC,  0x00000000, "vmmGCTestTrap0e_FaultEIP", "#PF (NULL)");
+        vmmR3DoTrapTest(pVM, 0xe, 1, VERR_TRPM_DONT_PANIC,  0x00000000, "vmmGCTestTrap0e_FaultEIP", "#PF (NULL) WP");
+        vmmR3DoTrapTest(pVM, 0xe, 2, VINF_SUCCESS,          0x00000000, NULL,                       "#PF w/Tmp Handler");
+        vmmR3DoTrapTest(pVM, 0xe, 4, VINF_SUCCESS,          0x00000000, NULL,                       "#PF w/Tmp Handler and bad fs");
+
+        /*
+         * Set a debug register and perform a context switch.
+         */
+        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
+        if (rc != VINF_SUCCESS)
+        {
+            RTPrintf("VMM: Nop test failed, rc=%Rrc not VINF_SUCCESS\n", rc);
+            return rc;
+        }
+
+        /* a harmless breakpoint */
+        RTPrintf("VMM: testing hardware bp at 0x10000 (not hit)\n");
+        DBGFADDRESS Addr;
+        DBGFR3AddrFromFlat(pVM, &Addr, 0x10000);
+        RTUINT iBp0;
+        rc = DBGFR3BpSetReg(pVM, &Addr, 0,  ~(uint64_t)0, X86_DR7_RW_EO, 1, &iBp0);
+        AssertReleaseRC(rc);
+        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
+        if (rc != VINF_SUCCESS)
+        {
+            RTPrintf("VMM: DR0=0x10000 test failed with rc=%Rrc!\n", rc);
+            return rc;
+        }
+
+        /* a bad one at VMMGCEntry */
+        RTPrintf("VMM: testing hardware bp at VMMGCEntry (hit)\n");
+        DBGFR3AddrFromFlat(pVM, &Addr, RCPtrEP);
+        RTUINT iBp1;
+        rc = DBGFR3BpSetReg(pVM, &Addr, 0,  ~(uint64_t)0, X86_DR7_RW_EO, 1, &iBp1);
+        AssertReleaseRC(rc);
+        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
+        if (rc != VINF_EM_DBG_HYPER_BREAKPOINT)
+        {
+            RTPrintf("VMM: DR1=VMMGCEntry test failed with rc=%Rrc! expected VINF_EM_RAW_BREAKPOINT_HYPER\n", rc);
+            return rc;
+        }
+
+        /* resume the breakpoint */
+        RTPrintf("VMM: resuming hyper after breakpoint\n");
+        CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_RF);
+        rc = VMMR3ResumeHyper(pVM, pVCpu);
+        if (rc != VINF_SUCCESS)
+        {
+            RTPrintf("VMM: failed to resume on hyper breakpoint, rc=%Rrc\n", rc);
+            return rc;
+        }
+
+        /* engage the breakpoint again and try single stepping. */
+        RTPrintf("VMM: testing hardware bp at VMMGCEntry + stepping\n");
+        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
+        if (rc != VINF_EM_DBG_HYPER_BREAKPOINT)
+        {
+            RTPrintf("VMM: DR1=VMMGCEntry test failed with rc=%Rrc! expected VINF_EM_RAW_BREAKPOINT_HYPER\n", rc);
+            return rc;
+        }
+
+        RTGCUINTREG OldPc = CPUMGetHyperEIP(pVCpu);
+        RTPrintf("%RGr=>", OldPc);
+        unsigned i;
+        for (i = 0; i < 8; i++)
+        {
+            CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
+            rc = VMMR3ResumeHyper(pVM, pVCpu);
+            if (rc != VINF_EM_DBG_HYPER_STEPPED)
+            {
+                RTPrintf("\nVMM: failed to step on hyper breakpoint, rc=%Rrc\n", rc);
+                return rc;
+            }
+            RTGCUINTREG Pc = CPUMGetHyperEIP(pVCpu);
+            RTPrintf("%RGr=>", Pc);
+            if (Pc == OldPc)
+            {
+                RTPrintf("\nVMM: step failed, PC: %RGr -> %RGr\n", OldPc, Pc);
+                return VERR_GENERAL_FAILURE;
+            }
+            OldPc = Pc;
+        }
+        RTPrintf("ok\n");
+
+        /* done, clear it */
+        if (    RT_FAILURE(DBGFR3BpClear(pVM, iBp0))
+            ||  RT_FAILURE(DBGFR3BpClear(pVM, iBp1)))
+        {
+            RTPrintf("VMM: Failed to clear breakpoints!\n");
+            return VERR_GENERAL_FAILURE;
+        }
+        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
+        if (rc != VINF_SUCCESS)
+        {
+            RTPrintf("VMM: NOP failed, rc=%Rrc\n", rc);
+            return rc;
+        }
+
+        /*
+         * Interrupt masking.
+         */
+        RTPrintf("VMM: interrupt masking...\n"); RTStrmFlush(g_pStdOut); RTThreadSleep(250);
+        for (i = 0; i < 10000; i++)
+        {
+            uint64_t StartTick = ASMReadTSC();
+            rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_INTERRUPT_MASKING, 0);
+            if (rc != VINF_SUCCESS)
+            {
+                RTPrintf("VMM: Interrupt masking failed: rc=%Rrc\n", rc);
+                return rc;
+            }
+            uint64_t Ticks = ASMReadTSC() - StartTick;
+            if (Ticks < (SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage) / 10000))
+                RTPrintf("Warning: Ticks=%RU64 (< %RU64)\n", Ticks, SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage) / 10000);
+        }
+
+        /*
+         * Interrupt forwarding.
+         */
+        CPUMHyperSetCtxCore(pVCpu, NULL);
+        CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
+        CPUMPushHyper(pVCpu, 0);
+        CPUMPushHyper(pVCpu, VMMGC_DO_TESTCASE_HYPER_INTERRUPT);
+        CPUMPushHyper(pVCpu, pVM->pVMRC);
+        CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
+        CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
+        CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
+        Log(("trampoline=%x\n", pVM->vmm.s.pfnCallTrampolineRC));
+
+        /*
+         * Switch and do da thing.
+         */
+        RTPrintf("VMM: interrupt forwarding...\n"); RTStrmFlush(g_pStdOut); RTThreadSleep(250);
+        i = 0;
+        uint64_t    tsBegin = RTTimeNanoTS();
+        uint64_t    TickStart = ASMReadTSC();
+        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
+        do
+        {
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                rc = pVCpu->vmm.s.iLastGZRc;
+            if (RT_FAILURE(rc))
+            {
+                Log(("VMM: GC returned fatal %Rra in iteration %d\n", rc, i));
+                VMMR3FatalDump(pVM, pVCpu, rc);
+                return rc;
+            }
+            i++;
+            if (!(i % 32))
+                Log(("VMM: iteration %d, esi=%08x edi=%08x ebx=%08x\n",
+                       i, CPUMGetHyperESI(pVCpu), CPUMGetHyperEDI(pVCpu), CPUMGetHyperEBX(pVCpu)));
+        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
+        uint64_t    TickEnd = ASMReadTSC();
+        uint64_t    tsEnd = RTTimeNanoTS();
+
+        uint64_t    Elapsed = tsEnd - tsBegin;
+        uint64_t    PerIteration = Elapsed / (uint64_t)i;
+        uint64_t    cTicksElapsed = TickEnd - TickStart;
+        uint64_t    cTicksPerIteration = cTicksElapsed / (uint64_t)i;
+
+        RTPrintf("VMM: %8d interrupts in %11llu ns (%11llu ticks),  %10llu ns/iteration (%11llu ticks)\n",
+                 i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration);
+        Log(("VMM: %8d interrupts in %11llu ns (%11llu ticks),  %10llu ns/iteration (%11llu ticks)\n",
+             i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration));
+
+        /*
+         * These forced actions are not necessary for the test and trigger breakpoints too.
+         */
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+
+        /*
+         * Profile switching.
+         */
+        RTPrintf("VMM: profiling switcher...\n");
+        Log(("VMM: profiling switcher...\n"));
+        uint64_t TickMin = ~0;
+        tsBegin = RTTimeNanoTS();
+        TickStart = ASMReadTSC();
+        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
+        for (i = 0; i < 1000000; i++)
+        {
+            CPUMHyperSetCtxCore(pVCpu, NULL);
+            CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
+            CPUMPushHyper(pVCpu, 0);
+            CPUMPushHyper(pVCpu, VMMGC_DO_TESTCASE_NOP);
+            CPUMPushHyper(pVCpu, pVM->pVMRC);
+            CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
+            CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
+            CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
+
+            uint64_t TickThisStart = ASMReadTSC();
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                rc = pVCpu->vmm.s.iLastGZRc;
+            uint64_t TickThisElapsed = ASMReadTSC() - TickThisStart;
+            if (RT_FAILURE(rc))
+            {
+                Log(("VMM: GC returned fatal %Rra in iteration %d\n", rc, i));
+                VMMR3FatalDump(pVM, pVCpu, rc);
+                return rc;
+            }
+            if (TickThisElapsed < TickMin)
+                TickMin = TickThisElapsed;
+        }
+        TickEnd = ASMReadTSC();
+        tsEnd = RTTimeNanoTS();
+
+        Elapsed = tsEnd - tsBegin;
+        PerIteration = Elapsed / (uint64_t)i;
+        cTicksElapsed = TickEnd - TickStart;
+        cTicksPerIteration = cTicksElapsed / (uint64_t)i;
+
+        RTPrintf("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
+                 i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin);
+        Log(("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
+             i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin));
+
+        rc = VINF_SUCCESS;
+    }
+    else
+        AssertMsgFailed(("Failed to resolved VMMGC.gc::VMMGCEntry(), rc=%Rrc\n", rc));
+#endif
+    return rc;
+}
+
+#define SYNC_SEL(pHyperCtx, reg)                                                        \
+        if (pHyperCtx->reg)                                                             \
+        {                                                                               \
+            DBGFSELINFO selInfo;                                                        \
+            int rc2 = SELMR3GetShadowSelectorInfo(pVM, pHyperCtx->reg, &selInfo);       \
+            AssertRC(rc2);                                                              \
+                                                                                        \
+            pHyperCtx->reg##Hid.u64Base              = selInfo.GCPtrBase;               \
+            pHyperCtx->reg##Hid.u32Limit             = selInfo.cbLimit;                 \
+            pHyperCtx->reg##Hid.Attr.n.u1Present     = selInfo.u.Raw.Gen.u1Present;     \
+            pHyperCtx->reg##Hid.Attr.n.u1DefBig      = selInfo.u.Raw.Gen.u1DefBig;      \
+            pHyperCtx->reg##Hid.Attr.n.u1Granularity = selInfo.u.Raw.Gen.u1Granularity; \
+            pHyperCtx->reg##Hid.Attr.n.u4Type        = selInfo.u.Raw.Gen.u4Type;        \
+            pHyperCtx->reg##Hid.Attr.n.u2Dpl         = selInfo.u.Raw.Gen.u2Dpl;         \
+            pHyperCtx->reg##Hid.Attr.n.u1DescType    = selInfo.u.Raw.Gen.u1DescType;    \
+            pHyperCtx->reg##Hid.Attr.n.u1Long        = selInfo.u.Raw.Gen.u1Long;        \
+        }
+
+/* execute the switch. */
+VMMR3DECL(int) VMMDoHwAccmTest(PVM pVM)
+{
+    uint32_t i;
+    int      rc;
+    PCPUMCTX pHyperCtx, pGuestCtx;
+    RTGCPHYS CR3Phys = 0x0; /* fake address */
+    PVMCPU   pVCpu = &pVM->aCpus[0];
+
+    if (!HWACCMR3IsAllowed(pVM))
+    {
+        RTPrintf("VMM: Hardware accelerated test not available!\n");
+        return VERR_ACCESS_DENIED;
+    }
+
+    /*
+     * These forced actions are not necessary for the test and trigger breakpoints too.
+     */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
+
+    /* Enable mapping of the hypervisor into the shadow page table. */
+    uint32_t cb;
+    rc = PGMR3MappingsSize(pVM, &cb);
+    AssertRCReturn(rc, rc);
+
+    /* Pretend the mappings are now fixed; to force a refresh of the reserved PDEs. */
+    rc = PGMR3MappingsFix(pVM, MM_HYPER_AREA_ADDRESS, cb);
+    AssertRCReturn(rc, rc);
+
+    CPUMQueryHyperCtxPtr(pVCpu, &pHyperCtx);
+
+    pHyperCtx->cr0 = X86_CR0_PE | X86_CR0_WP | X86_CR0_PG | X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
+    pHyperCtx->cr4 = X86_CR4_PGE | X86_CR4_OSFSXR | X86_CR4_OSXMMEEXCPT;
+    PGMChangeMode(pVCpu, pHyperCtx->cr0, pHyperCtx->cr4, pHyperCtx->msrEFER);
+    PGMSyncCR3(pVCpu, pHyperCtx->cr0, CR3Phys, pHyperCtx->cr4, true);
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TIMER);
+    VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+    VM_FF_CLEAR(pVM, VM_FF_REQUEST);
+
+    /*
+     * Setup stack for calling VMMGCEntry().
+     */
+    RTRCPTR RCPtrEP;
+    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
+    if (RT_SUCCESS(rc))
+    {
+        RTPrintf("VMM: VMMGCEntry=%RRv\n", RCPtrEP);
+
+        CPUMQueryHyperCtxPtr(pVCpu, &pHyperCtx);
+
+        /* Fill in hidden selector registers for the hypervisor state. */
+        SYNC_SEL(pHyperCtx, cs);
+        SYNC_SEL(pHyperCtx, ds);
+        SYNC_SEL(pHyperCtx, es);
+        SYNC_SEL(pHyperCtx, fs);
+        SYNC_SEL(pHyperCtx, gs);
+        SYNC_SEL(pHyperCtx, ss);
+        SYNC_SEL(pHyperCtx, tr);
+
+        /*
+         * Profile switching.
+         */
+        RTPrintf("VMM: profiling switcher...\n");
+        Log(("VMM: profiling switcher...\n"));
+        uint64_t TickMin = ~0;
+        uint64_t tsBegin = RTTimeNanoTS();
+        uint64_t TickStart = ASMReadTSC();
+        for (i = 0; i < 1000000; i++)
+        {
+            CPUMHyperSetCtxCore(pVCpu, NULL);
+
+            CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
+            CPUMPushHyper(pVCpu, 0);
+            CPUMPushHyper(pVCpu, VMMGC_DO_TESTCASE_HWACCM_NOP);
+            CPUMPushHyper(pVCpu, pVM->pVMRC);
+            CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
+            CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
+            CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
+
+            CPUMQueryHyperCtxPtr(pVCpu, &pHyperCtx);
+            pGuestCtx = CPUMQueryGuestCtxPtr(pVCpu);
+
+            /* Copy the hypervisor context to make sure we have a valid guest context. */
+            *pGuestCtx = *pHyperCtx;
+            pGuestCtx->cr3 = CR3Phys;
+
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TIMER);
+            VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
+
+            uint64_t TickThisStart = ASMReadTSC();
+            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HWACC_RUN, 0);
+            uint64_t TickThisElapsed = ASMReadTSC() - TickThisStart;
+            if (RT_FAILURE(rc))
+            {
+                Log(("VMM: R0 returned fatal %Rrc in iteration %d\n", rc, i));
+                VMMR3FatalDump(pVM, pVCpu, rc);
+                return rc;
+            }
+            if (TickThisElapsed < TickMin)
+                TickMin = TickThisElapsed;
+        }
+        uint64_t TickEnd = ASMReadTSC();
+        uint64_t tsEnd = RTTimeNanoTS();
+
+        uint64_t Elapsed = tsEnd - tsBegin;
+        uint64_t PerIteration = Elapsed / (uint64_t)i;
+        uint64_t cTicksElapsed = TickEnd - TickStart;
+        uint64_t cTicksPerIteration = cTicksElapsed / (uint64_t)i;
+
+        RTPrintf("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
+                 i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin);
+        Log(("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
+             i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin));
+
+        rc = VINF_SUCCESS;
+    }
+    else
+        AssertMsgFailed(("Failed to resolved VMMGC.gc::VMMGCEntry(), rc=%Rrc\n", rc));
+
+    return rc;
+}
+
Index: /trunk/src/VBox/VMM/VMMR3/VMReq.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMR3/VMReq.cpp	(revision 35333)
+++ /trunk/src/VBox/VMM/VMMR3/VMReq.cpp	(revision 35333)
@@ -0,0 +1,1273 @@
+/* $Id$ */
+/** @file
+ * VM - Virtual Machine
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include <VBox/mm.h>
+#include <VBox/vmm.h>
+#include "VMInternal.h"
+#include <VBox/vm.h>
+#include <VBox/uvm.h>
+
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/time.h>
+#include <iprt/semaphore.h>
+#include <iprt/thread.h>
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static int  vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq);
+
+
+/**
+ * Allocate and queue a call request.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completion. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pVM             The VM handle.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happens.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   fFlags          A combination of the VMREQFLAGS values.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCall(PVM pVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
+                           PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code,
+ * (2) that you want it's return code on success, and (3) that you wish to wait
+ * for ever for it to return.
+ *
+ * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
+ *          its status code is return.  Otherwise, the status of pfnFunction is
+ *          returned.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    if (RT_SUCCESS(rc))
+        rc = pReq->iStatus;
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code,
+ * (2) that you want it's return code on success, and (3) that you wish to wait
+ * for ever for it to return.
+ *
+ * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
+ *          its status code is return.  Otherwise, the status of pfnFunction is
+ *          returned.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    if (RT_SUCCESS(rc))
+        rc = pReq->iStatus;
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code
+ * and that you do not wish to wait for it to complete.
+ *
+ * @returns VBox status code returned by VMR3ReqCallVU.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallNoWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, NULL, 0, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_NO_WAIT,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns an VBox status code
+ * and that you do not wish to wait for it to complete.
+ *
+ * @returns VBox status code returned by VMR3ReqCallVU.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallNoWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, NULL, 0, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_NO_WAIT,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, and (2) that
+ * you wish to wait for ever for it to return.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallVoidWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, and (2) that
+ * you wish to wait for ever for it to return.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallVoidWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, and (2) that
+ * you do not wish to wait for it to complete.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pVM             Pointer to the shared VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallVoidNoWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID | VMREQFLAGS_NO_WAIT,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Convenience wrapper for VMR3ReqCallU.
+ *
+ * This assumes (1) you're calling a function that returns void, and (2) that
+ * you do not wish to wait for it to complete.
+ *
+ * @returns VBox status code of VMR3ReqCallVU.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallVoidNoWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    PVMREQ pReq;
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID | VMREQFLAGS_NO_WAIT,
+                           pfnFunction, cArgs, va);
+    va_end(va);
+    VMR3ReqFree(pReq);
+    return rc;
+}
+
+
+/**
+ * Allocate and queue a call request to a void function.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completion. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happens, unless fFlags
+ *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   fFlags          A combination of the VMREQFLAGS values.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ * @param   ...             Function arguments.
+ *
+ * @remarks See remarks on VMR3ReqCallVU.
+ */
+VMMR3DECL(int) VMR3ReqCallU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
+                            PFNRT pfnFunction, unsigned cArgs, ...)
+{
+    va_list va;
+    va_start(va, cArgs);
+    int rc = VMR3ReqCallVU(pUVM, idDstCpu, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * Allocate and queue a call request.
+ *
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completion. In the other case
+ * use RT_INDEFINITE_WAIT.
+ * The returned request packet must be freed using VMR3ReqFree().
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ * @param   ppReq           Where to store the pointer to the request.
+ *                          This will be NULL or a valid request pointer not matter what happens, unless fFlags
+ *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ * @param   pfnFunction     Pointer to the function to call.
+ * @param   fFlags          A combination of the VMREQFLAGS values.
+ * @param   cArgs           Number of arguments following in the ellipsis.
+ *                          Stuff which differs in size from uintptr_t is gonna make trouble, so don't try!
+ * @param   Args            Argument vector.
+ *
+ * @remarks Caveats:
+ *              - Do not pass anything which is larger than an uintptr_t.
+ *              - 64-bit integers are larger than uintptr_t on 32-bit hosts.
+ *                Pass integers > 32-bit by reference (pointers).
+ *              - Don't use NULL since it should be the integer 0 in C++ and may
+ *                therefore end up with garbage in the bits 63:32 on 64-bit
+ *                hosts because 'int' is 32-bit.
+ *                Use (void *)NULL or (uintptr_t)0 instead of NULL.
+ */
+VMMR3DECL(int) VMR3ReqCallVU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
+                             PFNRT pfnFunction, unsigned cArgs, va_list Args)
+{
+    LogFlow(("VMR3ReqCallV: idDstCpu=%u cMillies=%d fFlags=%#x pfnFunction=%p cArgs=%d\n", idDstCpu, cMillies, fFlags, pfnFunction, cArgs));
+
+    /*
+     * Validate input.
+     */
+    AssertPtrReturn(pfnFunction, VERR_INVALID_POINTER);
+    AssertPtrReturn(pUVM, VERR_INVALID_POINTER);
+    AssertReturn(!(fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE)), VERR_INVALID_PARAMETER);
+    if (!(fFlags & VMREQFLAGS_NO_WAIT) || ppReq)
+    {
+        AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
+        *ppReq = NULL;
+    }
+    PVMREQ pReq = NULL;
+    AssertMsgReturn(cArgs * sizeof(uintptr_t) <= sizeof(pReq->u.Internal.aArgs),
+                    ("cArg=%d\n", cArgs),
+                    VERR_TOO_MUCH_DATA);
+
+    /*
+     * Allocate request
+     */
+    int rc = VMR3ReqAllocU(pUVM, &pReq, VMREQTYPE_INTERNAL, idDstCpu);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Initialize the request data.
+     */
+    pReq->fFlags         = fFlags;
+    pReq->u.Internal.pfn = pfnFunction;
+    pReq->u.Internal.cArgs = cArgs;
+    for (unsigned iArg = 0; iArg < cArgs; iArg++)
+        pReq->u.Internal.aArgs[iArg] = va_arg(Args, uintptr_t);
+
+    /*
+     * Queue the request and return.
+     */
+    rc = VMR3ReqQueue(pReq, cMillies);
+    if (    RT_FAILURE(rc)
+        && rc != VERR_TIMEOUT)
+    {
+        VMR3ReqFree(pReq);
+        pReq = NULL;
+    }
+    if (!(fFlags & VMREQFLAGS_NO_WAIT))
+    {
+        *ppReq = pReq;
+        LogFlow(("VMR3ReqCallV: returns %Rrc *ppReq=%p\n", rc, pReq));
+    }
+    else
+        LogFlow(("VMR3ReqCallV: returns %Rrc\n", rc));
+    Assert(rc != VERR_INTERRUPTED);
+    return rc;
+}
+
+
+/**
+ * Joins the list pList with whatever is linked up at *pHead.
+ */
+static void vmr3ReqJoinFreeSub(volatile PVMREQ *ppHead, PVMREQ pList)
+{
+    for (unsigned cIterations = 0;; cIterations++)
+    {
+        PVMREQ pHead = ASMAtomicXchgPtrT(ppHead, pList, PVMREQ);
+        if (!pHead)
+            return;
+        PVMREQ pTail = pHead;
+        while (pTail->pNext)
+            pTail = pTail->pNext;
+        ASMAtomicWritePtr(&pTail->pNext, pList);
+        ASMCompilerBarrier();
+        if (ASMAtomicCmpXchgPtr(ppHead, pHead, pList))
+            return;
+        ASMAtomicWriteNullPtr(&pTail->pNext);
+        ASMCompilerBarrier();
+        if (ASMAtomicCmpXchgPtr(ppHead, pHead, NULL))
+            return;
+        pList = pHead;
+        Assert(cIterations != 32);
+        Assert(cIterations != 64);
+    }
+}
+
+
+/**
+ * Joins the list pList with whatever is linked up at *pHead.
+ */
+static void vmr3ReqJoinFree(PVMINTUSERPERVM pVMInt, PVMREQ pList)
+{
+    /*
+     * Split the list if it's too long.
+     */
+    unsigned cReqs = 1;
+    PVMREQ pTail = pList;
+    while (pTail->pNext)
+    {
+        if (cReqs++ > 25)
+        {
+            const uint32_t i = pVMInt->iReqFree;
+            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
+
+            pTail->pNext = NULL;
+            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2 + (i == pVMInt->iReqFree)) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
+            return;
+        }
+        pTail = pTail->pNext;
+    }
+    vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(pVMInt->iReqFree + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pList);
+}
+
+
+/**
+ * Allocates a request packet.
+ *
+ * The caller allocates a request packet, fills in the request data
+ * union and queues the request.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             VM handle.
+ * @param   ppReq           Where to store the pointer to the allocated packet.
+ * @param   enmType         Package type.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ */
+VMMR3DECL(int) VMR3ReqAlloc(PVM pVM, PVMREQ *ppReq, VMREQTYPE enmType, VMCPUID idDstCpu)
+{
+    return VMR3ReqAllocU(pVM->pUVM, ppReq, enmType, idDstCpu);
+}
+
+
+/**
+ * Allocates a request packet.
+ *
+ * The caller allocates a request packet, fills in the request data
+ * union and queues the request.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   ppReq           Where to store the pointer to the allocated packet.
+ * @param   enmType         Package type.
+ * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
+ *                          one of the following special values:
+ *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
+ */
+VMMR3DECL(int) VMR3ReqAllocU(PUVM pUVM, PVMREQ *ppReq, VMREQTYPE enmType, VMCPUID idDstCpu)
+{
+    /*
+     * Validate input.
+     */
+    AssertMsgReturn(enmType > VMREQTYPE_INVALID && enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusively.\n",
+                     enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
+    AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
+    AssertMsgReturn(    idDstCpu == VMCPUID_ANY
+                    ||  idDstCpu == VMCPUID_ANY_QUEUE
+                    ||  idDstCpu < pUVM->cCpus
+                    ||  idDstCpu == VMCPUID_ALL
+                    ||  idDstCpu == VMCPUID_ALL_REVERSE,
+                    ("Invalid destination %u (max=%u)\n", idDstCpu, pUVM->cCpus), VERR_INVALID_PARAMETER);
+
+    /*
+     * Try get a recycled packet.
+     * While this could all be solved with a single list with a lock, it's a sport
+     * of mine to avoid locks.
+     */
+    int cTries = RT_ELEMENTS(pUVM->vm.s.apReqFree) * 2;
+    while (--cTries >= 0)
+    {
+        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
+#if 0 /* sad, but this won't work safely because the reading of pReq->pNext. */
+        PVMREQ pNext = NULL;
+        PVMREQ pReq = *ppHead;
+        if (    pReq
+            &&  !ASMAtomicCmpXchgPtr(ppHead, (pNext = pReq->pNext), pReq)
+            &&  (pReq = *ppHead)
+            &&  !ASMAtomicCmpXchgPtr(ppHead, (pNext = pReq->pNext), pReq))
+            pReq = NULL;
+        if (pReq)
+        {
+            Assert(pReq->pNext == pNext); NOREF(pReq);
+#else
+        PVMREQ pReq = ASMAtomicXchgPtrT(ppHead, NULL, PVMREQ);
+        if (pReq)
+        {
+            PVMREQ pNext = pReq->pNext;
+            if (    pNext
+                &&  !ASMAtomicCmpXchgPtr(ppHead, pNext, NULL))
+            {
+                STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRaces);
+                vmr3ReqJoinFree(&pUVM->vm.s, pReq->pNext);
+            }
+#endif
+            ASMAtomicDecU32(&pUVM->vm.s.cReqFree);
+
+            /*
+             * Make sure the event sem is not signaled.
+             */
+            if (!pReq->fEventSemClear)
+            {
+                int rc = RTSemEventWait(pReq->EventSem, 0);
+                if (rc != VINF_SUCCESS && rc != VERR_TIMEOUT)
+                {
+                    /*
+                     * This shall not happen, but if it does we'll just destroy
+                     * the semaphore and create a new one.
+                     */
+                    AssertMsgFailed(("rc=%Rrc from RTSemEventWait(%#x).\n", rc, pReq->EventSem));
+                    RTSemEventDestroy(pReq->EventSem);
+                    rc = RTSemEventCreate(&pReq->EventSem);
+                    AssertRC(rc);
+                    if (RT_FAILURE(rc))
+                        return rc;
+#if 0 ///@todo @bugref{4725} - def RT_LOCK_STRICT
+                    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
+                        RTSemEventAddSignaller(pReq->EventSem, pUVM->aCpus[idCpu].vm.s.ThreadEMT);
+#endif
+                }
+                pReq->fEventSemClear = true;
+            }
+            else
+                Assert(RTSemEventWait(pReq->EventSem, 0) == VERR_TIMEOUT);
+
+            /*
+             * Initialize the packet and return it.
+             */
+            Assert(pReq->enmType == VMREQTYPE_INVALID);
+            Assert(pReq->enmState == VMREQSTATE_FREE);
+            Assert(pReq->pUVM == pUVM);
+            ASMAtomicXchgSize(&pReq->pNext, NULL);
+            pReq->enmState = VMREQSTATE_ALLOCATED;
+            pReq->iStatus  = VERR_VM_REQUEST_STATUS_STILL_PENDING;
+            pReq->fFlags   = VMREQFLAGS_VBOX_STATUS;
+            pReq->enmType  = enmType;
+            pReq->idDstCpu = idDstCpu;
+
+            *ppReq = pReq;
+            STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRecycled);
+            LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p recycled\n", pReq));
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * Ok allocate one.
+     */
+    PVMREQ pReq = (PVMREQ)MMR3HeapAllocU(pUVM, MM_TAG_VM_REQ, sizeof(*pReq));
+    if (!pReq)
+        return VERR_NO_MEMORY;
+
+    /*
+     * Create the semaphore.
+     */
+    int rc = RTSemEventCreate(&pReq->EventSem);
+    AssertRC(rc);
+    if (RT_FAILURE(rc))
+    {
+        MMR3HeapFree(pReq);
+        return rc;
+    }
+#if 0 ///@todo @bugref{4725} - def RT_LOCK_STRICT
+    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
+        RTSemEventAddSignaller(pReq->EventSem, pUVM->aCpus[idCpu].vm.s.ThreadEMT);
+#endif
+
+    /*
+     * Initialize the packet and return it.
+     */
+    pReq->pNext    = NULL;
+    pReq->pUVM     = pUVM;
+    pReq->enmState = VMREQSTATE_ALLOCATED;
+    pReq->iStatus  = VERR_VM_REQUEST_STATUS_STILL_PENDING;
+    pReq->fEventSemClear = true;
+    pReq->fFlags   = VMREQFLAGS_VBOX_STATUS;
+    pReq->enmType  = enmType;
+    pReq->idDstCpu = idDstCpu;
+
+    *ppReq = pReq;
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocNew);
+    LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p new\n", pReq));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Free a request packet.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pReq            Package to free.
+ * @remark  The request packet must be in allocated or completed state!
+ */
+VMMR3DECL(int) VMR3ReqFree(PVMREQ pReq)
+{
+    /*
+     * Ignore NULL (all free functions should do this imho).
+     */
+    if (!pReq)
+        return VINF_SUCCESS;
+
+    /*
+     * Check packet state.
+     */
+    switch (pReq->enmState)
+    {
+        case VMREQSTATE_ALLOCATED:
+        case VMREQSTATE_COMPLETED:
+            break;
+        default:
+            AssertMsgFailed(("Invalid state %d!\n", pReq->enmState));
+            return VERR_VM_REQUEST_STATE;
+    }
+
+    /*
+     * Make it a free packet and put it into one of the free packet lists.
+     */
+    pReq->enmState = VMREQSTATE_FREE;
+    pReq->iStatus  = VERR_VM_REQUEST_STATUS_FREED;
+    pReq->enmType  = VMREQTYPE_INVALID;
+
+    PUVM pUVM = pReq->pUVM;
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqFree);
+
+    if (pUVM->vm.s.cReqFree < 128)
+    {
+        ASMAtomicIncU32(&pUVM->vm.s.cReqFree);
+        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
+        PVMREQ pNext;
+        do
+        {
+            pNext = ASMAtomicUoReadPtrT(ppHead, PVMREQ);
+            ASMAtomicWritePtr(&pReq->pNext, pNext);
+            ASMCompilerBarrier();
+        } while (!ASMAtomicCmpXchgPtr(ppHead, pReq, pNext));
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pReq->pUVM->vm.s.StatReqFreeOverflow);
+        RTSemEventDestroy(pReq->EventSem);
+        MMR3HeapFree(pReq);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Queue a request.
+ *
+ * The quest must be allocated using VMR3ReqAlloc() and contain
+ * all the required data.
+ * If it's desired to poll on the completion of the request set cMillies
+ * to 0 and use VMR3ReqWait() to check for completion. In the other case
+ * use RT_INDEFINITE_WAIT.
+ *
+ * @returns VBox status code.
+ *          Will not return VERR_INTERRUPTED.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pReq            The request to queue.
+ * @param   cMillies        Number of milliseconds to wait for the request to
+ *                          be completed. Use RT_INDEFINITE_WAIT to only
+ *                          wait till it's completed.
+ */
+VMMR3DECL(int) VMR3ReqQueue(PVMREQ pReq, RTMSINTERVAL cMillies)
+{
+    LogFlow(("VMR3ReqQueue: pReq=%p cMillies=%d\n", pReq, cMillies));
+    /*
+     * Verify the supplied package.
+     */
+    AssertMsgReturn(pReq->enmState == VMREQSTATE_ALLOCATED, ("%d\n", pReq->enmState), VERR_VM_REQUEST_STATE);
+    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
+                    &&  !pReq->pNext
+                    &&  pReq->EventSem != NIL_RTSEMEVENT,
+                    ("Invalid request package! Anyone cooking their own packages???\n"),
+                    VERR_VM_REQUEST_INVALID_PACKAGE);
+    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
+                    &&  pReq->enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusively. This was verified on alloc too...\n",
+                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
+    Assert(!(pReq->fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE)));
+
+    /*
+     * Are we the EMT or not?
+     * Also, store pVM (and fFlags) locally since pReq may be invalid after queuing it.
+     */
+    int     rc      = VINF_SUCCESS;
+    PUVM    pUVM    = ((VMREQ volatile *)pReq)->pUVM;                 /* volatile paranoia */
+    PUVMCPU pUVCpu  = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
+
+    if (pReq->idDstCpu == VMCPUID_ALL)
+    {
+        /* One-by-one. */
+        Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
+        for (unsigned i = 0; i < pUVM->cCpus; i++)
+        {
+            /* Reinit some members. */
+            pReq->enmState = VMREQSTATE_ALLOCATED;
+            pReq->idDstCpu = i;
+            rc = VMR3ReqQueue(pReq, cMillies);
+            if (RT_FAILURE(rc))
+                break;
+        }
+    }
+    else if (pReq->idDstCpu == VMCPUID_ALL_REVERSE)
+    {
+        /* One-by-one. */
+        Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
+        for (int i = pUVM->cCpus-1; i >= 0; i--)
+        {
+            /* Reinit some members. */
+            pReq->enmState = VMREQSTATE_ALLOCATED;
+            pReq->idDstCpu = i;
+            rc = VMR3ReqQueue(pReq, cMillies);
+            if (RT_FAILURE(rc))
+                break;
+        }
+    }
+    else if (   pReq->idDstCpu != VMCPUID_ANY   /* for a specific VMCPU? */
+             && pReq->idDstCpu != VMCPUID_ANY_QUEUE
+             && (   !pUVCpu                     /* and it's not the current thread. */
+                 || pUVCpu->idCpu != pReq->idDstCpu))
+    {
+        VMCPUID  idTarget = pReq->idDstCpu;     Assert(idTarget < pUVM->cCpus);
+        PVMCPU   pVCpu = &pUVM->pVM->aCpus[idTarget];
+        unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags;     /* volatile paranoia */
+
+        /* Fetch the right UVMCPU */
+        pUVCpu = &pUVM->aCpus[idTarget];
+
+        /*
+         * Insert it.
+         */
+        pReq->enmState = VMREQSTATE_QUEUED;
+        PVMREQ pNext;
+        do
+        {
+            pNext = ASMAtomicUoReadPtrT(&pUVCpu->vm.s.pReqs, PVMREQ);
+            ASMAtomicWritePtr(&pReq->pNext, pNext);
+            ASMCompilerBarrier();
+        } while (!ASMAtomicCmpXchgPtr(&pUVCpu->vm.s.pReqs, pReq, pNext));
+
+        /*
+         * Notify EMT.
+         */
+        if (pUVM->pVM)
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_REQUEST);
+        VMR3NotifyCpuFFU(pUVCpu, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
+
+        /*
+         * Wait and return.
+         */
+        if (!(fFlags & VMREQFLAGS_NO_WAIT))
+            rc = VMR3ReqWait(pReq, cMillies);
+        LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
+    }
+    else if (   (    pReq->idDstCpu == VMCPUID_ANY
+                 && !pUVCpu /* only EMT threads have a valid pointer stored in the TLS slot. */)
+             || pReq->idDstCpu == VMCPUID_ANY_QUEUE)
+    {
+        unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags;     /* volatile paranoia */
+
+        /* Note: pUVCpu may or may not be NULL in the VMCPUID_ANY_QUEUE case; we don't care. */
+
+        /*
+         * Insert it.
+         */
+        pReq->enmState = VMREQSTATE_QUEUED;
+        PVMREQ pNext;
+        do
+        {
+            pNext = ASMAtomicUoReadPtrT(&pUVM->vm.s.pReqs, PVMREQ);
+            ASMAtomicWritePtr(&pReq->pNext, pNext);
+            ASMCompilerBarrier();
+        } while (!ASMAtomicCmpXchgPtr(&pUVM->vm.s.pReqs, pReq, pNext));
+
+        /*
+         * Notify EMT.
+         */
+        if (pUVM->pVM)
+            VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
+        VMR3NotifyGlobalFFU(pUVM, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
+
+        /*
+         * Wait and return.
+         */
+        if (!(fFlags & VMREQFLAGS_NO_WAIT))
+            rc = VMR3ReqWait(pReq, cMillies);
+        LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
+    }
+    else
+    {
+        Assert(pUVCpu);
+
+        /*
+         * The requester was an EMT, just execute it.
+         */
+        pReq->enmState = VMREQSTATE_QUEUED;
+        rc = vmR3ReqProcessOneU(pUVM, pReq);
+        LogFlow(("VMR3ReqQueue: returns %Rrc (processed)\n", rc));
+    }
+    return rc;
+}
+
+
+/**
+ * Wait for a request to be completed.
+ *
+ * @returns VBox status code.
+ * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
+ *
+ * @param   pReq            The request to wait for.
+ * @param   cMillies        Number of milliseconds to wait.
+ *                          Use RT_INDEFINITE_WAIT to only wait till it's completed.
+ */
+VMMR3DECL(int) VMR3ReqWait(PVMREQ pReq, RTMSINTERVAL cMillies)
+{
+    LogFlow(("VMR3ReqWait: pReq=%p cMillies=%d\n", pReq, cMillies));
+
+    /*
+     * Verify the supplied package.
+     */
+    AssertMsgReturn(    pReq->enmState == VMREQSTATE_QUEUED
+                    ||  pReq->enmState == VMREQSTATE_PROCESSING
+                    ||  pReq->enmState == VMREQSTATE_COMPLETED,
+                    ("Invalid state %d\n", pReq->enmState),
+                    VERR_VM_REQUEST_STATE);
+    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
+                    &&  pReq->EventSem != NIL_RTSEMEVENT,
+                    ("Invalid request package! Anyone cooking their own packages???\n"),
+                    VERR_VM_REQUEST_INVALID_PACKAGE);
+    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
+                    &&  pReq->enmType < VMREQTYPE_MAX,
+                    ("Invalid package type %d valid range %d-%d inclusively. This was verified on alloc too...\n",
+                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
+                    VERR_VM_REQUEST_INVALID_TYPE);
+
+    /*
+     * Check for deadlock condition
+     */
+    PUVM pUVM = pReq->pUVM;
+    NOREF(pUVM);
+
+    /*
+     * Wait on the package.
+     */
+    int rc;
+    if (cMillies != RT_INDEFINITE_WAIT)
+        rc = RTSemEventWait(pReq->EventSem, cMillies);
+    else
+    {
+        do
+        {
+            rc = RTSemEventWait(pReq->EventSem, RT_INDEFINITE_WAIT);
+            Assert(rc != VERR_TIMEOUT);
+        } while (   pReq->enmState != VMREQSTATE_COMPLETED
+                 && pReq->enmState != VMREQSTATE_INVALID);
+    }
+    if (RT_SUCCESS(rc))
+        ASMAtomicXchgSize(&pReq->fEventSemClear, true);
+    if (pReq->enmState == VMREQSTATE_COMPLETED)
+        rc = VINF_SUCCESS;
+    LogFlow(("VMR3ReqWait: returns %Rrc\n", rc));
+    Assert(rc != VERR_INTERRUPTED);
+    return rc;
+}
+
+
+/**
+ * VMR3ReqProcessU helper that handles cases where there are more than one
+ * pending request.
+ *
+ * @returns The oldest request.
+ * @param   pUVM                Pointer to the user mode VM structure
+ * @param   idDstCpu            VMCPUID_ANY or virtual CPU ID.
+ * @param   pReqList            The list of requests.
+ * @param   ppReqs              Pointer to the list head.
+ */
+static PVMREQ vmR3ReqProcessUTooManyHelper(PUVM pUVM, VMCPUID idDstCpu, PVMREQ pReqList, PVMREQ volatile *ppReqs)
+{
+    STAM_COUNTER_INC(&pUVM->vm.s.StatReqMoreThan1);
+    /* Chop off the last one (pReq). */
+    PVMREQ pPrev;
+    PVMREQ pReqRet = pReqList;
+    do
+    {
+        pPrev = pReqRet;
+        pReqRet = pReqRet->pNext;
+    } while (pReqRet->pNext);
+    ASMAtomicWriteNullPtr(&pPrev->pNext);
+
+    /* Push the others back onto the list (end of it). */
+    Log2(("VMR3ReqProcess: Pushing back %p %p...\n", pReqList, pReqList->pNext));
+    if (RT_UNLIKELY(!ASMAtomicCmpXchgPtr(ppReqs, pReqList, NULL)))
+    {
+        STAM_COUNTER_INC(&pUVM->vm.s.StatReqPushBackRaces);
+        do
+        {
+            ASMNopPause();
+            PVMREQ pReqList2 = ASMAtomicXchgPtrT(ppReqs, NULL, PVMREQ);
+            if (pReqList2)
+            {
+                PVMREQ pLast = pReqList2;
+                while (pLast->pNext)
+                    pLast = pLast->pNext;
+                ASMAtomicWritePtr(&pLast->pNext, pReqList);
+                pReqList = pReqList2;
+            }
+        } while (!ASMAtomicCmpXchgPtr(ppReqs, pReqList, NULL));
+    }
+
+    if (RT_LIKELY(pUVM->pVM))
+    {
+        if (idDstCpu == VMCPUID_ANY)
+            VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
+        else
+            VMCPU_FF_SET(&pUVM->pVM->aCpus[idDstCpu], VMCPU_FF_REQUEST);
+    }
+
+    return pReqRet;
+}
+
+
+/**
+ * Process pending request(s).
+ *
+ * This function is called from a forced action handler in the EMT
+ * or from one of the EMT loops.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pUVM            Pointer to the user mode VM structure.
+ * @param   idDstCpu        Pass VMCPUID_ANY to process the common request queue
+ *                          and the CPU ID for a CPU specific one. In the latter
+ *                          case the calling thread must be the EMT of that CPU.
+ *
+ * @note    SMP safe (multiple EMTs trying to satisfy VM_FF_REQUESTs).
+ *
+ * @remarks This was made reentrant for
+ */
+VMMR3DECL(int) VMR3ReqProcessU(PUVM pUVM, VMCPUID idDstCpu)
+{
+    LogFlow(("VMR3ReqProcessU: (enmVMState=%d) idDstCpu=%d\n", pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING, idDstCpu));
+
+    /*
+     * Process loop.
+     *
+     * We do not repeat the outer loop if we've got an informational status code
+     * since that code needs processing by our caller.
+     */
+    int rc = VINF_SUCCESS;
+    while (rc <= VINF_SUCCESS)
+    {
+        /*
+         * Get the pending requests.
+         * If there are more than one request, unlink the oldest and put the
+         * rest back so that we're reentrant.
+         */
+        PVMREQ volatile *ppReqs;
+        if (idDstCpu == VMCPUID_ANY)
+        {
+            ppReqs = &pUVM->vm.s.pReqs;
+            if (RT_LIKELY(pUVM->pVM))
+                VM_FF_CLEAR(pUVM->pVM, VM_FF_REQUEST);
+        }
+        else
+        {
+            Assert(idDstCpu < pUVM->cCpus);
+            Assert(pUVM->aCpus[idDstCpu].vm.s.NativeThreadEMT == RTThreadNativeSelf());
+            ppReqs = &pUVM->aCpus[idDstCpu].vm.s.pReqs;
+            if (RT_LIKELY(pUVM->pVM))
+                VMCPU_FF_CLEAR(&pUVM->pVM->aCpus[idDstCpu], VMCPU_FF_REQUEST);
+        }
+
+        PVMREQ pReq = ASMAtomicXchgPtrT(ppReqs, NULL, PVMREQ);
+        if (!pReq)
+            break;
+        if (RT_UNLIKELY(pReq->pNext))
+            pReq = vmR3ReqProcessUTooManyHelper(pUVM, idDstCpu, pReq, ppReqs);
+
+        /*
+         * Process the request.
+         * Note! The status code handling here extremely important and yet very
+         *       fragile.
+         */
+        STAM_COUNTER_INC(&pUVM->vm.s.StatReqProcessed);
+        int rc2 = vmR3ReqProcessOneU(pUVM, pReq);
+        if (    rc2 >= VINF_EM_FIRST
+            &&  rc2 <= VINF_EM_LAST
+            &&  (   rc == VINF_SUCCESS
+                 || rc2 < rc) )
+            rc = rc2;
+        /** @todo may have to abort processing to propagate EM scheduling status codes
+         *        up to the caller... See the ugly hacks after VMMR3EmtRendezvousFF
+         *        and VMR3ReqProcessU in EM.cpp. */
+    }
+
+    LogFlow(("VMR3ReqProcess: returns %Rrc (enmVMState=%d)\n", rc, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING));
+    return rc;
+}
+
+
+/**
+ * Process one request.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         VM handle.
+ * @param   pReq        Request packet to process.
+ */
+static int  vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq)
+{
+    LogFlow(("vmR3ReqProcessOneU: pReq=%p type=%d fFlags=%#x\n", pReq, pReq->enmType, pReq->fFlags));
+
+    /*
+     * Process the request.
+     */
+    Assert(pReq->enmState == VMREQSTATE_QUEUED);
+    pReq->enmState = VMREQSTATE_PROCESSING;
+    int     rcRet = VINF_SUCCESS;           /* the return code of this function. */
+    int     rcReq = VERR_NOT_IMPLEMENTED;   /* the request status. */
+    switch (pReq->enmType)
+    {
+        /*
+         * A packed down call frame.
+         */
+        case VMREQTYPE_INTERNAL:
+        {
+            uintptr_t *pauArgs = &pReq->u.Internal.aArgs[0];
+            union
+            {
+                PFNRT pfn;
+                DECLCALLBACKMEMBER(int, pfn00)(void);
+                DECLCALLBACKMEMBER(int, pfn01)(uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn02)(uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn03)(uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn04)(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn05)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn06)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn07)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn08)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn09)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn10)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn11)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn12)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn13)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn14)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+                DECLCALLBACKMEMBER(int, pfn15)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+            } u;
+            u.pfn = pReq->u.Internal.pfn;
+#ifdef RT_ARCH_AMD64
+            switch (pReq->u.Internal.cArgs)
+            {
+                case 0:  rcRet = u.pfn00(); break;
+                case 1:  rcRet = u.pfn01(pauArgs[0]); break;
+                case 2:  rcRet = u.pfn02(pauArgs[0], pauArgs[1]); break;
+                case 3:  rcRet = u.pfn03(pauArgs[0], pauArgs[1], pauArgs[2]); break;
+                case 4:  rcRet = u.pfn04(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3]); break;
+                case 5:  rcRet = u.pfn05(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4]); break;
+                case 6:  rcRet = u.pfn06(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5]); break;
+                case 7:  rcRet = u.pfn07(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6]); break;
+                case 8:  rcRet = u.pfn08(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7]); break;
+                case 9:  rcRet = u.pfn09(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8]); break;
+                case 10: rcRet = u.pfn10(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9]); break;
+                case 11: rcRet = u.pfn11(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10]); break;
+                case 12: rcRet = u.pfn12(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11]); break;
+                case 13: rcRet = u.pfn13(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12]); break;
+                case 14: rcRet = u.pfn14(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12], pauArgs[13]); break;
+                case 15: rcRet = u.pfn15(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12], pauArgs[13], pauArgs[14]); break;
+                default:
+                    AssertReleaseMsgFailed(("cArgs=%d\n", pReq->u.Internal.cArgs));
+                    rcRet = rcReq = VERR_INTERNAL_ERROR;
+                    break;
+            }
+#else /* x86: */
+            size_t cbArgs = pReq->u.Internal.cArgs * sizeof(uintptr_t);
+# ifdef __GNUC__
+            __asm__ __volatile__("movl  %%esp, %%edx\n\t"
+                                 "subl  %2, %%esp\n\t"
+                                 "andl  $0xfffffff0, %%esp\n\t"
+                                 "shrl  $2, %2\n\t"
+                                 "movl  %%esp, %%edi\n\t"
+                                 "rep movsl\n\t"
+                                 "movl  %%edx, %%edi\n\t"
+                                 "call  *%%eax\n\t"
+                                 "mov   %%edi, %%esp\n\t"
+                                 : "=a" (rcRet),
+                                   "=S" (pauArgs),
+                                   "=c" (cbArgs)
+                                 : "0" (u.pfn),
+                                   "1" (pauArgs),
+                                   "2" (cbArgs)
+                                 : "edi", "edx");
+# else
+            __asm
+            {
+                xor     edx, edx        /* just mess it up. */
+                mov     eax, u.pfn
+                mov     ecx, cbArgs
+                shr     ecx, 2
+                mov     esi, pauArgs
+                mov     ebx, esp
+                sub     esp, cbArgs
+                and     esp, 0xfffffff0
+                mov     edi, esp
+                rep movsd
+                call    eax
+                mov     esp, ebx
+                mov     rcRet, eax
+            }
+# endif
+#endif /* x86 */
+            if ((pReq->fFlags & (VMREQFLAGS_RETURN_MASK)) == VMREQFLAGS_VOID)
+                rcRet = VINF_SUCCESS;
+            rcReq = rcRet;
+            break;
+        }
+
+        default:
+            AssertMsgFailed(("pReq->enmType=%d\n", pReq->enmType));
+            rcReq = VERR_NOT_IMPLEMENTED;
+            break;
+    }
+
+    /*
+     * Complete the request.
+     */
+    pReq->iStatus  = rcReq;
+    pReq->enmState = VMREQSTATE_COMPLETED;
+    if (pReq->fFlags & VMREQFLAGS_NO_WAIT)
+    {
+        /* Free the packet, nobody is waiting. */
+        LogFlow(("vmR3ReqProcessOneU: Completed request %p: rcReq=%Rrc rcRet=%Rrc - freeing it\n",
+                 pReq, rcReq, rcRet));
+        VMR3ReqFree(pReq);
+    }
+    else
+    {
+        /* Notify the waiter and him free up the packet. */
+        LogFlow(("vmR3ReqProcessOneU: Completed request %p: rcReq=%Rrc rcRet=%Rrc - notifying waiting thread\n",
+                 pReq, rcReq, rcRet));
+        ASMAtomicXchgSize(&pReq->fEventSemClear, false);
+        int rc2 = RTSemEventSignal(pReq->EventSem);
+        if (RT_FAILURE(rc2))
+        {
+            AssertRC(rc2);
+            rcRet = rc2;
+        }
+    }
+    return rcRet;
+}
+
Index: /trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMRZ/PGMRZDynMap.cpp	(revision 35333)
@@ -22,7 +22,7 @@
 #define LOG_GROUP LOG_GROUP_PGM_DYNMAP
 #include <VBox/pgm.h>
-#include "../PGMInternal.h"
+#include "PGMInternal.h"
 #include <VBox/vm.h>
-#include "../PGMInline.h"
+#include "PGMInline.h"
 #include <VBox/err.h>
 #include <VBox/param.h>
Index: unk/src/VBox/VMM/VMMSwitcher.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMSwitcher.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1030 +1,0 @@
-/* $Id$ */
-/** @file
- * VMM - The Virtual Machine Monitor, World Switcher(s).
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_VMM
-#include <VBox/vmm.h>
-#include <VBox/pgm.h>
-#include <VBox/selm.h>
-#include <VBox/mm.h>
-#include <VBox/sup.h>
-#include "VMMInternal.h"
-#include "VMMSwitcher/VMMSwitcher.h"
-#include <VBox/vm.h>
-#include <VBox/dis.h>
-
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <iprt/assert.h>
-#include <iprt/alloc.h>
-#include <iprt/asm.h>
-#include <iprt/asm-amd64-x86.h>
-#include <iprt/string.h>
-#include <iprt/ctype.h>
-
-
-/*******************************************************************************
-*   Global Variables                                                           *
-*******************************************************************************/
-/** Array of switcher definitions.
- * The type and index shall match!
- */
-static PVMMSWITCHERDEF s_apSwitchers[VMMSWITCHER_MAX] =
-{
-    NULL, /* invalid entry */
-#ifdef VBOX_WITH_RAW_MODE
-# ifndef RT_ARCH_AMD64
-    &vmmR3Switcher32BitTo32Bit_Def,
-    &vmmR3Switcher32BitToPAE_Def,
-    &vmmR3Switcher32BitToAMD64_Def,
-    &vmmR3SwitcherPAETo32Bit_Def,
-    &vmmR3SwitcherPAEToPAE_Def,
-    &vmmR3SwitcherPAEToAMD64_Def,
-    NULL,   //&vmmR3SwitcherPAETo32Bit_Def,
-#  ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
-    &vmmR3SwitcherAMD64ToPAE_Def,
-#  else
-    NULL,   //&vmmR3SwitcherAMD64ToPAE_Def,
-#  endif
-    NULL    //&vmmR3SwitcherAMD64ToAMD64_Def,
-# else  /* RT_ARCH_AMD64 */
-    NULL,   //&vmmR3Switcher32BitTo32Bit_Def,
-    NULL,   //&vmmR3Switcher32BitToPAE_Def,
-    NULL,   //&vmmR3Switcher32BitToAMD64_Def,
-    NULL,   //&vmmR3SwitcherPAETo32Bit_Def,
-    NULL,   //&vmmR3SwitcherPAEToPAE_Def,
-    NULL,   //&vmmR3SwitcherPAEToAMD64_Def,
-    &vmmR3SwitcherAMD64To32Bit_Def,
-    &vmmR3SwitcherAMD64ToPAE_Def,
-    NULL    //&vmmR3SwitcherAMD64ToAMD64_Def,
-# endif /* RT_ARCH_AMD64 */
-#else  /* !VBOX_WITH_RAW_MODE */
-    NULL,
-    NULL,
-    NULL,
-    NULL,
-    NULL,
-    NULL,
-    NULL,
-    NULL,
-    NULL
-#endif /* !VBOX_WITH_RAW_MODE */
-};
-
-
-/**
- * VMMR3Init worker that initiates the switcher code (aka core code).
- *
- * This is core per VM code which might need fixups and/or for ease of use are
- * put on linear contiguous backing.
- *
- * @returns VBox status code.
- * @param   pVM     Pointer to the shared VM structure.
- */
-int vmmR3SwitcherInit(PVM pVM)
-{
-#ifndef VBOX_WITH_RAW_MODE
-    return VINF_SUCCESS;
-#else
-    /*
-     * Calc the size.
-     */
-    unsigned cbCoreCode = 0;
-    for (unsigned iSwitcher = 0; iSwitcher < RT_ELEMENTS(s_apSwitchers); iSwitcher++)
-    {
-        pVM->vmm.s.aoffSwitchers[iSwitcher] = cbCoreCode;
-        PVMMSWITCHERDEF pSwitcher = s_apSwitchers[iSwitcher];
-        if (pSwitcher)
-        {
-            AssertRelease((unsigned)pSwitcher->enmType == iSwitcher);
-            cbCoreCode += RT_ALIGN_32(pSwitcher->cbCode + 1, 32);
-        }
-    }
-
-    /*
-     * Allocate contiguous pages for switchers and deal with
-     * conflicts in the intermediate mapping of the code.
-     */
-    pVM->vmm.s.cbCoreCode = RT_ALIGN_32(cbCoreCode, PAGE_SIZE);
-    pVM->vmm.s.pvCoreCodeR3 = SUPR3ContAlloc(pVM->vmm.s.cbCoreCode >> PAGE_SHIFT, &pVM->vmm.s.pvCoreCodeR0, &pVM->vmm.s.HCPhysCoreCode);
-    int rc = VERR_NO_MEMORY;
-    if (pVM->vmm.s.pvCoreCodeR3)
-    {
-        rc = PGMR3MapIntermediate(pVM, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.HCPhysCoreCode, cbCoreCode);
-        if (rc == VERR_PGM_INTERMEDIATE_PAGING_CONFLICT)
-        {
-            /* try more allocations - Solaris, Linux.  */
-            const unsigned cTries = 8234;
-            struct VMMInitBadTry
-            {
-                RTR0PTR  pvR0;
-                void    *pvR3;
-                RTHCPHYS HCPhys;
-                RTUINT   cb;
-            } *paBadTries = (struct VMMInitBadTry *)RTMemTmpAlloc(sizeof(*paBadTries) * cTries);
-            AssertReturn(paBadTries, VERR_NO_TMP_MEMORY);
-            unsigned i = 0;
-            do
-            {
-                paBadTries[i].pvR3 = pVM->vmm.s.pvCoreCodeR3;
-                paBadTries[i].pvR0 = pVM->vmm.s.pvCoreCodeR0;
-                paBadTries[i].HCPhys = pVM->vmm.s.HCPhysCoreCode;
-                i++;
-                pVM->vmm.s.pvCoreCodeR0 = NIL_RTR0PTR;
-                pVM->vmm.s.HCPhysCoreCode = NIL_RTHCPHYS;
-                pVM->vmm.s.pvCoreCodeR3 = SUPR3ContAlloc(pVM->vmm.s.cbCoreCode >> PAGE_SHIFT, &pVM->vmm.s.pvCoreCodeR0, &pVM->vmm.s.HCPhysCoreCode);
-                if (!pVM->vmm.s.pvCoreCodeR3)
-                    break;
-                rc = PGMR3MapIntermediate(pVM, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.HCPhysCoreCode, cbCoreCode);
-            } while (   rc == VERR_PGM_INTERMEDIATE_PAGING_CONFLICT
-                     && i < cTries - 1);
-
-            /* cleanup */
-            if (RT_FAILURE(rc))
-            {
-                paBadTries[i].pvR3   = pVM->vmm.s.pvCoreCodeR3;
-                paBadTries[i].pvR0   = pVM->vmm.s.pvCoreCodeR0;
-                paBadTries[i].HCPhys = pVM->vmm.s.HCPhysCoreCode;
-                paBadTries[i].cb     = pVM->vmm.s.cbCoreCode;
-                i++;
-                LogRel(("Failed to allocated and map core code: rc=%Rrc\n", rc));
-            }
-            while (i-- > 0)
-            {
-                LogRel(("Core code alloc attempt #%d: pvR3=%p pvR0=%p HCPhys=%RHp\n",
-                        i, paBadTries[i].pvR3, paBadTries[i].pvR0, paBadTries[i].HCPhys));
-                SUPR3ContFree(paBadTries[i].pvR3, paBadTries[i].cb >> PAGE_SHIFT);
-            }
-            RTMemTmpFree(paBadTries);
-        }
-    }
-    if (RT_SUCCESS(rc))
-    {
-        /*
-         * copy the code.
-         */
-        for (unsigned iSwitcher = 0; iSwitcher < RT_ELEMENTS(s_apSwitchers); iSwitcher++)
-        {
-            PVMMSWITCHERDEF pSwitcher = s_apSwitchers[iSwitcher];
-            if (pSwitcher)
-                memcpy((uint8_t *)pVM->vmm.s.pvCoreCodeR3 + pVM->vmm.s.aoffSwitchers[iSwitcher],
-                       pSwitcher->pvCode, pSwitcher->cbCode);
-        }
-
-        /*
-         * Map the code into the GC address space.
-         */
-        RTGCPTR GCPtr;
-        rc = MMR3HyperMapHCPhys(pVM, pVM->vmm.s.pvCoreCodeR3, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.HCPhysCoreCode,
-                                cbCoreCode, "Core Code", &GCPtr);
-        if (RT_SUCCESS(rc))
-        {
-            pVM->vmm.s.pvCoreCodeRC = GCPtr;
-            MMR3HyperReserve(pVM, PAGE_SIZE, "fence", NULL);
-            LogRel(("CoreCode: R3=%RHv R0=%RHv RC=%RRv Phys=%RHp cb=%#x\n",
-                    pVM->vmm.s.pvCoreCodeR3, pVM->vmm.s.pvCoreCodeR0, pVM->vmm.s.pvCoreCodeRC, pVM->vmm.s.HCPhysCoreCode, pVM->vmm.s.cbCoreCode));
-
-            /*
-             * Finally, PGM probably has selected a switcher already but we need
-             * to get the routine addresses, so we'll reselect it.
-             * This may legally fail so, we're ignoring the rc.
-             */
-            VMMR3SelectSwitcher(pVM, pVM->vmm.s.enmSwitcher);
-            return rc;
-        }
-
-        /* shit */
-        AssertMsgFailed(("PGMR3Map(,%RRv, %RHp, %#x, 0) failed with rc=%Rrc\n", pVM->vmm.s.pvCoreCodeRC, pVM->vmm.s.HCPhysCoreCode, cbCoreCode, rc));
-        SUPR3ContFree(pVM->vmm.s.pvCoreCodeR3, pVM->vmm.s.cbCoreCode >> PAGE_SHIFT);
-    }
-    else
-        VMSetError(pVM, rc, RT_SRC_POS,
-                   N_("Failed to allocate %d bytes of contiguous memory for the world switcher code"),
-                   cbCoreCode);
-
-    pVM->vmm.s.pvCoreCodeR3 = NULL;
-    pVM->vmm.s.pvCoreCodeR0 = NIL_RTR0PTR;
-    pVM->vmm.s.pvCoreCodeRC = 0;
-    return rc;
-#endif
-}
-
-/**
- * Relocate the switchers, called by VMMR#Relocate.
- *
- * @param   pVM         Pointer to the shared VM structure.
- * @param   offDelta    The relocation delta.
- */
-void vmmR3SwitcherRelocate(PVM pVM, RTGCINTPTR offDelta)
-{
-#ifdef VBOX_WITH_RAW_MODE
-    /*
-     * Relocate all the switchers.
-     */
-    for (unsigned iSwitcher = 0; iSwitcher < RT_ELEMENTS(s_apSwitchers); iSwitcher++)
-    {
-        PVMMSWITCHERDEF pSwitcher = s_apSwitchers[iSwitcher];
-        if (pSwitcher && pSwitcher->pfnRelocate)
-        {
-            unsigned off = pVM->vmm.s.aoffSwitchers[iSwitcher];
-            pSwitcher->pfnRelocate(pVM,
-                                   pSwitcher,
-                                   pVM->vmm.s.pvCoreCodeR0 + off,
-                                   (uint8_t *)pVM->vmm.s.pvCoreCodeR3 + off,
-                                   pVM->vmm.s.pvCoreCodeRC + off,
-                                   pVM->vmm.s.HCPhysCoreCode + off);
-        }
-    }
-
-    /*
-     * Recalc the RC address for the current switcher.
-     */
-    PVMMSWITCHERDEF pSwitcher   = s_apSwitchers[pVM->vmm.s.enmSwitcher];
-    RTRCPTR         RCPtr       = pVM->vmm.s.pvCoreCodeRC + pVM->vmm.s.aoffSwitchers[pVM->vmm.s.enmSwitcher];
-    pVM->vmm.s.pfnGuestToHostRC         = RCPtr + pSwitcher->offGCGuestToHost;
-    pVM->vmm.s.pfnCallTrampolineRC      = RCPtr + pSwitcher->offGCCallTrampoline;
-    pVM->pfnVMMGCGuestToHostAsm         = RCPtr + pSwitcher->offGCGuestToHostAsm;
-    pVM->pfnVMMGCGuestToHostAsmHyperCtx = RCPtr + pSwitcher->offGCGuestToHostAsmHyperCtx;
-    pVM->pfnVMMGCGuestToHostAsmGuestCtx = RCPtr + pSwitcher->offGCGuestToHostAsmGuestCtx;
-
-//    AssertFailed();
-#endif
-}
-
-
-/**
- * Generic switcher code relocator.
- *
- * @param   pVM         The VM handle.
- * @param   pSwitcher   The switcher definition.
- * @param   pu8CodeR3   Pointer to the core code block for the switcher, ring-3 mapping.
- * @param   R0PtrCode   Pointer to the core code block for the switcher, ring-0 mapping.
- * @param   GCPtrCode   The guest context address corresponding to pu8Code.
- * @param   u32IDCode   The identity mapped (ID) address corresponding to pu8Code.
- * @param   SelCS       The hypervisor CS selector.
- * @param   SelDS       The hypervisor DS selector.
- * @param   SelTSS      The hypervisor TSS selector.
- * @param   GCPtrGDT    The GC address of the hypervisor GDT.
- * @param   SelCS64     The 64-bit mode hypervisor CS selector.
- */
-static void vmmR3SwitcherGenericRelocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode,
-                                         RTSEL SelCS, RTSEL SelDS, RTSEL SelTSS, RTGCPTR GCPtrGDT, RTSEL SelCS64)
-{
-    union
-    {
-        const uint8_t *pu8;
-        const uint16_t *pu16;
-        const uint32_t *pu32;
-        const uint64_t *pu64;
-        const void     *pv;
-        uintptr_t       u;
-    } u;
-    u.pv = pSwitcher->pvFixups;
-
-    /*
-     * Process fixups.
-     */
-    uint8_t u8;
-    while ((u8 = *u.pu8++) != FIX_THE_END)
-    {
-        /*
-         * Get the source (where to write the fixup).
-         */
-        uint32_t offSrc = *u.pu32++;
-        Assert(offSrc < pSwitcher->cbCode);
-        union
-        {
-            uint8_t    *pu8;
-            uint16_t   *pu16;
-            uint32_t   *pu32;
-            uint64_t   *pu64;
-            uintptr_t   u;
-        } uSrc;
-        uSrc.pu8 = pu8CodeR3 + offSrc;
-
-        /* The fixup target and method depends on the type. */
-        switch (u8)
-        {
-            /*
-             * 32-bit relative, source in HC and target in GC.
-             */
-            case FIX_HC_2_GC_NEAR_REL:
-            {
-                Assert(offSrc - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offSrc - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg - pSwitcher->offGCCode < pSwitcher->cbGCCode);
-                *uSrc.pu32 = (uint32_t)((GCPtrCode + offTrg) - (uSrc.u + 4));
-                break;
-            }
-
-            /*
-             * 32-bit relative, source in HC and target in ID.
-             */
-            case FIX_HC_2_ID_NEAR_REL:
-            {
-                Assert(offSrc - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offSrc - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
-                *uSrc.pu32 = (uint32_t)((u32IDCode + offTrg) - (R0PtrCode + offSrc + 4));
-                break;
-            }
-
-            /*
-             * 32-bit relative, source in GC and target in HC.
-             */
-            case FIX_GC_2_HC_NEAR_REL:
-            {
-                Assert(offSrc - pSwitcher->offGCCode < pSwitcher->cbGCCode);
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
-                *uSrc.pu32 = (uint32_t)((R0PtrCode + offTrg) - (GCPtrCode + offSrc + 4));
-                break;
-            }
-
-            /*
-             * 32-bit relative, source in GC and target in ID.
-             */
-            case FIX_GC_2_ID_NEAR_REL:
-            {
-                AssertMsg(offSrc - pSwitcher->offGCCode < pSwitcher->cbGCCode, ("%x - %x < %x\n", offSrc, pSwitcher->offGCCode, pSwitcher->cbGCCode));
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
-                *uSrc.pu32 = (uint32_t)((u32IDCode + offTrg) - (GCPtrCode + offSrc + 4));
-                break;
-            }
-
-            /*
-             * 32-bit relative, source in ID and target in HC.
-             */
-            case FIX_ID_2_HC_NEAR_REL:
-            {
-                Assert(offSrc - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offSrc - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
-                *uSrc.pu32 = (uint32_t)((R0PtrCode + offTrg) - (u32IDCode + offSrc + 4));
-                break;
-            }
-
-            /*
-             * 32-bit relative, source in ID and target in HC.
-             */
-            case FIX_ID_2_GC_NEAR_REL:
-            {
-                Assert(offSrc - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offSrc - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg - pSwitcher->offGCCode < pSwitcher->cbGCCode);
-                *uSrc.pu32 = (uint32_t)((GCPtrCode + offTrg) - (u32IDCode + offSrc + 4));
-                break;
-            }
-
-            /*
-             * 16:32 far jump, target in GC.
-             */
-            case FIX_GC_FAR32:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg - pSwitcher->offGCCode < pSwitcher->cbGCCode);
-                *uSrc.pu32++ = (uint32_t)(GCPtrCode + offTrg);
-                *uSrc.pu16++ = SelCS;
-                break;
-            }
-
-            /*
-             * Make 32-bit GC pointer given CPUM offset.
-             */
-            case FIX_GC_CPUM_OFF:
-            {
-                uint32_t offCPUM = *u.pu32++;
-                Assert(offCPUM < sizeof(pVM->cpum));
-                *uSrc.pu32 = (uint32_t)(VM_RC_ADDR(pVM, &pVM->cpum) + offCPUM);
-                break;
-            }
-
-            /*
-             * Make 32-bit GC pointer given CPUMCPU offset.
-             */
-            case FIX_GC_CPUMCPU_OFF:
-            {
-                uint32_t offCPUM = *u.pu32++;
-                Assert(offCPUM < sizeof(pVM->aCpus[0].cpum));
-                *uSrc.pu32 = (uint32_t)(VM_RC_ADDR(pVM, &pVM->aCpus[0].cpum) + offCPUM);
-                break;
-            }
-
-            /*
-             * Make 32-bit GC pointer given VM offset.
-             */
-            case FIX_GC_VM_OFF:
-            {
-                uint32_t offVM = *u.pu32++;
-                Assert(offVM < sizeof(VM));
-                *uSrc.pu32 = (uint32_t)(VM_RC_ADDR(pVM, pVM) + offVM);
-                break;
-            }
-
-            /*
-             * Make 32-bit HC pointer given CPUM offset.
-             */
-            case FIX_HC_CPUM_OFF:
-            {
-                uint32_t offCPUM = *u.pu32++;
-                Assert(offCPUM < sizeof(pVM->cpum));
-                *uSrc.pu32 = (uint32_t)pVM->pVMR0 + RT_OFFSETOF(VM, cpum) + offCPUM;
-                break;
-            }
-
-            /*
-             * Make 32-bit R0 pointer given VM offset.
-             */
-            case FIX_HC_VM_OFF:
-            {
-                uint32_t offVM = *u.pu32++;
-                Assert(offVM < sizeof(VM));
-                *uSrc.pu32 = (uint32_t)pVM->pVMR0 + offVM;
-                break;
-            }
-
-            /*
-             * Store the 32-Bit CR3 (32-bit) for the intermediate memory context.
-             */
-            case FIX_INTER_32BIT_CR3:
-            {
-
-                *uSrc.pu32 = PGMGetInter32BitCR3(pVM);
-                break;
-            }
-
-            /*
-             * Store the PAE CR3 (32-bit) for the intermediate memory context.
-             */
-            case FIX_INTER_PAE_CR3:
-            {
-
-                *uSrc.pu32 = PGMGetInterPaeCR3(pVM);
-                break;
-            }
-
-            /*
-             * Store the AMD64 CR3 (32-bit) for the intermediate memory context.
-             */
-            case FIX_INTER_AMD64_CR3:
-            {
-
-                *uSrc.pu32 = PGMGetInterAmd64CR3(pVM);
-                break;
-            }
-
-            /*
-             * Store Hypervisor CS (16-bit).
-             */
-            case FIX_HYPER_CS:
-            {
-                *uSrc.pu16 = SelCS;
-                break;
-            }
-
-            /*
-             * Store Hypervisor DS (16-bit).
-             */
-            case FIX_HYPER_DS:
-            {
-                *uSrc.pu16 = SelDS;
-                break;
-            }
-
-            /*
-             * Store Hypervisor TSS (16-bit).
-             */
-            case FIX_HYPER_TSS:
-            {
-                *uSrc.pu16 = SelTSS;
-                break;
-            }
-
-            /*
-             * Store the 32-bit GC address of the 2nd dword of the TSS descriptor (in the GDT).
-             */
-            case FIX_GC_TSS_GDTE_DW2:
-            {
-                RTGCPTR GCPtr = GCPtrGDT + (SelTSS & ~7) + 4;
-                *uSrc.pu32 = (uint32_t)GCPtr;
-                break;
-            }
-
-            /*
-             * Store the EFER or mask for the 32->64 bit switcher.
-             */
-            case FIX_EFER_OR_MASK:
-            {
-                uint32_t u32OrMask = MSR_K6_EFER_LME | MSR_K6_EFER_SCE;
-                /** note: we don't care if cpuid 0x8000001 isn't supported as that implies long mode isn't either, so this switcher would never be used. */
-                if (!!(ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_NX))
-                    u32OrMask |= MSR_K6_EFER_NXE;
-
-                *uSrc.pu32 = u32OrMask;
-                break;
-            }
-
-            /*
-             * Insert relative jump to specified target it FXSAVE/FXRSTOR isn't supported by the cpu.
-             */
-            case FIX_NO_FXSAVE_JMP:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg < pSwitcher->cbCode);
-                if (!CPUMSupportsFXSR(pVM))
-                {
-                    *uSrc.pu8++ = 0xe9; /* jmp rel32 */
-                    *uSrc.pu32++ = offTrg - (offSrc + 5);
-                }
-                else
-                {
-                    *uSrc.pu8++ = *((uint8_t *)pSwitcher->pvCode + offSrc);
-                    *uSrc.pu32++ = *(uint32_t *)((uint8_t *)pSwitcher->pvCode + offSrc + 1);
-                }
-                break;
-            }
-
-            /*
-             * Insert relative jump to specified target it SYSENTER isn't used by the host.
-             */
-            case FIX_NO_SYSENTER_JMP:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg < pSwitcher->cbCode);
-                if (!CPUMIsHostUsingSysEnter(pVM))
-                {
-                    *uSrc.pu8++ = 0xe9; /* jmp rel32 */
-                    *uSrc.pu32++ = offTrg - (offSrc + 5);
-                }
-                else
-                {
-                    *uSrc.pu8++ = *((uint8_t *)pSwitcher->pvCode + offSrc);
-                    *uSrc.pu32++ = *(uint32_t *)((uint8_t *)pSwitcher->pvCode + offSrc + 1);
-                }
-                break;
-            }
-
-            /*
-             * Insert relative jump to specified target it SYSCALL isn't used by the host.
-             */
-            case FIX_NO_SYSCALL_JMP:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offTrg < pSwitcher->cbCode);
-                if (!CPUMIsHostUsingSysCall(pVM))
-                {
-                    *uSrc.pu8++ = 0xe9; /* jmp rel32 */
-                    *uSrc.pu32++ = offTrg - (offSrc + 5);
-                }
-                else
-                {
-                    *uSrc.pu8++ = *((uint8_t *)pSwitcher->pvCode + offSrc);
-                    *uSrc.pu32++ = *(uint32_t *)((uint8_t *)pSwitcher->pvCode + offSrc + 1);
-                }
-                break;
-            }
-
-            /*
-             * 32-bit HC pointer fixup to (HC) target within the code (32-bit offset).
-             */
-            case FIX_HC_32BIT:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offSrc < pSwitcher->cbCode);
-                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
-                *uSrc.pu32 = R0PtrCode + offTrg;
-                break;
-            }
-
-#if defined(RT_ARCH_AMD64) || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-            /*
-             * 64-bit HC Code Selector (no argument).
-             */
-            case FIX_HC_64BIT_CS:
-            {
-                Assert(offSrc < pSwitcher->cbCode);
-# if defined(RT_OS_DARWIN) && defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
-                *uSrc.pu16 = 0x80; /* KERNEL64_CS from i386/seg.h */
-# else
-                AssertFatalMsgFailed(("FIX_HC_64BIT_CS not implemented for this host\n"));
-# endif
-                break;
-            }
-
-            /*
-             * 64-bit HC pointer to the CPUM instance data (no argument).
-             */
-            case FIX_HC_64BIT_CPUM:
-            {
-                Assert(offSrc < pSwitcher->cbCode);
-                *uSrc.pu64 = pVM->pVMR0 + RT_OFFSETOF(VM, cpum);
-                break;
-            }
-#endif
-            /*
-             * 64-bit HC pointer fixup to (HC) target within the code (32-bit offset).
-             */
-            case FIX_HC_64BIT:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offSrc < pSwitcher->cbCode);
-                Assert(offTrg - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0 || offTrg - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1);
-                *uSrc.pu64 = R0PtrCode + offTrg;
-                break;
-            }
-
-#ifdef RT_ARCH_X86
-            case FIX_GC_64_BIT_CPUM_OFF:
-            {
-                uint32_t offCPUM = *u.pu32++;
-                Assert(offCPUM < sizeof(pVM->cpum));
-                *uSrc.pu64 = (uint32_t)(VM_RC_ADDR(pVM, &pVM->cpum) + offCPUM);
-                break;
-            }
-#endif
-
-            /*
-             * 32-bit ID pointer to (ID) target within the code (32-bit offset).
-             */
-            case FIX_ID_32BIT:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offSrc < pSwitcher->cbCode);
-                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
-                *uSrc.pu32 = u32IDCode + offTrg;
-                break;
-            }
-
-            /*
-             * 64-bit ID pointer to (ID) target within the code (32-bit offset).
-             */
-            case FIX_ID_64BIT:
-            case FIX_HC_64BIT_NOCHECK:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offSrc < pSwitcher->cbCode);
-                Assert(u8 == FIX_HC_64BIT_NOCHECK || offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
-                *uSrc.pu64 = u32IDCode + offTrg;
-                break;
-            }
-
-            /*
-             * Far 16:32 ID pointer to 64-bit mode (ID) target within the code (32-bit offset).
-             */
-            case FIX_ID_FAR32_TO_64BIT_MODE:
-            {
-                uint32_t offTrg = *u.pu32++;
-                Assert(offSrc < pSwitcher->cbCode);
-                Assert(offTrg - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0 || offTrg - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1);
-                *uSrc.pu32++ = u32IDCode + offTrg;
-                *uSrc.pu16 = SelCS64;
-                AssertRelease(SelCS64);
-                break;
-            }
-
-#ifdef VBOX_WITH_NMI
-            /*
-             * 32-bit address to the APIC base.
-             */
-            case FIX_GC_APIC_BASE_32BIT:
-            {
-                *uSrc.pu32 = pVM->vmm.s.GCPtrApicBase;
-                break;
-            }
-#endif
-
-            default:
-                AssertReleaseMsgFailed(("Unknown fixup %d in switcher %s\n", u8, pSwitcher->pszDesc));
-                break;
-        }
-    }
-
-#ifdef LOG_ENABLED
-    /*
-     * If Log2 is enabled disassemble the switcher code.
-     *
-     * The switcher code have 1-2 HC parts, 1 GC part and 0-2 ID parts.
-     */
-    if (LogIs2Enabled())
-    {
-        RTLogPrintf("*** Disassembly of switcher %d '%s' %#x bytes ***\n"
-                    "   R0PtrCode   = %p\n"
-                    "   pu8CodeR3   = %p\n"
-                    "   GCPtrCode   = %RGv\n"
-                    "   u32IDCode   = %08x\n"
-                    "   pVMRC       = %RRv\n"
-                    "   pCPUMRC     = %RRv\n"
-                    "   pVMR3       = %p\n"
-                    "   pCPUMR3     = %p\n"
-                    "   GCPtrGDT    = %RGv\n"
-                    "   InterCR3s   = %08RHp, %08RHp, %08RHp (32-Bit, PAE, AMD64)\n"
-                    "   HyperCR3s   = %08RHp (32-Bit, PAE & AMD64)\n"
-                    "   SelCS       = %04x\n"
-                    "   SelDS       = %04x\n"
-                    "   SelCS64     = %04x\n"
-                    "   SelTSS      = %04x\n",
-                    pSwitcher->enmType, pSwitcher->pszDesc, pSwitcher->cbCode,
-                    R0PtrCode,
-                    pu8CodeR3,
-                    GCPtrCode,
-                    u32IDCode,
-                    VM_RC_ADDR(pVM, pVM),
-                    VM_RC_ADDR(pVM, &pVM->cpum),
-                    pVM,
-                    &pVM->cpum,
-                    GCPtrGDT,
-                    PGMGetInter32BitCR3(pVM), PGMGetInterPaeCR3(pVM), PGMGetInterAmd64CR3(pVM),
-                    PGMGetHyperCR3(VMMGetCpu(pVM)),
-                    SelCS, SelDS, SelCS64, SelTSS);
-
-        uint32_t offCode = 0;
-        while (offCode < pSwitcher->cbCode)
-        {
-            /*
-             * Figure out where this is.
-             */
-            const char *pszDesc = NULL;
-            RTUINTPTR   uBase;
-            uint32_t    cbCode;
-            if (offCode - pSwitcher->offHCCode0 < pSwitcher->cbHCCode0)
-            {
-                pszDesc = "HCCode0";
-                uBase   = R0PtrCode;
-                offCode = pSwitcher->offHCCode0;
-                cbCode  = pSwitcher->cbHCCode0;
-            }
-            else if (offCode - pSwitcher->offHCCode1 < pSwitcher->cbHCCode1)
-            {
-                pszDesc = "HCCode1";
-                uBase   = R0PtrCode;
-                offCode = pSwitcher->offHCCode1;
-                cbCode  = pSwitcher->cbHCCode1;
-            }
-            else if (offCode - pSwitcher->offGCCode < pSwitcher->cbGCCode)
-            {
-                pszDesc = "GCCode";
-                uBase   = GCPtrCode;
-                offCode = pSwitcher->offGCCode;
-                cbCode  = pSwitcher->cbGCCode;
-            }
-            else if (offCode - pSwitcher->offIDCode0 < pSwitcher->cbIDCode0)
-            {
-                pszDesc = "IDCode0";
-                uBase   = u32IDCode;
-                offCode = pSwitcher->offIDCode0;
-                cbCode  = pSwitcher->cbIDCode0;
-            }
-            else if (offCode - pSwitcher->offIDCode1 < pSwitcher->cbIDCode1)
-            {
-                pszDesc = "IDCode1";
-                uBase   = u32IDCode;
-                offCode = pSwitcher->offIDCode1;
-                cbCode  = pSwitcher->cbIDCode1;
-            }
-            else
-            {
-                RTLogPrintf("  %04x: %02x '%c' (nowhere)\n",
-                            offCode, pu8CodeR3[offCode], RT_C_IS_PRINT(pu8CodeR3[offCode]) ? pu8CodeR3[offCode] : ' ');
-                offCode++;
-                continue;
-            }
-
-            /*
-             * Disassemble it.
-             */
-            RTLogPrintf("  %s: offCode=%#x cbCode=%#x\n", pszDesc, offCode, cbCode);
-            DISCPUSTATE Cpu;
-
-            memset(&Cpu, 0, sizeof(Cpu));
-            Cpu.mode = CPUMODE_32BIT;
-            while (cbCode > 0)
-            {
-                /* try label it */
-                if (pSwitcher->offR0HostToGuest == offCode)
-                    RTLogPrintf(" *R0HostToGuest:\n");
-                if (pSwitcher->offGCGuestToHost == offCode)
-                    RTLogPrintf(" *GCGuestToHost:\n");
-                if (pSwitcher->offGCCallTrampoline == offCode)
-                    RTLogPrintf(" *GCCallTrampoline:\n");
-                if (pSwitcher->offGCGuestToHostAsm == offCode)
-                    RTLogPrintf(" *GCGuestToHostAsm:\n");
-                if (pSwitcher->offGCGuestToHostAsmHyperCtx == offCode)
-                    RTLogPrintf(" *GCGuestToHostAsmHyperCtx:\n");
-                if (pSwitcher->offGCGuestToHostAsmGuestCtx == offCode)
-                    RTLogPrintf(" *GCGuestToHostAsmGuestCtx:\n");
-
-                /* disas */
-                uint32_t cbInstr = 0;
-                char szDisas[256];
-                if (RT_SUCCESS(DISInstr(&Cpu, (uintptr_t)pu8CodeR3 + offCode, uBase - (uintptr_t)pu8CodeR3, &cbInstr, szDisas)))
-                    RTLogPrintf("  %04x: %s", offCode, szDisas); //for whatever reason szDisas includes '\n'.
-                else
-                {
-                    RTLogPrintf("  %04x: %02x '%c'\n",
-                                offCode, pu8CodeR3[offCode], RT_C_IS_PRINT(pu8CodeR3[offCode]) ? pu8CodeR3[offCode] : ' ');
-                    cbInstr = 1;
-                }
-                offCode += cbInstr;
-                cbCode -= RT_MIN(cbInstr, cbCode);
-            }
-        }
-    }
-#endif
-}
-
-
-/**
- * Relocator for the 32-Bit to 32-Bit world switcher.
- */
-DECLCALLBACK(void) vmmR3Switcher32BitTo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
-}
-
-
-/**
- * Relocator for the 32-Bit to PAE world switcher.
- */
-DECLCALLBACK(void) vmmR3Switcher32BitToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
-}
-
-
-/**
- * Relocator for the 32-Bit to AMD64 world switcher.
- */
-DECLCALLBACK(void) vmmR3Switcher32BitToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
-}
-
-
-/**
- * Relocator for the PAE to 32-Bit world switcher.
- */
-DECLCALLBACK(void) vmmR3SwitcherPAETo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
-}
-
-
-/**
- * Relocator for the PAE to PAE world switcher.
- */
-DECLCALLBACK(void) vmmR3SwitcherPAEToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), 0);
-}
-
-/**
- * Relocator for the PAE to AMD64 world switcher.
- */
-DECLCALLBACK(void) vmmR3SwitcherPAEToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
-}
-
-
-/**
- * Relocator for the AMD64 to 32-bit world switcher.
- */
-DECLCALLBACK(void) vmmR3SwitcherAMD64To32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
-}
-
-
-/**
- * Relocator for the AMD64 to PAE world switcher.
- */
-DECLCALLBACK(void) vmmR3SwitcherAMD64ToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IDCode)
-{
-    vmmR3SwitcherGenericRelocate(pVM, pSwitcher, R0PtrCode, pu8CodeR3, GCPtrCode, u32IDCode,
-                                 SELMGetHyperCS(pVM), SELMGetHyperDS(pVM), SELMGetHyperTSS(pVM), SELMGetHyperGDT(pVM), SELMGetHyperCS64(pVM));
-}
-
-
-/**
- * Selects the switcher to be used for switching to GC.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- * @param   enmSwitcher     The new switcher.
- * @remark  This function may be called before the VMM is initialized.
- */
-VMMR3_INT_DECL(int) VMMR3SelectSwitcher(PVM pVM, VMMSWITCHER enmSwitcher)
-{
-    /*
-     * Validate input.
-     */
-    if (    enmSwitcher < VMMSWITCHER_INVALID
-        ||  enmSwitcher >= VMMSWITCHER_MAX)
-    {
-        AssertMsgFailed(("Invalid input enmSwitcher=%d\n", enmSwitcher));
-        return VERR_INVALID_PARAMETER;
-    }
-
-    /* Do nothing if the switcher is disabled. */
-    if (pVM->vmm.s.fSwitcherDisabled)
-        return VINF_SUCCESS;
-
-    /*
-     * Select the new switcher.
-     */
-    PVMMSWITCHERDEF pSwitcher = s_apSwitchers[enmSwitcher];
-    if (pSwitcher)
-    {
-        Log(("VMMR3SelectSwitcher: enmSwitcher %d -> %d %s\n", pVM->vmm.s.enmSwitcher, enmSwitcher, pSwitcher->pszDesc));
-        pVM->vmm.s.enmSwitcher = enmSwitcher;
-
-        RTR0PTR     pbCodeR0 = (RTR0PTR)pVM->vmm.s.pvCoreCodeR0 + pVM->vmm.s.aoffSwitchers[enmSwitcher]; /** @todo fix the pvCoreCodeR0 type */
-        pVM->vmm.s.pfnHostToGuestR0 = pbCodeR0 + pSwitcher->offR0HostToGuest;
-
-        RTGCPTR     GCPtr = pVM->vmm.s.pvCoreCodeRC + pVM->vmm.s.aoffSwitchers[enmSwitcher];
-        pVM->vmm.s.pfnGuestToHostRC         = GCPtr + pSwitcher->offGCGuestToHost;
-        pVM->vmm.s.pfnCallTrampolineRC      = GCPtr + pSwitcher->offGCCallTrampoline;
-        pVM->pfnVMMGCGuestToHostAsm         = GCPtr + pSwitcher->offGCGuestToHostAsm;
-        pVM->pfnVMMGCGuestToHostAsmHyperCtx = GCPtr + pSwitcher->offGCGuestToHostAsmHyperCtx;
-        pVM->pfnVMMGCGuestToHostAsmGuestCtx = GCPtr + pSwitcher->offGCGuestToHostAsmGuestCtx;
-        return VINF_SUCCESS;
-    }
-
-    return VERR_NOT_IMPLEMENTED;
-}
-
-
-/**
- * Disable the switcher logic permanently.
- *
- * @returns VBox status code.
- * @param   pVM             VM handle.
- */
-VMMR3_INT_DECL(int) VMMR3DisableSwitcher(PVM pVM)
-{
-/** @todo r=bird: I would suggest that we create a dummy switcher which just does something like:
- * @code
- *       mov eax, VERR_INTERNAL_ERROR
- *       ret
- * @endcode
- * And then check for fSwitcherDisabled in VMMR3SelectSwitcher() in order to prevent it from being removed.
- */
-    pVM->vmm.s.fSwitcherDisabled = true;
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Gets the switcher to be used for switching to GC.
- *
- * @returns host to guest ring 0 switcher entrypoint
- * @param   pVM             VM handle.
- * @param   enmSwitcher     The new switcher.
- */
-VMMR3_INT_DECL(RTR0PTR) VMMR3GetHostToGuestSwitcher(PVM pVM, VMMSWITCHER enmSwitcher)
-{
-    /*
-     * Validate input.
-     */
-    if (    enmSwitcher < VMMSWITCHER_INVALID
-        ||  enmSwitcher >= VMMSWITCHER_MAX)
-    {
-        AssertMsgFailed(("Invalid input enmSwitcher=%d\n", enmSwitcher));
-        return NIL_RTR0PTR;
-    }
-
-    /*
-     * Select the new switcher.
-     */
-    PVMMSWITCHERDEF pSwitcher = s_apSwitchers[enmSwitcher];
-    if (pSwitcher)
-    {
-        RTR0PTR     pbCodeR0 = (RTR0PTR)pVM->vmm.s.pvCoreCodeR0 + pVM->vmm.s.aoffSwitchers[enmSwitcher]; /** @todo fix the pvCoreCodeR0 type */
-        return pbCodeR0 + pSwitcher->offR0HostToGuest;
-    }
-    return NIL_RTR0PTR;
-}
Index: /trunk/src/VBox/VMM/VMMSwitcher/AMD64andLegacy.mac
===================================================================
--- /trunk/src/VBox/VMM/VMMSwitcher/AMD64andLegacy.mac	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMSwitcher/AMD64andLegacy.mac	(revision 35333)
@@ -29,5 +29,5 @@
 %include "VBox/vm.mac"
 %include "CPUMInternal.mac"
-%include "VMMSwitcher/VMMSwitcher.mac"
+%include "VMMSwitcher.mac"
 
 
Index: /trunk/src/VBox/VMM/VMMSwitcher/LegacyandAMD64.mac
===================================================================
--- /trunk/src/VBox/VMM/VMMSwitcher/LegacyandAMD64.mac	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMSwitcher/LegacyandAMD64.mac	(revision 35333)
@@ -32,5 +32,5 @@
 %include "VBox/vm.mac"
 %include "CPUMInternal.mac"
-%include "VMMSwitcher/VMMSwitcher.mac"
+%include "VMMSwitcher.mac"
 
 
Index: /trunk/src/VBox/VMM/VMMSwitcher/PAEand32Bit.mac
===================================================================
--- /trunk/src/VBox/VMM/VMMSwitcher/PAEand32Bit.mac	(revision 35332)
+++ /trunk/src/VBox/VMM/VMMSwitcher/PAEand32Bit.mac	(revision 35333)
@@ -28,5 +28,5 @@
 %include "VBox/vm.mac"
 %include "CPUMInternal.mac"
-%include "VMMSwitcher/VMMSwitcher.mac"
+%include "VMMSwitcher.mac"
 
 %undef NEED_ID
Index: unk/src/VBox/VMM/VMMSwitcher/VMMSwitcher.h
===================================================================
--- /trunk/src/VBox/VMM/VMMSwitcher/VMMSwitcher.h	(revision 35332)
+++ 	(revision )
@@ -1,150 +1,0 @@
-/* $Id$ */
-/** @file
- * VMM - World Switchers.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-#ifndef ___VMMSwitcher_h
-#define ___VMMSwitcher_h
-
-#include <VBox/vmm.h>
-
-/** @name   Fixup Types.
- * @{
- */
-/** @todo document what arguments these take and what they do. */
-#define FIX_HC_2_GC_NEAR_REL    1
-#define FIX_HC_2_ID_NEAR_REL    2
-#define FIX_GC_2_HC_NEAR_REL    3
-#define FIX_GC_2_ID_NEAR_REL    4
-#define FIX_ID_2_HC_NEAR_REL    5
-#define FIX_ID_2_GC_NEAR_REL    6
-#define FIX_GC_FAR32            7
-#define FIX_GC_CPUM_OFF         8
-#define FIX_GC_VM_OFF           9   /** @todo cleanup fixup names GC->RC, HC->R0. */
-#define FIX_HC_CPUM_OFF         10
-#define FIX_HC_VM_OFF           11
-#define FIX_INTER_32BIT_CR3     12
-#define FIX_INTER_PAE_CR3       13
-#define FIX_INTER_AMD64_CR3     14
-#define FIX_HYPER_CS            18
-#define FIX_HYPER_DS            19
-#define FIX_HYPER_TSS           20
-#define FIX_GC_TSS_GDTE_DW2     21
-#define FIX_NO_FXSAVE_JMP       24
-#define FIX_NO_SYSENTER_JMP     25
-#define FIX_NO_SYSCALL_JMP      26
-#define FIX_HC_32BIT            27
-#define FIX_HC_64BIT            28
-#define FIX_HC_64BIT_CPUM       29
-#define FIX_HC_64BIT_CS         30
-#define FIX_ID_32BIT            31
-#define FIX_ID_64BIT            32
-#define FIX_ID_FAR32_TO_64BIT_MODE 33
-#define FIX_GC_APIC_BASE_32BIT  34
-#define FIX_HC_64BIT_NOCHECK    35
-#define FIX_GC_64_BIT_CPUM_OFF  36
-#define FIX_GC_CPUMCPU_OFF      37
-#define FIX_EFER_OR_MASK        38
-#define FIX_THE_END             255
-/** @} */
-
-
-/** Pointer to a switcher definition. */
-typedef struct VMMSWITCHERDEF *PVMMSWITCHERDEF;
-
-/**
- * Callback function for relocating the core code belonging to a switcher.
- *
- * @param   pVM         VM handle.
- * @param   pSwitcher   Pointer to the switcher structure.
- * @param   R0PtrCode   Pointer to the first code byte in the ring-0 mapping.
- * @param   pu8CodeR3   Pointer to the first code byte in the ring-3 mapping.
- * @param   GCPtrCode   The GC address of the first code byte.
- * @param   u32IDCode   The address of the identity mapped code (first byte).
- */
-typedef DECLCALLBACK(void) FNVMMSWITCHERRELOCATE(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3,
-                                                 RTGCPTR GCPtrCode, uint32_t u32IDCode);
-/** Pointer to a FNVMMSWITCHERRELOCATE(). */
-typedef FNVMMSWITCHERRELOCATE *PFNVMMSWITCHERRELOCATE;
-
-/**
- * VMM Switcher structure.
- */
-#pragma pack(1)
-typedef struct VMMSWITCHERDEF
-{
-    /** Pointer to the code. */
-    void       *pvCode;
-    /** Pointer to the fixup records. */
-    void       *pvFixups;
-    /** Pointer to the description. */
-    const char *pszDesc;
-    /** Function which performs the necessary relocations. */
-    PFNVMMSWITCHERRELOCATE pfnRelocate;
-    /** The switcher type. */
-    VMMSWITCHER enmType;
-    /** Size of the entire code chunk. */
-    uint32_t    cbCode;
-    /** vmmR0HostToGuest C entrypoint. */
-    uint32_t    offR0HostToGuest;
-    /** vmmGCGuestToHost C entrypoint. */
-    uint32_t    offGCGuestToHost;
-    /** vmmGCCallTrampoline address. */
-    uint32_t    offGCCallTrampoline;
-    /** vmmGCGuestToHostAsm assembly entrypoint. */
-    uint32_t    offGCGuestToHostAsm;
-    /** vmmGCGuestToHostAsmHyperCtx assembly entrypoint taking HyperCtx. */
-    uint32_t    offGCGuestToHostAsmHyperCtx;
-    /** vmmGCGuestToHostAsmGuestCtx assembly entrypoint taking GuestCtx. */
-    uint32_t    offGCGuestToHostAsmGuestCtx;
-    /** @name Disassembly Regions.
-     * @{ */
-    uint32_t    offHCCode0;
-    uint32_t    cbHCCode0;
-    uint32_t    offHCCode1;
-    uint32_t    cbHCCode1;
-    uint32_t    offIDCode0;
-    uint32_t    cbIDCode0;
-    uint32_t    offIDCode1;
-    uint32_t    cbIDCode1;
-    uint32_t    offGCCode;
-    uint32_t    cbGCCode;
-    /** @} */
-} VMMSWITCHERDEF;
-#pragma pack()
-
-RT_C_DECLS_BEGIN
-extern VMMSWITCHERDEF vmmR3Switcher32BitTo32Bit_Def;
-extern VMMSWITCHERDEF vmmR3Switcher32BitToPAE_Def;
-extern VMMSWITCHERDEF vmmR3Switcher32BitToAMD64_Def;
-extern VMMSWITCHERDEF vmmR3SwitcherPAETo32Bit_Def;
-extern VMMSWITCHERDEF vmmR3SwitcherPAEToPAE_Def;
-extern VMMSWITCHERDEF vmmR3SwitcherPAEToAMD64_Def;
-extern VMMSWITCHERDEF vmmR3SwitcherAMD64To32Bit_Def;
-extern VMMSWITCHERDEF vmmR3SwitcherAMD64ToPAE_Def;
-extern VMMSWITCHERDEF vmmR3SwitcherAMD64ToAMD64_Def;
-
-extern DECLCALLBACK(void) vmmR3Switcher32BitTo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3Switcher32BitToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3Switcher32BitToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3SwitcherPAETo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3SwitcherPAEToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3SwitcherPAEToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3SwitcherAMD64To32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3SwitcherAMD64ToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-extern DECLCALLBACK(void) vmmR3SwitcherAMD64ToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
-RT_C_DECLS_END
-
-#endif
Index: unk/src/VBox/VMM/VMMSwitcher/VMMSwitcher.mac
===================================================================
--- /trunk/src/VBox/VMM/VMMSwitcher/VMMSwitcher.mac	(revision 35332)
+++ 	(revision )
@@ -1,140 +1,0 @@
-; $Id$
-;; @file
-; VMM - World Switchers.
-;
-
-;
-; Copyright (C) 2006-2007 Oracle Corporation
-;
-; This file is part of VirtualBox Open Source Edition (OSE), as
-; available from http://www.virtualbox.org. This file is free software;
-; you can redistribute it and/or modify it under the terms of the GNU
-; General Public License (GPL) as published by the Free Software
-; Foundation, in version 2 as it comes in the "COPYING" file of the
-; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
-; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
-;
-
-%ifndef ___VMMSwitcher_mac
-%define ___VMMSwitcher_mac
-
-%include "iprt/asmdefs.mac"
-
-; enum VMMSWITCHER {
-%define VMMSWITCHER_INVALID             0
-%define VMMSWITCHER_32_TO_32            1
-%define VMMSWITCHER_32_TO_PAE           2
-%define VMMSWITCHER_32_TO_AMD64         3
-%define VMMSWITCHER_PAE_TO_32BIT        4
-%define VMMSWITCHER_PAE_TO_PAE          5
-%define VMMSWITCHER_PAE_TO_AMD64        6
-%define VMMSWITCHER_AMD64_TO_32         7
-%define VMMSWITCHER_AMD64_TO_PAE        8
-%define VMMSWITCHER_AMD64_TO_AMD64      9
-;; @todo the rest are all wrong. sync with vmm.h.
-%define VMMSWITCHER_HOST_TO_VMX         9
-%define VMMSWITCHER_HOST_TO_SVM         10
-%define VMMSWITCHER_MAX                 12
-; }
-
-
-struc VMMSWITCHERDEF
-    .pvCode:                        RTCCPTR_RES 1
-    .pvFixups:                      RTCCPTR_RES 1
-    .pszDesc:                       RTCCPTR_RES 1
-    .pfnRelocate:                   RTCCPTR_RES 1
-    .enmType:                       resd 1
-    .cbCode:                        resd 1
-    .offR0HostToGuest:              resd 1
-    .offGCGuestToHost:              resd 1
-    .offGCCallTrampoline:           resd 1
-    .offGCGuestToHostAsm:           resd 1
-    .offGCGuestToHostAsmHyperCtx:   resd 1
-    .offGCGuestToHostAsmGuestCtx:   resd 1
-    ; disasm help
-    .offHCCode0:                    resd 1
-    .cbHCCode0:                     resd 1
-    .offHCCode1:                    resd 1
-    .cbHCCode1:                     resd 1
-    .offIDCode0:                    resd 1
-    .cbIDCode0:                     resd 1
-    .offIDCode1:                    resd 1
-    .cbIDCode1:                     resd 1
-    .offGCCode:                     resd 1
-    .cbGCCode:                      resd 1
-endstruc
-
-
-;/** @name   Fixup Types.
-; * @{
-; */
-%define FIX_HC_2_GC_NEAR_REL    1
-%define FIX_HC_2_ID_NEAR_REL    2
-%define FIX_GC_2_HC_NEAR_REL    3
-%define FIX_GC_2_ID_NEAR_REL    4
-%define FIX_ID_2_HC_NEAR_REL    5
-%define FIX_ID_2_GC_NEAR_REL    6
-%define FIX_GC_FAR32            7
-%define FIX_GC_CPUM_OFF         8
-%define FIX_GC_VM_OFF           9
-%define FIX_HC_CPUM_OFF         10
-%define FIX_HC_VM_OFF           11
-%define FIX_INTER_32BIT_CR3     12
-%define FIX_INTER_PAE_CR3       13
-%define FIX_INTER_AMD64_CR3     14
-%define FIX_HYPER_CS            18
-%define FIX_HYPER_DS            19
-%define FIX_HYPER_TSS           20
-%define FIX_GC_TSS_GDTE_DW2     21
-%define FIX_NO_FXSAVE_JMP       24
-%define FIX_NO_SYSENTER_JMP     25
-%define FIX_NO_SYSCALL_JMP      26
-%define FIX_HC_32BIT            27
-%define FIX_HC_64BIT            28
-%define FIX_HC_64BIT_CPUM       29
-%define FIX_HC_64BIT_CS         30
-%define FIX_ID_32BIT            31
-%define FIX_ID_64BIT            32
-%define FIX_ID_FAR32_TO_64BIT_MODE 33
-%define FIX_GC_APIC_BASE_32BIT  34
-%define FIX_HC_64BIT_NOCHECK    35
-%define FIX_GC_64_BIT_CPUM_OFF  36
-%define FIX_GC_CPUMCPU_OFF      37
-%define FIX_EFER_OR_MASK        38
-%define FIX_THE_END             255
-;/** @} */
-
-
-;;
-; Generate a fixup label.
-; @param %1     Type of fixup (use one of those %defines)
-; @param %2     Offset into the instruction.
-; @param %3     Optional fixup data.
-%macro FIXUP 2-*
-BEGINDATA
-    db  %1  ; the type
-    dd  %%instruction + %2 - NAME(Start)
-    %rotate 2
-    %rep %0 - 2
-        dd      %1
-        %rotate 1
-    %endrep
-
-BEGINCODE
-%%instruction:
-%endmacro
-
-
-;; IMPORTANT all COM_ functions trashes esi, some edi and the LOOP_SHORT_WHILE kills ecx.
-;%define DEBUG_STUFF 1
-
-%ifdef DEBUG_STUFF
-    %define DEBUG_CHAR(ch) COM_CHAR ch
-    %define DEBUG_S_CHAR(ch) COM_CHAR ch
-%else
-    %define DEBUG_CHAR(ch)
-    %define DEBUG_S_CHAR(ch)
-%endif
-
-%endif ; !___VMMSwitcher_mac
-
Index: unk/src/VBox/VMM/VMMTests.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMMTests.cpp	(revision 35332)
+++ 	(revision )
@@ -1,592 +1,0 @@
-/* $Id$ */
-/** @file
- * VMM - The Virtual Machine Monitor Core, Tests.
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-//#define NO_SUPCALLR0VMM
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_VMM
-#include <iprt/asm-amd64-x86.h> /* for SUPGetCpuHzFromGIP */
-#include <VBox/vmm.h>
-#include <VBox/pdmapi.h>
-#include <VBox/cpum.h>
-#include <VBox/dbg.h>
-#include <VBox/mm.h>
-#include <VBox/trpm.h>
-#include <VBox/selm.h>
-#include "VMMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/x86.h>
-#include <VBox/hwaccm.h>
-
-#include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <iprt/time.h>
-#include <iprt/stream.h>
-#include <iprt/string.h>
-
-
-/**
- * Performs a testcase.
- *
- * @returns return value from the test.
- * @param   pVM         The VM handle.
- * @param   enmTestcase The testcase operation to perform.
- * @param   uVariation  The testcase variation id.
- */
-static int vmmR3DoGCTest(PVM pVM, VMMGCOPERATION enmTestcase, unsigned uVariation)
-{
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    RTRCPTR RCPtrEP;
-    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    CPUMHyperSetCtxCore(pVCpu, NULL);
-    memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE);
-    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
-    CPUMPushHyper(pVCpu, uVariation);
-    CPUMPushHyper(pVCpu, enmTestcase);
-    CPUMPushHyper(pVCpu, pVM->pVMRC);
-    CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
-    CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
-    CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
-    Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
-    rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
-    if (RT_LIKELY(rc == VINF_SUCCESS))
-        rc = pVCpu->vmm.s.iLastGZRc;
-    return rc;
-}
-
-
-/**
- * Performs a trap test.
- *
- * @returns Return value from the trap test.
- * @param   pVM         The VM handle.
- * @param   u8Trap      The trap number to test.
- * @param   uVariation  The testcase variation.
- * @param   rcExpect    The expected result.
- * @param   u32Eax      The expected eax value.
- * @param   pszFaultEIP The fault address. Pass NULL if this isn't available or doesn't apply.
- * @param   pszDesc     The test description.
- */
-static int vmmR3DoTrapTest(PVM pVM, uint8_t u8Trap, unsigned uVariation, int rcExpect, uint32_t u32Eax, const char *pszFaultEIP, const char *pszDesc)
-{
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-    RTPrintf("VMM: testing 0%x / %d - %s\n", u8Trap, uVariation, pszDesc);
-
-    RTRCPTR RCPtrEP;
-    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    CPUMHyperSetCtxCore(pVCpu, NULL);
-    memset(pVCpu->vmm.s.pbEMTStackR3, 0xaa, VMM_STACK_SIZE);
-    CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
-    CPUMPushHyper(pVCpu, uVariation);
-    CPUMPushHyper(pVCpu, u8Trap + VMMGC_DO_TESTCASE_TRAP_FIRST);
-    CPUMPushHyper(pVCpu, pVM->pVMRC);
-    CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
-    CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
-    CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
-    Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
-    rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
-    if (RT_LIKELY(rc == VINF_SUCCESS))
-        rc = pVCpu->vmm.s.iLastGZRc;
-    bool fDump = false;
-    if (rc != rcExpect)
-    {
-        RTPrintf("VMM: FAILURE - rc=%Rrc expected %Rrc\n", rc, rcExpect);
-        if (rc != VERR_NOT_IMPLEMENTED)
-            fDump = true;
-    }
-    else if (    rcExpect != VINF_SUCCESS
-             &&  u8Trap != 8 /* double fault doesn't dare set TrapNo. */
-             &&  u8Trap != 3 /* guest only, we're not in guest. */
-             &&  u8Trap != 1 /* guest only, we're not in guest. */
-             &&  u8Trap != TRPMGetTrapNo(pVCpu))
-    {
-        RTPrintf("VMM: FAILURE - Trap %#x expected %#x\n", TRPMGetTrapNo(pVCpu), u8Trap);
-        fDump = true;
-    }
-    else if (pszFaultEIP)
-    {
-        RTRCPTR RCPtrFault;
-        int rc2 = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, pszFaultEIP, &RCPtrFault);
-        if (RT_FAILURE(rc2))
-            RTPrintf("VMM: FAILURE - Failed to resolve symbol '%s', %Rrc!\n", pszFaultEIP, rc);
-        else if (RCPtrFault != CPUMGetHyperEIP(pVCpu))
-        {
-            RTPrintf("VMM: FAILURE - EIP=%08RX32 expected %RRv (%s)\n", CPUMGetHyperEIP(pVCpu), RCPtrFault, pszFaultEIP);
-            fDump = true;
-        }
-    }
-    else if (rcExpect != VINF_SUCCESS)
-    {
-        if (CPUMGetHyperSS(pVCpu) == SELMGetHyperDS(pVM))
-            RTPrintf("VMM: FAILURE - ss=%x expected %x\n", CPUMGetHyperSS(pVCpu), SELMGetHyperDS(pVM));
-        if (CPUMGetHyperES(pVCpu) == SELMGetHyperDS(pVM))
-            RTPrintf("VMM: FAILURE - es=%x expected %x\n", CPUMGetHyperES(pVCpu), SELMGetHyperDS(pVM));
-        if (CPUMGetHyperDS(pVCpu) == SELMGetHyperDS(pVM))
-            RTPrintf("VMM: FAILURE - ds=%x expected %x\n", CPUMGetHyperDS(pVCpu), SELMGetHyperDS(pVM));
-        if (CPUMGetHyperFS(pVCpu) == SELMGetHyperDS(pVM))
-            RTPrintf("VMM: FAILURE - fs=%x expected %x\n", CPUMGetHyperFS(pVCpu), SELMGetHyperDS(pVM));
-        if (CPUMGetHyperGS(pVCpu) == SELMGetHyperDS(pVM))
-            RTPrintf("VMM: FAILURE - gs=%x expected %x\n", CPUMGetHyperGS(pVCpu), SELMGetHyperDS(pVM));
-        if (CPUMGetHyperEDI(pVCpu) == 0x01234567)
-            RTPrintf("VMM: FAILURE - edi=%x expected %x\n", CPUMGetHyperEDI(pVCpu), 0x01234567);
-        if (CPUMGetHyperESI(pVCpu) == 0x42000042)
-            RTPrintf("VMM: FAILURE - esi=%x expected %x\n", CPUMGetHyperESI(pVCpu), 0x42000042);
-        if (CPUMGetHyperEBP(pVCpu) == 0xffeeddcc)
-            RTPrintf("VMM: FAILURE - ebp=%x expected %x\n", CPUMGetHyperEBP(pVCpu), 0xffeeddcc);
-        if (CPUMGetHyperEBX(pVCpu) == 0x89abcdef)
-            RTPrintf("VMM: FAILURE - ebx=%x expected %x\n", CPUMGetHyperEBX(pVCpu), 0x89abcdef);
-        if (CPUMGetHyperECX(pVCpu) == 0xffffaaaa)
-            RTPrintf("VMM: FAILURE - ecx=%x expected %x\n", CPUMGetHyperECX(pVCpu), 0xffffaaaa);
-        if (CPUMGetHyperEDX(pVCpu) == 0x77778888)
-            RTPrintf("VMM: FAILURE - edx=%x expected %x\n", CPUMGetHyperEDX(pVCpu), 0x77778888);
-        if (CPUMGetHyperEAX(pVCpu) == u32Eax)
-            RTPrintf("VMM: FAILURE - eax=%x expected %x\n", CPUMGetHyperEAX(pVCpu), u32Eax);
-    }
-    if (fDump)
-        VMMR3FatalDump(pVM, pVCpu, rc);
-    return rc;
-}
-
-
-/* execute the switch. */
-VMMR3DECL(int) VMMDoTest(PVM pVM)
-{
-#if 1
-    PVMCPU pVCpu = &pVM->aCpus[0];
-
-#ifdef NO_SUPCALLR0VMM
-    RTPrintf("NO_SUPCALLR0VMM\n");
-    return VINF_SUCCESS;
-#endif
-
-    /*
-     * Setup stack for calling VMMGCEntry().
-     */
-    RTRCPTR RCPtrEP;
-    int rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
-    if (RT_SUCCESS(rc))
-    {
-        RTPrintf("VMM: VMMGCEntry=%RRv\n", RCPtrEP);
-
-        /*
-         * Test various crashes which we must be able to recover from.
-         */
-        vmmR3DoTrapTest(pVM, 0x3, 0, VINF_EM_DBG_HYPER_ASSERTION,  0xf0f0f0f0, "vmmGCTestTrap3_FaultEIP", "int3");
-        vmmR3DoTrapTest(pVM, 0x3, 1, VINF_EM_DBG_HYPER_ASSERTION,  0xf0f0f0f0, "vmmGCTestTrap3_FaultEIP", "int3 WP");
-
-#if defined(DEBUG_bird) /* guess most people would like to skip these since they write to com1. */
-        vmmR3DoTrapTest(pVM, 0x8, 0, VERR_TRPM_PANIC,       0x00000000, "vmmGCTestTrap8_FaultEIP", "#DF [#PG]");
-        SELMR3Relocate(pVM); /* this resets the busy flag of the Trap 08 TSS */
-        bool f;
-        rc = CFGMR3QueryBool(CFGMR3GetRoot(pVM), "DoubleFault", &f);
-#if !defined(DEBUG_bird)
-        if (RT_SUCCESS(rc) && f)
-#endif
-        {
-            /* see triple fault warnings in SELM and VMMGC.cpp. */
-            vmmR3DoTrapTest(pVM, 0x8, 1, VERR_TRPM_PANIC,       0x00000000, "vmmGCTestTrap8_FaultEIP", "#DF [#PG] WP");
-            SELMR3Relocate(pVM); /* this resets the busy flag of the Trap 08 TSS */
-        }
-#endif
-
-        vmmR3DoTrapTest(pVM, 0xd, 0, VERR_TRPM_DONT_PANIC,  0xf0f0f0f0, "vmmGCTestTrap0d_FaultEIP", "ltr #GP");
-        ///@todo find a better \#GP case, on intel ltr will \#PF (busy update?) and not \#GP.
-        //vmmR3DoTrapTest(pVM, 0xd, 1, VERR_TRPM_DONT_PANIC,  0xf0f0f0f0, "vmmGCTestTrap0d_FaultEIP", "ltr #GP WP");
-
-        vmmR3DoTrapTest(pVM, 0xe, 0, VERR_TRPM_DONT_PANIC,  0x00000000, "vmmGCTestTrap0e_FaultEIP", "#PF (NULL)");
-        vmmR3DoTrapTest(pVM, 0xe, 1, VERR_TRPM_DONT_PANIC,  0x00000000, "vmmGCTestTrap0e_FaultEIP", "#PF (NULL) WP");
-        vmmR3DoTrapTest(pVM, 0xe, 2, VINF_SUCCESS,          0x00000000, NULL,                       "#PF w/Tmp Handler");
-        vmmR3DoTrapTest(pVM, 0xe, 4, VINF_SUCCESS,          0x00000000, NULL,                       "#PF w/Tmp Handler and bad fs");
-
-        /*
-         * Set a debug register and perform a context switch.
-         */
-        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
-        if (rc != VINF_SUCCESS)
-        {
-            RTPrintf("VMM: Nop test failed, rc=%Rrc not VINF_SUCCESS\n", rc);
-            return rc;
-        }
-
-        /* a harmless breakpoint */
-        RTPrintf("VMM: testing hardware bp at 0x10000 (not hit)\n");
-        DBGFADDRESS Addr;
-        DBGFR3AddrFromFlat(pVM, &Addr, 0x10000);
-        RTUINT iBp0;
-        rc = DBGFR3BpSetReg(pVM, &Addr, 0,  ~(uint64_t)0, X86_DR7_RW_EO, 1, &iBp0);
-        AssertReleaseRC(rc);
-        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
-        if (rc != VINF_SUCCESS)
-        {
-            RTPrintf("VMM: DR0=0x10000 test failed with rc=%Rrc!\n", rc);
-            return rc;
-        }
-
-        /* a bad one at VMMGCEntry */
-        RTPrintf("VMM: testing hardware bp at VMMGCEntry (hit)\n");
-        DBGFR3AddrFromFlat(pVM, &Addr, RCPtrEP);
-        RTUINT iBp1;
-        rc = DBGFR3BpSetReg(pVM, &Addr, 0,  ~(uint64_t)0, X86_DR7_RW_EO, 1, &iBp1);
-        AssertReleaseRC(rc);
-        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
-        if (rc != VINF_EM_DBG_HYPER_BREAKPOINT)
-        {
-            RTPrintf("VMM: DR1=VMMGCEntry test failed with rc=%Rrc! expected VINF_EM_RAW_BREAKPOINT_HYPER\n", rc);
-            return rc;
-        }
-
-        /* resume the breakpoint */
-        RTPrintf("VMM: resuming hyper after breakpoint\n");
-        CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_RF);
-        rc = VMMR3ResumeHyper(pVM, pVCpu);
-        if (rc != VINF_SUCCESS)
-        {
-            RTPrintf("VMM: failed to resume on hyper breakpoint, rc=%Rrc\n", rc);
-            return rc;
-        }
-
-        /* engage the breakpoint again and try single stepping. */
-        RTPrintf("VMM: testing hardware bp at VMMGCEntry + stepping\n");
-        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
-        if (rc != VINF_EM_DBG_HYPER_BREAKPOINT)
-        {
-            RTPrintf("VMM: DR1=VMMGCEntry test failed with rc=%Rrc! expected VINF_EM_RAW_BREAKPOINT_HYPER\n", rc);
-            return rc;
-        }
-
-        RTGCUINTREG OldPc = CPUMGetHyperEIP(pVCpu);
-        RTPrintf("%RGr=>", OldPc);
-        unsigned i;
-        for (i = 0; i < 8; i++)
-        {
-            CPUMSetHyperEFlags(pVCpu, CPUMGetHyperEFlags(pVCpu) | X86_EFL_TF | X86_EFL_RF);
-            rc = VMMR3ResumeHyper(pVM, pVCpu);
-            if (rc != VINF_EM_DBG_HYPER_STEPPED)
-            {
-                RTPrintf("\nVMM: failed to step on hyper breakpoint, rc=%Rrc\n", rc);
-                return rc;
-            }
-            RTGCUINTREG Pc = CPUMGetHyperEIP(pVCpu);
-            RTPrintf("%RGr=>", Pc);
-            if (Pc == OldPc)
-            {
-                RTPrintf("\nVMM: step failed, PC: %RGr -> %RGr\n", OldPc, Pc);
-                return VERR_GENERAL_FAILURE;
-            }
-            OldPc = Pc;
-        }
-        RTPrintf("ok\n");
-
-        /* done, clear it */
-        if (    RT_FAILURE(DBGFR3BpClear(pVM, iBp0))
-            ||  RT_FAILURE(DBGFR3BpClear(pVM, iBp1)))
-        {
-            RTPrintf("VMM: Failed to clear breakpoints!\n");
-            return VERR_GENERAL_FAILURE;
-        }
-        rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_NOP, 0);
-        if (rc != VINF_SUCCESS)
-        {
-            RTPrintf("VMM: NOP failed, rc=%Rrc\n", rc);
-            return rc;
-        }
-
-        /*
-         * Interrupt masking.
-         */
-        RTPrintf("VMM: interrupt masking...\n"); RTStrmFlush(g_pStdOut); RTThreadSleep(250);
-        for (i = 0; i < 10000; i++)
-        {
-            uint64_t StartTick = ASMReadTSC();
-            rc = vmmR3DoGCTest(pVM, VMMGC_DO_TESTCASE_INTERRUPT_MASKING, 0);
-            if (rc != VINF_SUCCESS)
-            {
-                RTPrintf("VMM: Interrupt masking failed: rc=%Rrc\n", rc);
-                return rc;
-            }
-            uint64_t Ticks = ASMReadTSC() - StartTick;
-            if (Ticks < (SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage) / 10000))
-                RTPrintf("Warning: Ticks=%RU64 (< %RU64)\n", Ticks, SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage) / 10000);
-        }
-
-        /*
-         * Interrupt forwarding.
-         */
-        CPUMHyperSetCtxCore(pVCpu, NULL);
-        CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
-        CPUMPushHyper(pVCpu, 0);
-        CPUMPushHyper(pVCpu, VMMGC_DO_TESTCASE_HYPER_INTERRUPT);
-        CPUMPushHyper(pVCpu, pVM->pVMRC);
-        CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
-        CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
-        CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
-        Log(("trampoline=%x\n", pVM->vmm.s.pfnCallTrampolineRC));
-
-        /*
-         * Switch and do da thing.
-         */
-        RTPrintf("VMM: interrupt forwarding...\n"); RTStrmFlush(g_pStdOut); RTThreadSleep(250);
-        i = 0;
-        uint64_t    tsBegin = RTTimeNanoTS();
-        uint64_t    TickStart = ASMReadTSC();
-        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
-        do
-        {
-            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
-            if (RT_LIKELY(rc == VINF_SUCCESS))
-                rc = pVCpu->vmm.s.iLastGZRc;
-            if (RT_FAILURE(rc))
-            {
-                Log(("VMM: GC returned fatal %Rra in iteration %d\n", rc, i));
-                VMMR3FatalDump(pVM, pVCpu, rc);
-                return rc;
-            }
-            i++;
-            if (!(i % 32))
-                Log(("VMM: iteration %d, esi=%08x edi=%08x ebx=%08x\n",
-                       i, CPUMGetHyperESI(pVCpu), CPUMGetHyperEDI(pVCpu), CPUMGetHyperEBX(pVCpu)));
-        } while (rc == VINF_EM_RAW_INTERRUPT_HYPER);
-        uint64_t    TickEnd = ASMReadTSC();
-        uint64_t    tsEnd = RTTimeNanoTS();
-
-        uint64_t    Elapsed = tsEnd - tsBegin;
-        uint64_t    PerIteration = Elapsed / (uint64_t)i;
-        uint64_t    cTicksElapsed = TickEnd - TickStart;
-        uint64_t    cTicksPerIteration = cTicksElapsed / (uint64_t)i;
-
-        RTPrintf("VMM: %8d interrupts in %11llu ns (%11llu ticks),  %10llu ns/iteration (%11llu ticks)\n",
-                 i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration);
-        Log(("VMM: %8d interrupts in %11llu ns (%11llu ticks),  %10llu ns/iteration (%11llu ticks)\n",
-             i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration));
-
-        /*
-         * These forced actions are not necessary for the test and trigger breakpoints too.
-         */
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-
-        /*
-         * Profile switching.
-         */
-        RTPrintf("VMM: profiling switcher...\n");
-        Log(("VMM: profiling switcher...\n"));
-        uint64_t TickMin = ~0;
-        tsBegin = RTTimeNanoTS();
-        TickStart = ASMReadTSC();
-        Assert(CPUMGetHyperCR3(pVCpu) && CPUMGetHyperCR3(pVCpu) == PGMGetHyperCR3(pVCpu));
-        for (i = 0; i < 1000000; i++)
-        {
-            CPUMHyperSetCtxCore(pVCpu, NULL);
-            CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
-            CPUMPushHyper(pVCpu, 0);
-            CPUMPushHyper(pVCpu, VMMGC_DO_TESTCASE_NOP);
-            CPUMPushHyper(pVCpu, pVM->pVMRC);
-            CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
-            CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
-            CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
-
-            uint64_t TickThisStart = ASMReadTSC();
-            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_RAW_RUN, 0);
-            if (RT_LIKELY(rc == VINF_SUCCESS))
-                rc = pVCpu->vmm.s.iLastGZRc;
-            uint64_t TickThisElapsed = ASMReadTSC() - TickThisStart;
-            if (RT_FAILURE(rc))
-            {
-                Log(("VMM: GC returned fatal %Rra in iteration %d\n", rc, i));
-                VMMR3FatalDump(pVM, pVCpu, rc);
-                return rc;
-            }
-            if (TickThisElapsed < TickMin)
-                TickMin = TickThisElapsed;
-        }
-        TickEnd = ASMReadTSC();
-        tsEnd = RTTimeNanoTS();
-
-        Elapsed = tsEnd - tsBegin;
-        PerIteration = Elapsed / (uint64_t)i;
-        cTicksElapsed = TickEnd - TickStart;
-        cTicksPerIteration = cTicksElapsed / (uint64_t)i;
-
-        RTPrintf("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
-                 i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin);
-        Log(("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
-             i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin));
-
-        rc = VINF_SUCCESS;
-    }
-    else
-        AssertMsgFailed(("Failed to resolved VMMGC.gc::VMMGCEntry(), rc=%Rrc\n", rc));
-#endif
-    return rc;
-}
-
-#define SYNC_SEL(pHyperCtx, reg)                                                        \
-        if (pHyperCtx->reg)                                                             \
-        {                                                                               \
-            DBGFSELINFO selInfo;                                                        \
-            int rc2 = SELMR3GetShadowSelectorInfo(pVM, pHyperCtx->reg, &selInfo);       \
-            AssertRC(rc2);                                                              \
-                                                                                        \
-            pHyperCtx->reg##Hid.u64Base              = selInfo.GCPtrBase;               \
-            pHyperCtx->reg##Hid.u32Limit             = selInfo.cbLimit;                 \
-            pHyperCtx->reg##Hid.Attr.n.u1Present     = selInfo.u.Raw.Gen.u1Present;     \
-            pHyperCtx->reg##Hid.Attr.n.u1DefBig      = selInfo.u.Raw.Gen.u1DefBig;      \
-            pHyperCtx->reg##Hid.Attr.n.u1Granularity = selInfo.u.Raw.Gen.u1Granularity; \
-            pHyperCtx->reg##Hid.Attr.n.u4Type        = selInfo.u.Raw.Gen.u4Type;        \
-            pHyperCtx->reg##Hid.Attr.n.u2Dpl         = selInfo.u.Raw.Gen.u2Dpl;         \
-            pHyperCtx->reg##Hid.Attr.n.u1DescType    = selInfo.u.Raw.Gen.u1DescType;    \
-            pHyperCtx->reg##Hid.Attr.n.u1Long        = selInfo.u.Raw.Gen.u1Long;        \
-        }
-
-/* execute the switch. */
-VMMR3DECL(int) VMMDoHwAccmTest(PVM pVM)
-{
-    uint32_t i;
-    int      rc;
-    PCPUMCTX pHyperCtx, pGuestCtx;
-    RTGCPHYS CR3Phys = 0x0; /* fake address */
-    PVMCPU   pVCpu = &pVM->aCpus[0];
-
-    if (!HWACCMR3IsAllowed(pVM))
-    {
-        RTPrintf("VMM: Hardware accelerated test not available!\n");
-        return VERR_ACCESS_DENIED;
-    }
-
-    /*
-     * These forced actions are not necessary for the test and trigger breakpoints too.
-     */
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TRPM_SYNC_IDT);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_SELM_SYNC_TSS);
-
-    /* Enable mapping of the hypervisor into the shadow page table. */
-    uint32_t cb;
-    rc = PGMR3MappingsSize(pVM, &cb);
-    AssertRCReturn(rc, rc);
-
-    /* Pretend the mappings are now fixed; to force a refresh of the reserved PDEs. */
-    rc = PGMR3MappingsFix(pVM, MM_HYPER_AREA_ADDRESS, cb);
-    AssertRCReturn(rc, rc);
-
-    CPUMQueryHyperCtxPtr(pVCpu, &pHyperCtx);
-
-    pHyperCtx->cr0 = X86_CR0_PE | X86_CR0_WP | X86_CR0_PG | X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
-    pHyperCtx->cr4 = X86_CR4_PGE | X86_CR4_OSFSXR | X86_CR4_OSXMMEEXCPT;
-    PGMChangeMode(pVCpu, pHyperCtx->cr0, pHyperCtx->cr4, pHyperCtx->msrEFER);
-    PGMSyncCR3(pVCpu, pHyperCtx->cr0, CR3Phys, pHyperCtx->cr4, true);
-
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
-    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TIMER);
-    VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
-    VM_FF_CLEAR(pVM, VM_FF_REQUEST);
-
-    /*
-     * Setup stack for calling VMMGCEntry().
-     */
-    RTRCPTR RCPtrEP;
-    rc = PDMR3LdrGetSymbolRC(pVM, VMMGC_MAIN_MODULE_NAME, "VMMGCEntry", &RCPtrEP);
-    if (RT_SUCCESS(rc))
-    {
-        RTPrintf("VMM: VMMGCEntry=%RRv\n", RCPtrEP);
-
-        CPUMQueryHyperCtxPtr(pVCpu, &pHyperCtx);
-
-        /* Fill in hidden selector registers for the hypervisor state. */
-        SYNC_SEL(pHyperCtx, cs);
-        SYNC_SEL(pHyperCtx, ds);
-        SYNC_SEL(pHyperCtx, es);
-        SYNC_SEL(pHyperCtx, fs);
-        SYNC_SEL(pHyperCtx, gs);
-        SYNC_SEL(pHyperCtx, ss);
-        SYNC_SEL(pHyperCtx, tr);
-
-        /*
-         * Profile switching.
-         */
-        RTPrintf("VMM: profiling switcher...\n");
-        Log(("VMM: profiling switcher...\n"));
-        uint64_t TickMin = ~0;
-        uint64_t tsBegin = RTTimeNanoTS();
-        uint64_t TickStart = ASMReadTSC();
-        for (i = 0; i < 1000000; i++)
-        {
-            CPUMHyperSetCtxCore(pVCpu, NULL);
-
-            CPUMSetHyperESP(pVCpu, pVCpu->vmm.s.pbEMTStackBottomRC); /* Clear the stack. */
-            CPUMPushHyper(pVCpu, 0);
-            CPUMPushHyper(pVCpu, VMMGC_DO_TESTCASE_HWACCM_NOP);
-            CPUMPushHyper(pVCpu, pVM->pVMRC);
-            CPUMPushHyper(pVCpu, 3 * sizeof(RTRCPTR));    /* stack frame size */
-            CPUMPushHyper(pVCpu, RCPtrEP);                /* what to call */
-            CPUMSetHyperEIP(pVCpu, pVM->vmm.s.pfnCallTrampolineRC);
-
-            CPUMQueryHyperCtxPtr(pVCpu, &pHyperCtx);
-            pGuestCtx = CPUMQueryGuestCtxPtr(pVCpu);
-
-            /* Copy the hypervisor context to make sure we have a valid guest context. */
-            *pGuestCtx = *pHyperCtx;
-            pGuestCtx->cr3 = CR3Phys;
-
-            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
-            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TIMER);
-            VM_FF_CLEAR(pVM, VM_FF_TM_VIRTUAL_SYNC);
-
-            uint64_t TickThisStart = ASMReadTSC();
-            rc = SUPR3CallVMMR0Fast(pVM->pVMR0, VMMR0_DO_HWACC_RUN, 0);
-            uint64_t TickThisElapsed = ASMReadTSC() - TickThisStart;
-            if (RT_FAILURE(rc))
-            {
-                Log(("VMM: R0 returned fatal %Rrc in iteration %d\n", rc, i));
-                VMMR3FatalDump(pVM, pVCpu, rc);
-                return rc;
-            }
-            if (TickThisElapsed < TickMin)
-                TickMin = TickThisElapsed;
-        }
-        uint64_t TickEnd = ASMReadTSC();
-        uint64_t tsEnd = RTTimeNanoTS();
-
-        uint64_t Elapsed = tsEnd - tsBegin;
-        uint64_t PerIteration = Elapsed / (uint64_t)i;
-        uint64_t cTicksElapsed = TickEnd - TickStart;
-        uint64_t cTicksPerIteration = cTicksElapsed / (uint64_t)i;
-
-        RTPrintf("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
-                 i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin);
-        Log(("VMM: %8d cycles     in %11llu ns (%11lld ticks),  %10llu ns/iteration (%11lld ticks)  Min %11lld ticks\n",
-             i, Elapsed, cTicksElapsed, PerIteration, cTicksPerIteration, TickMin));
-
-        rc = VINF_SUCCESS;
-    }
-    else
-        AssertMsgFailed(("Failed to resolved VMMGC.gc::VMMGCEntry(), rc=%Rrc\n", rc));
-
-    return rc;
-}
-
Index: unk/src/VBox/VMM/VMReq.cpp
===================================================================
--- /trunk/src/VBox/VMM/VMReq.cpp	(revision 35332)
+++ 	(revision )
@@ -1,1273 +1,0 @@
-/* $Id$ */
-/** @file
- * VM - Virtual Machine
- */
-
-/*
- * Copyright (C) 2006-2007 Oracle Corporation
- *
- * This file is part of VirtualBox Open Source Edition (OSE), as
- * available from http://www.virtualbox.org. This file is free software;
- * you can redistribute it and/or modify it under the terms of the GNU
- * General Public License (GPL) as published by the Free Software
- * Foundation, in version 2 as it comes in the "COPYING" file of the
- * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
- * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
- */
-
-
-/*******************************************************************************
-*   Header Files                                                               *
-*******************************************************************************/
-#define LOG_GROUP LOG_GROUP_VM
-#include <VBox/mm.h>
-#include <VBox/vmm.h>
-#include "VMInternal.h"
-#include <VBox/vm.h>
-#include <VBox/uvm.h>
-
-#include <VBox/err.h>
-#include <VBox/param.h>
-#include <VBox/log.h>
-#include <iprt/assert.h>
-#include <iprt/asm.h>
-#include <iprt/string.h>
-#include <iprt/time.h>
-#include <iprt/semaphore.h>
-#include <iprt/thread.h>
-
-
-/*******************************************************************************
-*   Internal Functions                                                         *
-*******************************************************************************/
-static int  vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq);
-
-
-/**
- * Allocate and queue a call request.
- *
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completion. In the other case
- * use RT_INDEFINITE_WAIT.
- * The returned request packet must be freed using VMR3ReqFree().
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pVM             The VM handle.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   ppReq           Where to store the pointer to the request.
- *                          This will be NULL or a valid request pointer not matter what happens.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- * @param   fFlags          A combination of the VMREQFLAGS values.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCall(PVM pVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
-                           PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns an VBox status code,
- * (2) that you want it's return code on success, and (3) that you wish to wait
- * for ever for it to return.
- *
- * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
- *          its status code is return.  Otherwise, the status of pfnFunction is
- *          returned.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    PVMREQ pReq;
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    if (RT_SUCCESS(rc))
-        rc = pReq->iStatus;
-    VMR3ReqFree(pReq);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns an VBox status code,
- * (2) that you want it's return code on success, and (3) that you wish to wait
- * for ever for it to return.
- *
- * @returns VBox status code.  In the unlikely event that VMR3ReqCallVU fails,
- *          its status code is return.  Otherwise, the status of pfnFunction is
- *          returned.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    PVMREQ pReq;
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VBOX_STATUS,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    if (RT_SUCCESS(rc))
-        rc = pReq->iStatus;
-    VMR3ReqFree(pReq);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns an VBox status code
- * and that you do not wish to wait for it to complete.
- *
- * @returns VBox status code returned by VMR3ReqCallVU.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallNoWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, NULL, 0, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_NO_WAIT,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns an VBox status code
- * and that you do not wish to wait for it to complete.
- *
- * @returns VBox status code returned by VMR3ReqCallVU.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallNoWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pUVM, idDstCpu, NULL, 0, VMREQFLAGS_VBOX_STATUS | VMREQFLAGS_NO_WAIT,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns void, and (2) that
- * you wish to wait for ever for it to return.
- *
- * @returns VBox status code of VMR3ReqCallVU.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallVoidWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    PVMREQ pReq;
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    VMR3ReqFree(pReq);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns void, and (2) that
- * you wish to wait for ever for it to return.
- *
- * @returns VBox status code of VMR3ReqCallVU.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallVoidWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    PVMREQ pReq;
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    VMR3ReqFree(pReq);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns void, and (2) that
- * you do not wish to wait for it to complete.
- *
- * @returns VBox status code of VMR3ReqCallVU.
- *
- * @param   pVM             Pointer to the shared VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallVoidNoWait(PVM pVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    PVMREQ pReq;
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pVM->pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID | VMREQFLAGS_NO_WAIT,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    VMR3ReqFree(pReq);
-    return rc;
-}
-
-
-/**
- * Convenience wrapper for VMR3ReqCallU.
- *
- * This assumes (1) you're calling a function that returns void, and (2) that
- * you do not wish to wait for it to complete.
- *
- * @returns VBox status code of VMR3ReqCallVU.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallVoidNoWaitU(PUVM pUVM, VMCPUID idDstCpu, PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    PVMREQ pReq;
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pUVM, idDstCpu, &pReq, RT_INDEFINITE_WAIT, VMREQFLAGS_VOID | VMREQFLAGS_NO_WAIT,
-                           pfnFunction, cArgs, va);
-    va_end(va);
-    VMR3ReqFree(pReq);
-    return rc;
-}
-
-
-/**
- * Allocate and queue a call request to a void function.
- *
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completion. In the other case
- * use RT_INDEFINITE_WAIT.
- * The returned request packet must be freed using VMR3ReqFree().
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   ppReq           Where to store the pointer to the request.
- *                          This will be NULL or a valid request pointer not matter what happens, unless fFlags
- *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- * @param   fFlags          A combination of the VMREQFLAGS values.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   cArgs           Number of arguments following in the ellipsis.
- * @param   ...             Function arguments.
- *
- * @remarks See remarks on VMR3ReqCallVU.
- */
-VMMR3DECL(int) VMR3ReqCallU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
-                            PFNRT pfnFunction, unsigned cArgs, ...)
-{
-    va_list va;
-    va_start(va, cArgs);
-    int rc = VMR3ReqCallVU(pUVM, idDstCpu, ppReq, cMillies, fFlags, pfnFunction, cArgs, va);
-    va_end(va);
-    return rc;
-}
-
-
-/**
- * Allocate and queue a call request.
- *
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completion. In the other case
- * use RT_INDEFINITE_WAIT.
- * The returned request packet must be freed using VMR3ReqFree().
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- * @param   ppReq           Where to store the pointer to the request.
- *                          This will be NULL or a valid request pointer not matter what happens, unless fFlags
- *                          contains VMREQFLAGS_NO_WAIT when it will be optional and always NULL.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- * @param   pfnFunction     Pointer to the function to call.
- * @param   fFlags          A combination of the VMREQFLAGS values.
- * @param   cArgs           Number of arguments following in the ellipsis.
- *                          Stuff which differs in size from uintptr_t is gonna make trouble, so don't try!
- * @param   Args            Argument vector.
- *
- * @remarks Caveats:
- *              - Do not pass anything which is larger than an uintptr_t.
- *              - 64-bit integers are larger than uintptr_t on 32-bit hosts.
- *                Pass integers > 32-bit by reference (pointers).
- *              - Don't use NULL since it should be the integer 0 in C++ and may
- *                therefore end up with garbage in the bits 63:32 on 64-bit
- *                hosts because 'int' is 32-bit.
- *                Use (void *)NULL or (uintptr_t)0 instead of NULL.
- */
-VMMR3DECL(int) VMR3ReqCallVU(PUVM pUVM, VMCPUID idDstCpu, PVMREQ *ppReq, RTMSINTERVAL cMillies, uint32_t fFlags,
-                             PFNRT pfnFunction, unsigned cArgs, va_list Args)
-{
-    LogFlow(("VMR3ReqCallV: idDstCpu=%u cMillies=%d fFlags=%#x pfnFunction=%p cArgs=%d\n", idDstCpu, cMillies, fFlags, pfnFunction, cArgs));
-
-    /*
-     * Validate input.
-     */
-    AssertPtrReturn(pfnFunction, VERR_INVALID_POINTER);
-    AssertPtrReturn(pUVM, VERR_INVALID_POINTER);
-    AssertReturn(!(fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE)), VERR_INVALID_PARAMETER);
-    if (!(fFlags & VMREQFLAGS_NO_WAIT) || ppReq)
-    {
-        AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
-        *ppReq = NULL;
-    }
-    PVMREQ pReq = NULL;
-    AssertMsgReturn(cArgs * sizeof(uintptr_t) <= sizeof(pReq->u.Internal.aArgs),
-                    ("cArg=%d\n", cArgs),
-                    VERR_TOO_MUCH_DATA);
-
-    /*
-     * Allocate request
-     */
-    int rc = VMR3ReqAllocU(pUVM, &pReq, VMREQTYPE_INTERNAL, idDstCpu);
-    if (RT_FAILURE(rc))
-        return rc;
-
-    /*
-     * Initialize the request data.
-     */
-    pReq->fFlags         = fFlags;
-    pReq->u.Internal.pfn = pfnFunction;
-    pReq->u.Internal.cArgs = cArgs;
-    for (unsigned iArg = 0; iArg < cArgs; iArg++)
-        pReq->u.Internal.aArgs[iArg] = va_arg(Args, uintptr_t);
-
-    /*
-     * Queue the request and return.
-     */
-    rc = VMR3ReqQueue(pReq, cMillies);
-    if (    RT_FAILURE(rc)
-        && rc != VERR_TIMEOUT)
-    {
-        VMR3ReqFree(pReq);
-        pReq = NULL;
-    }
-    if (!(fFlags & VMREQFLAGS_NO_WAIT))
-    {
-        *ppReq = pReq;
-        LogFlow(("VMR3ReqCallV: returns %Rrc *ppReq=%p\n", rc, pReq));
-    }
-    else
-        LogFlow(("VMR3ReqCallV: returns %Rrc\n", rc));
-    Assert(rc != VERR_INTERRUPTED);
-    return rc;
-}
-
-
-/**
- * Joins the list pList with whatever is linked up at *pHead.
- */
-static void vmr3ReqJoinFreeSub(volatile PVMREQ *ppHead, PVMREQ pList)
-{
-    for (unsigned cIterations = 0;; cIterations++)
-    {
-        PVMREQ pHead = ASMAtomicXchgPtrT(ppHead, pList, PVMREQ);
-        if (!pHead)
-            return;
-        PVMREQ pTail = pHead;
-        while (pTail->pNext)
-            pTail = pTail->pNext;
-        ASMAtomicWritePtr(&pTail->pNext, pList);
-        ASMCompilerBarrier();
-        if (ASMAtomicCmpXchgPtr(ppHead, pHead, pList))
-            return;
-        ASMAtomicWriteNullPtr(&pTail->pNext);
-        ASMCompilerBarrier();
-        if (ASMAtomicCmpXchgPtr(ppHead, pHead, NULL))
-            return;
-        pList = pHead;
-        Assert(cIterations != 32);
-        Assert(cIterations != 64);
-    }
-}
-
-
-/**
- * Joins the list pList with whatever is linked up at *pHead.
- */
-static void vmr3ReqJoinFree(PVMINTUSERPERVM pVMInt, PVMREQ pList)
-{
-    /*
-     * Split the list if it's too long.
-     */
-    unsigned cReqs = 1;
-    PVMREQ pTail = pList;
-    while (pTail->pNext)
-    {
-        if (cReqs++ > 25)
-        {
-            const uint32_t i = pVMInt->iReqFree;
-            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
-
-            pTail->pNext = NULL;
-            vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(i + 2 + (i == pVMInt->iReqFree)) % RT_ELEMENTS(pVMInt->apReqFree)], pTail->pNext);
-            return;
-        }
-        pTail = pTail->pNext;
-    }
-    vmr3ReqJoinFreeSub(&pVMInt->apReqFree[(pVMInt->iReqFree + 2) % RT_ELEMENTS(pVMInt->apReqFree)], pList);
-}
-
-
-/**
- * Allocates a request packet.
- *
- * The caller allocates a request packet, fills in the request data
- * union and queues the request.
- *
- * @returns VBox status code.
- *
- * @param   pVM             VM handle.
- * @param   ppReq           Where to store the pointer to the allocated packet.
- * @param   enmType         Package type.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- */
-VMMR3DECL(int) VMR3ReqAlloc(PVM pVM, PVMREQ *ppReq, VMREQTYPE enmType, VMCPUID idDstCpu)
-{
-    return VMR3ReqAllocU(pVM->pUVM, ppReq, enmType, idDstCpu);
-}
-
-
-/**
- * Allocates a request packet.
- *
- * The caller allocates a request packet, fills in the request data
- * union and queues the request.
- *
- * @returns VBox status code.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   ppReq           Where to store the pointer to the allocated packet.
- * @param   enmType         Package type.
- * @param   idDstCpu        The destination CPU(s). Either a specific CPU ID or
- *                          one of the following special values:
- *                              VMCPUID_ANY, VMCPUID_ANY_QUEUE, VMCPUID_ALL or VMCPUID_ALL_REVERSE.
- */
-VMMR3DECL(int) VMR3ReqAllocU(PUVM pUVM, PVMREQ *ppReq, VMREQTYPE enmType, VMCPUID idDstCpu)
-{
-    /*
-     * Validate input.
-     */
-    AssertMsgReturn(enmType > VMREQTYPE_INVALID && enmType < VMREQTYPE_MAX,
-                    ("Invalid package type %d valid range %d-%d inclusively.\n",
-                     enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
-                    VERR_VM_REQUEST_INVALID_TYPE);
-    AssertPtrReturn(ppReq, VERR_INVALID_POINTER);
-    AssertMsgReturn(    idDstCpu == VMCPUID_ANY
-                    ||  idDstCpu == VMCPUID_ANY_QUEUE
-                    ||  idDstCpu < pUVM->cCpus
-                    ||  idDstCpu == VMCPUID_ALL
-                    ||  idDstCpu == VMCPUID_ALL_REVERSE,
-                    ("Invalid destination %u (max=%u)\n", idDstCpu, pUVM->cCpus), VERR_INVALID_PARAMETER);
-
-    /*
-     * Try get a recycled packet.
-     * While this could all be solved with a single list with a lock, it's a sport
-     * of mine to avoid locks.
-     */
-    int cTries = RT_ELEMENTS(pUVM->vm.s.apReqFree) * 2;
-    while (--cTries >= 0)
-    {
-        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
-#if 0 /* sad, but this won't work safely because the reading of pReq->pNext. */
-        PVMREQ pNext = NULL;
-        PVMREQ pReq = *ppHead;
-        if (    pReq
-            &&  !ASMAtomicCmpXchgPtr(ppHead, (pNext = pReq->pNext), pReq)
-            &&  (pReq = *ppHead)
-            &&  !ASMAtomicCmpXchgPtr(ppHead, (pNext = pReq->pNext), pReq))
-            pReq = NULL;
-        if (pReq)
-        {
-            Assert(pReq->pNext == pNext); NOREF(pReq);
-#else
-        PVMREQ pReq = ASMAtomicXchgPtrT(ppHead, NULL, PVMREQ);
-        if (pReq)
-        {
-            PVMREQ pNext = pReq->pNext;
-            if (    pNext
-                &&  !ASMAtomicCmpXchgPtr(ppHead, pNext, NULL))
-            {
-                STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRaces);
-                vmr3ReqJoinFree(&pUVM->vm.s, pReq->pNext);
-            }
-#endif
-            ASMAtomicDecU32(&pUVM->vm.s.cReqFree);
-
-            /*
-             * Make sure the event sem is not signaled.
-             */
-            if (!pReq->fEventSemClear)
-            {
-                int rc = RTSemEventWait(pReq->EventSem, 0);
-                if (rc != VINF_SUCCESS && rc != VERR_TIMEOUT)
-                {
-                    /*
-                     * This shall not happen, but if it does we'll just destroy
-                     * the semaphore and create a new one.
-                     */
-                    AssertMsgFailed(("rc=%Rrc from RTSemEventWait(%#x).\n", rc, pReq->EventSem));
-                    RTSemEventDestroy(pReq->EventSem);
-                    rc = RTSemEventCreate(&pReq->EventSem);
-                    AssertRC(rc);
-                    if (RT_FAILURE(rc))
-                        return rc;
-#if 0 ///@todo @bugref{4725} - def RT_LOCK_STRICT
-                    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
-                        RTSemEventAddSignaller(pReq->EventSem, pUVM->aCpus[idCpu].vm.s.ThreadEMT);
-#endif
-                }
-                pReq->fEventSemClear = true;
-            }
-            else
-                Assert(RTSemEventWait(pReq->EventSem, 0) == VERR_TIMEOUT);
-
-            /*
-             * Initialize the packet and return it.
-             */
-            Assert(pReq->enmType == VMREQTYPE_INVALID);
-            Assert(pReq->enmState == VMREQSTATE_FREE);
-            Assert(pReq->pUVM == pUVM);
-            ASMAtomicXchgSize(&pReq->pNext, NULL);
-            pReq->enmState = VMREQSTATE_ALLOCATED;
-            pReq->iStatus  = VERR_VM_REQUEST_STATUS_STILL_PENDING;
-            pReq->fFlags   = VMREQFLAGS_VBOX_STATUS;
-            pReq->enmType  = enmType;
-            pReq->idDstCpu = idDstCpu;
-
-            *ppReq = pReq;
-            STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocRecycled);
-            LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p recycled\n", pReq));
-            return VINF_SUCCESS;
-        }
-    }
-
-    /*
-     * Ok allocate one.
-     */
-    PVMREQ pReq = (PVMREQ)MMR3HeapAllocU(pUVM, MM_TAG_VM_REQ, sizeof(*pReq));
-    if (!pReq)
-        return VERR_NO_MEMORY;
-
-    /*
-     * Create the semaphore.
-     */
-    int rc = RTSemEventCreate(&pReq->EventSem);
-    AssertRC(rc);
-    if (RT_FAILURE(rc))
-    {
-        MMR3HeapFree(pReq);
-        return rc;
-    }
-#if 0 ///@todo @bugref{4725} - def RT_LOCK_STRICT
-    for (VMCPUID idCpu = 0; idCpu < pUVM->cCpus; idCpu++)
-        RTSemEventAddSignaller(pReq->EventSem, pUVM->aCpus[idCpu].vm.s.ThreadEMT);
-#endif
-
-    /*
-     * Initialize the packet and return it.
-     */
-    pReq->pNext    = NULL;
-    pReq->pUVM     = pUVM;
-    pReq->enmState = VMREQSTATE_ALLOCATED;
-    pReq->iStatus  = VERR_VM_REQUEST_STATUS_STILL_PENDING;
-    pReq->fEventSemClear = true;
-    pReq->fFlags   = VMREQFLAGS_VBOX_STATUS;
-    pReq->enmType  = enmType;
-    pReq->idDstCpu = idDstCpu;
-
-    *ppReq = pReq;
-    STAM_COUNTER_INC(&pUVM->vm.s.StatReqAllocNew);
-    LogFlow(("VMR3ReqAlloc: returns VINF_SUCCESS *ppReq=%p new\n", pReq));
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Free a request packet.
- *
- * @returns VBox status code.
- *
- * @param   pReq            Package to free.
- * @remark  The request packet must be in allocated or completed state!
- */
-VMMR3DECL(int) VMR3ReqFree(PVMREQ pReq)
-{
-    /*
-     * Ignore NULL (all free functions should do this imho).
-     */
-    if (!pReq)
-        return VINF_SUCCESS;
-
-    /*
-     * Check packet state.
-     */
-    switch (pReq->enmState)
-    {
-        case VMREQSTATE_ALLOCATED:
-        case VMREQSTATE_COMPLETED:
-            break;
-        default:
-            AssertMsgFailed(("Invalid state %d!\n", pReq->enmState));
-            return VERR_VM_REQUEST_STATE;
-    }
-
-    /*
-     * Make it a free packet and put it into one of the free packet lists.
-     */
-    pReq->enmState = VMREQSTATE_FREE;
-    pReq->iStatus  = VERR_VM_REQUEST_STATUS_FREED;
-    pReq->enmType  = VMREQTYPE_INVALID;
-
-    PUVM pUVM = pReq->pUVM;
-    STAM_COUNTER_INC(&pUVM->vm.s.StatReqFree);
-
-    if (pUVM->vm.s.cReqFree < 128)
-    {
-        ASMAtomicIncU32(&pUVM->vm.s.cReqFree);
-        PVMREQ volatile *ppHead = &pUVM->vm.s.apReqFree[ASMAtomicIncU32(&pUVM->vm.s.iReqFree) % RT_ELEMENTS(pUVM->vm.s.apReqFree)];
-        PVMREQ pNext;
-        do
-        {
-            pNext = ASMAtomicUoReadPtrT(ppHead, PVMREQ);
-            ASMAtomicWritePtr(&pReq->pNext, pNext);
-            ASMCompilerBarrier();
-        } while (!ASMAtomicCmpXchgPtr(ppHead, pReq, pNext));
-    }
-    else
-    {
-        STAM_COUNTER_INC(&pReq->pUVM->vm.s.StatReqFreeOverflow);
-        RTSemEventDestroy(pReq->EventSem);
-        MMR3HeapFree(pReq);
-    }
-    return VINF_SUCCESS;
-}
-
-
-/**
- * Queue a request.
- *
- * The quest must be allocated using VMR3ReqAlloc() and contain
- * all the required data.
- * If it's desired to poll on the completion of the request set cMillies
- * to 0 and use VMR3ReqWait() to check for completion. In the other case
- * use RT_INDEFINITE_WAIT.
- *
- * @returns VBox status code.
- *          Will not return VERR_INTERRUPTED.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pReq            The request to queue.
- * @param   cMillies        Number of milliseconds to wait for the request to
- *                          be completed. Use RT_INDEFINITE_WAIT to only
- *                          wait till it's completed.
- */
-VMMR3DECL(int) VMR3ReqQueue(PVMREQ pReq, RTMSINTERVAL cMillies)
-{
-    LogFlow(("VMR3ReqQueue: pReq=%p cMillies=%d\n", pReq, cMillies));
-    /*
-     * Verify the supplied package.
-     */
-    AssertMsgReturn(pReq->enmState == VMREQSTATE_ALLOCATED, ("%d\n", pReq->enmState), VERR_VM_REQUEST_STATE);
-    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
-                    &&  !pReq->pNext
-                    &&  pReq->EventSem != NIL_RTSEMEVENT,
-                    ("Invalid request package! Anyone cooking their own packages???\n"),
-                    VERR_VM_REQUEST_INVALID_PACKAGE);
-    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
-                    &&  pReq->enmType < VMREQTYPE_MAX,
-                    ("Invalid package type %d valid range %d-%d inclusively. This was verified on alloc too...\n",
-                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
-                    VERR_VM_REQUEST_INVALID_TYPE);
-    Assert(!(pReq->fFlags & ~(VMREQFLAGS_RETURN_MASK | VMREQFLAGS_NO_WAIT | VMREQFLAGS_POKE)));
-
-    /*
-     * Are we the EMT or not?
-     * Also, store pVM (and fFlags) locally since pReq may be invalid after queuing it.
-     */
-    int     rc      = VINF_SUCCESS;
-    PUVM    pUVM    = ((VMREQ volatile *)pReq)->pUVM;                 /* volatile paranoia */
-    PUVMCPU pUVCpu  = (PUVMCPU)RTTlsGet(pUVM->vm.s.idxTLS);
-
-    if (pReq->idDstCpu == VMCPUID_ALL)
-    {
-        /* One-by-one. */
-        Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
-        for (unsigned i = 0; i < pUVM->cCpus; i++)
-        {
-            /* Reinit some members. */
-            pReq->enmState = VMREQSTATE_ALLOCATED;
-            pReq->idDstCpu = i;
-            rc = VMR3ReqQueue(pReq, cMillies);
-            if (RT_FAILURE(rc))
-                break;
-        }
-    }
-    else if (pReq->idDstCpu == VMCPUID_ALL_REVERSE)
-    {
-        /* One-by-one. */
-        Assert(!(pReq->fFlags & VMREQFLAGS_NO_WAIT));
-        for (int i = pUVM->cCpus-1; i >= 0; i--)
-        {
-            /* Reinit some members. */
-            pReq->enmState = VMREQSTATE_ALLOCATED;
-            pReq->idDstCpu = i;
-            rc = VMR3ReqQueue(pReq, cMillies);
-            if (RT_FAILURE(rc))
-                break;
-        }
-    }
-    else if (   pReq->idDstCpu != VMCPUID_ANY   /* for a specific VMCPU? */
-             && pReq->idDstCpu != VMCPUID_ANY_QUEUE
-             && (   !pUVCpu                     /* and it's not the current thread. */
-                 || pUVCpu->idCpu != pReq->idDstCpu))
-    {
-        VMCPUID  idTarget = pReq->idDstCpu;     Assert(idTarget < pUVM->cCpus);
-        PVMCPU   pVCpu = &pUVM->pVM->aCpus[idTarget];
-        unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags;     /* volatile paranoia */
-
-        /* Fetch the right UVMCPU */
-        pUVCpu = &pUVM->aCpus[idTarget];
-
-        /*
-         * Insert it.
-         */
-        pReq->enmState = VMREQSTATE_QUEUED;
-        PVMREQ pNext;
-        do
-        {
-            pNext = ASMAtomicUoReadPtrT(&pUVCpu->vm.s.pReqs, PVMREQ);
-            ASMAtomicWritePtr(&pReq->pNext, pNext);
-            ASMCompilerBarrier();
-        } while (!ASMAtomicCmpXchgPtr(&pUVCpu->vm.s.pReqs, pReq, pNext));
-
-        /*
-         * Notify EMT.
-         */
-        if (pUVM->pVM)
-            VMCPU_FF_SET(pVCpu, VMCPU_FF_REQUEST);
-        VMR3NotifyCpuFFU(pUVCpu, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
-
-        /*
-         * Wait and return.
-         */
-        if (!(fFlags & VMREQFLAGS_NO_WAIT))
-            rc = VMR3ReqWait(pReq, cMillies);
-        LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
-    }
-    else if (   (    pReq->idDstCpu == VMCPUID_ANY
-                 && !pUVCpu /* only EMT threads have a valid pointer stored in the TLS slot. */)
-             || pReq->idDstCpu == VMCPUID_ANY_QUEUE)
-    {
-        unsigned fFlags = ((VMREQ volatile *)pReq)->fFlags;     /* volatile paranoia */
-
-        /* Note: pUVCpu may or may not be NULL in the VMCPUID_ANY_QUEUE case; we don't care. */
-
-        /*
-         * Insert it.
-         */
-        pReq->enmState = VMREQSTATE_QUEUED;
-        PVMREQ pNext;
-        do
-        {
-            pNext = ASMAtomicUoReadPtrT(&pUVM->vm.s.pReqs, PVMREQ);
-            ASMAtomicWritePtr(&pReq->pNext, pNext);
-            ASMCompilerBarrier();
-        } while (!ASMAtomicCmpXchgPtr(&pUVM->vm.s.pReqs, pReq, pNext));
-
-        /*
-         * Notify EMT.
-         */
-        if (pUVM->pVM)
-            VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
-        VMR3NotifyGlobalFFU(pUVM, fFlags & VMREQFLAGS_POKE ? VMNOTIFYFF_FLAGS_POKE : 0);
-
-        /*
-         * Wait and return.
-         */
-        if (!(fFlags & VMREQFLAGS_NO_WAIT))
-            rc = VMR3ReqWait(pReq, cMillies);
-        LogFlow(("VMR3ReqQueue: returns %Rrc\n", rc));
-    }
-    else
-    {
-        Assert(pUVCpu);
-
-        /*
-         * The requester was an EMT, just execute it.
-         */
-        pReq->enmState = VMREQSTATE_QUEUED;
-        rc = vmR3ReqProcessOneU(pUVM, pReq);
-        LogFlow(("VMR3ReqQueue: returns %Rrc (processed)\n", rc));
-    }
-    return rc;
-}
-
-
-/**
- * Wait for a request to be completed.
- *
- * @returns VBox status code.
- * @returns VERR_TIMEOUT if cMillies was reached without the packet being completed.
- *
- * @param   pReq            The request to wait for.
- * @param   cMillies        Number of milliseconds to wait.
- *                          Use RT_INDEFINITE_WAIT to only wait till it's completed.
- */
-VMMR3DECL(int) VMR3ReqWait(PVMREQ pReq, RTMSINTERVAL cMillies)
-{
-    LogFlow(("VMR3ReqWait: pReq=%p cMillies=%d\n", pReq, cMillies));
-
-    /*
-     * Verify the supplied package.
-     */
-    AssertMsgReturn(    pReq->enmState == VMREQSTATE_QUEUED
-                    ||  pReq->enmState == VMREQSTATE_PROCESSING
-                    ||  pReq->enmState == VMREQSTATE_COMPLETED,
-                    ("Invalid state %d\n", pReq->enmState),
-                    VERR_VM_REQUEST_STATE);
-    AssertMsgReturn(    VALID_PTR(pReq->pUVM)
-                    &&  pReq->EventSem != NIL_RTSEMEVENT,
-                    ("Invalid request package! Anyone cooking their own packages???\n"),
-                    VERR_VM_REQUEST_INVALID_PACKAGE);
-    AssertMsgReturn(    pReq->enmType > VMREQTYPE_INVALID
-                    &&  pReq->enmType < VMREQTYPE_MAX,
-                    ("Invalid package type %d valid range %d-%d inclusively. This was verified on alloc too...\n",
-                     pReq->enmType, VMREQTYPE_INVALID + 1, VMREQTYPE_MAX - 1),
-                    VERR_VM_REQUEST_INVALID_TYPE);
-
-    /*
-     * Check for deadlock condition
-     */
-    PUVM pUVM = pReq->pUVM;
-    NOREF(pUVM);
-
-    /*
-     * Wait on the package.
-     */
-    int rc;
-    if (cMillies != RT_INDEFINITE_WAIT)
-        rc = RTSemEventWait(pReq->EventSem, cMillies);
-    else
-    {
-        do
-        {
-            rc = RTSemEventWait(pReq->EventSem, RT_INDEFINITE_WAIT);
-            Assert(rc != VERR_TIMEOUT);
-        } while (   pReq->enmState != VMREQSTATE_COMPLETED
-                 && pReq->enmState != VMREQSTATE_INVALID);
-    }
-    if (RT_SUCCESS(rc))
-        ASMAtomicXchgSize(&pReq->fEventSemClear, true);
-    if (pReq->enmState == VMREQSTATE_COMPLETED)
-        rc = VINF_SUCCESS;
-    LogFlow(("VMR3ReqWait: returns %Rrc\n", rc));
-    Assert(rc != VERR_INTERRUPTED);
-    return rc;
-}
-
-
-/**
- * VMR3ReqProcessU helper that handles cases where there are more than one
- * pending request.
- *
- * @returns The oldest request.
- * @param   pUVM                Pointer to the user mode VM structure
- * @param   idDstCpu            VMCPUID_ANY or virtual CPU ID.
- * @param   pReqList            The list of requests.
- * @param   ppReqs              Pointer to the list head.
- */
-static PVMREQ vmR3ReqProcessUTooManyHelper(PUVM pUVM, VMCPUID idDstCpu, PVMREQ pReqList, PVMREQ volatile *ppReqs)
-{
-    STAM_COUNTER_INC(&pUVM->vm.s.StatReqMoreThan1);
-    /* Chop off the last one (pReq). */
-    PVMREQ pPrev;
-    PVMREQ pReqRet = pReqList;
-    do
-    {
-        pPrev = pReqRet;
-        pReqRet = pReqRet->pNext;
-    } while (pReqRet->pNext);
-    ASMAtomicWriteNullPtr(&pPrev->pNext);
-
-    /* Push the others back onto the list (end of it). */
-    Log2(("VMR3ReqProcess: Pushing back %p %p...\n", pReqList, pReqList->pNext));
-    if (RT_UNLIKELY(!ASMAtomicCmpXchgPtr(ppReqs, pReqList, NULL)))
-    {
-        STAM_COUNTER_INC(&pUVM->vm.s.StatReqPushBackRaces);
-        do
-        {
-            ASMNopPause();
-            PVMREQ pReqList2 = ASMAtomicXchgPtrT(ppReqs, NULL, PVMREQ);
-            if (pReqList2)
-            {
-                PVMREQ pLast = pReqList2;
-                while (pLast->pNext)
-                    pLast = pLast->pNext;
-                ASMAtomicWritePtr(&pLast->pNext, pReqList);
-                pReqList = pReqList2;
-            }
-        } while (!ASMAtomicCmpXchgPtr(ppReqs, pReqList, NULL));
-    }
-
-    if (RT_LIKELY(pUVM->pVM))
-    {
-        if (idDstCpu == VMCPUID_ANY)
-            VM_FF_SET(pUVM->pVM, VM_FF_REQUEST);
-        else
-            VMCPU_FF_SET(&pUVM->pVM->aCpus[idDstCpu], VMCPU_FF_REQUEST);
-    }
-
-    return pReqRet;
-}
-
-
-/**
- * Process pending request(s).
- *
- * This function is called from a forced action handler in the EMT
- * or from one of the EMT loops.
- *
- * @returns VBox status code.
- *
- * @param   pUVM            Pointer to the user mode VM structure.
- * @param   idDstCpu        Pass VMCPUID_ANY to process the common request queue
- *                          and the CPU ID for a CPU specific one. In the latter
- *                          case the calling thread must be the EMT of that CPU.
- *
- * @note    SMP safe (multiple EMTs trying to satisfy VM_FF_REQUESTs).
- *
- * @remarks This was made reentrant for
- */
-VMMR3DECL(int) VMR3ReqProcessU(PUVM pUVM, VMCPUID idDstCpu)
-{
-    LogFlow(("VMR3ReqProcessU: (enmVMState=%d) idDstCpu=%d\n", pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING, idDstCpu));
-
-    /*
-     * Process loop.
-     *
-     * We do not repeat the outer loop if we've got an informational status code
-     * since that code needs processing by our caller.
-     */
-    int rc = VINF_SUCCESS;
-    while (rc <= VINF_SUCCESS)
-    {
-        /*
-         * Get the pending requests.
-         * If there are more than one request, unlink the oldest and put the
-         * rest back so that we're reentrant.
-         */
-        PVMREQ volatile *ppReqs;
-        if (idDstCpu == VMCPUID_ANY)
-        {
-            ppReqs = &pUVM->vm.s.pReqs;
-            if (RT_LIKELY(pUVM->pVM))
-                VM_FF_CLEAR(pUVM->pVM, VM_FF_REQUEST);
-        }
-        else
-        {
-            Assert(idDstCpu < pUVM->cCpus);
-            Assert(pUVM->aCpus[idDstCpu].vm.s.NativeThreadEMT == RTThreadNativeSelf());
-            ppReqs = &pUVM->aCpus[idDstCpu].vm.s.pReqs;
-            if (RT_LIKELY(pUVM->pVM))
-                VMCPU_FF_CLEAR(&pUVM->pVM->aCpus[idDstCpu], VMCPU_FF_REQUEST);
-        }
-
-        PVMREQ pReq = ASMAtomicXchgPtrT(ppReqs, NULL, PVMREQ);
-        if (!pReq)
-            break;
-        if (RT_UNLIKELY(pReq->pNext))
-            pReq = vmR3ReqProcessUTooManyHelper(pUVM, idDstCpu, pReq, ppReqs);
-
-        /*
-         * Process the request.
-         * Note! The status code handling here extremely important and yet very
-         *       fragile.
-         */
-        STAM_COUNTER_INC(&pUVM->vm.s.StatReqProcessed);
-        int rc2 = vmR3ReqProcessOneU(pUVM, pReq);
-        if (    rc2 >= VINF_EM_FIRST
-            &&  rc2 <= VINF_EM_LAST
-            &&  (   rc == VINF_SUCCESS
-                 || rc2 < rc) )
-            rc = rc2;
-        /** @todo may have to abort processing to propagate EM scheduling status codes
-         *        up to the caller... See the ugly hacks after VMMR3EmtRendezvousFF
-         *        and VMR3ReqProcessU in EM.cpp. */
-    }
-
-    LogFlow(("VMR3ReqProcess: returns %Rrc (enmVMState=%d)\n", rc, pUVM->pVM ? pUVM->pVM->enmVMState : VMSTATE_CREATING));
-    return rc;
-}
-
-
-/**
- * Process one request.
- *
- * @returns VBox status code.
- *
- * @param   pVM         VM handle.
- * @param   pReq        Request packet to process.
- */
-static int  vmR3ReqProcessOneU(PUVM pUVM, PVMREQ pReq)
-{
-    LogFlow(("vmR3ReqProcessOneU: pReq=%p type=%d fFlags=%#x\n", pReq, pReq->enmType, pReq->fFlags));
-
-    /*
-     * Process the request.
-     */
-    Assert(pReq->enmState == VMREQSTATE_QUEUED);
-    pReq->enmState = VMREQSTATE_PROCESSING;
-    int     rcRet = VINF_SUCCESS;           /* the return code of this function. */
-    int     rcReq = VERR_NOT_IMPLEMENTED;   /* the request status. */
-    switch (pReq->enmType)
-    {
-        /*
-         * A packed down call frame.
-         */
-        case VMREQTYPE_INTERNAL:
-        {
-            uintptr_t *pauArgs = &pReq->u.Internal.aArgs[0];
-            union
-            {
-                PFNRT pfn;
-                DECLCALLBACKMEMBER(int, pfn00)(void);
-                DECLCALLBACKMEMBER(int, pfn01)(uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn02)(uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn03)(uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn04)(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn05)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn06)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn07)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn08)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn09)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn10)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn11)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn12)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn13)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn14)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-                DECLCALLBACKMEMBER(int, pfn15)(uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t);
-            } u;
-            u.pfn = pReq->u.Internal.pfn;
-#ifdef RT_ARCH_AMD64
-            switch (pReq->u.Internal.cArgs)
-            {
-                case 0:  rcRet = u.pfn00(); break;
-                case 1:  rcRet = u.pfn01(pauArgs[0]); break;
-                case 2:  rcRet = u.pfn02(pauArgs[0], pauArgs[1]); break;
-                case 3:  rcRet = u.pfn03(pauArgs[0], pauArgs[1], pauArgs[2]); break;
-                case 4:  rcRet = u.pfn04(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3]); break;
-                case 5:  rcRet = u.pfn05(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4]); break;
-                case 6:  rcRet = u.pfn06(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5]); break;
-                case 7:  rcRet = u.pfn07(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6]); break;
-                case 8:  rcRet = u.pfn08(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7]); break;
-                case 9:  rcRet = u.pfn09(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8]); break;
-                case 10: rcRet = u.pfn10(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9]); break;
-                case 11: rcRet = u.pfn11(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10]); break;
-                case 12: rcRet = u.pfn12(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11]); break;
-                case 13: rcRet = u.pfn13(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12]); break;
-                case 14: rcRet = u.pfn14(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12], pauArgs[13]); break;
-                case 15: rcRet = u.pfn15(pauArgs[0], pauArgs[1], pauArgs[2], pauArgs[3], pauArgs[4], pauArgs[5], pauArgs[6], pauArgs[7], pauArgs[8], pauArgs[9], pauArgs[10], pauArgs[11], pauArgs[12], pauArgs[13], pauArgs[14]); break;
-                default:
-                    AssertReleaseMsgFailed(("cArgs=%d\n", pReq->u.Internal.cArgs));
-                    rcRet = rcReq = VERR_INTERNAL_ERROR;
-                    break;
-            }
-#else /* x86: */
-            size_t cbArgs = pReq->u.Internal.cArgs * sizeof(uintptr_t);
-# ifdef __GNUC__
-            __asm__ __volatile__("movl  %%esp, %%edx\n\t"
-                                 "subl  %2, %%esp\n\t"
-                                 "andl  $0xfffffff0, %%esp\n\t"
-                                 "shrl  $2, %2\n\t"
-                                 "movl  %%esp, %%edi\n\t"
-                                 "rep movsl\n\t"
-                                 "movl  %%edx, %%edi\n\t"
-                                 "call  *%%eax\n\t"
-                                 "mov   %%edi, %%esp\n\t"
-                                 : "=a" (rcRet),
-                                   "=S" (pauArgs),
-                                   "=c" (cbArgs)
-                                 : "0" (u.pfn),
-                                   "1" (pauArgs),
-                                   "2" (cbArgs)
-                                 : "edi", "edx");
-# else
-            __asm
-            {
-                xor     edx, edx        /* just mess it up. */
-                mov     eax, u.pfn
-                mov     ecx, cbArgs
-                shr     ecx, 2
-                mov     esi, pauArgs
-                mov     ebx, esp
-                sub     esp, cbArgs
-                and     esp, 0xfffffff0
-                mov     edi, esp
-                rep movsd
-                call    eax
-                mov     esp, ebx
-                mov     rcRet, eax
-            }
-# endif
-#endif /* x86 */
-            if ((pReq->fFlags & (VMREQFLAGS_RETURN_MASK)) == VMREQFLAGS_VOID)
-                rcRet = VINF_SUCCESS;
-            rcReq = rcRet;
-            break;
-        }
-
-        default:
-            AssertMsgFailed(("pReq->enmType=%d\n", pReq->enmType));
-            rcReq = VERR_NOT_IMPLEMENTED;
-            break;
-    }
-
-    /*
-     * Complete the request.
-     */
-    pReq->iStatus  = rcReq;
-    pReq->enmState = VMREQSTATE_COMPLETED;
-    if (pReq->fFlags & VMREQFLAGS_NO_WAIT)
-    {
-        /* Free the packet, nobody is waiting. */
-        LogFlow(("vmR3ReqProcessOneU: Completed request %p: rcReq=%Rrc rcRet=%Rrc - freeing it\n",
-                 pReq, rcReq, rcRet));
-        VMR3ReqFree(pReq);
-    }
-    else
-    {
-        /* Notify the waiter and him free up the packet. */
-        LogFlow(("vmR3ReqProcessOneU: Completed request %p: rcReq=%Rrc rcRet=%Rrc - notifying waiting thread\n",
-                 pReq, rcReq, rcRet));
-        ASMAtomicXchgSize(&pReq->fEventSemClear, false);
-        int rc2 = RTSemEventSignal(pReq->EventSem);
-        if (RT_FAILURE(rc2))
-        {
-            AssertRC(rc2);
-            rcRet = rc2;
-        }
-    }
-    return rcRet;
-}
-
Index: /trunk/src/VBox/VMM/include/CFGMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/CFGMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/CFGMInternal.h	(revision 35333)
@@ -0,0 +1,131 @@
+/* $Id$ */
+/** @file
+ * CFGM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___CFGMInternal_h
+#define ___CFGMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+
+
+/** @defgroup grp_cfgm_int Internals.
+ * @ingroup grp_cfgm
+ * @{
+ */
+
+
+/**
+ * Configuration manager propertype value.
+ */
+typedef union CFGMVALUE
+{
+    /** Integer value. */
+    struct CFGMVALUE_INTEGER
+    {
+        /** The integer represented as 64-bit unsigned. */
+        uint64_t        u64;
+    } Integer;
+
+    /** String value. (UTF-8 of course) */
+    struct CFGMVALUE_STRING
+    {
+        /** Length of string. (In bytes, including the terminator.) */
+        size_t          cb;
+        /** Pointer to the string. */
+        char           *psz;
+    } String;
+
+    /** Byte string value. */
+    struct CFGMVALUE_BYTES
+    {
+        /** Length of byte string. (in bytes) */
+        size_t          cb;
+        /** Pointer to the byte string. */
+        uint8_t        *pau8;
+    } Bytes;
+} CFGMVALUE;
+/** Pointer to configuration manager property value. */
+typedef CFGMVALUE *PCFGMVALUE;
+
+
+/**
+ * Configuration manager tree node.
+ */
+typedef struct CFGMLEAF
+{
+    /** Pointer to the next leaf. */
+    PCFGMLEAF       pNext;
+    /** Pointer to the previous leaf. */
+    PCFGMLEAF       pPrev;
+
+    /** Property type. */
+    CFGMVALUETYPE   enmType;
+    /** Property value. */
+    CFGMVALUE       Value;
+
+    /** Name length. (exclusive) */
+    size_t          cchName;
+    /** Name. */
+    char            szName[1];
+} CFGMLEAF;
+
+
+/**
+ * Configuration manager tree node.
+ */
+typedef struct CFGMNODE
+{
+    /** Pointer to the next node (on this level). */
+    PCFGMNODE       pNext;
+    /** Pointer to the previous node (on this level). */
+    PCFGMNODE       pPrev;
+    /** Pointer Parent node. */
+    PCFGMNODE       pParent;
+    /** Pointer to first child node. */
+    PCFGMNODE       pFirstChild;
+    /** Pointer to first property leaf. */
+    PCFGMLEAF       pFirstLeaf;
+
+    /** Pointer to the VM owning this node. */
+    PVM             pVM;
+
+    /** The root of a 'restricted' subtree, i.e. the parent is
+     * invisible to non-trusted users.
+     */
+    bool            fRestrictedRoot;
+
+    /** Name length. (exclusive) */
+    size_t          cchName;
+    /** Name. */
+    char            szName[1];
+} CFGMNODE;
+
+
+
+/**
+ * CFGM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct CFGM
+{
+    /** Pointer to root node. */
+    R3PTRTYPE(PCFGMNODE)    pRoot;
+} CFGM;
+
+/** @} */
+
+#endif
Index: /trunk/src/VBox/VMM/include/CPUMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/CPUMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/CPUMInternal.h	(revision 35333)
@@ -0,0 +1,430 @@
+/* $Id$ */
+/** @file
+ * CPUM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___CPUMInternal_h
+#define ___CPUMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/x86.h>
+
+
+
+/** @defgroup grp_cpum_int   Internals
+ * @ingroup grp_cpum
+ * @internal
+ * @{
+ */
+
+/** Flags and types for CPUM fault handlers
+ * @{ */
+/** Type: Load DS */
+#define CPUM_HANDLER_DS                 1
+/** Type: Load ES */
+#define CPUM_HANDLER_ES                 2
+/** Type: Load FS */
+#define CPUM_HANDLER_FS                 3
+/** Type: Load GS */
+#define CPUM_HANDLER_GS                 4
+/** Type: IRET */
+#define CPUM_HANDLER_IRET               5
+/** Type mask. */
+#define CPUM_HANDLER_TYPEMASK           0xff
+/** If set EBP points to the CPUMCTXCORE that's being used. */
+#define CPUM_HANDLER_CTXCORE_IN_EBP     RT_BIT(31)
+/** @} */
+
+
+/** Use flags (CPUM::fUseFlags).
+ * (Don't forget to sync this with CPUMInternal.mac!)
+ * @{ */
+/** Used the FPU, SSE or such stuff. */
+#define CPUM_USED_FPU                   RT_BIT(0)
+/** Used the FPU, SSE or such stuff since last we were in REM.
+ * REM syncing is clearing this, lazy FPU is setting it. */
+#define CPUM_USED_FPU_SINCE_REM         RT_BIT(1)
+/** Host OS is using SYSENTER and we must NULL the CS. */
+#define CPUM_USE_SYSENTER               RT_BIT(2)
+/** Host OS is using SYSENTER and we must NULL the CS. */
+#define CPUM_USE_SYSCALL                RT_BIT(3)
+/** Debug registers are used by host and must be disabled. */
+#define CPUM_USE_DEBUG_REGS_HOST        RT_BIT(4)
+/** Enabled use of debug registers in guest context. */
+#define CPUM_USE_DEBUG_REGS             RT_BIT(5)
+/** The XMM state was manually restored. (AMD only) */
+#define CPUM_MANUAL_XMM_RESTORE         RT_BIT(6)
+/** Sync the FPU state on entry (32->64 switcher only). */
+#define CPUM_SYNC_FPU_STATE             RT_BIT(7)
+/** Sync the debug state on entry (32->64 switcher only). */
+#define CPUM_SYNC_DEBUG_STATE           RT_BIT(8)
+/** Enabled use of hypervisor debug registers in guest context. */
+#define CPUM_USE_DEBUG_REGS_HYPER       RT_BIT(9)
+/** @} */
+
+/* Sanity check. */
+#if defined(VBOX_WITH_HYBRID_32BIT_KERNEL) && (HC_ARCH_BITS != 32 || R0_ARCH_BITS != 32)
+# error "VBOX_WITH_HYBRID_32BIT_KERNEL is only for 32 bit builds."
+#endif
+
+
+/**
+ * The saved host CPU state.
+ *
+ * @remark  The special VBOX_WITH_HYBRID_32BIT_KERNEL checks here are for the 10.4.x series
+ *          of Mac OS X where the OS is essentially 32-bit but the cpu mode can be 64-bit.
+ */
+typedef struct CPUMHOSTCTX
+{
+    /** FPU state. (16-byte alignment)
+     * @remark On x86, the format isn't necessarily X86FXSTATE (not important). */
+    X86FXSTATE      fpu;
+
+    /** General purpose register, selectors, flags and more
+     * @{ */
+#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    /** General purpose register ++
+     * { */
+    //uint64_t        rax; - scratch
+    uint64_t        rbx;
+    //uint64_t        rcx; - scratch
+    //uint64_t        rdx; - scratch
+    uint64_t        rdi;
+    uint64_t        rsi;
+    uint64_t        rbp;
+    uint64_t        rsp;
+    //uint64_t        r8; - scratch
+    //uint64_t        r9; - scratch
+    uint64_t        r10;
+    uint64_t        r11;
+    uint64_t        r12;
+    uint64_t        r13;
+    uint64_t        r14;
+    uint64_t        r15;
+    //uint64_t        rip; - scratch
+    uint64_t        rflags;
+#endif
+
+#if HC_ARCH_BITS == 32
+    //uint32_t        eax; - scratch
+    uint32_t        ebx;
+    //uint32_t        ecx; - scratch
+    //uint32_t        edx; - scratch
+    uint32_t        edi;
+    uint32_t        esi;
+    uint32_t        ebp;
+    X86EFLAGS       eflags;
+    //uint32_t        eip; - scratch
+    /* lss pair! */
+    uint32_t        esp;
+#endif
+    /** @} */
+
+    /** Selector registers
+     * @{ */
+    RTSEL           ss;
+    RTSEL           ssPadding;
+    RTSEL           gs;
+    RTSEL           gsPadding;
+    RTSEL           fs;
+    RTSEL           fsPadding;
+    RTSEL           es;
+    RTSEL           esPadding;
+    RTSEL           ds;
+    RTSEL           dsPadding;
+    RTSEL           cs;
+    RTSEL           csPadding;
+    /** @} */
+
+#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    /** Control registers.
+     * @{ */
+    uint32_t        cr0;
+    //uint32_t        cr2; - scratch
+    uint32_t        cr3;
+    uint32_t        cr4;
+    /** @} */
+
+    /** Debug registers.
+     * @{ */
+    uint32_t        dr0;
+    uint32_t        dr1;
+    uint32_t        dr2;
+    uint32_t        dr3;
+    uint32_t        dr6;
+    uint32_t        dr7;
+    /** @} */
+
+    /** Global Descriptor Table register. */
+    X86XDTR32       gdtr;
+    uint16_t        gdtrPadding;
+    /** Interrupt Descriptor Table register. */
+    X86XDTR32       idtr;
+    uint16_t        idtrPadding;
+    /** The task register. */
+    RTSEL           ldtr;
+    RTSEL           ldtrPadding;
+    /** The task register. */
+    RTSEL           tr;
+    RTSEL           trPadding;
+    uint32_t        SysEnterPadding;
+
+    /** The sysenter msr registers.
+     * This member is not used by the hypervisor context. */
+    CPUMSYSENTER    SysEnter;
+
+    /** MSRs
+     * @{ */
+    uint64_t        efer;
+    /** @} */
+
+    /* padding to get 64byte aligned size */
+    uint8_t         auPadding[16+32];
+
+#elif HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+
+    /** Control registers.
+     * @{ */
+    uint64_t        cr0;
+    //uint64_t        cr2; - scratch
+    uint64_t        cr3;
+    uint64_t        cr4;
+    uint64_t        cr8;
+    /** @} */
+
+    /** Debug registers.
+     * @{ */
+    uint64_t        dr0;
+    uint64_t        dr1;
+    uint64_t        dr2;
+    uint64_t        dr3;
+    uint64_t        dr6;
+    uint64_t        dr7;
+    /** @} */
+
+    /** Global Descriptor Table register. */
+    X86XDTR64       gdtr;
+    uint16_t        gdtrPadding;
+    /** Interrupt Descriptor Table register. */
+    X86XDTR64       idtr;
+    uint16_t        idtrPadding;
+    /** The task register. */
+    RTSEL           ldtr;
+    RTSEL           ldtrPadding;
+    /** The task register. */
+    RTSEL           tr;
+    RTSEL           trPadding;
+
+    /** MSRs
+     * @{ */
+    CPUMSYSENTER    SysEnter;
+    uint64_t        FSbase;
+    uint64_t        GSbase;
+    uint64_t        efer;
+    /** @} */
+
+    /* padding to get 32byte aligned size */
+# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
+    uint8_t         auPadding[16];
+# else
+    uint8_t         auPadding[8+32];
+# endif
+
+#else
+# error HC_ARCH_BITS not defined
+#endif
+} CPUMHOSTCTX;
+/** Pointer to the saved host CPU state. */
+typedef CPUMHOSTCTX *PCPUMHOSTCTX;
+
+
+/**
+ * CPUM Data (part of VM)
+ */
+typedef struct CPUM
+{
+    /** Offset from CPUM to CPUMCPU for the first CPU. */
+    uint32_t                offCPUMCPU0;
+
+    /** Use flags.
+     * These flags indicates which CPU features the host uses.
+     */
+    uint32_t                fHostUseFlags;
+
+    /** Host CPU Features - ECX */
+    struct
+    {
+        /** edx part */
+        X86CPUIDFEATEDX     edx;
+        /** ecx part */
+        X86CPUIDFEATECX     ecx;
+    } CPUFeatures;
+    /** Host extended CPU features. */
+    struct
+    {
+        /** edx part */
+        uint32_t            edx;
+        /** ecx part */
+        uint32_t            ecx;
+    } CPUFeaturesExt;
+
+    /** Host CPU manufacturer. */
+    CPUMCPUVENDOR           enmHostCpuVendor;
+    /** Guest CPU manufacturer. */
+    CPUMCPUVENDOR           enmGuestCpuVendor;
+
+    /** CR4 mask */
+    struct
+    {
+        uint32_t            AndMask;
+        uint32_t            OrMask;
+    } CR4;
+
+    /** Synthetic CPU type? */
+    bool                    fSyntheticCpu;
+    /** The (more) portable CPUID level.  */
+    uint8_t                 u8PortableCpuIdLevel;
+    /** Indicates that a state restore is pending.
+     * This is used to verify load order dependencies (PGM). */
+    bool                    fPendingRestore;
+    uint8_t                 abPadding[HC_ARCH_BITS == 64 ? 5 : 1];
+
+    /** The standard set of CpuId leafs. */
+    CPUMCPUID               aGuestCpuIdStd[6];
+    /** The extended set of CpuId leafs. */
+    CPUMCPUID               aGuestCpuIdExt[10];
+    /** The centaur set of CpuId leafs. */
+    CPUMCPUID               aGuestCpuIdCentaur[4];
+    /** The default set of CpuId leafs. */
+    CPUMCPUID               GuestCpuIdDef;
+
+#if HC_ARCH_BITS == 32
+    uint8_t                 abPadding2[4];
+#endif
+
+#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+    RTHCPTR                 pvApicBase;
+    uint32_t                fApicDisVectors;
+    uint8_t                 abPadding3[HC_ARCH_BITS == 32 ? 56 : 52];
+#endif
+
+    /**
+     * Guest context on raw mode entry. 64-byte aligned!
+     * This a debug feature, see CPUMR3SaveEntryCtx.
+     */
+    CPUMCTX                 GuestEntry;
+} CPUM;
+/** Pointer to the CPUM instance data residing in the shared VM structure. */
+typedef CPUM *PCPUM;
+
+/**
+ * CPUM Data (part of VMCPU)
+ */
+typedef struct CPUMCPU
+{
+    /**
+     * Hypervisor context.
+     * Aligned on a 64-byte boundary.
+     */
+    CPUMCTX                 Hyper;
+
+    /**
+     * Saved host context. Only valid while inside GC.
+     * Aligned on a 64-byte boundary.
+     */
+    CPUMHOSTCTX             Host;
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    uint8_t                 aMagic[56];
+    uint64_t                uMagic;
+#endif
+
+    /**
+     * Guest context.
+     * Aligned on a 64-byte boundary.
+     */
+    CPUMCTX                 Guest;
+
+    /**
+     * Guest context - misc MSRs
+     * Aligned on a 64-byte boundary.
+     */
+    CPUMCTXMSR              GuestMsr;
+
+    /** Pointer to the current hypervisor core context - R3Ptr. */
+    R3PTRTYPE(PCPUMCTXCORE) pHyperCoreR3;
+    /** Pointer to the current hypervisor core context - R0Ptr. */
+    R0PTRTYPE(PCPUMCTXCORE) pHyperCoreR0;
+    /** Pointer to the current hypervisor core context - RCPtr. */
+    RCPTRTYPE(PCPUMCTXCORE) pHyperCoreRC;
+
+    /** Use flags.
+     * These flags indicates both what is to be used and what has been used.
+     */
+    uint32_t                fUseFlags;
+
+    /** Changed flags.
+     * These flags indicates to REM (and others) which important guest
+     * registers which has been changed since last time the flags were cleared.
+     * See the CPUM_CHANGED_* defines for what we keep track of.
+     */
+    uint32_t                fChanged;
+
+    /** Offset from CPUM to CPUMCPU. */
+    uint32_t                offCPUM;
+
+    /** Temporary storage for the return code of the function called in the
+     * 32-64 switcher. */
+    uint32_t                u32RetCode;
+
+    /** Have we entered raw-mode? */
+    bool                    fRawEntered;
+    /** Have we entered the recompiler? */
+    bool                    fRemEntered;
+
+    /** Align the structure on a 64-byte boundary. */
+    uint8_t                 abPadding2[HC_ARCH_BITS == 32 ? 34 : 26];
+} CPUMCPU, *PCPUMCPU;
+/** Pointer to the CPUMCPU instance data residing in the shared VMCPU structure. */
+typedef CPUMCPU *PCPUMCPU;
+
+RT_C_DECLS_BEGIN
+
+DECLASM(int)      cpumHandleLazyFPUAsm(PCPUMCPU pCPUM);
+
+#ifdef IN_RING0
+DECLASM(int)      cpumR0SaveHostRestoreGuestFPUState(PCPUMCPU pCPUM);
+DECLASM(int)      cpumR0SaveGuestRestoreHostFPUState(PCPUMCPU pCPUM);
+DECLASM(int)      cpumR0SaveHostFPUState(PCPUMCPU pCPUM);
+DECLASM(int)      cpumR0RestoreHostFPUState(PCPUMCPU pCPUM);
+DECLASM(void)     cpumR0LoadFPU(PCPUMCTX pCtx);
+DECLASM(void)     cpumR0SaveFPU(PCPUMCTX pCtx);
+DECLASM(void)     cpumR0LoadXMM(PCPUMCTX pCtx);
+DECLASM(void)     cpumR0SaveXMM(PCPUMCTX pCtx);
+DECLASM(void)     cpumR0SetFCW(uint16_t u16FCW);
+DECLASM(uint16_t) cpumR0GetFCW(void);
+DECLASM(void)     cpumR0SetMXCSR(uint32_t u32MXCSR);
+DECLASM(uint32_t) cpumR0GetMXCSR(void);
+DECLASM(void)     cpumR0LoadDRx(uint64_t const *pa4Regs);
+DECLASM(void)     cpumR0SaveDRx(uint64_t *pa4Regs);
+#endif
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/CPUMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/include/CPUMInternal.mac	(revision 35333)
+++ /trunk/src/VBox/VMM/include/CPUMInternal.mac	(revision 35333)
@@ -0,0 +1,468 @@
+; $Id$
+;; @file
+; CPUM - Internal header file (asm).
+;
+
+;
+; Copyright (C) 2006-2010 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%include "VBox/asmdefs.mac"
+
+%define CPUM_USED_FPU                   RT_BIT(0)
+%define CPUM_USED_FPU_SINCE_REM         RT_BIT(1)
+%define CPUM_USE_SYSENTER               RT_BIT(2)
+%define CPUM_USE_SYSCALL                RT_BIT(3)
+%define CPUM_USE_DEBUG_REGS_HOST        RT_BIT(4)
+%define CPUM_USE_DEBUG_REGS             RT_BIT(5)
+%define CPUM_SYNC_FPU_STATE             RT_BIT(7)
+%define CPUM_SYNC_DEBUG_STATE           RT_BIT(8)
+
+%define CPUM_HANDLER_DS                 1
+%define CPUM_HANDLER_ES                 2
+%define CPUM_HANDLER_FS                 3
+%define CPUM_HANDLER_GS                 4
+%define CPUM_HANDLER_IRET               5
+%define CPUM_HANDLER_TYPEMASK           0ffh
+%define CPUM_HANDLER_CTXCORE_IN_EBP     RT_BIT(31)
+
+%define VMMGCRET_USED_FPU               040000000h
+
+%define FPUSTATE_SIZE                   512
+
+;; if anyone figures how to do %if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) in
+; nasm please tell / fix this hack.
+%ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
+ %define fVBOX_WITH_HYBRID_32BIT_KERNEL 1
+%else
+ %define fVBOX_WITH_HYBRID_32BIT_KERNEL 0
+%endif
+
+struc CPUM
+    ;...
+    .offCPUMCPU0          resd    1
+    .fHostUseFlags        resd    1
+
+    ; CPUID eax=1
+    .CPUFeatures.edx      resd    1
+    .CPUFeatures.ecx      resd    1
+
+    ; CPUID eax=0x80000001
+    .CPUFeaturesExt.edx   resd    1
+    .CPUFeaturesExt.ecx   resd    1
+
+    .enmHostCpuVendor     resd    1
+    .enmGuestCpuVendor    resd    1
+
+    ; CR4 masks
+    .CR4.AndMask          resd    1
+    .CR4.OrMask           resd    1
+    ; entered rawmode?
+    .fSyntheticCpu        resb    1
+    .u8PortableCpuIdLevel resb    1
+    .fPendingRestore      resb    1
+%if RTHCPTR_CB == 8
+    .abPadding            resb    5
+%else
+    .abPadding            resb    1
+%endif
+
+    ; CPUID leafs
+    .aGuestCpuIdStd       resb    16*6
+    .aGuestCpuIdExt       resb    16*10
+    .aGuestCpuIdCentaur   resb    16*4
+    .GuestCpuIdDef        resb    16
+
+%if HC_ARCH_BITS == 32
+    .abPadding2           resb    4
+%endif
+
+%ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
+    .pvApicBase           RTR0PTR_RES 1
+    .fApicDisVectors      resd    1
+%endif
+
+    alignb 64
+    ; CPUMCTX debug stuff...
+    .GuestEntry           resb   1024
+endstruc
+
+struc CPUMCPU
+    ;
+    ; Hypervisor Context.
+    ;
+    alignb 64                           ; the padding
+    .Hyper.fpu            resb    FPUSTATE_SIZE
+
+    .Hyper.edi            resq    1
+    .Hyper.esi            resq    1
+    .Hyper.ebp            resq    1
+    .Hyper.eax            resq    1
+    .Hyper.ebx            resq    1
+    .Hyper.edx            resq    1
+    .Hyper.ecx            resq    1
+    .Hyper.esp            resq    1
+    .Hyper.lss_esp        resd    1
+    .Hyper.ss             resw    1
+    .Hyper.ssPadding      resw    1
+    .Hyper.gs             resw    1
+    .Hyper.gsPadding      resw    1
+    .Hyper.fs             resw    1
+    .Hyper.fsPadding      resw    1
+    .Hyper.es             resw    1
+    .Hyper.esPadding      resw    1
+    .Hyper.ds             resw    1
+    .Hyper.dsPadding      resw    1
+    .Hyper.cs             resw    1
+    .Hyper.csPadding      resw    3
+    .Hyper.eflags         resq    1
+    .Hyper.eip            resq    1
+    .Hyper.r8             resq    1
+    .Hyper.r9             resq    1
+    .Hyper.r10            resq    1
+    .Hyper.r11            resq    1
+    .Hyper.r12            resq    1
+    .Hyper.r13            resq    1
+    .Hyper.r14            resq    1
+    .Hyper.r15            resq    1
+
+    .Hyper.esHid.u64Base      resq    1
+    .Hyper.esHid.u32Limit     resd    1
+    .Hyper.esHid.Attr         resd    1
+
+    .Hyper.csHid.u64Base      resq    1
+    .Hyper.csHid.u32Limit     resd    1
+    .Hyper.csHid.Attr         resd    1
+
+    .Hyper.ssHid.u64Base      resq    1
+    .Hyper.ssHid.u32Limit     resd    1
+    .Hyper.ssHid.Attr         resd    1
+
+    .Hyper.dsHid.u64Base      resq    1
+    .Hyper.dsHid.u32Limit     resd    1
+    .Hyper.dsHid.Attr         resd    1
+
+    .Hyper.fsHid.u64Base      resq    1
+    .Hyper.fsHid.u32Limit     resd    1
+    .Hyper.fsHid.Attr         resd    1
+
+    .Hyper.gsHid.u64Base      resq    1
+    .Hyper.gsHid.u32Limit     resd    1
+    .Hyper.gsHid.Attr         resd    1
+
+    .Hyper.cr0            resq    1
+    .Hyper.cr2            resq    1
+    .Hyper.cr3            resq    1
+    .Hyper.cr4            resq    1
+
+    .Hyper.dr             resq    8
+
+    .Hyper.gdtr           resb    10          ; GDT limit + linear address
+    .Hyper.gdtrPadding    resw    1
+    .Hyper.idtr           resb    10          ; IDT limit + linear address
+    .Hyper.idtrPadding    resw    1
+    .Hyper.ldtr           resw    1
+    .Hyper.ldtrPadding    resw    1
+    .Hyper.tr             resw    1
+    .Hyper.trPadding      resw    1
+
+    .Hyper.SysEnter.cs        resb    8
+    .Hyper.SysEnter.eip       resb    8
+    .Hyper.SysEnter.esp       resb    8
+
+    .Hyper.msrEFER            resb    8
+    .Hyper.msrSTAR            resb    8
+    .Hyper.msrPAT             resb    8
+    .Hyper.msrLSTAR           resb    8
+    .Hyper.msrCSTAR           resb    8
+    .Hyper.msrSFMASK          resb    8
+    .Hyper.msrKERNELGSBASE    resb    8
+
+    .Hyper.ldtrHid.u64Base    resq    1
+    .Hyper.ldtrHid.u32Limit   resd    1
+    .Hyper.ldtrHid.Attr       resd    1
+
+    .Hyper.trHid.u64Base      resq    1
+    .Hyper.trHid.u32Limit     resd    1
+    .Hyper.trHid.Attr         resd    1
+
+    ;
+    ; Host context state
+    ;
+    .Host.fpu            resb    FPUSTATE_SIZE
+
+%if HC_ARCH_BITS == 64 || fVBOX_WITH_HYBRID_32BIT_KERNEL
+    ;.Host.rax            resq    1 - scratch
+    .Host.rbx            resq    1
+    ;.Host.rcx            resq    1 - scratch
+    ;.Host.rdx            resq    1 - scratch
+    .Host.rdi            resq    1
+    .Host.rsi            resq    1
+    .Host.rbp            resq    1
+    .Host.rsp            resq    1
+    ;.Host.r8             resq    1 - scratch
+    ;.Host.r9             resq    1 - scratch
+    .Host.r10            resq    1
+    .Host.r11            resq    1
+    .Host.r12            resq    1
+    .Host.r13            resq    1
+    .Host.r14            resq    1
+    .Host.r15            resq    1
+    ;.Host.rip            resd    1 - scratch
+    .Host.rflags         resq    1
+%endif
+%if HC_ARCH_BITS == 32
+    ;.Host.eax            resd    1 - scratch
+    .Host.ebx            resd    1
+    ;.Host.edx            resd    1 - scratch
+    ;.Host.ecx            resd    1 - scratch
+    .Host.edi            resd    1
+    .Host.esi            resd    1
+    .Host.ebp            resd    1
+    .Host.eflags         resd    1
+    ;.Host.eip            resd    1 - scratch
+    ; lss pair!
+    .Host.esp            resd    1
+%endif
+    .Host.ss             resw    1
+    .Host.ssPadding      resw    1
+    .Host.gs             resw    1
+    .Host.gsPadding      resw    1
+    .Host.fs             resw    1
+    .Host.fsPadding      resw    1
+    .Host.es             resw    1
+    .Host.esPadding      resw    1
+    .Host.ds             resw    1
+    .Host.dsPadding      resw    1
+    .Host.cs             resw    1
+    .Host.csPadding      resw    1
+
+%if HC_ARCH_BITS == 32 && fVBOX_WITH_HYBRID_32BIT_KERNEL == 0
+    .Host.cr0            resd    1
+    ;.Host.cr2            resd    1 - scratch
+    .Host.cr3            resd    1
+    .Host.cr4            resd    1
+
+    .Host.dr0            resd    1
+    .Host.dr1            resd    1
+    .Host.dr2            resd    1
+    .Host.dr3            resd    1
+    .Host.dr6            resd    1
+    .Host.dr7            resd    1
+
+    .Host.gdtr           resb    6           ; GDT limit + linear address
+    .Host.gdtrPadding    resw    1
+    .Host.idtr           resb    6           ; IDT limit + linear address
+    .Host.idtrPadding    resw    1
+    .Host.ldtr           resw    1
+    .Host.ldtrPadding    resw    1
+    .Host.tr             resw    1
+    .Host.trPadding      resw    1
+
+    .Host.SysEnterPadding resd   1
+    .Host.SysEnter.cs    resq    1
+    .Host.SysEnter.eip   resq    1
+    .Host.SysEnter.esp   resq    1
+    .Host.efer           resq    1
+
+%else ; 64-bit
+
+    .Host.cr0            resq    1
+    ;.Host.cr2            resq    1 - scratch
+    .Host.cr3            resq    1
+    .Host.cr4            resq    1
+    .Host.cr8            resq    1
+
+    .Host.dr0            resq    1
+    .Host.dr1            resq    1
+    .Host.dr2            resq    1
+    .Host.dr3            resq    1
+    .Host.dr6            resq    1
+    .Host.dr7            resq    1
+
+    .Host.gdtr           resb    10         ; GDT limit + linear address
+    .Host.gdtrPadding    resw    1
+    .Host.idtr           resb    10         ; IDT limit + linear address
+    .Host.idtrPadding    resw    1
+    .Host.ldtr           resw    1
+    .Host.ldtrPadding    resw    1
+    .Host.tr             resw    1
+    .Host.trPadding      resw    1
+
+    .Host.SysEnter.cs    resq    1
+    .Host.SysEnter.eip   resq    1
+    .Host.SysEnter.esp   resq    1
+    .Host.FSbase         resq    1
+    .Host.GSbase         resq    1
+    .Host.efer           resq    1
+%endif ; 64-bit
+
+%ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    .aMagic              resb    56
+    .uMagic              resq    1
+%endif
+    ;
+    ; Guest context state
+    ; (Identical to the .Hyper chunk above.)
+    ;
+    alignb 64
+    .Guest.fpu            resb    FPUSTATE_SIZE
+
+    .Guest.edi            resq    1
+    .Guest.esi            resq    1
+    .Guest.ebp            resq    1
+    .Guest.eax            resq    1
+    .Guest.ebx            resq    1
+    .Guest.edx            resq    1
+    .Guest.ecx            resq    1
+    .Guest.esp            resq    1
+    .Guest.lss_esp        resd    1
+    .Guest.ss             resw    1
+    .Guest.ssPadding      resw    1
+    .Guest.gs             resw    1
+    .Guest.gsPadding      resw    1
+    .Guest.fs             resw    1
+    .Guest.fsPadding      resw    1
+    .Guest.es             resw    1
+    .Guest.esPadding      resw    1
+    .Guest.ds             resw    1
+    .Guest.dsPadding      resw    1
+    .Guest.cs             resw    1
+    .Guest.csPadding      resw    3
+    .Guest.eflags         resq    1
+    .Guest.eip            resq    1
+    .Guest.r8             resq    1
+    .Guest.r9             resq    1
+    .Guest.r10            resq    1
+    .Guest.r11            resq    1
+    .Guest.r12            resq    1
+    .Guest.r13            resq    1
+    .Guest.r14            resq    1
+    .Guest.r15            resq    1
+
+    .Guest.esHid.u64Base      resq    1
+    .Guest.esHid.u32Limit     resd    1
+    .Guest.esHid.Attr         resd    1
+
+    .Guest.csHid.u64Base      resq    1
+    .Guest.csHid.u32Limit     resd    1
+    .Guest.csHid.Attr         resd    1
+
+    .Guest.ssHid.u64Base      resq    1
+    .Guest.ssHid.u32Limit     resd    1
+    .Guest.ssHid.Attr         resd    1
+
+    .Guest.dsHid.u64Base      resq    1
+    .Guest.dsHid.u32Limit     resd    1
+    .Guest.dsHid.Attr         resd    1
+
+    .Guest.fsHid.u64Base      resq    1
+    .Guest.fsHid.u32Limit     resd    1
+    .Guest.fsHid.Attr         resd    1
+
+    .Guest.gsHid.u64Base      resq    1
+    .Guest.gsHid.u32Limit     resd    1
+    .Guest.gsHid.Attr         resd    1
+
+    .Guest.cr0            resq    1
+    .Guest.cr2            resq    1
+    .Guest.cr3            resq    1
+    .Guest.cr4            resq    1
+
+    .Guest.dr             resq    8
+
+    .Guest.gdtr.cbGdt     resw    1
+    .Guest.gdtr.pGdt      resq    1
+    .Guest.gdtrPadding    resw    1
+    .Guest.idtr.cbIdt     resw    1
+    .Guest.idtr.pIdt      resq    1
+    .Guest.idtrPadding    resw    1
+    .Guest.ldtr           resw    1
+    .Guest.ldtrPadding    resw    1
+    .Guest.tr             resw    1
+    .Guest.trPadding      resw    1
+
+    .Guest.SysEnter.cs        resb    8
+    .Guest.SysEnter.eip       resb    8
+    .Guest.SysEnter.esp       resb    8
+
+    .Guest.msrEFER            resb    8
+    .Guest.msrSTAR            resb    8
+    .Guest.msrPAT             resb    8
+    .Guest.msrLSTAR           resb    8
+    .Guest.msrCSTAR           resb    8
+    .Guest.msrSFMASK          resb    8
+    .Guest.msrKERNELGSBASE    resb    8
+
+    .Guest.ldtrHid.u64Base    resq    1
+    .Guest.ldtrHid.u32Limit   resd    1
+    .Guest.ldtrHid.Attr       resd    1
+
+    .Guest.trHid.u64Base      resq    1
+    .Guest.trHid.u32Limit     resd    1
+    .Guest.trHid.Attr         resd    1
+
+    .GuestMsr.au64            resq    64
+
+    ;
+    ; Other stuff.
+    ;
+    alignb 64
+    ; hypervisor core context.
+    .pHyperCoreR3         RTR3PTR_RES   1
+    .pHyperCoreR0         RTR0PTR_RES   1
+    .pHyperCoreRC         RTRCPTR_RES   1
+
+    .fUseFlags            resd    1
+    .fChanged             resd    1
+    .offCPUM              resd    1
+    .u32RetCode           resd    1
+    .fRawEntered          resb    1
+    .fRemEntered          resb    1
+
+%if RTHCPTR_CB == 8
+    .abPadding2           resb    26
+%else
+    .abPadding2           resb    34
+%endif
+
+endstruc
+
+
+;;
+; Converts the CPUM pointer to CPUMCPU
+; @param   %1   register name
+%macro CPUMCPU_FROM_CPUM 1
+    add     %1, dword [%1 + CPUM.offCPUMCPU0]
+%endmacro
+
+;;
+; Converts the CPUM pointer to CPUMCPU
+; @param   %1   register name (PVM)
+; @param   %2   register name (CPUMCPU offset)
+%macro CPUMCPU_FROM_CPUM_WITH_OFFSET 2
+    add     %1, %2
+%endmacro
+
+;;
+; Converts the CPUMCPU pointer to CPUM
+; @param   %1   register name
+%macro CPUM_FROM_CPUMCPU 1
+    sub     %1, dword [%1 + CPUMCPU.offCPUM]
+%endmacro
+
+;;
+; Converts the CPUMCPU pointer to CPUM
+; @param   %1   register name (PVM)
+; @param   %2   register name (CPUMCPU offset)
+%macro CPUM_FROM_CPUMCPU_WITH_OFFSET 2
+    sub     %1, %2
+%endmacro
Index: /trunk/src/VBox/VMM/include/DBGFInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/DBGFInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/DBGFInternal.h	(revision 35333)
@@ -0,0 +1,331 @@
+/* $Id$ */
+/** @file
+ * DBGF - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___DBGFInternal_h
+#define ___DBGFInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <iprt/semaphore.h>
+#include <iprt/critsect.h>
+#include <iprt/string.h>
+#include <iprt/avl.h>
+#include <VBox/dbgf.h>
+
+
+
+/** @defgroup grp_dbgf_int   Internals
+ * @ingroup grp_dbgf
+ * @internal
+ * @{
+ */
+
+
+/** VMM Debugger Command. */
+typedef enum DBGFCMD
+{
+    /** No command.
+     * This is assigned to the field by the emulation thread after
+     * a command has been completed. */
+    DBGFCMD_NO_COMMAND = 0,
+    /** Halt the VM. */
+    DBGFCMD_HALT,
+    /** Resume execution. */
+    DBGFCMD_GO,
+    /** Single step execution - stepping into calls. */
+    DBGFCMD_SINGLE_STEP,
+    /** Set a breakpoint. */
+    DBGFCMD_BREAKPOINT_SET,
+    /** Set a access breakpoint. */
+    DBGFCMD_BREAKPOINT_SET_ACCESS,
+    /** Set a REM breakpoint. */
+    DBGFCMD_BREAKPOINT_SET_REM,
+    /** Clear a breakpoint. */
+    DBGFCMD_BREAKPOINT_CLEAR,
+    /** Enable a breakpoint. */
+    DBGFCMD_BREAKPOINT_ENABLE,
+    /** Disable a breakpoint. */
+    DBGFCMD_BREAKPOINT_DISABLE,
+    /** List breakpoints. */
+    DBGFCMD_BREAKPOINT_LIST,
+
+    /** Detaches the debugger.
+     * Disabling all breakpoints, watch points and the like. */
+    DBGFCMD_DETACH_DEBUGGER = 0x7ffffffe,
+    /** Detached the debugger.
+     * The isn't a command as such, it's just that it's necessary for the
+     * detaching protocol to be racefree. */
+    DBGFCMD_DETACHED_DEBUGGER = 0x7fffffff
+} DBGFCMD;
+
+/**
+ * VMM Debugger Command.
+ */
+typedef union DBGFCMDDATA
+{
+    uint32_t    uDummy;
+} DBGFCMDDATA;
+/** Pointer to DBGF Command Data. */
+typedef DBGFCMDDATA *PDBGFCMDDATA;
+
+/**
+ * Info type.
+ */
+typedef enum DBGFINFOTYPE
+{
+    /** Invalid. */
+    DBGFINFOTYPE_INVALID = 0,
+    /** Device owner. */
+    DBGFINFOTYPE_DEV,
+    /** Driver owner. */
+    DBGFINFOTYPE_DRV,
+    /** Internal owner. */
+    DBGFINFOTYPE_INT,
+    /** External owner. */
+    DBGFINFOTYPE_EXT
+} DBGFINFOTYPE;
+
+
+/** Pointer to info structure. */
+typedef struct DBGFINFO *PDBGFINFO;
+
+/**
+ * Info structure.
+ */
+typedef struct DBGFINFO
+{
+    /** The flags. */
+    uint32_t        fFlags;
+    /** Owner type. */
+    DBGFINFOTYPE    enmType;
+    /** Per type data. */
+    union
+    {
+        /** DBGFINFOTYPE_DEV */
+        struct
+        {
+            /** Device info handler function. */
+            PFNDBGFHANDLERDEV   pfnHandler;
+            /** The device instance. */
+            PPDMDEVINS          pDevIns;
+        } Dev;
+
+        /** DBGFINFOTYPE_DRV */
+        struct
+        {
+            /** Driver info handler function. */
+            PFNDBGFHANDLERDRV   pfnHandler;
+            /** The driver instance. */
+            PPDMDRVINS          pDrvIns;
+        } Drv;
+
+        /** DBGFINFOTYPE_INT */
+        struct
+        {
+            /** Internal info handler function. */
+            PFNDBGFHANDLERINT   pfnHandler;
+        } Int;
+
+        /** DBGFINFOTYPE_EXT */
+        struct
+        {
+            /** External info handler function. */
+            PFNDBGFHANDLEREXT   pfnHandler;
+            /** The user argument. */
+            void               *pvUser;
+        } Ext;
+    } u;
+
+    /** Pointer to the description. */
+    const char     *pszDesc;
+    /** Pointer to the next info structure. */
+    PDBGFINFO       pNext;
+    /** The identifier name length. */
+    size_t          cchName;
+    /** The identifier name. (Extends 'beyond' the struct as usual.) */
+    char            szName[1];
+} DBGFINFO;
+
+
+/**
+ * Guest OS digger instance.
+ */
+typedef struct DBGFOS
+{
+    /** Pointer to the registration record. */
+    PCDBGFOSREG pReg;
+    /** Pointer to the next OS we've registered. */
+    struct DBGFOS *pNext;
+    /** The instance data (variable size). */
+    uint8_t abData[16];
+} DBGFOS;
+/** Pointer to guest OS digger instance. */
+typedef DBGFOS *PDBGFOS;
+/** Pointer to const guest OS digger instance. */
+typedef DBGFOS const *PCDBGFOS;
+
+
+/**
+ * Converts a DBGF pointer into a VM pointer.
+ * @returns Pointer to the VM structure the CPUM is part of.
+ * @param   pDBGF   Pointer to DBGF instance data.
+ */
+#define DBGF2VM(pDBGF)  ( (PVM)((char*)pDBGF - pDBGF->offVM) )
+
+
+/**
+ * DBGF Data (part of VM)
+ */
+typedef struct DBGF
+{
+    /** Offset to the VM structure. */
+    RTINT                   offVM;
+
+    /** Debugger Attached flag.
+     * Set if a debugger is attached, elsewise it's clear.
+     */
+    bool volatile           fAttached;
+
+    /** Stopped in the Hypervisor.
+     * Set if we're stopped on a trace, breakpoint or assertion inside
+     * the hypervisor and have to restrict the available operations.
+     */
+    bool volatile           fStoppedInHyper;
+
+    /**
+     * Ping-Pong construct where the Ping side is the VMM and the Pong side
+     * the Debugger.
+     */
+    RTPINGPONG              PingPong;
+
+    /** The Event to the debugger.
+     * The VMM will ping the debugger when the event is ready. The event is
+     * either a response to a command or to a break/watch point issued
+     * previously.
+     */
+    DBGFEVENT               DbgEvent;
+
+    /** The Command to the VMM.
+     * Operated in an atomic fashion since the VMM will poll on this.
+     * This means that a the command data must be written before this member
+     * is set. The VMM will reset this member to the no-command state
+     * when it have processed it.
+     */
+    DBGFCMD volatile        enmVMMCmd;
+    /** The Command data.
+     * Not all commands take data. */
+    DBGFCMDDATA             VMMCmdData;
+
+    /** List of registered info handlers. */
+    R3PTRTYPE(PDBGFINFO)    pInfoFirst;
+    /** Critical section protecting the above list. */
+    RTCRITSECT              InfoCritSect;
+
+    /** Range tree containing the loaded symbols of the a VM.
+     * This tree will never have blind spots. */
+    R3PTRTYPE(AVLRGCPTRTREE) SymbolTree;
+    /** Symbol name space. */
+    R3PTRTYPE(PRTSTRSPACE)  pSymbolSpace;
+    /** Indicates whether DBGFSym.cpp is initialized or not.
+     * This part is initialized in a lazy manner for performance reasons. */
+    bool                    fSymInited;
+    /** Alignment padding. */
+    RTUINT                  uAlignment0;
+
+    /** The number of hardware breakpoints. */
+    RTUINT                  cHwBreakpoints;
+    /** The number of active breakpoints. */
+    RTUINT                  cBreakpoints;
+    /** Array of hardware breakpoints. (0..3)
+     * This is shared among all the CPUs because life is much simpler that way. */
+    DBGFBP                  aHwBreakpoints[4];
+    /** Array of int 3 and REM breakpoints. (4..)
+     * @remark This is currently a fixed size array for reasons of simplicity. */
+    DBGFBP                  aBreakpoints[32];
+
+    /** The address space database lock. */
+    RTSEMRW                 hAsDbLock;
+    /** The address space handle database.      (Protected by hAsDbLock.) */
+    R3PTRTYPE(AVLPVTREE)    AsHandleTree;
+    /** The address space process id database.  (Protected by hAsDbLock.) */
+    R3PTRTYPE(AVLU32TREE)   AsPidTree;
+    /** The address space name database.        (Protected by hAsDbLock.) */
+    R3PTRTYPE(RTSTRSPACE)   AsNameSpace;
+    /** Special address space aliases.          (Protected by hAsDbLock.) */
+    RTDBGAS volatile        ahAsAliases[DBGF_AS_COUNT];
+    /** For lazily populating the aliased address spaces. */
+    bool volatile           afAsAliasPopuplated[DBGF_AS_COUNT];
+    /** Alignment padding. */
+    bool                    afAlignment[2];
+
+    /** The current Guest OS digger. */
+    R3PTRTYPE(PDBGFOS)      pCurOS;
+    /** The head of the Guest OS digger instances. */
+    R3PTRTYPE(PDBGFOS)      pOSHead;
+} DBGF;
+/** Pointer to DBGF Data. */
+typedef DBGF *PDBGF;
+
+
+/** Converts a DBGFCPU pointer into a VM pointer. */
+#define DBGFCPU_2_VM(pDbgfCpu) ((PVM)((uint8_t *)(pDbgfCpu) + (pDbgfCpu)->offVM))
+
+/**
+ * The per CPU data for DBGF.
+ */
+typedef struct DBGFCPU
+{
+    /** The offset into the VM structure.
+     * @see DBGFCPU_2_VM(). */
+    uint32_t                offVM;
+
+    /** Current active breakpoint (id).
+     * This is ~0U if not active. It is set when a execution engine
+     * encounters a breakpoint and returns VINF_EM_DBG_BREAKPOINT. This is
+     * currently not used for REM breakpoints because of the lazy coupling
+     * between VBox and REM. */
+    uint32_t                iActiveBp;
+    /** Set if we're singlestepping in raw mode.
+     * This is checked and cleared in the \#DB handler. */
+    bool                    fSingleSteppingRaw;
+
+    /** Padding the structure to 16 bytes. */
+    uint8_t                 abReserved[3];
+} DBGFCPU;
+/** Pointer to DBGFCPU data. */
+typedef DBGFCPU *PDBGFCPU;
+
+
+int  dbgfR3AsInit(PVM pVM);
+void dbgfR3AsTerm(PVM pVM);
+void dbgfR3AsRelocate(PVM pVM, RTGCUINTPTR offDelta);
+int  dbgfR3InfoInit(PVM pVM);
+int  dbgfR3InfoTerm(PVM pVM);
+void dbgfR3OSTerm(PVM pVM);
+int  dbgfR3SymInit(PVM pVM);
+int  dbgfR3SymTerm(PVM pVM);
+int  dbgfR3BpInit(PVM pVM);
+
+
+
+#ifdef IN_RING3
+
+#endif
+
+/** @} */
+
+#endif
Index: /trunk/src/VBox/VMM/include/EMHandleRCTmpl.h
===================================================================
--- /trunk/src/VBox/VMM/include/EMHandleRCTmpl.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/EMHandleRCTmpl.h	(revision 35333)
@@ -0,0 +1,359 @@
+/* $Id$ */
+/** @file
+ * EM - emR3[Raw|Hwaccm]HandleRC template.
+ */
+
+/*
+ * Copyright (C) 2006-2009 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___EMHandleRCTmpl_h
+#define ___EMHandleRCTmpl_h
+
+/**
+ * Process a subset of the raw-mode return code.
+ *
+ * Since we have to share this with raw-mode single stepping, this inline
+ * function has been created to avoid code duplication.
+ *
+ * @returns VINF_SUCCESS if it's ok to continue raw mode.
+ * @returns VBox status code to return to the EM main loop.
+ *
+ * @param   pVM     The VM handle
+ * @param   pVCpu   The VMCPU handle
+ * @param   rc      The return code.
+ * @param   pCtx    The guest cpu context.
+ */
+#ifdef EMHANDLERC_WITH_PATM
+int emR3RawHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
+#elif defined(EMHANDLERC_WITH_HWACCM)
+int emR3HwaccmHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc)
+#endif
+{
+    switch (rc)
+    {
+        /*
+         * Common & simple ones.
+         */
+        case VINF_SUCCESS:
+            break;
+        case VINF_EM_RESCHEDULE_RAW:
+        case VINF_EM_RESCHEDULE_HWACC:
+        case VINF_EM_RAW_INTERRUPT:
+        case VINF_EM_RAW_TO_R3:
+        case VINF_EM_RAW_TIMER_PENDING:
+        case VINF_EM_PENDING_REQUEST:
+            rc = VINF_SUCCESS;
+            break;
+
+#ifdef EMHANDLERC_WITH_PATM
+        /*
+         * Privileged instruction.
+         */
+        case VINF_EM_RAW_EXCEPTION_PRIVILEGED:
+        case VINF_PATM_PATCH_TRAP_GP:
+            rc = emR3RawPrivileged(pVM, pVCpu);
+            break;
+
+        case VINF_EM_RAW_GUEST_TRAP:
+            /*
+             * Got a trap which needs dispatching.
+             */
+            if (PATMR3IsInsidePatchJump(pVM, pCtx->eip, NULL))
+            {
+                AssertReleaseMsgFailed(("FATAL ERROR: executing random instruction inside generated patch jump %08X\n", CPUMGetGuestEIP(pVCpu)));
+                rc = VERR_EM_RAW_PATCH_CONFLICT;
+                break;
+            }
+            rc = emR3RawGuestTrap(pVM, pVCpu);
+            break;
+
+        /*
+         * Trap in patch code.
+         */
+        case VINF_PATM_PATCH_TRAP_PF:
+        case VINF_PATM_PATCH_INT3:
+            rc = emR3PatchTrap(pVM, pVCpu, pCtx, rc);
+            break;
+
+        case VINF_PATM_DUPLICATE_FUNCTION:
+            Assert(PATMIsPatchGCAddr(pVM, pCtx->eip));
+            rc = PATMR3DuplicateFunctionRequest(pVM, pCtx);
+            AssertRC(rc);
+            rc = VINF_SUCCESS;
+            break;
+
+        case VINF_PATM_CHECK_PATCH_PAGE:
+            rc = PATMR3HandleMonitoredPage(pVM);
+            AssertRC(rc);
+            rc = VINF_SUCCESS;
+            break;
+
+        /*
+         * Patch manager.
+         */
+        case VERR_EM_RAW_PATCH_CONFLICT:
+            AssertReleaseMsgFailed(("%Rrc handling is not yet implemented\n", rc));
+            break;
+#endif /* EMHANDLERC_WITH_PATM */
+
+#ifdef EMHANDLERC_WITH_PATM
+        /*
+         * Memory mapped I/O access - attempt to patch the instruction
+         */
+        case VINF_PATM_HC_MMIO_PATCH_READ:
+            rc = PATMR3InstallPatch(pVM, SELMToFlat(pVM, DIS_SELREG_CS, CPUMCTX2CORE(pCtx), pCtx->eip),
+                                    PATMFL_MMIO_ACCESS | ((SELMGetCpuModeFromSelector(pVM, pCtx->eflags, pCtx->cs, &pCtx->csHid) == CPUMODE_32BIT) ? PATMFL_CODE32 : 0));
+            if (RT_FAILURE(rc))
+                rc = emR3ExecuteInstruction(pVM, pVCpu, "MMIO");
+            break;
+
+        case VINF_PATM_HC_MMIO_PATCH_WRITE:
+            AssertFailed(); /* not yet implemented. */
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "MMIO");
+            break;
+#endif /* EMHANDLERC_WITH_PATM */
+
+        /*
+         * Conflict or out of page tables.
+         *
+         * VM_FF_PGM_SYNC_CR3 is set by the hypervisor and all we need to
+         * do here is to execute the pending forced actions.
+         */
+        case VINF_PGM_SYNC_CR3:
+            AssertMsg(VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL),
+                      ("VINF_PGM_SYNC_CR3 and no VMCPU_FF_PGM_SYNC_CR3*!\n"));
+            rc = VINF_SUCCESS;
+            break;
+
+        /*
+         * PGM pool flush pending (guest SMP only).
+         */
+        /** @todo jumping back and forth between ring 0 and 3 can burn a lot of cycles
+         * if the EMT thread that's supposed to handle the flush is currently not active
+         * (e.g. waiting to be scheduled) -> fix this properly!
+         *
+         * bird: Since the clearing is global and done via a rendezvous any CPU can do
+         *       it. They would have to choose who to call VMMR3EmtRendezvous and send
+         *       the rest to VMMR3EmtRendezvousFF ... Hmm ... that's not going to work
+         *       all that well since the the latter will race the setup done by the
+         *       first.  Guess that means we need some new magic in that area for
+         *       handling this case. :/
+         */
+        case VINF_PGM_POOL_FLUSH_PENDING:
+            rc = VINF_SUCCESS;
+            break;
+
+        /*
+         * Paging mode change.
+         */
+        case VINF_PGM_CHANGE_MODE:
+            rc = PGMChangeMode(pVCpu, pCtx->cr0, pCtx->cr4, pCtx->msrEFER);
+            if (rc == VINF_SUCCESS)
+                rc = VINF_EM_RESCHEDULE;
+            AssertMsg(RT_FAILURE(rc) || (rc >= VINF_EM_FIRST && rc <= VINF_EM_LAST), ("%Rrc\n", rc));
+            break;
+
+#ifdef EMHANDLERC_WITH_PATM
+        /*
+         * CSAM wants to perform a task in ring-3. It has set an FF action flag.
+         */
+        case VINF_CSAM_PENDING_ACTION:
+            rc = VINF_SUCCESS;
+            break;
+
+        /*
+         * Invoked Interrupt gate - must directly (!) go to the recompiler.
+         */
+        case VINF_EM_RAW_INTERRUPT_PENDING:
+        case VINF_EM_RAW_RING_SWITCH_INT:
+            Assert(TRPMHasTrap(pVCpu));
+            Assert(!PATMIsPatchGCAddr(pVM, pCtx->eip));
+
+            if (TRPMHasTrap(pVCpu))
+            {
+                /* If the guest gate is marked unpatched, then we will check again if we can patch it. */
+                uint8_t u8Interrupt = TRPMGetTrapNo(pVCpu);
+                if (TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) == TRPM_INVALID_HANDLER)
+                {
+                    CSAMR3CheckGates(pVM, u8Interrupt, 1);
+                    Log(("emR3RawHandleRC: recheck gate %x -> valid=%d\n", u8Interrupt, TRPMR3GetGuestTrapHandler(pVM, u8Interrupt) != TRPM_INVALID_HANDLER));
+                    /* Note: If it was successful, then we could go back to raw mode, but let's keep things simple for now. */
+                }
+            }
+            rc = VINF_EM_RESCHEDULE_REM;
+            break;
+
+        /*
+         * Other ring switch types.
+         */
+        case VINF_EM_RAW_RING_SWITCH:
+            rc = emR3RawRingSwitch(pVM, pVCpu);
+            break;
+#endif  /* EMHANDLERC_WITH_PATM */
+
+        /*
+         * I/O Port access - emulate the instruction.
+         */
+        case VINF_IOM_HC_IOPORT_READ:
+        case VINF_IOM_HC_IOPORT_WRITE:
+            rc = emR3ExecuteIOInstruction(pVM, pVCpu);
+            break;
+
+        /*
+         * Memory mapped I/O access - emulate the instruction.
+         */
+        case VINF_IOM_HC_MMIO_READ:
+        case VINF_IOM_HC_MMIO_WRITE:
+        case VINF_IOM_HC_MMIO_READ_WRITE:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "MMIO");
+            break;
+
+#ifdef EMHANDLERC_WITH_HWACCM
+        /*
+         * (MM)IO intensive code block detected; fall back to the recompiler for better performance
+         */
+        case VINF_EM_RAW_EMULATE_IO_BLOCK:
+            rc = HWACCMR3EmulateIoBlock(pVM, pCtx);
+            break;
+
+        case VINF_EM_HWACCM_PATCH_TPR_INSTR:
+            rc = HWACCMR3PatchTprInstr(pVM, pVCpu, pCtx);
+            break;
+#endif
+
+#ifdef EMHANDLERC_WITH_PATM
+        /*
+         * Execute instruction.
+         */
+        case VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "LDT FAULT: ");
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "GDT FAULT: ");
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "IDT FAULT: ");
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "TSS FAULT: ");
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_PD_FAULT:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "PD FAULT: ");
+            break;
+        case VINF_EM_RAW_EMULATE_INSTR_HLT:
+            /** @todo skip instruction and go directly to the halt state. (see REM for implementation details) */
+            rc = emR3RawPrivileged(pVM, pVCpu);
+            break;
+#endif
+
+#ifdef EMHANDLERC_WITH_PATM
+        case VINF_PATM_PENDING_IRQ_AFTER_IRET:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "EMUL: ", VINF_PATM_PENDING_IRQ_AFTER_IRET);
+            break;
+
+        case VINF_PATCH_EMULATE_INSTR:
+#else
+        case VINF_EM_RAW_GUEST_TRAP:
+#endif
+        case VINF_EM_RAW_EMULATE_INSTR:
+            rc = emR3ExecuteInstruction(pVM, pVCpu, "EMUL: ");
+            break;
+
+#ifdef EMHANDLERC_WITH_PATM
+        /*
+         * Stale selector and iret traps => REM.
+         */
+        case VINF_EM_RAW_STALE_SELECTOR:
+        case VINF_EM_RAW_IRET_TRAP:
+            /* We will not go to the recompiler if EIP points to patch code. */
+            if (PATMIsPatchGCAddr(pVM, pCtx->eip))
+            {
+                pCtx->eip = PATMR3PatchToGCPtr(pVM, (RTGCPTR)pCtx->eip, 0);
+            }
+            LogFlow(("emR3RawHandleRC: %Rrc -> %Rrc\n", rc, VINF_EM_RESCHEDULE_REM));
+            rc = VINF_EM_RESCHEDULE_REM;
+            break;
+#endif
+
+        /*
+         * Up a level.
+         */
+        case VINF_EM_TERMINATE:
+        case VINF_EM_OFF:
+        case VINF_EM_RESET:
+        case VINF_EM_SUSPEND:
+        case VINF_EM_HALT:
+        case VINF_EM_RESUME:
+        case VINF_EM_NO_MEMORY:
+        case VINF_EM_RESCHEDULE:
+        case VINF_EM_RESCHEDULE_REM:
+        case VINF_EM_WAIT_SIPI:
+            break;
+
+        /*
+         * Up a level and invoke the debugger.
+         */
+        case VINF_EM_DBG_STEPPED:
+        case VINF_EM_DBG_BREAKPOINT:
+        case VINF_EM_DBG_STEP:
+        case VINF_EM_DBG_HYPER_BREAKPOINT:
+        case VINF_EM_DBG_HYPER_STEPPED:
+        case VINF_EM_DBG_HYPER_ASSERTION:
+        case VINF_EM_DBG_STOP:
+            break;
+
+        /*
+         * Up a level, dump and debug.
+         */
+        case VERR_TRPM_DONT_PANIC:
+        case VERR_TRPM_PANIC:
+        case VERR_VMM_RING0_ASSERTION:
+        case VERR_VMM_HYPER_CR3_MISMATCH:
+        case VERR_VMM_RING3_CALL_DISABLED:
+            break;
+
+#ifdef EMHANDLERC_WITH_HWACCM
+        /*
+         * Up a level, after HwAccM have done some release logging.
+         */
+        case VERR_VMX_INVALID_VMCS_FIELD:
+        case VERR_VMX_INVALID_VMCS_PTR:
+        case VERR_VMX_INVALID_VMXON_PTR:
+        case VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE:
+        case VERR_VMX_UNEXPECTED_EXCEPTION:
+        case VERR_VMX_UNEXPECTED_EXIT_CODE:
+        case VERR_VMX_INVALID_GUEST_STATE:
+        case VERR_VMX_UNABLE_TO_START_VM:
+        case VERR_VMX_UNABLE_TO_RESUME_VM:
+            HWACCMR3CheckError(pVM, rc);
+            break;
+
+        /* Up a level; fatal */
+        case VERR_VMX_IN_VMX_ROOT_MODE:
+        case VERR_SVM_IN_USE:
+        case VERR_SVM_UNABLE_TO_START_VM:
+            break;
+#endif
+
+        /*
+         * Anything which is not known to us means an internal error
+         * and the termination of the VM!
+         */
+        default:
+            AssertMsgFailed(("Unknown GC return code: %Rra\n", rc));
+            break;
+    }
+    return rc;
+}
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/EMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/EMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/EMInternal.h	(revision 35333)
@@ -0,0 +1,463 @@
+/* $Id$ */
+/** @file
+ * EM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___EMInternal_h
+#define ___EMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/em.h>
+#include <VBox/stam.h>
+#include <VBox/patm.h>
+#include <VBox/dis.h>
+#include <VBox/pdmcritsect.h>
+#include <iprt/avl.h>
+#include <setjmp.h>
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_em_int       Internal
+ * @ingroup grp_em
+ * @internal
+ * @{
+ */
+
+/** The saved state version. */
+#define EM_SAVED_STATE_VERSION                          4
+#define EM_SAVED_STATE_VERSION_PRE_MWAIT                3
+#define EM_SAVED_STATE_VERSION_PRE_SMP                  2
+
+
+/**
+ * MWait state flags.
+ */
+/* MWait activated. */
+#define EMMWAIT_FLAG_ACTIVE             RT_BIT(0)
+/* MWait will continue when an interrupt is pending even when IF=0. */
+#define EMMWAIT_FLAG_BREAKIRQIF0        RT_BIT(1)
+/* Monitor instruction was executed previously. */
+#define EMMWAIT_FLAG_MONITOR_ACTIVE     RT_BIT(2)
+
+/** EM time slice in ms; used for capping execution time. */
+#define EM_TIME_SLICE                   100
+
+/**
+ * Cli node structure
+ */
+typedef struct CLISTAT
+{
+    /** The key is the cli address. */
+    AVLGCPTRNODECORE        Core;
+#if HC_ARCH_BITS == 32 && !defined(RT_OS_WINDOWS)
+    /** Padding. */
+    uint32_t                u32Padding;
+#endif
+    /** Occurrences. */
+    STAMCOUNTER             Counter;
+} CLISTAT, *PCLISTAT;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(CLISTAT, Counter, 8);
+#endif
+
+
+/**
+ * Excessive EM statistics.
+ */
+typedef struct EMSTATS
+{
+    /** GC: Profiling of EMInterpretInstruction(). */
+    STAMPROFILE             StatRZEmulate;
+    /** HC: Profiling of EMInterpretInstruction(). */
+    STAMPROFILE             StatR3Emulate;
+
+    /** @name Interpreter Instruction statistics.
+     * @{
+     */
+    STAMCOUNTER             StatRZInterpretSucceeded;
+    STAMCOUNTER             StatR3InterpretSucceeded;
+
+    STAMCOUNTER             StatRZAnd;
+    STAMCOUNTER             StatR3And;
+    STAMCOUNTER             StatRZCpuId;
+    STAMCOUNTER             StatR3CpuId;
+    STAMCOUNTER             StatRZDec;
+    STAMCOUNTER             StatR3Dec;
+    STAMCOUNTER             StatRZHlt;
+    STAMCOUNTER             StatR3Hlt;
+    STAMCOUNTER             StatRZInc;
+    STAMCOUNTER             StatR3Inc;
+    STAMCOUNTER             StatRZInvlPg;
+    STAMCOUNTER             StatR3InvlPg;
+    STAMCOUNTER             StatRZIret;
+    STAMCOUNTER             StatR3Iret;
+    STAMCOUNTER             StatRZLLdt;
+    STAMCOUNTER             StatR3LLdt;
+    STAMCOUNTER             StatRZLIdt;
+    STAMCOUNTER             StatR3LIdt;
+    STAMCOUNTER             StatRZLGdt;
+    STAMCOUNTER             StatR3LGdt;
+    STAMCOUNTER             StatRZMov;
+    STAMCOUNTER             StatR3Mov;
+    STAMCOUNTER             StatRZMovCRx;
+    STAMCOUNTER             StatR3MovCRx;
+    STAMCOUNTER             StatRZMovDRx;
+    STAMCOUNTER             StatR3MovDRx;
+    STAMCOUNTER             StatRZOr;
+    STAMCOUNTER             StatR3Or;
+    STAMCOUNTER             StatRZPop;
+    STAMCOUNTER             StatR3Pop;
+    STAMCOUNTER             StatRZSti;
+    STAMCOUNTER             StatR3Sti;
+    STAMCOUNTER             StatRZXchg;
+    STAMCOUNTER             StatR3Xchg;
+    STAMCOUNTER             StatRZXor;
+    STAMCOUNTER             StatR3Xor;
+    STAMCOUNTER             StatRZMonitor;
+    STAMCOUNTER             StatR3Monitor;
+    STAMCOUNTER             StatRZMWait;
+    STAMCOUNTER             StatR3MWait;
+    STAMCOUNTER             StatRZAdd;
+    STAMCOUNTER             StatR3Add;
+    STAMCOUNTER             StatRZSub;
+    STAMCOUNTER             StatR3Sub;
+    STAMCOUNTER             StatRZAdc;
+    STAMCOUNTER             StatR3Adc;
+    STAMCOUNTER             StatRZRdtsc;
+    STAMCOUNTER             StatR3Rdtsc;
+    STAMCOUNTER             StatRZRdpmc;
+    STAMCOUNTER             StatR3Rdpmc;
+    STAMCOUNTER             StatRZBtr;
+    STAMCOUNTER             StatR3Btr;
+    STAMCOUNTER             StatRZBts;
+    STAMCOUNTER             StatR3Bts;
+    STAMCOUNTER             StatRZBtc;
+    STAMCOUNTER             StatR3Btc;
+    STAMCOUNTER             StatRZCmpXchg;
+    STAMCOUNTER             StatR3CmpXchg;
+    STAMCOUNTER             StatRZCmpXchg8b;
+    STAMCOUNTER             StatR3CmpXchg8b;
+    STAMCOUNTER             StatRZXAdd;
+    STAMCOUNTER             StatR3XAdd;
+    STAMCOUNTER             StatRZClts;
+    STAMCOUNTER             StatR3Clts;
+    STAMCOUNTER             StatRZStosWD;
+    STAMCOUNTER             StatR3StosWD;
+    STAMCOUNTER             StatR3Rdmsr;
+    STAMCOUNTER             StatR3Wrmsr;
+    STAMCOUNTER             StatRZRdmsr;
+    STAMCOUNTER             StatRZWrmsr;
+    STAMCOUNTER             StatRZWbInvd;
+    STAMCOUNTER             StatR3WbInvd;
+    STAMCOUNTER             StatRZLmsw;
+    STAMCOUNTER             StatR3Lmsw;
+    STAMCOUNTER             StatRZSmsw;
+    STAMCOUNTER             StatR3Smsw;
+
+    STAMCOUNTER             StatRZInterpretFailed;
+    STAMCOUNTER             StatR3InterpretFailed;
+
+    STAMCOUNTER             StatRZFailedAnd;
+    STAMCOUNTER             StatR3FailedAnd;
+    STAMCOUNTER             StatRZFailedCpuId;
+    STAMCOUNTER             StatR3FailedCpuId;
+    STAMCOUNTER             StatRZFailedDec;
+    STAMCOUNTER             StatR3FailedDec;
+    STAMCOUNTER             StatRZFailedHlt;
+    STAMCOUNTER             StatR3FailedHlt;
+    STAMCOUNTER             StatRZFailedInc;
+    STAMCOUNTER             StatR3FailedInc;
+    STAMCOUNTER             StatRZFailedInvlPg;
+    STAMCOUNTER             StatR3FailedInvlPg;
+    STAMCOUNTER             StatRZFailedIret;
+    STAMCOUNTER             StatR3FailedIret;
+    STAMCOUNTER             StatRZFailedLLdt;
+    STAMCOUNTER             StatR3FailedLLdt;
+    STAMCOUNTER             StatRZFailedLGdt;
+    STAMCOUNTER             StatR3FailedLGdt;
+    STAMCOUNTER             StatRZFailedLIdt;
+    STAMCOUNTER             StatR3FailedLIdt;
+    STAMCOUNTER             StatRZFailedMisc;
+    STAMCOUNTER             StatR3FailedMisc;
+    STAMCOUNTER             StatRZFailedMov;
+    STAMCOUNTER             StatR3FailedMov;
+    STAMCOUNTER             StatRZFailedMovCRx;
+    STAMCOUNTER             StatR3FailedMovCRx;
+    STAMCOUNTER             StatRZFailedMovDRx;
+    STAMCOUNTER             StatR3FailedMovDRx;
+    STAMCOUNTER             StatRZFailedOr;
+    STAMCOUNTER             StatR3FailedOr;
+    STAMCOUNTER             StatRZFailedPop;
+    STAMCOUNTER             StatR3FailedPop;
+    STAMCOUNTER             StatRZFailedSti;
+    STAMCOUNTER             StatR3FailedSti;
+    STAMCOUNTER             StatRZFailedXchg;
+    STAMCOUNTER             StatR3FailedXchg;
+    STAMCOUNTER             StatRZFailedXor;
+    STAMCOUNTER             StatR3FailedXor;
+    STAMCOUNTER             StatRZFailedMonitor;
+    STAMCOUNTER             StatR3FailedMonitor;
+    STAMCOUNTER             StatRZFailedMWait;
+    STAMCOUNTER             StatR3FailedMWait;
+    STAMCOUNTER             StatR3FailedRdmsr;
+    STAMCOUNTER             StatR3FailedWrmsr;
+    STAMCOUNTER             StatRZFailedRdmsr;
+    STAMCOUNTER             StatRZFailedWrmsr;
+    STAMCOUNTER             StatRZFailedLmsw;
+    STAMCOUNTER             StatR3FailedLmsw;
+    STAMCOUNTER             StatRZFailedSmsw;
+    STAMCOUNTER             StatR3FailedSmsw;
+
+    STAMCOUNTER             StatRZFailedAdd;
+    STAMCOUNTER             StatR3FailedAdd;
+    STAMCOUNTER             StatRZFailedAdc;
+    STAMCOUNTER             StatR3FailedAdc;
+    STAMCOUNTER             StatRZFailedBtr;
+    STAMCOUNTER             StatR3FailedBtr;
+    STAMCOUNTER             StatRZFailedBts;
+    STAMCOUNTER             StatR3FailedBts;
+    STAMCOUNTER             StatRZFailedBtc;
+    STAMCOUNTER             StatR3FailedBtc;
+    STAMCOUNTER             StatRZFailedCli;
+    STAMCOUNTER             StatR3FailedCli;
+    STAMCOUNTER             StatRZFailedCmpXchg;
+    STAMCOUNTER             StatR3FailedCmpXchg;
+    STAMCOUNTER             StatRZFailedCmpXchg8b;
+    STAMCOUNTER             StatR3FailedCmpXchg8b;
+    STAMCOUNTER             StatRZFailedXAdd;
+    STAMCOUNTER             StatR3FailedXAdd;
+    STAMCOUNTER             StatR3FailedMovNTPS;
+    STAMCOUNTER             StatRZFailedMovNTPS;
+    STAMCOUNTER             StatRZFailedStosWD;
+    STAMCOUNTER             StatR3FailedStosWD;
+    STAMCOUNTER             StatRZFailedSub;
+    STAMCOUNTER             StatR3FailedSub;
+    STAMCOUNTER             StatRZFailedWbInvd;
+    STAMCOUNTER             StatR3FailedWbInvd;
+    STAMCOUNTER             StatRZFailedRdtsc;
+    STAMCOUNTER             StatR3FailedRdtsc;
+    STAMCOUNTER             StatRZFailedRdpmc;
+    STAMCOUNTER             StatR3FailedRdpmc;
+    STAMCOUNTER             StatRZFailedClts;
+    STAMCOUNTER             StatR3FailedClts;
+
+    STAMCOUNTER             StatRZFailedUserMode;
+    STAMCOUNTER             StatR3FailedUserMode;
+    STAMCOUNTER             StatRZFailedPrefix;
+    STAMCOUNTER             StatR3FailedPrefix;
+    /** @} */
+
+    /** @name Privileged Instructions Ending Up In HC.
+     * @{ */
+    STAMCOUNTER             StatCli;
+    STAMCOUNTER             StatSti;
+    STAMCOUNTER             StatIn;
+    STAMCOUNTER             StatIoRestarted;
+    STAMCOUNTER             StatOut;
+    STAMCOUNTER             StatInvlpg;
+    STAMCOUNTER             StatHlt;
+    STAMCOUNTER             StatMovReadCR[USE_REG_CR4 + 1];
+    STAMCOUNTER             StatMovWriteCR[USE_REG_CR4 + 1];
+    STAMCOUNTER             StatMovDRx;
+    STAMCOUNTER             StatIret;
+    STAMCOUNTER             StatMovLgdt;
+    STAMCOUNTER             StatMovLldt;
+    STAMCOUNTER             StatMovLidt;
+    STAMCOUNTER             StatMisc;
+    STAMCOUNTER             StatSysEnter;
+    STAMCOUNTER             StatSysExit;
+    STAMCOUNTER             StatSysCall;
+    STAMCOUNTER             StatSysRet;
+    /** @} */
+
+} EMSTATS;
+/** Pointer to the excessive EM statistics. */
+typedef EMSTATS *PEMSTATS;
+
+
+/**
+ * Converts a EM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the EM is part of.
+ * @param   pEM   Pointer to EM instance data.
+ */
+#define EM2VM(pEM)  ( (PVM)((char*)pEM - pEM->offVM) )
+
+/**
+ * EM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct EM
+{
+    /** Offset to the VM structure.
+     * See EM2VM(). */
+    RTUINT                  offVM;
+
+    /** Id of the VCPU that last executed code in the recompiler. */
+    VMCPUID                 idLastRemCpu;
+
+    /** REM critical section.
+     * This protects recompiler usage
+     */
+    PDMCRITSECT             CritSectREM;
+} EM;
+/** Pointer to EM VM instance data. */
+typedef EM *PEM;
+
+
+/**
+ * EM VMCPU Instance data.
+ */
+typedef struct EMCPU
+{
+    /** Offset to the VM structure.
+     * See EMCPU2VM(). */
+    RTUINT                  offVMCPU;
+
+    /** Execution Manager State. */
+    EMSTATE volatile        enmState;
+
+    /** Previous Execution Manager State. */
+    EMSTATE                 enmPrevState;
+
+    /** Force raw-mode execution.
+     * This is used to prevent REM from trying to execute patch code.
+     * The flag is cleared upon entering emR3RawExecute() and updated in certain return paths. */
+    bool                    fForceRAW;
+
+    uint8_t                 u8Padding[3];
+
+    /** Inhibit interrupts for this instruction. Valid only when VM_FF_INHIBIT_INTERRUPTS is set. */
+    RTGCUINTPTR             GCPtrInhibitInterrupts;
+
+    /** Pointer to the PATM status structure. (R3 Ptr) */
+    R3PTRTYPE(PPATMGCSTATE) pPatmGCState;
+
+    /** Pointer to the guest CPUM state. (R3 Ptr) */
+    R3PTRTYPE(PCPUMCTX)     pCtx;
+
+#if GC_ARCH_BITS == 64
+    RTGCPTR                 aPadding1;
+#endif
+
+    /** Start of the current time slice in ms. */
+    uint64_t                u64TimeSliceStart;
+    /** Start of the current time slice in thread execution time (ms). */
+    uint64_t                u64TimeSliceStartExec;
+    /** Current time slice value. */
+    uint64_t                u64TimeSliceExec;
+    uint64_t                u64Alignment;
+
+    /* MWait halt state. */
+    struct
+    {
+        uint32_t            fWait;          /* type of mwait; see EMMWAIT_FLAG_* */
+        uint32_t            a32Padding[1];
+        RTGCPTR             uMWaitEAX;      /* mwait hints */
+        RTGCPTR             uMWaitECX;      /* mwait extensions */
+        RTGCPTR             uMonitorEAX;    /* monitored address. */
+        RTGCPTR             uMonitorECX;    /* monitor extension. */
+        RTGCPTR             uMonitorEDX;    /* monitor hint. */
+    } mwait;
+
+    union
+    {
+        /** Padding used in the other rings.
+         * This must be larger than jmp_buf on any supported platform. */
+        char                achPaddingFatalLongJump[HC_ARCH_BITS == 32 ? 176 : 256];
+#ifdef IN_RING3
+        /** Long buffer jump for fatal VM errors.
+         * It will jump to before the outer EM loop is entered. */
+        jmp_buf             FatalLongJump;
+#endif
+    } u;
+
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack.
+     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
+    union
+    {
+        /** The disassembler scratch space. */
+        DISCPUSTATE         DisState;
+        /** Padding. */
+        uint8_t             abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
+    };
+
+    /** @name Execution profiling.
+     * @{ */
+    STAMPROFILE             StatForcedActions;
+    STAMPROFILE             StatHalted;
+    STAMPROFILEADV          StatCapped;
+    STAMPROFILEADV          StatHwAccEntry;
+    STAMPROFILE             StatHwAccExec;
+    STAMPROFILE             StatREMEmu;
+    STAMPROFILE             StatREMExec;
+    STAMPROFILE             StatREMSync;
+    STAMPROFILEADV          StatREMTotal;
+    STAMPROFILE             StatRAWExec;
+    STAMPROFILEADV          StatRAWEntry;
+    STAMPROFILEADV          StatRAWTail;
+    STAMPROFILEADV          StatRAWTotal;
+    STAMPROFILEADV          StatTotal;
+    /** @} */
+
+    /** R3: Profiling of emR3RawExecuteIOInstruction. */
+    STAMPROFILE             StatIOEmu;
+    /** R3: Profiling of emR3RawPrivileged. */
+    STAMPROFILE             StatPrivEmu;
+    /** R3: Number of time emR3HwAccExecute is called. */
+    STAMCOUNTER             StatHwAccExecuteEntry;
+
+    /** More statistics (R3). */
+    R3PTRTYPE(PEMSTATS)     pStatsR3;
+    /** More statistics (R0). */
+    R0PTRTYPE(PEMSTATS)     pStatsR0;
+    /** More statistics (RC). */
+    RCPTRTYPE(PEMSTATS)     pStatsRC;
+#if HC_ARCH_BITS == 64
+    RTRCPTR                 padding0;
+#endif
+
+    /** Tree for keeping track of cli occurrences (debug only). */
+    R3PTRTYPE(PAVLGCPTRNODECORE) pCliStatTree;
+    STAMCOUNTER             StatTotalClis;
+#if 0
+    /** 64-bit Visual C++ rounds the struct size up to 16 byte. */
+    uint64_t                padding1;
+#endif
+} EMCPU;
+/** Pointer to EM VM instance data. */
+typedef EMCPU *PEMCPU;
+
+/** @} */
+
+
+int     emR3HwAccExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
+int     emR3RawExecute(PVM pVM, PVMCPU pVCpu, bool *pfFFDone);
+int     emR3RawHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
+int     emR3HwaccmHandleRC(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
+EMSTATE emR3Reschedule(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
+int     emR3ForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
+int     emR3HighPriorityPostForcedActions(PVM pVM, PVMCPU pVCpu, int rc);
+int     emR3RawUpdateForceFlag(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rc);
+int     emR3RawResumeHyper(PVM pVM, PVMCPU pVCpu);
+int     emR3RawStep(PVM pVM, PVMCPU pVCpu);
+int     emR3SingleStepExecRem(PVM pVM, PVMCPU pVCpu, uint32_t cIterations);
+
+RT_C_DECLS_END
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/FTMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/FTMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/FTMInternal.h	(revision 35333)
@@ -0,0 +1,125 @@
+/* $Id$ */
+/** @file
+ * FTM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___FTMInternal_h
+#define ___FTMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/ftm.h>
+#include <VBox/stam.h>
+#include <VBox/pdmcritsect.h>
+#include <iprt/avl.h>
+
+/** @defgroup grp_ftm_int Internals.
+ * @ingroup grp_ftm
+ * @{
+ */
+
+/** Physical page tree node. */
+typedef struct FTMPHYSPAGETREENODE
+{
+    AVLGCPHYSNODECORE   Core;
+    void               *pPage;
+} FTMPHYSPAGETREENODE;
+/** Pointer to FTMPHYSPAGETREENODE */
+typedef FTMPHYSPAGETREENODE *PFTMPHYSPAGETREENODE;
+
+/**
+ * FTM VM Instance data.
+ * Changes to this must checked against the padding of the ftm union in VM!
+ */
+typedef struct FTM
+{
+    /** Address of the standby VM. */
+    R3PTRTYPE(char *)   pszAddress;
+    /** Password to access the syncing server of the standby VM. */
+    R3PTRTYPE(char *)   pszPassword;
+    /** Port of the standby VM. */
+    unsigned            uPort;
+    /** Syncing interval in ms. */
+    unsigned            uInterval;
+
+    /** Set when this VM is the standby FT node. */
+    bool                fIsStandbyNode;
+    /** Set when this master VM is busy with checkpointing. */
+    bool                fCheckpointingActive;
+    /** Set when VM save/restore should only include changed pages. */
+    bool                fDeltaLoadSaveActive;
+    /** Fallover to the standby VM. */
+    bool                fActivateStandby;
+    bool                fAlignment[4];
+
+    /** Current active socket. */
+    RTSOCKET            hSocket;
+
+    /* Shutdown event semaphore. */
+    RTSEMEVENT          hShutdownEvent;
+
+    /** State sync. */
+    struct
+    {
+        unsigned            uOffStream;
+        unsigned            cbReadBlock;
+        bool                fStopReading;
+        bool                fIOError;
+        bool                fEndOfStream;
+        bool                fAlignment[5];
+    } syncstate;
+
+    struct
+    {
+        R3PTRTYPE(PRTTCPSERVER)    hServer;
+        R3PTRTYPE(AVLGCPHYSTREE)   pPhysPageTree;
+        uint64_t                   u64LastHeartbeat;
+    } standby;
+
+    /*
+    struct
+    {
+    } master;
+    */
+
+    /** FTM critical section.
+     * This makes sure only the checkpoint or sync is active
+     */
+    PDMCRITSECT         CritSect;
+
+    STAMCOUNTER         StatReceivedMem;
+    STAMCOUNTER         StatReceivedState;
+    STAMCOUNTER         StatSentMem;
+    STAMCOUNTER         StatSentState;
+    STAMCOUNTER         StatDeltaMem;
+    STAMCOUNTER         StatDeltaVM;
+    STAMCOUNTER         StatFullSync;
+    STAMCOUNTER         StatCheckpointNetwork;
+    STAMCOUNTER         StatCheckpointStorage;
+#ifdef VBOX_WITH_STATISTICS
+    STAMPROFILE         StatCheckpoint;
+    STAMPROFILE         StatCheckpointResume;
+    STAMPROFILE         StatCheckpointPause;
+    STAMCOUNTER         StatSentMemRAM;
+    STAMCOUNTER         StatSentMemMMIO2;
+    STAMCOUNTER         StatSentMemShwROM;
+    STAMCOUNTER         StatSentStateWrite;
+#endif
+} FTM;
+AssertCompileMemberAlignment(FTM, CritSect, 8);
+
+/** @} */
+
+#endif
Index: /trunk/src/VBox/VMM/include/HWACCMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/HWACCMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/HWACCMInternal.h	(revision 35333)
@@ -0,0 +1,936 @@
+/* $Id$ */
+/** @file
+ * HWACCM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___HWACCMInternal_h
+#define ___HWACCMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/em.h>
+#include <VBox/stam.h>
+#include <VBox/dis.h>
+#include <VBox/hwaccm.h>
+#include <VBox/hwacc_vmx.h>
+#include <VBox/pgm.h>
+#include <VBox/cpum.h>
+#include <iprt/memobj.h>
+#include <iprt/cpuset.h>
+#include <iprt/mp.h>
+#include <iprt/avl.h>
+
+#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) || defined (VBOX_WITH_64_BITS_GUESTS)
+/* Enable 64 bits guest support. */
+# define VBOX_ENABLE_64_BITS_GUESTS
+#endif
+
+#define VMX_USE_CACHED_VMCS_ACCESSES
+#define HWACCM_VMX_EMULATE_REALMODE
+#define HWACCM_VTX_WITH_EPT
+#define HWACCM_VTX_WITH_VPID
+
+
+#if 0
+/* Seeing somewhat random behaviour on my Nehalem system with auto-save of guest MSRs;
+ * for some strange reason the CPU doesn't save the MSRs during the VM-exit.
+ * Clearly visible with a dual VCPU configured OpenSolaris 200906 live cd VM.
+ *
+ * Note: change the assembly files when enabling this! (remove the manual auto load/save)
+ */
+#define VBOX_WITH_AUTO_MSR_LOAD_RESTORE
+#endif
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_hwaccm_int       Internal
+ * @ingroup grp_hwaccm
+ * @internal
+ * @{
+ */
+
+
+/** Maximum number of exit reason statistics counters. */
+#define MAX_EXITREASON_STAT        0x100
+#define MASK_EXITREASON_STAT       0xff
+#define MASK_INJECT_IRQ_STAT       0xff
+
+/** @name Changed flags
+ * These flags are used to keep track of which important registers that
+ * have been changed since last they were reset.
+ * @{
+ */
+#define HWACCM_CHANGED_GUEST_FPU                RT_BIT(0)
+#define HWACCM_CHANGED_GUEST_CR0                RT_BIT(1)
+#define HWACCM_CHANGED_GUEST_CR3                RT_BIT(2)
+#define HWACCM_CHANGED_GUEST_CR4                RT_BIT(3)
+#define HWACCM_CHANGED_GUEST_GDTR               RT_BIT(4)
+#define HWACCM_CHANGED_GUEST_IDTR               RT_BIT(5)
+#define HWACCM_CHANGED_GUEST_LDTR               RT_BIT(6)
+#define HWACCM_CHANGED_GUEST_TR                 RT_BIT(7)
+#define HWACCM_CHANGED_GUEST_MSR                RT_BIT(8)
+#define HWACCM_CHANGED_GUEST_SEGMENT_REGS       RT_BIT(9)
+#define HWACCM_CHANGED_GUEST_DEBUG              RT_BIT(10)
+#define HWACCM_CHANGED_HOST_CONTEXT             RT_BIT(11)
+
+#define HWACCM_CHANGED_ALL                  (   HWACCM_CHANGED_GUEST_SEGMENT_REGS \
+                                            |   HWACCM_CHANGED_GUEST_CR0          \
+                                            |   HWACCM_CHANGED_GUEST_CR3          \
+                                            |   HWACCM_CHANGED_GUEST_CR4          \
+                                            |   HWACCM_CHANGED_GUEST_GDTR         \
+                                            |   HWACCM_CHANGED_GUEST_IDTR         \
+                                            |   HWACCM_CHANGED_GUEST_LDTR         \
+                                            |   HWACCM_CHANGED_GUEST_TR           \
+                                            |   HWACCM_CHANGED_GUEST_MSR          \
+                                            |   HWACCM_CHANGED_GUEST_FPU          \
+                                            |   HWACCM_CHANGED_GUEST_DEBUG        \
+                                            |   HWACCM_CHANGED_HOST_CONTEXT)
+
+#define HWACCM_CHANGED_ALL_GUEST            (   HWACCM_CHANGED_GUEST_SEGMENT_REGS \
+                                            |   HWACCM_CHANGED_GUEST_CR0          \
+                                            |   HWACCM_CHANGED_GUEST_CR3          \
+                                            |   HWACCM_CHANGED_GUEST_CR4          \
+                                            |   HWACCM_CHANGED_GUEST_GDTR         \
+                                            |   HWACCM_CHANGED_GUEST_IDTR         \
+                                            |   HWACCM_CHANGED_GUEST_LDTR         \
+                                            |   HWACCM_CHANGED_GUEST_TR           \
+                                            |   HWACCM_CHANGED_GUEST_MSR          \
+                                            |   HWACCM_CHANGED_GUEST_DEBUG        \
+                                            |   HWACCM_CHANGED_GUEST_FPU)
+
+/** @} */
+
+/** @name Intercepted traps
+ *  Traps that need to be intercepted so we can correctly dispatch them to the guest if required.
+ *  Currently #NM and #PF only
+ */
+#ifdef VBOX_STRICT
+#define HWACCM_VMX_TRAP_MASK                RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_PF) | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_MF)
+#define HWACCM_SVM_TRAP_MASK                HWACCM_VMX_TRAP_MASK
+#else
+#define HWACCM_VMX_TRAP_MASK                RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_PF)
+#define HWACCM_SVM_TRAP_MASK                RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_PF)
+#endif
+/* All exceptions have to be intercept in emulated real-mode (minus NM & PF as they are always intercepted. */
+#define HWACCM_VMX_TRAP_MASK_REALMODE       RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_DF) | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) | RT_BIT(X86_XCPT_XF)
+/** @} */
+
+
+/** Maximum number of page flushes we are willing to remember before considering a full TLB flush. */
+#define HWACCM_MAX_TLB_SHOOTDOWN_PAGES      8
+
+/** Size for the EPT identity page table (1024 4 MB pages to cover the entire address space). */
+#define HWACCM_EPT_IDENTITY_PG_TABLE_SIZE   PAGE_SIZE
+/** Size of the TSS structure + 2 pages for the IO bitmap + end byte. */
+#define HWACCM_VTX_TSS_SIZE                 (sizeof(VBOXTSS) + 2*PAGE_SIZE + 1)
+/** Total guest mapped memory needed. */
+#define HWACCM_VTX_TOTAL_DEVHEAP_MEM        (HWACCM_EPT_IDENTITY_PG_TABLE_SIZE + HWACCM_VTX_TSS_SIZE)
+
+/* Enable for TPR guest patching. */
+#define VBOX_HWACCM_WITH_GUEST_PATCHING
+
+/** HWACCM SSM version
+ */
+#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
+#define HWACCM_SSM_VERSION                  5
+#define HWACCM_SSM_VERSION_NO_PATCHING      4
+#else
+#define HWACCM_SSM_VERSION                  4
+#define HWACCM_SSM_VERSION_NO_PATCHING      4
+#endif
+#define HWACCM_SSM_VERSION_2_0_X            3
+
+/* Per-cpu information. (host) */
+typedef struct
+{
+    RTCPUID             idCpu;
+
+    RTR0MEMOBJ          pMemObj;
+    /* Current ASID (AMD-V)/VPID (Intel) */
+    uint32_t            uCurrentASID;
+    /* TLB flush count */
+    uint32_t            cTLBFlushes;
+
+    /* Set the first time a cpu is used to make sure we start with a clean TLB. */
+    bool                fFlushTLB;
+
+    /** Configured for VT-x or AMD-V. */
+    bool                fConfigured;
+
+    /** Set if the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE hack is active. */
+    bool                fIgnoreAMDVInUseError;
+
+    /** In use by our code. (for power suspend) */
+    volatile bool       fInUse;
+} HWACCM_CPUINFO;
+typedef HWACCM_CPUINFO *PHWACCM_CPUINFO;
+
+typedef enum
+{
+    HWACCMPENDINGIO_INVALID = 0,
+    HWACCMPENDINGIO_PORT_READ,
+    HWACCMPENDINGIO_PORT_WRITE,
+    HWACCMPENDINGIO_STRING_READ,
+    HWACCMPENDINGIO_STRING_WRITE,
+    /** The usual 32-bit paranoia. */
+    HWACCMPENDINGIO_32BIT_HACK   = 0x7fffffff
+} HWACCMPENDINGIO;
+
+
+typedef enum
+{
+    HWACCMTPRINSTR_INVALID,
+    HWACCMTPRINSTR_READ,
+    HWACCMTPRINSTR_READ_SHR4,
+    HWACCMTPRINSTR_WRITE_REG,
+    HWACCMTPRINSTR_WRITE_IMM,
+    HWACCMTPRINSTR_JUMP_REPLACEMENT,
+    /** The usual 32-bit paranoia. */
+    HWACCMTPRINSTR_32BIT_HACK   = 0x7fffffff
+} HWACCMTPRINSTR;
+
+typedef struct
+{
+    /** The key is the address of patched instruction. (32 bits GC ptr) */
+    AVLOU32NODECORE         Core;
+    /** Original opcode. */
+    uint8_t                 aOpcode[16];
+    /** Instruction size. */
+    uint32_t                cbOp;
+    /** Replacement opcode. */
+    uint8_t                 aNewOpcode[16];
+    /** Replacement instruction size. */
+    uint32_t                cbNewOp;
+    /** Instruction type. */
+    HWACCMTPRINSTR          enmType;
+    /** Source operand. */
+    uint32_t                uSrcOperand;
+    /** Destination operand. */
+    uint32_t                uDstOperand;
+    /** Number of times the instruction caused a fault. */
+    uint32_t                cFaults;
+    /** Patch address of the jump replacement. */
+    RTGCPTR32               pJumpTarget;
+} HWACCMTPRPATCH;
+/** Pointer to HWACCMTPRPATCH. */
+typedef HWACCMTPRPATCH *PHWACCMTPRPATCH;
+
+/**
+ * Switcher function, HC to RC.
+ *
+ * @param   pVM             The VM handle.
+ * @param   uOffsetVMCPU    VMCPU offset from pVM
+ * @returns Return code indicating the action to take.
+ */
+typedef DECLCALLBACK (int) FNHWACCMSWITCHERHC(PVM pVM, uint32_t uOffsetVMCPU);
+/** Pointer to switcher function. */
+typedef FNHWACCMSWITCHERHC *PFNHWACCMSWITCHERHC;
+
+/**
+ * HWACCM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct HWACCM
+{
+    /** Set when we've initialized VMX or SVM. */
+    bool                        fInitialized;
+
+    /** Set when hardware acceleration is allowed. */
+    bool                        fAllowed;
+
+    /** Set if nested paging is enabled. */
+    bool                        fNestedPaging;
+
+    /** Set if nested paging is allowed. */
+    bool                        fAllowNestedPaging;
+
+    /** Set if large pages are enabled (requires nested paging). */
+    bool                        fLargePages;
+
+    /** Set if we can support 64-bit guests or not. */
+    bool                        fAllow64BitGuests;
+
+    /** Set if an IO-APIC is configured for this VM. */
+    bool                        fHasIoApic;
+
+    /** Set when TPR patching is allowed. */
+    bool                        fTRPPatchingAllowed;
+
+    /** Set when we initialize VT-x or AMD-V once for all CPUs. */
+    bool                        fGlobalInit;
+
+    /** Set when TPR patching is active. */
+    bool                        fTPRPatchingActive;
+    bool                        u8Alignment[6];
+
+    /** And mask for copying register contents. */
+    uint64_t                    u64RegisterMask;
+
+    /** Maximum ASID allowed. */
+    RTUINT                      uMaxASID;
+
+    /** The maximum number of resumes loops allowed in ring-0 (safety precaution).
+     * This number is set much higher when RTThreadPreemptIsPending is reliable. */
+    uint32_t                    cMaxResumeLoops;
+
+    /** Guest allocated memory for patching purposes. */
+    RTGCPTR                     pGuestPatchMem;
+    /** Current free pointer inside the patch block. */
+    RTGCPTR                     pFreeGuestPatchMem;
+    /** Size of the guest patch memory block. */
+    uint32_t                    cbGuestPatchMem;
+    uint32_t                    uPadding1;
+
+#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    /** 32 to 64 bits switcher entrypoint. */
+    R0PTRTYPE(PFNHWACCMSWITCHERHC) pfnHost32ToGuest64R0;
+
+    /* AMD-V 64 bits vmrun handler */
+    RTRCPTR                     pfnSVMGCVMRun64;
+
+    /* VT-x 64 bits vmlaunch handler */
+    RTRCPTR                     pfnVMXGCStartVM64;
+
+    /* RC handler to setup the 64 bits FPU state. */
+    RTRCPTR                     pfnSaveGuestFPU64;
+
+    /* RC handler to setup the 64 bits debug state. */
+    RTRCPTR                     pfnSaveGuestDebug64;
+
+    /* Test handler */
+    RTRCPTR                     pfnTest64;
+
+    RTRCPTR                     uAlignment[2];
+/*#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    uint32_t                    u32Alignment[1]; */
+#endif
+
+    struct
+    {
+        /** Set by the ring-0 side of HWACCM to indicate VMX is supported by the
+         *  CPU. */
+        bool                        fSupported;
+
+        /** Set when we've enabled VMX. */
+        bool                        fEnabled;
+
+        /** Set if VPID is supported. */
+        bool                        fVPID;
+
+        /** Set if VT-x VPID is allowed. */
+        bool                        fAllowVPID;
+
+        /** Set if unrestricted guest execution is allowed (real and protected mode without paging). */
+        bool                        fUnrestrictedGuest;
+
+        /** Whether we're using the preemption timer or not. */
+        bool                        fUsePreemptTimer;
+        /** The shift mask employed by the VMX-Preemption timer. */
+        uint8_t                     cPreemptTimerShift;
+
+        bool                        uAlignment[1];
+
+        /** Virtual address of the TSS page used for real mode emulation. */
+        R3PTRTYPE(PVBOXTSS)         pRealModeTSS;
+
+        /** Virtual address of the identity page table used for real mode and protected mode without paging emulation in EPT mode. */
+        R3PTRTYPE(PX86PD)           pNonPagingModeEPTPageTable;
+
+        /** R0 memory object for the APIC physical page (serves for filtering accesses). */
+        RTR0MEMOBJ                  pMemObjAPIC;
+        /** Physical address of the APIC physical page (serves for filtering accesses). */
+        RTHCPHYS                    pAPICPhys;
+        /** Virtual address of the APIC physical page (serves for filtering accesses). */
+        R0PTRTYPE(uint8_t *)        pAPIC;
+
+        /** R0 memory object for the MSR entry load page (guest MSRs). */
+        RTR0MEMOBJ                  pMemObjMSREntryLoad;
+        /** Physical address of the MSR entry load page (guest MSRs). */
+        RTHCPHYS                    pMSREntryLoadPhys;
+        /** Virtual address of the MSR entry load page (guest MSRs). */
+        R0PTRTYPE(uint8_t *)        pMSREntryLoad;
+
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+        RTR0MEMOBJ                  pMemObjScratch;
+        RTHCPHYS                    pScratchPhys;
+        R0PTRTYPE(uint8_t *)        pScratch;
+#endif
+        /** R0 memory object for the MSR exit store page (guest MSRs). */
+        RTR0MEMOBJ                  pMemObjMSRExitStore;
+        /** Physical address of the MSR exit store page (guest MSRs). */
+        RTHCPHYS                    pMSRExitStorePhys;
+        /** Virtual address of the MSR exit store page (guest MSRs). */
+        R0PTRTYPE(uint8_t *)        pMSRExitStore;
+
+        /** R0 memory object for the MSR exit load page (host MSRs). */
+        RTR0MEMOBJ                  pMemObjMSRExitLoad;
+        /** Physical address of the MSR exit load page (host MSRs). */
+        RTHCPHYS                    pMSRExitLoadPhys;
+        /** Virtual address of the MSR exit load page (host MSRs). */
+        R0PTRTYPE(uint8_t *)        pMSRExitLoad;
+
+        /** Ring 0 handlers for VT-x. */
+        DECLR0CALLBACKMEMBER(void, pfnSetupTaggedTLB, (PVM pVM, PVMCPU pVCpu));
+
+#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
+        uint32_t                    u32Alignment;
+#endif
+        /** Host CR4 value (set by ring-0 VMX init) */
+        uint64_t                    hostCR4;
+
+        /** Host EFER value (set by ring-0 VMX init) */
+        uint64_t                    hostEFER;
+
+        /** VMX MSR values */
+        struct
+        {
+            uint64_t                feature_ctrl;
+            uint64_t                vmx_basic_info;
+            VMX_CAPABILITY          vmx_pin_ctls;
+            VMX_CAPABILITY          vmx_proc_ctls;
+            VMX_CAPABILITY          vmx_proc_ctls2;
+            VMX_CAPABILITY          vmx_exit;
+            VMX_CAPABILITY          vmx_entry;
+            uint64_t                vmx_misc;
+            uint64_t                vmx_cr0_fixed0;
+            uint64_t                vmx_cr0_fixed1;
+            uint64_t                vmx_cr4_fixed0;
+            uint64_t                vmx_cr4_fixed1;
+            uint64_t                vmx_vmcs_enum;
+            uint64_t                vmx_eptcaps;
+        } msr;
+
+        /** Flush types for invept & invvpid; they depend on capabilities. */
+        VMX_FLUSH                   enmFlushPage;
+        VMX_FLUSH                   enmFlushContext;
+    } vmx;
+
+    struct
+    {
+        /** Set by the ring-0 side of HWACCM to indicate SVM is supported by the
+         *  CPU. */
+        bool                        fSupported;
+        /** Set when we've enabled SVM. */
+        bool                        fEnabled;
+        /** Set if erratum 170 affects the AMD cpu. */
+        bool                        fAlwaysFlushTLB;
+        /** Set when the hack to ignore VERR_SVM_IN_USE is active. */
+        bool                        fIgnoreInUseError;
+
+        /** R0 memory object for the IO bitmap (12kb). */
+        RTR0MEMOBJ                  pMemObjIOBitmap;
+        /** Physical address of the IO bitmap (12kb). */
+        RTHCPHYS                    pIOBitmapPhys;
+        /** Virtual address of the IO bitmap. */
+        R0PTRTYPE(void *)           pIOBitmap;
+
+        /* HWCR msr (for diagnostics) */
+        uint64_t                    msrHWCR;
+
+        /** SVM revision. */
+        uint32_t                    u32Rev;
+
+        /** SVM feature bits from cpuid 0x8000000a */
+        uint32_t                    u32Features;
+    } svm;
+
+    /**
+     * AVL tree with all patches (active or disabled) sorted by guest instruction address
+     */
+    AVLOU32TREE                     PatchTree;
+    uint32_t                        cPatches;
+    HWACCMTPRPATCH                  aPatches[64];
+
+    struct
+    {
+        uint32_t                    u32AMDFeatureECX;
+        uint32_t                    u32AMDFeatureEDX;
+    } cpuid;
+
+    /** Saved error from detection */
+    int32_t                 lLastError;
+
+    /** HWACCMR0Init was run */
+    bool                    fHWACCMR0Init;
+    bool                    u8Alignment1[7];
+
+    STAMCOUNTER             StatTPRPatchSuccess;
+    STAMCOUNTER             StatTPRPatchFailure;
+    STAMCOUNTER             StatTPRReplaceSuccess;
+    STAMCOUNTER             StatTPRReplaceFailure;
+} HWACCM;
+/** Pointer to HWACCM VM instance data. */
+typedef HWACCM *PHWACCM;
+
+/* Maximum number of cached entries. */
+#define VMCSCACHE_MAX_ENTRY                             128
+
+/* Structure for storing read and write VMCS actions. */
+typedef struct VMCSCACHE
+{
+#ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    /* Magic marker for searching in crash dumps. */
+    uint8_t         aMagic[16];
+    uint64_t        uMagic;
+    uint64_t        u64TimeEntry;
+    uint64_t        u64TimeSwitch;
+    uint64_t        cResume;
+    uint64_t        interPD;
+    uint64_t        pSwitcher;
+    uint32_t        uPos;
+    uint32_t        idCpu;
+#endif
+    /* CR2 is saved here for EPT syncing. */
+    uint64_t        cr2;
+    struct
+    {
+        uint32_t    cValidEntries;
+        uint32_t    uAlignment;
+        uint32_t    aField[VMCSCACHE_MAX_ENTRY];
+        uint64_t    aFieldVal[VMCSCACHE_MAX_ENTRY];
+    } Write;
+    struct
+    {
+        uint32_t    cValidEntries;
+        uint32_t    uAlignment;
+        uint32_t    aField[VMCSCACHE_MAX_ENTRY];
+        uint64_t    aFieldVal[VMCSCACHE_MAX_ENTRY];
+    } Read;
+#ifdef DEBUG
+    struct
+    {
+        RTHCPHYS    pPageCpuPhys;
+        RTHCPHYS    pVMCSPhys;
+        RTGCPTR     pCache;
+        RTGCPTR     pCtx;
+    } TestIn;
+    struct
+    {
+        RTHCPHYS    pVMCSPhys;
+        RTGCPTR     pCache;
+        RTGCPTR     pCtx;
+        uint64_t    eflags;
+        uint64_t    cr8;
+    } TestOut;
+    struct
+    {
+        uint64_t    param1;
+        uint64_t    param2;
+        uint64_t    param3;
+        uint64_t    param4;
+    } ScratchPad;
+#endif
+} VMCSCACHE;
+/** Pointer to VMCSCACHE. */
+typedef VMCSCACHE *PVMCSCACHE;
+
+/** VMX StartVM function. */
+typedef DECLCALLBACK(int) FNHWACCMVMXSTARTVM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu);
+/** Pointer to a VMX StartVM function. */
+typedef R0PTRTYPE(FNHWACCMVMXSTARTVM *) PFNHWACCMVMXSTARTVM;
+
+/** SVM VMRun function. */
+typedef DECLCALLBACK(int) FNHWACCMSVMVMRUN(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu);
+/** Pointer to a SVM VMRun function. */
+typedef R0PTRTYPE(FNHWACCMSVMVMRUN *) PFNHWACCMSVMVMRUN;
+
+/**
+ * HWACCM VMCPU Instance data.
+ */
+typedef struct HWACCMCPU
+{
+    /** Old style FPU reporting trap mask override performed (optimization) */
+    bool                        fFPUOldStyleOverride;
+
+    /** Set if we don't have to flush the TLB on VM entry. */
+    bool                        fResumeVM;
+
+    /** Set if we need to flush the TLB during the world switch. */
+    bool                        fForceTLBFlush;
+
+    /** Set when we're using VT-x or AMD-V at that moment. */
+    bool                        fActive;
+
+    /** Set when the TLB has been checked until we return from the world switch. */
+    volatile uint8_t            fCheckedTLBFlush;
+    uint8_t                     bAlignment[3];
+
+    /** HWACCM_CHANGED_* flags. */
+    RTUINT                      fContextUseFlags;
+
+    /** Id of the last cpu we were executing code on (NIL_RTCPUID for the first time) */
+    RTCPUID                     idLastCpu;
+
+    /** TLB flush count */
+    RTUINT                      cTLBFlushes;
+
+    /** Current ASID in use by the VM */
+    RTUINT                      uCurrentASID;
+
+    /** World switch exit counter. */
+    volatile uint32_t           cWorldSwitchExit;
+    uint32_t                    u32Alignment;
+
+    struct
+    {
+        /** Physical address of the VM control structure (VMCS). */
+        RTHCPHYS                    pVMCSPhys;
+        /** R0 memory object for the VM control structure (VMCS). */
+        RTR0MEMOBJ                  pMemObjVMCS;
+        /** Virtual address of the VM control structure (VMCS). */
+        R0PTRTYPE(void *)           pVMCS;
+
+        /** Ring 0 handlers for VT-x. */
+        PFNHWACCMVMXSTARTVM         pfnStartVM;
+
+#if HC_ARCH_BITS == 32
+        uint32_t                    u32Alignment;
+#endif
+
+        /** Current VMX_VMCS_CTRL_PROC_EXEC_CONTROLS. */
+        uint64_t                    proc_ctls;
+
+        /** Current VMX_VMCS_CTRL_PROC_EXEC2_CONTROLS. */
+        uint64_t                    proc_ctls2;
+
+        /** Physical address of the virtual APIC page for TPR caching. */
+        RTHCPHYS                    pVAPICPhys;
+        /** R0 memory object for the virtual APIC page for TPR caching. */
+        RTR0MEMOBJ                  pMemObjVAPIC;
+        /** Virtual address of the virtual APIC page for TPR caching. */
+        R0PTRTYPE(uint8_t *)        pVAPIC;
+
+        /** Current CR0 mask. */
+        uint64_t                    cr0_mask;
+        /** Current CR4 mask. */
+        uint64_t                    cr4_mask;
+
+        /** Current EPTP. */
+        RTHCPHYS                    GCPhysEPTP;
+
+        /** Physical address of the MSR bitmap (1 page). */
+        RTHCPHYS                    pMSRBitmapPhys;
+        /** R0 memory object for the MSR bitmap (1 page). */
+        RTR0MEMOBJ                  pMemObjMSRBitmap;
+        /** Virtual address of the MSR bitmap (1 page). */
+        R0PTRTYPE(uint8_t *)        pMSRBitmap;
+
+#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
+        /** Physical address of the guest MSR load area (1 page). */
+        RTHCPHYS                    pGuestMSRPhys;
+        /** R0 memory object for the guest MSR load area (1 page). */
+        RTR0MEMOBJ                  pMemObjGuestMSR;
+        /** Virtual address of the guest MSR load area (1 page). */
+        R0PTRTYPE(uint8_t *)        pGuestMSR;
+
+        /** Physical address of the MSR load area (1 page). */
+        RTHCPHYS                    pHostMSRPhys;
+        /** R0 memory object for the MSR load area (1 page). */
+        RTR0MEMOBJ                  pMemObjHostMSR;
+        /** Virtual address of the MSR load area (1 page). */
+        R0PTRTYPE(uint8_t *)        pHostMSR;
+#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
+
+        /* Number of automatically loaded/restored MSRs. */
+        uint32_t                    cCachedMSRs;
+        uint32_t                    uAlignement;
+
+        /* Last use TSC offset value. (cached) */
+        uint64_t                    u64TSCOffset;
+
+        /** VMCS cache. */
+        VMCSCACHE                   VMCSCache;
+
+        /** Real-mode emulation state. */
+        struct
+        {
+            X86EFLAGS                   eflags;
+            uint32_t                    fValid;
+        } RealMode;
+
+        struct
+        {
+            uint64_t                u64VMCSPhys;
+            uint32_t                ulVMCSRevision;
+            uint32_t                ulInstrError;
+            uint32_t                ulExitReason;
+            RTCPUID                 idEnteredCpu;
+            RTCPUID                 idCurrentCpu;
+            uint32_t                padding;
+        } lasterror;
+
+        /** The last seen guest paging mode (by VT-x). */
+        PGMMODE                     enmLastSeenGuestMode;
+        /** Current guest paging mode (as seen by HWACCMR3PagingModeChanged). */
+        PGMMODE                     enmCurrGuestMode;
+        /** Previous guest paging mode (as seen by HWACCMR3PagingModeChanged). */
+        PGMMODE                     enmPrevGuestMode;
+    } vmx;
+
+    struct
+    {
+        /** R0 memory object for the host VM control block (VMCB). */
+        RTR0MEMOBJ                  pMemObjVMCBHost;
+        /** Physical address of the host VM control block (VMCB). */
+        RTHCPHYS                    pVMCBHostPhys;
+        /** Virtual address of the host VM control block (VMCB). */
+        R0PTRTYPE(void *)           pVMCBHost;
+
+        /** R0 memory object for the VM control block (VMCB). */
+        RTR0MEMOBJ                  pMemObjVMCB;
+        /** Physical address of the VM control block (VMCB). */
+        RTHCPHYS                    pVMCBPhys;
+        /** Virtual address of the VM control block (VMCB). */
+        R0PTRTYPE(void *)           pVMCB;
+
+        /** Ring 0 handlers for VT-x. */
+        PFNHWACCMSVMVMRUN           pfnVMRun;
+
+        /** R0 memory object for the MSR bitmap (8kb). */
+        RTR0MEMOBJ                  pMemObjMSRBitmap;
+        /** Physical address of the MSR bitmap (8kb). */
+        RTHCPHYS                    pMSRBitmapPhys;
+        /** Virtual address of the MSR bitmap. */
+        R0PTRTYPE(void *)           pMSRBitmap;
+    } svm;
+
+    /** Event injection state. */
+    struct
+    {
+        uint32_t                    fPending;
+        uint32_t                    errCode;
+        uint64_t                    intInfo;
+    } Event;
+
+    /** IO Block emulation state. */
+    struct
+    {
+        bool                    fEnabled;
+        uint8_t                 u8Align[7];
+
+        /** RIP at the start of the io code we wish to emulate in the recompiler. */
+        RTGCPTR                 GCPtrFunctionEip;
+
+        uint64_t                cr0;
+    } EmulateIoBlock;
+
+    struct
+    {
+        /* Pending IO operation type. */
+        HWACCMPENDINGIO         enmType;
+        uint32_t                uPadding;
+        RTGCPTR                 GCPtrRip;
+        RTGCPTR                 GCPtrRipNext;
+        union
+        {
+            struct
+            {
+                unsigned        uPort;
+                unsigned        uAndVal;
+                unsigned        cbSize;
+            } Port;
+            uint64_t            aRaw[2];
+        } s;
+    } PendingIO;
+
+    /** Currently shadow paging mode. */
+    PGMMODE                 enmShadowMode;
+
+    /** The CPU ID of the CPU currently owning the VMCS. Set in
+     * HWACCMR0Enter and cleared in HWACCMR0Leave. */
+    RTCPUID                 idEnteredCpu;
+
+    /** To keep track of pending TLB shootdown pages. (SMP guest only) */
+    struct
+    {
+        RTGCPTR             aPages[HWACCM_MAX_TLB_SHOOTDOWN_PAGES];
+        unsigned            cPages;
+    } TlbShootdown;
+
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack.
+     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
+    union
+    {
+        /** The disassembler scratch space. */
+        DISCPUSTATE         DisState;
+        /** Padding. */
+        uint8_t             abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
+    };
+
+    RTUINT                  padding2[1];
+
+    STAMPROFILEADV          StatEntry;
+    STAMPROFILEADV          StatExit1;
+    STAMPROFILEADV          StatExit2;
+#if 1 /* temporary for tracking down darwin issues. */
+    STAMPROFILEADV          StatExit2Sub1;
+    STAMPROFILEADV          StatExit2Sub2;
+    STAMPROFILEADV          StatExit2Sub3;
+#endif
+    STAMPROFILEADV          StatInGC;
+
+#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    STAMPROFILEADV          StatWorldSwitch3264;
+#endif
+    STAMPROFILEADV          StatPoke;
+    STAMPROFILEADV          StatSpinPoke;
+    STAMPROFILEADV          StatSpinPokeFailed;
+
+    STAMCOUNTER             StatIntInject;
+
+    STAMCOUNTER             StatExitShadowNM;
+    STAMCOUNTER             StatExitGuestNM;
+    STAMCOUNTER             StatExitShadowPF;
+    STAMCOUNTER             StatExitGuestPF;
+    STAMCOUNTER             StatExitGuestUD;
+    STAMCOUNTER             StatExitGuestSS;
+    STAMCOUNTER             StatExitGuestNP;
+    STAMCOUNTER             StatExitGuestGP;
+    STAMCOUNTER             StatExitGuestDE;
+    STAMCOUNTER             StatExitGuestDB;
+    STAMCOUNTER             StatExitGuestMF;
+    STAMCOUNTER             StatExitInvpg;
+    STAMCOUNTER             StatExitInvd;
+    STAMCOUNTER             StatExitCpuid;
+    STAMCOUNTER             StatExitRdtsc;
+    STAMCOUNTER             StatExitRdpmc;
+    STAMCOUNTER             StatExitCli;
+    STAMCOUNTER             StatExitSti;
+    STAMCOUNTER             StatExitPushf;
+    STAMCOUNTER             StatExitPopf;
+    STAMCOUNTER             StatExitIret;
+    STAMCOUNTER             StatExitInt;
+    STAMCOUNTER             StatExitCRxWrite[16];
+    STAMCOUNTER             StatExitCRxRead[16];
+    STAMCOUNTER             StatExitDRxWrite;
+    STAMCOUNTER             StatExitDRxRead;
+    STAMCOUNTER             StatExitRdmsr;
+    STAMCOUNTER             StatExitWrmsr;
+    STAMCOUNTER             StatExitCLTS;
+    STAMCOUNTER             StatExitHlt;
+    STAMCOUNTER             StatExitMwait;
+    STAMCOUNTER             StatExitMonitor;
+    STAMCOUNTER             StatExitLMSW;
+    STAMCOUNTER             StatExitIOWrite;
+    STAMCOUNTER             StatExitIORead;
+    STAMCOUNTER             StatExitIOStringWrite;
+    STAMCOUNTER             StatExitIOStringRead;
+    STAMCOUNTER             StatExitIrqWindow;
+    STAMCOUNTER             StatExitMaxResume;
+    STAMCOUNTER             StatExitPreemptPending;
+    STAMCOUNTER             StatIntReinject;
+    STAMCOUNTER             StatPendingHostIrq;
+
+    STAMCOUNTER             StatFlushPage;
+    STAMCOUNTER             StatFlushPageManual;
+    STAMCOUNTER             StatFlushPhysPageManual;
+    STAMCOUNTER             StatFlushTLB;
+    STAMCOUNTER             StatFlushTLBManual;
+    STAMCOUNTER             StatFlushPageInvlpg;
+    STAMCOUNTER             StatFlushTLBWorldSwitch;
+    STAMCOUNTER             StatNoFlushTLBWorldSwitch;
+    STAMCOUNTER             StatFlushTLBCRxChange;
+    STAMCOUNTER             StatFlushASID;
+    STAMCOUNTER             StatFlushTLBInvlpga;
+    STAMCOUNTER             StatTlbShootdown;
+    STAMCOUNTER             StatTlbShootdownFlush;
+
+    STAMCOUNTER             StatSwitchGuestIrq;
+    STAMCOUNTER             StatSwitchToR3;
+
+    STAMCOUNTER             StatTSCOffset;
+    STAMCOUNTER             StatTSCIntercept;
+    STAMCOUNTER             StatTSCInterceptOverFlow;
+
+    STAMCOUNTER             StatExitReasonNPF;
+    STAMCOUNTER             StatDRxArmed;
+    STAMCOUNTER             StatDRxContextSwitch;
+    STAMCOUNTER             StatDRxIOCheck;
+
+    STAMCOUNTER             StatLoadMinimal;
+    STAMCOUNTER             StatLoadFull;
+
+#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
+    STAMCOUNTER             StatFpu64SwitchBack;
+    STAMCOUNTER             StatDebug64SwitchBack;
+#endif
+
+#ifdef VBOX_WITH_STATISTICS
+    R3PTRTYPE(PSTAMCOUNTER) paStatExitReason;
+    R0PTRTYPE(PSTAMCOUNTER) paStatExitReasonR0;
+    R3PTRTYPE(PSTAMCOUNTER) paStatInjectedIrqs;
+    R0PTRTYPE(PSTAMCOUNTER) paStatInjectedIrqsR0;
+#endif
+} HWACCMCPU;
+/** Pointer to HWACCM VM instance data. */
+typedef HWACCMCPU *PHWACCMCPU;
+
+
+#ifdef IN_RING0
+
+VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpu();
+VMMR0DECL(PHWACCM_CPUINFO) HWACCMR0GetCurrentCpuEx(RTCPUID idCpu);
+
+
+#ifdef VBOX_STRICT
+VMMR0DECL(void) HWACCMDumpRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
+VMMR0DECL(void) HWACCMR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszMsg);
+#else
+# define HWACCMDumpRegs(a, b ,c)            do { } while (0)
+# define HWACCMR0DumpDescriptor(a, b, c)    do { } while (0)
+#endif
+
+/* Dummy callback handlers. */
+VMMR0DECL(int) HWACCMR0DummyEnter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu);
+VMMR0DECL(int) HWACCMR0DummyLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
+VMMR0DECL(int) HWACCMR0DummyEnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys);
+VMMR0DECL(int) HWACCMR0DummyDisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys);
+VMMR0DECL(int) HWACCMR0DummyInitVM(PVM pVM);
+VMMR0DECL(int) HWACCMR0DummyTermVM(PVM pVM);
+VMMR0DECL(int) HWACCMR0DummySetupVM(PVM pVM);
+VMMR0DECL(int) HWACCMR0DummyRunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
+VMMR0DECL(int) HWACCMR0DummySaveHostState(PVM pVM, PVMCPU pVCpu);
+VMMR0DECL(int) HWACCMR0DummyLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
+
+# ifdef VBOX_WITH_KERNEL_USING_XMM
+DECLASM(int)   hwaccmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu, PFNHWACCMVMXSTARTVM pfnStartVM);
+DECLASM(int)   hwaccmR0SVMRunWrapXMM(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu, PFNHWACCMSVMVMRUN pfnVMRun);
+# endif
+
+# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
+/**
+ * Gets 64-bit GDTR and IDTR on darwin.
+ * @param  pGdtr        Where to store the 64-bit GDTR.
+ * @param  pIdtr        Where to store the 64-bit IDTR.
+ */
+DECLASM(void) hwaccmR0Get64bitGDTRandIDTR(PX86XDTR64 pGdtr, PX86XDTR64 pIdtr);
+
+/**
+ * Gets 64-bit CR3 on darwin.
+ * @returns CR3
+ */
+DECLASM(uint64_t) hwaccmR0Get64bitCR3(void);
+# endif
+
+#endif /* IN_RING0 */
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/HWACCMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/include/HWACCMInternal.mac	(revision 35333)
+++ /trunk/src/VBox/VMM/include/HWACCMInternal.mac	(revision 35333)
@@ -0,0 +1,60 @@
+;$Id$
+;; @file
+; HWACCM - Internal header file.
+;
+;
+; Copyright (C) 2006-2007 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%define VMX_USE_CACHED_VMCS_ACCESSES
+
+;Maximum number of cached entries. 
+%define VMCSCACHE_MAX_ENTRY                             128
+
+; Structure for storing read and write VMCS actions. 
+struc VMCSCACHE
+%ifdef VBOX_WITH_CRASHDUMP_MAGIC
+    .aMagic                   resb    16
+    .uMagic                   resq    1
+    .u64TimeEntry             resq    1
+    .u64TimeSwitch            resq    1
+    .cResume                  resq    1
+    .interPD                  resq    1
+    .pSwitcher                resq    1
+    .uPos                     resd    1
+    .idCpu                    resd    1
+%endif
+    .cr2                      resq    1
+    .Write.cValidEntries      resd    1
+    .Write.uAlignment         resd    1
+    .Write.aField             resd    VMCSCACHE_MAX_ENTRY
+    .Write.aFieldVal          resq    VMCSCACHE_MAX_ENTRY
+    .Read.cValidEntries       resd    1
+    .Read.uAlignment          resd    1
+    .Read.aField              resd    VMCSCACHE_MAX_ENTRY
+    .Read.aFieldVal           resq    VMCSCACHE_MAX_ENTRY
+%ifdef DEBUG
+    .TestIn.pPageCpuPhys      resq    1
+    .TestIn.pVMCSPhys         resq    1
+    .TestIn.pCache            resq    1
+    .TestIn.pCtx              resq    1
+    .TestOut.pVMCSPhys        resq    1
+    .TestOut.pCache           resq    1
+    .TestOut.pCtx             resq    1
+    .TestOut.eflags           resq    1
+    .TestOut.cr8              resq    1
+    .ScratchPad.param1        resq    1
+    .ScratchPad.param2        resq    1
+    .ScratchPad.param3        resq    1
+    .ScratchPad.param4        resq    1
+%endif
+endstruc
+
Index: /trunk/src/VBox/VMM/include/IOMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/IOMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/IOMInternal.h	(revision 35333)
@@ -0,0 +1,567 @@
+/* $Id$ */
+/** @file
+ * IOM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___IOMInternal_h
+#define ___IOMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/iom.h>
+#include <VBox/stam.h>
+#include <VBox/pgm.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+
+
+
+/** @defgroup grp_iom_int   Internals
+ * @ingroup grp_iom
+ * @internal
+ * @{
+ */
+
+/**
+ * MMIO range descriptor.
+ */
+typedef struct IOMMMIORANGE
+{
+    /** Avl node core with GCPhys as Key and GCPhys + cbSize - 1 as KeyLast. */
+    AVLROGCPHYSNODECORE         Core;
+    /** Start physical address. */
+    RTGCPHYS                    GCPhys;
+    /** Size of the range. */
+    uint32_t                    cb;
+    uint32_t                    u32Alignment; /**< Alignment padding. */
+
+    /** Pointer to user argument - R3. */
+    RTR3PTR                     pvUserR3;
+    /** Pointer to device instance - R3. */
+    PPDMDEVINSR3                pDevInsR3;
+    /** Pointer to write callback function - R3. */
+    R3PTRTYPE(PFNIOMMMIOWRITE)  pfnWriteCallbackR3;
+    /** Pointer to read callback function - R3. */
+    R3PTRTYPE(PFNIOMMMIOREAD)   pfnReadCallbackR3;
+    /** Pointer to fill (memset) callback function - R3. */
+    R3PTRTYPE(PFNIOMMMIOFILL)   pfnFillCallbackR3;
+
+    /** Pointer to user argument - R0. */
+    RTR0PTR                     pvUserR0;
+    /** Pointer to device instance - R0. */
+    PPDMDEVINSR0                pDevInsR0;
+    /** Pointer to write callback function - R0. */
+    R0PTRTYPE(PFNIOMMMIOWRITE)  pfnWriteCallbackR0;
+    /** Pointer to read callback function - R0. */
+    R0PTRTYPE(PFNIOMMMIOREAD)   pfnReadCallbackR0;
+    /** Pointer to fill (memset) callback function - R0. */
+    R0PTRTYPE(PFNIOMMMIOFILL)   pfnFillCallbackR0;
+
+    /** Pointer to user argument - RC. */
+    RTRCPTR                     pvUserRC;
+    /** Pointer to device instance - RC. */
+    PPDMDEVINSRC                pDevInsRC;
+    /** Pointer to write callback function - RC. */
+    RCPTRTYPE(PFNIOMMMIOWRITE)  pfnWriteCallbackRC;
+    /** Pointer to read callback function - RC. */
+    RCPTRTYPE(PFNIOMMMIOREAD)   pfnReadCallbackRC;
+    /** Pointer to fill (memset) callback function - RC. */
+    RCPTRTYPE(PFNIOMMMIOFILL)   pfnFillCallbackRC;
+    /** Alignment padding. */
+    RTRCPTR                     RCPtrAlignment;
+
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)     pszDesc;
+} IOMMMIORANGE;
+/** Pointer to a MMIO range descriptor, R3 version. */
+typedef struct IOMMMIORANGE *PIOMMMIORANGE;
+
+
+/**
+ * MMIO address statistics. (one address)
+ *
+ * This is a simple way of making on demand statistics, however it's a
+ * bit free with the hypervisor heap memory.
+ */
+typedef struct IOMMMIOSTATS
+{
+    /** Avl node core with the address as Key. */
+    AVLOGCPHYSNODECORE          Core;
+
+    /** Number of accesses (subtract ReadRZToR3 and WriteRZToR3 to get the right
+     *  number). */
+    STAMCOUNTER                 Accesses;
+
+    /** Profiling read handler overhead in R3. */
+    STAMPROFILE                 ProfReadR3;
+    /** Profiling write handler overhead in R3. */
+    STAMPROFILE                 ProfWriteR3;
+    /** Counting and profiling reads in R0/RC. */
+    STAMPROFILE                 ProfReadRZ;
+    /** Counting and profiling writes in R0/RC. */
+    STAMPROFILE                 ProfWriteRZ;
+
+    /** Number of reads to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 ReadRZToR3;
+    /** Number of writes to this address from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 WriteRZToR3;
+} IOMMMIOSTATS;
+AssertCompileMemberAlignment(IOMMMIOSTATS, Accesses, 8);
+/** Pointer to I/O port statistics. */
+typedef IOMMMIOSTATS *PIOMMMIOSTATS;
+
+
+/**
+ * I/O port range descriptor, R3 version.
+ */
+typedef struct IOMIOPORTRANGER3
+{
+    /** Avl node core with Port as Key and Port + cPorts - 1 as KeyLast. */
+    AVLROIOPORTNODECORE         Core;
+#if HC_ARCH_BITS == 64 && !defined(RT_OS_WINDOWS)
+    uint32_t                    u32Alignment; /**< The sizeof(Core) differs. */
+#endif
+    /** Start I/O port address. */
+    RTIOPORT                    Port;
+    /** Size of the range. */
+    uint16_t                    cPorts;
+    /** Pointer to user argument. */
+    RTR3PTR                     pvUser;
+    /** Pointer to the associated device instance. */
+    R3PTRTYPE(PPDMDEVINS)       pDevIns;
+    /** Pointer to OUT callback function. */
+    R3PTRTYPE(PFNIOMIOPORTOUT)  pfnOutCallback;
+    /** Pointer to IN callback function. */
+    R3PTRTYPE(PFNIOMIOPORTIN)   pfnInCallback;
+    /** Pointer to string OUT callback function. */
+    R3PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback;
+    /** Pointer to string IN callback function. */
+    R3PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)     pszDesc;
+} IOMIOPORTRANGER3;
+/** Pointer to I/O port range descriptor, R3 version. */
+typedef IOMIOPORTRANGER3 *PIOMIOPORTRANGER3;
+
+/**
+ * I/O port range descriptor, R0 version.
+ */
+typedef struct IOMIOPORTRANGER0
+{
+    /** Avl node core with Port as Key and Port + cPorts - 1 as KeyLast. */
+    AVLROIOPORTNODECORE         Core;
+#if HC_ARCH_BITS == 64 && !defined(RT_OS_WINDOWS)
+    uint32_t                    u32Alignment; /**< The sizeof(Core) differs. */
+#endif
+    /** Start I/O port address. */
+    RTIOPORT                    Port;
+    /** Size of the range. */
+    uint16_t                    cPorts;
+    /** Pointer to user argument. */
+    RTR0PTR                     pvUser;
+    /** Pointer to the associated device instance. */
+    R0PTRTYPE(PPDMDEVINS)       pDevIns;
+    /** Pointer to OUT callback function. */
+    R0PTRTYPE(PFNIOMIOPORTOUT)  pfnOutCallback;
+    /** Pointer to IN callback function. */
+    R0PTRTYPE(PFNIOMIOPORTIN)   pfnInCallback;
+    /** Pointer to string OUT callback function. */
+    R0PTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback;
+    /** Pointer to string IN callback function. */
+    R0PTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)     pszDesc;
+} IOMIOPORTRANGER0;
+/** Pointer to I/O port range descriptor, R0 version. */
+typedef IOMIOPORTRANGER0 *PIOMIOPORTRANGER0;
+
+/**
+ * I/O port range descriptor, RC version.
+ */
+typedef struct IOMIOPORTRANGERC
+{
+    /** Avl node core with Port as Key and Port + cPorts - 1 as KeyLast. */
+    AVLROIOPORTNODECORE         Core;
+    /** Start I/O port address. */
+    RTIOPORT                    Port;
+    /** Size of the range. */
+    uint16_t                    cPorts;
+    /** Pointer to user argument. */
+    RTRCPTR                     pvUser;
+    /** Pointer to the associated device instance. */
+    RCPTRTYPE(PPDMDEVINS)       pDevIns;
+    /** Pointer to OUT callback function. */
+    RCPTRTYPE(PFNIOMIOPORTOUT)  pfnOutCallback;
+    /** Pointer to IN callback function. */
+    RCPTRTYPE(PFNIOMIOPORTIN)   pfnInCallback;
+    /** Pointer to string OUT callback function. */
+    RCPTRTYPE(PFNIOMIOPORTOUTSTRING) pfnOutStrCallback;
+    /** Pointer to string IN callback function. */
+    RCPTRTYPE(PFNIOMIOPORTINSTRING) pfnInStrCallback;
+#if HC_ARCH_BITS == 64
+    RTRCPTR                     RCPtrAlignment; /**< pszDesc is 8 byte aligned. */
+#endif
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)     pszDesc;
+} IOMIOPORTRANGERC;
+/** Pointer to I/O port range descriptor, RC version. */
+typedef IOMIOPORTRANGERC *PIOMIOPORTRANGERC;
+
+
+/**
+ * I/O port statistics. (one I/O port)
+ *
+ * This is a simple way of making on demand statistics, however it's a
+ * bit free with the hypervisor heap memory.
+ */
+typedef struct IOMIOPORTSTATS
+{
+    /** Avl node core with the port as Key. */
+    AVLOIOPORTNODECORE          Core;
+#if HC_ARCH_BITS != 64 || !defined(RT_OS_WINDOWS)
+    uint32_t                    u32Alignment; /**< The sizeof(Core) differs. */
+#endif
+    /** Number of INs to this port from R3. */
+    STAMCOUNTER                 InR3;
+    /** Profiling IN handler overhead in R3. */
+    STAMPROFILE                 ProfInR3;
+    /** Number of OUTs to this port from R3. */
+    STAMCOUNTER                 OutR3;
+    /** Profiling OUT handler overhead in R3. */
+    STAMPROFILE                 ProfOutR3;
+
+    /** Number of INs to this port from R0/RC. */
+    STAMCOUNTER                 InRZ;
+    /** Profiling IN handler overhead in R0/RC. */
+    STAMPROFILE                 ProfInRZ;
+    /** Number of INs to this port from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 InRZToR3;
+
+    /** Number of OUTs to this port from R0/RC. */
+    STAMCOUNTER                 OutRZ;
+    /** Profiling OUT handler overhead in R0/RC. */
+    STAMPROFILE                 ProfOutRZ;
+    /** Number of OUTs to this port from R0/RC which was serviced in R3. */
+    STAMCOUNTER                 OutRZToR3;
+} IOMIOPORTSTATS;
+AssertCompileMemberAlignment(IOMIOPORTSTATS, InR3, 8);
+/** Pointer to I/O port statistics. */
+typedef IOMIOPORTSTATS *PIOMIOPORTSTATS;
+
+
+/**
+ * The IOM trees.
+ * These are offset based the nodes and root must be in the same
+ * memory block in HC. The locations of IOM structure and the hypervisor heap
+ * are quite different in R3, R0 and RC.
+ */
+typedef struct IOMTREES
+{
+    /** Tree containing I/O port range descriptors registered for HC (IOMIOPORTRANGEHC). */
+    AVLROIOPORTTREE         IOPortTreeR3;
+    /** Tree containing I/O port range descriptors registered for R0 (IOMIOPORTRANGER0). */
+    AVLROIOPORTTREE         IOPortTreeR0;
+    /** Tree containing I/O port range descriptors registered for RC (IOMIOPORTRANGERC). */
+    AVLROIOPORTTREE         IOPortTreeRC;
+
+    /** Tree containing the MMIO range descriptors (IOMMMIORANGE). */
+    AVLROGCPHYSTREE         MMIOTree;
+
+    /** Tree containing I/O port statistics (IOMIOPORTSTATS). */
+    AVLOIOPORTTREE          IOPortStatTree;
+    /** Tree containing MMIO statistics (IOMMMIOSTATS). */
+    AVLOGCPHYSTREE          MMIOStatTree;
+} IOMTREES;
+/** Pointer to the IOM trees. */
+typedef IOMTREES *PIOMTREES;
+
+
+/**
+ * Converts an IOM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the PGM is part of.
+ * @param   pIOM   Pointer to IOM instance data.
+ */
+#define IOM2VM(pIOM)  ( (PVM)((char*)pIOM - pIOM->offVM) )
+
+/**
+ * IOM Data (part of VM)
+ */
+typedef struct IOM
+{
+    /** Offset to the VM structure. */
+    RTINT                           offVM;
+
+    /** Pointer to the trees - RC ptr. */
+    RCPTRTYPE(PIOMTREES)            pTreesRC;
+    /** Pointer to the trees - R3 ptr. */
+    R3PTRTYPE(PIOMTREES)            pTreesR3;
+    /** Pointer to the trees - R0 ptr. */
+    R0PTRTYPE(PIOMTREES)            pTreesR0;
+
+    /** The ring-0 address of IOMMMIOHandler. */
+    R0PTRTYPE(PFNPGMR0PHYSHANDLER)  pfnMMIOHandlerR0;
+    /** The RC address of IOMMMIOHandler. */
+    RCPTRTYPE(PFNPGMRCPHYSHANDLER)  pfnMMIOHandlerRC;
+#if HC_ARCH_BITS == 64
+    RTRCPTR                         padding;
+#endif
+
+    /** Lock serializing EMT access to IOM. */
+    PDMCRITSECT                     EmtLock;
+
+    /** @name Caching of I/O Port and MMIO ranges and statistics.
+     * (Saves quite some time in rep outs/ins instruction emulation.)
+     * @{ */
+    R3PTRTYPE(PIOMIOPORTRANGER3)    pRangeLastReadR3;
+    R3PTRTYPE(PIOMIOPORTRANGER3)    pRangeLastWriteR3;
+    R3PTRTYPE(PIOMIOPORTSTATS)      pStatsLastReadR3;
+    R3PTRTYPE(PIOMIOPORTSTATS)      pStatsLastWriteR3;
+    R3PTRTYPE(PIOMMMIORANGE)        pMMIORangeLastR3;
+    R3PTRTYPE(PIOMMMIOSTATS)        pMMIOStatsLastR3;
+
+    R0PTRTYPE(PIOMIOPORTRANGER0)    pRangeLastReadR0;
+    R0PTRTYPE(PIOMIOPORTRANGER0)    pRangeLastWriteR0;
+    R0PTRTYPE(PIOMIOPORTSTATS)      pStatsLastReadR0;
+    R0PTRTYPE(PIOMIOPORTSTATS)      pStatsLastWriteR0;
+    R0PTRTYPE(PIOMMMIORANGE)        pMMIORangeLastR0;
+    R0PTRTYPE(PIOMMMIOSTATS)        pMMIOStatsLastR0;
+
+    RCPTRTYPE(PIOMIOPORTRANGERC)    pRangeLastReadRC;
+    RCPTRTYPE(PIOMIOPORTRANGERC)    pRangeLastWriteRC;
+    RCPTRTYPE(PIOMIOPORTSTATS)      pStatsLastReadRC;
+    RCPTRTYPE(PIOMIOPORTSTATS)      pStatsLastWriteRC;
+    RCPTRTYPE(PIOMMMIORANGE)        pMMIORangeLastRC;
+    RCPTRTYPE(PIOMMMIOSTATS)        pMMIOStatsLastRC;
+    /** @} */
+
+    /** @name I/O Port statistics.
+     * @{ */
+    STAMCOUNTER                     StatInstIn;
+    STAMCOUNTER                     StatInstOut;
+    STAMCOUNTER                     StatInstIns;
+    STAMCOUNTER                     StatInstOuts;
+    /** @} */
+
+    /** @name MMIO statistics.
+     * @{ */
+    STAMPROFILE                     StatRZMMIOHandler;
+    STAMCOUNTER                     StatRZMMIOFailures;
+
+    STAMPROFILE                     StatRZInstMov;
+    STAMPROFILE                     StatRZInstCmp;
+    STAMPROFILE                     StatRZInstAnd;
+    STAMPROFILE                     StatRZInstOr;
+    STAMPROFILE                     StatRZInstXor;
+    STAMPROFILE                     StatRZInstBt;
+    STAMPROFILE                     StatRZInstTest;
+    STAMPROFILE                     StatRZInstXchg;
+    STAMPROFILE                     StatRZInstStos;
+    STAMPROFILE                     StatRZInstLods;
+#ifdef IOM_WITH_MOVS_SUPPORT
+    STAMPROFILEADV                  StatRZInstMovs;
+    STAMPROFILE                     StatRZInstMovsToMMIO;
+    STAMPROFILE                     StatRZInstMovsFromMMIO;
+    STAMPROFILE                     StatRZInstMovsMMIO;
+#endif
+    STAMCOUNTER                     StatRZInstOther;
+
+    STAMCOUNTER                     StatRZMMIO1Byte;
+    STAMCOUNTER                     StatRZMMIO2Bytes;
+    STAMCOUNTER                     StatRZMMIO4Bytes;
+    STAMCOUNTER                     StatRZMMIO8Bytes;
+
+    STAMCOUNTER                     StatR3MMIOHandler;
+
+    RTUINT                          cMovsMaxBytes;
+    RTUINT                          cStosMaxBytes;
+    /** @} */
+} IOM;
+/** Pointer to IOM instance data. */
+typedef IOM *PIOM;
+
+
+/**
+ * IOM per virtual CPU instance data.
+ */
+typedef struct IOMCPU
+{
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack.
+     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
+    union
+    {
+        /** The disassembler scratch space. */
+        DISCPUSTATE                 DisState;
+        /** Padding. */
+        uint8_t                     abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
+    };
+    uint8_t                         Dummy[16];
+} IOMCPU;
+/** Pointer to IOM per virtual CPU instance data. */
+typedef IOMCPU *PIOMCPU;
+
+
+RT_C_DECLS_BEGIN
+
+#ifdef IN_RING3
+PIOMIOPORTSTATS iomR3IOPortStatsCreate(PVM pVM, RTIOPORT Port, const char *pszDesc);
+PIOMMMIOSTATS   iomR3MMIOStatsCreate(PVM pVM, RTGCPHYS GCPhys, const char *pszDesc);
+#endif /* IN_RING3 */
+
+VMMDECL(int)        IOMMMIOHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
+#ifdef IN_RING3
+DECLCALLBACK(int)   IOMR3MMIOHandler(PVM pVM, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType, void *pvUser);
+#endif
+
+
+/**
+ * Gets the I/O port range for the specified I/O port in the current context.
+ *
+ * @returns Pointer to I/O port range.
+ * @returns NULL if no port registered.
+ *
+ * @param   pIOM    IOM instance data.
+ * @param   Port    Port to lookup.
+ */
+DECLINLINE(CTX_SUFF(PIOMIOPORTRANGE)) iomIOPortGetRange(PIOM pIOM, RTIOPORT Port)
+{
+#ifdef IN_RING3
+    if (PDMCritSectIsInitialized(&pIOM->EmtLock))
+#endif
+        Assert(IOMIsLockOwner(IOM2VM(pIOM)));
+    CTX_SUFF(PIOMIOPORTRANGE) pRange = (CTX_SUFF(PIOMIOPORTRANGE))RTAvlroIOPortRangeGet(&pIOM->CTX_SUFF(pTrees)->CTX_SUFF(IOPortTree), Port);
+    return pRange;
+}
+
+
+/**
+ * Gets the I/O port range for the specified I/O port in the HC.
+ *
+ * @returns Pointer to I/O port range.
+ * @returns NULL if no port registered.
+ *
+ * @param   pIOM    IOM instance data.
+ * @param   Port    Port to lookup.
+ */
+DECLINLINE(PIOMIOPORTRANGER3) iomIOPortGetRangeR3(PIOM pIOM, RTIOPORT Port)
+{
+#ifdef IN_RING3
+    if (PDMCritSectIsInitialized(&pIOM->EmtLock))
+#endif
+        Assert(IOMIsLockOwner(IOM2VM(pIOM)));
+    PIOMIOPORTRANGER3 pRange = (PIOMIOPORTRANGER3)RTAvlroIOPortRangeGet(&pIOM->CTX_SUFF(pTrees)->IOPortTreeR3, Port);
+    return pRange;
+}
+
+
+/**
+ * Gets the MMIO range for the specified physical address in the current context.
+ *
+ * @returns Pointer to MMIO range.
+ * @returns NULL if address not in a MMIO range.
+ *
+ * @param   pIOM    IOM instance data.
+ * @param   GCPhys  Physical address to lookup.
+ */
+DECLINLINE(PIOMMMIORANGE) iomMMIOGetRange(PIOM pIOM, RTGCPHYS GCPhys)
+{
+#ifdef IN_RING3
+    if (PDMCritSectIsInitialized(&pIOM->EmtLock))
+#endif
+        Assert(IOMIsLockOwner(IOM2VM(pIOM)));
+    PIOMMMIORANGE pRange = pIOM->CTX_SUFF(pMMIORangeLast);
+    if (    !pRange
+        ||  GCPhys - pRange->GCPhys >= pRange->cb)
+        pIOM->CTX_SUFF(pMMIORangeLast) = pRange = (PIOMMMIORANGE)RTAvlroGCPhysRangeGet(&pIOM->CTX_SUFF(pTrees)->MMIOTree, GCPhys);
+    return pRange;
+}
+
+#ifdef VBOX_STRICT
+/**
+ * Gets the MMIO range for the specified physical address in the current context.
+ *
+ * @returns Pointer to MMIO range.
+ * @returns NULL if address not in a MMIO range.
+ *
+ * @param   pIOM    IOM instance data.
+ * @param   GCPhys  Physical address to lookup.
+ */
+DECLINLINE(PIOMMMIORANGE) iomMMIOGetRangeUnsafe(PIOM pIOM, RTGCPHYS GCPhys)
+{
+    PIOMMMIORANGE pRange = pIOM->CTX_SUFF(pMMIORangeLast);
+    if (    !pRange
+        ||  GCPhys - pRange->GCPhys >= pRange->cb)
+        pIOM->CTX_SUFF(pMMIORangeLast) = pRange = (PIOMMMIORANGE)RTAvlroGCPhysRangeGet(&pIOM->CTX_SUFF(pTrees)->MMIOTree, GCPhys);
+    return pRange;
+}
+#endif
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * Gets the MMIO statistics record.
+ *
+ * In ring-3 this will lazily create missing records, while in GC/R0 the caller has to
+ * return the appropriate status to defer the operation to ring-3.
+ *
+ * @returns Pointer to MMIO stats.
+ * @returns NULL if not found (R0/GC), or out of memory (R3).
+ *
+ * @param   pIOM        IOM instance data.
+ * @param   GCPhys      Physical address to lookup.
+ * @param   pRange      The MMIO range.
+ */
+DECLINLINE(PIOMMMIOSTATS) iomMMIOGetStats(PIOM pIOM, RTGCPHYS GCPhys, PIOMMMIORANGE pRange)
+{
+    Assert(IOMIsLockOwner(IOM2VM(pIOM)));
+    /* For large ranges, we'll put everything on the first byte. */
+    if (pRange->cb > PAGE_SIZE)
+        GCPhys = pRange->GCPhys;
+
+    PIOMMMIOSTATS pStats = pIOM->CTX_SUFF(pMMIOStatsLast);
+    if (    !pStats
+        ||  pStats->Core.Key != GCPhys)
+    {
+        pStats = (PIOMMMIOSTATS)RTAvloGCPhysGet(&pIOM->CTX_SUFF(pTrees)->MMIOStatTree, GCPhys);
+# ifdef IN_RING3
+        if (!pStats)
+            pStats = iomR3MMIOStatsCreate(IOM2VM(pIOM), GCPhys, pRange->pszDesc);
+# endif
+    }
+    return pStats;
+}
+#endif
+
+/* IOM locking helpers. */
+int     iomLock(PVM pVM);
+int     iomTryLock(PVM pVM);
+void    iomUnlock(PVM pVM);
+
+/* Disassembly helpers used in IOMAll.cpp & IOMAllMMIO.cpp */
+bool    iomGetRegImmData(PDISCPUSTATE pCpu, PCOP_PARAMETER pParam, PCPUMCTXCORE pRegFrame, uint64_t *pu64Data, unsigned *pcbSize);
+bool    iomSaveDataToReg(PDISCPUSTATE pCpu, PCOP_PARAMETER pParam, PCPUMCTXCORE pRegFrame, uint64_t u32Data);
+
+RT_C_DECLS_END
+
+
+#ifdef IN_RING3
+
+#endif
+
+/** @} */
+
+#endif /* ___IOMInternal_h */
Index: /trunk/src/VBox/VMM/include/MMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/MMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/MMInternal.h	(revision 35333)
@@ -0,0 +1,848 @@
+/* $Id$ */
+/** @file
+ * MM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___MMInternal_h
+#define ___MMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/sup.h>
+#include <VBox/stam.h>
+#include <VBox/pdmcritsect.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+#include <iprt/critsect.h>
+
+
+
+/** @defgroup grp_mm_int   Internals
+ * @internal
+ * @ingroup grp_mm
+ * @{
+ */
+
+
+/** @name MMR3Heap - VM Ring-3 Heap Internals
+ * @{
+ */
+
+/** @def MMR3HEAP_SIZE_ALIGNMENT
+ * The allocation size alignment of the MMR3Heap.
+ */
+#define MMR3HEAP_SIZE_ALIGNMENT     16
+
+/** @def MMR3HEAP_WITH_STATISTICS
+ * Enable MMR3Heap statistics.
+ */
+#if !defined(MMR3HEAP_WITH_STATISTICS) && defined(VBOX_WITH_STATISTICS)
+# define MMR3HEAP_WITH_STATISTICS
+#endif
+
+/**
+ * Heap statistics record.
+ * There is one global and one per allocation tag.
+ */
+typedef struct MMHEAPSTAT
+{
+    /** Core avl node, key is the tag. */
+    AVLULNODECORE           Core;
+    /** Pointer to the heap the memory belongs to. */
+    struct MMHEAP          *pHeap;
+#ifdef MMR3HEAP_WITH_STATISTICS
+# if HC_ARCH_BITS == 32
+    /** Aligning the statistics on an 8 byte boundary (for uint64_t and STAM). */
+    void                   *pvAlignment;
+# endif
+    /** Number of allocation. */
+    uint64_t                cAllocations;
+    /** Number of reallocations. */
+    uint64_t                cReallocations;
+    /** Number of frees. */
+    uint64_t                cFrees;
+    /** Failures. */
+    uint64_t                cFailures;
+    /** Number of bytes allocated (sum). */
+    uint64_t                cbAllocated;
+    /** Number of bytes freed. */
+    uint64_t                cbFreed;
+    /** Number of bytes currently allocated. */
+    size_t                  cbCurAllocated;
+#endif
+} MMHEAPSTAT;
+#if defined(MMR3HEAP_WITH_STATISTICS) && defined(IN_RING3)
+AssertCompileMemberAlignment(MMHEAPSTAT, cAllocations, 8);
+#endif
+/** Pointer to heap statistics record. */
+typedef MMHEAPSTAT *PMMHEAPSTAT;
+
+
+
+
+/**
+ * Additional heap block header for relating allocations to the VM.
+ */
+typedef struct MMHEAPHDR
+{
+    /** Pointer to the next record. */
+    struct MMHEAPHDR       *pNext;
+    /** Pointer to the previous record. */
+    struct MMHEAPHDR       *pPrev;
+    /** Pointer to the heap statistics record.
+     * (Where the a PVM can be found.) */
+    PMMHEAPSTAT             pStat;
+    /** Size of the allocation (including this header). */
+    size_t                  cbSize;
+} MMHEAPHDR;
+/** Pointer to MM heap header. */
+typedef MMHEAPHDR *PMMHEAPHDR;
+
+
+/** MM Heap structure. */
+typedef struct MMHEAP
+{
+    /** Lock protecting the heap. */
+    RTCRITSECT              Lock;
+    /** Heap block list head. */
+    PMMHEAPHDR              pHead;
+    /** Heap block list tail. */
+    PMMHEAPHDR              pTail;
+    /** Heap per tag statistics tree. */
+    PAVLULNODECORE          pStatTree;
+    /** The VM handle. */
+    PUVM                    pUVM;
+    /** Heap global statistics. */
+    MMHEAPSTAT              Stat;
+} MMHEAP;
+/** Pointer to MM Heap structure. */
+typedef MMHEAP *PMMHEAP;
+
+/** @} */
+
+
+/** @name MMUkHeap - VM User-kernel Heap Internals
+ * @{
+ */
+
+/** @def MMUKHEAP_SIZE_ALIGNMENT
+ * The allocation size alignment of the MMR3UkHeap.
+ */
+#define MMUKHEAP_SIZE_ALIGNMENT   16
+
+/** @def MMUKHEAP_WITH_STATISTICS
+ * Enable MMUkHeap statistics.
+ */
+#if !defined(MMUKHEAP_WITH_STATISTICS) && defined(VBOX_WITH_STATISTICS)
+# define MMUKHEAP_WITH_STATISTICS
+#endif
+
+
+/**
+ * Heap statistics record.
+ * There is one global and one per allocation tag.
+ */
+typedef struct MMUKHEAPSTAT
+{
+    /** Core avl node, key is the tag. */
+    AVLULNODECORE           Core;
+    /** Number of allocation. */
+    uint64_t                cAllocations;
+    /** Number of reallocations. */
+    uint64_t                cReallocations;
+    /** Number of frees. */
+    uint64_t                cFrees;
+    /** Failures. */
+    uint64_t                cFailures;
+    /** Number of bytes allocated (sum). */
+    uint64_t                cbAllocated;
+    /** Number of bytes freed. */
+    uint64_t                cbFreed;
+    /** Number of bytes currently allocated. */
+    size_t                  cbCurAllocated;
+} MMUKHEAPSTAT;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(MMUKHEAPSTAT, cAllocations, 8);
+#endif
+/** Pointer to heap statistics record. */
+typedef MMUKHEAPSTAT *PMMUKHEAPSTAT;
+
+/**
+ * Sub heap tracking record.
+ */
+typedef struct MMUKHEAPSUB
+{
+    /** Pointer to the next sub-heap. */
+    struct MMUKHEAPSUB     *pNext;
+    /** The base address of the sub-heap. */
+    void                   *pv;
+    /** The size of the sub-heap.  */
+    size_t                  cb;
+    /** The handle of the simple block pointer. */
+    RTHEAPSIMPLE            hSimple;
+    /** The ring-0 address corresponding to MMUKHEAPSUB::pv. */
+    RTR0PTR                 pvR0;
+} MMUKHEAPSUB;
+/** Pointer to a sub-heap tracking record. */
+typedef MMUKHEAPSUB *PMMUKHEAPSUB;
+
+
+/** MM User-kernel Heap structure. */
+typedef struct MMUKHEAP
+{
+    /** Lock protecting the heap. */
+    RTCRITSECT              Lock;
+    /** Head of the sub-heap LIFO. */
+    PMMUKHEAPSUB            pSubHeapHead;
+    /** Heap per tag statistics tree. */
+    PAVLULNODECORE          pStatTree;
+    /** The VM handle. */
+    PUVM                    pUVM;
+#if HC_ARCH_BITS == 32
+    /** Aligning the statistics on an 8 byte boundary (for uint64_t and STAM). */
+    void                   *pvAlignment;
+#endif
+    /** Heap global statistics. */
+    MMUKHEAPSTAT            Stat;
+} MMUKHEAP;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(MMUKHEAP, Stat, 8);
+#endif
+/** Pointer to MM Heap structure. */
+typedef MMUKHEAP *PMMUKHEAP;
+
+/** @} */
+
+
+
+/** @name Hypervisor Heap Internals
+ * @{
+ */
+
+/** @def MMHYPER_HEAP_FREE_DELAY
+ * If defined, it indicates the number of frees that should be delayed.
+ */
+#if defined(DOXYGEN_RUNNING)
+# define MMHYPER_HEAP_FREE_DELAY            64
+#endif
+
+/** @def MMHYPER_HEAP_FREE_POISON
+ * If defined, it indicates that freed memory should be poisoned
+ * with the value it has.
+ */
+#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
+# define MMHYPER_HEAP_FREE_POISON           0xcb
+#endif
+
+/** @def MMHYPER_HEAP_STRICT
+ * Enables a bunch of assertions in the heap code. */
+#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
+# define MMHYPER_HEAP_STRICT 1
+# if 0 || defined(DOXYGEN_RUNNING)
+/** @def MMHYPER_HEAP_STRICT_FENCE
+ * Enables tail fence. */
+#  define MMHYPER_HEAP_STRICT_FENCE
+/** @def MMHYPER_HEAP_STRICT_FENCE_SIZE
+ * The fence size in bytes. */
+#  define MMHYPER_HEAP_STRICT_FENCE_SIZE    256
+/** @def MMHYPER_HEAP_STRICT_FENCE_U32
+ * The fence filler. */
+#  define MMHYPER_HEAP_STRICT_FENCE_U32     UINT32_C(0xdeadbeef)
+# endif
+#endif
+
+/**
+ * Hypervisor heap statistics record.
+ * There is one global and one per allocation tag.
+ */
+typedef struct MMHYPERSTAT
+{
+    /** Core avl node, key is the tag.
+     * @todo The type is wrong! Get your lazy a$$ over and create that offsetted uint32_t version we need here!  */
+    AVLOGCPHYSNODECORE      Core;
+    /** Aligning the 64-bit fields on a 64-bit line. */
+    uint32_t                u32Padding0;
+    /** Indicator for whether these statistics are registered with STAM or not. */
+    bool                    fRegistered;
+    /** Number of allocation. */
+    uint64_t                cAllocations;
+    /** Number of frees. */
+    uint64_t                cFrees;
+    /** Failures. */
+    uint64_t                cFailures;
+    /** Number of bytes allocated (sum). */
+    uint64_t                cbAllocated;
+    /** Number of bytes freed (sum). */
+    uint64_t                cbFreed;
+    /** Number of bytes currently allocated. */
+    uint32_t                cbCurAllocated;
+    /** Max number of bytes allocated. */
+    uint32_t                cbMaxAllocated;
+} MMHYPERSTAT;
+AssertCompileMemberAlignment(MMHYPERSTAT, cAllocations, 8);
+/** Pointer to hypervisor heap statistics record. */
+typedef MMHYPERSTAT *PMMHYPERSTAT;
+
+/**
+ * Hypervisor heap chunk.
+ */
+typedef struct MMHYPERCHUNK
+{
+    /** Previous block in the list of all blocks.
+     * This is relative to the start of the heap. */
+    uint32_t                offNext;
+    /** Offset to the previous block relative to this one. */
+    int32_t                 offPrev;
+    /** The statistics record this allocation belongs to (self relative). */
+    int32_t                 offStat;
+    /** Offset to the heap block (self relative). */
+    int32_t                 offHeap;
+} MMHYPERCHUNK;
+/** Pointer to a hypervisor heap chunk. */
+typedef MMHYPERCHUNK *PMMHYPERCHUNK;
+
+
+/**
+ * Hypervisor heap chunk.
+ */
+typedef struct MMHYPERCHUNKFREE
+{
+    /** Main list. */
+    MMHYPERCHUNK            core;
+    /** Offset of the next chunk in the list of free nodes. */
+    uint32_t                offNext;
+    /** Offset of the previous chunk in the list of free nodes. */
+    int32_t                 offPrev;
+    /** Size of the block. */
+    uint32_t                cb;
+} MMHYPERCHUNKFREE;
+/** Pointer to a free hypervisor heap chunk. */
+typedef MMHYPERCHUNKFREE *PMMHYPERCHUNKFREE;
+
+
+/**
+ * The hypervisor heap.
+ */
+typedef struct MMHYPERHEAP
+{
+    /** The typical magic (MMHYPERHEAP_MAGIC). */
+    uint32_t                u32Magic;
+    /** The heap size. (This structure is not included!) */
+    uint32_t                cbHeap;
+    /** Lock protecting the heap. */
+    PDMCRITSECT             Lock;
+    /** The HC ring-3 address of the heap. */
+    R3PTRTYPE(uint8_t *)    pbHeapR3;
+    /** The HC ring-3 address of the shared VM structure. */
+    PVMR3                   pVMR3;
+    /** The HC ring-0 address of the heap. */
+    R0PTRTYPE(uint8_t *)    pbHeapR0;
+    /** The HC ring-0 address of the shared VM structure. */
+    PVMR0                   pVMR0;
+    /** The RC address of the heap. */
+    RCPTRTYPE(uint8_t *)    pbHeapRC;
+    /** The RC address of the shared VM structure. */
+    PVMRC                   pVMRC;
+    /** The amount of free memory in the heap. */
+    uint32_t                cbFree;
+    /** Offset of the first free chunk in the heap.
+     * The offset is relative to the start of the heap. */
+    uint32_t                offFreeHead;
+    /** Offset of the last free chunk in the heap.
+     * The offset is relative to the start of the heap. */
+    uint32_t                offFreeTail;
+    /** Offset of the first page aligned block in the heap.
+     * The offset is equal to cbHeap initially. */
+    uint32_t                offPageAligned;
+    /** Tree of hypervisor heap statistics. */
+    AVLOGCPHYSTREE          HyperHeapStatTree;
+#ifdef MMHYPER_HEAP_FREE_DELAY
+    /** Where to insert the next free. */
+    uint32_t                iDelayedFree;
+    /** Array of delayed frees. Circular. Offsets relative to this structure. */
+    struct
+    {
+        /** The free caller address. */
+        RTUINTPTR           uCaller;
+        /** The offset of the freed chunk. */
+        uint32_t            offChunk;
+    } aDelayedFrees[MMHYPER_HEAP_FREE_DELAY];
+#else
+    /** Padding the structure to a 64-bit aligned size. */
+    uint32_t                u32Padding0;
+#endif
+    /** The heap physical pages. */
+    R3PTRTYPE(PSUPPAGE)     paPages;
+#if HC_ARCH_BITS == 32
+    /** Padding the structure to a 64-bit aligned size. */
+    uint32_t                u32Padding1;
+#endif
+} MMHYPERHEAP;
+/** Pointer to the hypervisor heap. */
+typedef MMHYPERHEAP *PMMHYPERHEAP;
+
+/** Magic value for MMHYPERHEAP. (C. S. Lewis) */
+#define MMHYPERHEAP_MAGIC               UINT32_C(0x18981129)
+
+
+/**
+ * Hypervisor heap minimum alignment (16 bytes).
+ */
+#define MMHYPER_HEAP_ALIGN_MIN          16
+
+/**
+ * The aligned size of the the MMHYPERHEAP structure.
+ */
+#define MMYPERHEAP_HDR_SIZE             RT_ALIGN_Z(sizeof(MMHYPERHEAP), MMHYPER_HEAP_ALIGN_MIN * 4)
+
+/** @name Hypervisor heap chunk flags.
+ * The flags are put in the first bits of the MMHYPERCHUNK::offPrev member.
+ * These bits aren't used anyway because of the chunk minimal alignment (16 bytes).
+ * @{ */
+/** The chunk is free. (The code ASSUMES this is 0!) */
+#define MMHYPERCHUNK_FLAGS_FREE         0x0
+/** The chunk is in use. */
+#define MMHYPERCHUNK_FLAGS_USED         0x1
+/** The type mask. */
+#define MMHYPERCHUNK_FLAGS_TYPE_MASK    0x1
+/** The flag mask */
+#define MMHYPERCHUNK_FLAGS_MASK         0x1
+
+/** Checks if the chunk is free. */
+#define MMHYPERCHUNK_ISFREE(pChunk)                 ( (((pChunk)->offPrev) & MMHYPERCHUNK_FLAGS_TYPE_MASK) == MMHYPERCHUNK_FLAGS_FREE )
+/** Checks if the chunk is used. */
+#define MMHYPERCHUNK_ISUSED(pChunk)                 ( (((pChunk)->offPrev) & MMHYPERCHUNK_FLAGS_TYPE_MASK) == MMHYPERCHUNK_FLAGS_USED )
+/** Toggles FREE/USED flag of a chunk. */
+#define MMHYPERCHUNK_SET_TYPE(pChunk, type)         do { (pChunk)->offPrev = ((pChunk)->offPrev & ~MMHYPERCHUNK_FLAGS_TYPE_MASK) | ((type) & MMHYPERCHUNK_FLAGS_TYPE_MASK); } while (0)
+
+/** Gets the prev offset without the flags. */
+#define MMHYPERCHUNK_GET_OFFPREV(pChunk)            ((int32_t)((pChunk)->offPrev & ~MMHYPERCHUNK_FLAGS_MASK))
+/** Sets the prev offset without changing the flags. */
+#define MMHYPERCHUNK_SET_OFFPREV(pChunk, off)       do { (pChunk)->offPrev = (off) | ((pChunk)->offPrev & MMHYPERCHUNK_FLAGS_MASK); } while (0)
+#if 0
+/** Clears one or more flags. */
+#define MMHYPERCHUNK_FLAGS_OP_CLEAR(pChunk, fFlags) do { ((pChunk)->offPrev) &= ~((fFlags) & MMHYPERCHUNK_FLAGS_MASK); } while (0)
+/** Sets one or more flags. */
+#define MMHYPERCHUNK_FLAGS_OP_SET(pChunk, fFlags)   do { ((pChunk)->offPrev) |=  ((fFlags) & MMHYPERCHUNK_FLAGS_MASK); } while (0)
+/** Checks if one is set. */
+#define MMHYPERCHUNK_FLAGS_OP_ISSET(pChunk, fFlag)  (!!(((pChunk)->offPrev) & ((fFlag) & MMHYPERCHUNK_FLAGS_MASK)))
+#endif
+/** @} */
+
+/** @} */
+
+
+/** @name Page Pool Internals
+ * @{
+ */
+
+/**
+ * Page sub pool
+ *
+ * About the allocation of this structure. To keep the number of heap blocks,
+ * the number of heap calls, and fragmentation low we allocate all the data
+ * related to a MMPAGESUBPOOL node in one chunk. That means that after the
+ * bitmap (which is of variable size) comes the SUPPAGE records and then
+ * follows the lookup tree nodes. (The heap in question is the hyper heap.)
+ */
+typedef struct MMPAGESUBPOOL
+{
+    /** Pointer to next sub pool. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(struct MMPAGESUBPOOL *)   pNext;
+#else
+    R3R0PTRTYPE(struct MMPAGESUBPOOL *) pNext;
+#endif
+    /** Pointer to next sub pool in the free chain.
+     * This is NULL if we're not in the free chain or at the end of it. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(struct MMPAGESUBPOOL *)   pNextFree;
+#else
+    R3R0PTRTYPE(struct MMPAGESUBPOOL *) pNextFree;
+#endif
+    /** Pointer to array of lock ranges.
+     * This is allocated together with the MMPAGESUBPOOL and thus needs no freeing.
+     * It follows immediately after the bitmap.
+     * The reserved field is a pointer to this structure.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(PSUPPAGE)                 paPhysPages;
+#else
+    R3R0PTRTYPE(PSUPPAGE)               paPhysPages;
+#endif
+    /** Pointer to the first page. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(void *)                   pvPages;
+#else
+    R3R0PTRTYPE(void *)                 pvPages;
+#endif
+    /** Size of the subpool. */
+    uint32_t                            cPages;
+    /** Number of free pages. */
+    uint32_t                            cPagesFree;
+    /** The allocation bitmap.
+     * This may extend beyond the end of the defined array size.
+     */
+    uint32_t                            auBitmap[1];
+    /* ... SUPPAGE                      aRanges[1]; */
+} MMPAGESUBPOOL;
+/** Pointer to page sub pool. */
+typedef MMPAGESUBPOOL *PMMPAGESUBPOOL;
+
+/**
+ * Page pool.
+ */
+typedef struct MMPAGEPOOL
+{
+    /** List of subpools. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(PMMPAGESUBPOOL)           pHead;
+#else
+    R3R0PTRTYPE(PMMPAGESUBPOOL)         pHead;
+#endif
+    /** Head of subpools with free pages. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(PMMPAGESUBPOOL)           pHeadFree;
+#else
+    R3R0PTRTYPE(PMMPAGESUBPOOL)         pHeadFree;
+#endif
+    /** AVLPV tree for looking up HC virtual addresses.
+     * The tree contains MMLOOKUPVIRTPP records.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(PAVLPVNODECORE)           pLookupVirt;
+#else
+    R3R0PTRTYPE(PAVLPVNODECORE)         pLookupVirt;
+#endif
+    /** Tree for looking up HC physical addresses.
+     * The tree contains MMLOOKUPPHYSHC records.
+     */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(AVLHCPHYSTREE)            pLookupPhys;
+#else
+    R3R0PTRTYPE(AVLHCPHYSTREE)          pLookupPhys;
+#endif
+    /** Pointer to the VM this pool belongs. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    PVMR3                               pVM;
+#else
+    R3R0PTRTYPE(PVM)                    pVM;
+#endif
+    /** Flag indicating the allocation method.
+     * Set: SUPR3LowAlloc().
+     * Clear: SUPR3PageAllocEx(). */
+    bool                                fLow;
+    /** Number of subpools. */
+    uint32_t                            cSubPools;
+    /** Number of pages in pool. */
+    uint32_t                            cPages;
+#ifdef VBOX_WITH_STATISTICS
+    /** Number of free pages in pool. */
+    uint32_t                            cFreePages;
+# if HC_ARCH_BITS == 32
+    /** Aligning the statistics on an 8 byte boundary. */
+    uint32_t                            u32Alignment;
+# endif
+    /** Number of alloc calls. */
+    STAMCOUNTER                         cAllocCalls;
+    /** Number of free calls. */
+    STAMCOUNTER                         cFreeCalls;
+    /** Number of to phys conversions. */
+    STAMCOUNTER                         cToPhysCalls;
+    /** Number of to virtual conversions. */
+    STAMCOUNTER                         cToVirtCalls;
+    /** Number of real errors. */
+    STAMCOUNTER                         cErrors;
+#endif
+} MMPAGEPOOL;
+#ifndef IN_RC
+AssertCompileMemberAlignment(MMPAGEPOOL, cSubPools, 4);
+# ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(MMPAGEPOOL, cAllocCalls, 8);
+# endif
+#endif
+/** Pointer to page pool. */
+typedef MMPAGEPOOL *PMMPAGEPOOL;
+
+/**
+ * Lookup record for HC virtual memory in the page pool.
+ */
+typedef struct MMPPLOOKUPHCPTR
+{
+    /** The key is virtual address. */
+    AVLPVNODECORE           Core;
+    /** Pointer to subpool if lookup record for a pool. */
+    struct MMPAGESUBPOOL   *pSubPool;
+} MMPPLOOKUPHCPTR;
+/** Pointer to virtual memory lookup record. */
+typedef MMPPLOOKUPHCPTR *PMMPPLOOKUPHCPTR;
+
+/**
+ * Lookup record for HC physical memory.
+ */
+typedef struct MMPPLOOKUPHCPHYS
+{
+    /** The key is physical address. */
+    AVLHCPHYSNODECORE       Core;
+    /** Pointer to SUPPAGE record for this physical address. */
+    PSUPPAGE                pPhysPage;
+} MMPPLOOKUPHCPHYS;
+/** Pointer to physical memory lookup record. */
+typedef MMPPLOOKUPHCPHYS *PMMPPLOOKUPHCPHYS;
+
+/** @} */
+
+
+/**
+ * Hypervisor memory mapping type.
+ */
+typedef enum MMLOOKUPHYPERTYPE
+{
+    /** Invalid record. This is used for record which are incomplete. */
+    MMLOOKUPHYPERTYPE_INVALID = 0,
+    /** Mapping of locked memory. */
+    MMLOOKUPHYPERTYPE_LOCKED,
+    /** Mapping of contiguous HC physical memory. */
+    MMLOOKUPHYPERTYPE_HCPHYS,
+    /** Mapping of contiguous GC physical memory. */
+    MMLOOKUPHYPERTYPE_GCPHYS,
+    /** Mapping of MMIO2 memory. */
+    MMLOOKUPHYPERTYPE_MMIO2,
+    /** Dynamic mapping area (MMR3HyperReserve).
+     * A conversion will require to check what's in the page table for the pages. */
+    MMLOOKUPHYPERTYPE_DYNAMIC
+} MMLOOKUPHYPERTYPE;
+
+/**
+ * Lookup record for the hypervisor memory area.
+ */
+typedef struct MMLOOKUPHYPER
+{
+    /** Byte offset from the start of this record to the next.
+     * If the value is NIL_OFFSET the chain is terminated. */
+    int32_t                 offNext;
+    /** Offset into the hypervisor memory area. */
+    uint32_t                off;
+    /** Size of this part. */
+    uint32_t                cb;
+    /** Locking type. */
+    MMLOOKUPHYPERTYPE       enmType;
+    /** Type specific data */
+    union
+    {
+        /** Locked memory. */
+        struct
+        {
+            /** Host context ring-3 pointer. */
+            R3PTRTYPE(void *)       pvR3;
+            /** Host context ring-0 pointer. Optional. */
+            RTR0PTR                 pvR0;
+            /** Pointer to an array containing the physical address of each page. */
+            R3PTRTYPE(PRTHCPHYS)    paHCPhysPages;
+        } Locked;
+
+        /** Contiguous physical memory. */
+        struct
+        {
+            /** Host context ring-3 pointer. */
+            R3PTRTYPE(void *)       pvR3;
+            /** Host context ring-0 pointer. Optional. */
+            RTR0PTR                 pvR0;
+            /** HC physical address corresponding to pvR3/pvR0. */
+            RTHCPHYS                HCPhys;
+        } HCPhys;
+
+        /** Contiguous guest physical memory. */
+        struct
+        {
+            /** The memory address (Guest Context). */
+            RTGCPHYS                GCPhys;
+        } GCPhys;
+
+        /** MMIO2 memory. */
+        struct
+        {
+            /** The device instance owning the MMIO2 region. */
+            PPDMDEVINSR3            pDevIns;
+            /** The region number. */
+            uint32_t                iRegion;
+            /** The offset into the MMIO2 region. */
+            RTGCPHYS                off;
+        } MMIO2;
+    } u;
+    /** Description. */
+    R3PTRTYPE(const char *) pszDesc;
+} MMLOOKUPHYPER;
+/** Pointer to a hypervisor memory lookup record. */
+typedef MMLOOKUPHYPER *PMMLOOKUPHYPER;
+
+
+/**
+ * Converts a MM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the MM is part of.
+ * @param   pMM   Pointer to MM instance data.
+ */
+#define MM2VM(pMM)  ( (PVM)((uint8_t *)pMM - pMM->offVM) )
+
+
+/**
+ * MM Data (part of VM)
+ */
+typedef struct MM
+{
+    /** Offset to the VM structure.
+     * See MM2VM(). */
+    RTINT                       offVM;
+
+    /** Set if MMR3InitPaging has been called. */
+    bool                        fDoneMMR3InitPaging;
+    /** Set if PGM has been initialized and we can safely call PGMR3Map(). */
+    bool                        fPGMInitialized;
+#if GC_ARCH_BITS == 64 || HC_ARCH_BITS == 64
+    uint32_t                    u32Padding1; /**< alignment padding. */
+#endif
+
+    /** Lookup list for the Hypervisor Memory Area.
+     * The offset is relative to the start of the heap.
+     * Use pHyperHeapR3, pHyperHeapR0 or pHypeRHeapRC to calculate the address.
+     */
+    RTUINT                      offLookupHyper;
+
+    /** The offset of the next static mapping in the Hypervisor Memory Area. */
+    RTUINT                      offHyperNextStatic;
+    /** The size of the HMA.
+     * Starts at 12MB and will be fixed late in the init process. */
+    RTUINT                      cbHyperArea;
+
+    /** Guest address of the Hypervisor Memory Area.
+     * @remarks It's still a bit open whether this should be change to RTRCPTR or
+     *          remain a RTGCPTR. */
+    RTGCPTR                     pvHyperAreaGC;
+
+    /** The hypervisor heap (GC Ptr). */
+    RCPTRTYPE(PMMHYPERHEAP)     pHyperHeapRC;
+#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 64
+    uint32_t                    u32Padding2;
+#endif
+
+    /** The hypervisor heap (R0 Ptr). */
+    R0PTRTYPE(PMMHYPERHEAP)     pHyperHeapR0;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** Page pool - R0 Ptr. */
+    R0PTRTYPE(PMMPAGEPOOL)      pPagePoolR0;
+    /** Page pool pages in low memory R0 Ptr. */
+    R0PTRTYPE(PMMPAGEPOOL)      pPagePoolLowR0;
+#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE */
+
+    /** The hypervisor heap (R3 Ptr). */
+    R3PTRTYPE(PMMHYPERHEAP)     pHyperHeapR3;
+    /** Page pool - R3 Ptr. */
+    R3PTRTYPE(PMMPAGEPOOL)      pPagePoolR3;
+    /** Page pool pages in low memory R3 Ptr. */
+    R3PTRTYPE(PMMPAGEPOOL)      pPagePoolLowR3;
+
+    /** Pointer to the dummy page.
+     * The dummy page is a paranoia thingy used for instance for pure MMIO RAM ranges
+     * to make sure any bugs will not harm whatever the system stores in the first
+     * physical page. */
+    R3PTRTYPE(void *)           pvDummyPage;
+    /** Physical address of the dummy page. */
+    RTHCPHYS                    HCPhysDummyPage;
+
+    /** Size of the base RAM in bytes. (The CFGM RamSize value.) */
+    uint64_t                    cbRamBase;
+    /** The number of base RAM pages that PGM has reserved (GMM).
+     * @remarks Shadow ROMs will be counted twice (RAM+ROM), so it won't be 1:1 with
+     *          what the guest sees. */
+    uint64_t                    cBasePages;
+    /** The number of handy pages that PGM has reserved (GMM).
+     * These are kept out of cBasePages and thus out of the saved state. */
+    uint32_t                    cHandyPages;
+    /** The number of shadow pages PGM has reserved (GMM). */
+    uint32_t                    cShadowPages;
+    /** The number of fixed pages we've reserved (GMM). */
+    uint32_t                    cFixedPages;
+    /** Padding. */
+    uint32_t                    u32Padding0;
+} MM;
+/** Pointer to MM Data (part of VM). */
+typedef MM *PMM;
+
+
+/**
+ * MM data kept in the UVM.
+ */
+typedef struct MMUSERPERVM
+{
+    /** Pointer to the MM R3 Heap. */
+    R3PTRTYPE(PMMHEAP)          pHeap;
+    /** Pointer to the MM Uk Heap. */
+    R3PTRTYPE(PMMUKHEAP)        pUkHeap;
+} MMUSERPERVM;
+/** Pointer to the MM data kept in the UVM. */
+typedef MMUSERPERVM *PMMUSERPERVM;
+
+
+RT_C_DECLS_BEGIN
+
+
+int  mmR3UpdateReservation(PVM pVM);
+
+int  mmR3PagePoolInit(PVM pVM);
+void mmR3PagePoolTerm(PVM pVM);
+
+int  mmR3HeapCreateU(PUVM pUVM, PMMHEAP *ppHeap);
+void mmR3HeapDestroy(PMMHEAP pHeap);
+
+void mmR3UkHeapDestroy(PMMUKHEAP pHeap);
+int  mmR3UkHeapCreateU(PUVM pUVM, PMMUKHEAP *ppHeap);
+
+
+int  mmR3HyperInit(PVM pVM);
+int  mmR3HyperTerm(PVM pVM);
+int  mmR3HyperInitPaging(PVM pVM);
+
+const char *mmGetTagName(MMTAG enmTag);
+
+/**
+ * Converts a pool address to a physical address.
+ * The specified allocation type must match with the address.
+ *
+ * @returns Physical address.
+ * @returns NIL_RTHCPHYS if not found or eType is not matching.
+ * @param   pPool   Pointer to the page pool.
+ * @param   pv      The address to convert.
+ * @thread  The Emulation Thread.
+ */
+RTHCPHYS mmPagePoolPtr2Phys(PMMPAGEPOOL pPool, void *pv);
+
+/**
+ * Converts a pool physical address to a linear address.
+ * The specified allocation type must match with the address.
+ *
+ * @returns Physical address.
+ * @returns NULL if not found or eType is not matching.
+ * @param   pPool       Pointer to the page pool.
+ * @param   HCPhys      The address to convert.
+ * @thread  The Emulation Thread.
+ */
+void *mmPagePoolPhys2Ptr(PMMPAGEPOOL pPool, RTHCPHYS HCPhys);
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/PDMAsyncCompletionFileInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/PDMAsyncCompletionFileInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/PDMAsyncCompletionFileInternal.h	(revision 35333)
@@ -0,0 +1,527 @@
+/* $Id$ */
+/** @file
+ * PDM Async I/O - Transport data asynchronous in R3 using EMT.
+ */
+
+/*
+ * Copyright (C) 2006-2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___PDMAsyncCompletionFileInternal_h
+#define ___PDMAsyncCompletionFileInternal_h
+
+#include <VBox/cfgm.h>
+#include <VBox/stam.h>
+#include <VBox/tm.h>
+#include <iprt/types.h>
+#include <iprt/file.h>
+#include <iprt/thread.h>
+#include <iprt/semaphore.h>
+#include <iprt/critsect.h>
+#include <iprt/avl.h>
+#include <iprt/list.h>
+#include <iprt/spinlock.h>
+#include <iprt/memcache.h>
+
+#include "PDMAsyncCompletionInternal.h"
+
+/** @todo: Revise the caching of tasks. We have currently four caches:
+ *  Per endpoint task cache
+ *  Per class cache
+ *  Per endpoint task segment cache
+ *  Per class task segment cache
+ *
+ *  We could use the RT heap for this probably or extend MMR3Heap (uses RTMemAlloc
+ *  instead of managing larger blocks) to have this global for the whole VM.
+ */
+
+RT_C_DECLS_BEGIN
+
+/**
+ * A few forward declarations.
+ */
+typedef struct PDMASYNCCOMPLETIONENDPOINTFILE *PPDMASYNCCOMPLETIONENDPOINTFILE;
+/** Pointer to a request segment. */
+typedef struct PDMACTASKFILE *PPDMACTASKFILE;
+/** Pointer to the endpoint class data. */
+typedef struct PDMASYNCCOMPLETIONTASKFILE *PPDMASYNCCOMPLETIONTASKFILE;
+/** Pointer to a cache LRU list. */
+typedef struct PDMACFILELRULIST *PPDMACFILELRULIST;
+/** Pointer to the global cache structure. */
+typedef struct PDMACFILECACHEGLOBAL *PPDMACFILECACHEGLOBAL;
+/** Pointer to a task segment. */
+typedef struct PDMACFILETASKSEG *PPDMACFILETASKSEG;
+
+/**
+ * Blocking event types.
+ */
+typedef enum PDMACEPFILEAIOMGRBLOCKINGEVENT
+{
+    /** Invalid tye */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_INVALID = 0,
+    /** An endpoint is added to the manager. */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_ADD_ENDPOINT,
+    /** An endpoint is removed from the manager. */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_REMOVE_ENDPOINT,
+    /** An endpoint is about to be closed. */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_CLOSE_ENDPOINT,
+    /** The manager is requested to terminate */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_SHUTDOWN,
+    /** The manager is requested to suspend */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_SUSPEND,
+    /** The manager is requested to resume */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_RESUME,
+    /** 32bit hack */
+    PDMACEPFILEAIOMGRBLOCKINGEVENT_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEAIOMGRBLOCKINGEVENT;
+
+/**
+ * I/O manager type.
+ */
+typedef enum PDMACEPFILEMGRTYPE
+{
+    /** Simple aka failsafe */
+    PDMACEPFILEMGRTYPE_SIMPLE = 0,
+    /** Async I/O with host cache enabled. */
+    PDMACEPFILEMGRTYPE_ASYNC,
+    /** 32bit hack */
+    PDMACEPFILEMGRTYPE_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEMGRTYPE;
+/** Pointer to a I/O manager type */
+typedef PDMACEPFILEMGRTYPE *PPDMACEPFILEMGRTYPE;
+
+/**
+ * States of the I/O manager.
+ */
+typedef enum PDMACEPFILEMGRSTATE
+{
+    /** Invalid state. */
+    PDMACEPFILEMGRSTATE_INVALID = 0,
+    /** Normal running state accepting new requests
+     * and processing them.
+     */
+    PDMACEPFILEMGRSTATE_RUNNING,
+    /** Fault state - not accepting new tasks for endpoints but waiting for
+     * remaining ones to finish.
+     */
+    PDMACEPFILEMGRSTATE_FAULT,
+    /** Suspending state - not accepting new tasks for endpoints but waiting
+     * for remaining ones to finish.
+     */
+    PDMACEPFILEMGRSTATE_SUSPENDING,
+    /** Shutdown state - not accepting new tasks for endpoints but waiting
+     * for remaining ones to finish.
+     */
+    PDMACEPFILEMGRSTATE_SHUTDOWN,
+    /** The I/O manager waits for all active requests to complete and doesn't queue
+     * new ones because it needs to grow to handle more requests.
+     */
+    PDMACEPFILEMGRSTATE_GROWING,
+    /** 32bit hack */
+    PDMACEPFILEMGRSTATE_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEMGRSTATE;
+
+/**
+ * State of a async I/O manager.
+ */
+typedef struct PDMACEPFILEMGR
+{
+    /** Next Aio manager in the list. */
+    R3PTRTYPE(struct PDMACEPFILEMGR *)     pNext;
+    /** Previous Aio manager in the list. */
+    R3PTRTYPE(struct PDMACEPFILEMGR *)     pPrev;
+    /** Manager type */
+    PDMACEPFILEMGRTYPE                     enmMgrType;
+    /** Current state of the manager. */
+    PDMACEPFILEMGRSTATE                    enmState;
+    /** Event semaphore the manager sleeps on when waiting for new requests. */
+    RTSEMEVENT                             EventSem;
+    /** Flag whether the thread waits in the event semaphore. */
+    volatile bool                          fWaitingEventSem;
+    /** Thread data */
+    RTTHREAD                               Thread;
+    /** The async I/O context for this manager. */
+    RTFILEAIOCTX                           hAioCtx;
+    /** Flag whether the I/O manager was woken up. */
+    volatile bool                          fWokenUp;
+    /** List of endpoints assigned to this manager. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointsHead;
+    /** Number of endpoints assigned to the manager. */
+    unsigned                               cEndpoints;
+    /** Number of requests active currently. */
+    unsigned                               cRequestsActive;
+    /** Number of maximum requests active. */
+    uint32_t                               cRequestsActiveMax;
+    /** Pointer to an array of free async I/O request handles. */
+    RTFILEAIOREQ                          *pahReqsFree;
+    /** Index of the next free entry in the cache. */
+    uint32_t                               iFreeEntry;
+    /** Size of the array. */
+    unsigned                               cReqEntries;
+    /** Memory cache for file range locks. */
+    RTMEMCACHE                             hMemCacheRangeLocks;
+    /** Number of milliseconds to wait until the bandwidth is refreshed for at least
+     * one endpoint and it is possible to process more requests. */
+    RTMSINTERVAL                           msBwLimitExpired;
+    /** Critical section protecting the blocking event handling. */
+    RTCRITSECT                             CritSectBlockingEvent;
+    /** Event semaphore for blocking external events.
+     * The caller waits on it until the async I/O manager
+     * finished processing the event. */
+    RTSEMEVENT                             EventSemBlock;
+    /** Flag whether a blocking event is pending and needs
+     * processing by the I/O manager. */
+    volatile bool                          fBlockingEventPending;
+    /** Blocking event type */
+    volatile PDMACEPFILEAIOMGRBLOCKINGEVENT enmBlockingEvent;
+    /** Event type data */
+    union
+    {
+        /** Add endpoint event. */
+        struct
+        {
+            /** The endpoint to be added */
+            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+        } AddEndpoint;
+        /** Remove endpoint event. */
+        struct
+        {
+            /** The endpoint to be removed */
+            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+        } RemoveEndpoint;
+        /** Close endpoint event. */
+        struct
+        {
+            /** The endpoint to be closed */
+            volatile PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint;
+        } CloseEndpoint;
+    } BlockingEventData;
+} PDMACEPFILEMGR;
+/** Pointer to a async I/O manager state. */
+typedef PDMACEPFILEMGR *PPDMACEPFILEMGR;
+/** Pointer to a async I/O manager state pointer. */
+typedef PPDMACEPFILEMGR *PPPDMACEPFILEMGR;
+
+/**
+ * A file access range lock.
+ */
+typedef struct PDMACFILERANGELOCK
+{
+    /** AVL node in the locked range tree of the endpoint. */
+    AVLRFOFFNODECORE            Core;
+    /** How many tasks have locked this range. */
+    uint32_t                    cRefs;
+    /** Flag whether this is a read or write lock. */
+    bool                        fReadLock;
+    /** List of tasks which are waiting that the range gets unlocked. */
+    PPDMACTASKFILE              pWaitingTasksHead;
+    /** List of tasks which are waiting that the range gets unlocked. */
+    PPDMACTASKFILE              pWaitingTasksTail;
+} PDMACFILERANGELOCK, *PPDMACFILERANGELOCK;
+
+/**
+ * Backend type for the endpoint.
+ */
+typedef enum PDMACFILEEPBACKEND
+{
+    /** Non buffered. */
+    PDMACFILEEPBACKEND_NON_BUFFERED = 0,
+    /** Buffered (i.e host cache enabled) */
+    PDMACFILEEPBACKEND_BUFFERED,
+    /** 32bit hack */
+    PDMACFILEEPBACKEND_32BIT_HACK = 0x7fffffff
+} PDMACFILEEPBACKEND;
+/** Pointer to a backend type. */
+typedef PDMACFILEEPBACKEND *PPDMACFILEEPBACKEND;
+
+/**
+ * Global data for the file endpoint class.
+ */
+typedef struct PDMASYNCCOMPLETIONEPCLASSFILE
+{
+    /** Common data. */
+    PDMASYNCCOMPLETIONEPCLASS           Core;
+    /** Override I/O manager type - set to SIMPLE after failure. */
+    PDMACEPFILEMGRTYPE                  enmMgrTypeOverride;
+    /** Default backend type for the endpoint. */
+    PDMACFILEEPBACKEND                  enmEpBackendDefault;
+    RTCRITSECT                          CritSect;
+    /** Pointer to the head of the async I/O managers. */
+    R3PTRTYPE(PPDMACEPFILEMGR)          pAioMgrHead;
+    /** Number of async I/O managers currently running. */
+    unsigned                            cAioMgrs;
+    /** Maximum number of segments to cache per endpoint */
+    unsigned                            cTasksCacheMax;
+    /** Maximum number of simultaneous outstandingrequests. */
+    uint32_t                            cReqsOutstandingMax;
+    /** Bitmask for checking the alignment of a buffer. */
+    RTR3UINTPTR                         uBitmaskAlignment;
+    /** Flag whether the out of resources warning was printed already. */
+    bool                                fOutOfResourcesWarningPrinted;
+} PDMASYNCCOMPLETIONEPCLASSFILE;
+/** Pointer to the endpoint class data. */
+typedef PDMASYNCCOMPLETIONEPCLASSFILE *PPDMASYNCCOMPLETIONEPCLASSFILE;
+
+typedef enum PDMACEPFILEBLOCKINGEVENT
+{
+    /** The invalid event type */
+    PDMACEPFILEBLOCKINGEVENT_INVALID = 0,
+    /** A task is about to be canceled */
+    PDMACEPFILEBLOCKINGEVENT_CANCEL,
+    /** Usual 32bit hack */
+    PDMACEPFILEBLOCKINGEVENT_32BIT_HACK = 0x7fffffff
+} PDMACEPFILEBLOCKINGEVENT;
+
+/**
+ * States of the endpoint.
+ */
+typedef enum PDMASYNCCOMPLETIONENDPOINTFILESTATE
+{
+    /** Invalid state. */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_INVALID = 0,
+    /** Normal running state accepting new requests
+     * and processing them.
+     */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_ACTIVE,
+    /** The endpoint is about to be closed - not accepting new tasks for endpoints but waiting for
+     *  remaining ones to finish.
+     */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_CLOSING,
+    /** Removing from current I/O manager state - not processing new tasks for endpoints but waiting
+     * for remaining ones to finish.
+     */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_REMOVING,
+    /** The current endpoint will be migrated to another I/O manager. */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_MIGRATING,
+    /** 32bit hack */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE_32BIT_HACK = 0x7fffffff
+} PDMASYNCCOMPLETIONENDPOINTFILESTATE;
+
+/**
+ * Data for the file endpoint.
+ */
+typedef struct PDMASYNCCOMPLETIONENDPOINTFILE
+{
+    /** Common data. */
+    PDMASYNCCOMPLETIONENDPOINT             Core;
+    /** Current state of the endpoint. */
+    PDMASYNCCOMPLETIONENDPOINTFILESTATE    enmState;
+    /** The backend to use for this endpoint. */
+    PDMACFILEEPBACKEND                     enmBackendType;
+    /** async I/O manager this endpoint is assigned to. */
+    R3PTRTYPE(volatile PPDMACEPFILEMGR)    pAioMgr;
+    /** Flags for opening the file. */
+    unsigned                               fFlags;
+    /** File handle. */
+    RTFILE                                 File;
+    /**
+     * Real size of the file. Only updated if
+     * data is appended.
+     */
+    volatile uint64_t                      cbFile;
+    /** List of new tasks. */
+    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksNewHead;
+
+    /** Head of the small cache for allocated task segments for exclusive
+     * use by this endpoint. */
+    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksFreeHead;
+    /** Tail of the small cache for allocated task segments for exclusive
+     * use by this endpoint. */
+    R3PTRTYPE(volatile PPDMACTASKFILE)     pTasksFreeTail;
+    /** Number of elements in the cache. */
+    volatile uint32_t                      cTasksCached;
+
+    /** Flag whether a flush request is currently active */
+    PPDMACTASKFILE                         pFlushReq;
+
+#ifdef VBOX_WITH_STATISTICS
+    /** Time spend in a read. */
+    STAMPROFILEADV                         StatRead;
+    /** Time spend in a write. */
+    STAMPROFILEADV                         StatWrite;
+#endif
+
+    /** Event semaphore for blocking external events.
+     * The caller waits on it until the async I/O manager
+     * finished processing the event. */
+    RTSEMEVENT                             EventSemBlock;
+    /** Flag whether caching is enabled for this file. */
+    bool                                   fCaching;
+    /** Flag whether the file was opened readonly. */
+    bool                                   fReadonly;
+    /** Flag whether the host supports the async flush API. */
+    bool                                   fAsyncFlushSupported;
+#ifdef VBOX_WITH_DEBUGGER
+    /** Status code to inject for the next complete read. */
+    volatile int                           rcReqRead;
+    /** Status code to inject for the next complete write. */
+    volatile int                           rcReqWrite;
+#endif
+    /** Flag whether a blocking event is pending and needs
+     * processing by the I/O manager. */
+    bool                                   fBlockingEventPending;
+    /** Blocking event type */
+    PDMACEPFILEBLOCKINGEVENT               enmBlockingEvent;
+
+    /** Additional data needed for the event types. */
+    union
+    {
+        /** Cancelation event. */
+        struct
+        {
+            /** The task to cancel. */
+            PPDMACTASKFILE                 pTask;
+        } Cancel;
+    } BlockingEventData;
+    /** Data for exclusive use by the assigned async I/O manager. */
+    struct
+    {
+        /** Pointer to the next endpoint assigned to the manager. */
+        R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointNext;
+        /** Pointer to the previous endpoint assigned to the manager. */
+        R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINTFILE) pEndpointPrev;
+        /** List of pending requests (not submitted due to usage restrictions
+         *  or a pending flush request) */
+        R3PTRTYPE(PPDMACTASKFILE)                  pReqsPendingHead;
+        /** Tail of pending requests. */
+        R3PTRTYPE(PPDMACTASKFILE)                  pReqsPendingTail;
+        /** Tree of currently locked ranges.
+         * If a write task is enqueued the range gets locked and any other
+         * task writing to that range has to wait until the task completes.
+         */
+        PAVLRFOFFTREE                              pTreeRangesLocked;
+        /** Number of requests currently being processed for this endpoint
+         * (excluded flush requests). */
+        unsigned                                   cRequestsActive;
+        /** Number of requests processed during the last second. */
+        unsigned                                   cReqsPerSec;
+        /** Current number of processed requests for the current update period. */
+        unsigned                                   cReqsProcessed;
+        /** Flag whether the endpoint is about to be moved to another manager. */
+        bool                                       fMoving;
+        /** Destination I/O manager. */
+        PPDMACEPFILEMGR                            pAioMgrDst;
+    } AioMgr;
+} PDMASYNCCOMPLETIONENDPOINTFILE;
+/** Pointer to the endpoint class data. */
+typedef PDMASYNCCOMPLETIONENDPOINTFILE *PPDMASYNCCOMPLETIONENDPOINTFILE;
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PDMASYNCCOMPLETIONENDPOINTFILE, StatRead, sizeof(uint64_t));
+#endif
+
+/** Request completion function */
+typedef DECLCALLBACK(void)   FNPDMACTASKCOMPLETED(PPDMACTASKFILE pTask, void *pvUser, int rc);
+/** Pointer to a request completion function. */
+typedef FNPDMACTASKCOMPLETED *PFNPDMACTASKCOMPLETED;
+
+/**
+ * Transfer type.
+ */
+typedef enum PDMACTASKFILETRANSFER
+{
+    /** Invalid. */
+    PDMACTASKFILETRANSFER_INVALID = 0,
+    /** Read transfer. */
+    PDMACTASKFILETRANSFER_READ,
+    /** Write transfer. */
+    PDMACTASKFILETRANSFER_WRITE,
+    /** Flush transfer. */
+    PDMACTASKFILETRANSFER_FLUSH
+} PDMACTASKFILETRANSFER;
+
+/**
+ * Data of a request.
+ */
+typedef struct PDMACTASKFILE
+{
+    /** Pointer to the range lock we are waiting for */
+    PPDMACFILERANGELOCK                  pRangeLock;
+    /** Next task in the list. (Depending on the state) */
+    struct PDMACTASKFILE                *pNext;
+    /** Endpoint */
+    PPDMASYNCCOMPLETIONENDPOINTFILE      pEndpoint;
+    /** Transfer type. */
+    PDMACTASKFILETRANSFER                enmTransferType;
+    /** Start offset */
+    RTFOFF                               Off;
+    /** Data segment. */
+    RTSGSEG                              DataSeg;
+    /** When non-zero the segment uses a bounce buffer because the provided buffer
+     * doesn't meet host requirements. */
+    size_t                               cbBounceBuffer;
+    /** Pointer to the used bounce buffer if any. */
+    void                                *pvBounceBuffer;
+    /** Start offset in the bounce buffer to copy from. */
+    uint32_t                             offBounceBuffer;
+    /** Flag whether this is a prefetch request. */
+    bool                                 fPrefetch;
+    /** Already prepared native I/O request.
+     * Used if the request is prepared already but
+     * was not queued because the host has not enough
+     * resources. */
+    RTFILEAIOREQ                         hReq;
+    /** Completion function to call on completion. */
+    PFNPDMACTASKCOMPLETED                pfnCompleted;
+    /** User data */
+    void                                *pvUser;
+} PDMACTASKFILE;
+
+/**
+ * Per task data.
+ */
+typedef struct PDMASYNCCOMPLETIONTASKFILE
+{
+    /** Common data. */
+    PDMASYNCCOMPLETIONTASK Core;
+    /** Number of bytes to transfer until this task completes. */
+    volatile int32_t      cbTransferLeft;
+    /** Flag whether the task completed. */
+    volatile bool         fCompleted;
+    /** Return code. */
+    volatile int          rc;
+} PDMASYNCCOMPLETIONTASKFILE;
+
+int pdmacFileAioMgrFailsafe(RTTHREAD ThreadSelf, void *pvUser);
+int pdmacFileAioMgrNormal(RTTHREAD ThreadSelf, void *pvUser);
+
+int pdmacFileAioMgrNormalInit(PPDMACEPFILEMGR pAioMgr);
+void pdmacFileAioMgrNormalDestroy(PPDMACEPFILEMGR pAioMgr);
+
+int pdmacFileAioMgrCreate(PPDMASYNCCOMPLETIONEPCLASSFILE pEpClass, PPPDMACEPFILEMGR ppAioMgr, PDMACEPFILEMGRTYPE enmMgrType);
+
+int pdmacFileAioMgrAddEndpoint(PPDMACEPFILEMGR pAioMgr, PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+PPDMACTASKFILE pdmacFileEpGetNewTasks(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+PPDMACTASKFILE pdmacFileTaskAlloc(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+void pdmacFileTaskFree(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint,
+                       PPDMACTASKFILE pTask);
+
+int pdmacFileEpAddTask(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMACTASKFILE pTask);
+
+void pdmacFileEpTaskCompleted(PPDMACTASKFILE pTask, void *pvUser, int rc);
+
+int pdmacFileCacheInit(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile, PCFGMNODE pCfgNode);
+void pdmacFileCacheDestroy(PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile);
+int pdmacFileEpCacheInit(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONEPCLASSFILE pClassFile);
+void pdmacFileEpCacheDestroy(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+int pdmacFileEpCacheRead(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
+                         RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
+                         size_t cbRead);
+int pdmacFileEpCacheWrite(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint, PPDMASYNCCOMPLETIONTASKFILE pTask,
+                          RTFOFF off, PCRTSGSEG paSegments, size_t cSegments,
+                          size_t cbWrite);
+int pdmacFileEpCacheFlush(PPDMASYNCCOMPLETIONENDPOINTFILE pEndpoint);
+
+RT_C_DECLS_END
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/PDMAsyncCompletionInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/PDMAsyncCompletionInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/PDMAsyncCompletionInternal.h	(revision 35333)
@@ -0,0 +1,277 @@
+/* $Id$ */
+/** @file
+ * PDM - Pluggable Device Manager, Async I/O Completion internal header.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___PDMAsyncCompletionInternal_h
+#define ___PDMAsyncCompletionInternal_h
+
+#include <iprt/critsect.h>
+#include <iprt/memcache.h>
+#include <iprt/sg.h>
+#include <VBox/types.h>
+#include <VBox/cfgm.h>
+#include <VBox/stam.h>
+#include <VBox/pdmasynccompletion.h>
+#include "PDMInternal.h"
+
+RT_C_DECLS_BEGIN
+
+
+/**
+ * PDM Async completion endpoint operations.
+ */
+typedef struct PDMASYNCCOMPLETIONEPCLASSOPS
+{
+    /** Version identifier. */
+    uint32_t                      u32Version;
+    /** Name of the endpoint class. */
+    const char                   *pcszName;
+    /** Class type. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE enmClassType;
+    /** Size of the global endpoint class data in bytes. */
+    size_t                        cbEndpointClassGlobal;
+    /** Size of an endpoint in bytes. */
+    size_t                        cbEndpoint;
+    /** size of a task in bytes. */
+    size_t                        cbTask;
+
+    /**
+     * Initializes the global data for a endpoint class.
+     *
+     * @returns VBox status code.
+     * @param   pClassGlobals    Pointer to the uninitialized globals data.
+     * @param   pCfgNode         Node for querying configuration data.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnInitialize, (PPDMASYNCCOMPLETIONEPCLASS pClassGlobals, PCFGMNODE pCfgNode));
+
+    /**
+     * Frees all allocated resources which were allocated during init.
+     *
+     * @returns VBox status code.
+     * @param   pClassGlobals    Pointer to the globals data.
+     */
+    DECLR3CALLBACKMEMBER(void, pfnTerminate, (PPDMASYNCCOMPLETIONEPCLASS pClassGlobals));
+
+    /**
+     * Initializes a given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Pointer to the uninitialized endpoint.
+     * @param   pszUri        Pointer to the string containing the endpoint
+     *                        destination (filename, IP address, ...)
+     * @param   fFlags        Creation flags.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpInitialize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                                const char *pszUri, uint32_t fFlags));
+
+    /**
+     * Closes a endpoint finishing all tasks.
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Pointer to the endpoint to be closed.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpClose, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint));
+
+    /**
+     * Initiates a read request from the given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pTask         Pointer to the task object associated with the request.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   off           Where to start reading from.
+     * @param   paSegments    Scatter gather list to store the data in.
+     * @param   cSegments     Number of segments in the list.
+     * @param   cbRead        The overall number of bytes to read.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpRead, (PPDMASYNCCOMPLETIONTASK pTask,
+                                          PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                          PCRTSGSEG paSegments, size_t cSegments,
+                                         size_t cbRead));
+
+    /**
+     * Initiates a write request to the given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pTask         Pointer to the task object associated with the request.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   off           Where to start writing to.
+     * @param   paSegments    Scatter gather list to store the data in.
+     * @param   cSegments     Number of segments in the list.
+     * @param   cbRead        The overall number of bytes to write.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpWrite, (PPDMASYNCCOMPLETIONTASK pTask,
+                                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint, RTFOFF off,
+                                           PCRTSGSEG paSegments, size_t cSegments,
+                                           size_t cbWrite));
+
+    /**
+     * Initiates a flush request on the given endpoint.
+     *
+     * @returns VBox status code.
+     * @param   pTask         Pointer to the task object associated with the request.
+     * @param   pEndpoint     Endpoint the request is for.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpFlush, (PPDMASYNCCOMPLETIONTASK pTask,
+                                           PPDMASYNCCOMPLETIONENDPOINT pEndpoint));
+
+    /**
+     * Queries the size of the endpoint. Optional.
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   pcbSize       Where to store the size of the endpoint.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpGetSize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                             uint64_t *pcbSize));
+
+    /**
+     * Sets the size of the endpoint. Optional.
+     * This is a synchronous operation.
+     *
+     *
+     * @returns VBox status code.
+     * @param   pEndpoint     Endpoint the request is for.
+     * @param   cbSize        New size for the endpoint.
+     */
+    DECLR3CALLBACKMEMBER(int, pfnEpSetSize, (PPDMASYNCCOMPLETIONENDPOINT pEndpoint,
+                                             uint64_t cbSize));
+
+    /** Initialization safety marker. */
+    uint32_t    u32VersionEnd;
+} PDMASYNCCOMPLETIONEPCLASSOPS;
+/** Pointer to a async completion endpoint class operation table. */
+typedef PDMASYNCCOMPLETIONEPCLASSOPS *PPDMASYNCCOMPLETIONEPCLASSOPS;
+/** Const pointer to a async completion endpoint class operation table. */
+typedef const PDMASYNCCOMPLETIONEPCLASSOPS *PCPDMASYNCCOMPLETIONEPCLASSOPS;
+
+/** Version for the endpoint class operations structure. */
+#define PDMAC_EPCLASS_OPS_VERSION 0x00000001
+
+/** Pointer to a bandwidth control manager. */
+typedef struct PDMACBWMGR *PPDMACBWMGR;
+
+/**
+ * PDM Async completion endpoint class.
+ * Common data.
+ */
+typedef struct PDMASYNCCOMPLETIONEPCLASS
+{
+    /** Pointer to the shared VM structure. */
+    PVM                                         pVM;
+    /** Critical section protecting the lists below. */
+    RTCRITSECT                                  CritSect;
+    /** Number of endpoints in the list. */
+    volatile unsigned                           cEndpoints;
+    /** Head of endpoints with this class. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pEndpointsHead;
+    /** Head of the bandwidth managers for this class. */
+    R3PTRTYPE(PPDMACBWMGR)                      pBwMgrsHead;
+    /** Pointer to the callback table. */
+    R3PTRTYPE(PCPDMASYNCCOMPLETIONEPCLASSOPS)   pEndpointOps;
+    /** Task cache. */
+    RTMEMCACHE                                  hMemCacheTasks;
+} PDMASYNCCOMPLETIONEPCLASS;
+/** Pointer to the PDM async completion endpoint class data. */
+typedef PDMASYNCCOMPLETIONEPCLASS *PPDMASYNCCOMPLETIONEPCLASS;
+
+/**
+ * A PDM Async completion endpoint.
+ * Common data.
+ */
+typedef struct PDMASYNCCOMPLETIONENDPOINT
+{
+    /** Next endpoint in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pNext;
+    /** Previous endpoint in the list. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)      pPrev;
+    /** Pointer to the class this endpoint belongs to. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONEPCLASS)       pEpClass;
+    /** Template associated with this endpoint. */
+    PPDMASYNCCOMPLETIONTEMPLATE                 pTemplate;
+    /** Reference count. */
+    unsigned                                    cUsers;
+    /** URI describing the endpoint */
+    char                                       *pszUri;
+    /** Pointer to the assigned bandwidth manager. */
+    volatile PPDMACBWMGR                        pBwMgr;
+#ifdef VBOX_WITH_STATISTICS
+    uint32_t                                    u32Alignment;
+    STAMCOUNTER                                 StatTaskRunTimesNs[10];
+    STAMCOUNTER                                 StatTaskRunTimesMicroSec[10];
+    STAMCOUNTER                                 StatTaskRunTimesMs[10];
+    STAMCOUNTER                                 StatTaskRunTimesSec[10];
+    STAMCOUNTER                                 StatTaskRunOver100Sec;
+    STAMCOUNTER                                 StatIoOpsPerSec;
+    STAMCOUNTER                                 StatIoOpsStarted;
+    STAMCOUNTER                                 StatIoOpsCompleted;
+    uint64_t                                    tsIntervalStartMs;
+    uint64_t                                    cIoOpsCompleted;
+#endif
+} PDMASYNCCOMPLETIONENDPOINT;
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PDMASYNCCOMPLETIONENDPOINT, StatTaskRunTimesNs, sizeof(uint64_t));
+#endif
+
+/**
+ * A PDM async completion task handle.
+ * Common data.
+ */
+typedef struct PDMASYNCCOMPLETIONTASK
+{
+    /** Next task in the list
+     * (for free and assigned tasks). */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTASK)      pNext;
+    /** Previous task in the list
+     * (for free and assigned tasks). */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTASK)      pPrev;
+    /** Endpoint this task is assigned to. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONENDPOINT)  pEndpoint;
+    /** Opaque user data for this task. */
+    void                                   *pvUser;
+    /** Start timestamp. */
+    uint64_t                                tsNsStart;
+} PDMASYNCCOMPLETIONTASK;
+
+/**
+ * Called by the endpoint if a task has finished.
+ *
+ * @returns nothing
+ * @param   pTask                     Pointer to the finished task.
+ * @param   rc                        Status code of the completed request.
+ * @param   fCallCompletionHandler    Flag whether the completion handler should be called to
+ *                                    inform the owner of the task that it has completed.
+ */
+void pdmR3AsyncCompletionCompleteTask(PPDMASYNCCOMPLETIONTASK pTask, int rc, bool fCallCompletionHandler);
+
+/**
+ * Checks if the endpoint is allowed to transfer the given amount of bytes.
+ *
+ * @returns true if the endpoint is allowed to transfer the data.
+ *          false otherwise
+ * @param   pEndpoint                 The endpoint.
+ * @param   cbTransfer                The number of bytes to transfer.
+ * @param   pmsWhenNext               Where to store the number of milliseconds
+ *                                    until the bandwidth is refreshed.
+ *                                    Only set if false is returned.
+ */
+bool pdmacEpIsTransferAllowed(PPDMASYNCCOMPLETIONENDPOINT pEndpoint, uint32_t cbTransfer, RTMSINTERVAL *pmsWhenNext);
+
+RT_C_DECLS_END
+
+extern const PDMASYNCCOMPLETIONEPCLASSOPS g_PDMAsyncCompletionEndpointClassFile;
+
+#endif /* !___PDMAsyncCompletionInternal_h */
+
Index: /trunk/src/VBox/VMM/include/PDMBlkCacheInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/PDMBlkCacheInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/PDMBlkCacheInternal.h	(revision 35333)
@@ -0,0 +1,320 @@
+/* $Id$ */
+/** @file
+ * PDM Block Cache.
+ */
+
+/*
+ * Copyright (C) 2006-2008 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___PDMBlkCacheInternal_h
+#define ___PDMBlkCacheInternal_h
+
+#include <VBox/cfgm.h>
+#include <VBox/stam.h>
+#include <VBox/tm.h>
+#include <VBox/pdmblkcache.h>
+#include <iprt/types.h>
+#include <iprt/file.h>
+#include <iprt/thread.h>
+#include <iprt/semaphore.h>
+#include <iprt/critsect.h>
+#include <iprt/avl.h>
+#include <iprt/list.h>
+#include <iprt/spinlock.h>
+#include <iprt/memcache.h>
+
+RT_C_DECLS_BEGIN
+
+/**
+ * A few forward declarations.
+ */
+/** Pointer to a cache LRU list. */
+typedef struct PDMBLKLRULIST *PPDMBLKLRULIST;
+/** Pointer to the global cache structure. */
+typedef struct PDMBLKCACHEGLOBAL *PPDMBLKCACHEGLOBAL;
+/** Pointer to a cache entry waiter structure. */
+typedef struct PDMBLKCACHEWAITER *PPDMBLKCACHEWAITER;
+
+/**
+ * A cache entry
+ */
+typedef struct PDMBLKCACHEENTRY
+{
+    /** The AVL entry data. */
+    AVLRU64NODECORE                 Core;
+    /** Pointer to the previous element. Used in one of the LRU lists.*/
+    struct PDMBLKCACHEENTRY        *pPrev;
+    /** Pointer to the next element. Used in one of the LRU lists.*/
+    struct PDMBLKCACHEENTRY        *pNext;
+    /** Pointer to the list the entry is in. */
+    PPDMBLKLRULIST                  pList;
+    /** Cache the entry belongs to. */
+    PPDMBLKCACHE                    pBlkCache;
+    /** Flags for this entry. Combinations of PDMACFILECACHE_* #defines */
+    volatile uint32_t               fFlags;
+    /** Reference counter. Prevents eviction of the entry if > 0. */
+    volatile uint32_t               cRefs;
+    /** Size of the entry. */
+    size_t                          cbData;
+    /** Pointer to the memory containing the data. */
+    uint8_t                        *pbData;
+    /** Head of list of tasks waiting for this one to finish. */
+    PPDMBLKCACHEWAITER              pWaitingHead;
+    /** Tail of list of tasks waiting for this one to finish. */
+    PPDMBLKCACHEWAITER              pWaitingTail;
+    /** Node for dirty but not yet committed entries list per endpoint. */
+    RTLISTNODE                      NodeNotCommitted;
+} PDMBLKCACHEENTRY, *PPDMBLKCACHEENTRY;
+/** I/O is still in progress for this entry. This entry is not evictable. */
+#define PDMBLKCACHE_ENTRY_IO_IN_PROGRESS RT_BIT(0)
+/** Entry is locked and thus not evictable. */
+#define PDMBLKCACHE_ENTRY_LOCKED         RT_BIT(1)
+/** Entry is dirty */
+#define PDMBLKCACHE_ENTRY_IS_DIRTY       RT_BIT(2)
+/** Entry is not evictable. */
+#define PDMBLKCACHE_NOT_EVICTABLE  (PDMBLKCACHE_ENTRY_LOCKED | PDMBLKCACHE_ENTRY_IO_IN_PROGRESS | PDMBLKCACHE_ENTRY_IS_DIRTY)
+
+/**
+ * LRU list data
+ */
+typedef struct PDMBLKLRULIST
+{
+    /** Head of the list. */
+    PPDMBLKCACHEENTRY pHead;
+    /** Tail of the list. */
+    PPDMBLKCACHEENTRY pTail;
+    /** Number of bytes cached in the list. */
+    uint32_t          cbCached;
+} PDMBLKLRULIST;
+
+/**
+ * Global cache data.
+ */
+typedef struct PDMBLKCACHEGLOBAL
+{
+    /** Pointer to the owning VM instance. */
+    PVM               pVM;
+    /** Maximum size of the cache in bytes. */
+    uint32_t          cbMax;
+    /** Current size of the cache in bytes. */
+    uint32_t          cbCached;
+    /** Critical section protecting the cache. */
+    RTCRITSECT        CritSect;
+    /** Maximum number of bytes cached. */
+    uint32_t          cbRecentlyUsedInMax;
+    /** Maximum number of bytes in the paged out list .*/
+    uint32_t          cbRecentlyUsedOutMax;
+    /** Recently used cache entries list */
+    PDMBLKLRULIST     LruRecentlyUsedIn;
+    /** Scorecard cache entry list. */
+    PDMBLKLRULIST     LruRecentlyUsedOut;
+    /** List of frequently used cache entries */
+    PDMBLKLRULIST     LruFrequentlyUsed;
+    /** Commit timeout in milli seconds */
+    uint32_t          u32CommitTimeoutMs;
+    /** Number of dirty bytes needed to start a commit of the data to the disk. */
+    uint32_t          cbCommitDirtyThreshold;
+    /** Current number of dirty bytes in the cache. */
+    volatile uint32_t cbDirty;
+    /** Flag whether a commit is currently in progress. */
+    volatile bool     fCommitInProgress;
+    /** Commit interval timer */
+    PTMTIMERR3        pTimerCommit;
+    /** Number of endpoints using the cache. */
+    uint32_t          cRefs;
+    /** List of all users of this cache. */
+    RTLISTNODE        ListUsers;
+#ifdef VBOX_WITH_STATISTICS
+    /** Hit counter. */
+    STAMCOUNTER       cHits;
+    /** Partial hit counter. */
+    STAMCOUNTER       cPartialHits;
+    /** Miss counter. */
+    STAMCOUNTER       cMisses;
+    /** Bytes read from cache. */
+    STAMCOUNTER       StatRead;
+    /** Bytes written to the cache. */
+    STAMCOUNTER       StatWritten;
+    /** Time spend to get an entry in the AVL tree. */
+    STAMPROFILEADV    StatTreeGet;
+    /** Time spend to insert an entry in the AVL tree. */
+    STAMPROFILEADV    StatTreeInsert;
+    /** Time spend to remove an entry in the AVL tree. */
+    STAMPROFILEADV    StatTreeRemove;
+    /** Number of times a buffer could be reused. */
+    STAMCOUNTER       StatBuffersReused;
+#endif
+    /** Flag whether the VM was suspended becaus of an I/O error. */
+    volatile bool     fIoErrorVmSuspended;
+} PDMBLKCACHEGLOBAL;
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PDMBLKCACHEGLOBAL, cHits, sizeof(uint64_t));
+#endif
+
+/**
+ * Block cache type.
+ */
+typedef enum PDMBLKCACHETYPE
+{
+    /** Device . */
+    PDMBLKCACHETYPE_DEV = 1,
+    /** Driver consumer. */
+    PDMBLKCACHETYPE_DRV,
+    /** Internal consumer. */
+    PDMBLKCACHETYPE_INTERNAL,
+    /** Usb consumer. */
+    PDMBLKCACHETYPE_USB
+} PDMBLKCACHETYPE;
+
+/**
+ * Per user cache data.
+ */
+typedef struct PDMBLKCACHE
+{
+    /** Pointer to the id for the cache. */
+    char                         *pszId;
+    /** AVL tree managing cache entries. */
+    PAVLRU64TREE                  pTree;
+    /** R/W semaphore protecting cached entries for this endpoint. */
+    RTSEMRW                       SemRWEntries;
+    /** Pointer to the gobal cache data */
+    PPDMBLKCACHEGLOBAL            pCache;
+    /** Lock protecting the dirty entries list. */
+    RTSPINLOCK                    LockList;
+    /** List of dirty but not committed entries for this endpoint. */
+    RTLISTNODE                    ListDirtyNotCommitted;
+    /** Node of the cache user list. */
+    RTLISTNODE                    NodeCacheUser;
+    /** Block cache type. */
+    PDMBLKCACHETYPE               enmType;
+    /** Type specific data. */
+    union
+    {
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DEV */
+        struct
+        {
+            /** Pointer to the device instance owning the block cache. */
+            R3PTRTYPE(PPDMDEVINS)                     pDevIns;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEDEV)  pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDEV)   pfnXferEnqueue;
+        } Dev;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_DRV */
+        struct
+        {
+            /** Pointer to the driver instance owning the block cache. */
+            R3PTRTYPE(PPDMDRVINS)                     pDrvIns;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEDRV)  pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEDRV)   pfnXferEnqueue;
+        } Drv;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_INTERNAL */
+        struct
+        {
+            /** Pointer to user data. */
+            R3PTRTYPE(void *)                         pvUser;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEINT)  pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEINT)   pfnXferEnqueue;
+        } Int;
+        /** PDMASYNCCOMPLETIONTEMPLATETYPE_USB */
+        struct
+        {
+            /** Pointer to the usb instance owning the template. */
+            R3PTRTYPE(PPDMUSBINS)                     pUsbIns;
+            /** Complete callback to the user. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERCOMPLETEUSB)  pfnXferComplete;
+            /** I/O enqueue callback. */
+            R3PTRTYPE(PFNPDMBLKCACHEXFERENQUEUEUSB)   pfnXferEnqueue;
+        } Usb;
+    } u;
+
+#ifdef VBOX_WITH_STATISTICS
+    uint32_t    u32Alignment;
+    /** Number of times a write was deferred because the cache entry was still in progress */
+    STAMCOUNTER StatWriteDeferred;
+#endif
+
+    /** Flag whether the cache was suspended. */
+    volatile bool                 fSuspended;
+
+} PDMBLKCACHE, *PPDMBLKCACHE;
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PDMBLKCACHE, StatWriteDeferred, sizeof(uint64_t));
+#endif
+
+/**
+ * I/O task.
+ */
+typedef struct PDMBLKCACHEREQ
+{
+    /** Opaque user data returned on completion. */
+    void             *pvUser;
+    /** Number of bytes to transfer. */
+    volatile uint32_t cbXfer;
+    /** Number of pending transfers (waiting for a cache entry and passed through). */
+    volatile uint32_t cXfersPending;
+    /** Status code. */
+    volatile int      rcReq;
+} PDMBLKCACHEREQ, *PPDMBLKCACHEREQ;
+
+/**
+ * I/O transfer from the cache to the underlying medium.
+ */
+typedef struct PDMBLKCACHEIOXFER
+{
+    /** Flag whether the I/O xfer updates a cache entry or updates the request directl. */
+    bool fIoCache;
+    /** Type dependent data. */
+    union
+    {
+        /** Pointer to the entry the transfer updates. */
+        PPDMBLKCACHEENTRY pEntry;
+        /** Pointer to the request the ztransfer updates. */
+        PPDMBLKCACHEREQ   pReq;
+    };
+    /** Segment used if a cache entry is updated. */
+    RTSGSEG SgSeg;
+    /** S/G buffer. */
+    RTSGBUF SgBuf;
+    /** Transfer direction. */
+    PDMBLKCACHEXFERDIR enmXferDir;
+    /** Size of the transfer. */
+    size_t             cbXfer;
+} PDMBLKCACHEIOXFER;
+
+/**
+ * Cache waiter
+ */
+typedef struct PDMBLKCACHEWAITER
+{
+    /* Next waiter in the list. */
+    struct PDMBLKCACHEWAITER *pNext;
+    /** S/G buffer holding or receiving data. */
+    RTSGBUF                   SgBuf;
+    /** Offset into the cache entry to start the transfer. */
+    uint32_t                  offCacheEntry;
+    /** How many bytes to transfer. */
+    size_t                    cbTransfer;
+    /** Flag whether the task wants to read or write into the entry. */
+    bool                      fWrite;
+    /** Task the waiter is for. */
+    PPDMBLKCACHEREQ           pReq;
+} PDMBLKCACHEWAITER;
+
+RT_C_DECLS_END
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/PDMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/PDMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/PDMInternal.h	(revision 35333)
@@ -0,0 +1,1192 @@
+/* $Id$ */
+/** @file
+ * PDM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___PDMInternal_h
+#define ___PDMInternal_h
+
+#include <VBox/types.h>
+#include <VBox/param.h>
+#include <VBox/cfgm.h>
+#include <VBox/stam.h>
+#include <VBox/vusb.h>
+#include <VBox/pdmasynccompletion.h>
+#include <VBox/pdmblkcache.h>
+#include <VBox/pdmcommon.h>
+#include <iprt/assert.h>
+#include <iprt/critsect.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_pdm_int       Internal
+ * @ingroup grp_pdm
+ * @internal
+ * @{
+ */
+
+/** @def PDM_WITH_R3R0_CRIT_SECT
+ * Enables or disabled ring-3/ring-0 critical sections. */
+#if defined(DOXYGEN_RUNNING) || 1
+# define PDM_WITH_R3R0_CRIT_SECT
+#endif
+
+/** @def PDMCRITSECT_STRICT
+ * Enables/disables PDM critsect strictness like deadlock detection. */
+#if (defined(RT_LOCK_STRICT) && defined(IN_RING3)) || defined(DOXYGEN_RUNNING)
+# define PDMCRITSECT_STRICT
+#endif
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+
+/** Pointer to a PDM Device. */
+typedef struct PDMDEV *PPDMDEV;
+/** Pointer to a pointer to a PDM Device. */
+typedef PPDMDEV *PPPDMDEV;
+
+/** Pointer to a PDM USB Device. */
+typedef struct PDMUSB *PPDMUSB;
+/** Pointer to a pointer to a PDM USB Device. */
+typedef PPDMUSB *PPPDMUSB;
+
+/** Pointer to a PDM Driver. */
+typedef struct PDMDRV *PPDMDRV;
+/** Pointer to a pointer to a PDM Driver. */
+typedef PPDMDRV *PPPDMDRV;
+
+/** Pointer to a PDM Logical Unit. */
+typedef struct PDMLUN *PPDMLUN;
+/** Pointer to a pointer to a PDM Logical Unit. */
+typedef PPDMLUN *PPPDMLUN;
+
+/** Pointer to a PDM PCI Bus instance. */
+typedef struct PDMPCIBUS *PPDMPCIBUS;
+/** Pointer to a DMAC instance. */
+typedef struct PDMDMAC *PPDMDMAC;
+/** Pointer to a RTC instance. */
+typedef struct PDMRTC *PPDMRTC;
+
+/** Pointer to an USB HUB registration record. */
+typedef struct PDMUSBHUB *PPDMUSBHUB;
+
+/**
+ * Supported asynchronous completion endpoint classes.
+ */
+typedef enum PDMASYNCCOMPLETIONEPCLASSTYPE
+{
+    /** File class. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_FILE = 0,
+    /** Number of supported classes. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_MAX,
+    /** 32bit hack. */
+    PDMASYNCCOMPLETIONEPCLASSTYPE_32BIT_HACK = 0x7fffffff
+} PDMASYNCCOMPLETIONEPCLASSTYPE;
+
+/**
+ * Private device instance data.
+ */
+typedef struct PDMDEVINSINT
+{
+    /** Pointer to the next instance (HC Ptr).
+     * (Head is pointed to by PDM::pDevInstances.) */
+    R3PTRTYPE(PPDMDEVINS)           pNextR3;
+    /** Pointer to the next per device instance (HC Ptr).
+     * (Head is pointed to by PDMDEV::pInstances.) */
+    R3PTRTYPE(PPDMDEVINS)           pPerDeviceNextR3;
+    /** Pointer to device structure - HC Ptr. */
+    R3PTRTYPE(PPDMDEV)              pDevR3;
+    /** Pointer to the list of logical units associated with the device. (FIFO) */
+    R3PTRTYPE(PPDMLUN)              pLunsR3;
+    /** Pointer to the asynchronous notification callback set while in
+     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
+    R3PTRTYPE(PFNPDMDEVASYNCNOTIFY) pfnAsyncNotify;
+    /** Configuration handle to the instance node. */
+    R3PTRTYPE(PCFGMNODE)            pCfgHandle;
+
+    /** R3 pointer to the VM this instance was created for. */
+    PVMR3                           pVMR3;
+    /** R3 pointer to associated PCI device structure. */
+    R3PTRTYPE(struct PCIDevice *)   pPciDeviceR3;
+    /** R3 pointer to associated PCI bus structure. */
+    R3PTRTYPE(PPDMPCIBUS)           pPciBusR3;
+
+    /** R0 pointer to the VM this instance was created for. */
+    PVMR0                           pVMR0;
+    /** R0 pointer to associated PCI device structure. */
+    R0PTRTYPE(struct PCIDevice *)   pPciDeviceR0;
+    /** R0 pointer to associated PCI bus structure. */
+    R0PTRTYPE(PPDMPCIBUS)           pPciBusR0;
+
+    /** RC pointer to the VM this instance was created for. */
+    PVMRC                           pVMRC;
+    /** RC pointer to associated PCI device structure. */
+    RCPTRTYPE(struct PCIDevice *)   pPciDeviceRC;
+    /** RC pointer to associated PCI bus structure. */
+    RCPTRTYPE(PPDMPCIBUS)           pPciBusRC;
+
+    /** Flags, see PDMDEVINSINT_FLAGS_XXX. */
+    uint32_t                        fIntFlags;
+} PDMDEVINSINT;
+
+/** @name PDMDEVINSINT::fIntFlags
+ * @{ */
+/** Used by pdmR3Load to mark device instances it found in the saved state. */
+#define PDMDEVINSINT_FLAGS_FOUND         RT_BIT_32(0)
+/** Indicates that the device hasn't been powered on or resumed.
+ * This is used by PDMR3PowerOn, PDMR3Resume, PDMR3Suspend and PDMR3PowerOff
+ * to make sure each device gets exactly one notification for each of those
+ * events.  PDMR3Resume and PDMR3PowerOn also makes use of it to bail out on
+ * a failure (already resumed/powered-on devices are suspended). */
+#define PDMDEVINSINT_FLAGS_SUSPENDED     RT_BIT_32(1)
+/** Indicates that the device has been reset already.  Used by PDMR3Reset. */
+#define PDMDEVINSINT_FLAGS_RESET         RT_BIT_32(2)
+/** @} */
+
+
+/**
+ * Private USB device instance data.
+ */
+typedef struct PDMUSBINSINT
+{
+    /** The UUID of this instance. */
+    RTUUID                          Uuid;
+    /** Pointer to the next instance.
+     * (Head is pointed to by PDM::pUsbInstances.) */
+    R3PTRTYPE(PPDMUSBINS)           pNext;
+    /** Pointer to the next per USB device instance.
+     * (Head is pointed to by PDMUSB::pInstances.) */
+    R3PTRTYPE(PPDMUSBINS)           pPerDeviceNext;
+
+    /** Pointer to device structure. */
+    R3PTRTYPE(PPDMUSB)              pUsbDev;
+
+    /** Pointer to the VM this instance was created for. */
+    PVMR3                           pVM;
+    /** Pointer to the list of logical units associated with the device. (FIFO) */
+    R3PTRTYPE(PPDMLUN)              pLuns;
+    /** The per instance device configuration. */
+    R3PTRTYPE(PCFGMNODE)            pCfg;
+    /** Same as pCfg if the configuration should be deleted when detaching the device. */
+    R3PTRTYPE(PCFGMNODE)            pCfgDelete;
+    /** The global device configuration. */
+    R3PTRTYPE(PCFGMNODE)            pCfgGlobal;
+
+    /** Pointer to the USB hub this device is attached to.
+     * This is NULL if the device isn't connected to any HUB. */
+    R3PTRTYPE(PPDMUSBHUB)           pHub;
+    /** The port number that we're connected to. */
+    uint32_t                        iPort;
+    /** Indicates that the USB device hasn't been powered on or resumed.
+     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
+    bool                            fVMSuspended;
+    /** Indicates that the USB device has been reset. */
+    bool                            fVMReset;
+    /** Pointer to the asynchronous notification callback set while in
+     * FNPDMDEVSUSPEND or FNPDMDEVPOWEROFF. */
+    R3PTRTYPE(PFNPDMUSBASYNCNOTIFY) pfnAsyncNotify;
+} PDMUSBINSINT;
+
+
+/**
+ * Private driver instance data.
+ */
+typedef struct PDMDRVINSINT
+{
+    /** Pointer to the driver instance above.
+     * This is NULL for the topmost drive. */
+    R3PTRTYPE(PPDMDRVINS)           pUp;
+    /** Pointer to the driver instance below.
+     * This is NULL for the bottommost driver. */
+    R3PTRTYPE(PPDMDRVINS)           pDown;
+    /** Pointer to the logical unit this driver chained on. */
+    R3PTRTYPE(PPDMLUN)              pLun;
+    /** Pointer to driver structure from which this was instantiated. */
+    R3PTRTYPE(PPDMDRV)              pDrv;
+    /** Pointer to the VM this instance was created for, ring-3 context. */
+    PVMR3                           pVMR3;
+    /** Pointer to the VM this instance was created for, ring-0 context. */
+    PVMR0                           pVMR0;
+    /** Pointer to the VM this instance was created for, raw-mode context. */
+    PVMRC                           pVMRC;
+    /** Flag indicating that the driver is being detached and destroyed.
+     * (Helps detect potential recursive detaching.) */
+    bool                            fDetaching;
+    /** Indicates that the driver hasn't been powered on or resumed.
+     * See PDMDEVINSINT_FLAGS_SUSPENDED. */
+    bool                            fVMSuspended;
+    /** Indicates that the driver has been reset already. */
+    bool                            fVMReset;
+    /** Set if allocated on the hyper heap, false if on the ring-3 heap. */
+    bool                            fHyperHeap;
+    /** Pointer to the asynchronous notification callback set while in
+     * PDMUSBREG::pfnVMSuspend or PDMUSBREG::pfnVMPowerOff. */
+    R3PTRTYPE(PFNPDMDRVASYNCNOTIFY) pfnAsyncNotify;
+    /** Configuration handle to the instance node. */
+    R3PTRTYPE(PCFGMNODE)            pCfgHandle;
+    /** Pointer to the ring-0 request handler function. */
+    PFNPDMDRVREQHANDLERR0           pfnReqHandlerR0;
+} PDMDRVINSINT;
+
+
+/**
+ * Private critical section data.
+ */
+typedef struct PDMCRITSECTINT
+{
+    /** The critical section core which is shared with IPRT. */
+    RTCRITSECT                      Core;
+    /** Pointer to the next critical section.
+     * This chain is used for relocating pVMRC and device cleanup. */
+    R3PTRTYPE(struct PDMCRITSECTINT *) pNext;
+    /** Owner identifier.
+     * This is pDevIns if the owner is a device. Similarly for a driver or service.
+     * PDMR3CritSectInit() sets this to point to the critsect itself. */
+    RTR3PTR                         pvKey;
+    /** Pointer to the VM - R3Ptr. */
+    PVMR3                           pVMR3;
+    /** Pointer to the VM - R0Ptr. */
+    PVMR0                           pVMR0;
+    /** Pointer to the VM - GCPtr. */
+    PVMRC                           pVMRC;
+    /** Alignment padding. */
+    uint32_t                        padding;
+    /** Event semaphore that is scheduled to be signaled upon leaving the
+     * critical section. This is Ring-3 only of course. */
+    RTSEMEVENT                      EventToSignal;
+    /** The lock name. */
+    R3PTRTYPE(const char *)         pszName;
+    /** R0/RC lock contention. */
+    STAMCOUNTER                     StatContentionRZLock;
+    /** R0/RC unlock contention. */
+    STAMCOUNTER                     StatContentionRZUnlock;
+    /** R3 lock contention. */
+    STAMCOUNTER                     StatContentionR3;
+    /** Profiling the time the section is locked. */
+    STAMPROFILEADV                  StatLocked;
+} PDMCRITSECTINT;
+AssertCompileMemberAlignment(PDMCRITSECTINT, StatContentionRZLock, 8);
+/** Pointer to private critical section data. */
+typedef PDMCRITSECTINT *PPDMCRITSECTINT;
+
+/** Indicates that the critical section is queued for unlock.
+ * PDMCritSectIsOwner and PDMCritSectIsOwned optimizations. */
+#define PDMCRITSECT_FLAGS_PENDING_UNLOCK    RT_BIT_32(17)
+
+
+/**
+ * The usual device/driver/internal/external stuff.
+ */
+typedef enum
+{
+    /** The usual invalid entry. */
+    PDMTHREADTYPE_INVALID = 0,
+    /** Device type. */
+    PDMTHREADTYPE_DEVICE,
+    /** USB Device type. */
+    PDMTHREADTYPE_USB,
+    /** Driver type. */
+    PDMTHREADTYPE_DRIVER,
+    /** Internal type. */
+    PDMTHREADTYPE_INTERNAL,
+    /** External type. */
+    PDMTHREADTYPE_EXTERNAL,
+    /** The usual 32-bit hack. */
+    PDMTHREADTYPE_32BIT_HACK = 0x7fffffff
+} PDMTHREADTYPE;
+
+
+/**
+ * The internal structure for the thread.
+ */
+typedef struct PDMTHREADINT
+{
+    /** The VM pointer. */
+    PVMR3                           pVM;
+    /** The event semaphore the thread blocks on when not running. */
+    RTSEMEVENTMULTI                 BlockEvent;
+    /** The event semaphore the thread sleeps on while running. */
+    RTSEMEVENTMULTI                 SleepEvent;
+    /** Pointer to the next thread. */
+    R3PTRTYPE(struct PDMTHREAD *)   pNext;
+    /** The thread type. */
+    PDMTHREADTYPE                   enmType;
+} PDMTHREADINT;
+
+
+
+/* Must be included after PDMDEVINSINT is defined. */
+#define PDMDEVINSINT_DECLARED
+#define PDMUSBINSINT_DECLARED
+#define PDMDRVINSINT_DECLARED
+#define PDMCRITSECTINT_DECLARED
+#define PDMTHREADINT_DECLARED
+#ifdef ___VBox_pdm_h
+# error "Invalid header PDM order. Include PDMInternal.h before VBox/pdm.h!"
+#endif
+RT_C_DECLS_END
+#include <VBox/pdm.h>
+RT_C_DECLS_BEGIN
+
+/**
+ * PDM Logical Unit.
+ *
+ * This typically the representation of a physical port on a
+ * device, like for instance the PS/2 keyboard port on the
+ * keyboard controller device. The LUNs are chained on the
+ * device the belong to (PDMDEVINSINT::pLunsR3).
+ */
+typedef struct PDMLUN
+{
+    /** The LUN - The Logical Unit Number. */
+    RTUINT                          iLun;
+    /** Pointer to the next LUN. */
+    PPDMLUN                         pNext;
+    /** Pointer to the top driver in the driver chain. */
+    PPDMDRVINS                      pTop;
+    /** Pointer to the bottom driver in the driver chain. */
+    PPDMDRVINS                      pBottom;
+    /** Pointer to the device instance which the LUN belongs to.
+     * Either this is set or pUsbIns is set. Both is never set at the same time. */
+    PPDMDEVINS                      pDevIns;
+    /** Pointer to the USB device instance which the LUN belongs to. */
+    PPDMUSBINS                      pUsbIns;
+    /** Pointer to the device base interface. */
+    PPDMIBASE                       pBase;
+    /** Description of this LUN. */
+    const char                     *pszDesc;
+} PDMLUN;
+
+
+/**
+ * PDM Device.
+ */
+typedef struct PDMDEV
+{
+    /** Pointer to the next device (R3 Ptr). */
+    R3PTRTYPE(PPDMDEV)              pNext;
+    /** Device name length. (search optimization) */
+    RTUINT                          cchName;
+    /** Registration structure. */
+    R3PTRTYPE(const struct PDMDEVREG *) pReg;
+    /** Number of instances. */
+    uint32_t                        cInstances;
+    /** Pointer to chain of instances (R3 Ptr). */
+    PPDMDEVINSR3                    pInstances;
+    /** The search path for raw-mode context modules (';' as separator). */
+    char                           *pszRCSearchPath;
+    /** The search path for ring-0 context modules (';' as separator). */
+    char                           *pszR0SearchPath;
+} PDMDEV;
+
+
+/**
+ * PDM USB Device.
+ */
+typedef struct PDMUSB
+{
+    /** Pointer to the next device (R3 Ptr). */
+    R3PTRTYPE(PPDMUSB)              pNext;
+    /** Device name length. (search optimization) */
+    RTUINT                          cchName;
+    /** Registration structure. */
+    R3PTRTYPE(const struct PDMUSBREG *) pReg;
+    /** Next instance number. */
+    uint32_t                        iNextInstance;
+    /** Pointer to chain of instances (R3 Ptr). */
+    R3PTRTYPE(PPDMUSBINS)           pInstances;
+} PDMUSB;
+
+
+/**
+ * PDM Driver.
+ */
+typedef struct PDMDRV
+{
+    /** Pointer to the next device. */
+    PPDMDRV                         pNext;
+    /** Registration structure. */
+    const struct PDMDRVREG *        pReg;
+    /** Current number of instances. */
+    uint32_t                        cInstances;
+    /** The next instance number. */
+    uint32_t                        iNextInstance;
+    /** The search path for raw-mode context modules (';' as separator). */
+    char                           *pszRCSearchPath;
+    /** The search path for ring-0 context modules (';' as separator). */
+    char                           *pszR0SearchPath;
+} PDMDRV;
+
+
+/**
+ * PDM registered PIC device.
+ */
+typedef struct PDMPIC
+{
+    /** Pointer to the PIC device instance - R3. */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** @copydoc PDMPICREG::pfnSetIrqR3 */
+    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
+    /** @copydoc PDMPICREG::pfnGetInterruptR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnGetInterruptR3,(PPDMDEVINS pDevIns));
+
+    /** Pointer to the PIC device instance - R0. */
+    PPDMDEVINSR0                    pDevInsR0;
+    /** @copydoc PDMPICREG::pfnSetIrqR3 */
+    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
+    /** @copydoc PDMPICREG::pfnGetInterruptR3 */
+    DECLR0CALLBACKMEMBER(int,       pfnGetInterruptR0,(PPDMDEVINS pDevIns));
+
+    /** Pointer to the PIC device instance - RC. */
+    PPDMDEVINSRC                    pDevInsRC;
+    /** @copydoc PDMPICREG::pfnSetIrqR3 */
+    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
+    /** @copydoc PDMPICREG::pfnGetInterruptR3 */
+    DECLRCCALLBACKMEMBER(int,       pfnGetInterruptRC,(PPDMDEVINS pDevIns));
+    /** Alignment padding. */
+    RTRCPTR                         RCPtrPadding;
+} PDMPIC;
+
+
+/**
+ * PDM registered APIC device.
+ */
+typedef struct PDMAPIC
+{
+    /** Pointer to the APIC device instance - R3 Ptr. */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** @copydoc PDMAPICREG::pfnGetInterruptR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnGetInterruptR3,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnHasPendingIrqR3 */
+    DECLR3CALLBACKMEMBER(bool,      pfnHasPendingIrqR3,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnSetBaseR3 */
+    DECLR3CALLBACKMEMBER(void,      pfnSetBaseR3,(PPDMDEVINS pDevIns, uint64_t u64Base));
+    /** @copydoc PDMAPICREG::pfnGetBaseR3 */
+    DECLR3CALLBACKMEMBER(uint64_t,  pfnGetBaseR3,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnSetTPRR3 */
+    DECLR3CALLBACKMEMBER(void,      pfnSetTPRR3,(PPDMDEVINS pDevIns, VMCPUID idCpu, uint8_t u8TPR));
+    /** @copydoc PDMAPICREG::pfnGetTPRR3 */
+    DECLR3CALLBACKMEMBER(uint8_t,   pfnGetTPRR3,(PPDMDEVINS pDevIns, VMCPUID idCpu));
+    /** @copydoc PDMAPICREG::pfnWriteMSRR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnWriteMSRR3, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t u64Value));
+    /** @copydoc PDMAPICREG::pfnReadMSRR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnReadMSRR3, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t *pu64Value));
+    /** @copydoc PDMAPICREG::pfnBusDeliverR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnBusDeliverR3,(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
+                                                     uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode));
+    /** @copydoc PDMAPICREG::pfnLocalInterruptR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnLocalInterruptR3,(PPDMDEVINS pDevIns, uint8_t u8Pin, uint8_t u8Level));
+
+    /** Pointer to the APIC device instance - R0 Ptr. */
+    PPDMDEVINSR0                    pDevInsR0;
+    /** @copydoc PDMAPICREG::pfnGetInterruptR3 */
+    DECLR0CALLBACKMEMBER(int,       pfnGetInterruptR0,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnHasPendingIrqR3 */
+    DECLR0CALLBACKMEMBER(bool,      pfnHasPendingIrqR0,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnSetBaseR3 */
+    DECLR0CALLBACKMEMBER(void,      pfnSetBaseR0,(PPDMDEVINS pDevIns, uint64_t u64Base));
+    /** @copydoc PDMAPICREG::pfnGetBaseR3 */
+    DECLR0CALLBACKMEMBER(uint64_t,  pfnGetBaseR0,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnSetTPRR3 */
+    DECLR0CALLBACKMEMBER(void,      pfnSetTPRR0,(PPDMDEVINS pDevIns, VMCPUID idCpu, uint8_t u8TPR));
+    /** @copydoc PDMAPICREG::pfnGetTPRR3 */
+    DECLR0CALLBACKMEMBER(uint8_t,   pfnGetTPRR0,(PPDMDEVINS pDevIns, VMCPUID idCpu));
+     /** @copydoc PDMAPICREG::pfnWriteMSRR3 */
+    DECLR0CALLBACKMEMBER(uint32_t,  pfnWriteMSRR0, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t u64Value));
+    /** @copydoc PDMAPICREG::pfnReadMSRR3 */
+    DECLR0CALLBACKMEMBER(uint32_t,  pfnReadMSRR0, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t *pu64Value));
+    /** @copydoc PDMAPICREG::pfnBusDeliverR3 */
+    DECLR0CALLBACKMEMBER(int,       pfnBusDeliverR0,(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
+                                                     uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode));
+    /** @copydoc PDMAPICREG::pfnLocalInterruptR3 */
+    DECLR0CALLBACKMEMBER(int,       pfnLocalInterruptR0,(PPDMDEVINS pDevIns, uint8_t u8Pin, uint8_t u8Level));
+
+    /** Pointer to the APIC device instance - RC Ptr. */
+    PPDMDEVINSRC                    pDevInsRC;
+    /** @copydoc PDMAPICREG::pfnGetInterruptR3 */
+    DECLRCCALLBACKMEMBER(int,       pfnGetInterruptRC,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnHasPendingIrqR3 */
+    DECLRCCALLBACKMEMBER(bool,      pfnHasPendingIrqRC,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnSetBaseR3 */
+    DECLRCCALLBACKMEMBER(void,      pfnSetBaseRC,(PPDMDEVINS pDevIns, uint64_t u64Base));
+    /** @copydoc PDMAPICREG::pfnGetBaseR3 */
+    DECLRCCALLBACKMEMBER(uint64_t,  pfnGetBaseRC,(PPDMDEVINS pDevIns));
+    /** @copydoc PDMAPICREG::pfnSetTPRR3 */
+    DECLRCCALLBACKMEMBER(void,      pfnSetTPRRC,(PPDMDEVINS pDevIns, VMCPUID idCpu, uint8_t u8TPR));
+    /** @copydoc PDMAPICREG::pfnGetTPRR3 */
+    DECLRCCALLBACKMEMBER(uint8_t,   pfnGetTPRRC,(PPDMDEVINS pDevIns, VMCPUID idCpu));
+    /** @copydoc PDMAPICREG::pfnWriteMSRR3 */
+    DECLRCCALLBACKMEMBER(uint32_t,  pfnWriteMSRRC, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t u64Value));
+    /** @copydoc PDMAPICREG::pfnReadMSRR3 */
+    DECLRCCALLBACKMEMBER(uint32_t,  pfnReadMSRRC, (PPDMDEVINS pDevIns, VMCPUID idCpu, uint32_t u32Reg, uint64_t *pu64Value));
+    /** @copydoc PDMAPICREG::pfnBusDeliverR3 */
+    DECLRCCALLBACKMEMBER(int,       pfnBusDeliverRC,(PPDMDEVINS pDevIns, uint8_t u8Dest, uint8_t u8DestMode, uint8_t u8DeliveryMode,
+                                                     uint8_t iVector, uint8_t u8Polarity, uint8_t u8TriggerMode));
+    /** @copydoc PDMAPICREG::pfnLocalInterruptR3 */
+    DECLRCCALLBACKMEMBER(int,       pfnLocalInterruptRC,(PPDMDEVINS pDevIns, uint8_t u8Pin, uint8_t u8Level));
+    RTRCPTR                         RCPtrAlignment;
+
+} PDMAPIC;
+
+
+/**
+ * PDM registered I/O APIC device.
+ */
+typedef struct PDMIOAPIC
+{
+    /** Pointer to the APIC device instance - R3 Ptr. */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** @copydoc PDMIOAPICREG::pfnSetIrqR3 */
+    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
+    /** @copydoc PDMIOAPICREG::pfnSendMsiR3 */
+    DECLR3CALLBACKMEMBER(void,      pfnSendMsiR3,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue));
+
+    /** Pointer to the PIC device instance - R0. */
+    PPDMDEVINSR0                    pDevInsR0;
+    /** @copydoc PDMIOAPICREG::pfnSetIrqR3 */
+    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
+    /** @copydoc PDMIOAPICREG::pfnSendMsiR3 */
+    DECLR0CALLBACKMEMBER(void,      pfnSendMsiR0,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue));
+
+    /** Pointer to the APIC device instance - RC Ptr. */
+    PPDMDEVINSRC                    pDevInsRC;
+    /** @copydoc PDMIOAPICREG::pfnSetIrqR3 */
+    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, int iIrq, int iLevel));
+     /** @copydoc PDMIOAPICREG::pfnSendMsiR3 */
+    DECLRCCALLBACKMEMBER(void,      pfnSendMsiRC,(PPDMDEVINS pDevIns, RTGCPHYS GCAddr, uint32_t uValue));
+
+    uint8_t                         Alignment[4];
+} PDMIOAPIC;
+
+/** Maximum number of PCI busses for a VM. */
+#define PDM_PCI_BUSSES_MAX 8
+
+/**
+ * PDM PCI Bus instance.
+ */
+typedef struct PDMPCIBUS
+{
+    /** PCI bus number. */
+    RTUINT          iBus;
+    RTUINT          uPadding0; /**< Alignment padding.*/
+
+    /** Pointer to PCI Bus device instance. */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** @copydoc PDMPCIBUSREG::pfnSetIrqR3 */
+    DECLR3CALLBACKMEMBER(void,      pfnSetIrqR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iIrq, int iLevel));
+    /** @copydoc PDMPCIBUSREG::pfnRegisterR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnRegisterR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, const char *pszName, int iDev));
+    /** @copydoc PDMPCIBUSREG::pfnPCIRegisterMsiR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnRegisterMsiR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PPDMMSIREG pMsiReg));
+    /** @copydoc PDMPCIBUSREG::pfnIORegionRegisterR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnIORegionRegisterR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iRegion, uint32_t cbRegion,
+                                                           PCIADDRESSSPACE enmType, PFNPCIIOREGIONMAP pfnCallback));
+    /** @copydoc PDMPCIBUSREG::pfnSetConfigCallbacksR3 */
+    DECLR3CALLBACKMEMBER(void,      pfnSetConfigCallbacksR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PFNPCICONFIGREAD pfnRead,
+                                                             PPFNPCICONFIGREAD ppfnReadOld, PFNPCICONFIGWRITE pfnWrite, PPFNPCICONFIGWRITE ppfnWriteOld));
+    /** @copydoc PDMPCIBUSREG::pfnSaveExecR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnSaveExecR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PSSMHANDLE pSSMHandle));
+    /** @copydoc PDMPCIBUSREG::pfnLoadExecR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnLoadExecR3,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, PSSMHANDLE pSSMHandle));
+    /** @copydoc PDMPCIBUSREG::pfnFakePCIBIOSR3 */
+    DECLR3CALLBACKMEMBER(int,       pfnFakePCIBIOSR3,(PPDMDEVINS pDevIns));
+
+    /** Pointer to the PIC device instance - R0. */
+    R0PTRTYPE(PPDMDEVINS)           pDevInsR0;
+    /** @copydoc PDMPCIBUSREG::pfnSetIrqR3 */
+    DECLR0CALLBACKMEMBER(void,      pfnSetIrqR0,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iIrq, int iLevel));
+
+    /** Pointer to PCI Bus device instance. */
+    PPDMDEVINSRC                    pDevInsRC;
+    /** @copydoc PDMPCIBUSREG::pfnSetIrqR3 */
+    DECLRCCALLBACKMEMBER(void,      pfnSetIrqRC,(PPDMDEVINS pDevIns, PPCIDEVICE pPciDev, int iIrq, int iLevel));
+} PDMPCIBUS;
+
+
+#ifdef IN_RING3
+/**
+ * PDM registered DMAC (DMA Controller) device.
+ */
+typedef struct PDMDMAC
+{
+    /** Pointer to the DMAC device instance. */
+    PPDMDEVINSR3                    pDevIns;
+    /** Copy of the registration structure. */
+    PDMDMACREG                      Reg;
+} PDMDMAC;
+
+
+/**
+ * PDM registered RTC (Real Time Clock) device.
+ */
+typedef struct PDMRTC
+{
+    /** Pointer to the RTC device instance. */
+    PPDMDEVINSR3                    pDevIns;
+    /** Copy of the registration structure. */
+    PDMRTCREG                       Reg;
+} PDMRTC;
+
+#endif /* IN_RING3 */
+
+/**
+ * Module type.
+ */
+typedef enum PDMMODTYPE
+{
+    /** Raw-mode (RC) context module. */
+    PDMMOD_TYPE_RC,
+    /** Ring-0 (host) context module. */
+    PDMMOD_TYPE_R0,
+    /** Ring-3 (host) context module. */
+    PDMMOD_TYPE_R3
+} PDMMODTYPE;
+
+
+/** The module name length including the terminator. */
+#define PDMMOD_NAME_LEN             32
+
+/**
+ * Loaded module instance.
+ */
+typedef struct PDMMOD
+{
+    /** Module name. This is used for referring to
+     * the module internally, sort of like a handle. */
+    char                            szName[PDMMOD_NAME_LEN];
+    /** Module type. */
+    PDMMODTYPE                      eType;
+    /** Loader module handle. Not used for R0 modules. */
+    RTLDRMOD                        hLdrMod;
+    /** Loaded address.
+     * This is the 'handle' for R0 modules. */
+    RTUINTPTR                       ImageBase;
+    /** Old loaded address.
+     * This is used during relocation of GC modules. Not used for R0 modules. */
+    RTUINTPTR                       OldImageBase;
+    /** Where the R3 HC bits are stored.
+     * This can be equal to ImageBase but doesn't have to. Not used for R0 modules. */
+    void                           *pvBits;
+
+    /** Pointer to next module. */
+    struct PDMMOD                  *pNext;
+    /** Module filename. */
+    char                            szFilename[1];
+} PDMMOD;
+/** Pointer to loaded module instance. */
+typedef PDMMOD *PPDMMOD;
+
+
+
+/** Extra space in the free array. */
+#define PDMQUEUE_FREE_SLACK         16
+
+/**
+ * Queue type.
+ */
+typedef enum PDMQUEUETYPE
+{
+    /** Device consumer. */
+    PDMQUEUETYPE_DEV = 1,
+    /** Driver consumer. */
+    PDMQUEUETYPE_DRV,
+    /** Internal consumer. */
+    PDMQUEUETYPE_INTERNAL,
+    /** External consumer. */
+    PDMQUEUETYPE_EXTERNAL
+} PDMQUEUETYPE;
+
+/** Pointer to a PDM Queue. */
+typedef struct PDMQUEUE *PPDMQUEUE;
+
+/**
+ * PDM Queue.
+ */
+typedef struct PDMQUEUE
+{
+    /** Pointer to the next queue in the list. */
+    R3PTRTYPE(PPDMQUEUE)            pNext;
+    /** Type specific data. */
+    union
+    {
+        /** PDMQUEUETYPE_DEV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEDEV)   pfnCallback;
+            /** Pointer to the device instance owning the queue. */
+            R3PTRTYPE(PPDMDEVINS)       pDevIns;
+        } Dev;
+        /** PDMQUEUETYPE_DRV */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEDRV)   pfnCallback;
+            /** Pointer to the driver instance owning the queue. */
+            R3PTRTYPE(PPDMDRVINS)       pDrvIns;
+        } Drv;
+        /** PDMQUEUETYPE_INTERNAL */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEINT)   pfnCallback;
+        } Int;
+        /** PDMQUEUETYPE_EXTERNAL */
+        struct
+        {
+            /** Pointer to consumer function. */
+            R3PTRTYPE(PFNPDMQUEUEEXT)   pfnCallback;
+            /** Pointer to user argument. */
+            R3PTRTYPE(void *)           pvUser;
+        } Ext;
+    } u;
+    /** Queue type. */
+    PDMQUEUETYPE                    enmType;
+    /** The interval between checking the queue for events.
+     * The realtime timer below is used to do the waiting.
+     * If 0, the queue will use the VM_FF_PDM_QUEUE forced action. */
+    uint32_t                        cMilliesInterval;
+    /** Interval timer. Only used if cMilliesInterval is non-zero. */
+    PTMTIMERR3                      pTimer;
+    /** Pointer to the VM - R3. */
+    PVMR3                           pVMR3;
+    /** LIFO of pending items - R3. */
+    R3PTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingR3;
+    /** Pointer to the VM - R0. */
+    PVMR0                           pVMR0;
+    /** LIFO of pending items - R0. */
+    R0PTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingR0;
+    /** Pointer to the GC VM and indicator for GC enabled queue.
+     * If this is NULL, the queue cannot be used in GC.
+     */
+    PVMRC                           pVMRC;
+    /** LIFO of pending items - GC. */
+    RCPTRTYPE(PPDMQUEUEITEMCORE) volatile pPendingRC;
+
+    /** Item size (bytes). */
+    uint32_t                        cbItem;
+    /** Number of items in the queue. */
+    uint32_t                        cItems;
+    /** Index to the free head (where we insert). */
+    uint32_t volatile               iFreeHead;
+    /** Index to the free tail (where we remove). */
+    uint32_t volatile               iFreeTail;
+
+    /** Unique queue name. */
+    R3PTRTYPE(const char *)         pszName;
+#if HC_ARCH_BITS == 32
+    RTR3PTR                         Alignment1;
+#endif
+    /** Stat: Times PDMQueueAlloc fails. */
+    STAMCOUNTER                     StatAllocFailures;
+    /** Stat: PDMQueueInsert calls. */
+    STAMCOUNTER                     StatInsert;
+    /** Stat: Queue flushes. */
+    STAMCOUNTER                     StatFlush;
+    /** Stat: Queue flushes with pending items left over. */
+    STAMCOUNTER                     StatFlushLeftovers;
+#ifdef VBOX_WITH_STATISTICS
+    /** State: Profiling the flushing. */
+    STAMPROFILE                     StatFlushPrf;
+    /** State: Pending items. */
+    uint32_t volatile               cStatPending;
+    uint32_t volatile               cAlignment;
+#endif
+
+    /** Array of pointers to free items. Variable size. */
+    struct PDMQUEUEFREEITEM
+    {
+        /** Pointer to the free item - HC Ptr. */
+        R3PTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemR3;
+        /** Pointer to the free item - HC Ptr. */
+        R0PTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemR0;
+        /** Pointer to the free item - GC Ptr. */
+        RCPTRTYPE(PPDMQUEUEITEMCORE) volatile   pItemRC;
+#if HC_ARCH_BITS == 64
+        RTRCPTR                                 Alignment0;
+#endif
+    }                               aFreeItems[1];
+} PDMQUEUE;
+
+/** @name PDM::fQueueFlushing
+ * @{ */
+/** Used to make sure only one EMT will flush the queues.
+ * Set when an EMT is flushing queues, clear otherwise.  */
+#define PDM_QUEUE_FLUSH_FLAG_ACTIVE_BIT     0
+/** Indicating there are queues with items pending.
+ * This is make sure we don't miss inserts happening during flushing.  The FF
+ * cannot be used for this since it has to be cleared immediately to prevent
+ * other EMTs from spinning. */
+#define PDM_QUEUE_FLUSH_FLAG_PENDING_BIT    1
+/** }@  */
+
+
+/**
+ * Queue device helper task operation.
+ */
+typedef enum PDMDEVHLPTASKOP
+{
+    /** The usual invalid 0 entry. */
+    PDMDEVHLPTASKOP_INVALID = 0,
+    /** ISASetIrq */
+    PDMDEVHLPTASKOP_ISA_SET_IRQ,
+    /** PCISetIrq */
+    PDMDEVHLPTASKOP_PCI_SET_IRQ,
+    /** PCISetIrq */
+    PDMDEVHLPTASKOP_IOAPIC_SET_IRQ,
+    /** The usual 32-bit hack. */
+    PDMDEVHLPTASKOP_32BIT_HACK = 0x7fffffff
+} PDMDEVHLPTASKOP;
+
+/**
+ * Queued Device Helper Task.
+ */
+typedef struct PDMDEVHLPTASK
+{
+    /** The queue item core (don't touch). */
+    PDMQUEUEITEMCORE                Core;
+    /** Pointer to the device instance (R3 Ptr). */
+    PPDMDEVINSR3                    pDevInsR3;
+    /** This operation to perform. */
+    PDMDEVHLPTASKOP                 enmOp;
+#if HC_ARCH_BITS == 64
+    uint32_t                        Alignment0;
+#endif
+    /** Parameters to the operation. */
+    union PDMDEVHLPTASKPARAMS
+    {
+        /**
+         * PDMDEVHLPTASKOP_ISA_SET_IRQ and PDMDEVHLPTASKOP_PCI_SET_IRQ.
+         */
+        struct PDMDEVHLPTASKSETIRQ
+        {
+            /** The IRQ */
+            int                     iIrq;
+            /** The new level. */
+            int                     iLevel;
+        } SetIRQ;
+    } u;
+} PDMDEVHLPTASK;
+/** Pointer to a queued Device Helper Task. */
+typedef PDMDEVHLPTASK *PPDMDEVHLPTASK;
+/** Pointer to a const queued Device Helper Task. */
+typedef const PDMDEVHLPTASK *PCPDMDEVHLPTASK;
+
+
+
+/**
+ * An USB hub registration record.
+ */
+typedef struct PDMUSBHUB
+{
+    /** The USB versions this hub support.
+     * Note that 1.1 hubs can take on 2.0 devices. */
+    uint32_t                        fVersions;
+    /** The number of ports on the hub. */
+    uint32_t                        cPorts;
+    /** The number of available ports (0..cPorts). */
+    uint32_t                        cAvailablePorts;
+    /** The driver instance of the hub. */
+    PPDMDRVINS                      pDrvIns;
+    /** Copy of the to the registration structure. */
+    PDMUSBHUBREG                    Reg;
+
+    /** Pointer to the next hub in the list. */
+    struct PDMUSBHUB               *pNext;
+} PDMUSBHUB;
+
+/** Pointer to a const USB HUB registration record. */
+typedef const PDMUSBHUB *PCPDMUSBHUB;
+
+/** Pointer to a PDM Async I/O template. */
+typedef struct PDMASYNCCOMPLETIONTEMPLATE *PPDMASYNCCOMPLETIONTEMPLATE;
+
+/** Pointer to the main PDM Async completion endpoint class. */
+typedef struct PDMASYNCCOMPLETIONEPCLASS *PPDMASYNCCOMPLETIONEPCLASS;
+
+/** Pointer to the global block cache structure. */
+typedef struct PDMBLKCACHEGLOBAL *PPDMBLKCACHEGLOBAL;
+
+/**
+ * PDM VMCPU Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VMCPU!
+ */
+typedef struct PDMCPU
+{
+    /** The number of entries in the apQueuedCritSectsLeaves table that's currently in use. */
+    uint32_t                        cQueuedCritSectLeaves;
+    uint32_t                        uPadding0; /**< Alignment padding.*/
+    /** Critical sections queued in RC/R0 because of contention preventing leave to complete. (R3 Ptrs)
+     * We will return to Ring-3 ASAP, so this queue doesn't have to be very long. */
+    R3PTRTYPE(PPDMCRITSECT)         apQueuedCritSectsLeaves[8];
+} PDMCPU;
+
+/**
+ * Converts a PDM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the PDM is part of.
+ * @param   pPDM   Pointer to PDM instance data.
+ */
+#define PDM2VM(pPDM)  ( (PVM)((char*)pPDM - pPDM->offVM) )
+
+
+/**
+ * PDM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct PDM
+{
+    /** Offset to the VM structure.
+     * See PDM2VM(). */
+    RTUINT                          offVM;
+    RTUINT                          uPadding0; /**< Alignment padding.*/
+
+    /** List of registered devices. (FIFO) */
+    R3PTRTYPE(PPDMDEV)              pDevs;
+    /** List of devices instances. (FIFO) */
+    R3PTRTYPE(PPDMDEVINS)           pDevInstances;
+    /** List of registered USB devices. (FIFO) */
+    R3PTRTYPE(PPDMUSB)              pUsbDevs;
+    /** List of USB devices instances. (FIFO) */
+    R3PTRTYPE(PPDMUSBINS)           pUsbInstances;
+    /** List of registered drivers. (FIFO) */
+    R3PTRTYPE(PPDMDRV)              pDrvs;
+    /** PCI Buses. */
+    PDMPCIBUS                       aPciBuses[PDM_PCI_BUSSES_MAX];
+    /** The register PIC device. */
+    PDMPIC                          Pic;
+    /** The registered APIC device. */
+    PDMAPIC                         Apic;
+    /** The registered I/O APIC device. */
+    PDMIOAPIC                       IoApic;
+    /** The registered DMAC device. */
+    R3PTRTYPE(PPDMDMAC)             pDmac;
+    /** The registered RTC device. */
+    R3PTRTYPE(PPDMRTC)              pRtc;
+    /** The registered USB HUBs. (FIFO) */
+    R3PTRTYPE(PPDMUSBHUB)           pUsbHubs;
+
+    /** Queue in which devhlp tasks are queued for R3 execution - R3 Ptr. */
+    R3PTRTYPE(PPDMQUEUE)            pDevHlpQueueR3;
+    /** Queue in which devhlp tasks are queued for R3 execution - R0 Ptr. */
+    R0PTRTYPE(PPDMQUEUE)            pDevHlpQueueR0;
+    /** Queue in which devhlp tasks are queued for R3 execution - RC Ptr. */
+    RCPTRTYPE(PPDMQUEUE)            pDevHlpQueueRC;
+    /** Pointer to the queue which should be manually flushed - RC Ptr.
+     * Only touched by EMT. */
+    RCPTRTYPE(struct PDMQUEUE *)    pQueueFlushRC;
+    /** Pointer to the queue which should be manually flushed - R0 Ptr.
+     * Only touched by EMT. */
+    R0PTRTYPE(struct PDMQUEUE *)    pQueueFlushR0;
+    /** Bitmask controlling the queue flushing.
+     * See PDM_QUEUE_FLUSH_FLAG_ACTIVE and PDM_QUEUE_FLUSH_FLAG_PENDING. */
+    uint32_t volatile               fQueueFlushing;
+    /** Alignment padding. */
+    uint32_t                        u32Padding2;
+
+    /** @name   VMM device heap
+     * @{ */
+    /** Pointer to the heap base (MMIO2 ring-3 mapping). NULL if not registered. */
+    RTR3PTR                         pvVMMDevHeap;
+    /** The heap size. */
+    uint32_t                        cbVMMDevHeap;
+    /** Free space. */
+    uint32_t                        cbVMMDevHeapLeft;
+    /** The current mapping. NIL_RTGCPHYS if not mapped or registered. */
+    RTGCPHYS                        GCPhysVMMDevHeap;
+    /** @} */
+
+    /** The PDM lock.
+     * This is used to protect everything that deals with interrupts, i.e.
+     * the PIC, APIC, IOAPIC and PCI devices plus some PDM functions. */
+    PDMCRITSECT                     CritSect;
+
+    /** Number of times a critical section leave request needed to be queued for ring-3 execution. */
+    STAMCOUNTER                     StatQueuedCritSectLeaves;
+} PDM;
+AssertCompileMemberAlignment(PDM, GCPhysVMMDevHeap, sizeof(RTGCPHYS));
+AssertCompileMemberAlignment(PDM, CritSect, 8);
+AssertCompileMemberAlignment(PDM, StatQueuedCritSectLeaves, 8);
+/** Pointer to PDM VM instance data. */
+typedef PDM *PPDM;
+
+
+
+/**
+ * PDM data kept in the UVM.
+ */
+typedef struct PDMUSERPERVM
+{
+    /** @todo move more stuff over here. */
+
+    /** Linked list of timer driven PDM queues.
+     * Currently serialized by PDM::CritSect.  */
+    R3PTRTYPE(struct PDMQUEUE *)    pQueuesTimer;
+    /** Linked list of force action driven PDM queues.
+     * Currently serialized by PDM::CritSect. */
+    R3PTRTYPE(struct PDMQUEUE *)    pQueuesForced;
+
+    /** Lock protecting the lists below it. */
+    RTCRITSECT                      ListCritSect;
+    /** Pointer to list of loaded modules. */
+    PPDMMOD                         pModules;
+    /** List of initialized critical sections. (LIFO) */
+    R3PTRTYPE(PPDMCRITSECTINT)      pCritSects;
+    /** Head of the PDM Thread list. (singly linked) */
+    R3PTRTYPE(PPDMTHREAD)           pThreads;
+    /** Tail of the PDM Thread list. (singly linked) */
+    R3PTRTYPE(PPDMTHREAD)           pThreadsTail;
+
+    /** @name   PDM Async Completion
+     * @{ */
+    /** Pointer to the array of supported endpoint classes. */
+    PPDMASYNCCOMPLETIONEPCLASS      apAsyncCompletionEndpointClass[PDMASYNCCOMPLETIONEPCLASSTYPE_MAX];
+    /** Head of the templates. Singly linked, protected by ListCritSect. */
+    R3PTRTYPE(PPDMASYNCCOMPLETIONTEMPLATE) pAsyncCompletionTemplates;
+    /** @} */
+
+    R3PTRTYPE(PPDMBLKCACHEGLOBAL)   pBlkCacheGlobal;
+
+} PDMUSERPERVM;
+/** Pointer to the PDM data kept in the UVM. */
+typedef PDMUSERPERVM *PPDMUSERPERVM;
+
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+#ifdef IN_RING3
+extern const PDMDRVHLPR3    g_pdmR3DrvHlp;
+extern const PDMDEVHLPR3    g_pdmR3DevHlpTrusted;
+extern const PDMDEVHLPR3    g_pdmR3DevHlpUnTrusted;
+extern const PDMPICHLPR3    g_pdmR3DevPicHlp;
+extern const PDMAPICHLPR3   g_pdmR3DevApicHlp;
+extern const PDMIOAPICHLPR3 g_pdmR3DevIoApicHlp;
+extern const PDMPCIHLPR3    g_pdmR3DevPciHlp;
+extern const PDMDMACHLP     g_pdmR3DevDmacHlp;
+extern const PDMRTCHLP      g_pdmR3DevRtcHlp;
+extern const PDMHPETHLPR3   g_pdmR3DevHpetHlp;
+#endif
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+/** @def PDMDEV_ASSERT_DEVINS
+ * Asserts the validity of the device instance.
+ */
+#ifdef VBOX_STRICT
+# define PDMDEV_ASSERT_DEVINS(pDevIns)   \
+    do { \
+        AssertPtr(pDevIns); \
+        Assert(pDevIns->u32Version == PDM_DEVINS_VERSION); \
+        Assert(pDevIns->CTX_SUFF(pvInstanceData) == (void *)&pDevIns->achInstanceData[0]); \
+    } while (0)
+#else
+# define PDMDEV_ASSERT_DEVINS(pDevIns)   do { } while (0)
+#endif
+
+/** @def PDMDRV_ASSERT_DRVINS
+ * Asserts the validity of the driver instance.
+ */
+#ifdef VBOX_STRICT
+# define PDMDRV_ASSERT_DRVINS(pDrvIns) \
+    do { \
+        AssertPtr(pDrvIns); \
+        Assert(pDrvIns->u32Version == PDM_DRVINS_VERSION); \
+        Assert(pDrvIns->CTX_SUFF(pvInstanceData) == (void *)&pDrvIns->achInstanceData[0]); \
+    } while (0)
+#else
+# define PDMDRV_ASSERT_DRVINS(pDrvIns)   do { } while (0)
+#endif
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+#ifdef IN_RING3
+int         pdmR3CritSectInitStats(PVM pVM);
+void        pdmR3CritSectRelocate(PVM pVM);
+int         pdmR3CritSectInitDevice(PVM pVM, PPDMDEVINS pDevIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, va_list va);
+int         pdmR3CritSectDeleteDevice(PVM pVM, PPDMDEVINS pDevIns);
+int         pdmR3CritSectInitDriver(PVM pVM, PPDMDRVINS pDrvIns, PPDMCRITSECT pCritSect, RT_SRC_POS_DECL, const char *pszNameFmt, ...);
+int         pdmR3CritSectDeleteDriver(PVM pVM, PPDMDRVINS pDrvIns);
+
+int         pdmR3DevInit(PVM pVM);
+PPDMDEV     pdmR3DevLookup(PVM pVM, const char *pszName);
+int         pdmR3DevFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMLUN *ppLun);
+DECLCALLBACK(bool) pdmR3DevHlpQueueConsumer(PVM pVM, PPDMQUEUEITEMCORE pItem);
+
+int         pdmR3UsbLoadModules(PVM pVM);
+int         pdmR3UsbInstantiateDevices(PVM pVM);
+PPDMUSB     pdmR3UsbLookup(PVM pVM, const char *pszName);
+int         pdmR3UsbFindLun(PVM pVM, const char *pszDevice, unsigned iInstance, unsigned iLun, PPDMLUN *ppLun);
+int         pdmR3UsbRegisterHub(PVM pVM, PPDMDRVINS pDrvIns, uint32_t fVersions, uint32_t cPorts, PCPDMUSBHUBREG pUsbHubReg, PPCPDMUSBHUBHLP ppUsbHubHlp);
+int         pdmR3UsbVMInitComplete(PVM pVM);
+
+int         pdmR3DrvInit(PVM pVM);
+int         pdmR3DrvInstantiate(PVM pVM, PCFGMNODE pNode, PPDMIBASE pBaseInterface, PPDMDRVINS pDrvAbove,
+                                PPDMLUN pLun, PPDMIBASE *ppBaseInterface);
+int         pdmR3DrvDetach(PPDMDRVINS pDrvIns, uint32_t fFlags);
+void        pdmR3DrvDestroyChain(PPDMDRVINS pDrvIns, uint32_t fFlags);
+PPDMDRV     pdmR3DrvLookup(PVM pVM, const char *pszName);
+
+int         pdmR3LdrInitU(PUVM pUVM);
+void        pdmR3LdrTermU(PUVM pUVM);
+char       *pdmR3FileR3(const char *pszFile, bool fShared);
+int         pdmR3LoadR3U(PUVM pUVM, const char *pszFilename, const char *pszName);
+
+void        pdmR3QueueRelocate(PVM pVM, RTGCINTPTR offDelta);
+
+int         pdmR3ThreadCreateDevice(PVM pVM, PPDMDEVINS pDevIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDEV pfnThread,
+                                    PFNPDMTHREADWAKEUPDEV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
+int         pdmR3ThreadCreateUsb(PVM pVM, PPDMDRVINS pUsbIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADUSB pfnThread,
+                                 PFNPDMTHREADWAKEUPUSB pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
+int         pdmR3ThreadCreateDriver(PVM pVM, PPDMDRVINS pDrvIns, PPPDMTHREAD ppThread, void *pvUser, PFNPDMTHREADDRV pfnThread,
+                                    PFNPDMTHREADWAKEUPDRV pfnWakeup, size_t cbStack, RTTHREADTYPE enmType, const char *pszName);
+int         pdmR3ThreadDestroyDevice(PVM pVM, PPDMDEVINS pDevIns);
+int         pdmR3ThreadDestroyUsb(PVM pVM, PPDMUSBINS pUsbIns);
+int         pdmR3ThreadDestroyDriver(PVM pVM, PPDMDRVINS pDrvIns);
+void        pdmR3ThreadDestroyAll(PVM pVM);
+int         pdmR3ThreadResumeAll(PVM pVM);
+int         pdmR3ThreadSuspendAll(PVM pVM);
+
+#ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
+int         pdmR3AsyncCompletionInit(PVM pVM);
+int         pdmR3AsyncCompletionTerm(PVM pVM);
+void        pdmR3AsyncCompletionResume(PVM pVM);
+#endif
+
+int         pdmR3BlkCacheInit(PVM pVM);
+void        pdmR3BlkCacheTerm(PVM pVM);
+int         pdmR3BlkCacheResume(PVM pVM);
+
+#endif /* IN_RING3 */
+
+void        pdmLock(PVM pVM);
+int         pdmLockEx(PVM pVM, int rc);
+void        pdmUnlock(PVM pVM);
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif
Index: /trunk/src/VBox/VMM/include/PGMGstDefs.h
===================================================================
--- /trunk/src/VBox/VMM/include/PGMGstDefs.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/PGMGstDefs.h	(revision 35333)
@@ -0,0 +1,231 @@
+/* $Id$ */
+/** @file
+ * VBox - Page Manager, Guest Paging Template - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#undef GSTPT
+#undef PGSTPT
+#undef GSTPTE
+#undef PGSTPTE
+#undef GSTPD
+#undef PGSTPD
+#undef GSTPDE
+#undef PGSTPDE
+#undef GSTPTWALK
+#undef PGSTPTWALK
+#undef PCGSTPTWALK
+#undef GST_BIG_PAGE_SIZE
+#undef GST_BIG_PAGE_OFFSET_MASK
+#undef GST_PDE_PG_MASK
+#undef GST_PDE_BIG_PG_MASK
+#undef GST_PD_SHIFT
+#undef GST_PD_MASK
+#undef GST_PTE_PG_MASK
+#undef GST_GET_PTE_SHW_FLAGS
+#undef GST_PT_SHIFT
+#undef GST_PT_MASK
+#undef GST_TOTAL_PD_ENTRIES
+#undef GST_CR3_PAGE_MASK
+#undef GST_PDPE_ENTRIES
+#undef GST_PDPT_SHIFT
+#undef GST_PDPT_MASK
+#undef GST_PDPE_PG_MASK
+#undef GST_GET_PTE_GCPHYS
+#undef GST_GET_PDE_GCPHYS
+#undef GST_GET_BIG_PDE_GCPHYS
+#undef GST_GET_PDE_SHW_FLAGS
+#undef GST_GET_BIG_PDE_SHW_FLAGS
+#undef GST_GET_BIG_PDE_SHW_FLAGS_4_PTE
+#undef GST_IS_PTE_VALID
+#undef GST_IS_PDE_VALID
+#undef GST_IS_BIG_PDE_VALID
+#undef GST_IS_PDPE_VALID
+#undef GST_IS_BIG_PDPE_VALID
+#undef GST_IS_PML4E_VALID
+#undef GST_IS_PSE_ACTIVE
+#undef GST_IS_NX_ACTIVE
+#undef BTH_IS_NP_ACTIVE
+
+#if PGM_GST_TYPE == PGM_TYPE_REAL \
+ || PGM_GST_TYPE == PGM_TYPE_PROT
+
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+#  define GSTPT                                 X86PTPAE
+#  define PGSTPT                                PX86PTPAE
+#  define GSTPTE                                X86PTEPAE
+#  define PGSTPTE                               PX86PTEPAE
+#  define GSTPD                                 X86PDPAE
+#  define PGSTPD                                PX86PDPAE
+#  define GSTPDE                                X86PDEPAE
+#  define PGSTPDE                               PX86PDEPAE
+#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
+#  define GST_IS_NX_ACTIVE(pVCpu)               (true && This_should_perhaps_not_be_used_in_this_context)
+#  define BTH_IS_NP_ACTIVE(pVM)                 (true)
+# else
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT /* Same as shadow paging, but no PGMSHWPTEPAE. */
+#   define GSTPT                                X86PT
+#   define PGSTPT                               PX86PT
+#   define GSTPTE                               X86PTE
+#   define PGSTPTE                              PX86PTE
+#   define GSTPD                                X86PD
+#   define PGSTPD                               PX86PD
+#   define GSTPDE                               X86PDE
+#   define PGSTPDE                              PX86PDE
+#   define GST_PTE_PG_MASK                      X86_PTE_PG_MASK
+#  else
+#   define GSTPT                                X86PTPAE
+#   define PGSTPT                               PX86PTPAE
+#   define GSTPTE                               X86PTEPAE
+#   define PGSTPTE                              PX86PTEPAE
+#   define GSTPD                                X86PDPAE
+#   define PGSTPD                               PX86PDPAE
+#   define GSTPDE                               X86PDEPAE
+#   define PGSTPDE                              PX86PDEPAE
+#   define GST_PTE_PG_MASK                      X86_PTE_PAE_PG_MASK
+#  endif
+#  define GST_IS_NX_ACTIVE(pVCpu)               (pgmGstIsNoExecuteActive(pVCpu))
+#  if PGM_GST_TYPE == PGM_TYPE_PROT             /* (comment at top of PGMAllBth.h) */
+#   define BTH_IS_NP_ACTIVE(pVM)                (pVM->pgm.s.fNestedPaging)
+#  else
+#   define BTH_IS_NP_ACTIVE(pVM)                (false)
+#  endif
+# endif
+# define GST_GET_PTE_GCPHYS(Pte)                ((Pte).u & GST_PTE_PG_MASK)
+# define GST_GET_PDE_GCPHYS(Pde)                (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_BIG_PDE_GCPHYS(Pde)            (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G)) /**< @todo Could return P|RW|US|A|D here without consulting the PTE. */
+# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde) (true && This_should_perhaps_not_be_used_in_this_context) //??
+# define GST_IS_PTE_VALID(pVCpu, Pte)           (true)
+# define GST_IS_PDE_VALID(pVCpu, Pde)           (true)
+# define GST_IS_BIG_PDE_VALID(pVCpu, Pde)       (true)
+# define GST_IS_PDPE_VALID(pVCpu, Pdpe)         (true)
+# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)     (true)
+# define GST_IS_PML4E_VALID(pVCpu, Pml4e)       (true)
+# define GST_IS_PSE_ACTIVE(pVCpu)               (false && This_should_not_be_used_in_this_context)
+
+#elif PGM_GST_TYPE == PGM_TYPE_32BIT
+# define GSTPT                                  X86PT
+# define PGSTPT                                 PX86PT
+# define GSTPTE                                 X86PTE
+# define PGSTPTE                                PX86PTE
+# define GSTPD                                  X86PD
+# define PGSTPD                                 PX86PD
+# define GSTPDE                                 X86PDE
+# define PGSTPDE                                PX86PDE
+# define GSTPTWALK                              PGMPTWALKGST32BIT
+# define PGSTPTWALK                             PPGMPTWALKGST32BIT
+# define PCGSTPTWALK                            PCPGMPTWALKGST32BIT
+# define GST_BIG_PAGE_SIZE                      X86_PAGE_4M_SIZE
+# define GST_BIG_PAGE_OFFSET_MASK               X86_PAGE_4M_OFFSET_MASK
+# define GST_PDE_PG_MASK                        X86_PDE_PG_MASK
+# define GST_PDE_BIG_PG_MASK                    X86_PDE4M_PG_MASK
+# define GST_GET_PTE_GCPHYS(Pte)                ((Pte).u & GST_PDE_PG_MASK)
+# define GST_GET_PDE_GCPHYS(Pde)                ((Pde).u & GST_PDE_PG_MASK)
+# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       pgmGstGet4MBPhysPage(&(pVM)->pgm.s, Pde)
+# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A))
+# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde) \
+    ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A))
+# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde) \
+    ((Pde).u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_A | X86_PDE4M_D | X86_PDE4M_G))
+# define GST_PD_SHIFT                           X86_PD_SHIFT
+# define GST_PD_MASK                            X86_PD_MASK
+# define GST_TOTAL_PD_ENTRIES                   X86_PG_ENTRIES
+# define GST_PTE_PG_MASK                        X86_PTE_PG_MASK
+# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G))
+# define GST_PT_SHIFT                           X86_PT_SHIFT
+# define GST_PT_MASK                            X86_PT_MASK
+# define GST_CR3_PAGE_MASK                      X86_CR3_PAGE_MASK
+# define GST_IS_PTE_VALID(pVCpu, Pte)           (true)
+# define GST_IS_PDE_VALID(pVCpu, Pde)           (true)
+# define GST_IS_BIG_PDE_VALID(pVCpu, Pde)       (!( (Pde).u & (pVCpu)->pgm.s.fGst32BitMbzBigPdeMask ))
+//# define GST_IS_PDPE_VALID(pVCpu, Pdpe)         (false)
+//# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)     (false)
+//# define GST_IS_PML4E_VALID(pVCpu, Pml4e)       (false)
+# define GST_IS_PSE_ACTIVE(pVCpu)               pgmGst32BitIsPageSizeExtActive(pVCpu)
+# define GST_IS_NX_ACTIVE(pVCpu)                (false)
+# define BTH_IS_NP_ACTIVE(pVM)                  (false)
+
+#elif   PGM_GST_TYPE == PGM_TYPE_PAE \
+     || PGM_GST_TYPE == PGM_TYPE_AMD64
+# define GSTPT                                  X86PTPAE
+# define PGSTPT                                 PX86PTPAE
+# define GSTPTE                                 X86PTEPAE
+# define PGSTPTE                                PX86PTEPAE
+# define GSTPD                                  X86PDPAE
+# define PGSTPD                                 PX86PDPAE
+# define GSTPDE                                 X86PDEPAE
+# define PGSTPDE                                PX86PDEPAE
+# define GST_BIG_PAGE_SIZE                      X86_PAGE_2M_SIZE
+# define GST_BIG_PAGE_OFFSET_MASK               X86_PAGE_2M_OFFSET_MASK
+# define GST_PDE_PG_MASK                        X86_PDE_PAE_PG_MASK
+# define GST_PDE_BIG_PG_MASK                    X86_PDE2M_PAE_PG_MASK
+# define GST_GET_PTE_GCPHYS(Pte)                ((Pte).u & GST_PTE_PG_MASK)
+# define GST_GET_PDE_GCPHYS(Pde)                ((Pde).u & GST_PDE_PG_MASK)
+# define GST_GET_BIG_PDE_GCPHYS(pVM, Pde)       ((Pde).u & GST_PDE_BIG_PG_MASK)
+# define GST_GET_PTE_SHW_FLAGS(pVCpu, Pte)      ((Pte).u & (pVCpu)->pgm.s.fGst64ShadowedPteMask )
+# define GST_GET_PDE_SHW_FLAGS(pVCpu, Pde)      ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedPdeMask )
+# define GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, Pde)  ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPdeMask )
+# define GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, Pde)  ((Pde).u & (pVCpu)->pgm.s.fGst64ShadowedBigPde4PteMask )
+
+# define GST_PD_SHIFT                           X86_PD_PAE_SHIFT
+# define GST_PD_MASK                            X86_PD_PAE_MASK
+# if PGM_GST_TYPE == PGM_TYPE_PAE
+#  define GSTPTWALK                             PGMPTWALKGSTPAE
+#  define PGSTPTWALK                            PPGMPTWALKGSTPAE
+#  define PCGSTPTWALK                           PCPGMPTWALKGSTPAE
+#  define GST_TOTAL_PD_ENTRIES                  (X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES)
+#  define GST_PDPE_ENTRIES                      X86_PG_PAE_PDPE_ENTRIES
+#  define GST_PDPE_PG_MASK                      X86_PDPE_PG_MASK
+#  define GST_PDPT_SHIFT                        X86_PDPT_SHIFT
+#  define GST_PDPT_MASK                         X86_PDPT_MASK_PAE
+#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
+#  define GST_CR3_PAGE_MASK                     X86_CR3_PAE_PAGE_MASK
+#  define GST_IS_PTE_VALID(pVCpu, Pte)          (!( (Pte).u   & (pVCpu)->pgm.s.fGstPaeMbzPteMask ))
+#  define GST_IS_PDE_VALID(pVCpu, Pde)          (!( (Pde).u   & (pVCpu)->pgm.s.fGstPaeMbzPdeMask ))
+#  define GST_IS_BIG_PDE_VALID(pVCpu, Pde)      (!( (Pde).u   & (pVCpu)->pgm.s.fGstPaeMbzBigPdeMask ))
+#  define GST_IS_PDPE_VALID(pVCpu, Pdpe)        (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstPaeMbzPdpeMask ))
+//# define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)    (false)
+//# define GST_IS_PML4E_VALID(pVCpu, Pml4e)      (false)
+# else
+#  define GSTPTWALK                             PGMPTWALKGSTAMD64
+#  define PGSTPTWALK                            PPGMPTWALKGSTAMD64
+#  define PCGSTPTWALK                           PCPGMPTWALKGSTAMD64
+#  define GST_TOTAL_PD_ENTRIES                  (X86_PG_AMD64_ENTRIES * X86_PG_AMD64_PDPE_ENTRIES)
+#  define GST_PDPE_ENTRIES                      X86_PG_AMD64_PDPE_ENTRIES
+#  define GST_PDPT_SHIFT                        X86_PDPT_SHIFT
+#  define GST_PDPE_PG_MASK                      X86_PDPE_PG_MASK
+#  define GST_PDPT_MASK                         X86_PDPT_MASK_AMD64
+#  define GST_PTE_PG_MASK                       X86_PTE_PAE_PG_MASK
+#  define GST_CR3_PAGE_MASK                     X86_CR3_AMD64_PAGE_MASK
+#  define GST_IS_PTE_VALID(pVCpu, Pte)          (!( (Pte).u   & (pVCpu)->pgm.s.fGstAmd64MbzPteMask ))
+#  define GST_IS_PDE_VALID(pVCpu, Pde)          (!( (Pde).u   & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask ))
+#  define GST_IS_BIG_PDE_VALID(pVCpu, Pde)      (!( (Pde).u   & (pVCpu)->pgm.s.fGstAmd64MbzBigPdeMask ))
+#  define GST_IS_PDPE_VALID(pVCpu, Pdpe)        (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstAmd64MbzPdpeMask ))
+#  define GST_IS_BIG_PDPE_VALID(pVCpu, Pdpe)    (!( (Pdpe).u  & (pVCpu)->pgm.s.fGstAmd64MbzBigPdpeMask ))
+#  define GST_IS_PML4E_VALID(pVCpu, Pml4e)      (!( (Pml4e).u & (pVCpu)->pgm.s.fGstAmd64MbzPml4eMask ))
+# endif
+# define GST_PT_SHIFT                           X86_PT_PAE_SHIFT
+# define GST_PT_MASK                            X86_PT_PAE_MASK
+# define GST_IS_PSE_ACTIVE(pVCpu)               (true)
+# define GST_IS_NX_ACTIVE(pVCpu)                (pgmGstIsNoExecuteActive(pVCpu))
+# define BTH_IS_NP_ACTIVE(pVM)                  (false)
+#endif
+
Index: /trunk/src/VBox/VMM/include/PGMInline.h
===================================================================
--- /trunk/src/VBox/VMM/include/PGMInline.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/PGMInline.h	(revision 35333)
@@ -0,0 +1,1542 @@
+/* $Id$ */
+/** @file
+ * PGM - Inlined functions.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___PGMInline_h
+#define ___PGMInline_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/err.h>
+#include <VBox/stam.h>
+#include <VBox/param.h>
+#include <VBox/vmm.h>
+#include <VBox/mm.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmapi.h>
+#include <VBox/dis.h>
+#include <VBox/dbgf.h>
+#include <VBox/log.h>
+#include <VBox/gmm.h>
+#include <VBox/hwaccm.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+#include <iprt/critsect.h>
+#include <iprt/sha.h>
+
+
+
+/** @addtogroup grp_pgm_int   Internals
+ * @internal
+ * @{
+ */
+
+/** @todo Split out all the inline stuff into a separate file.  Then we can
+ *        include it later when VM and VMCPU are defined and so avoid all that
+ *        &pVM->pgm.s and &pVCpu->pgm.s stuff.  It also chops ~1600 lines off
+ *        this file and will make it somewhat easier to navigate... */
+
+/**
+ * Gets the PGMRAMRANGE structure for a guest page.
+ *
+ * @returns Pointer to the RAM range on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+                break;
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    return pRam;
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ */
+DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+                return NULL;
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    return &pRam->aPages[off >> PAGE_SHIFT];
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * Old Phys code: Will make sure the page is present.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS and a valid *ppPage on success.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the page pointer on success.
+ */
+DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+            {
+                *ppPage = NULL; /* avoid incorrect and very annoying GCC warnings */
+                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+            }
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page.
+ *
+ * Old Phys code: Will make sure the page is present.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS and a valid *ppPage on success.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the page pointer on success.
+ * @param   ppRamHint   Where to read and store the ram list hint.
+ *                      The caller initializes this to NULL before the call.
+ */
+DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
+{
+    RTGCPHYS off;
+    PPGMRAMRANGE pRam = *ppRamHint;
+    if (    !pRam
+        ||  RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
+    {
+        pRam = pPGM->CTX_SUFF(pRamRanges);
+        off = GCPhys - pRam->GCPhys;
+        if (RT_UNLIKELY(off >= pRam->cb))
+        {
+            do
+            {
+                pRam = pRam->CTX_SUFF(pNext);
+                if (RT_UNLIKELY(!pRam))
+                {
+                    *ppPage = NULL; /* Kill the incorrect and extremely annoying GCC warnings. */
+                    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+                }
+                off = GCPhys - pRam->GCPhys;
+            } while (off >= pRam->cb);
+        }
+        *ppRamHint = pRam;
+    }
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppRam       Where to store the pointer to the PGMRAMRANGE.
+ */
+DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+                return NULL;
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    *ppRam = pRam;
+    return &pRam->aPages[off >> PAGE_SHIFT];
+}
+
+
+/**
+ * Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
+ *
+ * @returns Pointer to the page on success.
+ * @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
+ *
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   ppPage      Where to store the pointer to the PGMPAGE structure.
+ * @param   ppRam       Where to store the pointer to the PGMRAMRANGE structure.
+ */
+DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
+{
+    /*
+     * Optimize for the first range.
+     */
+    PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
+    RTGCPHYS off = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb))
+    {
+        do
+        {
+            pRam = pRam->CTX_SUFF(pNext);
+            if (RT_UNLIKELY(!pRam))
+            {
+                *ppRam = NULL;  /* Shut up silly GCC warnings. */
+                *ppPage = NULL; /* ditto */
+                return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+            }
+            off = GCPhys - pRam->GCPhys;
+        } while (off >= pRam->cb);
+    }
+    *ppRam = pRam;
+    *ppPage = &pRam->aPages[off >> PAGE_SHIFT];
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Convert GC Phys to HC Phys.
+ *
+ * @returns VBox status.
+ * @param   pPGM        PGM handle.
+ * @param   GCPhys      The GC physical address.
+ * @param   pHCPhys     Where to store the corresponding HC physical address.
+ *
+ * @deprecated  Doesn't deal with zero, shared or write monitored pages.
+ *              Avoid when writing new code!
+ */
+DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
+{
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
+    if (RT_FAILURE(rc))
+        return rc;
+    *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & PAGE_OFFSET_MASK);
+    return VINF_SUCCESS;
+}
+
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
+
+/**
+ * Inlined version of the ring-0 version of the host page mapping code
+ * that optimizes access to pages already in the set.
+ *
+ * @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
+ * @param   pVCpu       The current CPU.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmRZDynMapHCPageInlined(PVMCPU pVCpu, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    PPGMMAPSET  pSet    = &pVCpu->pgm.s.AutoSet;
+
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
+        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[iEntry].cInlinedRefs++;
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlHits);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInlMisses);
+        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapHCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined version of the guest page mapping code that optimizes access to pages
+ * already in the set.
+ *
+ * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPhys      The guest physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmRZDynMapGCPageV2Inlined(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
+    AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys));
+
+    /*
+     * Get the ram range.
+     */
+    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    RTGCPHYS        off  = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb
+        /** @todo   || page state stuff */))
+    {
+        /* This case is not counted into StatRZDynMapGCPageInl. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
+        return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
+
+    /*
+     * pgmRZDynMapHCPageInlined with out stats.
+     */
+    PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
+        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
+    {
+        pSet->aEntries[iEntry].cInlinedRefs++;
+        *ppv = pSet->aEntries[iEntry].pvPage;
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
+        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inlined version of the ring-0 version of guest page mapping that optimizes
+ * access to pages already in the set.
+ *
+ * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPhys      The guest physical address of the page.
+ * @param   ppv         Where to store the mapping address.
+ */
+DECLINLINE(int) pgmRZDynMapGCPageInlined(PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    return pgmRZDynMapGCPageV2Inlined(pVCpu->CTX_SUFF(pVM), pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+}
+
+
+/**
+ * Inlined version of the ring-0 version of the guest byte mapping code
+ * that optimizes access to pages already in the set.
+ *
+ * @returns VBox status code, see pgmRZDynMapGCPageCommon for details.
+ * @param   pVCpu       The current CPU.
+ * @param   HCPhys      The physical address of the page.
+ * @param   ppv         Where to store the mapping address. The offset is
+ *                      preserved.
+ */
+DECLINLINE(int) pgmRZDynMapGCPageOffInlined(PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.StatRZDynMapGCPageInl, a);
+
+    /*
+     * Get the ram range.
+     */
+    PVM             pVM  = pVCpu->CTX_SUFF(pVM);
+    PPGMRAMRANGE    pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
+    RTGCPHYS        off  = GCPhys - pRam->GCPhys;
+    if (RT_UNLIKELY(off >= pRam->cb
+        /** @todo   || page state stuff */))
+    {
+        /* This case is not counted into StatRZDynMapGCPageInl. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamMisses);
+        return pgmRZDynMapGCPageCommon(pVM, pVCpu, GCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+    }
+
+    RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
+    STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlRamHits);
+
+    /*
+     * pgmRZDynMapHCPageInlined with out stats.
+     */
+    PPGMMAPSET pSet = &pVCpu->pgm.s.AutoSet;
+    Assert(!(HCPhys & PAGE_OFFSET_MASK));
+    Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
+
+    unsigned    iHash   = PGMMAPSET_HASH(HCPhys);
+    unsigned    iEntry  = pSet->aiHashTable[iHash];
+    if (    iEntry < pSet->cEntries
+        &&  pSet->aEntries[iEntry].HCPhys == HCPhys
+        &&  pSet->aEntries[iEntry].cInlinedRefs < UINT16_MAX - 1)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlHits);
+        pSet->aEntries[iEntry].cInlinedRefs++;
+        *ppv = (void *)((uintptr_t)pSet->aEntries[iEntry].pvPage | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInlMisses);
+        pgmRZDynMapHCPageCommon(pSet, HCPhys, ppv RTLOG_COMMA_SRC_POS_ARGS);
+        *ppv = (void *)((uintptr_t)*ppv | (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
+    }
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZDynMapGCPageInl, a);
+    return VINF_SUCCESS;
+}
+
+#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
+#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+
+/**
+ * Maps the page into current context (RC and maybe R0).
+ *
+ * @returns pointer to the mapping.
+ * @param   pVM         Pointer to the PGM instance data.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void *) pgmPoolMapPageInlined(PVM pVM, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
+{
+    if (pPage->idx >= PGMPOOL_IDX_FIRST)
+    {
+        Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
+        void *pv;
+        pgmRZDynMapHCPageInlined(VMMGetCpu(pVM), pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
+        return pv;
+    }
+    AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
+}
+
+/**
+ * Maps the page into current context (RC and maybe R0).
+ *
+ * @returns pointer to the mapping.
+ * @param   pVM         Pointer to the PGM instance data.
+ * @param   pVCpu       The current CPU.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void *) pgmPoolMapPageV2Inlined(PVM pVM, PVMCPU pVCpu, PPGMPOOLPAGE pPage RTLOG_COMMA_SRC_POS_DECL)
+{
+    if (pPage->idx >= PGMPOOL_IDX_FIRST)
+    {
+        Assert(pPage->idx < pVM->pgm.s.CTX_SUFF(pPool)->cCurPages);
+        void *pv;
+        Assert(pVCpu == VMMGetCpu(pVM));
+        pgmRZDynMapHCPageInlined(pVCpu, pPage->Core.Key, &pv RTLOG_COMMA_SRC_POS_ARGS);
+        return pv;
+    }
+    AssertFatalMsgFailed(("pgmPoolMapPageV2Inlined invalid page index %x\n", pPage->idx));
+}
+
+#endif /*  VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 || IN_RC */
+#ifndef IN_RC
+
+/**
+ * Queries the Physical TLB entry for a physical guest page,
+ * attempting to load the TLB entry if necessary.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pPGM        The PGM instance handle.
+ * @param   GCPhys      The address of the guest page.
+ * @param   ppTlbe      Where to store the pointer to the TLB entry.
+ */
+DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+{
+    int rc;
+    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
+    {
+        STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
+    *ppTlbe = pTlbe;
+    return rc;
+}
+
+
+/**
+ * Queries the Physical TLB entry for a physical guest page,
+ * attempting to load the TLB entry if necessary.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pPGM        The PGM instance handle.
+ * @param   pPage       Pointer to the PGMPAGE structure corresponding to
+ *                      GCPhys.
+ * @param   GCPhys      The address of the guest page.
+ * @param   ppTlbe      Where to store the pointer to the TLB entry.
+ */
+DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
+{
+    int rc;
+    PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
+    {
+        STAM_COUNTER_INC(&pPGM->CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageMapTlbHits));
+        rc = VINF_SUCCESS;
+    }
+    else
+        rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
+    *ppTlbe = pTlbe;
+    return rc;
+}
+
+#endif /* !IN_RC */
+
+
+/**
+ * Checks if the no-execute (NX) feature is active (EFER.NXE=1).
+ *
+ * Only used when the guest is in PAE or long mode.  This is inlined so that we
+ * can perform consistency checks in debug builds.
+ *
+ * @returns true if it is, false if it isn't.
+ * @param   pVCpu       The current CPU.
+ */
+DECL_FORCE_INLINE(bool) pgmGstIsNoExecuteActive(PVMCPU pVCpu)
+{
+    Assert(pVCpu->pgm.s.fNoExecuteEnabled == CPUMIsGuestNXEnabled(pVCpu));
+    Assert(CPUMIsGuestInPAEMode(pVCpu) || CPUMIsGuestInLongMode(pVCpu));
+    return pVCpu->pgm.s.fNoExecuteEnabled;
+}
+
+
+/**
+ * Checks if the page size extension (PSE) is currently enabled (CR4.PSE=1).
+ *
+ * Only used when the guest is in paged 32-bit mode.  This is inlined so that
+ * we can perform consistency checks in debug builds.
+ *
+ * @returns true if it is, false if it isn't.
+ * @param   pVCpu       The current CPU.
+ */
+DECL_FORCE_INLINE(bool) pgmGst32BitIsPageSizeExtActive(PVMCPU pVCpu)
+{
+    Assert(pVCpu->pgm.s.fGst32BitPageSizeExtension == CPUMIsGuestPageSizeExtEnabled(pVCpu));
+    Assert(!CPUMIsGuestInPAEMode(pVCpu));
+    Assert(!CPUMIsGuestInLongMode(pVCpu));
+    return pVCpu->pgm.s.fGst32BitPageSizeExtension;
+}
+
+
+/**
+ * Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
+ * Takes PSE-36 into account.
+ *
+ * @returns guest physical address
+ * @param   pPGM        Pointer to the PGM instance data.
+ * @param   Pde         Guest Pde
+ */
+DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
+{
+    RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
+    GCPhys |= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
+
+    return GCPhys & pPGM->GCPhys4MBPSEMask;
+}
+
+
+/**
+ * Gets the address the guest page directory (32-bit paging).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The current CPU.
+ * @param   ppPd        Where to return the mapping. This is always set.
+ */
+DECLINLINE(int) pgmGstGet32bitPDPtrEx(PVMCPU pVCpu, PX86PD *ppPd)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPd RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        *ppPd = NULL;
+        return rc;
+    }
+#else
+    *ppPd = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
+    if (RT_UNLIKELY(!*ppPd))
+        return pgmGstLazyMap32BitPD(pVCpu, ppPd);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the address the guest page directory (32-bit paging).
+ *
+ * @returns Pointer the page directory entry in question.
+ * @param   pVCpu       The current CPU.
+ */
+DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PVMCPU pVCpu)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PD pGuestPD = NULL;
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPD RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+#else
+    PX86PD pGuestPD = pVCpu->pgm.s.CTX_SUFF(pGst32BitPd);
+    if (RT_UNLIKELY(!pGuestPD))
+    {
+        int rc = pgmGstLazyMap32BitPD(pVCpu, &pGuestPD);
+        if (RT_FAILURE(rc))
+            return NULL;
+    }
+#endif
+    return pGuestPD;
+}
+
+
+/**
+ * Gets the guest page directory pointer table.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The current CPU.
+ * @param   ppPdpt      Where to return the mapping.  This is always set.
+ */
+DECLINLINE(int) pgmGstGetPaePDPTPtrEx(PVMCPU pVCpu, PX86PDPT *ppPdpt)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPdpt RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        *ppPdpt = NULL;
+        return rc;
+    }
+#else
+    *ppPdpt = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
+    if (RT_UNLIKELY(!*ppPdpt))
+        return pgmGstLazyMapPaePDPT(pVCpu, ppPdpt);
+#endif
+    return VINF_SUCCESS;
+}
+
+/**
+ * Gets the guest page directory pointer table.
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The current CPU.
+ */
+DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PVMCPU pVCpu)
+{
+    PX86PDPT pGuestPdpt;
+    int rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGuestPdpt);
+    AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+    return pGuestPdpt;
+}
+
+
+/**
+ * Gets the guest page directory pointer table entry for the specified address.
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The current CPU
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PDPT pGuestPDPT = NULL;
+    int rc = pgmRZDynMapGCPageOffInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPDPT RTLOG_COMMA_SRC_POS);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
+    if (RT_UNLIKELY(!pGuestPDPT))
+    {
+        int rc = pgmGstLazyMapPaePDPT(pVCpu, &pGuestPDPT);
+        if (RT_FAILURE(rc))
+            return NULL;
+    }
+#endif
+    return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns The page directory entry in question.
+ * @returns A non-present entry if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The handle of the virtual CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    AssertGCPtr32(GCPtr);
+    PX86PDPT    pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
+    if (RT_LIKELY(pGuestPDPT))
+    {
+        const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+        if (    pGuestPDPT->a[iPdpt].n.u1Present
+            &&  !(pGuestPDPT->a[iPdpt].u & pVCpu->pgm.s.fGstPaeMbzPdpeMask) )
+        {
+            const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+            PX86PDPAE   pGuestPD = NULL;
+            int rc = pgmRZDynMapGCPageInlined(pVCpu,
+                                              pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
+                                              (void **)&pGuestPD
+                                              RTLOG_COMMA_SRC_POS);
+            if (RT_SUCCESS(rc))
+                return pGuestPD->a[iPD];
+            AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+#else
+            PX86PDPAE   pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
+            if (    !pGuestPD
+                ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
+                pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
+            if (pGuestPD)
+                return pGuestPD->a[iPD];
+#endif
+        }
+    }
+
+    X86PDEPAE ZeroPde = {0};
+    return ZeroPde;
+}
+
+
+/**
+ * Gets the page directory pointer table entry for the specified address
+ * and returns the index into the page directory
+ *
+ * @returns Pointer to the page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ * @param   piPD        Receives the index into the returned page directory
+ * @param   pPdpe       Receives the page directory pointer entry. Optional.
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
+{
+    AssertGCPtr32(GCPtr);
+
+    /* The PDPE. */
+    PX86PDPT        pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
+    if (RT_UNLIKELY(!pGuestPDPT))
+        return NULL;
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    if (pPdpe)
+        *pPdpe = pGuestPDPT->a[iPdpt];
+    if (!pGuestPDPT->a[iPdpt].n.u1Present)
+        return NULL;
+    if (RT_UNLIKELY(pVCpu->pgm.s.fGstPaeMbzPdpeMask & pGuestPDPT->a[iPdpt].u))
+        return NULL;
+
+    /* The PDE. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PDPAE   pGuestPD = NULL;
+    int rc = pgmRZDynMapGCPageInlined(pVCpu,
+                                      pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK,
+                                      (void **)&pGuestPD
+                                      RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+#else
+    PX86PDPAE   pGuestPD = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
+    if (    !pGuestPD
+        ||  (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt])
+        pgmGstLazyMapPaePD(pVCpu, iPdpt, &pGuestPD);
+#endif
+
+    *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+    return pGuestPD;
+}
+
+#ifndef IN_RC
+
+/**
+ * Gets the page map level-4 pointer for the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The current CPU.
+ * @param   ppPml4      Where to return the mapping.  Always set.
+ */
+DECLINLINE(int) pgmGstGetLongModePML4PtrEx(PVMCPU pVCpu, PX86PML4 *ppPml4)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)ppPml4 RTLOG_COMMA_SRC_POS);
+    if (RT_FAILURE(rc))
+    {
+        *ppPml4 = NULL;
+        return rc;
+    }
+#else
+    *ppPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
+    if (RT_UNLIKELY(!*ppPml4))
+        return pgmGstLazyMapPml4(pVCpu, ppPml4);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the page map level-4 pointer for the guest.
+ *
+ * @returns Pointer to the PML4 page.
+ * @param   pVCpu       The current CPU.
+ */
+DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PVMCPU pVCpu)
+{
+    PX86PML4 pGuestPml4;
+    int rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGuestPml4);
+    AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc);
+    return pGuestPml4;
+}
+
+
+/**
+ * Gets the pointer to a page map level-4 entry.
+ *
+ * @returns Pointer to the PML4 entry.
+ * @param   pVCpu       The current CPU.
+ * @param   iPml4       The index.
+ * @remarks Only used by AssertCR3.
+ */
+DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PVMCPU pVCpu, unsigned int iPml4)
+{
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
+    PX86PML4 pGuestPml4;
+    int rc = pgmRZDynMapGCPageInlined(pVCpu, pVCpu->pgm.s.GCPhysCR3, (void **)&pGuestPml4 RTLOG_COMMA_SRC_POS);
+    AssertRCReturn(rc, NULL);
+#else
+    PX86PML4 pGuestPml4 = pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4);
+    if (RT_UNLIKELY(!pGuestPml4))
+    {
+         int rc = pgmGstLazyMapPml4(pVCpu, &pGuestPml4);
+         AssertRCReturn(rc, NULL);
+    }
+#endif
+    return &pGuestPml4->a[iPml4];
+}
+
+
+/**
+ * Gets the page directory entry for the specified address.
+ *
+ * @returns The page directory entry in question.
+ * @returns A non-present entry if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PVMCPU pVCpu, RTGCPTR64 GCPtr)
+{
+    /*
+     * Note! To keep things simple, ASSUME invalid physical addresses will
+     *       cause X86_TRAP_PF_RSVD.  This isn't a problem until we start
+     *       supporting 52-bit wide physical guest addresses.
+     */
+    PCX86PML4       pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
+    const unsigned  iPml4      = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    if (    RT_LIKELY(pGuestPml4)
+        &&  pGuestPml4->a[iPml4].n.u1Present
+        &&  !(pGuestPml4->a[iPml4].u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask) )
+    {
+        PCX86PDPT   pPdptTemp;
+        int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
+        if (RT_SUCCESS(rc))
+        {
+            const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+            if (    pPdptTemp->a[iPdpt].n.u1Present
+                &&  !(pPdptTemp->a[iPdpt].u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask) )
+            {
+                PCX86PDPAE pPD;
+                rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+                if (RT_SUCCESS(rc))
+                {
+                    const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+                    return pPD->a[iPD];
+                }
+            }
+        }
+        AssertMsg(RT_SUCCESS(rc) || rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+    }
+
+    X86PDEPAE ZeroPde = {0};
+    return ZeroPde;
+}
+
+
+/**
+ * Gets the GUEST page directory pointer for the specified address.
+ *
+ * @returns The page directory in question.
+ * @returns NULL if the page directory is not present or on an invalid page.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ * @param   ppPml4e     Page Map Level-4 Entry (out)
+ * @param   pPdpe       Page directory pointer table entry (out)
+ * @param   piPD        Receives the index into the returned page directory
+ */
+DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe, unsigned *piPD)
+{
+    /* The PMLE4. */
+    PX86PML4        pGuestPml4 = pgmGstGetLongModePML4Ptr(pVCpu);
+    if (RT_UNLIKELY(!pGuestPml4))
+        return NULL;
+    const unsigned  iPml4      = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PCX86PML4E      pPml4e     = *ppPml4e = &pGuestPml4->a[iPml4];
+    if (!pPml4e->n.u1Present)
+        return NULL;
+    if (RT_UNLIKELY(pPml4e->u & pVCpu->pgm.s.fGstAmd64MbzPml4eMask))
+        return NULL;
+
+    /* The PDPE. */
+    PCX86PDPT       pPdptTemp;
+    int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+    const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    *pPdpe = pPdptTemp->a[iPdpt];
+    if (!pPdpe->n.u1Present)
+        return NULL;
+    if (RT_UNLIKELY(pPdpe->u & pVCpu->pgm.s.fGstAmd64MbzPdpeMask))
+        return NULL;
+
+    /* The PDE. */
+    PX86PDPAE pPD;
+    rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
+    if (RT_FAILURE(rc))
+    {
+        AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc));
+        return NULL;
+    }
+
+    *piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+    return pPD;
+}
+
+#endif /* !IN_RC */
+
+/**
+ * Gets the shadow page directory, 32-bit.
+ *
+ * @returns Pointer to the shadow 32-bit PD.
+ * @param   pVCpu       The current CPU.
+ */
+DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PVMCPU pVCpu)
+{
+    return (PX86PD)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page directory entry for the specified address, 32-bit.
+ *
+ * @returns Shadow 32-bit PDE.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDE) pgmShwGet32BitPDE(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
+
+    PX86PD pShwPde = pgmShwGet32BitPDPtr(pVCpu);
+    if (!pShwPde)
+    {
+        X86PDE ZeroPde = {0};
+        return ZeroPde;
+    }
+    return pShwPde->a[iPd];
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for the specified
+ * address, 32-bit.
+ *
+ * @returns Pointer to the shadow 32-bit PDE.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
+
+    PX86PD pPde = pgmShwGet32BitPDPtr(pVCpu);
+    AssertReturn(pPde, NULL);
+    return &pPde->a[iPd];
+}
+
+
+/**
+ * Gets the shadow page pointer table, PAE.
+ *
+ * @returns Pointer to the shadow PAE PDPT.
+ * @param   pVCpu       The current CPU.
+ */
+DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PVMCPU pVCpu)
+{
+    return (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    PX86PDPT        pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
+
+    if (!pPdpt->a[iPdpt].n.u1Present)
+        return NULL;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, NULL);
+
+    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PVMCPU pVCpu, PX86PDPT pPdpt, RTGCPTR GCPtr)
+{
+    const unsigned  iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+
+    if (!pPdpt->a[iPdpt].n.u1Present)
+        return NULL;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PVM             pVM     = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOLPAGE    pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, NULL);
+
+    return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+}
+
+
+/**
+ * Gets the shadow page directory entry, PAE.
+ *
+ * @returns PDE.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+
+    PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
+    if (!pShwPde)
+    {
+        X86PDEPAE ZeroPde = {0};
+        return ZeroPde;
+    }
+    return pShwPde->a[iPd];
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for an address, PAE.
+ *
+ * @returns Pointer to the PDE.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ * @remarks Only used by AssertCR3.
+ */
+DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
+
+    PX86PDPAE pPde = pgmShwGetPaePDPtr(pVCpu, GCPtr);
+    AssertReturn(pPde, NULL);
+    return &pPde->a[iPd];
+}
+
+#ifndef IN_RC
+
+/**
+ * Gets the shadow page map level-4 pointer.
+ *
+ * @returns Pointer to the shadow PML4.
+ * @param   pVCpu       The current CPU.
+ */
+DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PVMCPU pVCpu)
+{
+    return (PX86PML4)PGMPOOL_PAGE_2_PTR_V2(pVCpu->CTX_SUFF(pVM), pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+}
+
+
+/**
+ * Gets the shadow page map level-4 entry for the specified address.
+ *
+ * @returns The entry.
+ * @param   pVCpu       The current CPU.
+ * @param   GCPtr       The address.
+ */
+DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PVMCPU pVCpu, RTGCPTR GCPtr)
+{
+    const unsigned  iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PX86PML4        pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
+
+    if (!pShwPml4)
+    {
+        X86PML4E ZeroPml4e = {0};
+        return ZeroPml4e;
+    }
+    return pShwPml4->a[iPml4];
+}
+
+
+/**
+ * Gets the pointer to the specified shadow page map level-4 entry.
+ *
+ * @returns The entry.
+ * @param   pVCpu       The current CPU.
+ * @param   iPml4       The PML4 index.
+ */
+DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PVMCPU pVCpu, unsigned int iPml4)
+{
+    PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pVCpu);
+    if (!pShwPml4)
+        return NULL;
+    return &pShwPml4->a[iPml4];
+}
+
+#endif /* !IN_RC */
+
+
+/**
+ * Cached physical handler lookup.
+ *
+ * @returns Physical handler covering @a GCPhys.
+ * @param   pVM                 The VM handle.
+ * @param   GCPhys              The lookup address.
+ */
+DECLINLINE(PPGMPHYSHANDLER) pgmHandlerPhysicalLookup(PVM pVM, RTGCPHYS GCPhys)
+{
+    PPGMPHYSHANDLER pHandler = pVM->pgm.s.CTX_SUFF(pLastPhysHandler);
+    if (   pHandler
+        && GCPhys >= pHandler->Core.Key
+        && GCPhys < pHandler->Core.KeyLast)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupHits));
+        return pHandler;
+    }
+
+    STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PhysHandlerLookupMisses));
+    pHandler = (PPGMPHYSHANDLER)RTAvlroGCPhysRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (pHandler)
+        pVM->pgm.s.CTX_SUFF(pLastPhysHandler) = pHandler;
+    return pHandler;
+}
+
+
+/**
+ * Gets the page state for a physical handler.
+ *
+ * @returns The physical handler page state.
+ * @param   pCur    The physical handler in question.
+ */
+DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
+{
+    switch (pCur->enmType)
+    {
+        case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
+            return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
+
+        case PGMPHYSHANDLERTYPE_MMIO:
+        case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
+            return PGM_PAGE_HNDL_PHYS_STATE_ALL;
+
+        default:
+            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
+    }
+}
+
+
+/**
+ * Gets the page state for a virtual handler.
+ *
+ * @returns The virtual handler page state.
+ * @param   pCur    The virtual handler in question.
+ * @remarks This should never be used on a hypervisor access handler.
+ */
+DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
+{
+    switch (pCur->enmType)
+    {
+        case PGMVIRTHANDLERTYPE_WRITE:
+            return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
+        case PGMVIRTHANDLERTYPE_ALL:
+            return PGM_PAGE_HNDL_VIRT_STATE_ALL;
+        default:
+            AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
+    }
+}
+
+
+/**
+ * Clears one physical page of a virtual handler
+ *
+ * @param   pPGM    Pointer to the PGM instance.
+ * @param   pCur    Virtual handler structure
+ * @param   iPage   Physical page index
+ *
+ * @remark  Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
+ *          need to care about other handlers in the same page.
+ */
+DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
+{
+    const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
+
+    /*
+     * Remove the node from the tree (it's supposed to be in the tree if we get here!).
+     */
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+    AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
+                     ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                      pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
+#endif
+    if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
+    {
+        /* We're the head of the alias chain. */
+        PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+        AssertReleaseMsg(pRemove != NULL,
+                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
+        AssertReleaseMsg(pRemove == pPhys2Virt,
+                         ("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
+                          "   got:    pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
+                          pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
+#endif
+        if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
+        {
+            /* Insert the next list in the alias chain into the tree. */
+            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+            AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
+                             ("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
+                             pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
+#endif
+            pNext->offNextAlias |= PGMPHYS2VIRTHANDLER_IS_HEAD;
+            bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
+            AssertRelease(fRc);
+        }
+    }
+    else
+    {
+        /* Locate the previous node in the alias chain. */
+        PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+        AssertReleaseMsg(pPrev != pPhys2Virt,
+                         ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
+                          pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
+#endif
+        for (;;)
+        {
+            PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
+            if (pNext == pPhys2Virt)
+            {
+                /* unlink. */
+                LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
+                         pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
+                if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
+                    pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
+                else
+                {
+                    PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
+                    pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
+                                        | (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
+                }
+                break;
+            }
+
+            /* next */
+            if (pNext == pPrev)
+            {
+#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
+                AssertReleaseMsg(pNext != pPrev,
+                                 ("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
+                                  pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
+#endif
+                break;
+            }
+            pPrev = pNext;
+        }
+    }
+    Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
+          pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
+    pPhys2Virt->offNextAlias = 0;
+    pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
+
+    /*
+     * Clear the ram flags for this page.
+     */
+    PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
+    AssertReturnVoid(pPage);
+    PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
+}
+
+
+/**
+ * Internal worker for finding a 'in-use' shadow page give by it's physical address.
+ *
+ * @returns Pointer to the shadow page structure.
+ * @param   pPool       The pool.
+ * @param   idx         The pool page index.
+ */
+DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
+{
+    AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
+    return &pPool->aPages[idx];
+}
+
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPoolPage   The pool page.
+ * @param   pPhysPage   The physical guest page tracking structure.
+ * @param   iPte        Shadow PTE index
+ */
+DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte)
+{
+    /*
+     * Just deal with the simple case here.
+     */
+# ifdef LOG_ENABLED
+    const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
+# endif
+    const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
+    if (cRefs == 1)
+    {
+        Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
+        Assert(iPte == PGM_PAGE_GET_PTE_INDEX(pPhysPage));
+        /* Invalidate the tracking data. */
+        PGM_PAGE_SET_TRACKING(pPhysPage, 0);
+    }
+    else
+        pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage, iPte);
+    Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
+}
+
+
+/**
+ * Moves the page to the head of the age list.
+ *
+ * This is done when the cached page is used in one way or another.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
+
+    /*
+     * Move to the head of the age list.
+     */
+    if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
+    {
+        /* unlink */
+        pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
+        if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
+            pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
+        else
+            pPool->iAgeTail = pPage->iAgePrev;
+
+        /* insert at head */
+        pPage->iAgePrev = NIL_PGMPOOL_IDX;
+        pPage->iAgeNext = pPool->iAgeHead;
+        Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
+        pPool->iAgeHead = pPage->idx;
+        pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
+    }
+}
+
+/**
+ * Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
+ *
+ * @param   pVM         VM Handle.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
+    ASMAtomicIncU32(&pPage->cLocked);
+}
+
+
+/**
+ * Unlocks a page to allow flushing again
+ *
+ * @param   pVM         VM Handle.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
+    Assert(pPage->cLocked);
+    ASMAtomicDecU32(&pPage->cLocked);
+}
+
+
+/**
+ * Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
+ *
+ * @returns VBox status code.
+ * @param   pPage       PGM pool page
+ */
+DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
+{
+    if (pPage->cLocked)
+    {
+        LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
+        if (pPage->cModifications)
+            pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Tells if mappings are to be put into the shadow page table or not.
+ *
+ * @returns boolean result
+ * @param   pVM         VM handle.
+ */
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
+{
+#ifdef PGM_WITHOUT_MAPPINGS
+    /* There are no mappings in VT-x and AMD-V mode. */
+    Assert(pPGM->fMappingsDisabled);
+    return false;
+#else
+    return !pPGM->fMappingsDisabled;
+#endif
+}
+
+
+/**
+ * Checks if the mappings are floating and enabled.
+ *
+ * @returns true / false.
+ * @param   pVM         The VM handle.
+ */
+DECL_FORCE_INLINE(bool) pgmMapAreMappingsFloating(PPGM pPGM)
+{
+#ifdef PGM_WITHOUT_MAPPINGS
+    /* There are no mappings in VT-x and AMD-V mode. */
+    Assert(pPGM->fMappingsDisabled);
+    return false;
+#else
+    return !pPGM->fMappingsDisabled
+        && !pPGM->fMappingsFixed;
+#endif
+}
+
+/** @} */
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/PGMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/PGMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/PGMInternal.h	(revision 35333)
@@ -0,0 +1,3866 @@
+/* $Id$ */
+/** @file
+ * PGM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2010 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___PGMInternal_h
+#define ___PGMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/err.h>
+#include <VBox/dbg.h>
+#include <VBox/stam.h>
+#include <VBox/param.h>
+#include <VBox/vmm.h>
+#include <VBox/mm.h>
+#include <VBox/pdmcritsect.h>
+#include <VBox/pdmapi.h>
+#include <VBox/dis.h>
+#include <VBox/dbgf.h>
+#include <VBox/log.h>
+#include <VBox/gmm.h>
+#include <VBox/hwaccm.h>
+#include <VBox/hwacc_vmx.h>
+#include "internal/pgm.h"
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/avl.h>
+#include <iprt/critsect.h>
+#include <iprt/sha.h>
+
+
+
+/** @defgroup grp_pgm_int   Internals
+ * @ingroup grp_pgm
+ * @internal
+ * @{
+ */
+
+
+/** @name PGM Compile Time Config
+ * @{
+ */
+
+/**
+ * Indicates that there are no guest mappings to care about.
+ * Currently on raw-mode related code uses mappings, i.e. RC and R3 code.
+ */
+#if defined(IN_RING0) || !defined(VBOX_WITH_RAW_MODE)
+# define PGM_WITHOUT_MAPPINGS
+#endif
+
+/**
+ * Check and skip global PDEs for non-global flushes
+ */
+#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
+
+/**
+ * Optimization for PAE page tables that are modified often
+ */
+//#if 0 /* disabled again while debugging */
+#ifndef IN_RC
+# define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+#endif
+//#endif
+
+/**
+ * Large page support enabled only on 64 bits hosts; applies to nested paging only.
+ */
+#if (HC_ARCH_BITS == 64) && !defined(IN_RC)
+# define PGM_WITH_LARGE_PAGES
+#endif
+
+/**
+ * Enables optimizations for MMIO handlers that exploits X86_TRAP_PF_RSVD and
+ * VMX_EXIT_EPT_MISCONFIG.
+ */
+#if 1 /* testing */
+# define PGM_WITH_MMIO_OPTIMIZATIONS
+#endif
+
+/**
+ * Chunk unmapping code activated on 32-bit hosts for > 1.5/2 GB guest memory support
+ */
+#if (HC_ARCH_BITS == 32) && !defined(RT_OS_DARWIN)
+# define PGM_WITH_LARGE_ADDRESS_SPACE_ON_32_BIT_HOST
+#endif
+
+/**
+ * Sync N pages instead of a whole page table
+ */
+#define PGM_SYNC_N_PAGES
+
+/**
+ * Number of pages to sync during a page fault
+ *
+ * When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
+ * causes a lot of unnecessary extents and also is slower than taking more \#PFs.
+ *
+ * Note that \#PFs are much more expensive in the VT-x/AMD-V case due to
+ * world switch overhead, so let's sync more.
+ */
+# ifdef IN_RING0
+/* Chose 32 based on the compile test in #4219; 64 shows worse stats.
+ * 32 again shows better results than 16; slightly more overhead in the \#PF handler,
+ * but ~5% fewer faults.
+ */
+# define PGM_SYNC_NR_PAGES               32
+#else
+# define PGM_SYNC_NR_PAGES               8
+#endif
+
+/**
+ * Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
+ */
+#define PGM_MAX_PHYSCACHE_ENTRIES       64
+#define PGM_MAX_PHYSCACHE_ENTRIES_MASK  (PGM_MAX_PHYSCACHE_ENTRIES-1)
+
+
+/** @def PGMPOOL_CFG_MAX_GROW
+ * The maximum number of pages to add to the pool in one go.
+ */
+#define PGMPOOL_CFG_MAX_GROW            (_256K >> PAGE_SHIFT)
+
+/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
+ * Enables some extra assertions for virtual handlers (mainly phys2virt related).
+ */
+#ifdef VBOX_STRICT
+# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
+#endif
+
+/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+ * Enables the experimental lazy page allocation code. */
+/*#define VBOX_WITH_NEW_LAZY_PAGE_ALLOC */
+
+/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+ * Enables real write monitoring of pages, i.e. mapping them read-only and
+ * only making them writable when getting a write access #PF. */
+#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+
+/** @} */
+
+
+/** @name PDPT and PML4 flags.
+ * These are placed in the three bits available for system programs in
+ * the PDPT and PML4 entries.
+ * @{ */
+/** The entry is a permanent one and it's must always be present.
+ * Never free such an entry. */
+#define PGM_PLXFLAGS_PERMANENT          RT_BIT_64(10)
+/** Mapping (hypervisor allocated pagetable). */
+#define PGM_PLXFLAGS_MAPPING            RT_BIT_64(11)
+/** @} */
+
+/** @name Page directory flags.
+ * These are placed in the three bits available for system programs in
+ * the page directory entries.
+ * @{ */
+/** Mapping (hypervisor allocated pagetable). */
+#define PGM_PDFLAGS_MAPPING             RT_BIT_64(10)
+/** Made read-only to facilitate dirty bit tracking. */
+#define PGM_PDFLAGS_TRACK_DIRTY         RT_BIT_64(11)
+/** @} */
+
+/** @name Page flags.
+ * These are placed in the three bits available for system programs in
+ * the page entries.
+ * @{ */
+/** Made read-only to facilitate dirty bit tracking. */
+#define PGM_PTFLAGS_TRACK_DIRTY         RT_BIT_64(9)
+
+#ifndef PGM_PTFLAGS_CSAM_VALIDATED
+/** Scanned and approved by CSAM (tm).
+ * NOTE: Must be identical to the one defined in CSAMInternal.h!!
+ * @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
+#define PGM_PTFLAGS_CSAM_VALIDATED      RT_BIT_64(11)
+#endif
+
+/** @} */
+
+/** @name Defines used to indicate the shadow and guest paging in the templates.
+ * @{ */
+#define PGM_TYPE_REAL                   1
+#define PGM_TYPE_PROT                   2
+#define PGM_TYPE_32BIT                  3
+#define PGM_TYPE_PAE                    4
+#define PGM_TYPE_AMD64                  5
+#define PGM_TYPE_NESTED                 6
+#define PGM_TYPE_EPT                    7
+#define PGM_TYPE_MAX                    PGM_TYPE_EPT
+/** @} */
+
+/** Macro for checking if the guest is using paging.
+ * @param uGstType     PGM_TYPE_*
+ * @param uShwType     PGM_TYPE_*
+ * @remark  ASSUMES certain order of the PGM_TYPE_* values.
+ */
+#define PGM_WITH_PAGING(uGstType, uShwType)  \
+    (   (uGstType) >= PGM_TYPE_32BIT \
+     && (uShwType) != PGM_TYPE_NESTED \
+     && (uShwType) != PGM_TYPE_EPT)
+
+/** Macro for checking if the guest supports the NX bit.
+ * @param uGstType     PGM_TYPE_*
+ * @param uShwType     PGM_TYPE_*
+ * @remark  ASSUMES certain order of the PGM_TYPE_* values.
+ */
+#define PGM_WITH_NX(uGstType, uShwType)  \
+    (   (uGstType) >= PGM_TYPE_PAE \
+     && (uShwType) != PGM_TYPE_NESTED \
+     && (uShwType) != PGM_TYPE_EPT)
+
+
+/** @def PGM_HCPHYS_2_PTR
+ * Maps a HC physical page pool address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM handle.
+ * @param   pVCpu       The current CPU.
+ * @param   HCPhys      The HC physical address to map to a virtual one.
+ * @param   ppv         Where to store the virtual address. No need to cast
+ *                      this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
+# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
+     pgmRZDynMapHCPageInlined(pVCpu, HCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
+#else
+# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
+     MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
+#endif
+
+/** @def PGM_GCPHYS_2_PTR_V2
+ * Maps a GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The current CPU.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
+# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
+     pgmRZDynMapGCPageV2Inlined(pVM, pVCpu, GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
+#else
+# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
+     PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only */, (PRTR3PTR)(ppv)) /** @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
+#endif
+
+/** @def PGM_GCPHYS_2_PTR
+ * Maps a GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2(pVM, VMMGetCpu(pVM), GCPhys, ppv)
+
+/** @def PGM_GCPHYS_2_PTR_BY_VMCPU
+ * Maps a GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The current CPU.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#define PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2((pVCpu)->CTX_SUFF(pVM), pVCpu, GCPhys, ppv)
+
+/** @def PGM_GCPHYS_2_PTR_EX
+ * Maps a unaligned GC physical page address to a virtual address.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   GCPhys  The GC physical address to map to a virtual one.
+ * @param   ppv     Where to store the virtual address. No need to cast this.
+ *
+ * @remark  Use with care as we don't have so much dynamic mapping space in
+ *          ring-0 on 32-bit darwin and in RC.
+ * @remark  There is no need to assert on the result.
+ */
+#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
+     pgmRZDynMapGCPageOffInlined(VMMGetCpu(pVM), GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
+#else
+# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
+     PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only */, (PRTR3PTR)(ppv)) /** @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
+#endif
+
+/** @def PGM_DYNMAP_UNUSED_HINT
+ * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
+ * is no longer used.
+ *
+ * For best effect only apply this to the page that was mapped most recently.
+ *
+ * @param   pVCpu   The current CPU.
+ * @param   pPage   The pool page.
+ */
+#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+# ifdef LOG_ENABLED
+#  define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  pgmRZDynMapUnusedHint(pVCpu, pvPage, RT_SRC_POS)
+# else
+#  define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  pgmRZDynMapUnusedHint(pVCpu, pvPage)
+# endif
+#else
+# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage)  do {} while (0)
+#endif
+
+/** @def PGM_DYNMAP_UNUSED_HINT_VM
+ * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
+ * is no longer used.
+ *
+ * For best effect only apply this to the page that was mapped most recently.
+ *
+ * @param   pVM     The VM handle.
+ * @param   pPage   The pool page.
+ */
+#define PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvPage)  PGM_DYNMAP_UNUSED_HINT(VMMGetCpu(pVM), pvPage)
+
+
+/** @def PGM_INVL_PG
+ * Invalidates a page.
+ *
+ * @param   pVCpu       The VMCPU handle.
+ * @param   GCVirt      The virtual address of the page to invalidate.
+ */
+#ifdef IN_RC
+# define PGM_INVL_PG(pVCpu, GCVirt)             ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
+#elif defined(IN_RING0)
+# define PGM_INVL_PG(pVCpu, GCVirt)             HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
+#else
+# define PGM_INVL_PG(pVCpu, GCVirt)             HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
+#endif
+
+/** @def PGM_INVL_PG_ALL_VCPU
+ * Invalidates a page on all VCPUs
+ *
+ * @param   pVM         The VM handle.
+ * @param   GCVirt      The virtual address of the page to invalidate.
+ */
+#ifdef IN_RC
+# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
+#elif defined(IN_RING0)
+# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
+#else
+# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt)      HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
+#endif
+
+/** @def PGM_INVL_BIG_PG
+ * Invalidates a 4MB page directory entry.
+ *
+ * @param   pVCpu       The VMCPU handle.
+ * @param   GCVirt      The virtual address within the page directory to invalidate.
+ */
+#ifdef IN_RC
+# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         ASMReloadCR3()
+#elif defined(IN_RING0)
+# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         HWACCMFlushTLB(pVCpu)
+#else
+# define PGM_INVL_BIG_PG(pVCpu, GCVirt)         HWACCMFlushTLB(pVCpu)
+#endif
+
+/** @def PGM_INVL_VCPU_TLBS()
+ * Invalidates the TLBs of the specified VCPU
+ *
+ * @param   pVCpu       The VMCPU handle.
+ */
+#ifdef IN_RC
+# define PGM_INVL_VCPU_TLBS(pVCpu)             ASMReloadCR3()
+#elif defined(IN_RING0)
+# define PGM_INVL_VCPU_TLBS(pVCpu)             HWACCMFlushTLB(pVCpu)
+#else
+# define PGM_INVL_VCPU_TLBS(pVCpu)             HWACCMFlushTLB(pVCpu)
+#endif
+
+/** @def PGM_INVL_ALL_VCPU_TLBS()
+ * Invalidates the TLBs of all VCPUs
+ *
+ * @param   pVM         The VM handle.
+ */
+#ifdef IN_RC
+# define PGM_INVL_ALL_VCPU_TLBS(pVM)            ASMReloadCR3()
+#elif defined(IN_RING0)
+# define PGM_INVL_ALL_VCPU_TLBS(pVM)            HWACCMFlushTLBOnAllVCpus(pVM)
+#else
+# define PGM_INVL_ALL_VCPU_TLBS(pVM)            HWACCMFlushTLBOnAllVCpus(pVM)
+#endif
+
+
+/** @name Safer Shadow PAE PT/PTE
+ * For helping avoid misinterpreting invalid PAE/AMD64 page table entries as
+ * present.
+ *
+ * @{
+ */
+#if 1
+/**
+ * For making sure that u1Present and X86_PTE_P checks doesn't mistake
+ * invalid entries for present.
+ * @sa X86PTEPAE.
+ */
+typedef union PGMSHWPTEPAE
+{
+    /** Unsigned integer view */
+    X86PGPAEUINT    uCareful;
+    /* Not other views. */
+} PGMSHWPTEPAE;
+
+# define PGMSHWPTEPAE_IS_P(Pte)                 ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == X86_PTE_P )
+# define PGMSHWPTEPAE_IS_RW(Pte)                ( !!((Pte).uCareful & X86_PTE_RW))
+# define PGMSHWPTEPAE_IS_US(Pte)                ( !!((Pte).uCareful & X86_PTE_US))
+# define PGMSHWPTEPAE_IS_A(Pte)                 ( !!((Pte).uCareful & X86_PTE_A))
+# define PGMSHWPTEPAE_IS_D(Pte)                 ( !!((Pte).uCareful & X86_PTE_D))
+# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)       ( !!((Pte).uCareful & PGM_PTFLAGS_TRACK_DIRTY) )
+# define PGMSHWPTEPAE_IS_P_RW(Pte)              ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_RW | X86_PTE_PAE_MBZ_MASK_NX)) == (X86_PTE_P | X86_PTE_RW) )
+# define PGMSHWPTEPAE_GET_LOG(Pte)              ( (Pte).uCareful )
+# define PGMSHWPTEPAE_GET_HCPHYS(Pte)           ( (Pte).uCareful & X86_PTE_PAE_PG_MASK )
+# define PGMSHWPTEPAE_GET_U(Pte)                ( (Pte).uCareful ) /**< Use with care. */
+# define PGMSHWPTEPAE_SET(Pte, uVal)            do { (Pte).uCareful = (uVal); } while (0)
+# define PGMSHWPTEPAE_SET2(Pte, Pte2)           do { (Pte).uCareful = (Pte2).uCareful; } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal)     do { ASMAtomicWriteU64(&(Pte).uCareful, (uVal)); } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)    do { ASMAtomicWriteU64(&(Pte).uCareful, (Pte2).uCareful); } while (0)
+# define PGMSHWPTEPAE_SET_RO(Pte)               do { (Pte).uCareful &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
+# define PGMSHWPTEPAE_SET_RW(Pte)               do { (Pte).uCareful |= X86_PTE_RW; } while (0)
+
+/**
+ * For making sure that u1Present and X86_PTE_P checks doesn't mistake
+ * invalid entries for present.
+ * @sa X86PTPAE.
+ */
+typedef struct PGMSHWPTPAE
+{
+    PGMSHWPTEPAE  a[X86_PG_PAE_ENTRIES];
+} PGMSHWPTPAE;
+
+#else
+typedef X86PTEPAE           PGMSHWPTEPAE;
+typedef X86PTPAE            PGMSHWPTPAE;
+# define PGMSHWPTEPAE_IS_P(Pte)                 ( (Pte).n.u1Present )
+# define PGMSHWPTEPAE_IS_RW(Pte)                ( (Pte).n.u1Write )
+# define PGMSHWPTEPAE_IS_US(Pte)                ( (Pte).n.u1User )
+# define PGMSHWPTEPAE_IS_A(Pte)                 ( (Pte).n.u1Accessed )
+# define PGMSHWPTEPAE_IS_D(Pte)                 ( (Pte).n.u1Dirty )
+# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)       ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
+# define PGMSHWPTEPAE_IS_P_RW(Pte)              ( ((Pte).u & (X86_PTE_P | X86_PTE_RW)) == (X86_PTE_P | X86_PTE_RW) )
+# define PGMSHWPTEPAE_GET_LOG(Pte)              ( (Pte).u )
+# define PGMSHWPTEPAE_GET_HCPHYS(Pte)           ( (Pte).u & X86_PTE_PAE_PG_MASK )
+# define PGMSHWPTEPAE_GET_U(Pte)                ( (Pte).u ) /**< Use with care. */
+# define PGMSHWPTEPAE_SET(Pte, uVal)            do { (Pte).u = (uVal); } while (0)
+# define PGMSHWPTEPAE_SET2(Pte, Pte2)           do { (Pte).u = (Pte2).u; } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal)     do { ASMAtomicWriteU64(&(Pte).u, (uVal)); } while (0)
+# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)    do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
+# define PGMSHWPTEPAE_SET_RO(Pte)               do { (Pte).u &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
+# define PGMSHWPTEPAE_SET_RW(Pte)               do { (Pte).u |= X86_PTE_RW; } while (0)
+
+#endif
+
+/** Pointer to a shadow PAE PTE.  */
+typedef PGMSHWPTEPAE       *PPGMSHWPTEPAE;
+/** Pointer to a const shadow PAE PTE.  */
+typedef PGMSHWPTEPAE const *PCPGMSHWPTEPAE;
+
+/** Pointer to a shadow PAE page table.  */
+typedef PGMSHWPTPAE        *PPGMSHWPTPAE;
+/** Pointer to a const shadow PAE page table.  */
+typedef PGMSHWPTPAE const  *PCPGMSHWPTPAE;
+/** @}  */
+
+
+/** Size of the GCPtrConflict array in PGMMAPPING.
+ * @remarks Must be a power of two. */
+#define PGMMAPPING_CONFLICT_MAX         8
+
+/**
+ * Structure for tracking GC Mappings.
+ *
+ * This structure is used by linked list in both GC and HC.
+ */
+typedef struct PGMMAPPING
+{
+    /** Pointer to next entry. */
+    R3PTRTYPE(struct PGMMAPPING *)      pNextR3;
+    /** Pointer to next entry. */
+    R0PTRTYPE(struct PGMMAPPING *)      pNextR0;
+    /** Pointer to next entry. */
+    RCPTRTYPE(struct PGMMAPPING *)      pNextRC;
+    /** Indicate whether this entry is finalized. */
+    bool                                fFinalized;
+    /** Start Virtual address. */
+    RTGCPTR                             GCPtr;
+    /** Last Virtual address (inclusive). */
+    RTGCPTR                             GCPtrLast;
+    /** Range size (bytes). */
+    RTGCPTR                             cb;
+    /** Pointer to relocation callback function. */
+    R3PTRTYPE(PFNPGMRELOCATE)           pfnRelocate;
+    /** User argument to the callback. */
+    R3PTRTYPE(void *)                   pvUser;
+    /** Mapping description / name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** Last 8 addresses that caused conflicts. */
+    RTGCPTR                             aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
+    /** Number of conflicts for this hypervisor mapping. */
+    uint32_t                            cConflicts;
+    /** Number of page tables. */
+    uint32_t                            cPTs;
+
+    /** Array of page table mapping data. Each entry
+     * describes one page table. The array can be longer
+     * than the declared length.
+     */
+    struct
+    {
+        /** The HC physical address of the page table. */
+        RTHCPHYS                        HCPhysPT;
+        /** The HC physical address of the first PAE page table. */
+        RTHCPHYS                        HCPhysPaePT0;
+        /** The HC physical address of the second PAE page table. */
+        RTHCPHYS                        HCPhysPaePT1;
+        /** The HC virtual address of the 32-bit page table. */
+        R3PTRTYPE(PX86PT)               pPTR3;
+        /** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
+        R3PTRTYPE(PPGMSHWPTPAE)         paPaePTsR3;
+        /** The RC virtual address of the 32-bit page table. */
+        RCPTRTYPE(PX86PT)               pPTRC;
+        /** The RC virtual address of the two PAE page table. */
+        RCPTRTYPE(PPGMSHWPTPAE)         paPaePTsRC;
+        /** The R0 virtual address of the 32-bit page table. */
+        R0PTRTYPE(PX86PT)               pPTR0;
+        /** The R0 virtual address of the two PAE page table. */
+        R0PTRTYPE(PPGMSHWPTPAE)         paPaePTsR0;
+    } aPTs[1];
+} PGMMAPPING;
+/** Pointer to structure for tracking GC Mappings. */
+typedef struct PGMMAPPING *PPGMMAPPING;
+
+
+/**
+ * Physical page access handler structure.
+ *
+ * This is used to keep track of physical address ranges
+ * which are being monitored in some kind of way.
+ */
+typedef struct PGMPHYSHANDLER
+{
+    AVLROGCPHYSNODECORE                 Core;
+    /** Access type. */
+    PGMPHYSHANDLERTYPE                  enmType;
+    /** Number of pages to update. */
+    uint32_t                            cPages;
+    /** Set if we have pages that have been aliased. */
+    uint32_t                            cAliasedPages;
+    /** Set if we have pages that have temporarily been disabled. */
+    uint32_t                            cTmpOffPages;
+    /** Pointer to R3 callback function. */
+    R3PTRTYPE(PFNPGMR3PHYSHANDLER)      pfnHandlerR3;
+    /** User argument for R3 handlers. */
+    R3PTRTYPE(void *)                   pvUserR3;
+    /** Pointer to R0 callback function. */
+    R0PTRTYPE(PFNPGMR0PHYSHANDLER)      pfnHandlerR0;
+    /** User argument for R0 handlers. */
+    R0PTRTYPE(void *)                   pvUserR0;
+    /** Pointer to RC callback function. */
+    RCPTRTYPE(PFNPGMRCPHYSHANDLER)      pfnHandlerRC;
+    /** User argument for RC handlers. */
+    RCPTRTYPE(void *)                   pvUserRC;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+#ifdef VBOX_WITH_STATISTICS
+    /** Profiling of this handler. */
+    STAMPROFILE                         Stat;
+#endif
+} PGMPHYSHANDLER;
+/** Pointer to a physical page access handler structure. */
+typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
+
+
+/**
+ * Cache node for the physical addresses covered by a virtual handler.
+ */
+typedef struct PGMPHYS2VIRTHANDLER
+{
+    /** Core node for the tree based on physical ranges. */
+    AVLROGCPHYSNODECORE                 Core;
+    /** Offset from this struct to the PGMVIRTHANDLER structure. */
+    int32_t                             offVirtHandler;
+    /** Offset of the next alias relative to this one.
+     * Bit 0 is used for indicating whether we're in the tree.
+     * Bit 1 is used for indicating that we're the head node.
+     */
+    int32_t                             offNextAlias;
+} PGMPHYS2VIRTHANDLER;
+/** Pointer to a phys to virtual handler structure. */
+typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
+
+/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
+ * node is in the tree. */
+#define PGMPHYS2VIRTHANDLER_IN_TREE     RT_BIT(0)
+/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
+ * node is in the head of an alias chain.
+ * The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
+#define PGMPHYS2VIRTHANDLER_IS_HEAD     RT_BIT(1)
+/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
+#define PGMPHYS2VIRTHANDLER_OFF_MASK    (~(int32_t)3)
+
+
+/**
+ * Virtual page access handler structure.
+ *
+ * This is used to keep track of virtual address ranges
+ * which are being monitored in some kind of way.
+ */
+typedef struct PGMVIRTHANDLER
+{
+    /** Core node for the tree based on virtual ranges. */
+    AVLROGCPTRNODECORE                  Core;
+    /** Size of the range (in bytes). */
+    RTGCPTR                             cb;
+    /** Number of cache pages. */
+    uint32_t                            cPages;
+    /** Access type. */
+    PGMVIRTHANDLERTYPE                  enmType;
+    /** Pointer to the RC callback function. */
+    RCPTRTYPE(PFNPGMRCVIRTHANDLER)      pfnHandlerRC;
+#if HC_ARCH_BITS == 64
+    RTRCPTR                             padding;
+#endif
+    /** Pointer to the R3 callback function for invalidation. */
+    R3PTRTYPE(PFNPGMR3VIRTINVALIDATE)   pfnInvalidateR3;
+    /** Pointer to the R3 callback function. */
+    R3PTRTYPE(PFNPGMR3VIRTHANDLER)      pfnHandlerR3;
+    /** Description / Name. For easing debugging. */
+    R3PTRTYPE(const char *)             pszDesc;
+#ifdef VBOX_WITH_STATISTICS
+    /** Profiling of this handler. */
+    STAMPROFILE                         Stat;
+#endif
+    /** Array of cached physical addresses for the monitored ranged.  */
+    PGMPHYS2VIRTHANDLER                 aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
+} PGMVIRTHANDLER;
+/** Pointer to a virtual page access handler structure. */
+typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
+
+
+/** @name Page type predicates.
+ * @{ */
+#define PGMPAGETYPE_IS_READABLE(type)   ( (type) <= PGMPAGETYPE_ROM )
+#define PGMPAGETYPE_IS_WRITEABLE(type)  ( (type) <= PGMPAGETYPE_ROM_SHADOW )
+#define PGMPAGETYPE_IS_RWX(type)        ( (type) <= PGMPAGETYPE_ROM_SHADOW )
+#define PGMPAGETYPE_IS_ROX(type)        ( (type) == PGMPAGETYPE_ROM )
+#define PGMPAGETYPE_IS_NP(type)         ( (type) == PGMPAGETYPE_MMIO )
+/** @} */
+
+
+/**
+ * A Physical Guest Page tracking structure.
+ *
+ * The format of this structure is complicated because we have to fit a lot
+ * of information into as few bits as possible. The format is also subject
+ * to change (there is one coming up soon). Which means that for we'll be
+ * using PGM_PAGE_GET_*, PGM_PAGE_IS_ and PGM_PAGE_SET_* macros for *all*
+ * accesses to the structure.
+ */
+typedef struct PGMPAGE
+{
+    /** The physical address and the Page ID. */
+    RTHCPHYS    HCPhysAndPageID;
+    /** Combination of:
+     *  - [0-7]: u2HandlerPhysStateY - the physical handler state
+     *    (PGM_PAGE_HNDL_PHYS_STATE_*).
+     *  - [8-9]: u2HandlerVirtStateY - the virtual handler state
+     *    (PGM_PAGE_HNDL_VIRT_STATE_*).
+     *  - [10]:  u1FTDirty - indicator of dirty page for fault tolerance tracking
+     *  - [13-14]: u2PDEType  - paging structure needed to map the page (PGM_PAGE_PDE_TYPE_*)
+     *  - [15]:  fWrittenToY - flag indicating that a write monitored page was
+     *    written to when set.
+     *  - [11-13]: 3 unused bits.
+     * @remarks Warning! All accesses to the bits are hardcoded.
+     *
+     * @todo    Change this to a union with both bitfields, u8 and u accessors.
+     *          That'll help deal with some of the hardcoded accesses.
+     *
+     * @todo    Include uStateY and uTypeY as well so it becomes 32-bit.  This
+     *          will make it possible to turn some of the 16-bit accesses into
+     *          32-bit ones, which may be efficient (stalls).
+     */
+    RTUINT16U   u16MiscY;
+    /** The page state.
+     * Only 3 bits are really needed for this. */
+    uint16_t    uStateY   : 3;
+    /** The page type (PGMPAGETYPE).
+     * Only 3 bits are really needed for this. */
+    uint16_t    uTypeY    : 3;
+    /** PTE index for usage tracking (page pool). */
+    uint16_t    uPteIdx   : 10;
+    /** Usage tracking (page pool). */
+    uint16_t    u16TrackingY;
+    /** The number of read locks on this page. */
+    uint8_t     cReadLocksY;
+    /** The number of write locks on this page. */
+    uint8_t     cWriteLocksY;
+} PGMPAGE;
+AssertCompileSize(PGMPAGE, 16);
+/** Pointer to a physical guest page. */
+typedef PGMPAGE *PPGMPAGE;
+/** Pointer to a const physical guest page. */
+typedef const PGMPAGE *PCPGMPAGE;
+/** Pointer to a physical guest page pointer. */
+typedef PPGMPAGE *PPPGMPAGE;
+
+
+/**
+ * Clears the page structure.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_CLEAR(pPage) \
+    do { \
+        (pPage)->HCPhysAndPageID     = 0; \
+        (pPage)->uStateY             = 0; \
+        (pPage)->uTypeY              = 0; \
+        (pPage)->uPteIdx             = 0; \
+        (pPage)->u16MiscY.u          = 0; \
+        (pPage)->u16TrackingY        = 0; \
+        (pPage)->cReadLocksY         = 0; \
+        (pPage)->cWriteLocksY        = 0; \
+    } while (0)
+
+/**
+ * Initializes the page structure.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
+    do { \
+        RTHCPHYS SetHCPhysTmp = (_HCPhys); \
+        AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
+        (pPage)->HCPhysAndPageID     = (SetHCPhysTmp << (28-12)) | ((_idPage) & UINT32_C(0x0fffffff)); \
+        (pPage)->uStateY             = (_uState); \
+        (pPage)->uTypeY              = (_uType); \
+        (pPage)->uPteIdx             = 0; \
+        (pPage)->u16MiscY.u          = 0; \
+        (pPage)->u16TrackingY        = 0; \
+        (pPage)->cReadLocksY         = 0; \
+        (pPage)->cWriteLocksY        = 0; \
+    } while (0)
+
+/**
+ * Initializes the page structure of a ZERO page.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   pVM         The VM handle (for getting the zero page address).
+ * @param   uType       The page type (PGMPAGETYPE).
+ */
+#define PGM_PAGE_INIT_ZERO(pPage, pVM, uType)  \
+    PGM_PAGE_INIT((pPage), (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (uType), PGM_PAGE_STATE_ZERO)
+
+
+/** @name The Page state, PGMPAGE::uStateY.
+ * @{ */
+/** The zero page.
+ * This is a per-VM page that's never ever mapped writable. */
+#define PGM_PAGE_STATE_ZERO             0
+/** A allocated page.
+ * This is a per-VM page allocated from the page pool (or wherever
+ * we get MMIO2 pages from if the type is MMIO2).
+ */
+#define PGM_PAGE_STATE_ALLOCATED        1
+/** A allocated page that's being monitored for writes.
+ * The shadow page table mappings are read-only. When a write occurs, the
+ * fWrittenTo member is set, the page remapped as read-write and the state
+ * moved back to allocated. */
+#define PGM_PAGE_STATE_WRITE_MONITORED  2
+/** The page is shared, aka. copy-on-write.
+ * This is a page that's shared with other VMs. */
+#define PGM_PAGE_STATE_SHARED           3
+/** The page is ballooned, so no longer available for this VM. */
+#define PGM_PAGE_STATE_BALLOONED        4
+/** @} */
+
+
+/**
+ * Gets the page state.
+ * @returns page state (PGM_PAGE_STATE_*).
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_STATE(pPage)           ( (pPage)->uStateY )
+
+/**
+ * Sets the page state.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   _uState     The new page state.
+ */
+#define PGM_PAGE_SET_STATE(pPage, _uState)  do { (pPage)->uStateY = (_uState); } while (0)
+
+
+/**
+ * Gets the host physical address of the guest page.
+ * @returns host physical address (RTHCPHYS).
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_HCPHYS(pPage)          ( ((pPage)->HCPhysAndPageID >> 28) << 12 )
+
+/**
+ * Sets the host physical address of the guest page.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   _HCPhys     The new host physical address.
+ */
+#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
+    do { \
+        RTHCPHYS SetHCPhysTmp = (_HCPhys); \
+        AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
+        (pPage)->HCPhysAndPageID = ((pPage)->HCPhysAndPageID & UINT32_C(0x0fffffff)) \
+                                 | (SetHCPhysTmp << (28-12)); \
+    } while (0)
+
+/**
+ * Get the Page ID.
+ * @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PAGEID(pPage)          (  (uint32_t)((pPage)->HCPhysAndPageID & UINT32_C(0x0fffffff)) )
+
+/**
+ * Sets the Page ID.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_SET_PAGEID(pPage, _idPage) \
+    do { \
+        (pPage)->HCPhysAndPageID = (((pPage)->HCPhysAndPageID) & UINT64_C(0xfffffffff0000000)) \
+                                 | ((_idPage) & UINT32_C(0x0fffffff)); \
+    } while (0)
+
+/**
+ * Get the Chunk ID.
+ * @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_CHUNKID(pPage)         ( PGM_PAGE_GET_PAGEID(pPage) >> GMM_CHUNKID_SHIFT )
+
+/**
+ * Get the index of the page within the allocation chunk.
+ * @returns The page index.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage)   ( (uint32_t)((pPage)->HCPhysAndPageID & GMM_PAGEID_IDX_MASK) )
+
+/**
+ * Gets the page type.
+ * @returns The page type.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_TYPE(pPage)            (pPage)->uTypeY
+
+/**
+ * Sets the page type.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   _enmType    The new page type (PGMPAGETYPE).
+ */
+#define PGM_PAGE_SET_TYPE(pPage, _enmType)  do { (pPage)->uTypeY = (_enmType); } while (0)
+
+/**
+ * Gets the page table index
+ * @returns The page table index.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PTE_INDEX(pPage)            (pPage)->uPteIdx
+
+/**
+ * Sets the page table index
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   iPte        New page table index.
+ */
+#define PGM_PAGE_SET_PTE_INDEX(pPage, _iPte)  do { (pPage)->uPteIdx = (_iPte); } while (0)
+
+/**
+ * Checks if the page is marked for MMIO.
+ * @returns true/false.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_MMIO(pPage)             ( (pPage)->uTypeY == PGMPAGETYPE_MMIO )
+
+/**
+ * Checks if the page is backed by the ZERO page.
+ * @returns true/false.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_ZERO(pPage)             ( (pPage)->uStateY == PGM_PAGE_STATE_ZERO )
+
+/**
+ * Checks if the page is backed by a SHARED page.
+ * @returns true/false.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_SHARED(pPage)           ( (pPage)->uStateY == PGM_PAGE_STATE_SHARED )
+
+/**
+ * Checks if the page is ballooned.
+ * @returns true/false.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_BALLOONED(pPage)        ( (pPage)->uStateY == PGM_PAGE_STATE_BALLOONED )
+
+/**
+ * Marks the page as written to (for GMM change monitoring).
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_SET_WRITTEN_TO(pPage)      do { (pPage)->u16MiscY.au8[1] |= UINT8_C(0x80); } while (0)
+
+/**
+ * Clears the written-to indicator.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage)    do { (pPage)->u16MiscY.au8[1] &= UINT8_C(0x7f); } while (0)
+
+/**
+ * Checks if the page was marked as written-to.
+ * @returns true/false.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_WRITTEN_TO(pPage)       ( !!((pPage)->u16MiscY.au8[1] & UINT8_C(0x80)) )
+
+/**
+ * Marks the page as dirty for FTM
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_SET_FT_DIRTY(pPage)        do { (pPage)->u16MiscY.au8[1] |= UINT8_C(0x04); } while (0)
+
+/**
+ * Clears the FTM dirty indicator
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_CLEAR_FT_DIRTY(pPage)      do { (pPage)->u16MiscY.au8[1] &= UINT8_C(0xfb); } while (0)
+
+/**
+ * Checks if the page was marked as dirty for FTM
+ * @returns true/false.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_IS_FT_DIRTY(pPage)         ( !!((pPage)->u16MiscY.au8[1] & UINT8_C(0x04)) )
+
+
+/** @name PT usage values (PGMPAGE::u2PDEType).
+ *
+ * @{ */
+/** Either as a PT or PDE. */
+#define PGM_PAGE_PDE_TYPE_DONTCARE             0
+/** Must use a page table to map the range. */
+#define PGM_PAGE_PDE_TYPE_PT                   1
+/** Can use a page directory entry to map the continuous range. */
+#define PGM_PAGE_PDE_TYPE_PDE                  2
+/** Can use a page directory entry to map the continuous range - temporarily disabled (by page monitoring). */
+#define PGM_PAGE_PDE_TYPE_PDE_DISABLED         3
+/** @} */
+
+/**
+ * Set the PDE type of the page
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   uType       PGM_PAGE_PDE_TYPE_*
+ */
+#define PGM_PAGE_SET_PDE_TYPE(pPage, uType) \
+    do { \
+        (pPage)->u16MiscY.au8[1] = ((pPage)->u16MiscY.au8[1] & UINT8_C(0x9f)) \
+                                 | (((uType)                 & UINT8_C(0x03)) << 5); \
+    } while (0)
+
+/**
+ * Checks if the page was marked being part of a large page
+ * @returns true/false.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_PDE_TYPE(pPage)       ( ((pPage)->u16MiscY.au8[1] & UINT8_C(0x60)) >> 5)
+
+/** Enabled optimized access handler tests.
+ * These optimizations makes ASSUMPTIONS about the state values and the u16MiscY
+ * layout.  When enabled, the compiler should normally generate more compact
+ * code.
+ */
+#define PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS 1
+
+/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY).
+ *
+ * @remarks The values are assigned in order of priority, so we can calculate
+ *          the correct state for a page with different handlers installed.
+ * @{ */
+/** No handler installed. */
+#define PGM_PAGE_HNDL_PHYS_STATE_NONE       0
+/** Monitoring is temporarily disabled. */
+#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED   1
+/** Write access is monitored. */
+#define PGM_PAGE_HNDL_PHYS_STATE_WRITE      2
+/** All access is monitored. */
+#define PGM_PAGE_HNDL_PHYS_STATE_ALL        3
+/** @} */
+
+/**
+ * Gets the physical access handler state of a page.
+ * @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)  \
+    ( (pPage)->u16MiscY.au8[0] )
+
+/**
+ * Sets the physical access handler state of a page.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   _uState     The new state value.
+ */
+#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
+    do { (pPage)->u16MiscY.au8[0] = (_uState); } while (0)
+
+/**
+ * Checks if the page has any physical access handlers, including temporarily disabled ones.
+ * @returns true/false
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) \
+    ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
+
+/**
+ * Checks if the page has any active physical access handlers.
+ * @returns true/false
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) \
+    ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
+
+
+/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateY).
+ *
+ * @remarks The values are assigned in order of priority, so we can calculate
+ *          the correct state for a page with different handlers installed.
+ * @{ */
+/** No handler installed. */
+#define PGM_PAGE_HNDL_VIRT_STATE_NONE       0
+/* 1 is reserved so the lineup is identical with the physical ones. */
+/** Write access is monitored. */
+#define PGM_PAGE_HNDL_VIRT_STATE_WRITE      2
+/** All access is monitored. */
+#define PGM_PAGE_HNDL_VIRT_STATE_ALL        3
+/** @} */
+
+/**
+ * Gets the virtual access handler state of a page.
+ * @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ((uint8_t)( (pPage)->u16MiscY.au8[1] & UINT8_C(0x03) ))
+
+/**
+ * Sets the virtual access handler state of a page.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ * @param   _uState     The new state value.
+ */
+#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
+    do { \
+        (pPage)->u16MiscY.au8[1] = ((pPage)->u16MiscY.au8[1] & UINT8_C(0xfc)) \
+                                 | ((_uState)                & UINT8_C(0x03)); \
+    } while (0)
+
+/**
+ * Checks if the page has any virtual access handlers.
+ * @returns true/false
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) \
+    ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
+
+/**
+ * Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
+ * virtual handlers.
+ * @returns true/false
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) \
+    PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
+
+
+/**
+ * Checks if the page has any access handlers, including temporarily disabled ones.
+ * @returns true/false
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
+# define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
+    ( ((pPage)->u16MiscY.u & UINT16_C(0x0303)) != 0 )
+#else
+# define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
+    (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE \
+     || PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
+#endif
+
+/**
+ * Checks if the page has any active access handlers.
+ * @returns true/false
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
+# define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
+    ( ((pPage)->u16MiscY.u & UINT16_C(0x0202)) != 0 )
+#else
+# define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
+    (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
+     || PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
+#endif
+
+/**
+ * Checks if the page has any active access handlers catching all accesses.
+ * @returns true/false
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
+# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
+    (   ( ((pPage)->u16MiscY.au8[0] | (pPage)->u16MiscY.au8[1]) & UINT8_C(0x3) ) \
+     == PGM_PAGE_HNDL_PHYS_STATE_ALL )
+#else
+# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
+    (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL \
+     || PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL )
+#endif
+
+
+/** @def PGM_PAGE_GET_TRACKING
+ * Gets the packed shadow page pool tracking data associated with a guest page.
+ * @returns uint16_t containing the data.
+ * @param   pPage       Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_TRACKING(pPage)        ( (pPage)->u16TrackingY )
+
+/** @def PGM_PAGE_SET_TRACKING
+ * Sets the packed shadow page pool tracking data associated with a guest page.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ * @param   u16TrackingData     The tracking data to store.
+ */
+#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
+    do { (pPage)->u16TrackingY = (u16TrackingData); } while (0)
+
+/** @def PGM_PAGE_GET_TD_CREFS
+ * Gets the @a cRefs tracking data member.
+ * @returns cRefs.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_TD_CREFS(pPage) \
+    ((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
+
+/** @def PGM_PAGE_GET_TD_IDX
+ * Gets the @a idx tracking data member.
+ * @returns idx.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_TD_IDX(pPage) \
+    ((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT)   & PGMPOOL_TD_IDX_MASK)
+
+
+/** Max number of locks on a page. */
+#define PGM_PAGE_MAX_LOCKS                  UINT8_C(254)
+
+/** Get the read lock count.
+ * @returns count.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_READ_LOCKS(pPage)      ( (pPage)->cReadLocksY )
+
+/** Get the write lock count.
+ * @returns count.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_GET_WRITE_LOCKS(pPage)     ( (pPage)->cWriteLocksY )
+
+/** Decrement the read lock counter.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_DEC_READ_LOCKS(pPage)      do { --(pPage)->cReadLocksY; } while (0)
+
+/** Decrement the write lock counter.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_DEC_WRITE_LOCKS(pPage)     do { --(pPage)->cWriteLocksY; } while (0)
+
+/** Increment the read lock counter.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_INC_READ_LOCKS(pPage)      do { ++(pPage)->cReadLocksY; } while (0)
+
+/** Increment the write lock counter.
+ * @param   pPage               Pointer to the physical guest page tracking structure.
+ */
+#define PGM_PAGE_INC_WRITE_LOCKS(pPage)     do { ++(pPage)->cWriteLocksY; } while (0)
+
+
+#if 0
+/** Enables sanity checking of write monitoring using CRC-32.  */
+# define PGMLIVESAVERAMPAGE_WITH_CRC32
+#endif
+
+/**
+ * Per page live save tracking data.
+ */
+typedef struct PGMLIVESAVERAMPAGE
+{
+    /** Number of times it has been dirtied. */
+    uint32_t    cDirtied : 24;
+    /** Whether it is currently dirty. */
+    uint32_t    fDirty : 1;
+    /** Ignore the page.
+     *  This is used for pages that has been MMIO, MMIO2 or ROM pages once.  We will
+     *  deal with these after pausing the VM and DevPCI have said it bit about
+     *  remappings. */
+    uint32_t    fIgnore : 1;
+    /** Was a ZERO page last time around. */
+    uint32_t    fZero : 1;
+    /** Was a SHARED page last time around. */
+    uint32_t    fShared : 1;
+    /** Whether the page is/was write monitored in a previous pass. */
+    uint32_t    fWriteMonitored : 1;
+    /** Whether the page is/was write monitored earlier in this pass. */
+    uint32_t    fWriteMonitoredJustNow : 1;
+    /** Bits reserved for future use.  */
+    uint32_t    u2Reserved : 2;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+    /** CRC-32 for the page. This is for internal consistency checks.  */
+    uint32_t    u32Crc;
+#endif
+} PGMLIVESAVERAMPAGE;
+#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
+AssertCompileSize(PGMLIVESAVERAMPAGE, 8);
+#else
+AssertCompileSize(PGMLIVESAVERAMPAGE, 4);
+#endif
+/** Pointer to the per page live save tracking data. */
+typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE;
+
+/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */
+#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0
+
+
+/**
+ * Ram range for GC Phys to HC Phys conversion.
+ *
+ * Can be used for HC Virt to GC Phys and HC Virt to HC Phys
+ * conversions too, but we'll let MM handle that for now.
+ *
+ * This structure is used by linked lists in both GC and HC.
+ */
+typedef struct PGMRAMRANGE
+{
+    /** Start of the range. Page aligned. */
+    RTGCPHYS                            GCPhys;
+    /** Size of the range. (Page aligned of course). */
+    RTGCPHYS                            cb;
+    /** Pointer to the next RAM range - for R3. */
+    R3PTRTYPE(struct PGMRAMRANGE *)     pNextR3;
+    /** Pointer to the next RAM range - for R0. */
+    R0PTRTYPE(struct PGMRAMRANGE *)     pNextR0;
+    /** Pointer to the next RAM range - for RC. */
+    RCPTRTYPE(struct PGMRAMRANGE *)     pNextRC;
+    /** PGM_RAM_RANGE_FLAGS_* flags. */
+    uint32_t                            fFlags;
+    /** Last address in the range (inclusive). Page aligned (-1). */
+    RTGCPHYS                            GCPhysLast;
+    /** Start of the HC mapping of the range. This is only used for MMIO2. */
+    R3PTRTYPE(void *)                   pvR3;
+    /** Live save per page tracking data. */
+    R3PTRTYPE(PPGMLIVESAVERAMPAGE)      paLSPages;
+    /** The range description. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** Pointer to self - R0 pointer. */
+    R0PTRTYPE(struct PGMRAMRANGE *)     pSelfR0;
+    /** Pointer to self - RC pointer. */
+    RCPTRTYPE(struct PGMRAMRANGE *)     pSelfRC;
+    /** Padding to make aPage aligned on sizeof(PGMPAGE). */
+    uint32_t                            au32Alignment2[HC_ARCH_BITS == 32 ? 1 : 3];
+    /** Array of physical guest page tracking structures. */
+    PGMPAGE                             aPages[1];
+} PGMRAMRANGE;
+/** Pointer to Ram range for GC Phys to HC Phys conversion. */
+typedef PGMRAMRANGE *PPGMRAMRANGE;
+
+/** @name PGMRAMRANGE::fFlags
+ * @{ */
+/** The RAM range is floating around as an independent guest mapping. */
+#define PGM_RAM_RANGE_FLAGS_FLOATING        RT_BIT(20)
+/** Ad hoc RAM range for an ROM mapping. */
+#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM      RT_BIT(21)
+/** Ad hoc RAM range for an MMIO mapping. */
+#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO     RT_BIT(22)
+/** Ad hoc RAM range for an MMIO2 mapping. */
+#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2    RT_BIT(23)
+/** @} */
+
+/** Tests if a RAM range is an ad hoc one or not.
+ * @returns true/false.
+ * @param   pRam    The RAM range.
+ */
+#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
+    (!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2) ) )
+
+
+/**
+ * Per page tracking structure for ROM image.
+ *
+ * A ROM image may have a shadow page, in which case we may have two pages
+ * backing it.  This structure contains the PGMPAGE for both while
+ * PGMRAMRANGE have a copy of the active one.  It is important that these
+ * aren't out of sync in any regard other than page pool tracking data.
+ */
+typedef struct PGMROMPAGE
+{
+    /** The page structure for the virgin ROM page. */
+    PGMPAGE     Virgin;
+    /** The page structure for the shadow RAM page. */
+    PGMPAGE     Shadow;
+    /** The current protection setting. */
+    PGMROMPROT  enmProt;
+    /** Live save status information. Makes use of unused alignment space. */
+    struct
+    {
+        /** The previous protection value. */
+        uint8_t u8Prot;
+        /** Written to flag set by the handler. */
+        bool    fWrittenTo;
+        /** Whether the shadow page is dirty or not. */
+        bool    fDirty;
+        /** Whether it was dirtied in the recently. */
+        bool    fDirtiedRecently;
+    } LiveSave;
+} PGMROMPAGE;
+AssertCompileSizeAlignment(PGMROMPAGE, 8);
+/** Pointer to a ROM page tracking structure. */
+typedef PGMROMPAGE *PPGMROMPAGE;
+
+
+/**
+ * A registered ROM image.
+ *
+ * This is needed to keep track of ROM image since they generally intrude
+ * into a PGMRAMRANGE.  It also keeps track of additional info like the
+ * two page sets (read-only virgin and read-write shadow), the current
+ * state of each page.
+ *
+ * Because access handlers cannot easily be executed in a different
+ * context, the ROM ranges needs to be accessible and in all contexts.
+ */
+typedef struct PGMROMRANGE
+{
+    /** Pointer to the next range - R3. */
+    R3PTRTYPE(struct PGMROMRANGE *)     pNextR3;
+    /** Pointer to the next range - R0. */
+    R0PTRTYPE(struct PGMROMRANGE *)     pNextR0;
+    /** Pointer to the next range - RC. */
+    RCPTRTYPE(struct PGMROMRANGE *)     pNextRC;
+    /** Pointer alignment */
+    RTRCPTR                             RCPtrAlignment;
+    /** Address of the range. */
+    RTGCPHYS                            GCPhys;
+    /** Address of the last byte in the range. */
+    RTGCPHYS                            GCPhysLast;
+    /** Size of the range. */
+    RTGCPHYS                            cb;
+    /** The flags (PGMPHYS_ROM_FLAGS_*). */
+    uint32_t                            fFlags;
+    /** The saved state range ID. */
+    uint8_t                             idSavedState;
+    /** Alignment padding. */
+    uint8_t                             au8Alignment[3];
+    /** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
+    uint32_t                            au32Alignemnt[HC_ARCH_BITS == 32 ? 5 : 1];
+    /** The size bits pvOriginal points to. */
+    uint32_t                            cbOriginal;
+    /** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
+     * This is used for strictness checks. */
+    R3PTRTYPE(const void *)             pvOriginal;
+    /** The ROM description. */
+    R3PTRTYPE(const char *)             pszDesc;
+    /** The per page tracking structures. */
+    PGMROMPAGE                          aPages[1];
+} PGMROMRANGE;
+/** Pointer to a ROM range. */
+typedef PGMROMRANGE *PPGMROMRANGE;
+
+
+/**
+ * Live save per page data for an MMIO2 page.
+ *
+ * Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring
+ * of MMIO2 pages.  The current approach is using some optimistic SHA-1 +
+ * CRC-32 for detecting changes as well as special handling of zero pages.  This
+ * is a TEMPORARY measure which isn't perfect, but hopefully it is good enough
+ * for speeding things up.  (We're using SHA-1 and not SHA-256 or SHA-512
+ * because of speed (2.5x and 6x slower).)
+ *
+ * @todo Implement dirty MMIO2 page reporting that can be enabled during live
+ *       save but normally is disabled.  Since we can write monitor guest
+ *       accesses on our own, we only need this for host accesses.  Shouldn't be
+ *       too difficult for DevVGA, VMMDev might be doable, the planned
+ *       networking fun will be fun since it involves ring-0.
+ */
+typedef struct PGMLIVESAVEMMIO2PAGE
+{
+    /** Set if the page is considered dirty. */
+    bool        fDirty;
+    /** The number of scans this page has remained unchanged for.
+     * Only updated for dirty pages. */
+    uint8_t     cUnchangedScans;
+    /** Whether this page was zero at the last scan. */
+    bool        fZero;
+    /** Alignment padding. */
+    bool        fReserved;
+    /** CRC-32 for the first half of the page.
+     * This is used together with u32CrcH2 to quickly detect changes in the page
+     * during the non-final passes.  */
+    uint32_t    u32CrcH1;
+    /** CRC-32 for the second half of the page. */
+    uint32_t    u32CrcH2;
+    /** SHA-1 for the saved page.
+     * This is used in the final pass to skip pages without changes. */
+    uint8_t     abSha1Saved[RTSHA1_HASH_SIZE];
+} PGMLIVESAVEMMIO2PAGE;
+/** Pointer to a live save status data for an MMIO2 page. */
+typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE;
+
+/**
+ * A registered MMIO2 (= Device RAM) range.
+ *
+ * There are a few reason why we need to keep track of these
+ * registrations.  One of them is the deregistration & cleanup stuff,
+ * while another is that the PGMRAMRANGE associated with such a region may
+ * have to be removed from the ram range list.
+ *
+ * Overlapping with a RAM range has to be 100% or none at all.  The pages
+ * in the existing RAM range must not be ROM nor MMIO.  A guru meditation
+ * will be raised if a partial overlap or an overlap of ROM pages is
+ * encountered.  On an overlap we will free all the existing RAM pages and
+ * put in the ram range pages instead.
+ */
+typedef struct PGMMMIO2RANGE
+{
+    /** The owner of the range. (a device) */
+    PPDMDEVINSR3                        pDevInsR3;
+    /** Pointer to the ring-3 mapping of the allocation. */
+    RTR3PTR                             pvR3;
+    /** Pointer to the next range - R3. */
+    R3PTRTYPE(struct PGMMMIO2RANGE *)   pNextR3;
+    /** Whether it's mapped or not. */
+    bool                                fMapped;
+    /** Whether it's overlapping or not. */
+    bool                                fOverlapping;
+    /** The PCI region number.
+     * @remarks This ASSUMES that nobody will ever really need to have multiple
+     *          PCI devices with matching MMIO region numbers on a single device. */
+    uint8_t                             iRegion;
+    /** The saved state range ID. */
+    uint8_t                             idSavedState;
+    /** Alignment padding for putting the ram range on a PGMPAGE alignment boundary. */
+    uint8_t                             abAlignemnt[HC_ARCH_BITS == 32 ? 12 : 12];
+    /** Live save per page tracking data. */
+    R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE)    paLSPages;
+    /** The associated RAM range. */
+    PGMRAMRANGE                         RamRange;
+} PGMMMIO2RANGE;
+/** Pointer to a MMIO2 range. */
+typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
+
+
+
+
+/**
+ * PGMPhysRead/Write cache entry
+ */
+typedef struct PGMPHYSCACHEENTRY
+{
+    /** R3 pointer to physical page. */
+    R3PTRTYPE(uint8_t *)                pbR3;
+    /** GC Physical address for cache entry */
+    RTGCPHYS                            GCPhys;
+#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
+    RTGCPHYS                            u32Padding0; /**< alignment padding. */
+#endif
+} PGMPHYSCACHEENTRY;
+
+/**
+ * PGMPhysRead/Write cache to reduce REM memory access overhead
+ */
+typedef struct PGMPHYSCACHE
+{
+    /** Bitmap of valid cache entries */
+    uint64_t                            aEntries;
+    /** Cache entries */
+    PGMPHYSCACHEENTRY                   Entry[PGM_MAX_PHYSCACHE_ENTRIES];
+} PGMPHYSCACHE;
+
+
+/** Pointer to an allocation chunk ring-3 mapping. */
+typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
+/** Pointer to an allocation chunk ring-3 mapping pointer. */
+typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
+
+/**
+ * Ring-3 tracking structore for an allocation chunk ring-3 mapping.
+ *
+ * The primary tree (Core) uses the chunk id as key.
+ */
+typedef struct PGMCHUNKR3MAP
+{
+    /** The key is the chunk id. */
+    AVLU32NODECORE                      Core;
+    /** The current age thingy. */
+    uint32_t                            iAge;
+    /** The current reference count. */
+    uint32_t volatile                   cRefs;
+    /** The current permanent reference count. */
+    uint32_t volatile                   cPermRefs;
+    /** The mapping address. */
+    void                               *pv;
+} PGMCHUNKR3MAP;
+
+/**
+ * Allocation chunk ring-3 mapping TLB entry.
+ */
+typedef struct PGMCHUNKR3MAPTLBE
+{
+    /** The chunk id. */
+    uint32_t volatile                   idChunk;
+#if HC_ARCH_BITS == 64
+    uint32_t                            u32Padding; /**< alignment padding. */
+#endif
+    /** The chunk map. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(PPGMCHUNKR3MAP) volatile  pChunk;
+#else
+    R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile  pChunk;
+#endif
+} PGMCHUNKR3MAPTLBE;
+/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
+typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
+
+/** The number of TLB entries in PGMCHUNKR3MAPTLB.
+ * @remark Must be a power of two value. */
+#define PGM_CHUNKR3MAPTLB_ENTRIES   64
+
+/**
+ * Allocation chunk ring-3 mapping TLB.
+ *
+ * @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
+ *          At first glance this might look kinda odd since AVL trees are
+ *          supposed to give the most optimal lookup times of all trees
+ *          due to their balancing. However, take a tree with 1023 nodes
+ *          in it, that's 10 levels, meaning that most searches has to go
+ *          down 9 levels before they find what they want. This isn't fast
+ *          compared to a TLB hit. There is the factor of cache misses,
+ *          and of course the problem with trees and branch prediction.
+ *          This is why we use TLBs in front of most of the trees.
+ *
+ * @todo    Generalize this TLB + AVL stuff, shouldn't be all that
+ *          difficult when we switch to the new inlined AVL trees (from kStuff).
+ */
+typedef struct PGMCHUNKR3MAPTLB
+{
+    /** The TLB entries. */
+    PGMCHUNKR3MAPTLBE   aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
+} PGMCHUNKR3MAPTLB;
+
+/**
+ * Calculates the index of a guest page in the Ring-3 Chunk TLB.
+ * @returns Chunk TLB index.
+ * @param   idChunk         The Chunk ID.
+ */
+#define PGM_CHUNKR3MAPTLB_IDX(idChunk)     ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
+
+
+/**
+ * Ring-3 guest page mapping TLB entry.
+ * @remarks used in ring-0 as well at the moment.
+ */
+typedef struct PGMPAGER3MAPTLBE
+{
+    /** Address of the page. */
+    RTGCPHYS volatile                   GCPhys;
+    /** The guest page. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(PPGMPAGE) volatile        pPage;
+#else
+    R3R0PTRTYPE(PPGMPAGE) volatile      pPage;
+#endif
+    /** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(PPGMCHUNKR3MAP) volatile  pMap;
+#else
+    R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
+#endif
+    /** The address */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(void *) volatile          pv;
+#else
+    R3R0PTRTYPE(void *) volatile        pv;
+#endif
+#if HC_ARCH_BITS == 32
+    uint32_t                            u32Padding; /**< alignment padding. */
+#endif
+} PGMPAGER3MAPTLBE;
+/** Pointer to an entry in the HC physical TLB. */
+typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
+
+
+/** The number of entries in the ring-3 guest page mapping TLB.
+ * @remarks The value must be a power of two. */
+#define PGM_PAGER3MAPTLB_ENTRIES 256
+
+/**
+ * Ring-3 guest page mapping TLB.
+ * @remarks used in ring-0 as well at the moment.
+ */
+typedef struct PGMPAGER3MAPTLB
+{
+    /** The TLB entries. */
+    PGMPAGER3MAPTLBE            aEntries[PGM_PAGER3MAPTLB_ENTRIES];
+} PGMPAGER3MAPTLB;
+/** Pointer to the ring-3 guest page mapping TLB. */
+typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
+
+/**
+ * Calculates the index of the TLB entry for the specified guest page.
+ * @returns Physical TLB index.
+ * @param   GCPhys      The guest physical address.
+ */
+#define PGM_PAGER3MAPTLB_IDX(GCPhys)    ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
+
+
+/**
+ * Raw-mode context dynamic mapping cache entry.
+ *
+ * Because of raw-mode context being reloctable and all relocations are applied
+ * in ring-3, this has to be defined here and be RC specific.
+ *
+ * @sa PGMRZDYNMAPENTRY, PGMR0DYNMAPENTRY.
+ */
+typedef struct PGMRCDYNMAPENTRY
+{
+    /** The physical address of the currently mapped page.
+     * This is duplicate for three reasons: cache locality, cache policy of the PT
+     * mappings and sanity checks.   */
+    RTHCPHYS                    HCPhys;
+    /** Pointer to the page. */
+    RTRCPTR                     pvPage;
+    /** The number of references. */
+    int32_t volatile            cRefs;
+    /** PTE pointer union. */
+    union PGMRCDYNMAPENTRY_PPTE
+    {
+        /** PTE pointer, 32-bit legacy version. */
+        RCPTRTYPE(PX86PTE)      pLegacy;
+        /** PTE pointer, PAE version. */
+        RCPTRTYPE(PX86PTEPAE)   pPae;
+        /** PTE pointer, the void version. */
+        RTRCPTR                 pv;
+    } uPte;
+    /** Alignment padding. */
+    RTRCPTR                     RCPtrAlignment;
+} PGMRCDYNMAPENTRY;
+/** Pointer to a dynamic mapping cache entry for the raw-mode context. */
+typedef PGMRCDYNMAPENTRY *PPGMRCDYNMAPENTRY;
+
+
+/**
+ * Dynamic mapping cache for the raw-mode context.
+ *
+ * This is initialized during VMMRC init based upon the pbDynPageMapBaseGC and
+ * paDynPageMap* PGM members.  However, it has to be defined in PGMInternal.h
+ * so that we can perform relocations from PGMR3Relocate.  This has the
+ * consequence that we must have separate ring-0 and raw-mode context versions
+ * of this struct even if they share the basic elements.
+ *
+ * @sa PPGMRZDYNMAP, PGMR0DYNMAP.
+ */
+typedef struct PGMRCDYNMAP
+{
+    /** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
+    uint32_t                        u32Magic;
+    /** Array for tracking and managing the pages.  */
+    RCPTRTYPE(PPGMRCDYNMAPENTRY)    paPages;
+    /** The cache size given as a number of pages. */
+    uint32_t                        cPages;
+    /** Whether it's 32-bit legacy or PAE/AMD64 paging mode. */
+    bool                            fLegacyMode;
+    /** The current load.
+     * This does not include guard pages. */
+    uint32_t                        cLoad;
+    /** The max load ever.
+     * This is maintained to get trigger adding of more mapping space. */
+    uint32_t                        cMaxLoad;
+    /** The number of guard pages. */
+    uint32_t                        cGuardPages;
+    /** The number of users (protected by hInitLock). */
+    uint32_t                        cUsers;
+} PGMRCDYNMAP;
+/** Pointer to the dynamic cache for the raw-mode context. */
+typedef PGMRCDYNMAP *PPGMRCDYNMAP;
+
+
+/**
+ * Mapping cache usage set entry.
+ *
+ * @remarks 16-bit ints was chosen as the set is not expected to be used beyond
+ *          the dynamic ring-0 and (to some extent) raw-mode context mapping
+ *          cache.  If it's extended to include ring-3, well, then something
+ *          will have be changed here...
+ */
+typedef struct PGMMAPSETENTRY
+{
+    /** Pointer to the page. */
+#ifndef IN_RC
+    RTR0PTR                     pvPage;
+#else
+    RTRCPTR                     pvPage;
+# if HC_ARCH_BITS == 64
+    uint32_t                    u32Alignment2;
+# endif
+#endif
+    /** The mapping cache index. */
+    uint16_t                    iPage;
+    /** The number of references.
+     * The max is UINT16_MAX - 1. */
+    uint16_t                    cRefs;
+    /** The number inlined references.
+     * The max is UINT16_MAX - 1. */
+    uint16_t                    cInlinedRefs;
+    /** Unreferences.  */
+    uint16_t                    cUnrefs;
+
+#if HC_ARCH_BITS == 32
+    uint32_t                    u32Alignment1;
+#endif
+    /** The physical address for this entry. */
+    RTHCPHYS                    HCPhys;
+} PGMMAPSETENTRY;
+AssertCompileMemberOffset(PGMMAPSETENTRY, iPage, RT_MAX(sizeof(RTR0PTR), sizeof(RTRCPTR)));
+AssertCompileMemberAlignment(PGMMAPSETENTRY, HCPhys, sizeof(RTHCPHYS));
+/** Pointer to a mapping cache usage set entry. */
+typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
+
+/**
+ * Mapping cache usage set.
+ *
+ * This is used in ring-0 and the raw-mode context to track dynamic mappings
+ * done during exits / traps.  The set is
+ */
+typedef struct PGMMAPSET
+{
+    /** The number of occupied entries.
+     * This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
+     * dynamic mappings. */
+    uint32_t                    cEntries;
+    /** The start of the current subset.
+     * This is UINT32_MAX if no subset is currently open. */
+    uint32_t                    iSubset;
+    /** The index of the current CPU, only valid if the set is open. */
+    int32_t                     iCpu;
+    uint32_t                    alignment;
+    /** The entries. */
+    PGMMAPSETENTRY              aEntries[64];
+    /** HCPhys -> iEntry fast lookup table.
+     * Use PGMMAPSET_HASH for hashing.
+     * The entries may or may not be valid, check against cEntries. */
+    uint8_t                     aiHashTable[128];
+} PGMMAPSET;
+AssertCompileSizeAlignment(PGMMAPSET, 8);
+/** Pointer to the mapping cache set. */
+typedef PGMMAPSET *PPGMMAPSET;
+
+/** PGMMAPSET::cEntries value for a closed set. */
+#define PGMMAPSET_CLOSED            UINT32_C(0xdeadc0fe)
+
+/** Hash function for aiHashTable. */
+#define PGMMAPSET_HASH(HCPhys)      (((HCPhys) >> PAGE_SHIFT) & 127)
+
+
+/** @name Context neutral page mapper TLB.
+ *
+ * Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
+ * code is writting in a kind of context neutral way. Time will show whether
+ * this actually makes sense or not...
+ *
+ * @todo this needs to be reconsidered and dropped/redone since the ring-0
+ *       context ends up using a global mapping cache on some platforms
+ *       (darwin).
+ *
+ * @{ */
+/** @typedef PPGMPAGEMAPTLB
+ * The page mapper TLB pointer type for the current context. */
+/** @typedef PPGMPAGEMAPTLB
+ * The page mapper TLB entry pointer type for the current context. */
+/** @typedef PPGMPAGEMAPTLB
+ * The page mapper TLB entry pointer pointer type for the current context. */
+/** @def PGM_PAGEMAPTLB_ENTRIES
+ * The number of TLB entries in the page mapper TLB for the current context. */
+/** @def PGM_PAGEMAPTLB_IDX
+ * Calculate the TLB index for a guest physical address.
+ * @returns The TLB index.
+ * @param   GCPhys      The guest physical address. */
+/** @typedef PPGMPAGEMAP
+ * Pointer to a page mapper unit for current context. */
+/** @typedef PPPGMPAGEMAP
+ * Pointer to a page mapper unit pointer for current context. */
+#ifdef IN_RC
+// typedef PPGMPAGEGCMAPTLB               PPGMPAGEMAPTLB;
+// typedef PPGMPAGEGCMAPTLBE              PPGMPAGEMAPTLBE;
+// typedef PPGMPAGEGCMAPTLBE             *PPPGMPAGEMAPTLBE;
+# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGEGCMAPTLB_ENTRIES
+# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGEGCMAPTLB_IDX(GCPhys)
+ typedef void *                         PPGMPAGEMAP;
+ typedef void **                        PPPGMPAGEMAP;
+//#elif IN_RING0
+// typedef PPGMPAGER0MAPTLB               PPGMPAGEMAPTLB;
+// typedef PPGMPAGER0MAPTLBE              PPGMPAGEMAPTLBE;
+// typedef PPGMPAGER0MAPTLBE             *PPPGMPAGEMAPTLBE;
+//# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGER0MAPTLB_ENTRIES
+//# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGER0MAPTLB_IDX(GCPhys)
+// typedef PPGMCHUNKR0MAP                 PPGMPAGEMAP;
+// typedef PPPGMCHUNKR0MAP                PPPGMPAGEMAP;
+#else
+ typedef PPGMPAGER3MAPTLB               PPGMPAGEMAPTLB;
+ typedef PPGMPAGER3MAPTLBE              PPGMPAGEMAPTLBE;
+ typedef PPGMPAGER3MAPTLBE             *PPPGMPAGEMAPTLBE;
+# define PGM_PAGEMAPTLB_ENTRIES         PGM_PAGER3MAPTLB_ENTRIES
+# define PGM_PAGEMAPTLB_IDX(GCPhys)     PGM_PAGER3MAPTLB_IDX(GCPhys)
+ typedef PPGMCHUNKR3MAP                 PPGMPAGEMAP;
+ typedef PPPGMCHUNKR3MAP                PPPGMPAGEMAP;
+#endif
+/** @} */
+
+
+/** @name PGM Pool Indexes.
+ * Aka. the unique shadow page identifier.
+ * @{ */
+/** NIL page pool IDX. */
+#define NIL_PGMPOOL_IDX                 0
+/** The first normal index. */
+#define PGMPOOL_IDX_FIRST_SPECIAL       1
+/** Page directory (32-bit root). */
+#define PGMPOOL_IDX_PD                  1
+/** Page Directory Pointer Table (PAE root). */
+#define PGMPOOL_IDX_PDPT                2
+/** AMD64 CR3 level index.*/
+#define PGMPOOL_IDX_AMD64_CR3           3
+/** Nested paging root.*/
+#define PGMPOOL_IDX_NESTED_ROOT         4
+/** The first normal index. */
+#define PGMPOOL_IDX_FIRST               5
+/** The last valid index. (inclusive, 14 bits) */
+#define PGMPOOL_IDX_LAST                0x3fff
+/** @} */
+
+/** The NIL index for the parent chain. */
+#define NIL_PGMPOOL_USER_INDEX          ((uint16_t)0xffff)
+#define NIL_PGMPOOL_PRESENT_INDEX       ((uint16_t)0xffff)
+
+/**
+ * Node in the chain linking a shadowed page to it's parent (user).
+ */
+#pragma pack(1)
+typedef struct PGMPOOLUSER
+{
+    /** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
+    uint16_t            iNext;
+    /** The user page index. */
+    uint16_t            iUser;
+    /** Index into the user table. */
+    uint32_t            iUserTable;
+} PGMPOOLUSER, *PPGMPOOLUSER;
+typedef const PGMPOOLUSER *PCPGMPOOLUSER;
+#pragma pack()
+
+
+/** The NIL index for the phys ext chain. */
+#define NIL_PGMPOOL_PHYSEXT_INDEX       ((uint16_t)0xffff)
+/** The NIL pte index for a phys ext chain slot. */
+#define NIL_PGMPOOL_PHYSEXT_IDX_PTE     ((uint16_t)0xffff)
+
+/**
+ * Node in the chain of physical cross reference extents.
+ * @todo Calling this an 'extent' is not quite right, find a better name.
+ * @todo find out the optimal size of the aidx array
+ */
+#pragma pack(1)
+typedef struct PGMPOOLPHYSEXT
+{
+    /** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
+    uint16_t            iNext;
+    /** Alignment. */
+    uint16_t            u16Align;
+    /** The user page index. */
+    uint16_t            aidx[3];
+    /** The page table index or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown. */
+    uint16_t            apte[3];
+} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
+typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
+#pragma pack()
+
+
+/**
+ * The kind of page that's being shadowed.
+ */
+typedef enum PGMPOOLKIND
+{
+    /** The virtual invalid 0 entry. */
+    PGMPOOLKIND_INVALID = 0,
+    /** The entry is free (=unused). */
+    PGMPOOLKIND_FREE,
+
+    /** Shw: 32-bit page table;     Gst: no paging  */
+    PGMPOOLKIND_32BIT_PT_FOR_PHYS,
+    /** Shw: 32-bit page table;     Gst: 32-bit page table.  */
+    PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
+    /** Shw: 32-bit page table;     Gst: 4MB page.  */
+    PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
+    /** Shw: PAE page table;        Gst: no paging  */
+    PGMPOOLKIND_PAE_PT_FOR_PHYS,
+    /** Shw: PAE page table;        Gst: 32-bit page table. */
+    PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
+    /** Shw: PAE page table;        Gst: Half of a 4MB page.  */
+    PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
+    /** Shw: PAE page table;        Gst: PAE page table. */
+    PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
+    /** Shw: PAE page table;        Gst: 2MB page.  */
+    PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
+
+    /** Shw: 32-bit page directory. Gst: 32-bit page directory. */
+    PGMPOOLKIND_32BIT_PD,
+    /** Shw: 32-bit page directory. Gst: no paging. */
+    PGMPOOLKIND_32BIT_PD_PHYS,
+    /** Shw: PAE page directory 0;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
+    /** Shw: PAE page directory 1;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
+    /** Shw: PAE page directory 2;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
+    /** Shw: PAE page directory 3;  Gst: 32-bit page directory. */
+    PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
+    /** Shw: PAE page directory;    Gst: PAE page directory. */
+    PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
+    /** Shw: PAE page directory;    Gst: no paging.             Note: +NP. */
+    PGMPOOLKIND_PAE_PD_PHYS,
+
+    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst 32 bits paging. */
+    PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
+    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst PAE PDPT. */
+    PGMPOOLKIND_PAE_PDPT,
+    /** Shw: PAE page directory pointer table (legacy, 4 entries);  Gst: no paging. */
+    PGMPOOLKIND_PAE_PDPT_PHYS,
+
+    /** Shw: 64-bit page directory pointer table;   Gst: 64-bit page directory pointer table. */
+    PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
+    /** Shw: 64-bit page directory pointer table;   Gst: no paging  */
+    PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
+    /** Shw: 64-bit page directory table;           Gst: 64-bit page directory table. */
+    PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
+    /** Shw: 64-bit page directory table;           Gst: no paging  */
+    PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
+
+    /** Shw: 64-bit PML4;                           Gst: 64-bit PML4. */
+    PGMPOOLKIND_64BIT_PML4,
+
+    /** Shw: EPT page directory pointer table;      Gst: no paging  */
+    PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
+    /** Shw: EPT page directory table;              Gst: no paging  */
+    PGMPOOLKIND_EPT_PD_FOR_PHYS,
+    /** Shw: EPT page table;                        Gst: no paging  */
+    PGMPOOLKIND_EPT_PT_FOR_PHYS,
+
+    /** Shw: Root Nested paging table. */
+    PGMPOOLKIND_ROOT_NESTED,
+
+    /** The last valid entry. */
+    PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
+} PGMPOOLKIND;
+
+/**
+ * The access attributes of the page; only applies to big pages.
+ */
+typedef enum
+{
+    PGMPOOLACCESS_DONTCARE = 0,
+    PGMPOOLACCESS_USER_RW,
+    PGMPOOLACCESS_USER_R,
+    PGMPOOLACCESS_USER_RW_NX,
+    PGMPOOLACCESS_USER_R_NX,
+    PGMPOOLACCESS_SUPERVISOR_RW,
+    PGMPOOLACCESS_SUPERVISOR_R,
+    PGMPOOLACCESS_SUPERVISOR_RW_NX,
+    PGMPOOLACCESS_SUPERVISOR_R_NX
+} PGMPOOLACCESS;
+
+/**
+ * The tracking data for a page in the pool.
+ */
+typedef struct PGMPOOLPAGE
+{
+    /** AVL node code with the (R3) physical address of this page. */
+    AVLOHCPHYSNODECORE  Core;
+    /** Pointer to the R3 mapping of the page. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R3PTRTYPE(void *)   pvPageR3;
+#else
+    R3R0PTRTYPE(void *) pvPageR3;
+#endif
+    /** The guest physical address. */
+#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
+    uint32_t            Alignment0;
+#endif
+    RTGCPHYS            GCPhys;
+
+    /** Access handler statistics to determine whether the guest is (re)initializing a page table. */
+    RTGCPTR             pvLastAccessHandlerRip;
+    RTGCPTR             pvLastAccessHandlerFault;
+    uint64_t            cLastAccessHandlerCount;
+
+    /** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
+    uint8_t             enmKind;
+    /** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */
+    uint8_t             enmAccess;
+    /** The index of this page. */
+    uint16_t            idx;
+    /** The next entry in the list this page currently resides in.
+     * It's either in the free list or in the GCPhys hash. */
+    uint16_t            iNext;
+    /** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
+    uint16_t            iUserHead;
+    /** The number of present entries. */
+    uint16_t            cPresent;
+    /** The first entry in the table which is present. */
+    uint16_t            iFirstPresent;
+    /** The number of modifications to the monitored page. */
+    uint16_t            cModifications;
+    /** The next modified page. NIL_PGMPOOL_IDX if tail. */
+    uint16_t            iModifiedNext;
+    /** The previous modified page. NIL_PGMPOOL_IDX if head. */
+    uint16_t            iModifiedPrev;
+    /** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
+    uint16_t            iMonitoredNext;
+    /** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
+    uint16_t            iMonitoredPrev;
+    /** The next page in the age list. */
+    uint16_t            iAgeNext;
+    /** The previous page in the age list. */
+    uint16_t            iAgePrev;
+    /** Used to indicate that the page is zeroed. */
+    bool                fZeroed;
+    /** Used to indicate that a PT has non-global entries. */
+    bool                fSeenNonGlobal;
+    /** Used to indicate that we're monitoring writes to the guest page. */
+    bool                fMonitored;
+    /** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
+     * (All pages are in the age list.) */
+    bool                fCached;
+    /** This is used by the R3 access handlers when invoked by an async thread.
+     * It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
+    bool volatile       fReusedFlushPending;
+    /** Used to mark the page as dirty (write monitoring is temporarily
+     *  off). */
+    bool                fDirty;
+
+    /** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
+    uint32_t            cLocked;
+    uint32_t            idxDirty;
+    RTGCPTR             pvDirtyFault;
+} PGMPOOLPAGE, *PPGMPOOLPAGE, **PPPGMPOOLPAGE;
+/** Pointer to a const pool page. */
+typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
+
+
+/** The hash table size. */
+# define PGMPOOL_HASH_SIZE      0x40
+/** The hash function. */
+# define PGMPOOL_HASH(GCPhys)   ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
+
+
+/**
+ * The shadow page pool instance data.
+ *
+ * It's all one big allocation made at init time, except for the
+ * pages that is. The user nodes follows immediately after the
+ * page structures.
+ */
+typedef struct PGMPOOL
+{
+    /** The VM handle - R3 Ptr. */
+    PVMR3                       pVMR3;
+    /** The VM handle - R0 Ptr. */
+    PVMR0                       pVMR0;
+    /** The VM handle - RC Ptr. */
+    PVMRC                       pVMRC;
+    /** The max pool size. This includes the special IDs.  */
+    uint16_t                    cMaxPages;
+    /** The current pool size. */
+    uint16_t                    cCurPages;
+    /** The head of the free page list. */
+    uint16_t                    iFreeHead;
+    /* Padding. */
+    uint16_t                    u16Padding;
+    /** Head of the chain of free user nodes. */
+    uint16_t                    iUserFreeHead;
+    /** The number of user nodes we've allocated. */
+    uint16_t                    cMaxUsers;
+    /** The number of present page table entries in the entire pool. */
+    uint32_t                    cPresent;
+    /** Pointer to the array of user nodes - RC pointer. */
+    RCPTRTYPE(PPGMPOOLUSER)     paUsersRC;
+    /** Pointer to the array of user nodes - R3 pointer. */
+    R3PTRTYPE(PPGMPOOLUSER)     paUsersR3;
+    /** Pointer to the array of user nodes - R0 pointer. */
+    R0PTRTYPE(PPGMPOOLUSER)     paUsersR0;
+    /** Head of the chain of free phys ext nodes. */
+    uint16_t                    iPhysExtFreeHead;
+    /** The number of user nodes we've allocated. */
+    uint16_t                    cMaxPhysExts;
+    /** Pointer to the array of physical xref extent - RC pointer. */
+    RCPTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsRC;
+    /** Pointer to the array of physical xref extent nodes - R3 pointer. */
+    R3PTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsR3;
+    /** Pointer to the array of physical xref extent nodes - R0 pointer. */
+    R0PTRTYPE(PPGMPOOLPHYSEXT)  paPhysExtsR0;
+    /** Hash table for GCPhys addresses. */
+    uint16_t                    aiHash[PGMPOOL_HASH_SIZE];
+    /** The head of the age list. */
+    uint16_t                    iAgeHead;
+    /** The tail of the age list. */
+    uint16_t                    iAgeTail;
+    /** Set if the cache is enabled. */
+    bool                        fCacheEnabled;
+    /** Alignment padding. */
+    bool                        afPadding1[3];
+    /** Head of the list of modified pages. */
+    uint16_t                    iModifiedHead;
+    /** The current number of modified pages. */
+    uint16_t                    cModifiedPages;
+    /** Access handler, RC. */
+    RCPTRTYPE(PFNPGMRCPHYSHANDLER)  pfnAccessHandlerRC;
+    /** Access handler, R0. */
+    R0PTRTYPE(PFNPGMR0PHYSHANDLER)  pfnAccessHandlerR0;
+    /** Access handler, R3. */
+    R3PTRTYPE(PFNPGMR3PHYSHANDLER)  pfnAccessHandlerR3;
+    /** The access handler description (R3 ptr). */
+    R3PTRTYPE(const char *)         pszAccessHandler;
+# if HC_ARCH_BITS == 32
+    /** Alignment padding. */
+    uint32_t                    u32Padding2;
+# endif
+    /* Next available slot. */
+    uint32_t                    idxFreeDirtyPage;
+    /* Number of active dirty pages. */
+    uint32_t                    cDirtyPages;
+    /* Array of current dirty pgm pool page indices. */
+    struct
+    {
+        uint16_t                    uIdx;
+        uint16_t                    Alignment[3];
+        uint64_t                    aPage[512];
+    } aDirtyPages[16];
+    /** The number of pages currently in use. */
+    uint16_t                    cUsedPages;
+#ifdef VBOX_WITH_STATISTICS
+    /** The high water mark for cUsedPages. */
+    uint16_t                    cUsedPagesHigh;
+    uint32_t                    Alignment1;         /**< Align the next member on a 64-bit boundary. */
+    /** Profiling pgmPoolAlloc(). */
+    STAMPROFILEADV              StatAlloc;
+    /** Profiling pgmR3PoolClearDoIt(). */
+    STAMPROFILE                 StatClearAll;
+    /** Profiling pgmR3PoolReset(). */
+    STAMPROFILE                 StatR3Reset;
+    /** Profiling pgmPoolFlushPage(). */
+    STAMPROFILE                 StatFlushPage;
+    /** Profiling pgmPoolFree(). */
+    STAMPROFILE                 StatFree;
+    /** Counting explicit flushes by PGMPoolFlushPage(). */
+    STAMCOUNTER                 StatForceFlushPage;
+    /** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */
+    STAMCOUNTER                 StatForceFlushDirtyPage;
+    /** Counting flushes for reused pages. */
+    STAMCOUNTER                 StatForceFlushReused;
+    /** Profiling time spent zeroing pages. */
+    STAMPROFILE                 StatZeroPage;
+    /** Profiling of pgmPoolTrackDeref. */
+    STAMPROFILE                 StatTrackDeref;
+    /** Profiling pgmTrackFlushGCPhysPT. */
+    STAMPROFILE                 StatTrackFlushGCPhysPT;
+    /** Profiling pgmTrackFlushGCPhysPTs. */
+    STAMPROFILE                 StatTrackFlushGCPhysPTs;
+    /** Profiling pgmTrackFlushGCPhysPTsSlow. */
+    STAMPROFILE                 StatTrackFlushGCPhysPTsSlow;
+    /** Number of times we've been out of user records. */
+    STAMCOUNTER                 StatTrackFreeUpOneUser;
+    /** Nr of flushed entries. */
+    STAMCOUNTER                 StatTrackFlushEntry;
+    /** Nr of updated entries. */
+    STAMCOUNTER                 StatTrackFlushEntryKeep;
+    /** Profiling deref activity related tracking GC physical pages. */
+    STAMPROFILE                 StatTrackDerefGCPhys;
+    /** Number of linear searches for a HCPhys in the ram ranges. */
+    STAMCOUNTER                 StatTrackLinearRamSearches;
+    /** The number of failing pgmPoolTrackPhysExtAlloc calls. */
+    STAMCOUNTER                 StamTrackPhysExtAllocFailures;
+    /** Profiling the RC/R0 access handler. */
+    STAMPROFILE                 StatMonitorRZ;
+    /** Times we've failed interpreting the instruction. */
+    STAMCOUNTER                 StatMonitorRZEmulateInstr;
+    /** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
+    STAMPROFILE                 StatMonitorRZFlushPage;
+    /* Times we've detected a page table reinit. */
+    STAMCOUNTER                 StatMonitorRZFlushReinit;
+    /** Counting flushes for pages that are modified too often. */
+    STAMCOUNTER                 StatMonitorRZFlushModOverflow;
+    /** Times we've detected fork(). */
+    STAMCOUNTER                 StatMonitorRZFork;
+    /** Profiling the RC/R0 access we've handled (except REP STOSD). */
+    STAMPROFILE                 StatMonitorRZHandled;
+    /** Times we've failed interpreting a patch code instruction. */
+    STAMCOUNTER                 StatMonitorRZIntrFailPatch1;
+    /** Times we've failed interpreting a patch code instruction during flushing. */
+    STAMCOUNTER                 StatMonitorRZIntrFailPatch2;
+    /** The number of times we've seen rep prefixes we can't handle. */
+    STAMCOUNTER                 StatMonitorRZRepPrefix;
+    /** Profiling the REP STOSD cases we've handled. */
+    STAMPROFILE                 StatMonitorRZRepStosd;
+    /** Nr of handled PT faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPT;
+    /** Nr of handled PD faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPD;
+    /** Nr of handled PDPT faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPDPT;
+    /** Nr of handled PML4 faults. */
+    STAMCOUNTER                 StatMonitorRZFaultPML4;
+
+    /** Profiling the R3 access handler. */
+    STAMPROFILE                 StatMonitorR3;
+    /** Times we've failed interpreting the instruction. */
+    STAMCOUNTER                 StatMonitorR3EmulateInstr;
+    /** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
+    STAMPROFILE                 StatMonitorR3FlushPage;
+    /* Times we've detected a page table reinit. */
+    STAMCOUNTER                 StatMonitorR3FlushReinit;
+    /** Counting flushes for pages that are modified too often. */
+    STAMCOUNTER                 StatMonitorR3FlushModOverflow;
+    /** Times we've detected fork(). */
+    STAMCOUNTER                 StatMonitorR3Fork;
+    /** Profiling the R3 access we've handled (except REP STOSD). */
+    STAMPROFILE                 StatMonitorR3Handled;
+    /** The number of times we've seen rep prefixes we can't handle. */
+    STAMCOUNTER                 StatMonitorR3RepPrefix;
+    /** Profiling the REP STOSD cases we've handled. */
+    STAMPROFILE                 StatMonitorR3RepStosd;
+    /** Nr of handled PT faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPT;
+    /** Nr of handled PD faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPD;
+    /** Nr of handled PDPT faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPDPT;
+    /** Nr of handled PML4 faults. */
+    STAMCOUNTER                 StatMonitorR3FaultPML4;
+    /** The number of times we're called in an async thread an need to flush. */
+    STAMCOUNTER                 StatMonitorR3Async;
+    /** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
+    STAMCOUNTER                 StatResetDirtyPages;
+    /** Times we've called pgmPoolAddDirtyPage. */
+    STAMCOUNTER                 StatDirtyPage;
+    /** Times we've had to flush duplicates for dirty page management. */
+    STAMCOUNTER                 StatDirtyPageDupFlush;
+    /** Times we've had to flush because of overflow. */
+    STAMCOUNTER                 StatDirtyPageOverFlowFlush;
+
+    /** The high water mark for cModifiedPages. */
+    uint16_t                    cModifiedPagesHigh;
+    uint16_t                    Alignment2[3];      /**< Align the next member on a 64-bit boundary. */
+
+    /** The number of cache hits. */
+    STAMCOUNTER                 StatCacheHits;
+    /** The number of cache misses. */
+    STAMCOUNTER                 StatCacheMisses;
+    /** The number of times we've got a conflict of 'kind' in the cache. */
+    STAMCOUNTER                 StatCacheKindMismatches;
+    /** Number of times we've been out of pages. */
+    STAMCOUNTER                 StatCacheFreeUpOne;
+    /** The number of cacheable allocations. */
+    STAMCOUNTER                 StatCacheCacheable;
+    /** The number of uncacheable allocations. */
+    STAMCOUNTER                 StatCacheUncacheable;
+#else
+    uint32_t                    Alignment3;         /**< Align the next member on a 64-bit boundary. */
+#endif
+    /** The AVL tree for looking up a page by its HC physical address. */
+    AVLOHCPHYSTREE              HCPhysTree;
+    uint32_t                    Alignment4;         /**< Align the next member on a 64-bit boundary. */
+    /** Array of pages. (cMaxPages in length)
+     * The Id is the index into thist array.
+     */
+    PGMPOOLPAGE                 aPages[PGMPOOL_IDX_FIRST];
+} PGMPOOL, *PPGMPOOL, **PPPGMPOOL;
+AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8);
+AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8);
+AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8);
+#ifdef VBOX_WITH_STATISTICS
+AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8);
+#endif
+AssertCompileMemberAlignment(PGMPOOL, aPages, 8);
+
+
+/** @def PGMPOOL_PAGE_2_PTR
+ * Maps a pool page pool into the current context.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   pPage   The pool page.
+ *
+ * @remark  In RC this uses PGMGCDynMapHCPage(), so it will consume of the
+ *          small page window employeed by that function. Be careful.
+ * @remark  There is no need to assert on the result.
+ */
+#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+# define PGMPOOL_PAGE_2_PTR(pVM, pPage)  pgmPoolMapPageInlined((pVM), (pPage) RTLOG_COMMA_SRC_POS)
+#elif defined(VBOX_STRICT)
+# define PGMPOOL_PAGE_2_PTR(pVM, pPage)  pgmPoolMapPageStrict(pPage)
+DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
+{
+    Assert(pPage && pPage->pvPageR3);
+    return pPage->pvPageR3;
+}
+#else
+# define PGMPOOL_PAGE_2_PTR(pVM, pPage)  ((pPage)->pvPageR3)
+#endif
+
+
+/** @def PGMPOOL_PAGE_2_PTR_V2
+ * Maps a pool page pool into the current context, taking both VM and VMCPU.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ * @param   pVCpu   The current CPU.
+ * @param   pPage   The pool page.
+ *
+ * @remark  In RC this uses PGMGCDynMapHCPage(), so it will consume of the
+ *          small page window employeed by that function. Be careful.
+ * @remark  There is no need to assert on the result.
+ */
+#if defined(IN_RC) || defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
+# define PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPage)   pgmPoolMapPageV2Inlined((pVM), (pVCpu), (pPage) RTLOG_COMMA_SRC_POS)
+#else
+# define PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPage)   PGMPOOL_PAGE_2_PTR((pVM), (pPage))
+#endif
+
+
+/** @name Per guest page tracking data.
+ * This is currently as a 16-bit word in the PGMPAGE structure, the idea though
+ * is to use more bits for it and split it up later on. But for now we'll play
+ * safe and change as little as possible.
+ *
+ * The 16-bit word has two parts:
+ *
+ * The first 14-bit forms the @a idx field. It is either the index of a page in
+ * the shadow page pool, or and index into the extent list.
+ *
+ * The 2 topmost bits makes up the @a cRefs field, which counts the number of
+ * shadow page pool references to the page. If cRefs equals
+ * PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
+ * (misnomer) table and not the shadow page pool.
+ *
+ * See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
+ * the 16-bit word.
+ *
+ * @{ */
+/** The shift count for getting to the cRefs part. */
+#define PGMPOOL_TD_CREFS_SHIFT          14
+/** The mask applied after shifting the tracking data down by
+ * PGMPOOL_TD_CREFS_SHIFT. */
+#define PGMPOOL_TD_CREFS_MASK           0x3
+/** The cRefs value used to indicate that the idx is the head of a
+ * physical cross reference list. */
+#define PGMPOOL_TD_CREFS_PHYSEXT        PGMPOOL_TD_CREFS_MASK
+/** The shift used to get idx. */
+#define PGMPOOL_TD_IDX_SHIFT            0
+/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
+#define PGMPOOL_TD_IDX_MASK             0x3fff
+/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
+ * simply too many mappings of this page. */
+#define PGMPOOL_TD_IDX_OVERFLOWED       PGMPOOL_TD_IDX_MASK
+
+/** @def PGMPOOL_TD_MAKE
+ * Makes a 16-bit tracking data word.
+ *
+ * @returns tracking data.
+ * @param   cRefs       The @a cRefs field. Must be within bounds!
+ * @param   idx         The @a idx field. Must also be within bounds! */
+#define PGMPOOL_TD_MAKE(cRefs, idx)     ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) | (idx) )
+
+/** @def PGMPOOL_TD_GET_CREFS
+ * Get the @a cRefs field from a tracking data word.
+ *
+ * @returns The @a cRefs field
+ * @param   u16         The tracking data word.
+ * @remarks This will only return 1 or PGMPOOL_TD_CREFS_PHYSEXT for a
+ *          non-zero @a u16. */
+#define PGMPOOL_TD_GET_CREFS(u16)       ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
+
+/** @def PGMPOOL_TD_GET_IDX
+ * Get the @a idx field from a tracking data word.
+ *
+ * @returns The @a idx field
+ * @param   u16         The tracking data word. */
+#define PGMPOOL_TD_GET_IDX(u16)         ( ((u16) >> PGMPOOL_TD_IDX_SHIFT)   & PGMPOOL_TD_IDX_MASK   )
+/** @} */
+
+
+/**
+ * Trees are using self relative offsets as pointers.
+ * So, all its data, including the root pointer, must be in the heap for HC and GC
+ * to have the same layout.
+ */
+typedef struct PGMTREES
+{
+    /** Physical access handlers (AVL range+offsetptr tree). */
+    AVLROGCPHYSTREE                 PhysHandlers;
+    /** Virtual access handlers (AVL range + GC ptr tree). */
+    AVLROGCPTRTREE                  VirtHandlers;
+    /** Virtual access handlers (Phys range AVL range + offsetptr tree). */
+    AVLROGCPHYSTREE                 PhysToVirtHandlers;
+    /** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
+    AVLROGCPTRTREE                  HyperVirtHandlers;
+} PGMTREES;
+/** Pointer to PGM trees. */
+typedef PGMTREES *PPGMTREES;
+
+
+/**
+ * Page fault guest state for the AMD64 paging mode.
+ */
+typedef struct PGMPTWALKCORE
+{
+    /** The guest virtual address that is being resolved by the walk
+     *  (input). */
+    RTGCPTR         GCPtr;
+
+    /** The guest physical address that is the result of the walk.
+     * @remarks only valid if fSucceeded is set.  */
+    RTGCPHYS        GCPhys;
+
+    /** Set if the walk succeeded, i.d. GCPhys is valid. */
+    bool            fSucceeded;
+    /** The level problem arrised at.
+     * PTE is level 1, PDE is level 2, PDPE is level 3, PML4 is level 4, CR3 is
+     * level 8.  This is 0 on success. */
+    uint8_t         uLevel;
+    /** Set if the page isn't present. */
+    bool            fNotPresent;
+    /** Encountered a bad physical address. */
+    bool            fBadPhysAddr;
+    /** Set if there was reserved bit violations. */
+    bool            fRsvdError;
+    /** Set if it involves a big page (2/4 MB). */
+    bool            fBigPage;
+    /** Set if it involves a gigantic page (1 GB). */
+    bool            fGigantPage;
+    /** The effect X86_PTE_US flag for the address. */
+    bool            fEffectiveUS;
+    /** The effect X86_PTE_RW flag for the address. */
+    bool            fEffectiveRW;
+    /** The effect X86_PTE_NX flag for the address. */
+    bool            fEffectiveNX;
+} PGMPTWALKCORE;
+
+
+/**
+ * Guest page table walk for the AMD64 mode.
+ */
+typedef struct PGMPTWALKGSTAMD64
+{
+    /** The common core. */
+    PGMPTWALKCORE   Core;
+
+    PX86PML4        pPml4;
+    PX86PML4E       pPml4e;
+    X86PML4E        Pml4e;
+
+    PX86PDPT        pPdpt;
+    PX86PDPE        pPdpe;
+    X86PDPE         Pdpe;
+
+    PX86PDPAE       pPd;
+    PX86PDEPAE      pPde;
+    X86PDEPAE       Pde;
+
+    PX86PTPAE       pPt;
+    PX86PTEPAE      pPte;
+    X86PTEPAE       Pte;
+} PGMPTWALKGSTAMD64;
+/** Pointer to a AMD64 guest page table walk. */
+typedef PGMPTWALKGSTAMD64 *PPGMPTWALKGSTAMD64;
+/** Pointer to a const AMD64 guest page table walk. */
+typedef PGMPTWALKGSTAMD64 const *PCPGMPTWALKGSTAMD64;
+
+/**
+ * Guest page table walk for the PAE mode.
+ */
+typedef struct PGMPTWALKGSTPAE
+{
+    /** The common core. */
+    PGMPTWALKCORE   Core;
+
+    PX86PDPT        pPdpt;
+    PX86PDPE        pPdpe;
+    X86PDPE         Pdpe;
+
+    PX86PDPAE       pPd;
+    PX86PDEPAE      pPde;
+    X86PDEPAE       Pde;
+
+    PX86PTPAE       pPt;
+    PX86PTEPAE      pPte;
+    X86PTEPAE       Pte;
+} PGMPTWALKGSTPAE;
+/** Pointer to a PAE guest page table walk. */
+typedef PGMPTWALKGSTPAE *PPGMPTWALKGSTPAE;
+/** Pointer to a const AMD64 guest page table walk. */
+typedef PGMPTWALKGSTPAE const *PCPGMPTWALKGSTPAE;
+
+/**
+ * Guest page table walk for the 32-bit mode.
+ */
+typedef struct PGMPTWALKGST32BIT
+{
+    /** The common core. */
+    PGMPTWALKCORE   Core;
+
+    PX86PD          pPd;
+    PX86PDE         pPde;
+    X86PDE          Pde;
+
+    PX86PT          pPt;
+    PX86PTE         pPte;
+    X86PTE          Pte;
+} PGMPTWALKGST32BIT;
+/** Pointer to a 32-bit guest page table walk. */
+typedef PGMPTWALKGST32BIT *PPGMPTWALKGST32BIT;
+/** Pointer to a const 32-bit guest page table walk. */
+typedef PGMPTWALKGST32BIT const *PCPGMPTWALKGST32BIT;
+
+
+/** @name Paging mode macros
+ * @{
+ */
+#ifdef IN_RC
+# define PGM_CTX(a,b)                   a##RC##b
+# define PGM_CTX_STR(a,b)               a "GC" b
+# define PGM_CTX_DECL(type)             VMMRCDECL(type)
+#else
+# ifdef IN_RING3
+#  define PGM_CTX(a,b)                   a##R3##b
+#  define PGM_CTX_STR(a,b)               a "R3" b
+#  define PGM_CTX_DECL(type)             DECLCALLBACK(type)
+# else
+#  define PGM_CTX(a,b)                   a##R0##b
+#  define PGM_CTX_STR(a,b)               a "R0" b
+#  define PGM_CTX_DECL(type)             VMMDECL(type)
+# endif
+#endif
+
+#define PGM_GST_NAME_REAL(name)         PGM_CTX(pgm,GstReal##name)
+#define PGM_GST_NAME_RC_REAL_STR(name)  "pgmRCGstReal" #name
+#define PGM_GST_NAME_R0_REAL_STR(name)  "pgmR0GstReal" #name
+#define PGM_GST_NAME_PROT(name)         PGM_CTX(pgm,GstProt##name)
+#define PGM_GST_NAME_RC_PROT_STR(name)  "pgmRCGstProt" #name
+#define PGM_GST_NAME_R0_PROT_STR(name)  "pgmR0GstProt" #name
+#define PGM_GST_NAME_32BIT(name)        PGM_CTX(pgm,Gst32Bit##name)
+#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
+#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
+#define PGM_GST_NAME_PAE(name)          PGM_CTX(pgm,GstPAE##name)
+#define PGM_GST_NAME_RC_PAE_STR(name)   "pgmRCGstPAE" #name
+#define PGM_GST_NAME_R0_PAE_STR(name)   "pgmR0GstPAE" #name
+#define PGM_GST_NAME_AMD64(name)        PGM_CTX(pgm,GstAMD64##name)
+#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
+#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
+#define PGM_GST_PFN(name, pVCpu)        ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
+#define PGM_GST_DECL(type, name)        PGM_CTX_DECL(type) PGM_GST_NAME(name)
+
+#define PGM_SHW_NAME_32BIT(name)        PGM_CTX(pgm,Shw32Bit##name)
+#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
+#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
+#define PGM_SHW_NAME_PAE(name)          PGM_CTX(pgm,ShwPAE##name)
+#define PGM_SHW_NAME_RC_PAE_STR(name)   "pgmRCShwPAE" #name
+#define PGM_SHW_NAME_R0_PAE_STR(name)   "pgmR0ShwPAE" #name
+#define PGM_SHW_NAME_AMD64(name)        PGM_CTX(pgm,ShwAMD64##name)
+#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
+#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
+#define PGM_SHW_NAME_NESTED(name)        PGM_CTX(pgm,ShwNested##name)
+#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
+#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
+#define PGM_SHW_NAME_EPT(name)          PGM_CTX(pgm,ShwEPT##name)
+#define PGM_SHW_NAME_RC_EPT_STR(name)   "pgmRCShwEPT" #name
+#define PGM_SHW_NAME_R0_EPT_STR(name)   "pgmR0ShwEPT" #name
+#define PGM_SHW_DECL(type, name)        PGM_CTX_DECL(type) PGM_SHW_NAME(name)
+#define PGM_SHW_PFN(name, pVCpu)        ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
+
+/*                   Shw_Gst */
+#define PGM_BTH_NAME_32BIT_REAL(name)   PGM_CTX(pgm,Bth32BitReal##name)
+#define PGM_BTH_NAME_32BIT_PROT(name)   PGM_CTX(pgm,Bth32BitProt##name)
+#define PGM_BTH_NAME_32BIT_32BIT(name)  PGM_CTX(pgm,Bth32Bit32Bit##name)
+#define PGM_BTH_NAME_PAE_REAL(name)     PGM_CTX(pgm,BthPAEReal##name)
+#define PGM_BTH_NAME_PAE_PROT(name)     PGM_CTX(pgm,BthPAEProt##name)
+#define PGM_BTH_NAME_PAE_32BIT(name)    PGM_CTX(pgm,BthPAE32Bit##name)
+#define PGM_BTH_NAME_PAE_PAE(name)      PGM_CTX(pgm,BthPAEPAE##name)
+#define PGM_BTH_NAME_AMD64_PROT(name)   PGM_CTX(pgm,BthAMD64Prot##name)
+#define PGM_BTH_NAME_AMD64_AMD64(name)  PGM_CTX(pgm,BthAMD64AMD64##name)
+#define PGM_BTH_NAME_NESTED_REAL(name)  PGM_CTX(pgm,BthNestedReal##name)
+#define PGM_BTH_NAME_NESTED_PROT(name)  PGM_CTX(pgm,BthNestedProt##name)
+#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
+#define PGM_BTH_NAME_NESTED_PAE(name)   PGM_CTX(pgm,BthNestedPAE##name)
+#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
+#define PGM_BTH_NAME_EPT_REAL(name)     PGM_CTX(pgm,BthEPTReal##name)
+#define PGM_BTH_NAME_EPT_PROT(name)     PGM_CTX(pgm,BthEPTProt##name)
+#define PGM_BTH_NAME_EPT_32BIT(name)    PGM_CTX(pgm,BthEPT32Bit##name)
+#define PGM_BTH_NAME_EPT_PAE(name)      PGM_CTX(pgm,BthEPTPAE##name)
+#define PGM_BTH_NAME_EPT_AMD64(name)    PGM_CTX(pgm,BthEPTAMD64##name)
+
+#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name)    "pgmRCBth32BitReal" #name
+#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name)    "pgmRCBth32BitProt" #name
+#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name)   "pgmRCBth32Bit32Bit" #name
+#define PGM_BTH_NAME_RC_PAE_REAL_STR(name)      "pgmRCBthPAEReal" #name
+#define PGM_BTH_NAME_RC_PAE_PROT_STR(name)      "pgmRCBthPAEProt" #name
+#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name)     "pgmRCBthPAE32Bit" #name
+#define PGM_BTH_NAME_RC_PAE_PAE_STR(name)       "pgmRCBthPAEPAE" #name
+#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name)   "pgmRCBthAMD64AMD64" #name
+#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name)   "pgmRCBthNestedReal" #name
+#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name)   "pgmRCBthNestedProt" #name
+#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name)  "pgmRCBthNested32Bit" #name
+#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name)    "pgmRCBthNestedPAE" #name
+#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name)  "pgmRCBthNestedAMD64" #name
+#define PGM_BTH_NAME_RC_EPT_REAL_STR(name)      "pgmRCBthEPTReal" #name
+#define PGM_BTH_NAME_RC_EPT_PROT_STR(name)      "pgmRCBthEPTProt" #name
+#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name)     "pgmRCBthEPT32Bit" #name
+#define PGM_BTH_NAME_RC_EPT_PAE_STR(name)       "pgmRCBthEPTPAE" #name
+#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name)     "pgmRCBthEPTAMD64" #name
+#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name)    "pgmR0Bth32BitReal" #name
+#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name)    "pgmR0Bth32BitProt" #name
+#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name)   "pgmR0Bth32Bit32Bit" #name
+#define PGM_BTH_NAME_R0_PAE_REAL_STR(name)      "pgmR0BthPAEReal" #name
+#define PGM_BTH_NAME_R0_PAE_PROT_STR(name)      "pgmR0BthPAEProt" #name
+#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name)     "pgmR0BthPAE32Bit" #name
+#define PGM_BTH_NAME_R0_PAE_PAE_STR(name)       "pgmR0BthPAEPAE" #name
+#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name)    "pgmR0BthAMD64Prot" #name
+#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name)   "pgmR0BthAMD64AMD64" #name
+#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name)   "pgmR0BthNestedReal" #name
+#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name)   "pgmR0BthNestedProt" #name
+#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name)  "pgmR0BthNested32Bit" #name
+#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name)    "pgmR0BthNestedPAE" #name
+#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name)  "pgmR0BthNestedAMD64" #name
+#define PGM_BTH_NAME_R0_EPT_REAL_STR(name)      "pgmR0BthEPTReal" #name
+#define PGM_BTH_NAME_R0_EPT_PROT_STR(name)      "pgmR0BthEPTProt" #name
+#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name)     "pgmR0BthEPT32Bit" #name
+#define PGM_BTH_NAME_R0_EPT_PAE_STR(name)       "pgmR0BthEPTPAE" #name
+#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name)     "pgmR0BthEPTAMD64" #name
+
+#define PGM_BTH_DECL(type, name)        PGM_CTX_DECL(type) PGM_BTH_NAME(name)
+#define PGM_BTH_PFN(name, pVCpu)        ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
+/** @} */
+
+/**
+ * Data for each paging mode.
+ */
+typedef struct PGMMODEDATA
+{
+    /** The guest mode type. */
+    uint32_t                        uGstType;
+    /** The shadow mode type. */
+    uint32_t                        uShwType;
+
+    /** @name Function pointers for Shadow paging.
+     * @{
+     */
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwExit,(PVMCPU pVCpu));
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
+
+    DECLRCCALLBACKMEMBER(int,       pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
+    DECLRCCALLBACKMEMBER(int,       pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
+
+    DECLR0CALLBACKMEMBER(int,       pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
+    DECLR0CALLBACKMEMBER(int,       pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
+    /** @} */
+
+    /** @name Function pointers for Guest paging.
+     * @{
+     */
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstExit,(PVMCPU pVCpu));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
+    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
+    DECLRCCALLBACKMEMBER(int,       pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
+    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
+    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
+    DECLR0CALLBACKMEMBER(int,       pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
+    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
+    /** @} */
+
+    /** @name Function pointers for Both Shadow and Guest paging.
+     * @{
+     */
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
+    /*                           no pfnR3BthTrap0eHandler */
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
+#ifdef VBOX_STRICT
+    DECLR3CALLBACKMEMBER(unsigned,  pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
+#endif
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthUnmapCR3,(PVMCPU pVCpu));
+
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
+#ifdef VBOX_STRICT
+    DECLRCCALLBACKMEMBER(unsigned,  pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
+#endif
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthUnmapCR3,(PVMCPU pVCpu));
+
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
+#ifdef VBOX_STRICT
+    DECLR0CALLBACKMEMBER(unsigned,  pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
+#endif
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthUnmapCR3,(PVMCPU pVCpu));
+    /** @} */
+} PGMMODEDATA, *PPGMMODEDATA;
+
+
+#ifdef VBOX_WITH_STATISTICS
+/**
+ * PGM statistics.
+ *
+ * These lives on the heap when compiled in as they would otherwise waste
+ * unnecessary space in release builds.
+ */
+typedef struct PGMSTATS
+{
+    /* R3 only: */
+    STAMCOUNTER StatR3DetectedConflicts;            /**< R3: Number of times PGMR3MapHasConflicts() detected a conflict. */
+    STAMPROFILE StatR3ResolveConflict;              /**< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). */
+
+    /* R3+RZ */
+    STAMCOUNTER StatRZChunkR3MapTlbHits;            /**< RC/R0: Ring-3/0 chunk mapper TLB hits. */
+    STAMCOUNTER StatRZChunkR3MapTlbMisses;          /**< RC/R0: Ring-3/0 chunk mapper TLB misses. */
+    STAMCOUNTER StatRZPageMapTlbHits;               /**< RC/R0: Ring-3/0 page mapper TLB hits. */
+    STAMCOUNTER StatRZPageMapTlbMisses;             /**< RC/R0: Ring-3/0 page mapper TLB misses. */
+    STAMCOUNTER StatPageMapTlbFlushes;              /**< ALL: Ring-3/0 page mapper TLB flushes. */
+    STAMCOUNTER StatPageMapTlbFlushEntry;           /**< ALL: Ring-3/0 page mapper TLB flushes. */
+    STAMCOUNTER StatR3ChunkR3MapTlbHits;            /**< R3: Ring-3/0 chunk mapper TLB hits. */
+    STAMCOUNTER StatR3ChunkR3MapTlbMisses;          /**< R3: Ring-3/0 chunk mapper TLB misses. */
+    STAMCOUNTER StatR3PageMapTlbHits;               /**< R3: Ring-3/0 page mapper TLB hits. */
+    STAMCOUNTER StatR3PageMapTlbMisses;             /**< R3: Ring-3/0 page mapper TLB misses. */
+    STAMPROFILE StatRZSyncCR3HandlerVirtualReset;   /**< RC/R0: Profiling of the virtual handler resets. */
+    STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate;  /**< RC/R0: Profiling of the virtual handler updates. */
+    STAMPROFILE StatR3SyncCR3HandlerVirtualReset;   /**< R3: Profiling of the virtual handler resets. */
+    STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate;  /**< R3: Profiling of the virtual handler updates. */
+    STAMCOUNTER StatR3PhysHandlerReset;             /**< R3: The number of times PGMHandlerPhysicalReset is called. */
+    STAMCOUNTER StatRZPhysHandlerReset;             /**< RC/R0: The number of times PGMHandlerPhysicalReset is called. */
+    STAMCOUNTER StatR3PhysHandlerLookupHits;        /**< R3: Number of cache hits when looking up physical handlers. */
+    STAMCOUNTER StatR3PhysHandlerLookupMisses;      /**< R3: Number of cache misses when looking up physical handlers. */
+    STAMCOUNTER StatRZPhysHandlerLookupHits;        /**< RC/R0: Number of cache hits when lookup up physical handlers. */
+    STAMCOUNTER StatRZPhysHandlerLookupMisses;      /**< RC/R0: Number of cache misses when looking up physical handlers */
+    STAMPROFILE StatRZVirtHandlerSearchByPhys;      /**< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. */
+    STAMPROFILE StatR3VirtHandlerSearchByPhys;      /**< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. */
+    STAMCOUNTER StatRZPageReplaceShared;            /**< RC/R0: Times a shared page has been replaced by a private one. */
+    STAMCOUNTER StatRZPageReplaceZero;              /**< RC/R0: Times the zero page has been replaced by a private one. */
+/// @todo    STAMCOUNTER StatRZPageHandyAllocs;              /**< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. */
+    STAMCOUNTER StatR3PageReplaceShared;            /**< R3: Times a shared page has been replaced by a private one. */
+    STAMCOUNTER StatR3PageReplaceZero;              /**< R3: Times the zero page has been replaced by a private one. */
+/// @todo    STAMCOUNTER StatR3PageHandyAllocs;              /**< R3: The number of times we've executed GMMR3AllocateHandyPages. */
+
+    /* RC only: */
+    STAMCOUNTER StatRCInvlPgConflict;               /**< RC: Number of times PGMInvalidatePage() detected a mapping conflict. */
+    STAMCOUNTER StatRCInvlPgSyncMonCR3;             /**< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. */
+
+    STAMCOUNTER StatRZPhysRead;
+    STAMCOUNTER StatRZPhysReadBytes;
+    STAMCOUNTER StatRZPhysWrite;
+    STAMCOUNTER StatRZPhysWriteBytes;
+    STAMCOUNTER StatR3PhysRead;
+    STAMCOUNTER StatR3PhysReadBytes;
+    STAMCOUNTER StatR3PhysWrite;
+    STAMCOUNTER StatR3PhysWriteBytes;
+    STAMCOUNTER StatRCPhysRead;
+    STAMCOUNTER StatRCPhysReadBytes;
+    STAMCOUNTER StatRCPhysWrite;
+    STAMCOUNTER StatRCPhysWriteBytes;
+
+    STAMCOUNTER StatRZPhysSimpleRead;
+    STAMCOUNTER StatRZPhysSimpleReadBytes;
+    STAMCOUNTER StatRZPhysSimpleWrite;
+    STAMCOUNTER StatRZPhysSimpleWriteBytes;
+    STAMCOUNTER StatR3PhysSimpleRead;
+    STAMCOUNTER StatR3PhysSimpleReadBytes;
+    STAMCOUNTER StatR3PhysSimpleWrite;
+    STAMCOUNTER StatR3PhysSimpleWriteBytes;
+    STAMCOUNTER StatRCPhysSimpleRead;
+    STAMCOUNTER StatRCPhysSimpleReadBytes;
+    STAMCOUNTER StatRCPhysSimpleWrite;
+    STAMCOUNTER StatRCPhysSimpleWriteBytes;
+
+    STAMCOUNTER StatTrackVirgin;                    /**< The number of first time shadowings. */
+    STAMCOUNTER StatTrackAliased;                   /**< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. */
+    STAMCOUNTER StatTrackAliasedMany;               /**< The number of times we're tracking using cRef2. */
+    STAMCOUNTER StatTrackAliasedLots;               /**< The number of times we're hitting pages which has overflowed cRef2. */
+    STAMCOUNTER StatTrackNoExtentsLeft;             /**< The number of times the extent list was exhausted. */
+    STAMCOUNTER StatTrackOverflows;                 /**< The number of times the extent list grows to long. */
+    STAMPROFILE StatTrackDeref;                     /**< Profiling of SyncPageWorkerTrackDeref (expensive). */
+
+    /** Time spent by the host OS for large page allocation. */
+    STAMPROFILE                 StatAllocLargePage;
+    /** Time spent clearing the newly allocated large pages. */
+    STAMPROFILE                 StatClearLargePage;
+    /** The number of times allocating a large pages takes more than the allowed period. */
+    STAMCOUNTER                 StatLargePageOverflow;
+    /** pgmPhysIsValidLargePage profiling - R3 */
+    STAMPROFILE                 StatR3IsValidLargePage;
+    /** pgmPhysIsValidLargePage profiling - RZ*/
+    STAMPROFILE                 StatRZIsValidLargePage;
+
+    STAMPROFILE                 StatChunkAging;
+    STAMPROFILE                 StatChunkFindCandidate;
+    STAMPROFILE                 StatChunkUnmap;
+    STAMPROFILE                 StatChunkMap;
+} PGMSTATS;
+#endif /* VBOX_WITH_STATISTICS */
+
+
+/**
+ * Converts a PGM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the PGM is part of.
+ * @param   pPGM   Pointer to PGM instance data.
+ */
+#define PGM2VM(pPGM)  ( (PVM)((char*)pPGM - pPGM->offVM) )
+
+/**
+ * PGM Data (part of VM)
+ */
+typedef struct PGM
+{
+    /** Offset to the VM structure. */
+    int32_t                         offVM;
+    /** Offset of the PGMCPU structure relative to VMCPU. */
+    int32_t                         offVCpuPGM;
+
+    /** @cfgm{RamPreAlloc, boolean, false}
+     * Indicates whether the base RAM should all be allocated before starting
+     * the VM (default), or if it should be allocated when first written to.
+     */
+    bool                            fRamPreAlloc;
+    /** Indicates whether write monitoring is currently in use.
+     * This is used to prevent conflicts between live saving and page sharing
+     * detection. */
+    bool                            fPhysWriteMonitoringEngaged;
+    /** Set if the CPU has less than 52-bit physical address width.
+     * This is used  */
+    bool                            fLessThan52PhysicalAddressBits;
+    /** Set when nested paging is active.
+     * This is meant to save calls to HWACCMIsNestedPagingActive and let the
+     * compilers optimize the code better.  Whether we use nested paging or
+     * not is something we find out during VMM initialization and we won't
+     * change this later on. */
+    bool                            fNestedPaging;
+    /** The host paging mode. (This is what SUPLib reports.) */
+    SUPPAGINGMODE                   enmHostMode;
+    /** We're not in a state which permits writes to guest memory.
+     * (Only used in strict builds.) */
+    bool                            fNoMorePhysWrites;
+    /** Alignment padding that makes the next member start on a 8 byte boundary. */
+    bool                            afAlignment1[3];
+
+    /** Indicates that PGMR3FinalizeMappings has been called and that further
+     * PGMR3MapIntermediate calls will be rejected. */
+    bool                            fFinalizedMappings;
+    /** If set no conflict checks are required. */
+    bool                            fMappingsFixed;
+    /** If set if restored as fixed but we were unable to re-fixate at the old
+     *  location because of room or address incompatibilities. */
+    bool                            fMappingsFixedRestored;
+    /** If set, then no mappings are put into the shadow page table.
+     * Use pgmMapAreMappingsEnabled() instead of direct access. */
+    bool                            fMappingsDisabled;
+    /** Size of fixed mapping.
+     * This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
+    uint32_t                        cbMappingFixed;
+    /** Generation ID for the RAM ranges. This member is incremented everytime
+     * a RAM range is linked or unlinked. */
+    uint32_t volatile               idRamRangesGen;
+
+    /** Base address (GC) of fixed mapping.
+     * This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
+    RTGCPTR                         GCPtrMappingFixed;
+    /** The address of the previous RAM range mapping. */
+    RTGCPTR                         GCPtrPrevRamRangeMapping;
+
+    /** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
+    RTGCPHYS                        GCPhys4MBPSEMask;
+    /** Mask containing the invalid bits of a guest physical address.
+     * @remarks this does not stop at bit 52.  */
+    RTGCPHYS                        GCPhysInvAddrMask;
+
+
+    /** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
+     * This is sorted by physical address and contains no overlapping ranges. */
+    R3PTRTYPE(PPGMRAMRANGE)         pRamRangesR3;
+    /** PGM offset based trees - R3 Ptr. */
+    R3PTRTYPE(PPGMTREES)            pTreesR3;
+    /** Caching the last physical handler we looked up in R3. */
+    R3PTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerR3;
+    /** Shadow Page Pool - R3 Ptr. */
+    R3PTRTYPE(PPGMPOOL)             pPoolR3;
+    /** Linked list of GC mappings - for HC.
+     * The list is sorted ascending on address. */
+    R3PTRTYPE(PPGMMAPPING)          pMappingsR3;
+    /** Pointer to the list of ROM ranges - for R3.
+     * This is sorted by physical address and contains no overlapping ranges. */
+    R3PTRTYPE(PPGMROMRANGE)         pRomRangesR3;
+    /** Pointer to the list of MMIO2 ranges - for R3.
+     * Registration order. */
+    R3PTRTYPE(PPGMMMIO2RANGE)       pMmio2RangesR3;
+    /** Pointer to SHW+GST mode data (function pointers).
+     * The index into this table is made up from */
+    R3PTRTYPE(PPGMMODEDATA)         paModeData;
+    /*RTR3PTR                         R3PtrAlignment0;*/
+
+
+    /** R0 pointer corresponding to PGM::pRamRangesR3. */
+    R0PTRTYPE(PPGMRAMRANGE)         pRamRangesR0;
+    /** PGM offset based trees - R0 Ptr. */
+    R0PTRTYPE(PPGMTREES)            pTreesR0;
+    /** Caching the last physical handler we looked up in R0. */
+    R0PTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerR0;
+    /** Shadow Page Pool - R0 Ptr. */
+    R0PTRTYPE(PPGMPOOL)             pPoolR0;
+    /** Linked list of GC mappings - for R0.
+     * The list is sorted ascending on address. */
+    R0PTRTYPE(PPGMMAPPING)          pMappingsR0;
+    /** R0 pointer corresponding to PGM::pRomRangesR3. */
+    R0PTRTYPE(PPGMROMRANGE)         pRomRangesR0;
+    /*RTR0PTR                         R0PtrAlignment0;*/
+
+
+    /** RC pointer corresponding to PGM::pRamRangesR3. */
+    RCPTRTYPE(PPGMRAMRANGE)         pRamRangesRC;
+    /** PGM offset based trees - RC Ptr. */
+    RCPTRTYPE(PPGMTREES)            pTreesRC;
+    /** Caching the last physical handler we looked up in RC. */
+    RCPTRTYPE(PPGMPHYSHANDLER)      pLastPhysHandlerRC;
+    /** Shadow Page Pool - RC Ptr. */
+    RCPTRTYPE(PPGMPOOL)             pPoolRC;
+    /** Linked list of GC mappings - for RC.
+     * The list is sorted ascending on address. */
+    RCPTRTYPE(PPGMMAPPING)          pMappingsRC;
+    /** RC pointer corresponding to PGM::pRomRangesR3. */
+    RCPTRTYPE(PPGMROMRANGE)         pRomRangesRC;
+    /*RTRCPTR                         RCPtrAlignment0;*/
+    /** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
+    RCPTRTYPE(PX86PTE)              paDynPageMap32BitPTEsGC;
+    /** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
+    RCPTRTYPE(PPGMSHWPTEPAE)        paDynPageMapPaePTEsGC;
+
+
+    /** Pointer to the 5 page CR3 content mapping.
+     * The first page is always the CR3 (in some form) while the 4 other pages
+     * are used of the PDs in PAE mode. */
+    RTGCPTR                         GCPtrCR3Mapping;
+
+    /** @name Intermediate Context
+     * @{ */
+    /** Pointer to the intermediate page directory - Normal. */
+    R3PTRTYPE(PX86PD)               pInterPD;
+    /** Pointer to the intermediate page tables - Normal.
+     * There are two page tables, one for the identity mapping and one for
+     * the host context mapping (of the core code). */
+    R3PTRTYPE(PX86PT)               apInterPTs[2];
+    /** Pointer to the intermediate page tables - PAE. */
+    R3PTRTYPE(PX86PTPAE)            apInterPaePTs[2];
+    /** Pointer to the intermediate page directory - PAE. */
+    R3PTRTYPE(PX86PDPAE)            apInterPaePDs[4];
+    /** Pointer to the intermediate page directory - PAE. */
+    R3PTRTYPE(PX86PDPT)             pInterPaePDPT;
+    /** Pointer to the intermediate page-map level 4 - AMD64. */
+    R3PTRTYPE(PX86PML4)             pInterPaePML4;
+    /** Pointer to the intermediate page directory - AMD64. */
+    R3PTRTYPE(PX86PDPT)             pInterPaePDPT64;
+    /** The Physical Address (HC) of the intermediate Page Directory - Normal. */
+    RTHCPHYS                        HCPhysInterPD;
+    /** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
+    RTHCPHYS                        HCPhysInterPaePDPT;
+    /** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
+    RTHCPHYS                        HCPhysInterPaePML4;
+    /** @} */
+
+    /** Base address of the dynamic page mapping area.
+     * The array is MM_HYPER_DYNAMIC_SIZE bytes big.
+     *
+     * @todo The plan of keeping PGMRCDYNMAP private to PGMRZDynMap.cpp didn't
+     *       work out.  Some cleaning up of the initialization that would
+     *       remove this memory is yet to be done...
+     */
+    RCPTRTYPE(uint8_t *)            pbDynPageMapBaseGC;
+    /** The address of the raw-mode context mapping cache. */
+    RCPTRTYPE(PPGMRCDYNMAP)         pRCDynMap;
+    /** The address of the ring-0 mapping cache if we're making use of it.  */
+    RTR0PTR                         pvR0DynMapUsed;
+#if HC_ARCH_BITS == 32
+    /** Alignment padding that makes the next member start on a 8 byte boundary. */
+    uint32_t                        u32Alignment2;
+#endif
+
+    /** PGM critical section.
+     * This protects the physical & virtual access handlers, ram ranges,
+     * and the page flag updating (some of it anyway).
+     */
+    PDMCRITSECT                     CritSect;
+
+    /**
+     * Data associated with managing the ring-3 mappings of the allocation chunks.
+     */
+    struct
+    {
+        /** The chunk tree, ordered by chunk id. */
+#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
+        R3PTRTYPE(PAVLU32NODECORE)  pTree;
+#else
+        R3R0PTRTYPE(PAVLU32NODECORE) pTree;
+#endif
+#if HC_ARCH_BITS == 32
+        uint32_t                    u32Alignment;
+#endif
+        /** The chunk mapping TLB. */
+        PGMCHUNKR3MAPTLB            Tlb;
+        /** The number of mapped chunks. */
+        uint32_t                    c;
+        /** The maximum number of mapped chunks.
+         * @cfgm    PGM/MaxRing3Chunks */
+        uint32_t                    cMax;
+        /** The current time. */
+        uint32_t                    iNow;
+        /** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
+        uint32_t                    AgeingCountdown;
+    } ChunkR3Map;
+
+    /**
+     * The page mapping TLB for ring-3 and (for the time being) ring-0.
+     */
+    PGMPAGER3MAPTLB                 PhysTlbHC;
+
+    /** @name   The zero page.
+     * @{ */
+    /** The host physical address of the zero page. */
+    RTHCPHYS                        HCPhysZeroPg;
+    /** The ring-3 mapping of the zero page. */
+    RTR3PTR                         pvZeroPgR3;
+    /** The ring-0 mapping of the zero page. */
+    RTR0PTR                         pvZeroPgR0;
+    /** The GC mapping of the zero page. */
+    RTRCPTR                         pvZeroPgRC;
+    RTRCPTR                         RCPtrAlignment3;
+    /** @}*/
+
+    /** @name   The Invalid MMIO page.
+     * This page is filled with 0xfeedface.
+     * @{ */
+    /** The host physical address of the invalid MMIO page. */
+    RTHCPHYS                        HCPhysMmioPg;
+    /** The host pysical address of the invalid MMIO page plus all invalid
+     * physical address bits set.  This is used to trigger X86_TRAP_PF_RSVD.
+     * @remarks Check fLessThan52PhysicalAddressBits before use. */
+    RTHCPHYS                        HCPhysInvMmioPg;
+    /** The ring-3 mapping of the invalid MMIO page. */
+    RTR3PTR                         pvMmioPgR3;
+#if HC_ARCH_BITS == 32
+    RTR3PTR                         R3PtrAlignment4;
+#endif
+    /** @} */
+
+
+    /** The number of handy pages. */
+    uint32_t                        cHandyPages;
+
+    /** The number of large handy pages. */
+    uint32_t                        cLargeHandyPages;
+
+    /**
+     * Array of handy pages.
+     *
+     * This array is used in a two way communication between pgmPhysAllocPage
+     * and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
+     * an intermediary.
+     *
+     * The size of this array is important, see pgmPhysEnsureHandyPage for details.
+     * (The current size of 32 pages, means 128 KB of handy memory.)
+     */
+    GMMPAGEDESC                     aHandyPages[PGM_HANDY_PAGES];
+
+    /**
+     * Array of large handy pages. (currently size 1)
+     *
+     * This array is used in a two way communication between pgmPhysAllocLargePage
+     * and GMMR0AllocateLargePage, with PGMR3PhysAllocateLargePage serving as
+     * an intermediary.
+     */
+    GMMPAGEDESC                     aLargeHandyPage[1];
+
+    /**
+     * Live save data.
+     */
+    struct
+    {
+        /** Per type statistics. */
+        struct
+        {
+            /** The number of ready pages.  */
+            uint32_t                cReadyPages;
+            /** The number of dirty pages. */
+            uint32_t                cDirtyPages;
+            /** The number of ready zero pages.  */
+            uint32_t                cZeroPages;
+            /** The number of write monitored pages. */
+            uint32_t                cMonitoredPages;
+        }                           Rom,
+                                    Mmio2,
+                                    Ram;
+        /** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */
+        uint32_t                    cIgnoredPages;
+        /** Indicates that a live save operation is active.  */
+        bool                        fActive;
+        /** Padding. */
+        bool                        afReserved[2];
+        /** The next history index. */
+        uint8_t                     iDirtyPagesHistory;
+        /** History of the total amount of dirty pages. */
+        uint32_t                    acDirtyPagesHistory[64];
+        /** Short term dirty page average. */
+        uint32_t                    cDirtyPagesShort;
+        /** Long term dirty page average. */
+        uint32_t                    cDirtyPagesLong;
+        /** The number of saved pages.  This is used to get some kind of estimate of the
+         * link speed so we can decide when we're done.  It is reset after the first
+         * 7 passes so the speed estimate doesn't get inflated by the initial set of
+         * zero pages.   */
+        uint64_t                    cSavedPages;
+        /** The nanosecond timestamp when cSavedPages was 0. */
+        uint64_t                    uSaveStartNS;
+        /** Pages per second (for statistics). */
+        uint32_t                    cPagesPerSecond;
+        uint32_t                    cAlignment;
+    } LiveSave;
+
+    /** @name   Error injection.
+     * @{ */
+    /** Inject handy page allocation errors pretending we're completely out of
+     * memory. */
+    bool volatile                   fErrInjHandyPages;
+    /** Padding. */
+    bool                            afReserved[3];
+    /** @} */
+
+    /** @name Release Statistics
+     * @{ */
+    uint32_t                        cAllPages;          /**< The total number of pages. (Should be Private + Shared + Zero + Pure MMIO.) */
+    uint32_t                        cPrivatePages;      /**< The number of private pages. */
+    uint32_t                        cSharedPages;       /**< The number of shared pages. */
+    uint32_t                        cReusedSharedPages; /**< The number of reused shared pages. */
+    uint32_t                        cZeroPages;         /**< The number of zero backed pages. */
+    uint32_t                        cPureMmioPages;     /**< The number of pure MMIO pages. */
+    uint32_t                        cMonitoredPages;    /**< The number of write monitored pages. */
+    uint32_t                        cWrittenToPages;    /**< The number of previously write monitored pages. */
+    uint32_t                        cWriteLockedPages;  /**< The number of write locked pages. */
+    uint32_t                        cReadLockedPages;   /**< The number of read locked pages. */
+    uint32_t                        cBalloonedPages;    /**< The number of ballooned pages. */
+    uint32_t                        cMappedChunks;      /**< Number of times we mapped a chunk. */
+    uint32_t                        cUnmappedChunks;    /**< Number of times we unmapped a chunk. */
+/*    uint32_t                        aAlignment4[1]; */
+
+    /** The number of times we were forced to change the hypervisor region location. */
+    STAMCOUNTER                     cRelocations;
+
+    STAMCOUNTER                     StatLargePageAlloc;                 /**< The number of large pages we've allocated.*/
+    STAMCOUNTER                     StatLargePageReused;                /**< The number of large pages we've reused.*/
+    STAMCOUNTER                     StatLargePageRefused;               /**< The number of times we couldn't use a large page.*/
+    STAMCOUNTER                     StatLargePageRecheck;               /**< The number of times we rechecked a disabled large page.*/
+    /** @} */
+
+#ifdef VBOX_WITH_STATISTICS
+    /** @name Statistics on the heap.
+     * @{ */
+    R3PTRTYPE(PGMSTATS *)           pStatsR3;
+    R0PTRTYPE(PGMSTATS *)           pStatsR0;
+    RCPTRTYPE(PGMSTATS *)           pStatsRC;
+    RTRCPTR                         RCPtrAlignment;
+    /** @}  */
+#endif
+} PGM;
+#ifndef IN_TSTVMSTRUCTGC /* HACK */
+AssertCompileMemberAlignment(PGM, paDynPageMap32BitPTEsGC, 8);
+AssertCompileMemberAlignment(PGM, GCPtrMappingFixed, sizeof(RTGCPTR));
+AssertCompileMemberAlignment(PGM, HCPhysInterPD, 8);
+AssertCompileMemberAlignment(PGM, CritSect, 8);
+AssertCompileMemberAlignment(PGM, ChunkR3Map, 8);
+AssertCompileMemberAlignment(PGM, PhysTlbHC, 8);
+AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8);
+AssertCompileMemberAlignment(PGM, aHandyPages, 8);
+AssertCompileMemberAlignment(PGM, cRelocations, 8);
+#endif /* !IN_TSTVMSTRUCTGC */
+/** Pointer to the PGM instance data. */
+typedef PGM *PPGM;
+
+
+
+typedef struct PGMCPUSTATS
+{
+    /* Common */
+    STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES];       /**< SyncPT - PD distribution. */
+    STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES];     /**< SyncPage - PD distribution. */
+
+    /* R0 only: */
+    STAMPROFILE StatR0NpMiscfg;                     /**< R0: PGMR0Trap0eHandlerNPMisconfig() profiling. */
+    STAMCOUNTER StatR0NpMiscfgSyncPage;             /**< R0: SyncPage calls from PGMR0Trap0eHandlerNPMisconfig(). */
+
+    /* RZ only: */
+    STAMPROFILE StatRZTrap0e;                       /**< RC/R0: PGMTrap0eHandler() profiling. */
+    STAMPROFILE StatRZTrap0eTime2Ballooned;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is read access to a ballooned page. */
+    STAMPROFILE StatRZTrap0eTime2CSAM;              /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. */
+    STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. */
+    STAMPROFILE StatRZTrap0eTime2GuestTrap;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. */
+    STAMPROFILE StatRZTrap0eTime2HndPhys;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. */
+    STAMPROFILE StatRZTrap0eTime2HndVirt;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. */
+    STAMPROFILE StatRZTrap0eTime2HndUnhandled;      /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. */
+    STAMPROFILE StatRZTrap0eTime2InvalidPhys;       /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address. */
+    STAMPROFILE StatRZTrap0eTime2MakeWritable;      /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a page that needed to be made writable. */
+    STAMPROFILE StatRZTrap0eTime2Mapping;           /**< RC/R0: Profiling of the Trap0eHandler body when the cause is the guest mappings. */
+    STAMPROFILE StatRZTrap0eTime2Misc;              /**< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. */
+    STAMPROFILE StatRZTrap0eTime2OutOfSync;         /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. */
+    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. */
+    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt;  /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. */
+    STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs;   /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. */
+    STAMPROFILE StatRZTrap0eTime2SyncPT;            /**< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. */
+    STAMPROFILE StatRZTrap0eTime2WPEmulation;       /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP emulation. */
+    STAMCOUNTER StatRZTrap0eConflicts;              /**< RC/R0: The number of times \#PF was caused by an undetected conflict. */
+    STAMCOUNTER StatRZTrap0eHandlersMapping;        /**< RC/R0: Number of traps due to access handlers in mappings. */
+    STAMCOUNTER StatRZTrap0eHandlersOutOfSync;      /**< RC/R0: Number of out-of-sync handled pages. */
+    STAMCOUNTER StatRZTrap0eHandlersPhysAll;        /**< RC/R0: Number of traps due to physical all-access handlers. */
+    STAMCOUNTER StatRZTrap0eHandlersPhysAllOpt;     /**< RC/R0: Number of the physical all-access handler traps using the optimization. */
+    STAMCOUNTER StatRZTrap0eHandlersPhysWrite;      /**< RC/R0: Number of traps due to write-physical access handlers. */
+    STAMCOUNTER StatRZTrap0eHandlersVirtual;        /**< RC/R0: Number of traps due to virtual access handlers. */
+    STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys;  /**< RC/R0: Number of traps due to virtual access handlers found by physical address. */
+    STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/**< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). */
+    STAMCOUNTER StatRZTrap0eHandlersUnhandled;      /**< RC/R0: Number of traps due to access outside range of monitored page(s). */
+    STAMCOUNTER StatRZTrap0eHandlersInvalid;        /**< RC/R0: Number of traps due to access to invalid physical memory. */
+    STAMCOUNTER StatRZTrap0eUSNotPresentRead;       /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSNotPresentWrite;      /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSWrite;                /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSReserved;             /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSNXE;                  /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eUSRead;                 /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVNotPresentRead;       /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVNotPresentWrite;      /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVWrite;                /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSVReserved;             /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eSNXE;                   /**< RC/R0: \#PF err kind */
+    STAMCOUNTER StatRZTrap0eGuestPF;                /**< RC/R0: Real guest \#PFs. */
+    STAMCOUNTER StatRZTrap0eGuestPFMapping;         /**< RC/R0: Real guest \#PF to HMA or other mapping. */
+    STAMCOUNTER StatRZTrap0eWPEmulInRZ;             /**< RC/R0: WP=0 virtualization trap, handled. */
+    STAMCOUNTER StatRZTrap0eWPEmulToR3;             /**< RC/R0: WP=0 virtualization trap, chickened out. */
+    STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES];     /**< RC/R0: PD distribution of the \#PFs. */
+    STAMCOUNTER StatRZGuestCR3WriteHandled;         /**< RC/R0: The number of times WriteHandlerCR3() was successfully called. */
+    STAMCOUNTER StatRZGuestCR3WriteUnhandled;       /**< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. */
+    STAMCOUNTER StatRZGuestCR3WriteConflict;        /**< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. */
+    STAMCOUNTER StatRZGuestROMWriteHandled;         /**< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. */
+    STAMCOUNTER StatRZGuestROMWriteUnhandled;       /**< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler */
+    STAMCOUNTER StatRZDynMapMigrateInvlPg;          /**< RZ: invlpg in PGMR0DynMapMigrateAutoSet. */
+    STAMPROFILE StatRZDynMapGCPageInl;              /**< RZ: Calls to pgmRZDynMapGCPageInlined. */
+    STAMCOUNTER StatRZDynMapGCPageInlHits;          /**< RZ: Hash table lookup hits. */
+    STAMCOUNTER StatRZDynMapGCPageInlMisses;        /**< RZ: Misses that falls back to the code common. */
+    STAMCOUNTER StatRZDynMapGCPageInlRamHits;       /**< RZ: 1st ram range hits. */
+    STAMCOUNTER StatRZDynMapGCPageInlRamMisses;     /**< RZ: 1st ram range misses, takes slow path. */
+    STAMPROFILE StatRZDynMapHCPageInl;              /**< RZ: Calls to pgmRZDynMapHCPageInlined. */
+    STAMCOUNTER StatRZDynMapHCPageInlHits;          /**< RZ: Hash table lookup hits. */
+    STAMCOUNTER StatRZDynMapHCPageInlMisses;        /**< RZ: Misses that falls back to the code common. */
+    STAMPROFILE StatRZDynMapHCPage;                 /**< RZ: Calls to pgmRZDynMapHCPageCommon. */
+    STAMCOUNTER StatRZDynMapSetOptimize;            /**< RZ: Calls to pgmRZDynMapOptimizeAutoSet. */
+    STAMCOUNTER StatRZDynMapSetSearchFlushes;       /**< RZ: Set search restoring to subset flushes. */
+    STAMCOUNTER StatRZDynMapSetSearchHits;          /**< RZ: Set search hits. */
+    STAMCOUNTER StatRZDynMapSetSearchMisses;        /**< RZ: Set search misses. */
+    STAMCOUNTER StatRZDynMapPage;                   /**< RZ: Calls to pgmR0DynMapPage. */
+    STAMCOUNTER StatRZDynMapPageHits0;              /**< RZ: Hits at iPage+0. */
+    STAMCOUNTER StatRZDynMapPageHits1;              /**< RZ: Hits at iPage+1. */
+    STAMCOUNTER StatRZDynMapPageHits2;              /**< RZ: Hits at iPage+2. */
+    STAMCOUNTER StatRZDynMapPageInvlPg;             /**< RZ: invlpg. */
+    STAMCOUNTER StatRZDynMapPageSlow;               /**< RZ: Calls to pgmR0DynMapPageSlow. */
+    STAMCOUNTER StatRZDynMapPageSlowLoopHits;       /**< RZ: Hits in the pgmR0DynMapPageSlow search loop. */
+    STAMCOUNTER StatRZDynMapPageSlowLoopMisses;     /**< RZ: Misses in the pgmR0DynMapPageSlow search loop. */
+    //STAMCOUNTER StatRZDynMapPageSlowLostHits;       /**< RZ: Lost hits. */
+    STAMCOUNTER StatRZDynMapSubsets;                /**< RZ: Times PGMDynMapPushAutoSubset was called. */
+    STAMCOUNTER StatRZDynMapPopFlushes;             /**< RZ: Times PGMDynMapPopAutoSubset flushes the subset. */
+    STAMCOUNTER aStatRZDynMapSetFilledPct[11];      /**< RZ: Set fill distribution, percent. */
+
+    /* HC - R3 and (maybe) R0: */
+
+    /* RZ & R3: */
+    STAMPROFILE StatRZSyncCR3;                      /**< RC/R0: PGMSyncCR3() profiling. */
+    STAMPROFILE StatRZSyncCR3Handlers;              /**< RC/R0: Profiling of the PGMSyncCR3() update handler section. */
+    STAMCOUNTER StatRZSyncCR3Global;                /**< RC/R0: The number of global CR3 syncs. */
+    STAMCOUNTER StatRZSyncCR3NotGlobal;             /**< RC/R0: The number of non-global CR3 syncs. */
+    STAMCOUNTER StatRZSyncCR3DstCacheHit;           /**< RC/R0: The number of times we got some kind of cache hit on a page table. */
+    STAMCOUNTER StatRZSyncCR3DstFreed;              /**< RC/R0: The number of times we've had to free a shadow entry. */
+    STAMCOUNTER StatRZSyncCR3DstFreedSrcNP;         /**< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. */
+    STAMCOUNTER StatRZSyncCR3DstNotPresent;         /**< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. */
+    STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD;    /**< RC/R0: The number of times a global page directory wasn't flushed. */
+    STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT;    /**< RC/R0: The number of times a page table with only global entries wasn't flushed. */
+    STAMPROFILE StatRZSyncPT;                       /**< RC/R0: PGMSyncPT() profiling. */
+    STAMCOUNTER StatRZSyncPTFailed;                 /**< RC/R0: The number of times PGMSyncPT() failed. */
+    STAMCOUNTER StatRZSyncPT4K;                     /**< RC/R0: Number of 4KB syncs. */
+    STAMCOUNTER StatRZSyncPT4M;                     /**< RC/R0: Number of 4MB syncs. */
+    STAMCOUNTER StatRZSyncPagePDNAs;                /**< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
+    STAMCOUNTER StatRZSyncPagePDOutOfSync;          /**< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. */
+    STAMCOUNTER StatRZAccessedPage;                 /**< RC/R0: The number of pages marked not present for accessed bit emulation. */
+    STAMPROFILE StatRZDirtyBitTracking;             /**< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. */
+    STAMCOUNTER StatRZDirtyPage;                    /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatRZDirtyPageBig;                 /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatRZDirtyPageSkipped;             /**< RC/R0: The number of pages already dirty or readonly. */
+    STAMCOUNTER StatRZDirtyPageTrap;                /**< RC/R0: The number of traps generated for dirty bit tracking. */
+    STAMCOUNTER StatRZDirtyPageStale;               /**< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) */
+    STAMCOUNTER StatRZDirtyTrackRealPF;             /**< RC/R0: The number of real pages faults during dirty bit tracking. */
+    STAMCOUNTER StatRZDirtiedPage;                  /**< RC/R0: The number of pages marked dirty because of write accesses. */
+    STAMCOUNTER StatRZPageAlreadyDirty;             /**< RC/R0: The number of pages already marked dirty because of write accesses. */
+    STAMPROFILE StatRZInvalidatePage;               /**< RC/R0: PGMInvalidatePage() profiling. */
+    STAMCOUNTER StatRZInvalidatePage4KBPages;       /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. */
+    STAMCOUNTER StatRZInvalidatePage4MBPages;       /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. */
+    STAMCOUNTER StatRZInvalidatePage4MBPagesSkip;   /**< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. */
+    STAMCOUNTER StatRZInvalidatePagePDMappings;     /**< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). */
+    STAMCOUNTER StatRZInvalidatePagePDNAs;          /**< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
+    STAMCOUNTER StatRZInvalidatePagePDNPs;          /**< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. */
+    STAMCOUNTER StatRZInvalidatePagePDOutOfSync;    /**< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
+    STAMCOUNTER StatRZInvalidatePageSkipped;        /**< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
+    STAMCOUNTER StatRZPageOutOfSyncUser;            /**< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatRZPageOutOfSyncSupervisor;      /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatRZPageOutOfSyncUserWrite;       /**< RC/R0: The number of times user page is out of sync was detected in \#PF. */
+    STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF. */
+    STAMCOUNTER StatRZPageOutOfSyncBallloon;        /**< RC/R0: The number of times a ballooned page was accessed (read). */
+    STAMPROFILE StatRZPrefetch;                     /**< RC/R0: PGMPrefetchPage. */
+    STAMPROFILE StatRZFlushTLB;                     /**< RC/R0: Profiling of the PGMFlushTLB() body. */
+    STAMCOUNTER StatRZFlushTLBNewCR3;               /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
+    STAMCOUNTER StatRZFlushTLBNewCR3Global;         /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
+    STAMCOUNTER StatRZFlushTLBSameCR3;              /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
+    STAMCOUNTER StatRZFlushTLBSameCR3Global;        /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
+    STAMPROFILE StatRZGstModifyPage;                /**< RC/R0: Profiling of the PGMGstModifyPage() body */
+
+    STAMPROFILE StatR3SyncCR3;                      /**< R3: PGMSyncCR3() profiling. */
+    STAMPROFILE StatR3SyncCR3Handlers;              /**< R3: Profiling of the PGMSyncCR3() update handler section. */
+    STAMCOUNTER StatR3SyncCR3Global;                /**< R3: The number of global CR3 syncs. */
+    STAMCOUNTER StatR3SyncCR3NotGlobal;             /**< R3: The number of non-global CR3 syncs. */
+    STAMCOUNTER StatR3SyncCR3DstFreed;              /**< R3: The number of times we've had to free a shadow entry. */
+    STAMCOUNTER StatR3SyncCR3DstFreedSrcNP;         /**< R3: The number of times we've had to free a shadow entry for which the source entry was not present. */
+    STAMCOUNTER StatR3SyncCR3DstNotPresent;         /**< R3: The number of times we've encountered a not present shadow entry for a present guest entry. */
+    STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD;    /**< R3: The number of times a global page directory wasn't flushed. */
+    STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT;    /**< R3: The number of times a page table with only global entries wasn't flushed. */
+    STAMCOUNTER StatR3SyncCR3DstCacheHit;           /**< R3: The number of times we got some kind of cache hit on a page table. */
+    STAMPROFILE StatR3SyncPT;                       /**< R3: PGMSyncPT() profiling. */
+    STAMCOUNTER StatR3SyncPTFailed;                 /**< R3: The number of times PGMSyncPT() failed. */
+    STAMCOUNTER StatR3SyncPT4K;                     /**< R3: Number of 4KB syncs. */
+    STAMCOUNTER StatR3SyncPT4M;                     /**< R3: Number of 4MB syncs. */
+    STAMCOUNTER StatR3SyncPagePDNAs;                /**< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
+    STAMCOUNTER StatR3SyncPagePDOutOfSync;          /**< R3: The number of time we've encountered an out-of-sync PD in SyncPage. */
+    STAMCOUNTER StatR3AccessedPage;                 /**< R3: The number of pages marked not present for accessed bit emulation. */
+    STAMPROFILE StatR3DirtyBitTracking;             /**< R3: Profiling the dirty bit tracking in CheckPageFault(). */
+    STAMCOUNTER StatR3DirtyPage;                    /**< R3: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatR3DirtyPageBig;                 /**< R3: The number of pages marked read-only for dirty bit tracking. */
+    STAMCOUNTER StatR3DirtyPageSkipped;             /**< R3: The number of pages already dirty or readonly. */
+    STAMCOUNTER StatR3DirtyPageTrap;                /**< R3: The number of traps generated for dirty bit tracking. */
+    STAMCOUNTER StatR3DirtyTrackRealPF;             /**< R3: The number of real pages faults during dirty bit tracking. */
+    STAMCOUNTER StatR3DirtiedPage;                  /**< R3: The number of pages marked dirty because of write accesses. */
+    STAMCOUNTER StatR3PageAlreadyDirty;             /**< R3: The number of pages already marked dirty because of write accesses. */
+    STAMPROFILE StatR3InvalidatePage;               /**< R3: PGMInvalidatePage() profiling. */
+    STAMCOUNTER StatR3InvalidatePage4KBPages;       /**< R3: The number of times PGMInvalidatePage() was called for a 4KB page. */
+    STAMCOUNTER StatR3InvalidatePage4MBPages;       /**< R3: The number of times PGMInvalidatePage() was called for a 4MB page. */
+    STAMCOUNTER StatR3InvalidatePage4MBPagesSkip;   /**< R3: The number of times PGMInvalidatePage() skipped a 4MB page. */
+    STAMCOUNTER StatR3InvalidatePagePDNAs;          /**< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
+    STAMCOUNTER StatR3InvalidatePagePDNPs;          /**< R3: The number of times PGMInvalidatePage() was called for a not present page directory. */
+    STAMCOUNTER StatR3InvalidatePagePDMappings;     /**< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). */
+    STAMCOUNTER StatR3InvalidatePagePDOutOfSync;    /**< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
+    STAMCOUNTER StatR3InvalidatePageSkipped;        /**< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
+    STAMCOUNTER StatR3PageOutOfSyncUser;            /**< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatR3PageOutOfSyncSupervisor;      /**< R3: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
+    STAMCOUNTER StatR3PageOutOfSyncUserWrite;       /**< R3: The number of times user page is out of sync was detected in \#PF. */
+    STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /**< R3: The number of times supervisor page is out of sync was detected in in \#PF. */
+    STAMCOUNTER StatR3PageOutOfSyncBallloon;        /**< R3: The number of times a ballooned page was accessed (read). */
+    STAMPROFILE StatR3Prefetch;                     /**< R3: PGMPrefetchPage. */
+    STAMPROFILE StatR3FlushTLB;                     /**< R3: Profiling of the PGMFlushTLB() body. */
+    STAMCOUNTER StatR3FlushTLBNewCR3;               /**< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
+    STAMCOUNTER StatR3FlushTLBNewCR3Global;         /**< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
+    STAMCOUNTER StatR3FlushTLBSameCR3;              /**< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
+    STAMCOUNTER StatR3FlushTLBSameCR3Global;        /**< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
+    STAMPROFILE StatR3GstModifyPage;                /**< R3: Profiling of the PGMGstModifyPage() body */
+    /** @} */
+} PGMCPUSTATS;
+
+
+/**
+ * Converts a PGMCPU pointer into a VM pointer.
+ * @returns Pointer to the VM structure the PGM is part of.
+ * @param   pPGM   Pointer to PGMCPU instance data.
+ */
+#define PGMCPU2VM(pPGM)         ( (PVM)((char*)(pPGM) - (pPGM)->offVM) )
+
+/**
+ * Converts a PGMCPU pointer into a PGM pointer.
+ * @returns Pointer to the VM structure the PGM is part of.
+ * @param   pPGM   Pointer to PGMCPU instance data.
+ */
+#define PGMCPU2PGM(pPGMCpu)     ( (PPGM)((char *)(pPGMCpu) - (pPGMCpu)->offPGM) )
+
+/**
+ * PGMCPU Data (part of VMCPU).
+ */
+typedef struct PGMCPU
+{
+    /** Offset to the VM structure. */
+    int32_t                         offVM;
+    /** Offset to the VMCPU structure. */
+    int32_t                         offVCpu;
+    /** Offset of the PGM structure relative to VMCPU. */
+    int32_t                         offPGM;
+    uint32_t                        uPadding0;      /**< structure size alignment. */
+
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) || defined(VBOX_WITH_RAW_MODE)
+    /** Automatically tracked physical memory mapping set.
+     * Ring-0 and strict raw-mode builds. */
+    PGMMAPSET                       AutoSet;
+#endif
+
+    /** A20 gate mask.
+     * Our current approach to A20 emulation is to let REM do it and don't bother
+     * anywhere else. The interesting Guests will be operating with it enabled anyway.
+     * But whould need arrise, we'll subject physical addresses to this mask. */
+    RTGCPHYS                        GCPhysA20Mask;
+    /** A20 gate state - boolean! */
+    bool                            fA20Enabled;
+    /** Mirror of the EFER.NXE bit.  Managed by PGMNotifyNxeChanged. */
+    bool                            fNoExecuteEnabled;
+    /** Unused bits. */
+    bool                            afUnused[2];
+
+    /** What needs syncing (PGM_SYNC_*).
+     * This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
+     * PGMFlushTLB, and PGMR3Load. */
+    RTUINT                          fSyncFlags;
+
+    /** The shadow paging mode. */
+    PGMMODE                         enmShadowMode;
+    /** The guest paging mode. */
+    PGMMODE                         enmGuestMode;
+
+    /** The current physical address representing in the guest CR3 register. */
+    RTGCPHYS                        GCPhysCR3;
+
+    /** @name 32-bit Guest Paging.
+     * @{ */
+    /** The guest's page directory, R3 pointer. */
+    R3PTRTYPE(PX86PD)               pGst32BitPdR3;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** The guest's page directory, R0 pointer. */
+    R0PTRTYPE(PX86PD)               pGst32BitPdR0;
+#endif
+    /** The guest's page directory, static RC mapping. */
+    RCPTRTYPE(PX86PD)               pGst32BitPdRC;
+    /** Mask containing the MBZ bits of a big page PDE. */
+    uint32_t                        fGst32BitMbzBigPdeMask;
+    /** Set if the page size extension (PSE) is enabled. */
+    bool                            fGst32BitPageSizeExtension;
+    /** Alignment padding. */
+    bool                            afAlignment2[3];
+    /** @} */
+
+    /** @name PAE Guest Paging.
+     * @{ */
+    /** The guest's page directory pointer table, static RC mapping. */
+    RCPTRTYPE(PX86PDPT)             pGstPaePdptRC;
+    /** The guest's page directory pointer table, R3 pointer. */
+    R3PTRTYPE(PX86PDPT)             pGstPaePdptR3;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** The guest's page directory pointer table, R0 pointer. */
+    R0PTRTYPE(PX86PDPT)             pGstPaePdptR0;
+#endif
+
+    /** The guest's page directories, R3 pointers.
+     * These are individual pointers and don't have to be adjacent.
+     * These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
+    R3PTRTYPE(PX86PDPAE)            apGstPaePDsR3[4];
+    /** The guest's page directories, R0 pointers.
+     * Same restrictions as apGstPaePDsR3. */
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    R0PTRTYPE(PX86PDPAE)            apGstPaePDsR0[4];
+#endif
+    /** The guest's page directories, static GC mapping.
+     * Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
+     * These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
+    RCPTRTYPE(PX86PDPAE)            apGstPaePDsRC[4];
+    /** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
+    RTGCPHYS                        aGCPhysGstPaePDs[4];
+    /** The physical addresses of the monitored guest page directories (PAE). */
+    RTGCPHYS                        aGCPhysGstPaePDsMonitored[4];
+    /** Mask containing the MBZ PTE bits. */
+    uint64_t                        fGstPaeMbzPteMask;
+    /** Mask containing the MBZ PDE bits. */
+    uint64_t                        fGstPaeMbzPdeMask;
+    /** Mask containing the MBZ big page PDE bits. */
+    uint64_t                        fGstPaeMbzBigPdeMask;
+    /** Mask containing the MBZ PDPE bits. */
+    uint64_t                        fGstPaeMbzPdpeMask;
+    /** @} */
+
+    /** @name AMD64 Guest Paging.
+     * @{ */
+    /** The guest's page directory pointer table, R3 pointer. */
+    R3PTRTYPE(PX86PML4)             pGstAmd64Pml4R3;
+#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
+    /** The guest's page directory pointer table, R0 pointer. */
+    R0PTRTYPE(PX86PML4)             pGstAmd64Pml4R0;
+#else
+    RTR0PTR                         alignment6b; /**< alignment equalizer. */
+#endif
+    /** Mask containing the MBZ PTE bits. */
+    uint64_t                        fGstAmd64MbzPteMask;
+    /** Mask containing the MBZ PDE bits. */
+    uint64_t                        fGstAmd64MbzPdeMask;
+    /** Mask containing the MBZ big page PDE bits. */
+    uint64_t                        fGstAmd64MbzBigPdeMask;
+    /** Mask containing the MBZ PDPE bits. */
+    uint64_t                        fGstAmd64MbzPdpeMask;
+    /** Mask containing the MBZ big page PDPE bits. */
+    uint64_t                        fGstAmd64MbzBigPdpeMask;
+    /** Mask containing the MBZ PML4E bits. */
+    uint64_t                        fGstAmd64MbzPml4eMask;
+    /** Mask containing the PDPE bits that we shadow. */
+    uint64_t                        fGstAmd64ShadowedPdpeMask;
+    /** Mask containing the PML4E bits that we shadow. */
+    uint64_t                        fGstAmd64ShadowedPml4eMask;
+    /** @} */
+
+    /** @name PAE and AMD64 Guest Paging.
+     * @{ */
+    /** Mask containing the PTE bits that we shadow. */
+    uint64_t                        fGst64ShadowedPteMask;
+    /** Mask containing the PDE bits that we shadow. */
+    uint64_t                        fGst64ShadowedPdeMask;
+    /** Mask containing the big page PDE bits that we shadow in the PDE. */
+    uint64_t                        fGst64ShadowedBigPdeMask;
+    /** Mask containing the big page PDE bits that we shadow in the PTE. */
+    uint64_t                        fGst64ShadowedBigPde4PteMask;
+    /** @}  */
+
+    /** Pointer to the page of the current active CR3 - R3 Ptr. */
+    R3PTRTYPE(PPGMPOOLPAGE)         pShwPageCR3R3;
+    /** Pointer to the page of the current active CR3 - R0 Ptr. */
+    R0PTRTYPE(PPGMPOOLPAGE)         pShwPageCR3R0;
+    /** Pointer to the page of the current active CR3 - RC Ptr. */
+    RCPTRTYPE(PPGMPOOLPAGE)         pShwPageCR3RC;
+    /* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
+    uint32_t                        iShwUser;
+    /* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
+    uint32_t                        iShwUserTable;
+# if HC_ARCH_BITS == 64
+    RTRCPTR                         alignment6; /**< structure size alignment. */
+# endif
+    /** @} */
+
+    /** @name Function pointers for Shadow paging.
+     * @{
+     */
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwExit,(PVMCPU pVCpu));
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
+    DECLR3CALLBACKMEMBER(int,       pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
+
+    DECLRCCALLBACKMEMBER(int,       pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
+    DECLRCCALLBACKMEMBER(int,       pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
+
+    DECLR0CALLBACKMEMBER(int,       pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
+    DECLR0CALLBACKMEMBER(int,       pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
+
+    /** @} */
+
+    /** @name Function pointers for Guest paging.
+     * @{
+     */
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstExit,(PVMCPU pVCpu));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
+    DECLR3CALLBACKMEMBER(int,       pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
+    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
+    DECLRCCALLBACKMEMBER(int,       pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
+    DECLRCCALLBACKMEMBER(int,       pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
+#if HC_ARCH_BITS == 64
+    RTRCPTR                         alignment3; /**< structure size alignment. */
+#endif
+
+    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
+    DECLR0CALLBACKMEMBER(int,       pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
+    DECLR0CALLBACKMEMBER(int,       pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
+    /** @} */
+
+    /** @name Function pointers for Both Shadow and Guest paging.
+     * @{
+     */
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
+    /*                           no pfnR3BthTrap0eHandler */
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
+    DECLR3CALLBACKMEMBER(unsigned,  pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
+    DECLR3CALLBACKMEMBER(int,       pfnR3BthUnmapCR3,(PVMCPU pVCpu));
+
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
+    DECLR0CALLBACKMEMBER(unsigned,  pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
+    DECLR0CALLBACKMEMBER(int,       pfnR0BthUnmapCR3,(PVMCPU pVCpu));
+
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
+    DECLRCCALLBACKMEMBER(unsigned,  pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
+    DECLRCCALLBACKMEMBER(int,       pfnRCBthUnmapCR3,(PVMCPU pVCpu));
+#if 0
+    RTRCPTR                         alignment2; /**< structure size alignment. */
+#endif
+    /** @} */
+
+    /** For saving stack space, the disassembler state is allocated here instead of
+     * on the stack.
+     * @note The DISCPUSTATE structure is not R3/R0/RZ clean!  */
+    union
+    {
+        /** The disassembler scratch space. */
+        DISCPUSTATE                 DisState;
+        /** Padding. */
+        uint8_t                     abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
+    };
+
+    /** Count the number of pgm pool access handler calls. */
+    uint64_t                        cPoolAccessHandler;
+
+    /** @name Release Statistics
+     * @{ */
+    /** The number of times the guest has switched mode since last reset or statistics reset. */
+    STAMCOUNTER                     cGuestModeChanges;
+    /** @} */
+
+#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap.  */
+    /** @name Statistics
+     * @{ */
+    /** RC: Pointer to the statistics. */
+    RCPTRTYPE(PGMCPUSTATS *)        pStatsRC;
+    /** RC: Which statistic this \#PF should be attributed to. */
+    RCPTRTYPE(PSTAMPROFILE)         pStatTrap0eAttributionRC;
+    /** R0: Pointer to the statistics. */
+    R0PTRTYPE(PGMCPUSTATS *)        pStatsR0;
+    /** R0: Which statistic this \#PF should be attributed to. */
+    R0PTRTYPE(PSTAMPROFILE)         pStatTrap0eAttributionR0;
+    /** R3: Pointer to the statistics. */
+    R3PTRTYPE(PGMCPUSTATS *)        pStatsR3;
+    /** Alignment padding. */
+    RTR3PTR                         pPaddingR3;
+    /** @}  */
+#endif /* VBOX_WITH_STATISTICS */
+} PGMCPU;
+/** Pointer to the per-cpu PGM data. */
+typedef PGMCPU *PPGMCPU;
+
+
+/** @name PGM::fSyncFlags Flags
+ * @{
+ */
+/** Updates the virtual access handler state bit in PGMPAGE. */
+#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL        RT_BIT(0)
+/** Always sync CR3. */
+#define PGM_SYNC_ALWAYS                         RT_BIT(1)
+/** Check monitoring on next CR3 (re)load and invalidate page.
+ * @todo This is obsolete now. Remove after 2.2.0 is branched off. */
+#define PGM_SYNC_MONITOR_CR3                    RT_BIT(2)
+/** Check guest mapping in SyncCR3. */
+#define PGM_SYNC_MAP_CR3                        RT_BIT(3)
+/** Clear the page pool (a light weight flush). */
+#define PGM_SYNC_CLEAR_PGM_POOL_BIT             8
+#define PGM_SYNC_CLEAR_PGM_POOL                 RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT)
+/** @} */
+
+
+RT_C_DECLS_BEGIN
+
+int             pgmLock(PVM pVM);
+void            pgmUnlock(PVM pVM);
+
+int             pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb);
+int             pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
+int             pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
+PPGMMAPPING     pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
+int             pgmMapResolveConflicts(PVM pVM);
+DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+
+void            pgmR3HandlerPhysicalUpdateAll(PVM pVM);
+bool            pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
+void            pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, bool fDoAccounting);
+int             pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER *ppVirt, unsigned *piPage);
+DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
+#if defined(VBOX_STRICT) || defined(LOG_ENABLED)
+void            pgmHandlerVirtualDumpPhysPages(PVM pVM);
+#else
+# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
+#endif
+DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
+int             pgmR3InitSavedState(PVM pVM, uint64_t cbRam);
+
+int             pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+int             pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys);
+int             pgmPhysIsValidLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage);
+int             pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
+int             pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+void            pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage);
+int             pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
+int             pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
+int             pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
+int             pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv);
+int             pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
+int             pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
+int             pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
+VMMDECL(int)    pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
+VMMDECL(int)    pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
+int             pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys);
+
+#ifdef IN_RING3
+void            pgmR3PhysRelinkRamRanges(PVM pVM);
+int             pgmR3PhysRamPreAllocate(PVM pVM);
+int             pgmR3PhysRamReset(PVM pVM);
+int             pgmR3PhysRomReset(PVM pVM);
+int             pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
+int             pgmR3PhysRamTerm(PVM pVM);
+void            pgmR3PhysRomTerm(PVM pVM);
+
+int             pgmR3PoolInit(PVM pVM);
+void            pgmR3PoolRelocate(PVM pVM);
+void            pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu);
+void            pgmR3PoolReset(PVM pVM);
+void            pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb);
+DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl);
+void            pgmR3PoolWriteProtectPages(PVM pVM);
+
+#endif /* IN_RING3 */
+#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) || defined(IN_RC)
+int             pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
+int             pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
+# ifdef LOG_ENABLED
+void            pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL);
+# else
+void            pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint);
+# endif
+#endif
+int             pgmPoolAllocEx(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser,
+                               uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage);
+
+DECLINLINE(int) pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable,
+                             PPPGMPOOLPAGE ppPage)
+{
+    return pgmPoolAllocEx(pVM, GCPhys, enmKind, PGMPOOLACCESS_DONTCARE, iUser, iUserTable, false, ppPage);
+}
+
+void            pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
+void            pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
+int             pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush = true /* DO NOT USE false UNLESS YOU KNOWN WHAT YOU'RE DOING!! */);
+void            pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys);
+PPGMPOOLPAGE    pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
+PPGMPOOLPAGE    pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys);
+int             pgmPoolSyncCR3(PVMCPU pVCpu);
+bool            pgmPoolIsDirtyPage(PVM pVM, RTGCPHYS GCPhys);
+void            pgmPoolInvalidateDirtyPage(PVM pVM, RTGCPHYS GCPhysPT);
+int             pgmPoolTrackUpdateGCPhys(PVM pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs);
+void            pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte);
+uint16_t        pgmPoolTrackPhysExtAddref(PVM pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte);
+void            pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte);
+void            pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, unsigned cbWrite);
+int             pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+void            pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+
+void            pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+void            pgmPoolResetDirtyPages(PVM pVM);
+void            pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage);
+
+int             pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
+int             pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
+
+void            pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
+void            pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
+int             pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
+int             pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
+
+int             pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
+int             pgmShwSyncNestedPageLocked(PVMCPU pVCpu, RTGCPHYS GCPhysFault, uint32_t cPages, PGMMODE enmShwPagingMode);
+
+int             pgmGstLazyMap32BitPD(PVMCPU pVCpu, PX86PD *ppPd);
+int             pgmGstLazyMapPaePDPT(PVMCPU pVCpu, PX86PDPT *ppPdpt);
+int             pgmGstLazyMapPaePD(PVMCPU pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd);
+int             pgmGstLazyMapPml4(PVMCPU pVCpu, PX86PML4 *ppPml4);
+
+# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
+DECLCALLBACK(int)  pgmR3CmdCheckDuplicatePages(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+DECLCALLBACK(int)  pgmR3CmdShowSharedModules(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs, PDBGCVAR pResult);
+# endif
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/REMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/REMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/REMInternal.h	(revision 35333)
@@ -0,0 +1,286 @@
+/* $Id$ */
+/** @file
+ * REM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___REMInternal_h
+#define ___REMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/cpum.h>
+#include <VBox/stam.h>
+#include <VBox/pgm.h>
+#include <VBox/pdmcritsect.h>
+#ifdef REM_INCLUDE_CPU_H
+# include "target-i386/cpu.h"
+#endif
+
+
+
+/** @defgroup grp_rem_int   Internals
+ * @ingroup grp_rem
+ * @internal
+ * @{
+ */
+
+/** The saved state version number. */
+#define REM_SAVED_STATE_VERSION_VER1_6   6
+#define REM_SAVED_STATE_VERSION          7
+
+
+/** @def REM_MONITOR_CODE_PAGES
+ * Enable to monitor code pages that have been translated by the recompiler. */
+/** Currently broken and interferes with CSAM monitoring (see #2784) */
+////#define REM_MONITOR_CODE_PAGES
+#ifdef DOXYGEN_RUNNING
+# define REM_MONITOR_CODE_PAGES
+#endif
+
+typedef enum REMHANDLERNOTIFICATIONKIND
+{
+    /** The usual invalid 0 entry. */
+    REMHANDLERNOTIFICATIONKIND_INVALID = 0,
+    /** REMR3NotifyHandlerPhysicalRegister. */
+    REMHANDLERNOTIFICATIONKIND_PHYSICAL_REGISTER,
+    /** REMR3NotifyHandlerPhysicalDeregister. */
+    REMHANDLERNOTIFICATIONKIND_PHYSICAL_DEREGISTER,
+    /** REMR3NotifyHandlerPhysicalModify. */
+    REMHANDLERNOTIFICATIONKIND_PHYSICAL_MODIFY,
+    /** The usual 32-bit hack. */
+    REMHANDLERNOTIFICATIONKIND_32BIT_HACK = 0x7fffffff
+} REMHANDLERNOTIFICATIONKIND;
+
+
+/**
+ * A recorded handler notification.
+ */
+typedef struct REMHANDLERNOTIFICATION
+{
+    /** The notification kind. */
+    REMHANDLERNOTIFICATIONKIND  enmKind;
+    uint32_t                    padding;
+    /** Type specific data. */
+    union
+    {
+        struct
+        {
+            RTGCPHYS            GCPhys;
+            RTGCPHYS            cb;
+            PGMPHYSHANDLERTYPE  enmType;
+            bool                fHasHCHandler;
+        } PhysicalRegister;
+
+        struct
+        {
+            RTGCPHYS            GCPhys;
+            RTGCPHYS            cb;
+            PGMPHYSHANDLERTYPE  enmType;
+            bool                fHasHCHandler;
+            bool                fRestoreAsRAM;
+        } PhysicalDeregister;
+
+        struct
+        {
+            RTGCPHYS            GCPhysOld;
+            RTGCPHYS            GCPhysNew;
+            RTGCPHYS            cb;
+            PGMPHYSHANDLERTYPE  enmType;
+            bool                fHasHCHandler;
+            bool                fRestoreAsRAM;
+        } PhysicalModify;
+        uint64_t                padding[5];
+    } u;
+    uint32_t                    idxSelf;
+    uint32_t volatile           idxNext;
+} REMHANDLERNOTIFICATION;
+/** Pointer to a handler notification record. */
+typedef REMHANDLERNOTIFICATION *PREMHANDLERNOTIFICATION;
+
+/**
+ * Converts a REM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the REM is part of.
+ * @param   pREM    Pointer to REM instance data.
+ */
+#define REM2VM(pREM)  ( (PVM)((char*)pREM - pREM->offVM) )
+
+
+/**
+ * REM Data (part of VM)
+ */
+typedef struct REM
+{
+    /** Offset to the VM structure. */
+    RTINT                   offVM;
+    /** Alignment padding. */
+    RTUINT                  uPadding0;
+
+    /** Cached pointer of the register context of the current VCPU. */
+    R3PTRTYPE(PCPUMCTX)     pCtx;
+
+    /** In REM mode.
+     * I.e. the correct CPU state and some other bits are with REM. */
+    bool volatile           fInREM;
+    /** In REMR3State. */
+    bool                    fInStateSync;
+
+    /** Set when the translation blocks cache need to be flushed. */
+    bool                    fFlushTBs;
+
+    /** Ignore CR3 load notifications from the REM. */
+    bool                    fIgnoreCR3Load;
+    /** Ignore invlpg notifications from the REM. */
+    bool                    fIgnoreInvlPg;
+    /** Ignore CR0, CR4 and EFER load. */
+    bool                    fIgnoreCpuMode;
+    /** Ignore set page. */
+    bool                    fIgnoreSetPage;
+    bool                    bPadding1;
+
+    /** Ignore all that can be ignored. */
+    uint32_t                cIgnoreAll;
+
+    /** Number of times REMR3CanExecuteRaw has been called.
+     * It is used to prevent rescheduling on the first call. */
+    uint32_t                cCanExecuteRaw;
+
+    /** Pending interrupt (~0 -> nothing). */
+    uint32_t                u32PendingInterrupt;
+
+    /** Number of recorded invlpg instructions. */
+    uint32_t volatile       cInvalidatedPages;
+#if HC_ARCH_BITS == 32
+    uint32_t                uPadding2;
+#endif
+    /** Array of recorded invlpg instruction.
+     * These instructions are replayed when entering REM. */
+    RTGCPTR                 aGCPtrInvalidatedPages[48];
+
+    /** Array of recorded handler notifications.
+     * These are replayed when entering REM. */
+    REMHANDLERNOTIFICATION  aHandlerNotifications[64];
+    volatile uint32_t       idxPendingList;
+    volatile uint32_t       idxFreeList;
+
+    /** MMIO memory type.
+     * This is used to register MMIO physical access handlers. */
+    int32_t                 iMMIOMemType;
+    /** Handler memory type.
+     * This is used to register non-MMIO physical access handlers which are executed in HC. */
+    int32_t                 iHandlerMemType;
+
+    /** Pending exception */
+    uint32_t                uPendingException;
+    /** Nr of pending exceptions */
+    uint32_t                cPendingExceptions;
+    /** Pending exception's EIP */
+    RTGCPTR                 uPendingExcptEIP;
+    /** Pending exception's CR2 */
+    RTGCPTR                 uPendingExcptCR2;
+
+    /** The highest known RAM address. */
+    RTGCPHYS                GCPhysLastRam;
+    /** Whether GCPhysLastRam has been fixed (see REMR3Init()). */
+    bool                    fGCPhysLastRamFixed;
+
+    /** Pending rc. */
+    int32_t                 rc;
+
+    /** REM critical section.
+     * This protects cpu_register_physical_memory usage
+     */
+    PDMCRITSECT             CritSectRegister;
+
+    /** Time spent in QEMU. */
+    STAMPROFILEADV          StatsInQEMU;
+    /** Time spent in rawmode.c. */
+    STAMPROFILEADV          StatsInRAWEx;
+    /** Time spent switching state. */
+    STAMPROFILE             StatsState;
+    /** Time spent switching state back. */
+    STAMPROFILE             StatsStateBack;
+
+    /** Padding the CPUX86State structure to 64 byte. */
+    uint32_t                abPadding[HC_ARCH_BITS == 32 ? 4 : 4];
+
+# define REM_ENV_SIZE       0xff00
+
+    /** Recompiler CPU state. */
+#ifdef REM_INCLUDE_CPU_H
+    CPUX86State             Env;
+#else
+    struct FakeEnv
+    {
+        char                achPadding[REM_ENV_SIZE];
+    }                       Env;
+#endif /* !REM_INCLUDE_CPU_H */
+} REM;
+
+/** Pointer to the REM Data. */
+typedef REM *PREM;
+
+
+#ifdef REM_INCLUDE_CPU_H
+bool    remR3CanExecuteRaw(CPUState *env, RTGCPTR eip, unsigned fFlags, int *piException);
+void    remR3CSAMCheckEIP(CPUState *env, RTGCPTR GCPtrCode);
+bool    remR3GetOpcode(CPUState *env, RTGCPTR GCPtrInstr, uint8_t *pu8Byte);
+bool    remR3DisasInstr(CPUState *env, int f32BitCode, char *pszPrefix);
+void    remR3FlushPage(CPUState *env, RTGCPTR GCPtr);
+void    remR3SetPage(CPUState *env, CPUTLBEntry *pRead,  CPUTLBEntry *pWrite, int prot, int is_user);
+void    remR3FlushTLB(CPUState *env, bool fGlobal);
+void    remR3ProtectCode(CPUState *env, RTGCPTR GCPtr);
+void    remR3ChangeCpuMode(CPUState *env);
+void    remR3DmaRun(CPUState *env);
+void    remR3TimersRun(CPUState *env);
+int     remR3NotifyTrap(CPUState *env, uint32_t uTrap, uint32_t uErrorCode, RTGCPTR pvNextEIP);
+void    remR3TrapStat(CPUState *env, uint32_t uTrap);
+void    remR3CpuId(CPUState *env, unsigned uOperator, void *pvEAX, void *pvEBX, void *pvECX, void *pvEDX);
+void    remR3RecordCall(CPUState *env);
+#endif /* REM_INCLUDE_CPU_H */
+void    remR3TrapClear(PVM pVM);
+void    remR3RaiseRC(PVM pVM, int rc);
+void    remR3DumpLnxSyscall(PVMCPU pVCpu);
+void    remR3DumpOBsdSyscall(PVMCPU pVCpu);
+
+
+/** @todo r=bird: clean up the RAWEx stats. */
+/* temporary hacks */
+#define RAWEx_ProfileStart(a, b)    remR3ProfileStart(b)
+#define RAWEx_ProfileStop(a, b)     remR3ProfileStop(b)
+
+
+#ifdef VBOX_WITH_STATISTICS
+
+# define STATS_EMULATE_SINGLE_INSTR         1
+# define STATS_QEMU_COMPILATION             2
+# define STATS_QEMU_RUN_EMULATED_CODE       3
+# define STATS_QEMU_TOTAL                   4
+# define STATS_QEMU_RUN_TIMERS              5
+# define STATS_TLB_LOOKUP                   6
+# define STATS_IRQ_HANDLING                 7
+# define STATS_RAW_CHECK                    8
+
+void remR3ProfileStart(int statcode);
+void remR3ProfileStop(int statcode);
+
+#else  /* !VBOX_WITH_STATISTICS */
+# define remR3ProfileStart(c)   do { } while (0)
+# define remR3ProfileStop(c)    do { } while (0)
+#endif /* !VBOX_WITH_STATISTICS */
+
+/** @} */
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/SELMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/SELMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/SELMInternal.h	(revision 35333)
@@ -0,0 +1,193 @@
+/* $Id$ */
+/** @file
+ * SELM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___SELMInternal_h
+#define ___SELMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/stam.h>
+#include <VBox/cpum.h>
+
+
+
+/** @defgroup grp_selm_int   Internals
+ * @ingroup grp_selm
+ * @internal
+ * @{
+ */
+
+/** The number of GDTS allocated for our GDT. (full size) */
+#define SELM_GDT_ELEMENTS                   8192
+
+/** aHyperSel index to retrieve hypervisor selectors */
+/** The Flat CS selector used by the VMM inside the GC. */
+#define SELM_HYPER_SEL_CS                   0
+/** The Flat DS selector used by the VMM inside the GC. */
+#define SELM_HYPER_SEL_DS                   1
+/** The 64-bit mode CS selector used by the VMM inside the GC. */
+#define SELM_HYPER_SEL_CS64                 2
+/** The TSS selector used by the VMM inside the GC. */
+#define SELM_HYPER_SEL_TSS                  3
+/** The TSS selector for taking trap 08 (\#DF). */
+#define SELM_HYPER_SEL_TSS_TRAP08           4
+/** Number of GDTs we need for internal use */
+#define SELM_HYPER_SEL_MAX                  (SELM_HYPER_SEL_TSS_TRAP08 + 1)
+
+
+/** Default GDT selectors we use for the hypervisor. */
+#define SELM_HYPER_DEFAULT_SEL_CS           ((SELM_GDT_ELEMENTS - 0x1) << 3)
+#define SELM_HYPER_DEFAULT_SEL_DS           ((SELM_GDT_ELEMENTS - 0x2) << 3)
+#define SELM_HYPER_DEFAULT_SEL_CS64         ((SELM_GDT_ELEMENTS - 0x3) << 3)
+#define SELM_HYPER_DEFAULT_SEL_TSS          ((SELM_GDT_ELEMENTS - 0x4) << 3)
+#define SELM_HYPER_DEFAULT_SEL_TSS_TRAP08   ((SELM_GDT_ELEMENTS - 0x5) << 3)
+/** The lowest value default we use. */
+#define SELM_HYPER_DEFAULT_BASE             SELM_HYPER_DEFAULT_SEL_TSS_TRAP08
+
+/**
+ * Converts a SELM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the SELM is part of.
+ * @param   pSELM   Pointer to SELM instance data.
+ */
+#define SELM2VM(pSELM)  ( (PVM)((char *)pSELM - pSELM->offVM) )
+
+
+
+/**
+ * SELM Data (part of VM)
+ */
+typedef struct SELM
+{
+    /** Offset to the VM structure.
+     * See SELM2VM(). */
+    RTINT                   offVM;
+
+    /** Flat CS, DS, 64 bit mode CS, TSS & trap 8 TSS. */
+    RTSEL                   aHyperSel[SELM_HYPER_SEL_MAX];
+
+    /** Pointer to the GCs - R3 Ptr.
+     * This size is governed by SELM_GDT_ELEMENTS. */
+    R3PTRTYPE(PX86DESC)     paGdtR3;
+    /** Pointer to the GCs - RC Ptr.
+     * This is not initialized until the first relocation because it's used to
+     * check if the shadow GDT virtual handler requires deregistration. */
+    RCPTRTYPE(PX86DESC)     paGdtRC;
+    /** Current (last) Guest's GDTR.
+     * The pGdt member is set to RTRCPTR_MAX if we're not monitoring the guest GDT. */
+    VBOXGDTR                GuestGdtr;
+    /** The current (last) effective Guest GDT size. */
+    RTUINT                  cbEffGuestGdtLimit;
+
+    uint32_t                padding0;
+
+    /** R3 pointer to the LDT shadow area in HMA. */
+    R3PTRTYPE(void *)       pvLdtR3;
+    /** RC pointer to the LDT shadow area in HMA. */
+    RCPTRTYPE(void *)       pvLdtRC;
+#if GC_ARCH_BITS == 64
+    RTRCPTR                 padding1;
+#endif
+    /** The address of the guest LDT.
+     * RTRCPTR_MAX if not monitored. */
+    RTGCPTR                 GCPtrGuestLdt;
+    /** Current LDT limit, both Guest and Shadow. */
+    RTUINT                  cbLdtLimit;
+    /** Current LDT offset relative to pvLdtR3/pvLdtRC. */
+    RTUINT                  offLdtHyper;
+#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
+    uint32_t                padding2[2];
+#endif
+    /** TSS. (This is 16 byte aligned!)
+      * @todo I/O bitmap & interrupt redirection table? */
+    VBOXTSS                 Tss;
+
+    /** TSS for trap 08 (\#DF). */
+    VBOXTSS                 TssTrap08;
+
+    /** Monitored shadow TSS address. */
+    RCPTRTYPE(void *)       pvMonShwTssRC;
+#if GC_ARCH_BITS == 64
+    RTRCPTR                 padding3;
+#endif
+    /** GC Pointer to the current Guest's TSS.
+     * RTRCPTR_MAX if not monitored. */
+    RTGCPTR                 GCPtrGuestTss;
+    /** The size of the guest TSS. */
+    RTUINT                  cbGuestTss;
+    /** Set if it's a 32-bit TSS. */
+    bool                    fGuestTss32Bit;
+    /** The size of the Guest's TSS part we're monitoring. */
+    RTUINT                  cbMonitoredGuestTss;
+    /** The guest TSS selector at last sync (part of monitoring).
+     * Contains RTSEL_MAX if not set. */
+    RTSEL                   GCSelTss;
+    /** The last known offset of the I/O bitmap.
+     * This is only used if we monitor the bitmap. */
+    uint16_t                offGuestIoBitmap;
+
+    /** Indicates that the Guest GDT access handler have been registered. */
+    bool                    fGDTRangeRegistered;
+
+    /** Indicates whether LDT/GDT/TSS monitoring and syncing is disabled. */
+    bool                    fDisableMonitoring;
+
+    /** Indicates whether the TSS stack selector & base address need to be refreshed.  */
+    bool                    fSyncTSSRing0Stack;
+    bool                    fPadding2[1+2];
+
+    /** SELMR3UpdateFromCPUM() profiling. */
+    STAMPROFILE             StatUpdateFromCPUM;
+    /** SELMR3SyncTSS() profiling. */
+    STAMPROFILE             StatTSSSync;
+
+    /** GC: The number of handled writes to the Guest's GDT. */
+    STAMCOUNTER             StatRCWriteGuestGDTHandled;
+    /** GC: The number of unhandled write to the Guest's GDT. */
+    STAMCOUNTER             StatRCWriteGuestGDTUnhandled;
+    /** GC: The number of times writes to Guest's LDT was detected. */
+    STAMCOUNTER             StatRCWriteGuestLDT;
+    /** GC: The number of handled writes to the Guest's TSS. */
+    STAMCOUNTER             StatRCWriteGuestTSSHandled;
+    /** GC: The number of handled writes to the Guest's TSS where we detected a change. */
+    STAMCOUNTER             StatRCWriteGuestTSSHandledChanged;
+    /** GC: The number of handled redir writes to the Guest's TSS where we detected a change. */
+    STAMCOUNTER             StatRCWriteGuestTSSRedir;
+    /** GC: The number of unhandled writes to the Guest's TSS. */
+    STAMCOUNTER             StatRCWriteGuestTSSUnhandled;
+    /** The number of times we had to relocate our hypervisor selectors. */
+    STAMCOUNTER             StatHyperSelsChanged;
+    /** The number of times we had find free hypervisor selectors. */
+    STAMCOUNTER             StatScanForHyperSels;
+} SELM, *PSELM;
+
+RT_C_DECLS_BEGIN
+
+VMMRCDECL(int) selmRCGuestGDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+VMMRCDECL(int) selmRCGuestLDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+VMMRCDECL(int) selmRCGuestTSSWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+
+VMMRCDECL(int) selmRCShadowGDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+VMMRCDECL(int) selmRCShadowLDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+VMMRCDECL(int) selmRCShadowTSSWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+
+void           selmSetRing1Stack(PVM pVM, uint32_t ss, RTGCPTR32 esp);
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif
Index: /trunk/src/VBox/VMM/include/SSMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/SSMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/SSMInternal.h	(revision 35333)
@@ -0,0 +1,288 @@
+/* $Id$ */
+/** @file
+ * SSM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___SSMInternal_h
+#define ___SSMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/ssm.h>
+#include <iprt/critsect.h>
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_ssm_int       Internals
+ * @ingroup grp_ssm
+ * @internal
+ * @{
+ */
+
+
+/**
+ * Data unit callback type.
+ */
+typedef enum SSMUNITTYPE
+{
+    /** PDM Device . */
+    SSMUNITTYPE_DEV = 1,
+    /** PDM Driver. */
+    SSMUNITTYPE_DRV,
+    /** VM Internal. */
+    SSMUNITTYPE_INTERNAL,
+    /** External Wrapper. */
+    SSMUNITTYPE_EXTERNAL
+} SSMUNITTYPE;
+
+/** Pointer to a data unit descriptor. */
+typedef struct SSMUNIT *PSSMUNIT;
+
+/**
+ * Data unit descriptor.
+ */
+typedef struct SSMUNIT
+{
+    /** Pointer ot the next one in the list. */
+    PSSMUNIT                pNext;
+
+    /** Called in this save/load operation.
+     * The flag is used to determine whether there is need for a call to
+     * done or not. */
+    bool                    fCalled;
+    /** Finished its live part.
+     * This is used to handle VERR_SSM_VOTE_FOR_GIVING_UP.  */
+    bool                    fDoneLive;
+    /** Callback interface type. */
+    SSMUNITTYPE             enmType;
+    /** Type specific data. */
+    union
+    {
+        /** SSMUNITTYPE_DEV. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMDEVLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMDEVLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMDEVLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMDEVSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMDEVSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMDEVSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMDEVLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMDEVLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMDEVLOADDONE   pfnLoadDone;
+            /** Device instance. */
+            PPDMDEVINS          pDevIns;
+        } Dev;
+
+        /** SSMUNITTYPE_DRV. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMDRVLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMDRVLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMDRVLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMDRVSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMDRVSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMDRVSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMDRVLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMDRVLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMDRVLOADDONE   pfnLoadDone;
+            /** Driver instance. */
+            PPDMDRVINS          pDrvIns;
+        } Drv;
+
+        /** SSMUNITTYPE_INTERNAL. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMINTLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMINTLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMINTLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMINTSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMINTSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMINTSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMINTLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMINTLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMINTLOADDONE   pfnLoadDone;
+        } Internal;
+
+        /** SSMUNITTYPE_EXTERNAL. */
+        struct
+        {
+            /** Prepare live save. */
+            PFNSSMEXTLIVEPREP   pfnLivePrep;
+            /** Execute live save. */
+            PFNSSMEXTLIVEEXEC   pfnLiveExec;
+            /** Vote live save complete. */
+            PFNSSMEXTLIVEVOTE   pfnLiveVote;
+            /** Prepare save. */
+            PFNSSMEXTSAVEPREP   pfnSavePrep;
+            /** Execute save. */
+            PFNSSMEXTSAVEEXEC   pfnSaveExec;
+            /** Done save. */
+            PFNSSMEXTSAVEDONE   pfnSaveDone;
+            /** Prepare load. */
+            PFNSSMEXTLOADPREP   pfnLoadPrep;
+            /** Execute load. */
+            PFNSSMEXTLOADEXEC   pfnLoadExec;
+            /** Done load. */
+            PFNSSMEXTLOADDONE   pfnLoadDone;
+            /** User data. */
+            void               *pvUser;
+        } External;
+
+        struct
+        {
+            /** Prepare live save. */
+            PFNRT               pfnLivePrep;
+            /** Execute live save. */
+            PFNRT               pfnLiveExec;
+            /** Vote live save complete. */
+            PFNRT               pfnLiveVote;
+            /** Prepare save. */
+            PFNRT               pfnSavePrep;
+            /** Execute save. */
+            PFNRT               pfnSaveExec;
+            /** Done save. */
+            PFNRT               pfnSaveDone;
+            /** Prepare load. */
+            PFNRT               pfnLoadPrep;
+            /** Execute load. */
+            PFNRT               pfnLoadExec;
+            /** Done load. */
+            PFNRT               pfnLoadDone;
+            /** User data. */
+            void               *pvKey;
+        } Common;
+    } u;
+    /** Data layout version. */
+    uint32_t                u32Version;
+    /** Instance number. */
+    uint32_t                u32Instance;
+    /** The offset of the final data unit.
+     * This is used for constructing the directory. */
+    RTFOFF                  offStream;
+    /** The guessed size of the data unit - used only for progress indication. */
+    size_t                  cbGuess;
+    /** Name size. (bytes) */
+    size_t                  cchName;
+    /** Name of this unit. (extends beyond the defined size) */
+    char                    szName[1];
+} SSMUNIT;
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Dev.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Dev.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Dev.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Dev.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Dev.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Dev.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Dev.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Dev.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Dev.pfnLoadDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.Dev.pDevIns);
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Drv.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Drv.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Drv.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Drv.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Drv.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Drv.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Drv.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Drv.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Drv.pfnLoadDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.Drv.pDrvIns);
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.Internal.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.Internal.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.Internal.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.Internal.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.Internal.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.Internal.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.Internal.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.Internal.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.Internal.pfnLoadDone);
+
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLivePrep, u.External.pfnLivePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveExec, u.External.pfnLiveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLiveVote, u.External.pfnLiveVote);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSavePrep, u.External.pfnSavePrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveExec, u.External.pfnSaveExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnSaveDone, u.External.pfnSaveDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadPrep, u.External.pfnLoadPrep);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadExec, u.External.pfnLoadExec);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pfnLoadDone, u.External.pfnLoadDone);
+AssertCompile2MemberOffsets(SSMUNIT, u.Common.pvKey,       u.External.pvUser);
+
+
+/**
+ * SSM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ *
+ * @todo Move this to UVM.
+ */
+typedef struct SSM
+{
+    /** Critical section for serializing cancellation (pSSM). */
+    RTCRITSECT              CancelCritSect;
+    /** The handle of the current save or load operation.
+     * This is used by SSMR3Cancel.  */
+    PSSMHANDLE volatile     pSSM;
+
+    /** FIFO of data entity descriptors. */
+    R3PTRTYPE(PSSMUNIT)     pHead;
+    /** The number of register units. */
+    uint32_t                cUnits;
+    /** For lazy init. */
+    bool                    fInitialized;
+    /** Current pass (for STAM). */
+    uint32_t                uPass;
+    uint32_t                u32Alignment;
+} SSM;
+/** Pointer to SSM VM instance data. */
+typedef SSM *PSSM;
+
+
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif /* !___SSMInternal_h */
+
Index: /trunk/src/VBox/VMM/include/STAMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/STAMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/STAMInternal.h	(revision 35333)
@@ -0,0 +1,125 @@
+/* $Id$ */
+/** @file
+ * STAM Internal Header.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___STAMInternal_h
+#define ___STAMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/stam.h>
+#include <VBox/gvmm.h>
+#include <iprt/semaphore.h>
+
+
+
+RT_C_DECLS_BEGIN
+
+/** @defgroup grp_stam_int   Internals
+ * @ingroup grp_stam
+ * @internal
+ * @{
+ */
+
+/**
+ * Sample descriptor.
+ */
+typedef struct STAMDESC
+{
+    /** Pointer to the next sample. */
+    struct STAMDESC    *pNext;
+    /** Sample name. */
+    const char         *pszName;
+    /** Sample type. */
+    STAMTYPE            enmType;
+    /** Visibility type. */
+    STAMVISIBILITY      enmVisibility;
+    /** Pointer to the sample data. */
+    union STAMDESCSAMPLEDATA
+    {
+        /** Counter. */
+        PSTAMCOUNTER    pCounter;
+        /** Profile. */
+        PSTAMPROFILE    pProfile;
+        /** Advanced profile. */
+        PSTAMPROFILEADV pProfileAdv;
+        /** Ratio, unsigned 32-bit. */
+        PSTAMRATIOU32   pRatioU32;
+        /** unsigned 8-bit. */
+        uint8_t        *pu8;
+        /** unsigned 16-bit. */
+        uint16_t       *pu16;
+        /** unsigned 32-bit. */
+        uint32_t       *pu32;
+        /** unsigned 64-bit. */
+        uint64_t       *pu64;
+        /** Simple void pointer. */
+        void           *pv;
+        /** */
+        struct STAMDESCSAMPLEDATACALLBACKS
+        {
+            /** The same pointer. */
+            void                   *pvSample;
+            /** Pointer to the reset callback. */
+            PFNSTAMR3CALLBACKRESET  pfnReset;
+            /** Pointer to the print callback. */
+            PFNSTAMR3CALLBACKPRINT  pfnPrint;
+        }               Callback;
+    }                   u;
+    /** Unit. */
+    STAMUNIT            enmUnit;
+    /** Description. */
+    const char         *pszDesc;
+} STAMDESC;
+/** Pointer to sample descriptor. */
+typedef STAMDESC        *PSTAMDESC;
+/** Pointer to const sample descriptor. */
+typedef const STAMDESC  *PCSTAMDESC;
+
+
+/**
+ * STAM data kept in the UVM.
+ */
+typedef struct STAMUSERPERVM
+{
+    /** Pointer to the first sample. */
+    R3PTRTYPE(PSTAMDESC)    pHead;
+    /** RW Lock for the list. */
+    RTSEMRW                 RWSem;
+
+    /** The copy of the GVMM statistics. */
+    GVMMSTATS               GVMMStats;
+    /** The number of registered host CPU leaves. */
+    uint32_t                cRegisteredHostCpus;
+} STAMUSERPERVM;
+/** Pointer to the STAM data kept in the UVM. */
+typedef STAMUSERPERVM *PSTAMUSERPERVM;
+
+
+/** Locks the sample descriptors for reading. */
+#define STAM_LOCK_RD(pUVM)      do { int rcSem = RTSemRWRequestRead(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT);  AssertRC(rcSem); } while (0)
+/** Locks the sample descriptors for writing. */
+#define STAM_LOCK_WR(pUVM)      do { int rcSem = RTSemRWRequestWrite(pUVM->stam.s.RWSem, RT_INDEFINITE_WAIT); AssertRC(rcSem); } while (0)
+/** UnLocks the sample descriptors after reading. */
+#define STAM_UNLOCK_RD(pUVM)    do { int rcSem = RTSemRWReleaseRead(pUVM->stam.s.RWSem);  AssertRC(rcSem); } while (0)
+/** UnLocks the sample descriptors after writing. */
+#define STAM_UNLOCK_WR(pUVM)    do { int rcSem = RTSemRWReleaseWrite(pUVM->stam.s.RWSem); AssertRC(rcSem); } while (0)
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif
Index: /trunk/src/VBox/VMM/include/TMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/TMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/TMInternal.h	(revision 35333)
@@ -0,0 +1,726 @@
+/* $Id$ */
+/** @file
+ * TM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___TMInternal_h
+#define ___TMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <iprt/time.h>
+#include <iprt/timer.h>
+#include <iprt/assert.h>
+#include <VBox/stam.h>
+#include <VBox/pdmcritsect.h>
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_tm_int       Internal
+ * @ingroup grp_tm
+ * @internal
+ * @{
+ */
+
+/** Frequency of the real clock. */
+#define TMCLOCK_FREQ_REAL       UINT32_C(1000)
+/** Frequency of the virtual clock. */
+#define TMCLOCK_FREQ_VIRTUAL    UINT32_C(1000000000)
+
+
+/**
+ * Timer type.
+ */
+typedef enum TMTIMERTYPE
+{
+    /** Device timer. */
+    TMTIMERTYPE_DEV = 1,
+    /** Driver timer. */
+    TMTIMERTYPE_DRV,
+    /** Internal timer . */
+    TMTIMERTYPE_INTERNAL,
+    /** External timer. */
+    TMTIMERTYPE_EXTERNAL
+} TMTIMERTYPE;
+
+/**
+ * Timer state
+ */
+typedef enum TMTIMERSTATE
+{
+    /** Timer is stopped. */
+    TMTIMERSTATE_STOPPED = 1,
+    /** Timer is active. */
+    TMTIMERSTATE_ACTIVE,
+    /** Timer is expired, getting expire and unlinking. */
+    TMTIMERSTATE_EXPIRED_GET_UNLINK,
+    /** Timer is expired and is being delivered. */
+    TMTIMERSTATE_EXPIRED_DELIVER,
+
+    /** Timer is stopped but still in the active list.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_STOP,
+    /** Timer is stopped but needs unlinking from the ScheduleTimers list.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_STOP_SCHEDULE,
+    /** Timer is being modified and will soon be pending scheduling.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE,
+    /** Timer is pending scheduling.
+     * Currently in the ScheduleTimers list. */
+    TMTIMERSTATE_PENDING_SCHEDULE,
+    /** Timer is being modified and will soon be pending rescheduling.
+     * Currently in the ScheduleTimers list and the active list. */
+    TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE,
+    /** Timer is modified and is now pending rescheduling.
+     * Currently in the ScheduleTimers list and the active list. */
+    TMTIMERSTATE_PENDING_RESCHEDULE,
+    /** Timer is being destroyed. */
+    TMTIMERSTATE_DESTROY,
+    /** Timer is free. */
+    TMTIMERSTATE_FREE
+} TMTIMERSTATE;
+
+/** Predicate that returns true if the give state is pending scheduling or
+ *  rescheduling of any kind. Will reference the argument more than once! */
+#define TMTIMERSTATE_IS_PENDING_SCHEDULING(enmState) \
+    (   (enmState) <= TMTIMERSTATE_PENDING_RESCHEDULE \
+     && (enmState) >= TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE)
+
+
+/**
+ * Internal representation of a timer.
+ *
+ * For correct serialization (without the use of semaphores and
+ * other blocking/slow constructs) certain rules applies to updating
+ * this structure:
+ *      - For thread other than EMT only u64Expire, enmState and pScheduleNext*
+ *        are changeable. Everything else is out of bounds.
+ *      - Updating of u64Expire timer can only happen in the TMTIMERSTATE_STOPPED
+ *        and TMTIMERSTATE_PENDING_RESCHEDULING_SET_EXPIRE states.
+ *      - Timers in the TMTIMERSTATE_EXPIRED state are only accessible from EMT.
+ *      - Actual destruction of a timer can only be done at scheduling time.
+ */
+typedef struct TMTIMER
+{
+    /** Expire time. */
+    volatile uint64_t       u64Expire;
+    /** Clock to apply to u64Expire. */
+    TMCLOCK                 enmClock;
+    /** Timer callback type. */
+    TMTIMERTYPE             enmType;
+    /** Type specific data. */
+    union
+    {
+        /** TMTIMERTYPE_DEV. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMERDEV)    pfnTimer;
+            /** Device instance. */
+            PPDMDEVINSR3                pDevIns;
+        } Dev;
+
+        /** TMTIMERTYPE_DRV. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMERDRV)    pfnTimer;
+            /** Device instance. */
+            R3PTRTYPE(PPDMDRVINS)       pDrvIns;
+        } Drv;
+
+        /** TMTIMERTYPE_INTERNAL. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMERINT)    pfnTimer;
+        } Internal;
+
+        /** TMTIMERTYPE_EXTERNAL. */
+        struct
+        {
+            /** Callback. */
+            R3PTRTYPE(PFNTMTIMEREXT)    pfnTimer;
+        } External;
+    } u;
+
+    /** Timer state. */
+    volatile TMTIMERSTATE   enmState;
+    /** Timer relative offset to the next timer in the schedule list. */
+    int32_t volatile        offScheduleNext;
+
+    /** Timer relative offset to the next timer in the chain. */
+    int32_t                 offNext;
+    /** Timer relative offset to the previous timer in the chain. */
+    int32_t                 offPrev;
+
+    /** Pointer to the VM the timer belongs to - R3 Ptr. */
+    PVMR3                   pVMR3;
+    /** Pointer to the VM the timer belongs to - R0 Ptr. */
+    PVMR0                   pVMR0;
+    /** Pointer to the VM the timer belongs to - RC Ptr. */
+    PVMRC                   pVMRC;
+    /** The timer frequency hint.  This is 0 if not hint was given. */
+    uint32_t volatile       uHzHint;
+
+    /** User argument. */
+    RTR3PTR                 pvUser;
+    /** The critical section associated with the lock. */
+    R3PTRTYPE(PPDMCRITSECT) pCritSect;
+
+    /** Pointer to the next timer in the list of created or free timers. (TM::pTimers or TM::pFree) */
+    PTMTIMERR3              pBigNext;
+    /** Pointer to the previous timer in the list of all created timers. (TM::pTimers) */
+    PTMTIMERR3              pBigPrev;
+    /** Pointer to the timer description. */
+    R3PTRTYPE(const char *) pszDesc;
+#if HC_ARCH_BITS == 32
+    uint32_t                padding0; /**< pad structure to multiple of 8 bytes. */
+#endif
+} TMTIMER;
+AssertCompileMemberSize(TMTIMER, enmState, sizeof(uint32_t));
+
+
+/**
+ * Updates a timer state in the correct atomic manner.
+ */
+#if 1
+# define TM_SET_STATE(pTimer, state) \
+    ASMAtomicWriteU32((uint32_t volatile *)&(pTimer)->enmState, state)
+#else
+# define TM_SET_STATE(pTimer, state) \
+    do { \
+        uint32_t uOld1 = (pTimer)->enmState; \
+        Log(("%s: %p: %d -> %d\n", __FUNCTION__, (pTimer), (pTimer)->enmState, state)); \
+        uint32_t uOld2 = ASMAtomicXchgU32((uint32_t volatile *)&(pTimer)->enmState, state); \
+        Assert(uOld1 == uOld2); \
+    } while (0)
+#endif
+
+/**
+ * Tries to updates a timer state in the correct atomic manner.
+ */
+#if 1
+# define TM_TRY_SET_STATE(pTimer, StateNew, StateOld, fRc) \
+    (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)&(pTimer)->enmState, StateNew, StateOld)
+#else
+# define TM_TRY_SET_STATE(pTimer, StateNew, StateOld, fRc) \
+    do { (fRc) = ASMAtomicCmpXchgU32((uint32_t volatile *)&(pTimer)->enmState, StateNew, StateOld); \
+         Log(("%s: %p: %d -> %d %RTbool\n", __FUNCTION__, (pTimer), StateOld, StateNew, fRc)); \
+    } while (0)
+#endif
+
+/** Get the previous timer. */
+#define TMTIMER_GET_PREV(pTimer) ((PTMTIMER)((pTimer)->offPrev ? (intptr_t)(pTimer) + (pTimer)->offPrev : 0))
+/** Get the next timer. */
+#define TMTIMER_GET_NEXT(pTimer) ((PTMTIMER)((pTimer)->offNext ? (intptr_t)(pTimer) + (pTimer)->offNext : 0))
+/** Set the previous timer link. */
+#define TMTIMER_SET_PREV(pTimer, pPrev) ((pTimer)->offPrev = (pPrev) ? (intptr_t)(pPrev) - (intptr_t)(pTimer) : 0)
+/** Set the next timer link. */
+#define TMTIMER_SET_NEXT(pTimer, pNext) ((pTimer)->offNext = (pNext) ? (intptr_t)(pNext) - (intptr_t)(pTimer) : 0)
+
+
+/**
+ * A timer queue.
+ *
+ * This is allocated on the hyper heap.
+ */
+typedef struct TMTIMERQUEUE
+{
+    /** The cached expire time for this queue.
+     * Updated by EMT when scheduling the queue or modifying the head timer.
+     * Assigned UINT64_MAX when there is no head timer. */
+    uint64_t                u64Expire;
+    /** Doubly linked list of active timers.
+     *
+     * When no scheduling is pending, this list is will be ordered by expire time (ascending).
+     * Access is serialized by only letting the emulation thread (EMT) do changes.
+     *
+     * The offset is relative to the queue structure.
+     */
+    int32_t                 offActive;
+    /** List of timers pending scheduling of some kind.
+     *
+     * Timer stats allowed in the list are TMTIMERSTATE_PENDING_STOPPING,
+     * TMTIMERSTATE_PENDING_DESTRUCTION, TMTIMERSTATE_PENDING_STOPPING_DESTRUCTION,
+     * TMTIMERSTATE_PENDING_RESCHEDULING and TMTIMERSTATE_PENDING_SCHEDULE.
+     *
+     * The offset is relative to the queue structure.
+     */
+    int32_t volatile        offSchedule;
+    /** The clock for this queue. */
+    TMCLOCK                 enmClock;
+    /** Pad the structure up to 32 bytes. */
+    uint32_t                au32Padding[3];
+} TMTIMERQUEUE;
+
+/** Pointer to a timer queue. */
+typedef TMTIMERQUEUE *PTMTIMERQUEUE;
+
+/** Get the head of the active timer list. */
+#define TMTIMER_GET_HEAD(pQueue)        ((PTMTIMER)((pQueue)->offActive ? (intptr_t)(pQueue) + (pQueue)->offActive : 0))
+/** Set the head of the active timer list. */
+#define TMTIMER_SET_HEAD(pQueue, pHead) ((pQueue)->offActive = pHead ? (intptr_t)pHead - (intptr_t)(pQueue) : 0)
+
+
+/**
+ * CPU load data set.
+ * Mainly used by tmR3CpuLoadTimer.
+ */
+typedef struct TMCPULOADSTATE
+{
+    /** The percent of the period spent executing guest code. */
+    uint8_t                 cPctExecuting;
+    /** The percent of the period spent halted. */
+    uint8_t                 cPctHalted;
+    /** The percent of the period spent on other things. */
+    uint8_t                 cPctOther;
+    /** Explicit alignment padding */
+    uint8_t                 au8Alignment[5];
+
+    /** Previous cNsTotal value. */
+    uint64_t                cNsPrevTotal;
+    /** Previous cNsExecuting value. */
+    uint64_t                cNsPrevExecuting;
+    /** Previous cNsHalted value. */
+    uint64_t                cNsPrevHalted;
+} TMCPULOADSTATE;
+AssertCompileSizeAlignment(TMCPULOADSTATE, 8);
+AssertCompileMemberAlignment(TMCPULOADSTATE, cNsPrevTotal, 8);
+/** Pointer to a CPU load data set. */
+typedef TMCPULOADSTATE *PTMCPULOADSTATE;
+
+/**
+ * Converts a TM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the TM is part of.
+ * @param   pTM   Pointer to TM instance data.
+ */
+#define TM2VM(pTM)  ( (PVM)((char*)pTM - pTM->offVM) )
+
+
+/**
+ * TM VM Instance data.
+ * Changes to this must checked against the padding of the cfgm union in VM!
+ */
+typedef struct TM
+{
+    /** Offset to the VM structure.
+     * See TM2VM(). */
+    RTUINT                      offVM;
+
+    /** Set if we fully virtualize the TSC, i.e. intercept all rdtsc instructions.
+     * Config variable: TSCVirtualized (bool) */
+    bool                        fTSCVirtualized;
+    /** Set if we use the real TSC as time source or if we use the virtual clock.
+     * If fTSCVirtualized is set we maintain a offset to the TSC and pausing/resuming the
+     * ticking. fTSCVirtualized = false implies fTSCUseRealTSC = true.
+     * Config variable: TSCUseRealTSC (bool) */
+    bool                        fTSCUseRealTSC;
+    /** Flag indicating that the host TSC is suitable for use in AMD-V and VT-x mode.
+     * Config variable: MaybeUseOffsettedHostTSC (boolean) */
+    bool                        fMaybeUseOffsettedHostTSC;
+    /** Whether the TSC is tied to the execution of code.
+     * Config variable: TSCTiedToExecution (bool) */
+    bool                        fTSCTiedToExecution;
+    /** Modifier for fTSCTiedToExecution which pauses the TSC while halting if true.
+     * Config variable: TSCNotTiedToHalt (bool) */
+    bool                        fTSCNotTiedToHalt;
+    bool                        afAlignment0[2]; /**< alignment padding */
+    /** The ID of the virtual CPU that normally runs the timers. */
+    VMCPUID                     idTimerCpu;
+    /** The number of CPU clock ticks per second (TMCLOCK_TSC).
+     * Config variable: TSCTicksPerSecond (64-bit unsigned int)
+     * The config variable implies fTSCVirtualized = true and fTSCUseRealTSC = false. */
+    uint64_t                    cTSCTicksPerSecond;
+
+    /** Virtual time ticking enabled indicator (counter for each VCPU). (TMCLOCK_VIRTUAL) */
+    uint32_t volatile           cVirtualTicking;
+    /** Virtual time is not running at 100%. */
+    bool                        fVirtualWarpDrive;
+    /** Virtual timer synchronous time ticking enabled indicator (bool). (TMCLOCK_VIRTUAL_SYNC) */
+    bool volatile               fVirtualSyncTicking;
+    /** Virtual timer synchronous time catch-up active. */
+    bool volatile               fVirtualSyncCatchUp;
+    bool                        afAlignment1[5]; /**< alignment padding */
+    /** WarpDrive percentage.
+     * 100% is normal (fVirtualSyncNormal == true). When other than 100% we apply
+     * this percentage to the raw time source for the period it's been valid in,
+     * i.e. since u64VirtualWarpDriveStart. */
+    uint32_t                    u32VirtualWarpDrivePercentage;
+
+    /** The offset of the virtual clock relative to it's timesource.
+     * Only valid if fVirtualTicking is set. */
+    uint64_t                    u64VirtualOffset;
+    /** The guest virtual time when fVirtualTicking is cleared. */
+    uint64_t                    u64Virtual;
+    /** When the Warp drive was started or last adjusted.
+     * Only valid when fVirtualWarpDrive is set. */
+    uint64_t                    u64VirtualWarpDriveStart;
+    /** The previously returned nano TS.
+     * This handles TSC drift on SMP systems and expired interval.
+     * This is a valid range u64NanoTS to u64NanoTS + 1000000000 (ie. 1sec). */
+    uint64_t volatile           u64VirtualRawPrev;
+    /** The ring-3 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
+    RTTIMENANOTSDATAR3          VirtualGetRawDataR3;
+    /** The ring-0 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
+    RTTIMENANOTSDATAR0          VirtualGetRawDataR0;
+    /** The ring-0 data structure for the RTTimeNanoTS workers used by tmVirtualGetRawNanoTS. */
+    RTTIMENANOTSDATARC          VirtualGetRawDataRC;
+    /** Pointer to the ring-3 tmVirtualGetRawNanoTS worker function. */
+    R3PTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawR3;
+    /** Pointer to the ring-3 tmVirtualGetRawNanoTS worker function. */
+    R0PTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawR0;
+    /** Pointer to the ring-3 tmVirtualGetRawNanoTS worker function. */
+    RCPTRTYPE(PFNTIMENANOTSINTERNAL) pfnVirtualGetRawRC;
+    /** Alignment. */
+    RTRCPTR                     AlignmentRCPtr;
+    /** The guest virtual timer synchronous time when fVirtualSyncTicking is cleared. */
+    uint64_t volatile           u64VirtualSync;
+    /** The offset of the timer synchronous virtual clock (TMCLOCK_VIRTUAL_SYNC) relative
+     * to the virtual clock (TMCLOCK_VIRTUAL).
+     * (This is accessed by the timer thread and must be updated atomically.) */
+    uint64_t volatile           offVirtualSync;
+    /** The offset into offVirtualSync that's been irrevocably given up by failed catch-up attempts.
+     * Thus the current lag is offVirtualSync - offVirtualSyncGivenUp. */
+    uint64_t                    offVirtualSyncGivenUp;
+    /** The TMCLOCK_VIRTUAL at the previous TMVirtualGetSync call when catch-up is active. */
+    uint64_t volatile           u64VirtualSyncCatchUpPrev;
+    /** The current catch-up percentage. */
+    uint32_t volatile           u32VirtualSyncCatchUpPercentage;
+    /** How much slack when processing timers. */
+    uint32_t                    u32VirtualSyncScheduleSlack;
+    /** When to stop catch-up. */
+    uint64_t                    u64VirtualSyncCatchUpStopThreshold;
+    /** When to give up catch-up. */
+    uint64_t                    u64VirtualSyncCatchUpGiveUpThreshold;
+/** @def TM_MAX_CATCHUP_PERIODS
+ * The number of catchup rates. */
+#define TM_MAX_CATCHUP_PERIODS  10
+    /** The aggressiveness of the catch-up relative to how far we've lagged behind.
+     * The idea is to have increasing catch-up percentage as the lag increases. */
+    struct TMCATCHUPPERIOD
+    {
+        uint64_t                u64Start;       /**< When this period starts. (u64VirtualSyncOffset). */
+        uint32_t                u32Percentage;  /**< The catch-up percent to apply. */
+        uint32_t                u32Alignment;   /**< Structure alignment */
+    }                           aVirtualSyncCatchUpPeriods[TM_MAX_CATCHUP_PERIODS];
+
+    /** The current max timer Hz hint. */
+    uint32_t volatile           uMaxHzHint;
+    /** Whether to recalulate the HzHint next time its queried. */
+    bool volatile               fHzHintNeedsUpdating;
+    /** Alignment */
+    bool                        afAlignment2[3];
+    /** @cfgm{TM/HostHzMax, uint32_t, Hz, 0, UINT32_MAX, 20000}
+     * The max host Hz frequency hint returned by TMCalcHostTimerFrequency.  */
+    uint32_t                    cHostHzMax;
+    /** @cfgm{TM/HostHzFudgeFactorTimerCpu, uint32_t, Hz, 0, UINT32_MAX, 111}
+     * The number of Hz TMCalcHostTimerFrequency adds for the timer CPU.  */
+    uint32_t                    cPctHostHzFudgeFactorTimerCpu;
+    /** @cfgm{TM/HostHzFudgeFactorOtherCpu, uint32_t, Hz, 0, UINT32_MAX, 110}
+     * The number of Hz TMCalcHostTimerFrequency adds for the other CPUs. */
+    uint32_t                    cPctHostHzFudgeFactorOtherCpu;
+    /** @cfgm{TM/HostHzFudgeFactorCatchUp100, uint32_t, Hz, 0, UINT32_MAX, 300}
+     *  The fudge factor (expressed in percent) that catch-up percentages below
+     * 100% is multiplied by. */
+    uint32_t                    cPctHostHzFudgeFactorCatchUp100;
+    /** @cfgm{TM/HostHzFudgeFactorCatchUp200, uint32_t, Hz, 0, UINT32_MAX, 250}
+     * The fudge factor (expressed in percent) that catch-up percentages
+     * 100%-199% is multiplied by. */
+    uint32_t                    cPctHostHzFudgeFactorCatchUp200;
+    /** @cfgm{TM/HostHzFudgeFactorCatchUp400, uint32_t, Hz, 0, UINT32_MAX, 200}
+     * The fudge factor (expressed in percent) that catch-up percentages
+     * 200%-399% is multiplied by. */
+    uint32_t                    cPctHostHzFudgeFactorCatchUp400;
+
+    /** The UTC offset in ns.
+     * This is *NOT* for converting UTC to local time. It is for converting real
+     * world UTC time to VM UTC time. This feature is indented for doing date
+     * testing of software and similar.
+     * @todo Implement warpdrive on UTC. */
+    int64_t                     offUTC;
+
+    /** Timer queues for the different clock types - R3 Ptr */
+    R3PTRTYPE(PTMTIMERQUEUE)    paTimerQueuesR3;
+    /** Timer queues for the different clock types - R0 Ptr */
+    R0PTRTYPE(PTMTIMERQUEUE)    paTimerQueuesR0;
+    /** Timer queues for the different clock types - RC Ptr */
+    RCPTRTYPE(PTMTIMERQUEUE)    paTimerQueuesRC;
+
+    /** Pointer to our RC mapping of the GIP. */
+    RCPTRTYPE(void *)           pvGIPRC;
+    /** Pointer to our R3 mapping of the GIP. */
+    R3PTRTYPE(void *)           pvGIPR3;
+
+    /** Pointer to a singly linked list of free timers.
+     * This chain is using the TMTIMER::pBigNext members.
+     * Only accessible from the emulation thread. */
+    PTMTIMERR3                  pFree;
+
+    /** Pointer to a doubly linked list of created timers.
+     * This chain is using the TMTIMER::pBigNext and TMTIMER::pBigPrev members.
+     * Only accessible from the emulation thread. */
+    PTMTIMERR3                  pCreated;
+
+    /** The schedule timer timer handle (runtime timer).
+     * This timer will do frequent check on pending queue schedules and
+     * raise VM_FF_TIMER to pull EMTs attention to them.
+     */
+    R3PTRTYPE(PRTTIMER)         pTimer;
+    /** Interval in milliseconds of the pTimer timer. */
+    uint32_t                    u32TimerMillies;
+
+    /** Indicates that queues are being run. */
+    bool volatile               fRunningQueues;
+    /** Indicates that the virtual sync queue is being run. */
+    bool volatile               fRunningVirtualSyncQueue;
+    /** Alignment */
+    bool                        afAlignment3[2];
+
+    /** Lock serializing access to the timer lists. */
+    PDMCRITSECT                 TimerCritSect;
+    /** Lock serializing access to the VirtualSync clock. */
+    PDMCRITSECT                 VirtualSyncLock;
+
+    /** CPU load state for all the virtual CPUs (tmR3CpuLoadTimer). */
+    TMCPULOADSTATE              CpuLoad;
+
+    /** TMR3TimerQueuesDo
+     * @{ */
+    STAMPROFILE                 StatDoQueues;
+    STAMPROFILEADV              aStatDoQueues[TMCLOCK_MAX];
+    /** @} */
+    /** tmSchedule
+     * @{ */
+    STAMPROFILE                 StatScheduleOneRZ;
+    STAMPROFILE                 StatScheduleOneR3;
+    STAMCOUNTER                 StatScheduleSetFF;
+    STAMCOUNTER                 StatPostponedR3;
+    STAMCOUNTER                 StatPostponedRZ;
+    /** @} */
+    /** Read the time
+     * @{ */
+    STAMCOUNTER                 StatVirtualGet;
+    STAMCOUNTER                 StatVirtualGetSetFF;
+    STAMCOUNTER                 StatVirtualSyncGet;
+    STAMCOUNTER                 StatVirtualSyncGetELoop;
+    STAMCOUNTER                 StatVirtualSyncGetExpired;
+    STAMCOUNTER                 StatVirtualSyncGetLockless;
+    STAMCOUNTER                 StatVirtualSyncGetLocked;
+    STAMCOUNTER                 StatVirtualSyncGetSetFF;
+    STAMCOUNTER                 StatVirtualPause;
+    STAMCOUNTER                 StatVirtualResume;
+    /* @} */
+    /** TMTimerPoll
+     * @{ */
+    STAMCOUNTER                 StatPoll;
+    STAMCOUNTER                 StatPollAlreadySet;
+    STAMCOUNTER                 StatPollELoop;
+    STAMCOUNTER                 StatPollMiss;
+    STAMCOUNTER                 StatPollRunning;
+    STAMCOUNTER                 StatPollSimple;
+    STAMCOUNTER                 StatPollVirtual;
+    STAMCOUNTER                 StatPollVirtualSync;
+    /** @} */
+    /** TMTimerSet
+     * @{ */
+    STAMCOUNTER                 StatTimerSet;
+    STAMCOUNTER                 StatTimerSetOpt;
+    STAMPROFILE                 StatTimerSetRZ;
+    STAMPROFILE                 StatTimerSetR3;
+    STAMCOUNTER                 StatTimerSetStStopped;
+    STAMCOUNTER                 StatTimerSetStExpDeliver;
+    STAMCOUNTER                 StatTimerSetStActive;
+    STAMCOUNTER                 StatTimerSetStPendStop;
+    STAMCOUNTER                 StatTimerSetStPendStopSched;
+    STAMCOUNTER                 StatTimerSetStPendSched;
+    STAMCOUNTER                 StatTimerSetStPendResched;
+    STAMCOUNTER                 StatTimerSetStOther;
+    /** @} */
+    /** TMTimerSetRelative
+     * @{ */
+    STAMCOUNTER                 StatTimerSetRelative;
+    STAMPROFILE                 StatTimerSetRelativeRZ;
+    STAMPROFILE                 StatTimerSetRelativeR3;
+    STAMCOUNTER                 StatTimerSetRelativeOpt;
+    STAMCOUNTER                 StatTimerSetRelativeRacyVirtSync;
+    STAMCOUNTER                 StatTimerSetRelativeStStopped;
+    STAMCOUNTER                 StatTimerSetRelativeStExpDeliver;
+    STAMCOUNTER                 StatTimerSetRelativeStActive;
+    STAMCOUNTER                 StatTimerSetRelativeStPendStop;
+    STAMCOUNTER                 StatTimerSetRelativeStPendStopSched;
+    STAMCOUNTER                 StatTimerSetRelativeStPendSched;
+    STAMCOUNTER                 StatTimerSetRelativeStPendResched;
+    STAMCOUNTER                 StatTimerSetRelativeStOther;
+    /** @} */
+    /** TMTimerStop
+     * @{ */
+    STAMPROFILE                 StatTimerStopRZ;
+    STAMPROFILE                 StatTimerStopR3;
+    /** @} */
+    /** VirtualSync - Running and Catching Up
+     * @{ */
+    STAMCOUNTER                 StatVirtualSyncRun;
+    STAMCOUNTER                 StatVirtualSyncRunRestart;
+    STAMPROFILE                 StatVirtualSyncRunSlack;
+    STAMCOUNTER                 StatVirtualSyncRunStop;
+    STAMCOUNTER                 StatVirtualSyncRunStoppedAlready;
+    STAMCOUNTER                 StatVirtualSyncGiveUp;
+    STAMCOUNTER                 StatVirtualSyncGiveUpBeforeStarting;
+    STAMPROFILEADV              StatVirtualSyncCatchup;
+    STAMCOUNTER                 aStatVirtualSyncCatchupInitial[TM_MAX_CATCHUP_PERIODS];
+    STAMCOUNTER                 aStatVirtualSyncCatchupAdjust[TM_MAX_CATCHUP_PERIODS];
+    /** @} */
+    /** TMR3VirtualSyncFF (non dedicated EMT). */
+    STAMPROFILE                 StatVirtualSyncFF;
+    /** The timer callback. */
+    STAMCOUNTER                 StatTimerCallbackSetFF;
+
+    /** Calls to TMCpuTickSet. */
+    STAMCOUNTER                 StatTSCSet;
+
+    /** @name Reasons for refusing TSC offsetting in TMCpuTickCanUseRealTSC.
+     * @{ */
+    STAMCOUNTER                 StatTSCNotFixed;
+    STAMCOUNTER                 StatTSCNotTicking;
+    STAMCOUNTER                 StatTSCCatchupLE010;
+    STAMCOUNTER                 StatTSCCatchupLE025;
+    STAMCOUNTER                 StatTSCCatchupLE100;
+    STAMCOUNTER                 StatTSCCatchupOther;
+    STAMCOUNTER                 StatTSCWarp;
+    STAMCOUNTER                 StatTSCUnderflow;
+    STAMCOUNTER                 StatTSCSyncNotTicking;
+    /** @} */
+} TM;
+/** Pointer to TM VM instance data. */
+typedef TM *PTM;
+
+/**
+ * TM VMCPU Instance data.
+ * Changes to this must checked against the padding of the tm union in VM!
+ */
+typedef struct TMCPU
+{
+    /** Offset to the VMCPU structure.
+     * See TMCPU2VM(). */
+    RTUINT                      offVMCPU;
+
+    /** CPU timestamp ticking enabled indicator (bool). (RDTSC) */
+    bool                        fTSCTicking;
+    bool                        afAlignment0[3]; /**< alignment padding */
+
+    /** The offset between the raw TSC source and the Guest TSC.
+     * Only valid if fTicking is set and and fTSCUseRealTSC is clear. */
+    uint64_t                    offTSCRawSrc;
+
+    /** The guest TSC when fTicking is cleared. */
+    uint64_t                    u64TSC;
+
+    /** The last seen TSC by the guest. */
+    uint64_t                    u64TSCLastSeen;
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /** The nanosecond timestamp of the CPU start or resume.
+     * This is recalculated when the VM is started so that
+     * cNsTotal = RTTimeNanoTS() - u64NsTsStartCpu. */
+    uint64_t                    u64NsTsStartTotal;
+    /** The nanosecond timestamp of the last start-execute notification. */
+    uint64_t                    u64NsTsStartExecuting;
+    /** The nanosecond timestamp of the last start-halt notification. */
+    uint64_t                    u64NsTsStartHalting;
+    /** The cNsXXX generation. */
+    uint32_t volatile           uTimesGen;
+    /** Explicit alignment padding.  */
+    uint32_t                    u32Alignment;
+    /** The number of nanoseconds total run time.
+     * @remarks This is updated when cNsExecuting and cNsHalted are updated. */
+    uint64_t                    cNsTotal;
+    /** The number of nanoseconds spent executing. */
+    uint64_t                    cNsExecuting;
+    /** The number of nanoseconds being halted. */
+    uint64_t                    cNsHalted;
+    /** The number of nanoseconds spent on other things.
+     * @remarks This is updated when cNsExecuting and cNsHalted are updated. */
+    uint64_t                    cNsOther;
+    /** The number of halts. */
+    uint64_t                    cPeriodsHalted;
+    /** The number of guest execution runs. */
+    uint64_t                    cPeriodsExecuting;
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+    /** Resettable version of cNsTotal. */
+    STAMCOUNTER                 StatNsTotal;
+    /** Resettable version of cNsExecuting. */
+    STAMPROFILE                 StatNsExecuting;
+    /** Long execution intervals. */
+    STAMPROFILE                 StatNsExecLong;
+    /** Short execution intervals . */
+    STAMPROFILE                 StatNsExecShort;
+    /** Tiny execution intervals . */
+    STAMPROFILE                 StatNsExecTiny;
+    /** Resettable version of cNsHalted. */
+    STAMPROFILE                 StatNsHalted;
+    /** Resettable version of cNsOther. */
+    STAMPROFILE                 StatNsOther;
+# endif
+
+    /** CPU load state for this virtual CPU (tmR3CpuLoadTimer). */
+    TMCPULOADSTATE              CpuLoad;
+#endif
+} TMCPU;
+/** Pointer to TM VMCPU instance data. */
+typedef TMCPU *PTMCPU;
+
+#if 0 /* enable this to rule out locking bugs on single cpu guests. */
+# define tmTimerLock(pVM)                VINF_SUCCESS
+# define tmTimerTryLock(pVM)             VINF_SUCCESS
+# define tmTimerUnlock(pVM)              ((void)0)
+# define tmVirtualSyncLock(pVM)     VINF_SUCCESS
+# define tmVirtualSyncTryLock(pVM)  VINF_SUCCESS
+# define tmVirtualSyncUnlock(pVM)   ((void)0)
+# define TM_ASSERT_LOCK(pVM)        VM_ASSERT_EMT(pVM)
+#else
+int                     tmTimerLock(PVM pVM);
+int                     tmTimerTryLock(PVM pVM);
+void                    tmTimerUnlock(PVM pVM);
+/** Checks that the caller owns the timer lock.  */
+#define TM_ASSERT_LOCK(pVM) Assert(PDMCritSectIsOwner(&pVM->tm.s.TimerCritSect))
+int                     tmVirtualSyncLock(PVM pVM);
+int                     tmVirtualSyncTryLock(PVM pVM);
+void                    tmVirtualSyncUnlock(PVM pVM);
+#endif
+
+const char             *tmTimerState(TMTIMERSTATE enmState);
+void                    tmTimerQueueSchedule(PVM pVM, PTMTIMERQUEUE pQueue);
+#ifdef VBOX_STRICT
+void                    tmTimerQueuesSanityChecks(PVM pVM, const char *pszWhere);
+#endif
+
+int                     tmCpuTickPause(PVM pVM, PVMCPU pVCpu);
+int                     tmCpuTickResume(PVM pVM, PVMCPU pVCpu);
+
+int                     tmVirtualPauseLocked(PVM pVM);
+int                     tmVirtualResumeLocked(PVM pVM);
+DECLEXPORT(void)        tmVirtualNanoTSBad(PRTTIMENANOTSDATA pData, uint64_t u64NanoTS, uint64_t u64DeltaPrev, uint64_t u64PrevNanoTS);
+DECLEXPORT(uint64_t)    tmVirtualNanoTSRediscover(PRTTIMENANOTSDATA pData);
+
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/TRPMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/TRPMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/TRPMInternal.h	(revision 35333)
@@ -0,0 +1,287 @@
+/* $Id$ */
+/** @file
+ * TRPM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___TRPMInternal_h
+#define ___TRPMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/types.h>
+#include <VBox/stam.h>
+#include <VBox/cpum.h>
+
+
+
+/* Enable to allow trap forwarding in GC. */
+#define TRPM_FORWARD_TRAPS_IN_GC
+
+/** First interrupt handler. Used for validating input. */
+#define TRPM_HANDLER_INT_BASE  0x20
+
+RT_C_DECLS_BEGIN
+
+
+/** @defgroup grp_trpm_int   Internals
+ * @ingroup grp_trpm
+ * @internal
+ * @{
+ */
+
+/** @name   TRPMGCTrapIn* flags.
+ * The lower bits are offsets into the CPUMCTXCORE structure.
+ * @{ */
+/** The mask for the operation. */
+#define TRPM_TRAP_IN_OP_MASK    0xffff
+/** Traps on MOV GS, eax. */
+#define TRPM_TRAP_IN_MOV_GS     1
+/** Traps on MOV FS, eax. */
+#define TRPM_TRAP_IN_MOV_FS     2
+/** Traps on MOV ES, eax. */
+#define TRPM_TRAP_IN_MOV_ES     3
+/** Traps on MOV DS, eax. */
+#define TRPM_TRAP_IN_MOV_DS     4
+/** Traps on IRET. */
+#define TRPM_TRAP_IN_IRET       5
+/** Set if this is a V86 resume. */
+#define TRPM_TRAP_IN_V86        RT_BIT(30)
+/** If set this is a hypervisor register set. If cleared it's a guest set. */
+#define TRPM_TRAP_IN_HYPER      RT_BIT(31)
+/** @} */
+
+
+#if 0 /* not used */
+/**
+ * Converts a TRPM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the TRPM is part of.
+ * @param   pTRPM   Pointer to TRPM instance data.
+ */
+#define TRPM_2_VM(pTRPM)            ( (PVM)((uint8_t *)(pTRPM) - (pTRPM)->offVM) )
+#endif
+
+/**
+ * Converts a TRPM pointer into a TRPMCPU pointer.
+ * @returns Pointer to the VM structure the TRPMCPU is part of.
+ * @param   pTRPM   Pointer to TRPMCPU instance data.
+ * @remarks Raw-mode only, not SMP safe.
+ */
+#define TRPM_2_TRPMCPU(pTrpmCpu)     ( (PTRPMCPU)((uint8_t *)(pTrpmCpu) + (pTrpmCpu)->offTRPMCPU) )
+
+
+/**
+ * TRPM Data (part of VM)
+ *
+ * IMPORTANT! Keep the nasm version of this struct up-to-date.
+ */
+#pragma pack(4)
+typedef struct TRPM
+{
+    /** Offset to the VM structure.
+     * See TRPM_2_VM(). */
+    RTINT                   offVM;
+    /** Offset to the TRPMCPU structure.
+     * See TRPM2TRPMCPU(). */
+    RTINT                   offTRPMCPU;
+
+    /** IDT monitoring and sync flag (HWACC). */
+    bool                    fDisableMonitoring; /** @todo r=bird: bool and 7 byte achPadding1. */
+
+    /** Whether monitoring of the guest IDT is enabled or not.
+     *
+     * This configuration option is provided for speeding up guest like Solaris
+     * that put the IDT on the same page as a whole lot of other data that is
+     * frequently updated. The updates will cause #PFs and have to be interpreted
+     * by PGMInterpretInstruction which is slow compared to raw execution.
+     *
+     * If the guest is well behaved and doesn't change the IDT after loading it,
+     * there is no problem with dropping the IDT monitoring.
+     *
+     * @cfgm    /TRPM/SafeToDropGuestIDTMonitoring   boolean     defaults to false.
+     */
+    bool                    fSafeToDropGuestIDTMonitoring;
+
+    /** Padding to get the IDTs at a 16 byte alignment. */
+    uint8_t                 abPadding1[6];
+    /** IDTs. Aligned at 16 byte offset for speed. */
+    VBOXIDTE                aIdt[256];
+
+    /** Bitmap for IDTEs that contain PATM handlers. (needed for relocation) */
+    uint32_t                au32IdtPatched[8];
+
+    /** Temporary Hypervisor trap handlers.
+     * NULL means default action. */
+    RCPTRTYPE(void *)       aTmpTrapHandlers[256];
+
+    /** RC Pointer to the IDT shadow area (aIdt) in HMA. */
+    RCPTRTYPE(void *)       pvMonShwIdtRC;
+    /** Current (last) Guest's IDTR. */
+    VBOXIDTR                GuestIdtr;
+
+    /** padding. */
+    uint8_t                 au8Padding[2];
+
+    /** Checked trap & interrupt handler array */
+    RCPTRTYPE(void *)       aGuestTrapHandler[256];
+
+    /** RC: The number of times writes to the Guest IDT were detected. */
+    STAMCOUNTER             StatRCWriteGuestIDTFault;
+    STAMCOUNTER             StatRCWriteGuestIDTHandled;
+
+    /** HC: Profiling of the TRPMR3SyncIDT() method. */
+    STAMPROFILE             StatSyncIDT;
+    /** GC: Statistics for the trap handlers. */
+    STAMPROFILEADV          aStatGCTraps[0x14];
+
+    STAMPROFILEADV          StatForwardProfR3;
+    STAMPROFILEADV          StatForwardProfRZ;
+    STAMCOUNTER             StatForwardFailNoHandler;
+    STAMCOUNTER             StatForwardFailPatchAddr;
+    STAMCOUNTER             StatForwardFailR3;
+    STAMCOUNTER             StatForwardFailRZ;
+
+    STAMPROFILE             StatTrap0dDisasm;
+    STAMCOUNTER             StatTrap0dRdTsc;    /**< Number of RDTSC #GPs. */
+
+#ifdef VBOX_WITH_STATISTICS
+    /* R3: Statistics for interrupt handlers (allocated on the hypervisor heap). */
+    R3PTRTYPE(PSTAMCOUNTER) paStatForwardedIRQR3;
+    /* R0: Statistics for interrupt handlers (allocated on the hypervisor heap). */
+    R0PTRTYPE(PSTAMCOUNTER) paStatForwardedIRQR0;
+    /* RC: Statistics for interrupt handlers (allocated on the hypervisor heap). */
+    RCPTRTYPE(PSTAMCOUNTER) paStatForwardedIRQRC;
+#endif
+} TRPM;
+
+/** Pointer to TRPM Data. */
+typedef TRPM *PTRPM;
+
+
+/**
+ * Converts a TRPMCPU pointer into a VM pointer.
+ * @returns Pointer to the VM structure the TRPMCPU is part of.
+ * @param   pTRPM   Pointer to TRPMCPU instance data.
+ */
+#define TRPMCPU_2_VM(pTrpmCpu)      ( (PVM)((uint8_t *)(pTrpmCpu) - (pTrpmCpu)->offVM) )
+
+/**
+ * Converts a TRPMCPU pointer into a VMCPU pointer.
+ * @returns Pointer to the VMCPU structure the TRPMCPU is part of.
+ * @param   pTRPM   Pointer to TRPMCPU instance data.
+ */
+#define TRPMCPU_2_VMCPU(pTrpmCpu)   ( (PVMCPU)((uint8_t *)(pTrpmCpu) - (pTrpmCpu)->offVMCpu) )
+
+
+/**
+ * Per CPU data for TRPM.
+ */
+typedef struct TRPMCPU
+{
+    /** Offset into the VM structure.
+     * See TRPMCPU_2_VM(). */
+    uint32_t                offVM;
+    /** Offset into the VMCPU structure.
+     * See TRPMCPU_2_VMCPU().  */
+    uint32_t                offVMCpu;
+
+    /** Active Interrupt or trap vector number.
+     * If not ~0U this indicates that we're currently processing
+     * a interrupt, trap, fault, abort, whatever which have arrived
+     * at that vector number.
+     */
+    RTUINT                  uActiveVector;
+
+    /** Active trap type. */
+    TRPMEVENT               enmActiveType;
+
+    /** Errorcode for the active interrupt/trap. */
+    RTGCUINT                uActiveErrorCode; /**< @todo don't use RTGCUINT */
+
+    /** CR2 at the time of the active exception. */
+    RTGCUINTPTR             uActiveCR2;
+
+    /** Saved trap vector number. */
+    RTGCUINT                uSavedVector; /**< @todo don't use RTGCUINT */
+
+    /** Saved trap type. */
+    TRPMEVENT               enmSavedType;
+
+    /** Saved errorcode. */
+    RTGCUINT                uSavedErrorCode;
+
+    /** Saved cr2. */
+    RTGCUINTPTR             uSavedCR2;
+
+    /** Previous trap vector # - for debugging. */
+    RTGCUINT                uPrevVector;
+} TRPMCPU;
+
+/** Pointer to TRPMCPU Data. */
+typedef TRPMCPU *PTRPMCPU;
+
+#pragma pack()
+
+
+VMMRCDECL(int) trpmRCGuestIDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+VMMRCDECL(int) trpmRCShadowIDTWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPTR pvRange, uintptr_t offRange);
+
+/**
+ * Clear guest trap/interrupt gate handler
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   iTrap       Interrupt/trap number.
+ */
+VMMDECL(int) trpmClearGuestTrapHandler(PVM pVM, unsigned iTrap);
+
+
+#ifdef IN_RING3
+
+/**
+ * Clear passthrough interrupt gate handler (reset to default handler)
+ *
+ * @returns VBox status code.
+ * @param   pVM         The VM to operate on.
+ * @param   iTrap       Trap/interrupt gate number.
+ */
+VMMR3DECL(int) trpmR3ClearPassThroughHandler(PVM pVM, unsigned iTrap);
+
+#endif
+
+
+#ifdef IN_RING0
+
+/**
+ * Calls the interrupt gate as if we received an interrupt while in Ring-0.
+ *
+ * @param   uIP     The interrupt gate IP.
+ * @param   SelCS   The interrupt gate CS.
+ * @param   RSP     The interrupt gate RSP. ~0 if no stack switch should take place. (only AMD64)
+ */
+DECLASM(void) trpmR0DispatchHostInterrupt(RTR0UINTPTR uIP, RTSEL SelCS, RTR0UINTPTR RSP);
+
+/**
+ * Issues a software interrupt to the specified interrupt vector.
+ *
+ * @param   uActiveVector   The vector number.
+ */
+DECLASM(void) trpmR0DispatchHostInterruptSimple(RTUINT uActiveVector);
+
+#endif /* IN_RING0 */
+
+/** @} */
+
+RT_C_DECLS_END
+
+#endif
Index: /trunk/src/VBox/VMM/include/TRPMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/include/TRPMInternal.mac	(revision 35333)
+++ /trunk/src/VBox/VMM/include/TRPMInternal.mac	(revision 35333)
@@ -0,0 +1,123 @@
+; $Id$
+;; @file
+; TRPM - Internal header file.
+;
+
+;
+; Copyright (C) 2006-2007 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%include "VBox/asmdefs.mac"
+%include "VBox/stam.mac"
+
+%define TRPM_TRAP_IN_OP_MASK    0xffff
+%define TRPM_TRAP_IN_MOV_GS     1
+%define TRPM_TRAP_IN_MOV_FS     2
+%define TRPM_TRAP_IN_MOV_ES     3
+%define TRPM_TRAP_IN_MOV_DS     4
+%define TRPM_TRAP_IN_IRET       5
+%define TRPM_TRAP_IN_V86        RT_BIT(30)
+%define TRPM_TRAP_IN_HYPER      RT_BIT(31)
+
+struc TRPM
+    .offVM              resd 1
+    .offTRPMCPU         resd 1
+    .fDisableMonitoring resb 1
+    .fSafeToDropGuestIDTMonitoring resb 1
+    .abPadding1         resb 6
+    .aIdt               resd 512
+    .au32IdtPatched     resd 8
+    .aTmpTrapHandlers   RTRCPTR_RES 256
+
+    .pvMonShwIdtRC            RTRCPTR_RES 1
+    .GuestIdtr          resb 10
+    .au8Padding         resb 2
+
+    .aGuestTrapHandler  RTRCPTR_RES 256
+
+    .StatRCWriteGuestIDTFault  resb STAMCOUNTER_size
+    .StatRCWriteGuestIDTHandled  resb STAMCOUNTER_size
+    .StatSyncIDT        resb STAMPROFILE_size
+    .aStatGCTraps       resb STAMPROFILEADV_size * 0x14
+
+    .StatForwardProfR3        resb STAMPROFILEADV_size
+    .StatForwardProfRZ        resb STAMPROFILEADV_size
+    .StatForwardFailNoHandler resb STAMCOUNTER_size
+    .StatForwardFailPatchAddr resb STAMCOUNTER_size
+    .StatForwardFailR3        resb STAMCOUNTER_size
+    .StatForwardFailRZ        resb STAMCOUNTER_size
+
+    .StatTrap0dDisasm         resb STAMPROFILE_size
+    .StatTrap0dRdTsc          resb STAMCOUNTER_size
+
+%ifdef VBOX_WITH_STATISTICS
+    .paStatForwardedIRQR3  RTR3PTR_RES 1
+    .paStatForwardedIRQR0  RTR0PTR_RES 1
+    .paStatForwardedIRQRC  RTRCPTR_RES 1
+%endif
+endstruc
+
+struc TRPMCPU
+    .offVM              resd 1
+    .offVMCpu           resd 1
+    .uActiveVector      resd 1
+    .enmActiveType      resd 1
+    .uActiveErrorCode   RTGCPTR_RES 1
+    .uActiveCR2         RTGCPTR_RES 1
+    .uSavedVector       RTGCPTR_RES 1
+    .enmSavedType       resd 1
+    .uSavedErrorCode    RTGCPTR_RES 1
+    .uSavedCR2          RTGCPTR_RES 1
+    .uPrevVector        RTGCPTR_RES 1
+endstruc
+
+struc VBOXTSS
+    .selPrev            resw 1
+    .padding1           resw 1
+    .esp0               resd 1
+    .ss0                resw 1
+    .padding_ss0        resw 1
+    .esp1               resd 1
+    .ss1                resw 1
+    .padding_ss1        resw 1
+    .esp2               resd 1
+    .ss2                resw 1
+    .padding_ss2        resw 1
+    .cr3                resd 1
+    .eip                resd 1
+    .eflags             resd 1
+    .eax                resd 1
+    .ecx                resd 1
+    .edx                resd 1
+    .ebx                resd 1
+    .esp                resd 1
+    .ebp                resd 1
+    .esi                resd 1
+    .edi                resd 1
+    .es                 resw 1
+    .padding_es         resw 1
+    .cs                 resw 1
+    .padding_cs         resw 1
+    .ss                 resw 1
+    .padding_ss         resw 1
+    .ds                 resw 1
+    .padding_ds         resw 1
+    .fs                 resw 1
+    .padding_fs         resw 1
+    .gs                 resw 1
+    .padding_gs         resw 1
+    .selLdt             resw 1
+    .padding_ldt        resw 1
+    .fDebugTrap         resw 1
+    .offIoBitmap        resw 1
+    .IntRedirBitmap     resb 32
+endstruc
+
Index: /trunk/src/VBox/VMM/include/VMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/VMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/VMInternal.h	(revision 35333)
@@ -0,0 +1,444 @@
+/* $Id$ */
+/** @file
+ * VM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___VMInternal_h
+#define ___VMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/vmapi.h>
+#include <iprt/assert.h>
+#include <iprt/critsect.h>
+#include <setjmp.h>
+
+
+
+/** @defgroup grp_vm_int   Internals
+ * @ingroup grp_vm
+ * @internal
+ * @{
+ */
+
+
+/**
+ * VM state change callback.
+ */
+typedef struct VMATSTATE
+{
+    /** Pointer to the next one. */
+    struct VMATSTATE               *pNext;
+    /** Pointer to the callback. */
+    PFNVMATSTATE                    pfnAtState;
+    /** The user argument. */
+    void                           *pvUser;
+} VMATSTATE;
+/** Pointer to a VM state change callback. */
+typedef VMATSTATE *PVMATSTATE;
+
+
+/**
+ * VM error callback.
+ */
+typedef struct VMATERROR
+{
+    /** Pointer to the next one. */
+    struct VMATERROR               *pNext;
+    /** Pointer to the callback. */
+    PFNVMATERROR                    pfnAtError;
+    /** The user argument. */
+    void                           *pvUser;
+} VMATERROR;
+/** Pointer to a VM error callback. */
+typedef VMATERROR *PVMATERROR;
+
+
+/**
+ * Chunk of memory allocated off the hypervisor heap in which
+ * we copy the error details.
+ */
+typedef struct VMERROR
+{
+    /** The size of the chunk. */
+    uint32_t                        cbAllocated;
+    /** The current offset into the chunk.
+     * We start by putting the filename and function immediately
+     * after the end of the buffer. */
+    uint32_t                        off;
+    /** Offset from the start of this structure to the file name. */
+    uint32_t                        offFile;
+    /** The line number. */
+    uint32_t                        iLine;
+    /** Offset from the start of this structure to the function name. */
+    uint32_t                        offFunction;
+    /** Offset from the start of this structure to the formatted message text. */
+    uint32_t                        offMessage;
+    /** The VBox status code. */
+    int32_t                         rc;
+} VMERROR, *PVMERROR;
+
+
+/**
+ * VM runtime error callback.
+ */
+typedef struct VMATRUNTIMEERROR
+{
+    /** Pointer to the next one. */
+    struct VMATRUNTIMEERROR         *pNext;
+    /** Pointer to the callback. */
+    PFNVMATRUNTIMEERROR              pfnAtRuntimeError;
+    /** The user argument. */
+    void                            *pvUser;
+} VMATRUNTIMEERROR;
+/** Pointer to a VM error callback. */
+typedef VMATRUNTIMEERROR *PVMATRUNTIMEERROR;
+
+
+/**
+ * Chunk of memory allocated off the hypervisor heap in which
+ * we copy the runtime error details.
+ */
+typedef struct VMRUNTIMEERROR
+{
+    /** The size of the chunk. */
+    uint32_t                        cbAllocated;
+    /** The current offset into the chunk.
+     * We start by putting the error ID immediately
+     * after the end of the buffer. */
+    uint32_t                        off;
+    /** Offset from the start of this structure to the error ID. */
+    uint32_t                        offErrorId;
+    /** Offset from the start of this structure to the formatted message text. */
+    uint32_t                        offMessage;
+    /** Error flags. */
+    uint32_t                        fFlags;
+} VMRUNTIMEERROR, *PVMRUNTIMEERROR;
+
+/** The halt method. */
+typedef enum
+{
+    /** The usual invalid value. */
+    VMHALTMETHOD_INVALID = 0,
+    /** Use the method used during bootstrapping. */
+    VMHALTMETHOD_BOOTSTRAP,
+    /** Use the default method. */
+    VMHALTMETHOD_DEFAULT,
+    /** The old spin/yield/block method. */
+    VMHALTMETHOD_OLD,
+    /** The first go at a block/spin method. */
+    VMHALTMETHOD_1,
+    /** The first go at a more global approach. */
+    VMHALTMETHOD_GLOBAL_1,
+    /** The end of valid methods. (not inclusive of course) */
+    VMHALTMETHOD_END,
+    /** The usual 32-bit max value. */
+    VMHALTMETHOD_32BIT_HACK = 0x7fffffff
+} VMHALTMETHOD;
+
+
+/**
+ * VM Internal Data (part of the VM structure).
+ *
+ * @todo Move this and all related things to VMM. The VM component was, to some
+ *       extent at least, a bad ad hoc design which should all have been put in
+ *       VMM. @see pg_vm.
+ */
+typedef struct VMINT
+{
+    /** VM Error Message. */
+    R3PTRTYPE(PVMERROR)             pErrorR3;
+    /** VM Runtime Error Message. */
+    R3PTRTYPE(PVMRUNTIMEERROR)      pRuntimeErrorR3;
+    /** The VM was/is-being teleported and has not yet been fully resumed. */
+    bool                            fTeleportedAndNotFullyResumedYet;
+} VMINT;
+/** Pointer to the VM Internal Data (part of the VM structure). */
+typedef VMINT *PVMINT;
+
+
+/**
+ * VM internal data kept in the UVM.
+ */
+typedef struct VMINTUSERPERVM
+{
+    /** Head of the request queue. Atomic. */
+    volatile PVMREQ                 pReqs;
+    /** The last index used during alloc/free. */
+    volatile uint32_t               iReqFree;
+    /** Number of free request packets. */
+    volatile uint32_t               cReqFree;
+    /** Array of pointers to lists of free request packets. Atomic. */
+    volatile PVMREQ                 apReqFree[9];
+
+#ifdef VBOX_WITH_STATISTICS
+    /** Number of VMR3ReqAlloc returning a new packet. */
+    STAMCOUNTER                     StatReqAllocNew;
+    /** Number of VMR3ReqAlloc causing races. */
+    STAMCOUNTER                     StatReqAllocRaces;
+    /** Number of VMR3ReqAlloc returning a recycled packet. */
+    STAMCOUNTER                     StatReqAllocRecycled;
+    /** Number of VMR3ReqFree calls. */
+    STAMCOUNTER                     StatReqFree;
+    /** Number of times the request was actually freed. */
+    STAMCOUNTER                     StatReqFreeOverflow;
+    /** Number of requests served. */
+    STAMCOUNTER                     StatReqProcessed;
+    /** Number of times there are more than one request and the others needed to be
+     * pushed back onto the list. */
+    STAMCOUNTER                     StatReqMoreThan1;
+    /** Number of times we've raced someone when pushing the other requests back
+     * onto the list. */
+    STAMCOUNTER                     StatReqPushBackRaces;
+#endif
+
+    /** Pointer to the support library session.
+     * Mainly for creation and destruction. */
+    PSUPDRVSESSION                  pSession;
+
+    /** Force EMT to terminate. */
+    bool volatile                   fTerminateEMT;
+
+    /** Critical section for pAtState and enmPrevVMState. */
+    RTCRITSECT                      AtStateCritSect;
+    /** List of registered state change callbacks. */
+    PVMATSTATE                      pAtState;
+    /** List of registered state change callbacks. */
+    PVMATSTATE                     *ppAtStateNext;
+    /** The previous VM state.
+     * This is mainly used for the 'Resetting' state, but may come in handy later
+     * and when debugging. */
+    VMSTATE                         enmPrevVMState;
+
+    /** Critical section for pAtError and pAtRuntimeError. */
+    RTCRITSECT                      AtErrorCritSect;
+
+    /** List of registered error callbacks. */
+    PVMATERROR                      pAtError;
+    /** List of registered error callbacks. */
+    PVMATERROR                     *ppAtErrorNext;
+    /** The error message count.
+     * This is incremented every time an error is raised.  */
+    uint32_t volatile               cErrors;
+
+    /** The runtime error message count.
+     * This is incremented every time a runtime error is raised.  */
+    uint32_t volatile               cRuntimeErrors;
+    /** List of registered error callbacks. */
+    PVMATRUNTIMEERROR               pAtRuntimeError;
+    /** List of registered error callbacks. */
+    PVMATRUNTIMEERROR              *ppAtRuntimeErrorNext;
+
+    /** @name Generic Halt data
+     * @{
+     */
+    /** The current halt method.
+     * Can be selected by CFGM option 'VM/HaltMethod'. */
+    VMHALTMETHOD                    enmHaltMethod;
+    /** The index into g_aHaltMethods of the current halt method. */
+    uint32_t volatile               iHaltMethod;
+    /** @} */
+
+    /** @todo Do NOT add new members here or reuse the current, we need to store the config for
+     *  each halt method separately because we're racing on SMP guest rigs. */
+    union
+    {
+       /**
+        * Method 1 & 2 - Block whenever possible, and when lagging behind
+        * switch to spinning with regular blocking every 5-200ms (defaults)
+        * depending on the accumulated lag. The blocking interval is adjusted
+        * with the average oversleeping of the last 64 times.
+        *
+        * The difference between 1 and 2 is that we use native absolute
+        * time APIs for the blocking instead of the millisecond based IPRT
+        * interface.
+        */
+        struct
+        {
+            /** The max interval without blocking (when spinning). */
+            uint32_t                u32MinBlockIntervalCfg;
+            /** The minimum interval between blocking (when spinning). */
+            uint32_t                u32MaxBlockIntervalCfg;
+            /** The value to divide the current lag by to get the raw blocking interval (when spinning). */
+            uint32_t                u32LagBlockIntervalDivisorCfg;
+            /** When to start spinning (lag / nano secs). */
+            uint32_t                u32StartSpinningCfg;
+            /** When to stop spinning (lag / nano secs). */
+            uint32_t                u32StopSpinningCfg;
+        }                           Method12;
+
+       /**
+        * The GVMM manages halted and waiting EMTs.
+        */
+        struct
+        {
+            /** The threshold between spinning and blocking. */
+            uint32_t                cNsSpinBlockThresholdCfg;
+        }                           Global1;
+    }                               Halt;
+
+    /** Pointer to the DBGC instance data. */
+    void                           *pvDBGC;
+
+    /** TLS index for the VMINTUSERPERVMCPU pointer. */
+    RTTLS                           idxTLS;
+} VMINTUSERPERVM;
+
+/** Pointer to the VM internal data kept in the UVM. */
+typedef VMINTUSERPERVM *PVMINTUSERPERVM;
+
+
+/**
+ * VMCPU internal data kept in the UVM.
+ *
+ * Almost a copy of VMINTUSERPERVM. Separate data properly later on.
+ */
+typedef struct VMINTUSERPERVMCPU
+{
+    /** Head of the request queue. Atomic. */
+    volatile PVMREQ                 pReqs;
+
+    /** The handle to the EMT thread. */
+    RTTHREAD                        ThreadEMT;
+    /** The native of the EMT thread. */
+    RTNATIVETHREAD                  NativeThreadEMT;
+    /** Wait event semaphore. */
+    RTSEMEVENT                      EventSemWait;
+    /** Wait/Idle indicator. */
+    bool volatile                   fWait;
+    /** Align the next bit. */
+    bool                            afAlignment[7];
+
+    /** @name Generic Halt data
+     * @{
+     */
+    /** The average time (ns) between two halts in the last second. (updated once per second) */
+    uint32_t                        HaltInterval;
+    /** The average halt frequency for the last second. (updated once per second) */
+    uint32_t                        HaltFrequency;
+    /** The number of halts in the current period. */
+    uint32_t                        cHalts;
+    uint32_t                        padding; /**< alignment padding. */
+    /** When we started counting halts in cHalts (RTTimeNanoTS). */
+    uint64_t                        u64HaltsStartTS;
+    /** @} */
+
+    /** Union containing data and config for the different halt algorithms. */
+    union
+    {
+       /**
+        * Method 1 & 2 - Block whenever possible, and when lagging behind
+        * switch to spinning with regular blocking every 5-200ms (defaults)
+        * depending on the accumulated lag. The blocking interval is adjusted
+        * with the average oversleeping of the last 64 times.
+        *
+        * The difference between 1 and 2 is that we use native absolute
+        * time APIs for the blocking instead of the millisecond based IPRT
+        * interface.
+        */
+        struct
+        {
+            /** How many times we've blocked while cBlockedNS and cBlockedTooLongNS has been accumulating. */
+            uint32_t                cBlocks;
+            /** Align the next member. */
+            uint32_t                u32Alignment;
+            /** Avg. time spend oversleeping when blocking. (Re-calculated every so often.) */
+            uint64_t                cNSBlockedTooLongAvg;
+            /** Total time spend oversleeping when blocking. */
+            uint64_t                cNSBlockedTooLong;
+            /** Total time spent blocking. */
+            uint64_t                cNSBlocked;
+            /** The timestamp (RTTimeNanoTS) of the last block. */
+            uint64_t                u64LastBlockTS;
+
+            /** When we started spinning relentlessly in order to catch up some of the oversleeping.
+             * This is 0 when we're not spinning. */
+            uint64_t                u64StartSpinTS;
+        }                           Method12;
+
+#if 0
+       /**
+        * Method 3 & 4 - Same as method 1 & 2 respectivly, except that we
+        * sprinkle it with yields.
+        */
+       struct
+       {
+           /** How many times we've blocked while cBlockedNS and cBlockedTooLongNS has been accumulating. */
+           uint32_t                 cBlocks;
+           /** Avg. time spend oversleeping when blocking. (Re-calculated every so often.) */
+           uint64_t                 cBlockedTooLongNSAvg;
+           /** Total time spend oversleeping when blocking. */
+           uint64_t                 cBlockedTooLongNS;
+           /** Total time spent blocking. */
+           uint64_t                 cBlockedNS;
+           /** The timestamp (RTTimeNanoTS) of the last block. */
+           uint64_t                 u64LastBlockTS;
+
+           /** How many times we've yielded while cBlockedNS and cBlockedTooLongNS has been accumulating. */
+           uint32_t                 cYields;
+           /** Avg. time spend oversleeping when yielding. */
+           uint32_t                 cYieldTooLongNSAvg;
+           /** Total time spend oversleeping when yielding. */
+           uint64_t                 cYieldTooLongNS;
+           /** Total time spent yielding. */
+           uint64_t                 cYieldedNS;
+           /** The timestamp (RTTimeNanoTS) of the last block. */
+           uint64_t                 u64LastYieldTS;
+
+           /** When we started spinning relentlessly in order to catch up some of the oversleeping. */
+           uint64_t                 u64StartSpinTS;
+       }                            Method34;
+#endif
+    }                               Halt;
+
+    /** Profiling the halted state; yielding vs blocking.
+     * @{ */
+    STAMPROFILE                     StatHaltYield;
+    STAMPROFILE                     StatHaltBlock;
+    STAMPROFILE                     StatHaltBlockOverslept;
+    STAMPROFILE                     StatHaltBlockInsomnia;
+    STAMPROFILE                     StatHaltBlockOnTime;
+    STAMPROFILE                     StatHaltTimers;
+    STAMPROFILE                     StatHaltPoll;
+    /** @} */
+} VMINTUSERPERVMCPU;
+#ifdef IN_RING3
+AssertCompileMemberAlignment(VMINTUSERPERVMCPU, u64HaltsStartTS, 8);
+AssertCompileMemberAlignment(VMINTUSERPERVMCPU, Halt.Method12.cNSBlockedTooLongAvg, 8);
+AssertCompileMemberAlignment(VMINTUSERPERVMCPU, StatHaltYield, 8);
+#endif
+
+/** Pointer to the VM internal data kept in the UVM. */
+typedef VMINTUSERPERVMCPU *PVMINTUSERPERVMCPU;
+
+RT_C_DECLS_BEGIN
+
+DECLCALLBACK(int)   vmR3EmulationThread(RTTHREAD ThreadSelf, void *pvArg);
+int                 vmR3SetHaltMethodU(PUVM pUVM, VMHALTMETHOD enmHaltMethod);
+DECLCALLBACK(int)   vmR3Destroy(PVM pVM);
+DECLCALLBACK(void)  vmR3SetErrorUV(PUVM pUVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list *args);
+void                vmSetErrorCopy(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args);
+DECLCALLBACK(int)   vmR3SetRuntimeError(PVM pVM, uint32_t fFlags, const char *pszErrorId, char *pszMessage);
+DECLCALLBACK(int)   vmR3SetRuntimeErrorV(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list *pVa);
+void                vmSetRuntimeErrorCopy(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va);
+void                vmR3SetGuruMeditation(PVM pVM);
+void                vmR3SetTerminated(PVM pVM);
+
+RT_C_DECLS_END
+
+
+/** @} */
+
+#endif
+
Index: /trunk/src/VBox/VMM/include/VMMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/VMMInternal.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/VMMInternal.h	(revision 35333)
@@ -0,0 +1,644 @@
+/* $Id$ */
+/** @file
+ * VMM - Internal header file.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___VMMInternal_h
+#define ___VMMInternal_h
+
+#include <VBox/cdefs.h>
+#include <VBox/stam.h>
+#include <VBox/log.h>
+#include <iprt/critsect.h>
+
+
+#if !defined(IN_VMM_R3) && !defined(IN_VMM_R0) && !defined(IN_VMM_RC)
+# error "Not in VMM! This is an internal header!"
+#endif
+
+
+/** @defgroup grp_vmm_int   Internals
+ * @ingroup grp_vmm
+ * @internal
+ * @{
+ */
+
+/** @def VBOX_WITH_RC_RELEASE_LOGGING
+ * Enables RC release logging. */
+#define VBOX_WITH_RC_RELEASE_LOGGING
+
+/** @def VBOX_WITH_R0_LOGGING
+ * Enables Ring-0 logging (non-release).
+ *
+ * Ring-0 logging isn't 100% safe yet (thread id reuse / process exit cleanup),
+ * so you have to sign up here by adding your defined(DEBUG_<userid>) to the
+ * #if, or by adding VBOX_WITH_R0_LOGGING to your LocalConfig.kmk.
+ */
+#if defined(DEBUG_sandervl) || defined(DEBUG_frank) || defined(DOXYGEN_RUNNING)
+# define VBOX_WITH_R0_LOGGING
+#endif
+
+/** @def VBOX_STRICT_VMM_STACK
+ * Enables VMM stack guard pages to catch stack over- and underruns. */
+#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
+# define VBOX_STRICT_VMM_STACK
+#endif
+
+
+/**
+ * Converts a VMM pointer into a VM pointer.
+ * @returns Pointer to the VM structure the VMM is part of.
+ * @param   pVMM   Pointer to VMM instance data.
+ */
+#define VMM2VM(pVMM)  ( (PVM)((char*)pVMM - pVMM->offVM) )
+
+
+/**
+ * Switcher function, HC to RC.
+ *
+ * @param   pVM         The VM handle.
+ * @returns Return code indicating the action to take.
+ */
+typedef DECLASMTYPE(int) FNVMMSWITCHERHC(PVM pVM);
+/** Pointer to switcher function. */
+typedef FNVMMSWITCHERHC *PFNVMMSWITCHERHC;
+
+/**
+ * Switcher function, RC to HC.
+ *
+ * @param   rc          VBox status code.
+ */
+typedef DECLASMTYPE(void) FNVMMSWITCHERRC(int rc);
+/** Pointer to switcher function. */
+typedef FNVMMSWITCHERRC *PFNVMMSWITCHERRC;
+
+
+/**
+ * The ring-0 logger instance wrapper.
+ *
+ * We need to be able to find the VM handle from the logger instance, so we wrap
+ * it in this structure.
+ */
+typedef struct VMMR0LOGGER
+{
+    /** Pointer to the VM handle. */
+    R0PTRTYPE(PVM)              pVM;
+    /** Size of the allocated logger instance (Logger). */
+    uint32_t                    cbLogger;
+    /** Flag indicating whether we've create the logger Ring-0 instance yet. */
+    bool                        fCreated;
+    /** Flag indicating whether we've disabled flushing (world switch) or not. */
+    bool                        fFlushingDisabled;
+    /** Flag indicating whether we've registered the instance already. */
+    bool                        fRegistered;
+    bool                        a8Alignment;
+    /** The CPU ID. */
+    VMCPUID                     idCpu;
+#if HC_ARCH_BITS == 64
+    uint32_t                    u32Alignment;
+#endif
+    /** The ring-0 logger instance. This extends beyond the size.  */
+    RTLOGGER                    Logger;
+} VMMR0LOGGER;
+/** Pointer to a ring-0 logger instance wrapper. */
+typedef VMMR0LOGGER *PVMMR0LOGGER;
+
+
+/**
+ * Jump buffer for the setjmp/longjmp like constructs used to
+ * quickly 'call' back into Ring-3.
+ */
+typedef struct VMMR0JMPBUF
+{
+    /** Traditional jmp_buf stuff
+     * @{ */
+#if HC_ARCH_BITS == 32
+    uint32_t                    ebx;
+    uint32_t                    esi;
+    uint32_t                    edi;
+    uint32_t                    ebp;
+    uint32_t                    esp;
+    uint32_t                    eip;
+    uint32_t                    eflags;
+#endif
+#if HC_ARCH_BITS == 64
+    uint64_t                    rbx;
+# ifdef RT_OS_WINDOWS
+    uint64_t                    rsi;
+    uint64_t                    rdi;
+# endif
+    uint64_t                    rbp;
+    uint64_t                    r12;
+    uint64_t                    r13;
+    uint64_t                    r14;
+    uint64_t                    r15;
+    uint64_t                    rsp;
+    uint64_t                    rip;
+# ifdef RT_OS_WINDOWS
+    uint128_t                   xmm6;
+    uint128_t                   xmm7;
+    uint128_t                   xmm8;
+    uint128_t                   xmm9;
+    uint128_t                   xmm10;
+    uint128_t                   xmm11;
+    uint128_t                   xmm12;
+    uint128_t                   xmm13;
+    uint128_t                   xmm14;
+    uint128_t                   xmm15;
+# endif
+    uint64_t                    rflags;
+#endif
+    /** @} */
+
+    /** Flag that indicates that we've done a ring-3 call. */
+    bool                        fInRing3Call;
+    /** The number of bytes we've saved. */
+    uint32_t                    cbSavedStack;
+    /** Pointer to the buffer used to save the stack.
+     * This is assumed to be 8KB. */
+    RTR0PTR                     pvSavedStack;
+    /** Esp we we match against esp on resume to make sure the stack wasn't relocated. */
+    RTHCUINTREG                 SpCheck;
+    /** The esp we should resume execution with after the restore. */
+    RTHCUINTREG                 SpResume;
+    /** ESP/RSP at the time of the jump to ring 3. */
+    RTHCUINTREG                 SavedEsp;
+    /** EBP/RBP at the time of the jump to ring 3. */
+    RTHCUINTREG                 SavedEbp;
+
+    /** Stats: Max amount of stack used. */
+    uint32_t                    cbUsedMax;
+    /** Stats: Average stack usage. (Avg = cbUsedTotal / cUsedTotal) */
+    uint32_t                    cbUsedAvg;
+    /** Stats: Total amount of stack used. */
+    uint64_t                    cbUsedTotal;
+    /** Stats: Number of stack usages. */
+    uint64_t                    cUsedTotal;
+} VMMR0JMPBUF;
+/** Pointer to a ring-0 jump buffer. */
+typedef VMMR0JMPBUF *PVMMR0JMPBUF;
+
+
+/**
+ * VMM Data (part of VM)
+ */
+typedef struct VMM
+{
+    /** Offset to the VM structure.
+     * See VMM2VM(). */
+    RTINT                       offVM;
+
+    /** @name World Switcher and Related
+     * @{
+     */
+    /** Size of the core code. */
+    RTUINT                      cbCoreCode;
+    /** Physical address of core code. */
+    RTHCPHYS                    HCPhysCoreCode;
+    /** Pointer to core code ring-3 mapping - contiguous memory.
+     * At present this only means the context switcher code. */
+    RTR3PTR                     pvCoreCodeR3;
+    /** Pointer to core code ring-0 mapping - contiguous memory.
+     * At present this only means the context switcher code. */
+    RTR0PTR                     pvCoreCodeR0;
+    /** Pointer to core code guest context mapping. */
+    RTRCPTR                     pvCoreCodeRC;
+    RTRCPTR                     pRCPadding0; /**< Alignment padding */
+#ifdef VBOX_WITH_NMI
+    /** The guest context address of the APIC (host) mapping. */
+    RTRCPTR                     GCPtrApicBase;
+    RTRCPTR                     pRCPadding1; /**< Alignment padding */
+#endif
+    /** The current switcher.
+     * This will be set before the VMM is fully initialized. */
+    VMMSWITCHER                 enmSwitcher;
+    /** Flag to disable the switcher permanently (VMX) (boolean) */
+    bool                        fSwitcherDisabled;
+    /** Array of offsets to the different switchers within the core code. */
+    RTUINT                      aoffSwitchers[VMMSWITCHER_MAX];
+
+    /** Resume Guest Execution. See CPUMGCResumeGuest(). */
+    RTRCPTR                     pfnCPUMRCResumeGuest;
+    /** Resume Guest Execution in V86 mode. See CPUMGCResumeGuestV86(). */
+    RTRCPTR                     pfnCPUMRCResumeGuestV86;
+    /** Call Trampoline. See vmmGCCallTrampoline(). */
+    RTRCPTR                     pfnCallTrampolineRC;
+    /** Guest to host switcher entry point. */
+    RCPTRTYPE(PFNVMMSWITCHERRC) pfnGuestToHostRC;
+    /** Host to guest switcher entry point. */
+    R0PTRTYPE(PFNVMMSWITCHERHC) pfnHostToGuestR0;
+    /** @}  */
+
+    /** @name Logging
+     * @{
+     */
+    /** Size of the allocated logger instance (pRCLoggerRC/pRCLoggerR3). */
+    uint32_t                    cbRCLogger;
+    /** Pointer to the RC logger instance - RC Ptr.
+     * This is NULL if logging is disabled. */
+    RCPTRTYPE(PRTLOGGERRC)      pRCLoggerRC;
+    /** Pointer to the GC logger instance - R3 Ptr.
+     * This is NULL if logging is disabled. */
+    R3PTRTYPE(PRTLOGGERRC)      pRCLoggerR3;
+    /** Pointer to the GC release logger instance - R3 Ptr. */
+    R3PTRTYPE(PRTLOGGERRC)      pRCRelLoggerR3;
+    /** Pointer to the GC release logger instance - RC Ptr. */
+    RCPTRTYPE(PRTLOGGERRC)      pRCRelLoggerRC;
+    /** Size of the allocated release logger instance (pRCRelLoggerRC/pRCRelLoggerR3).
+     * This may differ from cbRCLogger. */
+    uint32_t                    cbRCRelLogger;
+    /** Whether log flushing has been disabled or not. */
+    bool                        fRCLoggerFlushingDisabled;
+    bool                        afAlignment[5]; /**< Alignment padding. */
+    /** @} */
+
+    /** Whether the stack guard pages have been stationed or not. */
+    bool                        fStackGuardsStationed;
+    /** Whether we should use the periodic preemption timers. */
+    bool                        fUsePeriodicPreemptionTimers;
+
+    /** The EMT yield timer. */
+    PTMTIMERR3                  pYieldTimer;
+    /** The period to the next timeout when suspended or stopped.
+     * This is 0 when running. */
+    uint32_t                    cYieldResumeMillies;
+    /** The EMT yield timer interval (milliseconds). */
+    uint32_t                    cYieldEveryMillies;
+    /** The timestamp of the previous yield. (nano) */
+    uint64_t                    u64LastYield;
+
+    /** Critical section.
+     * Use for synchronizing all VCPUs
+     */
+    RTCRITSECT                  CritSectSync;
+
+    /** @name EMT Rendezvous
+     * @{ */
+    /** Semaphore to wait on upon entering ordered execution. */
+    R3PTRTYPE(PRTSEMEVENT)      pahEvtRendezvousEnterOrdered;
+    /** Semaphore to wait on upon entering for one-by-one execution. */
+    RTSEMEVENT                  hEvtRendezvousEnterOneByOne;
+    /** Semaphore to wait on upon entering for all-at-once execution. */
+    RTSEMEVENTMULTI             hEvtMulRendezvousEnterAllAtOnce;
+    /** Semaphore to wait on when done. */
+    RTSEMEVENTMULTI             hEvtMulRendezvousDone;
+    /** Semaphore the VMMR3EmtRendezvous caller waits on at the end. */
+    RTSEMEVENT                  hEvtRendezvousDoneCaller;
+    /** Callback. */
+    R3PTRTYPE(PFNVMMEMTRENDEZVOUS) volatile pfnRendezvous;
+    /** The user argument for the callback. */
+    RTR3PTR volatile            pvRendezvousUser;
+    /** Flags. */
+    volatile uint32_t           fRendezvousFlags;
+    /** The number of EMTs that has entered. */
+    volatile uint32_t           cRendezvousEmtsEntered;
+    /** The number of EMTs that has done their job. */
+    volatile uint32_t           cRendezvousEmtsDone;
+    /** The number of EMTs that has returned. */
+    volatile uint32_t           cRendezvousEmtsReturned;
+    /** The status code. */
+    volatile int32_t            i32RendezvousStatus;
+    /** Spin lock. */
+    volatile uint32_t           u32RendezvousLock;
+    /** @} */
+
+#if HC_ARCH_BITS == 32
+    uint32_t                    u32Alignment; /**< Alignment padding. */
+#endif
+
+    /** Buffer for storing the standard assertion message for a ring-0 assertion.
+     * Used for saving the assertion message text for the release log and guru
+     * meditation dump. */
+    char                        szRing0AssertMsg1[512];
+    /** Buffer for storing the custom message for a ring-0 assertion. */
+    char                        szRing0AssertMsg2[256];
+
+    /** Number of VMMR0_DO_RUN_GC calls. */
+    STAMCOUNTER                 StatRunRC;
+
+    /** Statistics for each of the RC/R0 return codes.
+     * @{ */
+    STAMCOUNTER                 StatRZRetNormal;
+    STAMCOUNTER                 StatRZRetInterrupt;
+    STAMCOUNTER                 StatRZRetInterruptHyper;
+    STAMCOUNTER                 StatRZRetGuestTrap;
+    STAMCOUNTER                 StatRZRetRingSwitch;
+    STAMCOUNTER                 StatRZRetRingSwitchInt;
+    STAMCOUNTER                 StatRZRetStaleSelector;
+    STAMCOUNTER                 StatRZRetIRETTrap;
+    STAMCOUNTER                 StatRZRetEmulate;
+    STAMCOUNTER                 StatRZRetIOBlockEmulate;
+    STAMCOUNTER                 StatRZRetPatchEmulate;
+    STAMCOUNTER                 StatRZRetIORead;
+    STAMCOUNTER                 StatRZRetIOWrite;
+    STAMCOUNTER                 StatRZRetMMIORead;
+    STAMCOUNTER                 StatRZRetMMIOWrite;
+    STAMCOUNTER                 StatRZRetMMIOPatchRead;
+    STAMCOUNTER                 StatRZRetMMIOPatchWrite;
+    STAMCOUNTER                 StatRZRetMMIOReadWrite;
+    STAMCOUNTER                 StatRZRetLDTFault;
+    STAMCOUNTER                 StatRZRetGDTFault;
+    STAMCOUNTER                 StatRZRetIDTFault;
+    STAMCOUNTER                 StatRZRetTSSFault;
+    STAMCOUNTER                 StatRZRetPDFault;
+    STAMCOUNTER                 StatRZRetCSAMTask;
+    STAMCOUNTER                 StatRZRetSyncCR3;
+    STAMCOUNTER                 StatRZRetMisc;
+    STAMCOUNTER                 StatRZRetPatchInt3;
+    STAMCOUNTER                 StatRZRetPatchPF;
+    STAMCOUNTER                 StatRZRetPatchGP;
+    STAMCOUNTER                 StatRZRetPatchIretIRQ;
+    STAMCOUNTER                 StatRZRetRescheduleREM;
+    STAMCOUNTER                 StatRZRetToR3;
+    STAMCOUNTER                 StatRZRetToR3Unknown;
+    STAMCOUNTER                 StatRZRetToR3TMVirt;
+    STAMCOUNTER                 StatRZRetToR3HandyPages;
+    STAMCOUNTER                 StatRZRetToR3PDMQueues;
+    STAMCOUNTER                 StatRZRetToR3Rendezvous;
+    STAMCOUNTER                 StatRZRetToR3Timer;
+    STAMCOUNTER                 StatRZRetToR3DMA;
+    STAMCOUNTER                 StatRZRetToR3CritSect;
+    STAMCOUNTER                 StatRZRetTimerPending;
+    STAMCOUNTER                 StatRZRetInterruptPending;
+    STAMCOUNTER                 StatRZRetCallRing3;
+    STAMCOUNTER                 StatRZRetPATMDuplicateFn;
+    STAMCOUNTER                 StatRZRetPGMChangeMode;
+    STAMCOUNTER                 StatRZRetPendingRequest;
+    STAMCOUNTER                 StatRZRetPGMFlushPending;
+    STAMCOUNTER                 StatRZRetPatchTPR;
+    STAMCOUNTER                 StatRZCallPDMLock;
+    STAMCOUNTER                 StatRZCallLogFlush;
+    STAMCOUNTER                 StatRZCallPGMPoolGrow;
+    STAMCOUNTER                 StatRZCallPGMMapChunk;
+    STAMCOUNTER                 StatRZCallPGMAllocHandy;
+    STAMCOUNTER                 StatRZCallRemReplay;
+    STAMCOUNTER                 StatRZCallVMSetError;
+    STAMCOUNTER                 StatRZCallVMSetRuntimeError;
+    STAMCOUNTER                 StatRZCallPGMLock;
+    /** @} */
+} VMM;
+/** Pointer to VMM. */
+typedef VMM *PVMM;
+
+
+/**
+ * VMMCPU Data (part of VMCPU)
+ */
+typedef struct VMMCPU
+{
+    /** Offset to the VMCPU structure.
+     * See VMM2VMCPU(). */
+    RTINT                       offVMCPU;
+
+    /** The last RC/R0 return code. */
+    int32_t                     iLastGZRc;
+
+    /** VMM stack, pointer to the top of the stack in R3.
+     * Stack is allocated from the hypervisor heap and is page aligned
+     * and always writable in RC. */
+    R3PTRTYPE(uint8_t *)        pbEMTStackR3;
+    /** Pointer to the bottom of the stack - needed for doing relocations. */
+    RCPTRTYPE(uint8_t *)        pbEMTStackRC;
+    /** Pointer to the bottom of the stack - needed for doing relocations. */
+    RCPTRTYPE(uint8_t *)        pbEMTStackBottomRC;
+
+#ifdef LOG_ENABLED
+    /** Pointer to the R0 logger instance - R3 Ptr.
+     * This is NULL if logging is disabled. */
+    R3PTRTYPE(PVMMR0LOGGER)     pR0LoggerR3;
+    /** Pointer to the R0 logger instance - R0 Ptr.
+     * This is NULL if logging is disabled. */
+    R0PTRTYPE(PVMMR0LOGGER)     pR0LoggerR0;
+#endif
+
+    /** @name Call Ring-3
+     * Formerly known as host calls.
+     * @{ */
+    /** The disable counter. */
+    uint32_t                    cCallRing3Disabled;
+    /** The pending operation. */
+    VMMCALLRING3                enmCallRing3Operation;
+    /** The result of the last operation. */
+    int32_t                     rcCallRing3;
+#if HC_ARCH_BITS == 64
+    uint32_t                    padding;
+#endif
+    /** The argument to the operation. */
+    uint64_t                    u64CallRing3Arg;
+    /** The Ring-0 jmp buffer. */
+    VMMR0JMPBUF                 CallRing3JmpBufR0;
+    /** @} */
+
+} VMMCPU;
+/** Pointer to VMMCPU. */
+typedef VMMCPU *PVMMCPU;
+
+
+/**
+ * The VMMGCEntry() codes.
+ */
+typedef enum VMMGCOPERATION
+{
+    /** Do GC module init. */
+    VMMGC_DO_VMMGC_INIT = 1,
+
+    /** The first Trap testcase. */
+    VMMGC_DO_TESTCASE_TRAP_FIRST = 0x0dead000,
+    /** Trap 0 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_0 = VMMGC_DO_TESTCASE_TRAP_FIRST,
+    /** Trap 1 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_1,
+    /** Trap 2 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_2,
+    /** Trap 3 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_3,
+    /** Trap 4 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_4,
+    /** Trap 5 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_5,
+    /** Trap 6 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_6,
+    /** Trap 7 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_7,
+    /** Trap 8 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_8,
+    /** Trap 9 testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_9,
+    /** Trap 0a testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_0A,
+    /** Trap 0b testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_0B,
+    /** Trap 0c testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_0C,
+    /** Trap 0d testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_0D,
+    /** Trap 0e testcases, uArg selects the variation. */
+    VMMGC_DO_TESTCASE_TRAP_0E,
+    /** The last trap testcase (exclusive). */
+    VMMGC_DO_TESTCASE_TRAP_LAST,
+    /** Testcase for checking interrupt forwarding. */
+    VMMGC_DO_TESTCASE_HYPER_INTERRUPT,
+    /** Switching testing and profiling stub. */
+    VMMGC_DO_TESTCASE_NOP,
+    /** Testcase for checking interrupt masking.. */
+    VMMGC_DO_TESTCASE_INTERRUPT_MASKING,
+    /** Switching testing and profiling stub. */
+    VMMGC_DO_TESTCASE_HWACCM_NOP,
+
+    /** The usual 32-bit hack. */
+    VMMGC_DO_32_BIT_HACK = 0x7fffffff
+} VMMGCOPERATION;
+
+
+RT_C_DECLS_BEGIN
+
+#ifdef IN_RING3
+int  vmmR3SwitcherInit(PVM pVM);
+void vmmR3SwitcherRelocate(PVM pVM, RTGCINTPTR offDelta);
+#endif /* IN_RING3 */
+
+#ifdef IN_RING0
+/**
+ * World switcher assembly routine.
+ * It will call VMMGCEntry().
+ *
+ * @returns return code from VMMGCEntry().
+ * @param   pVM     The VM in question.
+ * @param   uArg    See VMMGCEntry().
+ * @internal
+ */
+DECLASM(int)    vmmR0WorldSwitch(PVM pVM, unsigned uArg);
+
+/**
+ * Callback function for vmmR0CallRing3SetJmp.
+ *
+ * @returns VBox status code.
+ * @param   pVM     The VM handle.
+ */
+typedef DECLCALLBACK(int) FNVMMR0SETJMP(PVM pVM, PVMCPU pVCpu);
+/** Pointer to FNVMMR0SETJMP(). */
+typedef FNVMMR0SETJMP *PFNVMMR0SETJMP;
+
+/**
+ * The setjmp variant used for calling Ring-3.
+ *
+ * This differs from the normal setjmp in that it will resume VMMRZCallRing3 if we're
+ * in the middle of a ring-3 call. Another differences is the function pointer and
+ * argument. This has to do with resuming code and the stack frame of the caller.
+ *
+ * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+ * @param   pJmpBuf     The jmp_buf to set.
+ * @param   pfn         The function to be called when not resuming..
+ * @param   pVM         The argument of that function.
+ */
+DECLASM(int)    vmmR0CallRing3SetJmp(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMP pfn, PVM pVM, PVMCPU pVCpu);
+
+/**
+ * Callback function for vmmR0CallRing3SetJmpEx.
+ *
+ * @returns VBox status code.
+ * @param   pvUser      The user argument.
+ */
+typedef DECLCALLBACK(int) FNVMMR0SETJMPEX(void *pvUser);
+/** Pointer to FNVMMR0SETJMP(). */
+typedef FNVMMR0SETJMPEX *PFNVMMR0SETJMPEX;
+
+/**
+ * Same as vmmR0CallRing3SetJmp except for the function signature.
+ *
+ * @returns VINF_SUCCESS on success or whatever is passed to vmmR0CallRing3LongJmp.
+ * @param   pJmpBuf     The jmp_buf to set.
+ * @param   pfn         The function to be called when not resuming..
+ * @param   pvUser      The argument of that function.
+ */
+DECLASM(int)    vmmR0CallRing3SetJmpEx(PVMMR0JMPBUF pJmpBuf, PFNVMMR0SETJMPEX pfn, void *pvUser);
+
+
+/**
+ * Worker for VMMRZCallRing3.
+ * This will save the stack and registers.
+ *
+ * @returns rc.
+ * @param   pJmpBuf         Pointer to the jump buffer.
+ * @param   rc              The return code.
+ */
+DECLASM(int)    vmmR0CallRing3LongJmp(PVMMR0JMPBUF pJmpBuf, int rc);
+
+/**
+ * Internal R0 logger worker: Logger wrapper.
+ */
+VMMR0DECL(void) vmmR0LoggerWrapper(const char *pszFormat, ...);
+
+/**
+ * Internal R0 logger worker: Flush logger.
+ *
+ * @param   pLogger     The logger instance to flush.
+ * @remark  This function must be exported!
+ */
+VMMR0DECL(void) vmmR0LoggerFlush(PRTLOGGER pLogger);
+
+/**
+ * Internal R0 logger worker: Custom prefix.
+ *
+ * @returns Number of chars written.
+ *
+ * @param   pLogger     The logger instance.
+ * @param   pchBuf      The output buffer.
+ * @param   cchBuf      The size of the buffer.
+ * @param   pvUser      User argument (ignored).
+ */
+VMMR0DECL(size_t) vmmR0LoggerPrefix(PRTLOGGER pLogger, char *pchBuf, size_t cchBuf, void *pvUser);
+
+#endif /* IN_RING0 */
+#ifdef IN_RC
+
+/**
+ * Internal GC logger worker: Logger wrapper.
+ */
+VMMRCDECL(void) vmmGCLoggerWrapper(const char *pszFormat, ...);
+
+/**
+ * Internal GC release logger worker: Logger wrapper.
+ */
+VMMRCDECL(void) vmmGCRelLoggerWrapper(const char *pszFormat, ...);
+
+/**
+ * Internal GC logger worker: Flush logger.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pLogger     The logger instance to flush.
+ * @remark  This function must be exported!
+ */
+VMMRCDECL(int)  vmmGCLoggerFlush(PRTLOGGERRC pLogger);
+
+/** @name Trap testcases and related labels.
+ * @{ */
+DECLASM(void)   vmmGCEnableWP(void);
+DECLASM(void)   vmmGCDisableWP(void);
+DECLASM(int)    vmmGCTestTrap3(void);
+DECLASM(int)    vmmGCTestTrap8(void);
+DECLASM(int)    vmmGCTestTrap0d(void);
+DECLASM(int)    vmmGCTestTrap0e(void);
+DECLASM(int)    vmmGCTestTrap0e_FaultEIP(void); /**< a label */
+DECLASM(int)    vmmGCTestTrap0e_ResumeEIP(void); /**< a label */
+/** @} */
+
+#endif /* IN_RC */
+
+RT_C_DECLS_END
+
+/** @} */
+
+#endif
Index: /trunk/src/VBox/VMM/include/VMMInternal.mac
===================================================================
--- /trunk/src/VBox/VMM/include/VMMInternal.mac	(revision 35333)
+++ /trunk/src/VBox/VMM/include/VMMInternal.mac	(revision 35333)
@@ -0,0 +1,96 @@
+; $Id$
+;; @file
+; VMM - Internal header file.
+;
+
+;
+; Copyright (C) 2006-2009 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%include "VBox/asmdefs.mac"
+
+;
+; Determine the default stack switching unless specified explicitly.
+;
+%ifndef VMM_R0_SWITCH_STACK
+ %ifndef VMM_R0_NO_SWITCH_STACK
+  %ifdef RT_OS_DARWIN
+   %define VMM_R0_SWITCH_STACK
+  %endif
+ %endif
+%endif
+
+
+struc VMMR0JMPBUF
+%ifdef RT_ARCH_X86
+    ; traditional jmp_buf
+    .ebx            resd 1
+    .esi            resd 1
+    .edi            resd 1
+    .ebp            resd 1
+    .esp            resd 1
+    .eip            resd 1
+    .eflags         resd 1
+
+    ; additional state and stack info.
+    .fInRing3Call   resd 1
+    .cbSavedStack   resd 1
+    .pvSavedStack   resd 1
+    .SpCheck        resd 1
+    .SpResume       resd 1
+    .SavedEsp       resd 1
+    .SavedEbp       resd 1
+%endif
+%ifdef RT_ARCH_AMD64
+    ; traditional jmp_buf
+    .rbx            resq 1
+ %ifdef RT_OS_WINDOWS
+    .rsi            resq 1
+    .rdi            resq 1
+ %endif
+    .rbp            resq 1
+    .r12            resq 1
+    .r13            resq 1
+    .r14            resq 1
+    .r15            resq 1
+    .rsp            resq 1
+    .rip            resq 1
+ %ifdef RT_OS_WINDOWS
+    .xmm6           resq 2
+    .xmm7           resq 2
+    .xmm8           resq 2
+    .xmm9           resq 2
+    .xmm10          resq 2
+    .xmm11          resq 2
+    .xmm12          resq 2
+    .xmm13          resq 2
+    .xmm14          resq 2
+    .xmm15          resq 2
+ %endif
+    .rflags         resq 1
+
+    ; additional state and stack info.
+    .fInRing3Call   resd 1
+    .cbSavedStack   resd 1
+    .pvSavedStack   resq 1
+    .SpCheck        resq 1
+    .SpResume       resq 1
+    .SavedEsp       resq 1
+    .SavedEbp       resq 1
+%endif
+
+    ; Statistics
+    .cbUsedMax      resd 1
+    .cbUsedAvg      resd 1
+    .cbUsedTotal    resq 1
+    .cUsedTotal     resq 1
+endstruc
+
Index: /trunk/src/VBox/VMM/include/VMMSwitcher.h
===================================================================
--- /trunk/src/VBox/VMM/include/VMMSwitcher.h	(revision 35333)
+++ /trunk/src/VBox/VMM/include/VMMSwitcher.h	(revision 35333)
@@ -0,0 +1,150 @@
+/* $Id$ */
+/** @file
+ * VMM - World Switchers.
+ */
+
+/*
+ * Copyright (C) 2006-2007 Oracle Corporation
+ *
+ * This file is part of VirtualBox Open Source Edition (OSE), as
+ * available from http://www.virtualbox.org. This file is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU
+ * General Public License (GPL) as published by the Free Software
+ * Foundation, in version 2 as it comes in the "COPYING" file of the
+ * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+ * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+ */
+
+#ifndef ___VMMSwitcher_h
+#define ___VMMSwitcher_h
+
+#include <VBox/vmm.h>
+
+/** @name   Fixup Types.
+ * @{
+ */
+/** @todo document what arguments these take and what they do. */
+#define FIX_HC_2_GC_NEAR_REL    1
+#define FIX_HC_2_ID_NEAR_REL    2
+#define FIX_GC_2_HC_NEAR_REL    3
+#define FIX_GC_2_ID_NEAR_REL    4
+#define FIX_ID_2_HC_NEAR_REL    5
+#define FIX_ID_2_GC_NEAR_REL    6
+#define FIX_GC_FAR32            7
+#define FIX_GC_CPUM_OFF         8
+#define FIX_GC_VM_OFF           9   /** @todo cleanup fixup names GC->RC, HC->R0. */
+#define FIX_HC_CPUM_OFF         10
+#define FIX_HC_VM_OFF           11
+#define FIX_INTER_32BIT_CR3     12
+#define FIX_INTER_PAE_CR3       13
+#define FIX_INTER_AMD64_CR3     14
+#define FIX_HYPER_CS            18
+#define FIX_HYPER_DS            19
+#define FIX_HYPER_TSS           20
+#define FIX_GC_TSS_GDTE_DW2     21
+#define FIX_NO_FXSAVE_JMP       24
+#define FIX_NO_SYSENTER_JMP     25
+#define FIX_NO_SYSCALL_JMP      26
+#define FIX_HC_32BIT            27
+#define FIX_HC_64BIT            28
+#define FIX_HC_64BIT_CPUM       29
+#define FIX_HC_64BIT_CS         30
+#define FIX_ID_32BIT            31
+#define FIX_ID_64BIT            32
+#define FIX_ID_FAR32_TO_64BIT_MODE 33
+#define FIX_GC_APIC_BASE_32BIT  34
+#define FIX_HC_64BIT_NOCHECK    35
+#define FIX_GC_64_BIT_CPUM_OFF  36
+#define FIX_GC_CPUMCPU_OFF      37
+#define FIX_EFER_OR_MASK        38
+#define FIX_THE_END             255
+/** @} */
+
+
+/** Pointer to a switcher definition. */
+typedef struct VMMSWITCHERDEF *PVMMSWITCHERDEF;
+
+/**
+ * Callback function for relocating the core code belonging to a switcher.
+ *
+ * @param   pVM         VM handle.
+ * @param   pSwitcher   Pointer to the switcher structure.
+ * @param   R0PtrCode   Pointer to the first code byte in the ring-0 mapping.
+ * @param   pu8CodeR3   Pointer to the first code byte in the ring-3 mapping.
+ * @param   GCPtrCode   The GC address of the first code byte.
+ * @param   u32IDCode   The address of the identity mapped code (first byte).
+ */
+typedef DECLCALLBACK(void) FNVMMSWITCHERRELOCATE(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3,
+                                                 RTGCPTR GCPtrCode, uint32_t u32IDCode);
+/** Pointer to a FNVMMSWITCHERRELOCATE(). */
+typedef FNVMMSWITCHERRELOCATE *PFNVMMSWITCHERRELOCATE;
+
+/**
+ * VMM Switcher structure.
+ */
+#pragma pack(1)
+typedef struct VMMSWITCHERDEF
+{
+    /** Pointer to the code. */
+    void       *pvCode;
+    /** Pointer to the fixup records. */
+    void       *pvFixups;
+    /** Pointer to the description. */
+    const char *pszDesc;
+    /** Function which performs the necessary relocations. */
+    PFNVMMSWITCHERRELOCATE pfnRelocate;
+    /** The switcher type. */
+    VMMSWITCHER enmType;
+    /** Size of the entire code chunk. */
+    uint32_t    cbCode;
+    /** vmmR0HostToGuest C entrypoint. */
+    uint32_t    offR0HostToGuest;
+    /** vmmGCGuestToHost C entrypoint. */
+    uint32_t    offGCGuestToHost;
+    /** vmmGCCallTrampoline address. */
+    uint32_t    offGCCallTrampoline;
+    /** vmmGCGuestToHostAsm assembly entrypoint. */
+    uint32_t    offGCGuestToHostAsm;
+    /** vmmGCGuestToHostAsmHyperCtx assembly entrypoint taking HyperCtx. */
+    uint32_t    offGCGuestToHostAsmHyperCtx;
+    /** vmmGCGuestToHostAsmGuestCtx assembly entrypoint taking GuestCtx. */
+    uint32_t    offGCGuestToHostAsmGuestCtx;
+    /** @name Disassembly Regions.
+     * @{ */
+    uint32_t    offHCCode0;
+    uint32_t    cbHCCode0;
+    uint32_t    offHCCode1;
+    uint32_t    cbHCCode1;
+    uint32_t    offIDCode0;
+    uint32_t    cbIDCode0;
+    uint32_t    offIDCode1;
+    uint32_t    cbIDCode1;
+    uint32_t    offGCCode;
+    uint32_t    cbGCCode;
+    /** @} */
+} VMMSWITCHERDEF;
+#pragma pack()
+
+RT_C_DECLS_BEGIN
+extern VMMSWITCHERDEF vmmR3Switcher32BitTo32Bit_Def;
+extern VMMSWITCHERDEF vmmR3Switcher32BitToPAE_Def;
+extern VMMSWITCHERDEF vmmR3Switcher32BitToAMD64_Def;
+extern VMMSWITCHERDEF vmmR3SwitcherPAETo32Bit_Def;
+extern VMMSWITCHERDEF vmmR3SwitcherPAEToPAE_Def;
+extern VMMSWITCHERDEF vmmR3SwitcherPAEToAMD64_Def;
+extern VMMSWITCHERDEF vmmR3SwitcherAMD64To32Bit_Def;
+extern VMMSWITCHERDEF vmmR3SwitcherAMD64ToPAE_Def;
+extern VMMSWITCHERDEF vmmR3SwitcherAMD64ToAMD64_Def;
+
+extern DECLCALLBACK(void) vmmR3Switcher32BitTo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3Switcher32BitToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3Switcher32BitToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3SwitcherPAETo32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3SwitcherPAEToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3SwitcherPAEToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3SwitcherAMD64To32Bit_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3SwitcherAMD64ToPAE_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+extern DECLCALLBACK(void) vmmR3SwitcherAMD64ToAMD64_Relocate(PVM pVM, PVMMSWITCHERDEF pSwitcher, RTR0PTR R0PtrCode, uint8_t *pu8CodeR3, RTGCPTR GCPtrCode, uint32_t u32IdCode);
+RT_C_DECLS_END
+
+#endif
Index: /trunk/src/VBox/VMM/include/VMMSwitcher.mac
===================================================================
--- /trunk/src/VBox/VMM/include/VMMSwitcher.mac	(revision 35333)
+++ /trunk/src/VBox/VMM/include/VMMSwitcher.mac	(revision 35333)
@@ -0,0 +1,140 @@
+; $Id$
+;; @file
+; VMM - World Switchers.
+;
+
+;
+; Copyright (C) 2006-2007 Oracle Corporation
+;
+; This file is part of VirtualBox Open Source Edition (OSE), as
+; available from http://www.virtualbox.org. This file is free software;
+; you can redistribute it and/or modify it under the terms of the GNU
+; General Public License (GPL) as published by the Free Software
+; Foundation, in version 2 as it comes in the "COPYING" file of the
+; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+;
+
+%ifndef ___VMMSwitcher_mac
+%define ___VMMSwitcher_mac
+
+%include "iprt/asmdefs.mac"
+
+; enum VMMSWITCHER {
+%define VMMSWITCHER_INVALID             0
+%define VMMSWITCHER_32_TO_32            1
+%define VMMSWITCHER_32_TO_PAE           2
+%define VMMSWITCHER_32_TO_AMD64         3
+%define VMMSWITCHER_PAE_TO_32BIT        4
+%define VMMSWITCHER_PAE_TO_PAE          5
+%define VMMSWITCHER_PAE_TO_AMD64        6
+%define VMMSWITCHER_AMD64_TO_32         7
+%define VMMSWITCHER_AMD64_TO_PAE        8
+%define VMMSWITCHER_AMD64_TO_AMD64      9
+;; @todo the rest are all wrong. sync with vmm.h.
+%define VMMSWITCHER_HOST_TO_VMX         9
+%define VMMSWITCHER_HOST_TO_SVM         10
+%define VMMSWITCHER_MAX                 12
+; }
+
+
+struc VMMSWITCHERDEF
+    .pvCode:                        RTCCPTR_RES 1
+    .pvFixups:                      RTCCPTR_RES 1
+    .pszDesc:                       RTCCPTR_RES 1
+    .pfnRelocate:                   RTCCPTR_RES 1
+    .enmType:                       resd 1
+    .cbCode:                        resd 1
+    .offR0HostToGuest:              resd 1
+    .offGCGuestToHost:              resd 1
+    .offGCCallTrampoline:           resd 1
+    .offGCGuestToHostAsm:           resd 1
+    .offGCGuestToHostAsmHyperCtx:   resd 1
+    .offGCGuestToHostAsmGuestCtx:   resd 1
+    ; disasm help
+    .offHCCode0:                    resd 1
+    .cbHCCode0:                     resd 1
+    .offHCCode1:                    resd 1
+    .cbHCCode1:                     resd 1
+    .offIDCode0:                    resd 1
+    .cbIDCode0:                     resd 1
+    .offIDCode1:                    resd 1
+    .cbIDCode1:                     resd 1
+    .offGCCode:                     resd 1
+    .cbGCCode:                      resd 1
+endstruc
+
+
+;/** @name   Fixup Types.
+; * @{
+; */
+%define FIX_HC_2_GC_NEAR_REL    1
+%define FIX_HC_2_ID_NEAR_REL    2
+%define FIX_GC_2_HC_NEAR_REL    3
+%define FIX_GC_2_ID_NEAR_REL    4
+%define FIX_ID_2_HC_NEAR_REL    5
+%define FIX_ID_2_GC_NEAR_REL    6
+%define FIX_GC_FAR32            7
+%define FIX_GC_CPUM_OFF         8
+%define FIX_GC_VM_OFF           9
+%define FIX_HC_CPUM_OFF         10
+%define FIX_HC_VM_OFF           11
+%define FIX_INTER_32BIT_CR3     12
+%define FIX_INTER_PAE_CR3       13
+%define FIX_INTER_AMD64_CR3     14
+%define FIX_HYPER_CS            18
+%define FIX_HYPER_DS            19
+%define FIX_HYPER_TSS           20
+%define FIX_GC_TSS_GDTE_DW2     21
+%define FIX_NO_FXSAVE_JMP       24
+%define FIX_NO_SYSENTER_JMP     25
+%define FIX_NO_SYSCALL_JMP      26
+%define FIX_HC_32BIT            27
+%define FIX_HC_64BIT            28
+%define FIX_HC_64BIT_CPUM       29
+%define FIX_HC_64BIT_CS         30
+%define FIX_ID_32BIT            31
+%define FIX_ID_64BIT            32
+%define FIX_ID_FAR32_TO_64BIT_MODE 33
+%define FIX_GC_APIC_BASE_32BIT  34
+%define FIX_HC_64BIT_NOCHECK    35
+%define FIX_GC_64_BIT_CPUM_OFF  36
+%define FIX_GC_CPUMCPU_OFF      37
+%define FIX_EFER_OR_MASK        38
+%define FIX_THE_END             255
+;/** @} */
+
+
+;;
+; Generate a fixup label.
+; @param %1     Type of fixup (use one of those %defines)
+; @param %2     Offset into the instruction.
+; @param %3     Optional fixup data.
+%macro FIXUP 2-*
+BEGINDATA
+    db  %1  ; the type
+    dd  %%instruction + %2 - NAME(Start)
+    %rotate 2
+    %rep %0 - 2
+        dd      %1
+        %rotate 1
+    %endrep
+
+BEGINCODE
+%%instruction:
+%endmacro
+
+
+;; IMPORTANT all COM_ functions trashes esi, some edi and the LOOP_SHORT_WHILE kills ecx.
+;%define DEBUG_STUFF 1
+
+%ifdef DEBUG_STUFF
+    %define DEBUG_CHAR(ch) COM_CHAR ch
+    %define DEBUG_S_CHAR(ch) COM_CHAR ch
+%else
+    %define DEBUG_CHAR(ch)
+    %define DEBUG_S_CHAR(ch)
+%endif
+
+%endif ; !___VMMSwitcher_mac
+
Index: /trunk/src/VBox/VMM/testcase/Makefile.kmk
===================================================================
--- /trunk/src/VBox/VMM/testcase/Makefile.kmk	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/Makefile.kmk	(revision 35333)
@@ -120,5 +120,7 @@
  endif
  tstVMStructRC_SOURCES   = tstVMStructRC.cpp
- tstVMStructRC_INCS      = $(VBOX_PATH_VMM_SRC) $(VBOX_PATH_VMM_SRC)/PATM
+ tstVMStructRC_INCS      = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_PATH_VMM_SRC)/PATM
 endif
 
@@ -133,5 +135,8 @@
  tstVMStructSize_DEFS  += VBOX_WITH_RAW_MODE
 endif
-tstVMStructSize_INCS    = $(VBOX_PATH_VMM_SRC) $(VBOX_PATH_VMM_SRC)/PATM $(VBOX_VMM_TESTCASE_OUT_DIR)
+tstVMStructSize_INCS    = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_PATH_VMM_SRC)/PATM \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)
 tstVMStructSize_SOURCES = tstVMStructSize.cpp
 ifdef VBOX_WITH_RAW_MODE
@@ -156,5 +161,7 @@
  tstAsmStructs_DEFS    += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
 endif
-tstAsmStructs_INCS      = $(VBOX_PATH_VMM_SRC) $(VBOX_VMM_TESTCASE_OUT_DIR)
+tstAsmStructs_INCS      = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)
 tstAsmStructs_SOURCES   = tstAsmStructs.cpp
 tstAsmStructs.cpp_DEPS  = $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsHC.h
@@ -169,5 +176,7 @@
   tstAsmStructsRC_DEFS  += VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
  endif
- tstAsmStructsRC_INCS    = $(VBOX_PATH_VMM_SRC) $(VBOX_VMM_TESTCASE_OUT_DIR)
+ tstAsmStructsRC_INCS    = \
+	$(VBOX_PATH_VMM_SRC)/include \
+	$(VBOX_VMM_TESTCASE_OUT_DIR)
  tstAsmStructsRC_SOURCES = tstAsmStructs.cpp
  tstAsmStructs.cpp_DEPS += $(VBOX_VMM_TESTCASE_OUT_DIR)/tstAsmStructsRC.h
@@ -184,4 +193,5 @@
 
 tstSSM_TEMPLATE         = VBOXR3TSTEXE
+tstSSM_INCS             = $(VBOX_PATH_VMM_SRC)/include
 tstSSM_SOURCES          = tstSSM.cpp
 tstSSM_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
@@ -200,5 +210,5 @@
 tstVMMR0CallHost-1_TEMPLATE = VBOXR3TSTEXE
 tstVMMR0CallHost-1_DEFS = VMM_R0_NO_SWITCH_STACK
-tstVMMR0CallHost-1_INCS = $(VBOX_PATH_VMM_SRC)
+tstVMMR0CallHost-1_INCS = $(VBOX_PATH_VMM_SRC)/include
 tstVMMR0CallHost-1_SOURCES = \
 	tstVMMR0CallHost-1.cpp
@@ -263,8 +273,10 @@
 ifdef VBOX_WITH_PDM_ASYNC_COMPLETION
  tstPDMAsyncCompletion_TEMPLATE         = VBOXR3EXE
+ tstPDMAsyncCompletion_INCS             = $(VBOX_PATH_VMM_SRC)/include
  tstPDMAsyncCompletion_SOURCES          = tstPDMAsyncCompletion.cpp
  tstPDMAsyncCompletion_LIBS             = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
 
  tstPDMAsyncCompletionStress_TEMPLATE   = VBOXR3EXE
+ tstPDMAsyncCompletionStress_INCS       = $(VBOX_PATH_VMM_SRC)/include
  tstPDMAsyncCompletionStress_SOURCES    = tstPDMAsyncCompletionStress.cpp
  tstPDMAsyncCompletionStress_LIBS       = $(LIB_VMM) $(LIB_REM) $(LIB_RUNTIME)
@@ -310,5 +322,6 @@
 		) \
 		-f $(if $(eq $(KBUILD_TARGET),darwin),macho,elf) \
-		$(foreach inc,$(INCS) $(VBOX_PATH_VMM_SRC)/testcase $(VBOX_VMM_TESTCASE_OUT_DIR),-I$(inc)/)
+		$(foreach inc,$(INCS) $(VBOX_PATH_VMM_SRC)/testcase $(VBOX_PATH_VMM_SRC)/include $(VBOX_VMM_TESTCASE_OUT_DIR)\
+			,-I$(inc)/)
 
 # 1a. make a header file which makes all the structures+members globals.
@@ -320,8 +333,8 @@
 		$(DEPTH)/include/VBox/sup.mac \
 		$(DEPTH)/include/VBox/vm.mac \
-		$(VBOX_PATH_VMM_SRC)/CPUMInternal.mac \
-		$(VBOX_PATH_VMM_SRC)/TRPMInternal.mac \
-		$(VBOX_PATH_VMM_SRC)/HWACCMInternal.mac \
-		$(VBOX_PATH_VMM_SRC)/VMMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/CPUMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/TRPMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/HWACCMInternal.mac \
+		$(VBOX_PATH_VMM_SRC)/include/VMMInternal.mac \
 		$(VBOX_PATH_VMM_SRC)/testcase/Makefile.kmk \
 		$(PATH_ROOT)/Config.kmk $(LOCALCFG) $(AUTOCFG) \
Index: /trunk/src/VBox/VMM/testcase/tstAsmStructs.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstAsmStructs.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/tstAsmStructs.cpp	(revision 35333)
@@ -24,5 +24,5 @@
 #include "TRPMInternal.h"
 #include "HWACCMInternal.h"
-#include "../VMMSwitcher/VMMSwitcher.h"
+#include "VMMSwitcher.h"
 #include "VMMInternal.h"
 #include <VBox/vm.h>
Index: /trunk/src/VBox/VMM/testcase/tstAsmStructsAsm.asm
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstAsmStructsAsm.asm	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/tstAsmStructsAsm.asm	(revision 35333)
@@ -23,12 +23,12 @@
 %endif
 
-%include "../CPUMInternal.mac"
-%include "../HWACCMInternal.mac"
-%include "../TRPMInternal.mac"
-%include "../VMMInternal.mac"
+%include "CPUMInternal.mac"
+%include "HWACCMInternal.mac"
+%include "TRPMInternal.mac"
+%include "VMMInternal.mac"
 %include "VBox/cpum.mac"
 %include "VBox/vm.mac"
 %include "VBox/sup.mac"
-%include "../VMMSwitcher/VMMSwitcher.mac"
+%include "VMMSwitcher.mac"
 %ifdef DO_GLOBALS
  %include "tstAsmStructsAsm.mac"
Index: /trunk/src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/tstPDMAsyncCompletion.cpp	(revision 35333)
@@ -26,5 +26,5 @@
 #define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
 
-#include "../VMInternal.h" /* UVM */
+#include "VMInternal.h" /* UVM */
 #include <VBox/vm.h>
 #include <VBox/uvm.h>
Index: /trunk/src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/tstPDMAsyncCompletionStress.cpp	(revision 35333)
@@ -23,5 +23,5 @@
 #define LOG_GROUP LOG_GROUP_PDM_ASYNC_COMPLETION
 
-#include "../VMInternal.h" /* UVM */
+#include "VMInternal.h" /* UVM */
 #include <VBox/vm.h>
 #include <VBox/uvm.h>
Index: /trunk/src/VBox/VMM/testcase/tstSSM.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstSSM.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/tstSSM.cpp	(revision 35333)
@@ -21,5 +21,5 @@
 *******************************************************************************/
 #include <VBox/ssm.h>
-#include "../VMInternal.h" /* createFakeVM */
+#include "VMInternal.h" /* createFakeVM */
 #include <VBox/vm.h>
 #include <VBox/uvm.h>
Index: /trunk/src/VBox/VMM/testcase/tstVMStructRC.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstVMStructRC.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/tstVMStructRC.cpp	(revision 35333)
@@ -59,23 +59,23 @@
 #include <VBox/vmm.h>
 #include <VBox/stam.h>
-#include "../PDMInternal.h"
+#include "PDMInternal.h"
 #include <VBox/pdm.h>
-#include "../CFGMInternal.h"
-#include "../CPUMInternal.h"
-#include "../MMInternal.h"
-#include "../PGMInternal.h"
-#include "../SELMInternal.h"
-#include "../TRPMInternal.h"
-#include "../TMInternal.h"
-#include "../IOMInternal.h"
-#include "../REMInternal.h"
-#include "../HWACCMInternal.h"
-#include "../PATM/PATMInternal.h"
-#include "../VMMInternal.h"
-#include "../DBGFInternal.h"
-#include "../STAMInternal.h"
-#include "../PATM/CSAMInternal.h"
-#include "../EMInternal.h"
-#include "../REMInternal.h"
+#include "CFGMInternal.h"
+#include "CPUMInternal.h"
+#include "MMInternal.h"
+#include "PGMInternal.h"
+#include "SELMInternal.h"
+#include "TRPMInternal.h"
+#include "TMInternal.h"
+#include "IOMInternal.h"
+#include "REMInternal.h"
+#include "HWACCMInternal.h"
+#include "PATMInternal.h"
+#include "VMMInternal.h"
+#include "DBGFInternal.h"
+#include "STAMInternal.h"
+#include "CSAMInternal.h"
+#include "EMInternal.h"
+#include "REMInternal.h"
 #include <VBox/vm.h>
 #include <VBox/param.h>
Index: /trunk/src/VBox/VMM/testcase/tstVMStructSize.cpp
===================================================================
--- /trunk/src/VBox/VMM/testcase/tstVMStructSize.cpp	(revision 35332)
+++ /trunk/src/VBox/VMM/testcase/tstVMStructSize.cpp	(revision 35333)
@@ -50,6 +50,6 @@
 #include "EMInternal.h"
 #include "REMInternal.h"
-#include "VMMR0/GMMR0Internal.h"
-#include "VMMR0/GVMMR0Internal.h"
+#include "../VMMR0/GMMR0Internal.h"
+#include "../VMMR0/GVMMR0Internal.h"
 #include <VBox/vm.h>
 #include <VBox/uvm.h>
Index: /trunk/src/recompiler/Makefile.kmk
===================================================================
--- /trunk/src/recompiler/Makefile.kmk	(revision 35332)
+++ /trunk/src/recompiler/Makefile.kmk	(revision 35333)
@@ -81,5 +81,5 @@
 	fpu \
 	$(PATH_VBoxRemPrimary) \
-	$(PATH_ROOT)/src/VBox/VMM \
+	$(PATH_ROOT)/src/VBox/VMM/include \
 	.
 VBoxRemPrimary_INCS.amd64     += tcg/x86_64
